Commit 12090f98 authored by Manuel Wudka-Robles's avatar Manuel Wudka-Robles
Browse files

WIP: embed protobufutil

parent df7f4368

CMakeLists.txt

+93 −28
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ endif(BUILD_PIC) 


if(APPLE)

  # On OSX we must explicitly specify that we want to build for x86-64

  set(OPENSSL_CONFIGURE ./Configure darwin64-x86_64-cc)

  set(OPENSSL_CONFIGURE_COMMAND ./Configure darwin64-x86_64-cc)

else(APPLE)

  if(${BUILD_FOR_ARM})

    set(CMAKE_C_COMPILER "arm-marvell-linux-gnueabi-gcc")
@@ -48,46 +48,105 @@ set(TEST_LIBRARIES 
set(GENERATED_SOURCES_PATH ${kinetic_cpp_client_SOURCE_DIR}/src/main/generated)



set(PREFIX "${CMAKE_BINARY_DIR}/vendor")

set(EXTERNAL_PREFIX "${kinetic_cpp_client_BINARY_DIR}/vendor")



include(ExternalProject)



set(PROTOBUFUTIL_VERSION "v0.2.4")

set(PROTOBUFUTIL_MD5 "5d21e7ad0737463b6629ac3e2c8f4e8d")

set(KINETIC_PROTO_VERSION "2.0.2")

set(KINETIC_PROTO_MD5 "f5ff4cbea0e909362c46032fb3775c49")



ExternalProject_add(

    protobufutil

    kinetic-proto

    PREFIX ${PREFIX}

    # cmake's use of curl doesn't work right on macs, so we use a DOWNLOAD_COMMAND instead

    # be warned... cmake's arg parsing is really simplistic: it just splits on spaces regardless of quotes.

    DOWNLOAD_COMMAND curl -L https://github.com/palominolabs/protobuf-util/archive/${PROTOBUFUTIL_VERSION}.tar.gz -o protobufutil-src.tar.gz && openssl md5 protobufutil-src.tar.gz | grep -q ${PROTOBUFUTIL_MD5} && mkdir -p protobufutil && tar -xz --strip-components 1 -C protobufutil -f protobufutil-src.tar.gz

    DOWNLOAD_COMMAND curl -L https://github.com/Seagate/kinetic-protocol/archive/${KINETIC_PROTO_VERSION}.tar.gz -o kinetic-proto.tar.gz && openssl md5 kinetic-proto.tar.gz | grep -q ${KINETIC_PROTO_MD5} && rm -rf kinetic-proto && mkdir -p kinetic-proto && tar -xz --strip-components 1 -C kinetic-proto -f kinetic-proto.tar.gz

    BUILD_IN_SOURCE 1

    CONFIGURE_COMMAND cmake -DBUILD_FOR_ARM=${BUILD_FOR_ARM} -DBUILD_PIC=${BUILD_PIC}

    CONFIGURE_COMMAND ""

    BUILD_COMMAND ""

    INSTALL_COMMAND ""

)



set(KINETIC_PROTO_VERSION "2.0.2")

set(KINETIC_PROTO_MD5 "f5ff4cbea0e909362c46032fb3775c49")

ExternalProject_add(

    gflags

    PREFIX ${EXTERNAL_PREFIX}

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/gflags-2.0-no-svn-files.tar.gz"

    URL_MD5 "9084829124e02a7e6be0f0f824523423"

    CONFIGURE_COMMAND ../gflags/configure --prefix=${EXTERNAL_PREFIX} --enable-static ${CONFIG_HOST_FLAG} ${CHILD_MAKE_FLAGS} ${PIC_MAKE_FLAGS}

)



ExternalProject_add(

    kinetic-proto

    PREFIX ${PREFIX}

    DOWNLOAD_COMMAND curl -L https://github.com/Seagate/kinetic-protocol/archive/${KINETIC_PROTO_VERSION}.tar.gz -o kinetic-proto.tar.gz && openssl md5 kinetic-proto.tar.gz | grep -q ${KINETIC_PROTO_MD5} && rm -rf kinetic-proto && mkdir -p kinetic-proto && tar -xz --strip-components 1 -C kinetic-proto -f kinetic-proto.tar.gz

    glog

    PREFIX ${EXTERNAL_PREFIX}

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/glog-0.3.3.tar.gz"

    URL_MD5 "a6fd2c22f8996846e34c763422717c18"

    PATCH_COMMAND sh ${kinetic_cpp_client_SOURCE_DIR}/patches/apply-glog-patches.sh ${kinetic_cpp_client_SOURCE_DIR}

    CONFIGURE_COMMAND ../glog/configure --prefix=${EXTERNAL_PREFIX} --with-gflags=${EXTERNAL_PREFIX} --enable-static ${CONFIG_HOST_FLAG} ${CHILD_MAKE_FLAGS} ${PIC_MAKE_FLAGS}

    DEPENDS gflags

)



ExternalProject_add(

    gtest

    PREFIX ${EXTERNAL_PREFIX}

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/gtest-1.6.0.zip"

    URL_MD5 "4577b49f2973c90bf9ba69aa8166b786"

    BUILD_IN_SOURCE 1

    CONFIGURE_COMMAND ""

    BUILD_COMMAND ""

    BUILD_COMMAND ${CMAKE_CXX_COMPILER} -DGTEST_USE_OWN_TR1_TUPLE=1 -I../gtest -I../gtest/include -c ../gtest/src/gtest-all.cc && ar -rv libgtest.a gtest-all.o && ranlib libgtest.a

    INSTALL_COMMAND ""

)



ExternalProject_add(

    gmock

    PREFIX ${EXTERNAL_PREFIX}

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/gmock-1.6.0.zip"

    URL_MD5 "f547f47321ca88d3965ca2efdcc2a3c1"

    BUILD_IN_SOURCE 1

    CONFIGURE_COMMAND ""

    BUILD_COMMAND ${CMAKE_CXX_COMPILER} -DGTEST_USE_OWN_TR1_TUPLE=1 -I../gmock -I../gmock/include -I../gtest -I../gtest/include -c ../gmock/src/gmock-all.cc && ar -rv libgmock.a gmock-all.o && ranlib libgmock.a

    INSTALL_COMMAND ""

    DEPENDS gtest

)



ExternalProject_add(

    openssl

    PREFIX ${EXTERNAL_PREFIX}

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/openssl-1.0.1g.tar.gz"

    URL_MD5 "de62b43dfcd858e66a74bee1c834e959"

    BUILD_IN_SOURCE 1

    CONFIGURE_COMMAND ${OPENSSL_CONFIGURE_COMMAND} --prefix=${EXTERNAL_PREFIX} ${BUILD_PIC_COMPILER_FLAG}

    BUILD_COMMAND touch apps/openssl && touch openssl.pc && make ${CHILD_MAKE_FLAGS} build_libs libssl.pc libcrypto.pc

    INSTALL_COMMAND make install_sw

)



# The protobuf build requires the existence of a protoc binary that can be

# executed on the host machine. To handle cross compilation, we always build

# protobuf once for the host so that we have a suitable copy of protoc.

ExternalProject_add(

    protoc

    PREFIX ${EXTERNAL_PREFIX}/host

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/protobuf-2.5.0.tar.bz2"

    URL_MD5 "a72001a9067a4c2c4e0e836d0f92ece4"

    CONFIGURE_COMMAND ../protoc/configure --prefix=${EXTERNAL_PREFIX}/host --enable-static

)



# Protobuf code generation rules

set(PROTOC_PATH "${PREFIX}/src/protobufutil/vendor/host/bin/protoc")

set(PROTOC_PATH "${PREFIX}/host/bin/protoc")

set(PROTO_DIR "${CMAKE_BINARY_DIR}/vendor/src/kinetic-proto")

set(PROTO_ORIG_PATH "${PROTO_DIR}/kinetic.proto")

set(PROTO_MODIFIED_PATH "${PROTO_DIR}/kinetic_client.proto")

ExternalProject_add(

    protobuf

    PREFIX ${EXTERNAL_PREFIX}

    URL "${kinetic_cpp_client_SOURCE_DIR}/tarballs/protobuf-2.5.0.tar.bz2"

    URL_MD5 "a72001a9067a4c2c4e0e836d0f92ece4"

    CONFIGURE_COMMAND ../protobuf/configure --prefix=${EXTERNAL_PREFIX} --enable-static --with-protoc=${PROTOC_PATH} ${CONFIG_HOST_FLAG} ${CHILD_MAKE_FLAGS} ${PIC_MAKE_FLAGS}

    DEPENDS protoc

)



add_custom_command(

    COMMENT "Compiling protobuf"

    OUTPUT ${GENERATED_SOURCES_PATH}/kinetic_client.pb.h ${GENERATED_SOURCES_PATH}/kinetic_client.pb.cc

    COMMAND mkdir -p ${GENERATED_SOURCES_PATH} && sed 's/com\\.seagate\\.kinetic\\.proto/com.seagate.kinetic.client.proto/' ${PROTO_ORIG_PATH} > ${PROTO_MODIFIED_PATH} && ${PROTOC_PATH} -I=${PROTO_DIR} --cpp_out=${GENERATED_SOURCES_PATH} ${PROTO_MODIFIED_PATH}

    DEPENDS protobufutil kinetic-proto

    DEPENDS kinetic-proto protoc protobuf

)

set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${GENERATED_SOURCES_PATH})
@@ -99,22 +158,20 @@ include_directories( 
    src/test/mock

    src/test



    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/include

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/include

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gmock/include

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gtest/include

    ${EXTERNAL_PREFIX}/include

    ${EXTERNAL_PREFIX}/src/gmock/include

    ${EXTERNAL_PREFIX}/src/gtest/include

)



set(LIBRARY_DEPENDENCIES

    kinetic_client

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/libprotobufutil.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libglog.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libgflags.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libssl.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libcrypto.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libprotobuf.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gtest/libgtest.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gmock/libgmock.a

    ${CMAKE_BINARY_DIR}/vendor/lib/libglog.a

    ${CMAKE_BINARY_DIR}/vendor/lib/libgflags.a

    ${CMAKE_BINARY_DIR}/vendor/lib/libssl.a

    ${CMAKE_BINARY_DIR}/vendor/lib/libcrypto.a

    ${CMAKE_BINARY_DIR}/vendor/lib/libprotobuf.a

    ${CMAKE_BINARY_DIR}/vendor/src/gtest/libgtest.a

    ${CMAKE_BINARY_DIR}/vendor/src/gmock/libgmock.a

    ${CMAKE_THREAD_LIBS_INIT}

    dl

)
@@ -135,9 +192,15 @@ add_library(kinetic_client 
    src/main/blocking_kinetic_connection.cc

    src/main/threadsafe_blocking_kinetic_connection.cc

    src/main/status_code.cc

    src/main/byte_stream.cc

    src/main/incoming_string_value.cc

    src/main/message_stream.cc

    src/main/outgoing_string_value.cc

    src/main/reader_writer.cc



    src/main/key_range_iterator.cc

)

add_dependencies(kinetic_client openssl)



add_executable(${TEST_BINARY}

    src/test/kinetic_cpp_client_test.cc
@@ -148,6 +211,8 @@ add_executable(${TEST_BINARY} 
    src/test/nonblocking_packet_test.cc

    src/test/nonblocking_string_test.cc

    src/test/hmac_provider_test.cc

    src/test/message_stream_test.cc

    src/test/string_value_test.cc

)

target_link_libraries(${TEST_BINARY} ${LIBRARY_DEPENDENCIES})

include/kinetic/blocking_kinetic_connection.h

+1 −1
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ 
#include "kinetic/status.h"

#include "kinetic/kinetic_connection.h"

#include "kinetic/key_range_iterator.h"

#include "protobufutil/common.h"

#include "kinetic/common.h"





namespace kinetic {

include/kinetic/byte_stream.h

0 → 100644
+71 −0
Original line number Diff line number Diff line 
/*

 * kinetic-cpp-client

 * Copyright (C) 2014 Seagate Technology.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

 *

 */



#ifndef KINETIC_CPP_CLIENT_BYTE_STREAM_H_

#define KINETIC_CPP_CLIENT_BYTE_STREAM_H_



#include "openssl/ssl.h"



#include "common.h"

#include "incoming_value.h"

#include "outgoing_value.h"



namespace kinetic {



class ByteStreamInterface {

    public:

    virtual ~ByteStreamInterface() {}

    virtual bool Read(void *buf, size_t n) = 0;

    virtual bool Write(const void *buf, size_t n) = 0;

    virtual IncomingValueInterface *ReadValue(size_t n) = 0;

    virtual bool WriteValue(const OutgoingValueInterface &value) = 0;

};



class PlainByteStream : public ByteStreamInterface {

    public:

    PlainByteStream(int fd, IncomingValueFactoryInterface &value_factory);

    bool Read(void *buf, size_t n);

    bool Write(const void *buf, size_t n);

    IncomingValueInterface *ReadValue(size_t n);

    bool WriteValue(const OutgoingValueInterface &value);



    private:

    int fd_;

    IncomingValueFactoryInterface &value_factory_;

    DISALLOW_COPY_AND_ASSIGN(PlainByteStream);

};



class SslByteStream : public ByteStreamInterface {

    public:

    explicit SslByteStream(SSL *ssl);

    virtual ~SslByteStream();

    bool Read(void *buf, size_t n);

    bool Write(const void *buf, size_t n);

    IncomingValueInterface *ReadValue(size_t n);

    bool WriteValue(const OutgoingValueInterface &value);



    private:

    SSL *ssl_;

    DISALLOW_COPY_AND_ASSIGN(SslByteStream);

};



} // namespace kinetic



#endif  // KINETIC_CPP_CLIENT_BYTE_STREAM_H_

include/kinetic/common.h

0 → 100644
+30 −0
Original line number Diff line number Diff line 
/*

 * kinetic-cpp-client

 * Copyright (C) 2014 Seagate Technology.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

 *

 */



#ifndef KINETIC_CPP_CLIENT_COMMON_H_

#define KINETIC_CPP_CLIENT_COMMON_H_



// Most classes do not need copy or assignment constructors and should use this

// macro to disable them.

#define DISALLOW_COPY_AND_ASSIGN(TypeName) \

    TypeName(const TypeName&); \

    void operator=(const TypeName&)



#endif  // KINETIC_CPP_CLIENT_COMMON_H_

include/kinetic/incoming_value.h

0 → 100644
+92 −0
Original line number Diff line number Diff line 
/*

 * kinetic-cpp-client

 * Copyright (C) 2014 Seagate Technology.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

 *

 */



#ifndef KINETIC_CPP_CLIENT_INCOMING_VALUE_H_

#define KINETIC_CPP_CLIENT_INCOMING_VALUE_H_



#include <string>



#include "common.h"



namespace kinetic {



/*

 * IncomingValueInterface represents a byte-array value arriving in a PUT

 * request. This can take the form of either a string in memory or a socket

 * file descriptor from which we can read a specified number of bytes.

 */

class IncomingValueInterface {

    public:

    virtual ~IncomingValueInterface() {}



    virtual size_t size() = 0;



    /*

     * TransferToFile transfers the contents of the value to a file represented

     * by the given file descriptor. As soon as this method has been called,

     * the object should be considered defunct and all further calls to

     * TransferToFile and ToString will fail.

     */

    virtual bool TransferToFile(int fd) = 0;



    /*

     * ToString copies the value to the string pointed to by the result

     * parameter.

     */

    virtual bool ToString(std::string *result) = 0;



    /*

     * Consume does whatever is necessary to consume the resources underlying

     * the value. In the SpliceableValue implementation this consists of

     * reading the appropriate bytes from the socket and throwing them away.

     * After this method has been called, the object should be considered

     * defunct and all further calls to TransferToFile and ToString will fail.

     */

    virtual void Consume() = 0;

};



/*

 * IncomingStringValue represents a value stored internally as a plain string.

 * It's preferable to use SpliceableValue whenever possible because of its

 * performance benefits.

 */

class IncomingStringValue : public IncomingValueInterface {

    public:

    explicit IncomingStringValue(const std::string &s);

    size_t size();

    bool TransferToFile(int fd);

    bool ToString(std::string *result);

    void Consume();



    private:

    const std::string s_;

    bool defunct_;

    DISALLOW_COPY_AND_ASSIGN(IncomingStringValue);

};



class IncomingValueFactoryInterface {

    public:

    virtual ~IncomingValueFactoryInterface() {}

    virtual IncomingValueInterface *NewValue(int fd, size_t n) = 0;

};



} // namespace kinetic



#endif  // KINETIC_CPP_CLIENT_INCOMING_VALUE_H_