Commit 93c9ef85 authored by Andrew Mitchell's avatar Andrew Mitchell
Browse files

Initial open source commit

parents

.gitignore

0 → 100644
+41 −0
Original line number Diff line number Diff line 
*.swp

.idea

.project

.cproject



# Build output

*.o

*.a

*.lo

*.la

*.pc

.libs

.deps

kinetic_client_test

kinetic_integration_test

/vendor

kinetic_cpp_client.build

kinetic_cpp_client.xcodeproj

kinetic_client.proto



# gtest report xml

/gtestresults.xml

/integrationresults.xml

test_detail.xml



# CMake artifacts

CMakeCache.txt

CMakeFiles

cmake_install.cmake

/Makefile

CMakeScripts



# Doxygen output

docs

doxygen_warnings.log



# Compiled protobuf files

*.pb.cc

*.pb.h

tmp
 
 No newline at end of file

CMakeLists.txt

0 → 100644
+204 −0
Original line number Diff line number Diff line 
cmake_minimum_required(VERSION 2.8.6)

project(kinetic_cpp_client CXX C)



find_package (Threads)



set(CMAKE_POSITION_INDEPENDENT_CODE ON) 



option(BUILD_FOR_ARM "Build for ARM instead of x86" off)

option(BUILD_PIC "Build static PIC" off)



set(BUILD_PIC_COMPILER_FLAGS "")



if(BUILD_PIC)

  set(BUILD_PIC_COMPILER_FLAGS "-fPIC")

endif(BUILD_PIC)



if(APPLE)

  # On OSX we must explicitly specify that we want to build for x86-64

  set(OPENSSL_CONFIGURE ./Configure darwin64-x86_64-cc)

else(APPLE)

  if(${BUILD_FOR_ARM})

    set(CMAKE_C_COMPILER "arm-marvell-linux-gnueabi-gcc")

    set(CMAKE_CXX_COMPILER "arm-marvell-linux-gnueabi-g++")

    set(CMAKE_RANLIB "arm-marvell-linux-gnueabi-ranlib")

    set(OPENSSL_CONFIGURE_COMMAND ../openssl/Configure linux-armv4)

    set(CONFIG_HOST_FLAG --host=arm)

    set(CHILD_MAKE_FLAGS CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} RANLIB=${CMAKE_RANLIB})

  else(${BUILD_FOR_ARM})

    set(CMAKE_C_COMPILER "gcc")

    set(CMAKE_CXX_COMPILER "g++")

    set(OPENSSL_CONFIGURE_COMMAND ../openssl/config -DPURIFY)

  endif(${BUILD_FOR_ARM})

endif(APPLE)



set(CMAKE_CXX_FLAGS "--std=c++0x -Wall -Wextra -Werror -Wno-unknown-warning-option -Wno-unused-parameter -Wno-null-dereference -Wno-unused-local-typedefs -DGTEST_USE_OWN_TR1_TUPLE=1 ${BUILD_PIC_COMPILER_FLAGS}")



set(TEST_BINARY "kinetic_client_test")

set(TEST_BINARY_PATH ${kinetic_cpp_client_BINARY_DIR}/${TEST_BINARY})

set(INTEGRATION_TEST_BINARY "kinetic_integration_test")

set(INTEGRATION_TEST_BINARY_PATH ${kinetic_cpp_client_BINARY_DIR}/${INTEGRATION_TEST_BINARY})

set(TEST_LIBRARIES

    glog

    gtest

    gmock

    openssl

)



set(GENERATED_SOURCES_PATH ${kinetic_cpp_client_SOURCE_DIR}/src/main/generated)



set(PREFIX "${CMAKE_BINARY_DIR}/vendor")



include(ExternalProject)



set(PROTOBUFUTIL_VERSION "v0.2.2")

set(PROTOBUFUTIL_MD5 "e1de2f4b5068b1d6464bacd3b3af3b72")



ExternalProject_add(

    protobufutil

    PREFIX ${PREFIX}

    # cmake's use of curl doesn't work right on macs, so we use a DOWNLOAD_COMMAND instead

    # be warned... cmake's arg parsing is really simplistic: it just splits on spaces regardless of quotes.

    DOWNLOAD_COMMAND curl -L https://github.com/palominolabs/protobuf-util/archive/${PROTOBUFUTIL_VERSION}.tar.gz -o protobufutil-src.tar.gz && openssl md5 protobufutil-src.tar.gz | grep -q ${PROTOBUFUTIL_MD5} && mkdir -p protobufutil && tar -xz --strip-components 1 -C protobufutil -f protobufutil-src.tar.gz

    BUILD_IN_SOURCE 1

    CONFIGURE_COMMAND cmake -DBUILD_FOR_ARM=${BUILD_FOR_ARM} -DBUILD_PIC=${BUILD_PIC}

    INSTALL_COMMAND ""

)



set(KINETIC_PROTO_VERSION "9c6b4a180a70f8488c5d8ec8e8a6464c4ff63f84")

set(KINETIC_PROTO_MD5 "17d68ba828e3c0dbcd98c2c9c52ab73a")



ExternalProject_add(

    kinetic-proto

    PREFIX ${PREFIX}

    DOWNLOAD_COMMAND curl -L https://github.com/Seagate/kinetic-protocol/archive/${KINETIC_PROTO_VERSION}.tar.gz -o kinetic-proto.tar.gz && openssl md5 kinetic-proto.tar.gz | grep -q ${KINETIC_PROTO_MD5} && rm -rf kinetic-proto && mkdir -p kinetic-proto && tar -xz --strip-components 1 -C kinetic-proto -f kinetic-proto.tar.gz

    BUILD_IN_SOURCE 1

    CONFIGURE_COMMAND ""

    BUILD_COMMAND ""

    INSTALL_COMMAND ""

)



# Protobuf code generation rules

set(PROTOC_PATH "${PREFIX}/src/protobufutil/vendor/host/bin/protoc")

set(PROTO_DIR "${CMAKE_BINARY_DIR}/vendor/src/kinetic-proto")

set(PROTO_ORIG_PATH "${PROTO_DIR}/kinetic.proto")

set(PROTO_MODIFIED_PATH "${PROTO_DIR}/kinetic_client.proto")

add_custom_command(

    COMMENT "Compiling protobuf"

    OUTPUT ${GENERATED_SOURCES_PATH}/kinetic_client.pb.h ${GENERATED_SOURCES_PATH}/kinetic_client.pb.cc

    COMMAND mkdir -p ${GENERATED_SOURCES_PATH} && sed 's/com\\.seagate\\.kinetic\\.proto/com.seagate.kinetic.client.proto/' ${PROTO_ORIG_PATH} > ${PROTO_MODIFIED_PATH} && ${PROTOC_PATH} -I=${PROTO_DIR} --cpp_out=${GENERATED_SOURCES_PATH} ${PROTO_MODIFIED_PATH}

    DEPENDS protobufutil kinetic-proto

)

set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${GENERATED_SOURCES_PATH})



include_directories(

    include

    src/main/generated

    src/main



    src/test/mock

    src/test



    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/include

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/include

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gmock/include

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gtest/include

)



set(LIBRARY_DEPENDENCIES

    kinetic_client

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/libprotobufutil.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libglog.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libgflags.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libssl.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libcrypto.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/lib/libprotobuf.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gtest/libgtest.a

    ${CMAKE_BINARY_DIR}/vendor/src/protobufutil/vendor/src/gmock/libgmock.a

    ${CMAKE_THREAD_LIBS_INIT}

    dl

)



add_library(kinetic_client

    src/main/generated/kinetic_client.pb.cc

    src/main/hmac_provider.cc

    src/main/kinetic_connection_factory.cc

    src/main/nonblocking_kinetic_connection.cc

    src/main/threadsafe_nonblocking_kinetic_connection.cc

    src/main/nonblocking_packet.cc

    src/main/nonblocking_packet_writer_factory.cc

    src/main/nonblocking_packet_service.cc

    src/main/nonblocking_packet_sender.cc

    src/main/nonblocking_packet_receiver.cc

    src/main/nonblocking_string.cc

    src/main/socket_wrapper.cc

    src/main/blocking_kinetic_connection.cc

    src/main/connection_handle.cc

    src/main/key_range_iterator.cc

)

add_dependencies(kinetic_client

    glog

    openssl

    protobuf

)



add_executable(${TEST_BINARY}

    src/test/kinetic_cpp_client_test.cc

    src/test/nonblocking_kinetic_connection_test.cc

    src/test/nonblocking_packet_service_test.cc

    src/test/nonblocking_packet_sender_test.cc

    src/test/nonblocking_packet_receiver_test.cc

    src/test/nonblocking_packet_test.cc

    src/test/nonblocking_string_test.cc

    src/test/hmac_provider_test.cc

)

add_dependencies(${TEST_BINARY} ${TEST_LIBRARIES})

target_link_libraries(${TEST_BINARY} ${LIBRARY_DEPENDENCIES})



add_executable(${INTEGRATION_TEST_BINARY}

    src/integration_test/delete_test.cc

    src/integration_test/get_test.cc

    src/integration_test/nonexistent_server_test.cc

    src/integration_test/put_test.cc

    src/integration_test/blocking_smoketest.cc

    src/test/kinetic_cpp_client_test.cc

)

add_dependencies(${INTEGRATION_TEST_BINARY} ${TEST_LIBRARIES})

target_link_libraries(${INTEGRATION_TEST_BINARY} ${LIBRARY_DEPENDENCIES})



# Rule for running unit tests

add_custom_target(test

    COMMAND ${TEST_BINARY_PATH} --gtest_output=xml:gtestresults.xml

    DEPENDS ${TEST_BINARY_PATH}

)



# Rule for running integration tests

add_custom_target(integration_test

    COMMAND ${INTEGRATION_TEST_BINARY_PATH} --gtest_output=xml:integrationresults.xml

    DEPENDS ${INTEGRATION_TEST_BINARY_PATH}

)



# Rules for running unit and integration tests under Valgrind

add_custom_target(test_valgrind

    COMMAND valgrind --leak-check=full --show-reachable=yes --track-fds=yes --suppressions=${kinetic_cpp_client_SOURCE_DIR}/valgrind_linux.supp ${TEST_BINARY_PATH}

    DEPENDS ${TEST_BINARY_PATH}

)

add_custom_target(integration_test_valgrind

    COMMAND valgrind --leak-check=full --show-reachable=yes --track-fds=yes --suppressions=${kinetic_cpp_client_SOURCE_DIR}/valgrind_linux.supp ${INTEGRATION_TEST_BINARY_PATH}

    DEPENDS ${INTEGRATION_TEST_BINARY_PATH}

)



# Rule for generating docs

add_custom_target(doc

    doxygen ${kinetic_cpp_client_SOURCE_DIR}/Doxyfile

    WORKING_DIRECTORY ${kinetic_cpp_client_SOURCE_DIR}

    COMMENT "Generating API documentation with Doxygen" VERBATIM

)

set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES docs)



# Rule for linting

add_custom_target(lint

    ./bin/lint.sh

    WORKING_DIRECTORY ${kinetic_cpp_client_SOURCE_DIR}

    COMMENT "Running style checker" VERBATIM

)

Doxyfile

0 → 100644
+0 −0

File added.

Preview size limit exceeded, changes collapsed.

DoxygenMainPage.md

0 → 100644
+129 −0
Original line number Diff line number Diff line 
Quickstart {#mainpage}

======================

This library allows building C++ programs that interact with a Kinetic API implementation. It provides blocking and non-blocking styles of interaction depending on application needs.



Including the library

---------------------

The library can be built as a static or shared library. For examples of how to include the library in a CMake-based project see the examples repo at [https://github.com/Seagate/kinetic-cpp-examples](https://github.com/Seagate/kinetic-cpp-examples). Applications also need to add the `include/kinetic` directory to the include path and include `kinetic/kinetic.h`.



Opening a connection

--------------------

Before creating a connection, first create a `KineticConnectionFactory` using the handy helper method:



    kinetic::KineticConnectionFactory kinetic_connection_factory = kinetic::NewKineticConnectionFactory();



A single `KineticConnectionFactory` can be used to create many connections. Deleting the connection factory won't affect existing connections.



Next, create a connection options object to indicate the IP, port, user identity, and so on:



    kinetic::ConnectionOptions options;

    options.host = my_host;

    options.port = my_port;

    options.user_id = my_user_id;

    options.hmac_key = my_hmac_key;



Finally, open the connection:



    std::unique_ptr<kinetic::ConnectionHandle> connection;

    kinetic::KineticStatus status = kinetic_connection_factory.NewConnection(options, timeout_in_seconds, connection);



*Note*: If the connection needs to be shared between threads, call `NewThreadsafeConnection` instead.



To check whether the connection succeeded, check the value of `status.ok()`. If it's false, additional error information is available by calling `status.statusCode()` and `status.message()`.



The underlying connection can be accessed by calling `connection.blocking()` or `connection.nonblocking()`. The interface returned by `blocking()` and `nonblocking()` both access the same underlying TCP connection.



Performing blocking GET/PUT operations

--------------------------------------

PUT and GET calls make use of the `kientic::KineticRecord` class, which bundles the value/version/tag/algorithm fourpule allows construction via a series of `shared_ptr` instances (avoiding a copy) or a series of `string`s (avoiding extra typing).



To PUT a key:

    

    connection->blocking().Put(

        key,

        version,

        kinetic::IGNORE_VERSION,

        KineticRecord(value, version, tag, Message_Algorithm_SHA1));



Like most operations in the Kinetic C++ client, `Put` returns a `KineticStatus` object that indicates success and provides error information.



*Note*: The Kinetic C++ client makes extensive use of `shared_ptr` and `unique_ptr` to make memory transfer and ownership explicit while avoiding unnecessary copies. However, applications that prefer to avoid the complexity of `shared_ptr` and `unique_ptr` can just pass in `std::string` instances and accept the additional overhead.



GETting a key works similarly:



    std::unique_ptr<KineticRecord> record;

    connection->blocking().Get(key, record);



Performing non-blocking GET/PUT operations

------------------------------------------

The blocking API is convenient for simplicity or thread-per-connection style designs, but many applications use a non-blocking event-driven design. For these applications, it makes more sense to use a non-blocking API. The Kinetic C++ client's non-blocking API is very similar to the blocking API, with two important differences. First, all methods take an additional callback parameter that will be called once the operation succeeds or fails. Secondly, to actually perform IO the `Run` method needs to be called repeatedly.



As a simple example to perform several writes, first create a callback

class to receive success and failure messages:



    // PutCallback definition:

    class PutCallback : public PutCallbackInterface {

    public:

        void Success() {

            printf("Success!\n");

        }

        void Failure(KineticStatus error) {

            printf("Dismal failure!\n");

        }

    };

    //

    // ...

    //

    // PutCallback instantiation

    auto callback = make_shared<PutCallback>();



Then enqueue a series of operations:



    auto record = make_shared<KineticRecord>(value, version, tag, Message_Algorithm_SHA1);

        connection->nonblocking().Put(key1, version, kinetic::IGNORE_VERSION, record, callback);

    Message_Algorithm_SHA1);

        connection->nonblocking().Put(key2, version, kinetic::IGNORE_VERSION, record, callback);

    Message_Algorithm_SHA1);

        connection->nonblocking().Put(key3, version, kinetic::IGNORE_VERSION, record, callback);



Finally, call `Run` repeatedly to actually execute the operations:

    

    fd_set read_fds, write_fds;

    int nfd = 0;

    connection->nonblocking().Run(&read_fds, &write_fds, &nfd);

    while (there_is_work_to_do) {

        while (select(nfd + 1, &read_fds, &write_fds, NULL, NULL) <= 0);

        connection->nonblocking().Run(&read_fds, &write_fds, &num_fds);

    }



If desired, other fds can be added to the `fd_set`s so that the `select` call can wait for IO to be ready on fds controlled by the Kinetic API or other parts of the application.



GETs work very similarly. First, a callback implementation:



    class GetCallback : public GetCallbackInterface {

    public:

        void Success(const std::string &key, std::unique_ptr<KineticRecord> record) {

            printf("Good\n");

        }

        void Failure(KineticStatus error) {

            printf("Bad\n");

        }

    };

    //

    // ...

    //

    // GetCallback instantiation

    auto callback = make_shared<GetCallback>();



Next, enqueue some operations:



        connection->nonblocking().Get(key_1, callback);

        connection->nonblocking().Get(key_2, callback);

        connection->nonblocking().Get(key_3, callback);



The `Run` loop works exactly the same way as it does for the PUT case. Multiple GET/PUT/management operations can all be enqueued and they will be executed one at a time in order by repeated `Run` calls.



Closing a connection

--------------------

The `ConnectionHandle` destructor closes the connection. This means that the he connection remains open until the `unique_ptr` goes out of scope or the `ConnectionHandle` is manually freed.
 
 No newline at end of file

LICENSE

0 → 100644
+0 −0

File added.

Preview size limit exceeded, changes collapsed.