CPP build in pure cmake

.github/workflows/ci-cpu-cpp.yml

@@ -53,5 +53,10 @@ jobs:
           python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
-          ./build.sh
+          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir build && sudo mount --bind /mnt/_build build
+          cd build && cmake ..
+          make -j && make install
+          make test
+      - name: Run test
+        run: |
+          cd test && pytest pytest/test_cpp_backend.py

.gitmodules

@@ -1,18 +1,3 @@
 [submodule "third_party/google/rpc"]
 	path = third_party/google/rpc
 	url = https://github.com/googleapis/googleapis.git
-[submodule "cpp/third-party/llama2.c"]
-	path = cpp/third-party/llama2.c
-	url = https://github.com/karpathy/llama2.c
-[submodule "cpp/third-party/llama2.so"]
-	path = cpp/third-party/llama2.so
-	url = https://github.com/mreso/llama2.so.git
-[submodule "cpp/third-party/yaml-cpp"]
-	path = cpp/third-party/yaml-cpp
-	url = https://github.com/jbeder/yaml-cpp.git
-[submodule "cpp/third-party/tokenizers-cpp"]
-	path = cpp/third-party/tokenizers-cpp
-	url = https://github.com/mlc-ai/tokenizers-cpp.git
-[submodule "cpp/third-party/kineto"]
-	path = cpp/third-party/kineto
-	url = https://github.com/pytorch/kineto.git

cpp/CMakeLists.txt

@@ -1,10 +1,13 @@
 cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 project(torchserve_cpp VERSION 0.1)
 
+enable_testing()
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC")
+set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
 
 find_program(CLANG_TIDY_EXE NAMES "clang-tidy" REQUIRED)
 set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE}")
@@ -20,10 +23,29 @@ if(CLANG_FORMAT_EXE)
 
 endif()
 
+execute_process(COMMAND python -c "import importlib.util; print(importlib.util.find_spec('nvidia') is not None)"
+                OUTPUT_VARIABLE NVIDIA_AVAILABLE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(${NVIDIA_AVAILABLE} STREQUAL "True")
+  execute_process(COMMAND python -c "import nvidia;from pathlib import Path;print(Path(nvidia.__file__).parent/'nccl'/'lib')"
+                  OUTPUT_VARIABLE NCCL_PATH
+                  OUTPUT_STRIP_TRAILING_WHITESPACE)
+  list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.2")
+endif()
+
+execute_process(COMMAND python -c "import torch;print(torch.utils.cmake_prefix_path)"
+                OUTPUT_VARIABLE TORCH_CMAKE_PREFIX_PATH
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+list(APPEND CMAKE_PREFIX_PATH ${TORCH_CMAKE_PREFIX_PATH})
 
 find_package(Boost REQUIRED)
 find_package(Torch REQUIRED)
 
+find_library(NCCL_LIBRARY nccl HINTS ${NCCL_PATH})
+if("${NCCL_LIBRARY}" STREQUAL "NCCL_LIBRARY-NOTFOUND")
+  set(NCCL_LIBRARY "")
+endif()
+
 include(FetchContent)
 
 FetchContent_Declare(
@@ -71,3 +93,5 @@ add_subdirectory(src/examples)
 add_subdirectory(test)
 
 FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
+
+install(SCRIPT "install.cmake")

cpp/README.md

@@ -17,9 +17,9 @@ cd serve/docker
 Start the container and optionally bind mount a build directory into the container to persist build artifacts across container runs
 ```
 # For CPU support
-docker run [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
+docker run [-v /path/to/build/dir:/serve/cpp/build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
 # For GPU support
-docker run --gpus all [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
+docker run --gpus all [-v /path/to/build/dir:/serve/cpp/build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
 ```
 `Warning`: The dev docker container does not install all necessary dependencies or build Torchserve CPP. Please follow the steps below after starting the container.
 
@@ -41,7 +41,12 @@ Then build the backend:
 ```
 ## Dev Build
 cd cpp
-./build.sh
+mkdir build && cd build
+# Optionally, you can skip building the tests by adding: -DBUILD_TESTS=OFF
+cmake ..
+make -j && make install
+## Optionally, you can run the tests with
+make test
 ```
 
 ### Run TorchServe

cpp/install.cmake

@@ -0,0 +1,14 @@
+execute_process(COMMAND python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"
+                OUTPUT_VARIABLE TARGET_DIR
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+message("Installing cpp backend into ${TARGET_DIR}")
+
+if(EXISTS ${TARGET_DIR})
+    execute_process(COMMAND rm -rf ${TARGET_DIR})
+endif()
+
+execute_process(COMMAND mkdir ${TARGET_DIR})
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/bin ${TARGET_DIR}/bin)
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/libs ${TARGET_DIR}/lib)
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/resources ${TARGET_DIR}/resources)

cpp/src/backends/CMakeLists.txt

@@ -26,7 +26,7 @@ list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/so
 add_library(ts_backends_protocol SHARED ${TS_BACKENDS_PROTOCOL_SOURCE_FILES})
 target_include_directories(ts_backends_protocol PUBLIC ${TS_BACKENDS_PROTOCOL_SRC_DIR})
 target_link_libraries(ts_backends_protocol PRIVATE ts_utils)
-install(TARGETS ts_backends_protocol DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_backends_protocol DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 # build library ts_backend_core
 set(BACKEND_SOURCE_FILES "")
@@ -37,7 +37,7 @@ list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/torch_scripted_h
 add_library(ts_backends_core SHARED ${BACKEND_SOURCE_FILES})
 target_include_directories(ts_backends_core PUBLIC ${TS_BACKENDS_CORE_SRC_DIR})
 target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${TORCH_LIBRARIES})
-install(TARGETS ts_backends_core DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_backends_core DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 # build exe model_worker_socket
 add_executable(model_worker_socket
@@ -51,5 +51,5 @@ target_include_directories(model_worker_socket PRIVATE
   ${TS_BACKENDS_TORCH_SCRIPTED_SRC_DIR}
 )
 target_link_libraries(model_worker_socket
-  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags)
-install(TARGETS model_worker_socket DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/bin)
+  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags ${NCCL_LIBRARY})
+install(TARGETS model_worker_socket DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)

cpp/src/utils/CMakeLists.txt

@@ -47,7 +47,7 @@ else()
   target_link_libraries(ts_utils ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} yaml-cpp nlohmann_json fmt)
 endif()
 
-install(TARGETS ts_utils DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_utils DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/ifoo.hh)
 list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/foo.hh)

cpp/test/CMakeLists.txt

@@ -6,13 +6,13 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(googletest)
 # run google test
-enable_testing()
+
 
 set(TEST_BINARY ${CMAKE_PROJECT_NAME}_test)
 file(GLOB_RECURSE TEST_SOURCES LIST_DIRECTORIES false *.cc *.hh)
 
 add_executable(${TEST_BINARY} ${TEST_SOURCES})
-target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES})
+target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES} ${NCCL_LIBRARY})
 
 include(GoogleTest)
 gtest_discover_tests(${TEST_BINARY})

cpp/test/backends/otf_protocol_and_handler_test.cc

@@ -24,7 +24,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
       // model_name length
       .WillOnce(::testing::Return(5))
       // model_path length
-      .WillOnce(::testing::Return(42))
+      .WillOnce(::testing::Return(45))
       // batch_size
       .WillOnce(::testing::Return(1))
       // handler length
@@ -44,8 +44,8 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
         strncpy(data, "mnist", length);
       }))
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
-        ASSERT_EQ(length, 42);
-        strncpy(data, "test/resources/examples/mnist/base_handler", length);
+        ASSERT_EQ(length, 45);
+        strncpy(data, "../test/resources/examples/mnist/base_handler", length);
       }))
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
         ASSERT_EQ(length, 11);
@@ -59,7 +59,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
   EXPECT_CALL(*client_socket, SendAll(testing::_, testing::_)).Times(1);
   auto load_model_request = OTFMessage::RetrieveLoadMsg(*client_socket);
   ASSERT_EQ(load_model_request->model_dir,
-            "test/resources/examples/mnist/base_handler");
+            "../test/resources/examples/mnist/base_handler");
   ASSERT_EQ(load_model_request->model_name, "mnist");
   ASSERT_EQ(load_model_request->envelope, "");
   ASSERT_EQ(load_model_request->model_name, "mnist");
@@ -68,9 +68,9 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
 
   // initialize backend
   auto backend = std::make_shared<torchserve::Backend>();
-  MetricsRegistry::Initialize("test/resources/metrics/default_config.yaml",
+  MetricsRegistry::Initialize("../test/resources/metrics/default_config.yaml",
                               MetricsContext::BACKEND);
-  backend->Initialize("test/resources/examples/mnist/base_handler");
+  backend->Initialize("../test/resources/examples/mnist/base_handler");
 
   // load the model
   auto load_model_response = backend->LoadModel(load_model_request);
@@ -125,7 +125,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
         ASSERT_EQ(length, 3883);
         // strncpy(data, "valu", length);
-        std::ifstream input("test/resources/examples/mnist/0_png.pt",
+        std::ifstream input("../test/resources/examples/mnist/0_png.pt",
                             std::ios::in | std::ios::binary);
         std::vector<char> image((std::istreambuf_iterator<char>(input)),
                                 (std::istreambuf_iterator<char>()));