CPP build in pure cmake (#3021)

* Move cpp install into cmake file

* add pytest to cpp ci workflow

* Fix cpp ci

* Add debugging

* Change exec dir for test

* Start running of tests to ctest; removed from build.sh start fixing unit tests

* fix tests

* Enable test before adding subdirectory; use make test in build.sh

* Remove build.sh

* Fix import error in python 3.9

* Fix examples + halde nccl not found

* Fix lib not found

* convert tokenizer-cpp from submodule to fetch_content

* Change skip condition for cpp pytest

* Remove kineto submodule

* Remove yaml-cpp submodule

* Convert llama2.so to fetch_content

* Convert llama2.c submodule into fetch_content

* fix relateive path

---------

Co-authored-by: lxning <[email protected]>
Co-authored-by: lxning <[email protected]>

Author: mreso

.github/workflows/ci-cpu-cpp.yml

@@ -53,5 +53,10 @@ jobs:
           python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
-          ./build.sh
+          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir build && sudo mount --bind /mnt/_build build
+          cd build && cmake ..
+          make -j && make install
+          make test
+      - name: Run test
+        run: |
+          cd test && pytest pytest/test_cpp_backend.py

.gitmodules

@@ -1,18 +1,3 @@
 [submodule "third_party/google/rpc"]
 	path = third_party/google/rpc
 	url = https://github.com/googleapis/googleapis.git
-[submodule "cpp/third-party/llama2.c"]
-	path = cpp/third-party/llama2.c
-	url = https://github.com/karpathy/llama2.c
-[submodule "cpp/third-party/llama2.so"]
-	path = cpp/third-party/llama2.so
-	url = https://github.com/mreso/llama2.so.git
-[submodule "cpp/third-party/yaml-cpp"]
-	path = cpp/third-party/yaml-cpp
-	url = https://github.com/jbeder/yaml-cpp.git
-[submodule "cpp/third-party/tokenizers-cpp"]
-	path = cpp/third-party/tokenizers-cpp
-	url = https://github.com/mlc-ai/tokenizers-cpp.git
-[submodule "cpp/third-party/kineto"]
-	path = cpp/third-party/kineto
-	url = https://github.com/pytorch/kineto.git

cpp/CMakeLists.txt

@@ -1,10 +1,13 @@
 cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 project(torchserve_cpp VERSION 0.1)
 
+enable_testing()
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC")
+set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
 
 find_program(CLANG_TIDY_EXE NAMES "clang-tidy" REQUIRED)
 set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE}")
@@ -20,10 +23,29 @@ if(CLANG_FORMAT_EXE)
 
 endif()
 
+execute_process(COMMAND python -c "import importlib.util; print(importlib.util.find_spec('nvidia') is not None)"
+                OUTPUT_VARIABLE NVIDIA_AVAILABLE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(${NVIDIA_AVAILABLE} STREQUAL "True")
+  execute_process(COMMAND python -c "import nvidia;from pathlib import Path;print(Path(nvidia.__file__).parent/'nccl'/'lib')"
+                  OUTPUT_VARIABLE NCCL_PATH
+                  OUTPUT_STRIP_TRAILING_WHITESPACE)
+  list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.2")
+endif()
+
+execute_process(COMMAND python -c "import torch;print(torch.utils.cmake_prefix_path)"
+                OUTPUT_VARIABLE TORCH_CMAKE_PREFIX_PATH
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+list(APPEND CMAKE_PREFIX_PATH ${TORCH_CMAKE_PREFIX_PATH})
 
 find_package(Boost REQUIRED)
 find_package(Torch REQUIRED)
 
+find_library(NCCL_LIBRARY nccl HINTS ${NCCL_PATH})
+if("${NCCL_LIBRARY}" STREQUAL "NCCL_LIBRARY-NOTFOUND")
+  set(NCCL_LIBRARY "")
+endif()
+
 include(FetchContent)
 
 FetchContent_Declare(
@@ -71,3 +93,5 @@ add_subdirectory(src/examples)
 add_subdirectory(test)
 
 FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
+
+install(SCRIPT "install.cmake")

cpp/README.md

@@ -17,9 +17,9 @@ cd serve/docker
 Start the container and optionally bind mount a build directory into the container to persist build artifacts across container runs
 ```
 # For CPU support
-docker run [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
+docker run [-v /path/to/build/dir:/serve/cpp/build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
 # For GPU support
-docker run --gpus all [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
+docker run --gpus all [-v /path/to/build/dir:/serve/cpp/build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
 ```
 `Warning`: The dev docker container does not install all necessary dependencies or build Torchserve CPP. Please follow the steps below after starting the container.
 
@@ -41,7 +41,12 @@ Then build the backend:
 ```
 ## Dev Build
 cd cpp
-./build.sh
+mkdir build && cd build
+# Optionally, you can skip building the tests by adding: -DBUILD_TESTS=OFF
+cmake ..
+make -j && make install
+## Optionally, you can run the tests with
+make test
 ```
 
 ### Run TorchServe

cpp/build.sh

@@ -0,0 +1,14 @@
+execute_process(COMMAND python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"
+                OUTPUT_VARIABLE TARGET_DIR
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+message("Installing cpp backend into ${TARGET_DIR}")
+
+if(EXISTS ${TARGET_DIR})
+    execute_process(COMMAND rm -rf ${TARGET_DIR})
+endif()
+
+execute_process(COMMAND mkdir ${TARGET_DIR})
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/bin ${TARGET_DIR}/bin)
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/libs ${TARGET_DIR}/lib)
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/resources ${TARGET_DIR}/resources)

cpp/install.cmake

@@ -26,7 +26,7 @@ list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/so
 add_library(ts_backends_protocol SHARED ${TS_BACKENDS_PROTOCOL_SOURCE_FILES})
 target_include_directories(ts_backends_protocol PUBLIC ${TS_BACKENDS_PROTOCOL_SRC_DIR})
 target_link_libraries(ts_backends_protocol PRIVATE ts_utils)
-install(TARGETS ts_backends_protocol DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_backends_protocol DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 # build library ts_backend_core
 set(BACKEND_SOURCE_FILES "")
@@ -37,7 +37,7 @@ list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/torch_scripted_h
 add_library(ts_backends_core SHARED ${BACKEND_SOURCE_FILES})
 target_include_directories(ts_backends_core PUBLIC ${TS_BACKENDS_CORE_SRC_DIR})
 target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${TORCH_LIBRARIES})
-install(TARGETS ts_backends_core DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_backends_core DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 # build exe model_worker_socket
 add_executable(model_worker_socket
@@ -51,5 +51,5 @@ target_include_directories(model_worker_socket PRIVATE
   ${TS_BACKENDS_TORCH_SCRIPTED_SRC_DIR}
 )
 target_link_libraries(model_worker_socket
-  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags)
-install(TARGETS model_worker_socket DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/bin)
+  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags ${NCCL_LIBRARY})
+install(TARGETS model_worker_socket DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)

cpp/src/backends/CMakeLists.txt

@@ -1,6 +1,3 @@
-
-file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/../../test/resources/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/)
-
 add_subdirectory("../../../examples/cpp/babyllama/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/babyllama/babyllama_handler/")
 
 add_subdirectory("../../../examples/cpp/llamacpp/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/llamacpp/llamacpp_handler/")

cpp/src/examples/CMakeLists.txt

@@ -47,7 +47,7 @@ else()
   target_link_libraries(ts_utils ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} yaml-cpp nlohmann_json fmt)
 endif()
 
-install(TARGETS ts_utils DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_utils DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/ifoo.hh)
 list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/foo.hh)

cpp/src/utils/CMakeLists.txt

@@ -6,13 +6,14 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(googletest)
 # run google test
-enable_testing()
+
 
 set(TEST_BINARY ${CMAKE_PROJECT_NAME}_test)
 file(GLOB_RECURSE TEST_SOURCES LIST_DIRECTORIES false *.cc *.hh)
+file(COPY resources/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/resources/)
 
 add_executable(${TEST_BINARY} ${TEST_SOURCES})
-target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES})
+target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES} ${NCCL_LIBRARY})
 
 include(GoogleTest)
 gtest_discover_tests(${TEST_BINARY})