Merge branch 'master' into m1

Author: udaij12

.github/workflows/ci-cpu-cpp.yml

@@ -16,17 +16,42 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macOS-latest]
+        os: [ubuntu-20.04]
     steps:
+      # - name: Setup Python for M1
+      #   if: matrix.os == 'macos-14'
+      #   uses: actions/setup-python@v5
+      #   with:
+      #     python-version: '3.10'
+      - name: Setup Python for all other OS
+        if: matrix.os != 'macos-14'
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+          architecture: x64
+      - name: Setup Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
       - name: Checkout TorchServe
-        uses: actions/checkout@v2
-      - name: Install libtorch - macOS
-        if: matrix.os == 'macOS-latest'
-        run: |
-            brew install libtorch
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      # - name: Install libtorch - macOS
+      #   if: matrix.os == 'macOS-latest'
+      #   run: |
+      #       brew install libtorch
       - name: Install dependencies
         run: |
-          python ts_scripts/install_dependencies.py --environment=dev --cpp
+          sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
+      - name: Install TorchServe
+        run: |
+          python ts_scripts/install_from_src.py
+      - name: Print Env
+        run: |
+          python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && ./build.sh
+          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
+          ./build.sh

.github/workflows/torchserve-nightly-build.yml

@@ -10,6 +10,10 @@ jobs:
     steps:
       - name: Setup Conda
         uses: s-weigand/setup-conda@v1
+        with:
+          activate-conda: true
+          update-conda: false
+          python-version: "3.9"
       - run: conda --version
       - run: conda install -y conda-build anaconda-client
       - name: Checkout TorchServe

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "third_party/google/rpc"]
 	path = third_party/google/rpc
 	url = https://github.com/googleapis/googleapis.git
-[submodule "cpp/third-party/llama.cpp"]
-	path = cpp/third-party/llama.cpp
-	url = https://github.com/ggerganov/llama.cpp.git
 [submodule "cpp/third-party/llama2.c"]
 	path = cpp/third-party/llama2.c
 	url = https://github.com/karpathy/llama2.c

MANIFEST.in

@@ -1,7 +1,7 @@
 include ts/frontend/model-server.jar
 include ts/cpp/bin/*
 include ts/cpp/lib/*
-include ts/cpp/resources/logging.config
+include ts/cpp/resources/logging.yaml
 include PyPiDescription.rst
 include ts/configs/*
-include ts/version.txt
+include ts/version.txt

cpp/CMakeLists.txt

@@ -4,7 +4,7 @@ project(torchserve_cpp VERSION 0.1)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC -D_GLIBCXX_USE_CXX11_ABI=1")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC")
 
 find_program(CLANG_TIDY_EXE NAMES "clang-tidy" REQUIRED)
 set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE}")
@@ -18,35 +18,56 @@ if(CLANG_FORMAT_EXE)
     ${PROJECT_SOURCE_DIR}/test/*.hh
   )
 
-  add_custom_target(format
-    COMMAND
-      ${CLANG_FORMAT_EXE} -i -style=google ${ALL_CXX_SOURCE_FILES}
-  )
 endif()
 
 
 find_package(Boost REQUIRED)
-find_package(folly REQUIRED)
-find_package(fmt REQUIRED)
-find_package(gflags REQUIRED)
 find_package(Torch REQUIRED)
 
+include(FetchContent)
+
+FetchContent_Declare(
+  spdlog
+  GIT_REPOSITORY https://github.com/gabime/spdlog
+  GIT_TAG v1.13.0
+)
+FetchContent_GetProperties(spdlog)
+
+if(NOT spdlog_POPULATED)
+  message(STATUS "Fetching spdlog...")
+  FetchContent_Populate(spdlog)
+  add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR})
+endif()
+
+
+FetchContent_Declare(
+  json
+  GIT_REPOSITORY https://github.com/nlohmann/json
+  GIT_TAG v3.11.3
+)
+FetchContent_GetProperties(json)
+
+if(NOT json_POPULATED)
+  message(STATUS "Fetching json...")
+  FetchContent_Populate(json)
+  add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR})
+endif()
+
+include_directories("${json_SOURCE_DIR}/include" "${spdlog_SOURCE_DIR}/include")
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 
 include_directories(${TORCH_INCLUDE_DIRS})
-include_directories(${FOLLY_INCLUDE_DIRS})
 include_directories(${GTEST_INCLUDE_DIRS})
 include_directories(${GMOCK_INCLUDE_DIRS})
 
 include_directories("${CMAKE_INSTALL_PREFIX}/_deps")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
-set(FOLLY_LIBRARIES Folly::folly)
-
 # add subdirectories in src
 add_subdirectory(src/utils)
 add_subdirectory(src/backends)
 add_subdirectory(src/examples)
 add_subdirectory(test)
 
-FILE(COPY src/resources/logging.config DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
+FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")

cpp/README.md

@@ -3,38 +3,41 @@
 * C++17
 * GCC version: gcc-9
 * cmake version: 3.18+
+* Linux
 ## Installation and Running TorchServe CPP
+This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.
 
 ### Install dependencies
 ```
 cd serve
 python ts_scripts/install_dependencies.py --cpp --environment dev [--cuda=cu121|cu118]
 ```
 ### Building the backend
+Don't forget to install or update TorchServe at this point if it wasn't previously installed. E.g. with:
 ```
-## Dev Build
-cd cpp
-./build.sh [-g cu121|cu118]
-
-## Install TorchServe from source
-cd ..
 python ts_scripts/install_from_src.py
 ```
-### Set Environment Var
-#### On Mac
-```
-export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/_build/_deps/libtorch/lib
-```
-#### On Ubuntu
+
+Then build the backend:
 ```
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/_build/_deps/libtorch/lib
+## Dev Build
+cd cpp
+./build.sh [-g cu121|cu118]
 ```
 
 ### Run TorchServe
 ```
 mkdir model_store
+export LD_LIBRARY_PATH=`python -c "import torch;from pathlib import Path;p=Path(torch.__file__);print(f\"{(p.parent / 'lib').as_posix()}:{(p.parents[1] / 'nvidia/nccl/lib').as_posix()}\")"`:$LD_LIBRARY_PATH
 torchserve --ncs --start --model-store model_store
 ```
+
+### Clean the build directory
+To clean the build directory in order to rebuild from scratch simply delete the cpp/_build directory with
+```
+rm -rf cpp/_build
+```
+
 ## Backend
 TorchServe cpp backend can run as a process, which is similar to [TorchServe Python backend](https://github.com/pytorch/serve/tree/master/ts). By default, TorchServe supports torch scripted model in cpp backend. Other platforms such as MxNet, ONNX can be supported through custom handlers following the TorchScript example [src/backends/handler/torch_scripted_handler.hh](https://github.com/pytorch/serve/blob/master/cpp/src/backends/handler/torch_scripted_handler.hh).
 ### Custom Handler
@@ -89,11 +92,11 @@ python -c "import ts; from pathlib import Path; print((Path(ts.__file__).parent
 3. Make sure you have the right conda/venv environment activated during building that you're also using to run TorchServe.
 
 Q: Build on Mac fails with `Library not loaded: @rpath/libomp.dylib`
-A: Install libomp with brew and link in /usr/local/lib 
+A: Install libomp with brew and link in /usr/local/lib
 ```bash
 brew install libomp
 sudo ln -s /opt/homebrew/opt/libomp/lib/libomp.dylib /usr/local/lib/libomp.dylib
 ```
 
 Q: When loading a handler which uses a model exported with torch._export.aot_compile the handler dies with "error: Error in dlopen: MODEL.SO : undefined symbol: SOME_SYMBOL".
-A: Make sure that you are using matching libtorch and Pytorch versions for inference and export, respectively.
+A: Make sure that you are using matching libtorch and Pytorch versions for inference and export, respectively.