Remove folly dependency + fix aot on gpu (#3006)

* Adjust cpp backend integration test

* Fix test_cpp_backend.py

* Move some headers from logging.hh into cc

* Exchange folly logging against spdlog

* Remove default logging config in cpp worker + adjust config parameter in frontend

* Remove folly usage in handler

* Remove folly

* Remove separate libtorch

* Add missing yaml file

* Set ld_library_path in cpp test

* Support PT 2.3 and 2.2 in aot examples

* Fix cpp_backend test

* Add env command to READMEs

* Fix linter issue

* Update logging config name in MANIFEST

* Add note about gpu placement + simplify call to model->run

Author: mreso

MANIFEST.in

@@ -1,7 +1,7 @@
 include ts/frontend/model-server.jar
 include ts/cpp/bin/*
 include ts/cpp/lib/*
-include ts/cpp/resources/logging.config
+include ts/cpp/resources/logging.yaml
 include PyPiDescription.rst
 include ts/configs/*
-include ts/version.txt
+include ts/version.txt

cpp/CMakeLists.txt

@@ -4,7 +4,7 @@ project(torchserve_cpp VERSION 0.1)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC -D_GLIBCXX_USE_CXX11_ABI=1")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC")
 
 find_program(CLANG_TIDY_EXE NAMES "clang-tidy" REQUIRED)
 set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE}")
@@ -22,42 +22,52 @@ endif()
 
 
 find_package(Boost REQUIRED)
-find_package(folly REQUIRED)
-find_package(fmt REQUIRED)
-find_package(gflags REQUIRED)
 find_package(Torch REQUIRED)
 
 include(FetchContent)
 
 FetchContent_Declare(
-  yaml-cpp
-  GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
-  GIT_TAG 0.8.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master)
+  spdlog
+  GIT_REPOSITORY https://github.com/gabime/spdlog
+  GIT_TAG v1.13.0
 )
-FetchContent_GetProperties(yaml-cpp)
+FetchContent_GetProperties(spdlog)
 
-if(NOT yaml-cpp_POPULATED)
-  message(STATUS "Fetching yaml-cpp...")
-  FetchContent_Populate(yaml-cpp)
-  add_subdirectory(${yaml-cpp_SOURCE_DIR} ${yaml-cpp_BINARY_DIR})
+if(NOT spdlog_POPULATED)
+  message(STATUS "Fetching spdlog...")
+  FetchContent_Populate(spdlog)
+  add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR})
 endif()
 
+
+FetchContent_Declare(
+  json
+  GIT_REPOSITORY https://github.com/nlohmann/json
+  GIT_TAG v3.11.3
+)
+FetchContent_GetProperties(json)
+
+if(NOT json_POPULATED)
+  message(STATUS "Fetching json...")
+  FetchContent_Populate(json)
+  add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR})
+endif()
+
+include_directories("${json_SOURCE_DIR}/include" "${spdlog_SOURCE_DIR}/include")
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 
 include_directories(${TORCH_INCLUDE_DIRS})
-include_directories(${FOLLY_INCLUDE_DIRS})
 include_directories(${GTEST_INCLUDE_DIRS})
 include_directories(${GMOCK_INCLUDE_DIRS})
 
 include_directories("${CMAKE_INSTALL_PREFIX}/_deps")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
-set(FOLLY_LIBRARIES Folly::folly)
-
 # add subdirectories in src
 add_subdirectory(src/utils)
 add_subdirectory(src/backends)
 add_subdirectory(src/examples)
 add_subdirectory(test)
 
-FILE(COPY src/resources/logging.config DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
+FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")

cpp/README.md

@@ -3,8 +3,8 @@
 * C++17
 * GCC version: gcc-9
 * cmake version: 3.18+
+* Linux
 ## Installation and Running TorchServe CPP
-
 This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.
 
 ### Install dependencies
@@ -13,17 +13,22 @@ cd serve
 python ts_scripts/install_dependencies.py --cpp --environment dev [--cuda=cu121|cu118]
 ```
 ### Building the backend
-Don't forget to install or update TorchServe at this point if it wasn't previously installed.
+Don't forget to install or update TorchServe at this point if it wasn't previously installed. E.g. with:
+```
+python ts_scripts/install_from_src.py
+```
+
+Then build the backend:
 ```
 ## Dev Build
 cd cpp
 ./build.sh [-g cu121|cu118]
-
 ```
 
 ### Run TorchServe
 ```
 mkdir model_store
+export LD_LIBRARY_PATH=`python -c "import torch;from pathlib import Path;p=Path(torch.__file__);print(f\"{(p.parent / 'lib').as_posix()}:{(p.parents[1] / 'nvidia/nccl/lib').as_posix()}\")"`:$LD_LIBRARY_PATH
 torchserve --ncs --start --model-store model_store
 ```
 

cpp/build.sh

@@ -20,79 +20,6 @@ function detect_platform() {
   echo -e "${COLOR_GREEN}Detected platform: $PLATFORM ${COLOR_OFF}"
 }
 
-function install_folly() {
-  FOLLY_SRC_DIR=$BASE_DIR/third-party/folly
-  FOLLY_BUILD_DIR=$DEPS_DIR/folly-build
-
-  if [ ! -d "$FOLLY_BUILD_DIR" ] ; then
-    echo -e "${COLOR_GREEN}[ INFO ] Building Folly ${COLOR_OFF}"
-    cd $FOLLY_SRC_DIR
-
-    ./build/fbcode_builder/getdeps.py install-system-deps --recursive
-
-    ./build/fbcode_builder/getdeps.py build \
-    --allow-system-packages \
-    --scratch-path $FOLLY_BUILD_DIR \
-    --extra-cmake-defines='{"CMAKE_CXX_FLAGS": "-fPIC -D_GLIBCXX_USE_CXX11_ABI=1"}'
-
-    echo -e "${COLOR_GREEN}[ INFO ] Folly is installed ${COLOR_OFF}"
-  fi
-
-  cd "$BWD" || exit
-  echo "$FOLLY_BUILD_DIR/installed"
-}
-
-function install_libtorch() {
-  cd "$DEPS_DIR" || exit
-  TORCH_VERSION="2.2.1"
-  if [ -d "$DEPS_DIR/libtorch" ]; then
-    RAW_VERSION=`cat "$DEPS_DIR/libtorch/build-version"`
-    VERSION=`cat "$DEPS_DIR/libtorch/build-version" | cut -d "+" -f 1`
-    if [ "$USE_NIGHTLIES" = "true" ] && [[ ! "${RAW_VERSION}" =~ .*"dev".* ]]; then
-      rm -rf "$DEPS_DIR/libtorch"
-    elif [ "$USE_NIGHTLIES" == "" ] && [ "$VERSION" != "$TORCH_VERSION" ]; then
-      rm -rf "$DEPS_DIR/libtorch"
-    fi
-  fi
-  if [ "$PLATFORM" = "Mac" ]; then
-    if [ ! -d "$DEPS_DIR/libtorch" ]; then
-      if [[ $(uname -m) == 'x86_64' ]]; then
-        echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac x86_64 ${COLOR_OFF}"
-        wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
-        unzip -q libtorch-macos-x86_64-${TORCH_VERSION}.zip
-        rm libtorch-macos-x86_64-${TORCH_VERSION}.zip
-      else
-        echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac arm64 ${COLOR_OFF}"
-        wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
-        unzip -q libtorch-macos-arm64-${TORCH_VERSION}.zip
-        rm libtorch-macos-arm64-${TORCH_VERSION}.zip
-      fi
-    fi
-  elif [ "$PLATFORM" = "Windows" ]; then
-      echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Windows ${COLOR_OFF}"
-      # TODO: Windows
-      echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
-      exit 1
-  else  # Linux
-    if [ ! -d "$DEPS_DIR/libtorch" ]; then
-      echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Linux ${COLOR_OFF}"
-      if [ "$USE_NIGHTLIES" == true ]; then
-        URL=https://download.pytorch.org/libtorch/nightly/${CUDA}/libtorch-cxx11-abi-shared-with-deps-latest.zip
-      else
-        URL=https://download.pytorch.org/libtorch/${CUDA}/libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}%2B${CUDA}.zip
-      fi
-      wget -q $URL
-      ZIP_FILE=$(basename "$URL")
-      ZIP_FILE="${ZIP_FILE//%2B/+}"
-      unzip -q $ZIP_FILE
-      rm $ZIP_FILE
-    fi
-    echo -e "${COLOR_GREEN}[ INFO ] libtorch is installed ${COLOR_OFF}"
-  fi
-
-  cd "$BWD" || exit
-}
-
 function prepare_test_files() {
   echo -e "${COLOR_GREEN}[ INFO ]Preparing test files ${COLOR_OFF}"
   local EX_DIR="${TR_DIR}/examples/"
@@ -123,7 +50,7 @@ function prepare_test_files() {
       mv Transformer_model/tokenizer.json ${HANDLER_DIR}/tokenizer.json
       export TOKENIZERS_PARALLELISM=""
     fi
-    if [ ! -f "${EX_DIR}/aot_inductor/resnet_handler/resne50_pt2.so" ]; then
+    if [ ! -f "${EX_DIR}/aot_inductor/resnet_handler/resnet50_pt2.so" ]; then
       local HANDLER_DIR=${EX_DIR}/aot_inductor/resnet_handler/
       cd ${HANDLER_DIR}
       python ${BASE_DIR}/../examples/cpp/aot_inductor/resnet/resnet50_torch_export.py
@@ -133,6 +60,7 @@ function prepare_test_files() {
 }
 
 function build() {
+  echo -e "${COLOR_GREEN}[ INFO ]Building backend ${COLOR_OFF}"
   MAYBE_BUILD_QUIC=""
   if [ "$WITH_QUIC" == true ] ; then
     setup_mvfst
@@ -151,50 +79,37 @@ function build() {
     PREFIX=$BWD
   fi
 
-  MAYBE_CUDA_COMPILER=""
-  if [ "$CUDA" != "" ]; then
-    MAYBE_CUDA_COMPILER='-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc'
-  fi
-
-  MAYBE_NIGHTLIES="-Dnightlies=OFF"
-  if [ "$USE_NIGHTLIES" == true ]; then
-    MAYBE_NIGHTLIES="-Dnightlies=ON"
-  fi
-
   # Build torchserve_cpp with cmake
   cd "$BWD" || exit
-  FOLLY_CMAKE_DIR=$DEPS_DIR/folly-build/installed
-  find $FOLLY_CMAKE_DIR -name "lib*.*"  -exec ln -s "{}" $LIBS_DIR/ \;
+
+  CMAKE_PREFIX_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'`
+
   if [ "$PLATFORM" = "Linux" ]; then
+    NCCL_PATH=`python3 -c 'import torch;from pathlib import Path;print(Path(torch.__file__).parents[1]/"nvidia"/"nccl"/"lib")'`
+    export LD_LIBRARY_PATH=${NCCL_PATH}:${LD_LIBRARY_PATH}
     cmake                                                                                     \
-    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch"                       \
+    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH"                                                           \
     -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
     "$MAYBE_BUILD_QUIC"                                                                       \
     "$MAYBE_BUILD_TESTS"                                                                      \
     "$MAYBE_BUILD_SHARED_LIBS"                                                                \
     "$MAYBE_OVERRIDE_CXX_FLAGS"                                                               \
     "$MAYBE_USE_STATIC_DEPS"                                                                  \
     "$MAYBE_LIB_FUZZING_ENGINE"                                                               \
-    "$MAYBE_CUDA_COMPILER"                                                                    \
-    "$MAYBE_NIGHTLIES"                                                                        \
     ..
 
-    if [ "$CUDA" = "cu118" ] || [ "$CUDA" = "cu121" ]; then
-      export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/bin/nvcc
-    fi
   elif [ "$PLATFORM" = "Mac" ]; then
     export LIBRARY_PATH=${LIBRARY_PATH}:`brew --prefix icu4c`/lib:`brew --prefix libomp`/lib
 
     cmake                                                                                     \
-    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch"                       \
+    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH"                                        \
     -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
     "$MAYBE_BUILD_QUIC"                                                                       \
     "$MAYBE_BUILD_TESTS"                                                                      \
     "$MAYBE_BUILD_SHARED_LIBS"                                                                \
     "$MAYBE_OVERRIDE_CXX_FLAGS"                                                               \
     "$MAYBE_USE_STATIC_DEPS"                                                                  \
     "$MAYBE_LIB_FUZZING_ENGINE"                                                               \
-    "$MAYBE_NIGHTLIES"                                                                        \
     "-DLLAMA_METAL=OFF"                                                                       \
     ..
 
@@ -220,12 +135,6 @@ function build() {
   fi
 }
 
-function symlink_torch_libs() {
-  if [ "$PLATFORM" = "Linux" ]; then
-    ln -sf ${DEPS_DIR}/libtorch/lib/*.so* ${LIBS_DIR}
-  fi
-}
-
 function install_torchserve_cpp() {
   TARGET_DIR=`python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"`
 
@@ -244,16 +153,12 @@ WITH_QUIC=false
 INSTALL_DEPENDENCIES=false
 PREFIX=""
 COMPILER_FLAGS=""
-CUDA="cpu"
-USAGE="./build.sh [-j num_jobs] [-g cu118|cu121] [-q|--with-quic] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags] [-n|--nighlies]"
+USAGE="./build.sh [-j num_jobs] [-q|--with-quic] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags]"
 while [ "$1" != "" ]; do
   case $1 in
     -j | --jobs ) shift
                   JOBS=$1
                   ;;
-    -g | --cuda-version ) shift
-                  CUDA=$1
-                  ;;
     -q | --with-quic )
                   WITH_QUIC=true
                   ;;
@@ -268,9 +173,6 @@ while [ "$1" != "" ]; do
                   shift
                   COMPILER_FLAGS=$1
       ;;
-    -n | --nightlies )
-                  USE_NIGHTLIES=true
-      ;;
     * )           echo $USAGE
                   exit 1
 esac
@@ -305,9 +207,6 @@ cd $BASE_DIR
 
 git submodule update --init --recursive
 
-install_folly
-install_libtorch
 prepare_test_files
 build
-symlink_torch_libs
 install_torchserve_cpp

cpp/src/backends/CMakeLists.txt

@@ -1,3 +1,17 @@
+FetchContent_Declare(
+  gflags
+  GIT_REPOSITORY https://github.com/gflags/gflags
+  GIT_TAG v2.2.2
+)
+FetchContent_GetProperties(spdlgflagsog)
+
+if(NOT gflags_POPULATED)
+  message(STATUS "Fetching gflags...")
+  FetchContent_Populate(gflags)
+  add_subdirectory(${gflags_SOURCE_DIR} ${gflags_BINARY_DIR})
+endif()
+
+
 set(TS_BACKENDS_SRC_DIR "${torchserve_cpp_SOURCE_DIR}/src/backends")
 set(TS_BACKENDS_CORE_SRC_DIR "${TS_BACKENDS_SRC_DIR}/core")
 set(TS_BACKENDS_PROTOCOL_SRC_DIR "${TS_BACKENDS_SRC_DIR}/protocol")
@@ -11,7 +25,7 @@ list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/ot
 list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/socket.cc)
 add_library(ts_backends_protocol SHARED ${TS_BACKENDS_PROTOCOL_SOURCE_FILES})
 target_include_directories(ts_backends_protocol PUBLIC ${TS_BACKENDS_PROTOCOL_SRC_DIR})
-target_link_libraries(ts_backends_protocol PRIVATE ts_utils ${FOLLY_LIBRARIES})
+target_link_libraries(ts_backends_protocol PRIVATE ts_utils)
 install(TARGETS ts_backends_protocol DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
 
 # build library ts_backend_core
@@ -22,7 +36,7 @@ list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/base_handler.cc)
 list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/torch_scripted_handler.cc)
 add_library(ts_backends_core SHARED ${BACKEND_SOURCE_FILES})
 target_include_directories(ts_backends_core PUBLIC ${TS_BACKENDS_CORE_SRC_DIR})
-target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${FOLLY_LIBRARIES} ${TORCH_LIBRARIES})
+target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${TORCH_LIBRARIES})
 install(TARGETS ts_backends_core DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
 
 # build exe model_worker_socket
@@ -37,5 +51,5 @@ target_include_directories(model_worker_socket PRIVATE
   ${TS_BACKENDS_TORCH_SCRIPTED_SRC_DIR}
 )
 target_link_libraries(model_worker_socket
-  PRIVATE ts_backends_core ts_backends_protocol ${FOLLY_LIBRARIES} ${TORCH_LIBRARIES})
+  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags)
 install(TARGETS model_worker_socket DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/bin)