Skip to content

Commit b92cf42

Browse files
authored
Remove folly dependency + fix aot on gpu (#3006)
* Adjust cpp backend integration test * Fix test_cpp_backend.py * Move some headers from logging.hh into cc * Exchange folly logging against spdlog * Remove default logging config in cpp worker + adjust config parameter in frontend * Remove folly usage in handler * Remove folly * Remove separate libtorch * Add missing yaml file * Set ld_library_path in cpp test * Support PT 2.3 and 2.2 in aot examples * Fix cpp_backend test * Add env command to READMEs * Fix linter issue * Update logging config name in MANIFEST * Add note about gpu placement + simplify call to model->run
1 parent ce3832a commit b92cf42

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+553
-388
lines changed

MANIFEST.in

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
include ts/frontend/model-server.jar
22
include ts/cpp/bin/*
33
include ts/cpp/lib/*
4-
include ts/cpp/resources/logging.config
4+
include ts/cpp/resources/logging.yaml
55
include PyPiDescription.rst
66
include ts/configs/*
7-
include ts/version.txt
7+
include ts/version.txt

cpp/CMakeLists.txt

+26-16
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ project(torchserve_cpp VERSION 0.1)
44
set(CMAKE_CXX_STANDARD 17)
55
set(CMAKE_CXX_STANDARD_REQUIRED True)
66
set(CMAKE_CXX_EXTENSIONS OFF)
7-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC -D_GLIBCXX_USE_CXX11_ABI=1")
7+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC")
88

99
find_program(CLANG_TIDY_EXE NAMES "clang-tidy" REQUIRED)
1010
set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE}")
@@ -22,42 +22,52 @@ endif()
2222

2323

2424
find_package(Boost REQUIRED)
25-
find_package(folly REQUIRED)
26-
find_package(fmt REQUIRED)
27-
find_package(gflags REQUIRED)
2825
find_package(Torch REQUIRED)
2926

3027
include(FetchContent)
3128

3229
FetchContent_Declare(
33-
yaml-cpp
34-
GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
35-
GIT_TAG 0.8.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master)
30+
spdlog
31+
GIT_REPOSITORY https://github.com/gabime/spdlog
32+
GIT_TAG v1.13.0
3633
)
37-
FetchContent_GetProperties(yaml-cpp)
34+
FetchContent_GetProperties(spdlog)
3835

39-
if(NOT yaml-cpp_POPULATED)
40-
message(STATUS "Fetching yaml-cpp...")
41-
FetchContent_Populate(yaml-cpp)
42-
add_subdirectory(${yaml-cpp_SOURCE_DIR} ${yaml-cpp_BINARY_DIR})
36+
if(NOT spdlog_POPULATED)
37+
message(STATUS "Fetching spdlog...")
38+
FetchContent_Populate(spdlog)
39+
add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR})
4340
endif()
4441

42+
43+
FetchContent_Declare(
44+
json
45+
GIT_REPOSITORY https://github.com/nlohmann/json
46+
GIT_TAG v3.11.3
47+
)
48+
FetchContent_GetProperties(json)
49+
50+
if(NOT json_POPULATED)
51+
message(STATUS "Fetching json...")
52+
FetchContent_Populate(json)
53+
add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR})
54+
endif()
55+
56+
include_directories("${json_SOURCE_DIR}/include" "${spdlog_SOURCE_DIR}/include")
57+
4558
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
4659

4760
include_directories(${TORCH_INCLUDE_DIRS})
48-
include_directories(${FOLLY_INCLUDE_DIRS})
4961
include_directories(${GTEST_INCLUDE_DIRS})
5062
include_directories(${GMOCK_INCLUDE_DIRS})
5163

5264
include_directories("${CMAKE_INSTALL_PREFIX}/_deps")
5365
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
5466

55-
set(FOLLY_LIBRARIES Folly::folly)
56-
5767
# add subdirectories in src
5868
add_subdirectory(src/utils)
5969
add_subdirectory(src/backends)
6070
add_subdirectory(src/examples)
6171
add_subdirectory(test)
6272

63-
FILE(COPY src/resources/logging.config DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
73+
FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")

cpp/README.md

+8-3
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
* C++17
44
* GCC version: gcc-9
55
* cmake version: 3.18+
6+
* Linux
67
## Installation and Running TorchServe CPP
7-
88
This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.
99

1010
### Install dependencies
@@ -13,17 +13,22 @@ cd serve
1313
python ts_scripts/install_dependencies.py --cpp --environment dev [--cuda=cu121|cu118]
1414
```
1515
### Building the backend
16-
Don't forget to install or update TorchServe at this point if it wasn't previously installed.
16+
Don't forget to install or update TorchServe at this point if it wasn't previously installed. E.g. with:
17+
```
18+
python ts_scripts/install_from_src.py
19+
```
20+
21+
Then build the backend:
1722
```
1823
## Dev Build
1924
cd cpp
2025
./build.sh [-g cu121|cu118]
21-
2226
```
2327

2428
### Run TorchServe
2529
```
2630
mkdir model_store
31+
export LD_LIBRARY_PATH=`python -c "import torch;from pathlib import Path;p=Path(torch.__file__);print(f\"{(p.parent / 'lib').as_posix()}:{(p.parents[1] / 'nvidia/nccl/lib').as_posix()}\")"`:$LD_LIBRARY_PATH
2732
torchserve --ncs --start --model-store model_store
2833
```
2934

cpp/build.sh

+10-111
Original file line numberDiff line numberDiff line change
@@ -20,79 +20,6 @@ function detect_platform() {
2020
echo -e "${COLOR_GREEN}Detected platform: $PLATFORM ${COLOR_OFF}"
2121
}
2222

23-
function install_folly() {
24-
FOLLY_SRC_DIR=$BASE_DIR/third-party/folly
25-
FOLLY_BUILD_DIR=$DEPS_DIR/folly-build
26-
27-
if [ ! -d "$FOLLY_BUILD_DIR" ] ; then
28-
echo -e "${COLOR_GREEN}[ INFO ] Building Folly ${COLOR_OFF}"
29-
cd $FOLLY_SRC_DIR
30-
31-
./build/fbcode_builder/getdeps.py install-system-deps --recursive
32-
33-
./build/fbcode_builder/getdeps.py build \
34-
--allow-system-packages \
35-
--scratch-path $FOLLY_BUILD_DIR \
36-
--extra-cmake-defines='{"CMAKE_CXX_FLAGS": "-fPIC -D_GLIBCXX_USE_CXX11_ABI=1"}'
37-
38-
echo -e "${COLOR_GREEN}[ INFO ] Folly is installed ${COLOR_OFF}"
39-
fi
40-
41-
cd "$BWD" || exit
42-
echo "$FOLLY_BUILD_DIR/installed"
43-
}
44-
45-
function install_libtorch() {
46-
cd "$DEPS_DIR" || exit
47-
TORCH_VERSION="2.2.1"
48-
if [ -d "$DEPS_DIR/libtorch" ]; then
49-
RAW_VERSION=`cat "$DEPS_DIR/libtorch/build-version"`
50-
VERSION=`cat "$DEPS_DIR/libtorch/build-version" | cut -d "+" -f 1`
51-
if [ "$USE_NIGHTLIES" = "true" ] && [[ ! "${RAW_VERSION}" =~ .*"dev".* ]]; then
52-
rm -rf "$DEPS_DIR/libtorch"
53-
elif [ "$USE_NIGHTLIES" == "" ] && [ "$VERSION" != "$TORCH_VERSION" ]; then
54-
rm -rf "$DEPS_DIR/libtorch"
55-
fi
56-
fi
57-
if [ "$PLATFORM" = "Mac" ]; then
58-
if [ ! -d "$DEPS_DIR/libtorch" ]; then
59-
if [[ $(uname -m) == 'x86_64' ]]; then
60-
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac x86_64 ${COLOR_OFF}"
61-
wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
62-
unzip -q libtorch-macos-x86_64-${TORCH_VERSION}.zip
63-
rm libtorch-macos-x86_64-${TORCH_VERSION}.zip
64-
else
65-
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac arm64 ${COLOR_OFF}"
66-
wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
67-
unzip -q libtorch-macos-arm64-${TORCH_VERSION}.zip
68-
rm libtorch-macos-arm64-${TORCH_VERSION}.zip
69-
fi
70-
fi
71-
elif [ "$PLATFORM" = "Windows" ]; then
72-
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Windows ${COLOR_OFF}"
73-
# TODO: Windows
74-
echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
75-
exit 1
76-
else # Linux
77-
if [ ! -d "$DEPS_DIR/libtorch" ]; then
78-
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Linux ${COLOR_OFF}"
79-
if [ "$USE_NIGHTLIES" == true ]; then
80-
URL=https://download.pytorch.org/libtorch/nightly/${CUDA}/libtorch-cxx11-abi-shared-with-deps-latest.zip
81-
else
82-
URL=https://download.pytorch.org/libtorch/${CUDA}/libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}%2B${CUDA}.zip
83-
fi
84-
wget -q $URL
85-
ZIP_FILE=$(basename "$URL")
86-
ZIP_FILE="${ZIP_FILE//%2B/+}"
87-
unzip -q $ZIP_FILE
88-
rm $ZIP_FILE
89-
fi
90-
echo -e "${COLOR_GREEN}[ INFO ] libtorch is installed ${COLOR_OFF}"
91-
fi
92-
93-
cd "$BWD" || exit
94-
}
95-
9623
function prepare_test_files() {
9724
echo -e "${COLOR_GREEN}[ INFO ]Preparing test files ${COLOR_OFF}"
9825
local EX_DIR="${TR_DIR}/examples/"
@@ -123,7 +50,7 @@ function prepare_test_files() {
12350
mv Transformer_model/tokenizer.json ${HANDLER_DIR}/tokenizer.json
12451
export TOKENIZERS_PARALLELISM=""
12552
fi
126-
if [ ! -f "${EX_DIR}/aot_inductor/resnet_handler/resne50_pt2.so" ]; then
53+
if [ ! -f "${EX_DIR}/aot_inductor/resnet_handler/resnet50_pt2.so" ]; then
12754
local HANDLER_DIR=${EX_DIR}/aot_inductor/resnet_handler/
12855
cd ${HANDLER_DIR}
12956
python ${BASE_DIR}/../examples/cpp/aot_inductor/resnet/resnet50_torch_export.py
@@ -133,6 +60,7 @@ function prepare_test_files() {
13360
}
13461

13562
function build() {
63+
echo -e "${COLOR_GREEN}[ INFO ]Building backend ${COLOR_OFF}"
13664
MAYBE_BUILD_QUIC=""
13765
if [ "$WITH_QUIC" == true ] ; then
13866
setup_mvfst
@@ -151,50 +79,37 @@ function build() {
15179
PREFIX=$BWD
15280
fi
15381

154-
MAYBE_CUDA_COMPILER=""
155-
if [ "$CUDA" != "" ]; then
156-
MAYBE_CUDA_COMPILER='-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc'
157-
fi
158-
159-
MAYBE_NIGHTLIES="-Dnightlies=OFF"
160-
if [ "$USE_NIGHTLIES" == true ]; then
161-
MAYBE_NIGHTLIES="-Dnightlies=ON"
162-
fi
163-
16482
# Build torchserve_cpp with cmake
16583
cd "$BWD" || exit
166-
FOLLY_CMAKE_DIR=$DEPS_DIR/folly-build/installed
167-
find $FOLLY_CMAKE_DIR -name "lib*.*" -exec ln -s "{}" $LIBS_DIR/ \;
84+
85+
CMAKE_PREFIX_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'`
86+
16887
if [ "$PLATFORM" = "Linux" ]; then
88+
NCCL_PATH=`python3 -c 'import torch;from pathlib import Path;print(Path(torch.__file__).parents[1]/"nvidia"/"nccl"/"lib")'`
89+
export LD_LIBRARY_PATH=${NCCL_PATH}:${LD_LIBRARY_PATH}
16990
cmake \
170-
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch" \
91+
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH" \
17192
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
17293
"$MAYBE_BUILD_QUIC" \
17394
"$MAYBE_BUILD_TESTS" \
17495
"$MAYBE_BUILD_SHARED_LIBS" \
17596
"$MAYBE_OVERRIDE_CXX_FLAGS" \
17697
"$MAYBE_USE_STATIC_DEPS" \
17798
"$MAYBE_LIB_FUZZING_ENGINE" \
178-
"$MAYBE_CUDA_COMPILER" \
179-
"$MAYBE_NIGHTLIES" \
18099
..
181100

182-
if [ "$CUDA" = "cu118" ] || [ "$CUDA" = "cu121" ]; then
183-
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/bin/nvcc
184-
fi
185101
elif [ "$PLATFORM" = "Mac" ]; then
186102
export LIBRARY_PATH=${LIBRARY_PATH}:`brew --prefix icu4c`/lib:`brew --prefix libomp`/lib
187103

188104
cmake \
189-
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch" \
105+
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH" \
190106
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
191107
"$MAYBE_BUILD_QUIC" \
192108
"$MAYBE_BUILD_TESTS" \
193109
"$MAYBE_BUILD_SHARED_LIBS" \
194110
"$MAYBE_OVERRIDE_CXX_FLAGS" \
195111
"$MAYBE_USE_STATIC_DEPS" \
196112
"$MAYBE_LIB_FUZZING_ENGINE" \
197-
"$MAYBE_NIGHTLIES" \
198113
"-DLLAMA_METAL=OFF" \
199114
..
200115

@@ -220,12 +135,6 @@ function build() {
220135
fi
221136
}
222137

223-
function symlink_torch_libs() {
224-
if [ "$PLATFORM" = "Linux" ]; then
225-
ln -sf ${DEPS_DIR}/libtorch/lib/*.so* ${LIBS_DIR}
226-
fi
227-
}
228-
229138
function install_torchserve_cpp() {
230139
TARGET_DIR=`python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"`
231140

@@ -244,16 +153,12 @@ WITH_QUIC=false
244153
INSTALL_DEPENDENCIES=false
245154
PREFIX=""
246155
COMPILER_FLAGS=""
247-
CUDA="cpu"
248-
USAGE="./build.sh [-j num_jobs] [-g cu118|cu121] [-q|--with-quic] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags] [-n|--nighlies]"
156+
USAGE="./build.sh [-j num_jobs] [-q|--with-quic] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags]"
249157
while [ "$1" != "" ]; do
250158
case $1 in
251159
-j | --jobs ) shift
252160
JOBS=$1
253161
;;
254-
-g | --cuda-version ) shift
255-
CUDA=$1
256-
;;
257162
-q | --with-quic )
258163
WITH_QUIC=true
259164
;;
@@ -268,9 +173,6 @@ while [ "$1" != "" ]; do
268173
shift
269174
COMPILER_FLAGS=$1
270175
;;
271-
-n | --nightlies )
272-
USE_NIGHTLIES=true
273-
;;
274176
* ) echo $USAGE
275177
exit 1
276178
esac
@@ -305,9 +207,6 @@ cd $BASE_DIR
305207

306208
git submodule update --init --recursive
307209

308-
install_folly
309-
install_libtorch
310210
prepare_test_files
311211
build
312-
symlink_torch_libs
313212
install_torchserve_cpp

cpp/src/backends/CMakeLists.txt

+17-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
1+
FetchContent_Declare(
2+
gflags
3+
GIT_REPOSITORY https://github.com/gflags/gflags
4+
GIT_TAG v2.2.2
5+
)
6+
FetchContent_GetProperties(spdlgflagsog)
7+
8+
if(NOT gflags_POPULATED)
9+
message(STATUS "Fetching gflags...")
10+
FetchContent_Populate(gflags)
11+
add_subdirectory(${gflags_SOURCE_DIR} ${gflags_BINARY_DIR})
12+
endif()
13+
14+
115
set(TS_BACKENDS_SRC_DIR "${torchserve_cpp_SOURCE_DIR}/src/backends")
216
set(TS_BACKENDS_CORE_SRC_DIR "${TS_BACKENDS_SRC_DIR}/core")
317
set(TS_BACKENDS_PROTOCOL_SRC_DIR "${TS_BACKENDS_SRC_DIR}/protocol")
@@ -11,7 +25,7 @@ list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/ot
1125
list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/socket.cc)
1226
add_library(ts_backends_protocol SHARED ${TS_BACKENDS_PROTOCOL_SOURCE_FILES})
1327
target_include_directories(ts_backends_protocol PUBLIC ${TS_BACKENDS_PROTOCOL_SRC_DIR})
14-
target_link_libraries(ts_backends_protocol PRIVATE ts_utils ${FOLLY_LIBRARIES})
28+
target_link_libraries(ts_backends_protocol PRIVATE ts_utils)
1529
install(TARGETS ts_backends_protocol DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
1630

1731
# build library ts_backend_core
@@ -22,7 +36,7 @@ list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/base_handler.cc)
2236
list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/torch_scripted_handler.cc)
2337
add_library(ts_backends_core SHARED ${BACKEND_SOURCE_FILES})
2438
target_include_directories(ts_backends_core PUBLIC ${TS_BACKENDS_CORE_SRC_DIR})
25-
target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${FOLLY_LIBRARIES} ${TORCH_LIBRARIES})
39+
target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${TORCH_LIBRARIES})
2640
install(TARGETS ts_backends_core DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
2741

2842
# build exe model_worker_socket
@@ -37,5 +51,5 @@ target_include_directories(model_worker_socket PRIVATE
3751
${TS_BACKENDS_TORCH_SCRIPTED_SRC_DIR}
3852
)
3953
target_link_libraries(model_worker_socket
40-
PRIVATE ts_backends_core ts_backends_protocol ${FOLLY_LIBRARIES} ${TORCH_LIBRARIES})
54+
PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags)
4155
install(TARGETS model_worker_socket DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/bin)

0 commit comments

Comments
 (0)