Skip to content

Commit 2e26323

Browse files
mresomreso
authored
Fix cpp ci (#2958)
* Remove install_dependencies from buld.sh call * remove sudo from folly build * Align cpp wf file with cpu wf file * Make yaml statically linked dependency * move cpp binaries to location based on ts.__file__ * Add FAQ to cpp readme * Update readme to expect installed TS * Fix libtorch install on mac * Add cpp build faqs * Remove llama.cpp as submodule and add through fetch_content * Disable metal in llamacpp example * Fix dangling pointer error in cpp worker * Remove kineto from mac build * Add llvm as mac cpp dependencies (req for clang-tidy) * Enable fPIC for llama.cpp example * Add sudo to clang-tidy link creation * Add undefined symbol faq * fix llv typo * Add install from source to cpp_ci * Correct install from source to cpp_ci * bump up pyyaml version to avoid cython 3 issue yaml/pyyaml#702 * Move cpp ci to M1 mac * Run workflow on self hosted runner * Disable mac ci for cpp * Fix workflow syntax * Run on cpp-ci * Remove sudo * Add apt update for act docker * print library_path in print_env_info.py * print end in cpp ci workflow * Run on github runner * Add upterm session to workflow * Move post mortem upterm session before build * Remove first upterm session * ci debugging * Observe disk space * move _build to /mnt on github runner * fix permission denied * use mount instead of ln * Adjust llamacpp api * Reactivate set -e * Remove note on env variable in cpp readme * Fix linting issue in print_env_info.py * Cleanup ci-cpu-cpp.yml * quieten wget * Add build clean section in cpp/readme * Readjust to llama.cpp api
1 parent 8b51d1e commit 2e26323

File tree

10 files changed

+137
-144
lines changed

10 files changed

+137
-144
lines changed

‎.github/workflows/ci-cpu-cpp.yml

+33-8
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,42 @@ jobs:
1616
strategy:
1717
fail-fast: false
1818
matrix:
19-
os: [ubuntu-20.04, macOS-latest]
19+
os: [ubuntu-20.04]
2020
steps:
21+
# - name: Setup Python for M1
22+
# if: matrix.os == 'macos-14'
23+
# uses: actions/setup-python@v5
24+
# with:
25+
# python-version: '3.10'
26+
- name: Setup Python for all other OS
27+
if: matrix.os != 'macos-14'
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: 3.9
31+
architecture: x64
32+
- name: Setup Java 17
33+
uses: actions/setup-java@v3
34+
with:
35+
distribution: 'zulu'
36+
java-version: '17'
2137
- name: Checkout TorchServe
22-
uses: actions/checkout@v2
23-
- name: Install libtorch - macOS
24-
if: matrix.os == 'macOS-latest'
25-
run: |
26-
brew install libtorch
38+
uses: actions/checkout@v3
39+
with:
40+
submodules: recursive
41+
# - name: Install libtorch - macOS
42+
# if: matrix.os == 'macOS-latest'
43+
# run: |
44+
# brew install libtorch
2745
- name: Install dependencies
2846
run: |
29-
python ts_scripts/install_dependencies.py --environment=dev --cpp
47+
sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
48+
- name: Install TorchServe
49+
run: |
50+
python ts_scripts/install_from_src.py
51+
- name: Print Env
52+
run: |
53+
python ts_scripts/print_env_info.py
3054
- name: Build
3155
run: |
32-
cd cpp && ./build.sh
56+
cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
57+
./build.sh

‎.gitmodules

-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
[submodule "third_party/google/rpc"]
22
path = third_party/google/rpc
33
url = https://github.com/googleapis/googleapis.git
4-
[submodule "cpp/third-party/llama.cpp"]
5-
path = cpp/third-party/llama.cpp
6-
url = https://github.com/ggerganov/llama.cpp.git
74
[submodule "cpp/third-party/llama2.c"]
85
path = cpp/third-party/llama2.c
96
url = https://github.com/karpathy/llama2.c

‎cpp/CMakeLists.txt

+15-4
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,6 @@ if(CLANG_FORMAT_EXE)
1818
${PROJECT_SOURCE_DIR}/test/*.hh
1919
)
2020

21-
add_custom_target(format
22-
COMMAND
23-
${CLANG_FORMAT_EXE} -i -style=google ${ALL_CXX_SOURCE_FILES}
24-
)
2521
endif()
2622

2723

@@ -31,6 +27,21 @@ find_package(fmt REQUIRED)
3127
find_package(gflags REQUIRED)
3228
find_package(Torch REQUIRED)
3329

30+
include(FetchContent)
31+
32+
FetchContent_Declare(
33+
yaml-cpp
34+
GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
35+
GIT_TAG 0.8.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master)
36+
)
37+
FetchContent_GetProperties(yaml-cpp)
38+
39+
if(NOT yaml-cpp_POPULATED)
40+
message(STATUS "Fetching yaml-cpp...")
41+
FetchContent_Populate(yaml-cpp)
42+
add_subdirectory(${yaml-cpp_SOURCE_DIR} ${yaml-cpp_BINARY_DIR})
43+
endif()
44+
3445
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
3546

3647
include_directories(${TORCH_INCLUDE_DIRS})

‎cpp/README.md

+12-14
Original file line numberDiff line numberDiff line change
@@ -5,36 +5,34 @@
55
* cmake version: 3.18+
66
## Installation and Running TorchServe CPP
77

8+
This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.
9+
810
### Install dependencies
911
```
1012
cd serve
1113
python ts_scripts/install_dependencies.py --cpp --environment dev [--cuda=cu121|cu118]
1214
```
1315
### Building the backend
16+
Don't forget to install or update TorchServe at this point if it wasn't previously installed.
1417
```
1518
## Dev Build
1619
cd cpp
1720
./build.sh [-g cu121|cu118]
1821
19-
## Install TorchServe from source
20-
cd ..
21-
python ts_scripts/install_from_src.py
22-
```
23-
### Set Environment Var
24-
#### On Mac
25-
```
26-
export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/_build/_deps/libtorch/lib
27-
```
28-
#### On Ubuntu
29-
```
30-
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/_build/_deps/libtorch/lib
3122
```
3223

3324
### Run TorchServe
3425
```
3526
mkdir model_store
3627
torchserve --ncs --start --model-store model_store
3728
```
29+
30+
### Clean the build directory
31+
To clean the build directory in order to rebuild from scratch simply delete the cpp/_build directory with
32+
```
33+
rm -rf cpp/_build
34+
```
35+
3836
## Backend
3937
TorchServe cpp backend can run as a process, which is similar to [TorchServe Python backend](https://github.com/pytorch/serve/tree/master/ts). By default, TorchServe supports torch scripted model in cpp backend. Other platforms such as MxNet, ONNX can be supported through custom handlers following the TorchScript example [src/backends/handler/torch_scripted_handler.hh](https://github.com/pytorch/serve/blob/master/cpp/src/backends/handler/torch_scripted_handler.hh).
4038
### Custom Handler
@@ -89,11 +87,11 @@ python -c "import ts; from pathlib import Path; print((Path(ts.__file__).parent
8987
3. Make sure you have the right conda/venv environment activated during building that you're also using to run TorchServe.
9088
9189
Q: Build on Mac fails with `Library not loaded: @rpath/libomp.dylib`
92-
A: Install libomp with brew and link in /usr/local/lib
90+
A: Install libomp with brew and link in /usr/local/lib
9391
```bash
9492
brew install libomp
9593
sudo ln -s /opt/homebrew/opt/libomp/lib/libomp.dylib /usr/local/lib/libomp.dylib
9694
```
9795
9896
Q: When loading a handler which uses a model exported with torch._export.aot_compile the handler dies with "error: Error in dlopen: MODEL.SO : undefined symbol: SOME_SYMBOL".
99-
A: Make sure that you are using matching libtorch and Pytorch versions for inference and export, respectively.
97+
A: Make sure that you are using matching libtorch and Pytorch versions for inference and export, respectively.

‎cpp/build.sh

+28-96
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,9 @@ function install_folly() {
2828
echo -e "${COLOR_GREEN}[ INFO ] Building Folly ${COLOR_OFF}"
2929
cd $FOLLY_SRC_DIR
3030

31-
if [ "$PLATFORM" = "Linux" ]; then
32-
SUDO="sudo"
33-
elif [ "$PLATFORM" = "Mac" ]; then
34-
SUDO=""
35-
fi
36-
$SUDO ./build/fbcode_builder/getdeps.py install-system-deps --recursive
31+
./build/fbcode_builder/getdeps.py install-system-deps --recursive
3732

38-
$SUDO ./build/fbcode_builder/getdeps.py build \
33+
./build/fbcode_builder/getdeps.py build \
3934
--allow-system-packages \
4035
--scratch-path $FOLLY_BUILD_DIR \
4136
--extra-cmake-defines='{"CMAKE_CXX_FLAGS": "-fPIC -D_GLIBCXX_USE_CXX11_ABI=1"}'
@@ -47,36 +42,29 @@ function install_folly() {
4742
echo "$FOLLY_BUILD_DIR/installed"
4843
}
4944

50-
function install_kineto() {
51-
if [ "$PLATFORM" = "Linux" ]; then
52-
echo -e "${COLOR_GREEN}[ INFO ] Skip install kineto on Linux ${COLOR_OFF}"
53-
elif [ "$PLATFORM" = "Mac" ]; then
54-
KINETO_SRC_DIR=$BASE_DIR/third-party/kineto
55-
56-
if [ ! -d "$KINETO_SRC_DIR/libkineto/build" ] ; then
57-
cd $KINETO_SRC_DIR/libkineto
58-
mkdir build && cd build
59-
cmake ..
60-
make install
61-
fi
62-
fi
63-
64-
cd "$BWD" || exit
65-
}
66-
6745
function install_libtorch() {
46+
cd "$DEPS_DIR" || exit
6847
TORCH_VERSION="2.2.1"
48+
if [ -d "$DEPS_DIR/libtorch" ]; then
49+
RAW_VERSION=`cat "$DEPS_DIR/libtorch/build-version"`
50+
VERSION=`cat "$DEPS_DIR/libtorch/build-version" | cut -d "+" -f 1`
51+
if [ "$USE_NIGHTLIES" = "true" ] && [[ ! "${RAW_VERSION}" =~ .*"dev".* ]]; then
52+
rm -rf "$DEPS_DIR/libtorch"
53+
elif [ "$USE_NIGHTLIES" == "" ] && [ "$VERSION" != "$TORCH_VERSION" ]; then
54+
rm -rf "$DEPS_DIR/libtorch"
55+
fi
56+
fi
6957
if [ "$PLATFORM" = "Mac" ]; then
7058
if [ ! -d "$DEPS_DIR/libtorch" ]; then
7159
if [[ $(uname -m) == 'x86_64' ]]; then
7260
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac x86_64 ${COLOR_OFF}"
73-
wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
74-
unzip libtorch-macos-x86_64-${TORCH_VERSION}.zip
61+
wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
62+
unzip -q libtorch-macos-x86_64-${TORCH_VERSION}.zip
7563
rm libtorch-macos-x86_64-${TORCH_VERSION}.zip
7664
else
7765
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac arm64 ${COLOR_OFF}"
78-
wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
79-
unzip libtorch-macos-arm64-${TORCH_VERSION}.zip
66+
wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
67+
unzip -q libtorch-macos-arm64-${TORCH_VERSION}.zip
8068
rm libtorch-macos-arm64-${TORCH_VERSION}.zip
8169
fi
8270
fi
@@ -86,27 +74,17 @@ function install_libtorch() {
8674
echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
8775
exit 1
8876
else # Linux
89-
if [ -d "$DEPS_DIR/libtorch" ]; then
90-
RAW_VERSION=`cat "$DEPS_DIR/libtorch/build-version"`
91-
VERSION=`cat "$DEPS_DIR/libtorch/build-version" | cut -d "+" -f 1`
92-
if [ "$USE_NIGHTLIES" = "true" ] && [[ ! "${RAW_VERSION}" =~ .*"dev".* ]]; then
93-
rm -rf "$DEPS_DIR/libtorch"
94-
elif [ "$USE_NIGHTLIES" == "" ] && [ "$VERSION" != "$TORCH_VERSION" ]; then
95-
rm -rf "$DEPS_DIR/libtorch"
96-
fi
97-
fi
9877
if [ ! -d "$DEPS_DIR/libtorch" ]; then
99-
cd "$DEPS_DIR" || exit
10078
echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Linux ${COLOR_OFF}"
10179
if [ "$USE_NIGHTLIES" == true ]; then
10280
URL=https://download.pytorch.org/libtorch/nightly/${CUDA}/libtorch-cxx11-abi-shared-with-deps-latest.zip
10381
else
10482
URL=https://download.pytorch.org/libtorch/${CUDA}/libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}%2B${CUDA}.zip
10583
fi
106-
wget $URL
84+
wget -q $URL
10785
ZIP_FILE=$(basename "$URL")
10886
ZIP_FILE="${ZIP_FILE//%2B/+}"
109-
unzip $ZIP_FILE
87+
unzip -q $ZIP_FILE
11088
rm $ZIP_FILE
11189
fi
11290
echo -e "${COLOR_GREEN}[ INFO ] libtorch is installed ${COLOR_OFF}"
@@ -115,58 +93,22 @@ function install_libtorch() {
11593
cd "$BWD" || exit
11694
}
11795

118-
function install_yaml_cpp() {
119-
YAML_CPP_SRC_DIR=$BASE_DIR/third-party/yaml-cpp
120-
YAML_CPP_BUILD_DIR=$DEPS_DIR/yaml-cpp-build
121-
122-
if [ ! -d "$YAML_CPP_BUILD_DIR" ] ; then
123-
echo -e "${COLOR_GREEN}[ INFO ] Building yaml-cpp ${COLOR_OFF}"
124-
125-
if [ "$PLATFORM" = "Linux" ]; then
126-
SUDO="sudo"
127-
elif [ "$PLATFORM" = "Mac" ]; then
128-
SUDO=""
129-
fi
130-
131-
mkdir $YAML_CPP_BUILD_DIR
132-
cd $YAML_CPP_BUILD_DIR
133-
cmake $YAML_CPP_SRC_DIR -DYAML_BUILD_SHARED_LIBS=ON -DYAML_CPP_BUILD_TESTS=OFF -DCMAKE_CXX_FLAGS="-fPIC"
134-
$SUDO make install
135-
136-
echo -e "${COLOR_GREEN}[ INFO ] yaml-cpp is installed ${COLOR_OFF}"
137-
fi
138-
139-
cd "$BWD" || exit
140-
}
141-
142-
function build_llama_cpp() {
143-
BWD=$(pwd)
144-
LLAMA_CPP_SRC_DIR=$BASE_DIR/third-party/llama.cpp
145-
cd "${LLAMA_CPP_SRC_DIR}"
146-
if [ "$PLATFORM" = "Mac" ]; then
147-
make LLAMA_METAL=OFF -j
148-
else
149-
make -j
150-
fi
151-
cd "$BWD" || exit
152-
}
153-
15496
function prepare_test_files() {
15597
echo -e "${COLOR_GREEN}[ INFO ]Preparing test files ${COLOR_OFF}"
15698
local EX_DIR="${TR_DIR}/examples/"
15799
rsync -a --link-dest=../../test/resources/ ${BASE_DIR}/test/resources/ ${TR_DIR}/
158100
if [ ! -f "${EX_DIR}/babyllama/babyllama_handler/tokenizer.bin" ]; then
159-
wget https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin -O "${EX_DIR}/babyllama/babyllama_handler/tokenizer.bin"
101+
wget -q https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin -O "${EX_DIR}/babyllama/babyllama_handler/tokenizer.bin"
160102
fi
161103
if [ ! -f "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin" ]; then
162-
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin -O "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin"
104+
wget -q https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin -O "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin"
163105
fi
164106
# PT2.2 torch.expport does not support Mac
165107
if [ "$PLATFORM" = "Linux" ]; then
166108
if [ ! -f "${EX_DIR}/aot_inductor/llama_handler/stories15M.so" ]; then
167109
local HANDLER_DIR=${EX_DIR}/aot_inductor/llama_handler/
168110
if [ ! -f "${HANDLER_DIR}/stories15M.pt" ]; then
169-
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true -O "${HANDLER_DIR}/stories15M.pt"
111+
wget -q https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true -O "${HANDLER_DIR}/stories15M.pt"
170112
fi
171113
local LLAMA_SO_DIR=${BASE_DIR}/third-party/llama2.so/
172114
PYTHONPATH=${LLAMA_SO_DIR}:${PYTHONPATH} python ${BASE_DIR}/../examples/cpp/aot_inductor/llama2/compile.py --checkpoint ${HANDLER_DIR}/stories15M.pt ${HANDLER_DIR}/stories15M.so
@@ -221,12 +163,11 @@ function build() {
221163

222164
# Build torchserve_cpp with cmake
223165
cd "$BWD" || exit
224-
YAML_CPP_CMAKE_DIR=$DEPS_DIR/yaml-cpp-build
225166
FOLLY_CMAKE_DIR=$DEPS_DIR/folly-build/installed
226167
find $FOLLY_CMAKE_DIR -name "lib*.*" -exec ln -s "{}" $LIBS_DIR/ \;
227168
if [ "$PLATFORM" = "Linux" ]; then
228169
cmake \
229-
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$YAML_CPP_CMAKE_DIR;$DEPS_DIR/libtorch" \
170+
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch" \
230171
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
231172
"$MAYBE_BUILD_QUIC" \
232173
"$MAYBE_BUILD_TESTS" \
@@ -242,8 +183,10 @@ function build() {
242183
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/bin/nvcc
243184
fi
244185
elif [ "$PLATFORM" = "Mac" ]; then
186+
export LIBRARY_PATH=${LIBRARY_PATH}:`brew --prefix icu4c`/lib:`brew --prefix libomp`/lib
187+
245188
cmake \
246-
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$YAML_CPP_CMAKE_DIR;$DEPS_DIR/libtorch" \
189+
-DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch" \
247190
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
248191
"$MAYBE_BUILD_QUIC" \
249192
"$MAYBE_BUILD_TESTS" \
@@ -252,9 +195,10 @@ function build() {
252195
"$MAYBE_USE_STATIC_DEPS" \
253196
"$MAYBE_LIB_FUZZING_ENGINE" \
254197
"$MAYBE_NIGHTLIES" \
198+
"-DLLAMA_METAL=OFF" \
255199
..
256200

257-
export LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/opt/icu4c/lib
201+
258202
else
259203
# TODO: Windows
260204
echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
@@ -282,16 +226,8 @@ function symlink_torch_libs() {
282226
fi
283227
}
284228

285-
function symlink_yaml_cpp_lib() {
286-
if [ "$PLATFORM" = "Linux" ]; then
287-
ln -sf ${DEPS_DIR}/yaml-cpp-build/*.so* ${LIBS_DIR}
288-
elif [ "$PLATFORM" = "Mac" ]; then
289-
ln -sf ${DEPS_DIR}/yaml-cpp-build/*.dylib* ${LIBS_DIR}
290-
fi
291-
}
292-
293229
function install_torchserve_cpp() {
294-
TARGET_DIR=$BASE_DIR/../ts/cpp/
230+
TARGET_DIR=`python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"`
295231

296232
if [ -d $TARGET_DIR ]; then
297233
rm -rf $TARGET_DIR
@@ -370,12 +306,8 @@ cd $BASE_DIR
370306
git submodule update --init --recursive
371307

372308
install_folly
373-
#install_kineto
374309
install_libtorch
375-
install_yaml_cpp
376-
build_llama_cpp
377310
prepare_test_files
378311
build
379312
symlink_torch_libs
380-
symlink_yaml_cpp_lib
381313
install_torchserve_cpp

‎cpp/third-party/llama.cpp

-1
This file was deleted.

0 commit comments

Comments
 (0)