Skip to content

Commit 2f1f358

Browse files
lxningmreso
lxning
and
mreso
authored
AOTInductor Resnet CPP example (#2944)
* fmt * fix compile error * fmt * fmt * fmt * fix path * remove sentencepiece * add test file * fmt * fmt * fix typo in readme * update readme * fix lint * fix typo * add model-config.yaml --------- Co-authored-by: Matthias Reso <[email protected]>
1 parent acc3f34 commit 2f1f358

File tree

17 files changed

+458
-42
lines changed

17 files changed

+458
-42
lines changed

‎cpp/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ find_package(folly REQUIRED)
3030
find_package(fmt REQUIRED)
3131
find_package(gflags REQUIRED)
3232
find_package(Torch REQUIRED)
33-
find_package(yaml-cpp REQUIRED NO_CMAKE_PATH)
33+
3434
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
3535

3636
include_directories(${TORCH_INCLUDE_DIRS})

‎cpp/build.sh

+15-38
Original file line numberDiff line numberDiff line change
@@ -155,37 +155,6 @@ function install_yaml_cpp() {
155155
cd "$BWD" || exit
156156
}
157157

158-
function install_sentencepiece() {
159-
SENTENCEPIECE_SRC_DIR=$BASE_DIR/third-party/sentencepiece
160-
SENTENCEPIECE_BUILD_DIR=$DEPS_DIR/sentencepiece-build
161-
162-
if [ ! -d "$SENTENCEPIECE_SRC_DIR" ] ; then
163-
echo -e "${COLOR_GREEN}[ INFO ] Cloning sentencepiece repo ${COLOR_OFF}"
164-
git clone https://github.com/google/sentencepiece.git "$SENTENCEPIECE_SRC_DIR"
165-
cd $SENTENCEPIECE_SRC_DIR
166-
git checkout tags/v0.1.99
167-
fi
168-
169-
if [ ! -d "$SENTENCEPIECE_BUILD_DIR" ] ; then
170-
echo -e "${COLOR_GREEN}[ INFO ] Building sentencepiece ${COLOR_OFF}"
171-
172-
mkdir $SENTENCEPIECE_BUILD_DIR
173-
cd $SENTENCEPIECE_BUILD_DIR
174-
cmake $SENTENCEPIECE_SRC_DIR
175-
make -i $(nproc)
176-
if [ "$PLATFORM" = "Linux" ]; then
177-
sudo make install
178-
sudo ldconfig -v
179-
elif [ "$PLATFORM" = "Mac" ]; then
180-
make install
181-
fi
182-
183-
echo -e "${COLOR_GREEN}[ INFO ] sentencepiece is installed ${COLOR_OFF}"
184-
fi
185-
186-
cd "$BWD" || exit
187-
}
188-
189158
function build_llama_cpp() {
190159
BWD=$(pwd)
191160
LLAMA_CPP_SRC_DIR=$BASE_DIR/third-party/llama.cpp
@@ -208,14 +177,23 @@ function prepare_test_files() {
208177
if [ ! -f "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin" ]; then
209178
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin -O "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin"
210179
fi
211-
if [ ! -f "${EX_DIR}/aot_inductor/llama_handler/stories15M.so" ]; then
212-
local HANDLER_DIR=${EX_DIR}/aot_inductor/llama_handler/
213-
if [ ! -f "${HANDLER_DIR}/stories15M.pt" ]; then
214-
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true -O "${HANDLER_DIR}/stories15M.pt"
180+
# PT2.2 torch.expport does not support Mac
181+
if [ "$PLATFORM" = "Linux" ]; then
182+
if [ ! -f "${EX_DIR}/aot_inductor/llama_handler/stories15M.so" ]; then
183+
local HANDLER_DIR=${EX_DIR}/aot_inductor/llama_handler/
184+
if [ ! -f "${HANDLER_DIR}/stories15M.pt" ]; then
185+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true -O "${HANDLER_DIR}/stories15M.pt"
186+
fi
187+
local LLAMA_SO_DIR=${BASE_DIR}/third-party/llama2.so/
188+
PYTHONPATH=${LLAMA_SO_DIR}:${PYTHONPATH} python ${BASE_DIR}/../examples/cpp/aot_inductor/llama2/compile.py --checkpoint ${HANDLER_DIR}/stories15M.pt ${HANDLER_DIR}/stories15M.so
189+
fi
190+
if [ ! -f "${EX_DIR}/aot_inductor/resnet_handler/resne50_pt2.so" ]; then
191+
local HANDLER_DIR=${EX_DIR}/aot_inductor/resnet_handler/
192+
cd ${HANDLER_DIR}
193+
python ${BASE_DIR}/../examples/cpp/aot_inductor/resnet/resnet50_torch_export.py
215194
fi
216-
local LLAMA_SO_DIR=${BASE_DIR}/third-party/llama2.so/
217-
PYTHONPATH=${LLAMA_SO_DIR}:${PYTHONPATH} python ${BASE_DIR}/../examples/cpp/aot_inductor/llama2/compile.py --checkpoint ${HANDLER_DIR}/stories15M.pt ${HANDLER_DIR}/stories15M.so
218195
fi
196+
cd "$BWD" || exit
219197
}
220198

221199
function build() {
@@ -401,7 +379,6 @@ install_folly
401379
install_kineto
402380
install_libtorch
403381
install_yaml_cpp
404-
install_sentencepiece
405382
build_llama_cpp
406383
prepare_test_files
407384
build

‎cpp/src/examples/CMakeLists.txt

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11

22
add_subdirectory("../../../examples/cpp/babyllama/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/babyllama/babyllama_handler/")
33

4-
add_subdirectory("../../../examples/cpp/aot_inductor/llama2/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/aot_inductor/llama_handler/")
5-
64
add_subdirectory("../../../examples/cpp/llamacpp/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/llamacpp/llamacpp_handler/")
75

86
add_subdirectory("../../../examples/cpp/mnist/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/mnist/mnist_handler/")
7+
8+
9+
# PT2.2 torch.expport does not support Mac
10+
if(CMAKE_SYSTEM_NAME MATCHES "Linux")
11+
add_subdirectory("../../../examples/cpp/aot_inductor/llama2/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/aot_inductor/llama_handler/")
12+
13+
add_subdirectory("../../../examples/cpp/aot_inductor/resnet" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/aot_inductor/resnet_handler/")
14+
endif()

‎cpp/src/utils/CMakeLists.txt

+6-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ list(APPEND TS_UTILS_SOURCE_FILES ${TS_UTILS_SRC_DIR}/metrics/registry.cc)
1212
add_library(ts_utils SHARED ${TS_UTILS_SOURCE_FILES})
1313
target_include_directories(ts_utils PUBLIC ${TS_UTILS_SRC_DIR})
1414
target_include_directories(ts_utils PRIVATE ${Boost_INCLUDE_DIRS})
15-
target_link_libraries(ts_utils ${FOLLY_LIBRARIES} ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} yaml-cpp::yaml-cpp)
15+
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
16+
target_link_libraries(ts_utils ${FOLLY_LIBRARIES} ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} yaml-cpp::yaml-cpp)
17+
else()
18+
target_link_libraries(ts_utils ${FOLLY_LIBRARIES} ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} yaml-cpp)
19+
endif()
20+
1621
install(TARGETS ts_utils DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
1722

1823
list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/ifoo.hh)

‎cpp/test/examples/examples_test.cc

+21
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,24 @@ TEST_F(ModelPredictTest, TestLoadPredictLlamaCppHandler) {
5959
base_dir + "llamacpp_handler", "llamacpp", -1, "", "", 1, false),
6060
base_dir + "llamacpp_handler", base_dir + "prompt.txt", "llm_ts", 200);
6161
}
62+
63+
TEST_F(ModelPredictTest, TestLoadPredictAotInductorResnetHandler) {
64+
std::string base_dir = "_build/test/resources/examples/aot_inductor/";
65+
std::string file1 = base_dir + "resnet_handler/resnet50_pt2.so";
66+
67+
std::ifstream f1(file1);
68+
69+
if (!f1.good())
70+
GTEST_SKIP() << "Skipping TestLoadPredictAotInductorResnetHandler because "
71+
"of missing files: "
72+
<< file1;
73+
74+
this->LoadPredict(
75+
std::make_shared<torchserve::LoadModelRequest>(
76+
base_dir + "resnet_handler", "resnet50_aot",
77+
torch::cuda::is_available() ? 0 : -1, "", "", 1, false),
78+
base_dir + "resnet_handler",
79+
base_dir + "resnet_handler/0_png.pt",
80+
"resnet_ts",
81+
200);
82+
}

‎cpp/test/resources/examples/aot_inductor/resnet_handler/0_png.pt

589 KB
Binary file not shown.

‎cpp/test/resources/examples/aot_inductor/resnet_handler/MAR-INF/MANIFEST.json

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"createdOn": "14/02/2024 05:57:14",
3+
"runtime": "LSP",
4+
"model": {
5+
"modelName": "resnetcppaot",
6+
"handler": "libresnet_handler:ResnetCppHandler",
7+
"modelVersion": "1.0",
8+
"configFile": "model-config.yaml"
9+
},
10+
"archiverVersion": "0.9.0"
11+
}

‎cpp/test/resources/examples/aot_inductor/resnet_handler/index_to_name.json

+1
Large diffs are not rendered by default.

‎cpp/test/resources/examples/aot_inductor/resnet_handler/model-config.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
minWorkers: 1
2+
maxWorkers: 1
3+
batchSize: 2
4+
5+
handler:
6+
model_so_path: "resnet50_pt2.so"
7+
mapping: "index_to_name.json"

‎examples/cpp/aot_inductor/resnet/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
add_library(resnet_handler SHARED src/resnet_handler.cc)
2+
target_link_libraries(resnet_handler PRIVATE ts_backends_core ts_utils ${TORCH_LIBRARIES})

‎examples/cpp/aot_inductor/resnet/README.md

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
This example uses AOTInductor to compile the Resnet50 into an so file which is then executed using libtorch.
2+
The handler C++ source code for this examples can be found [here](src).
3+
4+
### Setup
5+
1. Follow the instructions in [README.md](../../../../cpp/README.md) to build the TorchServe C++ backend.
6+
7+
```
8+
cd serve/cpp
9+
./builld.sh
10+
```
11+
12+
The build script will create the necessary artifact for this example.
13+
To recreate these by hand you can follow the prepare_test_files function of the [build.sh](../../../../cpp/build.sh) script.
14+
We will need the handler .so file as well as the resnet50_pt2.so file containing the model and weights.
15+
16+
2. Create a [model-config.yaml](model-config.yaml)
17+
18+
```yaml
19+
minWorkers: 1
20+
maxWorkers: 1
21+
batchSize: 2
22+
23+
handler:
24+
model_so_path: "resnet50_pt2.so"
25+
mapping: "index_to_name.json"
26+
```
27+
28+
### Generate Model Artifacts Folder
29+
30+
```bash
31+
torch-model-archiver --model-name resnetcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/resnet_handler/libresnet_handler:ResnetCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/_build/test/resources/examples/aot_inductor/resnet_handler/resnet50_pt2.so --config-file model-config.yaml --archive-format no-archive
32+
```
33+
34+
Create model store directory and move the folder `resnetcppaot`
35+
36+
```
37+
mkdir model_store
38+
mv resnetcppaot model_store/
39+
```
40+
41+
### Inference
42+
43+
Start torchserve using the following command
44+
45+
```
46+
torchserve --ncs --model-store model_store/ --models resnetcppaot
47+
```
48+
49+
50+
Infer the model using the following command
51+
52+
```
53+
curl http://localhost:8080/predictions/resnetcppaot -T ../../../../cpp/test/resources/examples/aot_inductor/resnet_handler/0_png.pt
54+
{
55+
"lens_cap": 0.0022578993812203407,
56+
"lynx": 0.0032067005522549152,
57+
"Egyptian_cat": 0.046274684369564056,
58+
"tiger_cat": 0.13740436732769012,
59+
"tabby": 0.2724998891353607
60+
}
61+
```

‎examples/cpp/aot_inductor/resnet/index_to_name.json

+1
Large diffs are not rendered by default.

‎examples/cpp/aot_inductor/resnet/model-config.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
minWorkers: 1
2+
maxWorkers: 1
3+
batchSize: 2
4+
5+
handler:
6+
model_so_path: "resnet50_pt2.so"
7+
mapping: "index_to_name.json"

‎examples/cpp/aot_inductor/resnet/resnet50_torch_export.py

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import os
2+
3+
import torch
4+
from torchvision.models import ResNet50_Weights, resnet50
5+
6+
torch.set_float32_matmul_precision("high")
7+
8+
MAX_BATCH_SIZE = 15
9+
10+
model = resnet50(weights=ResNet50_Weights.DEFAULT)
11+
model.eval()
12+
13+
with torch.no_grad():
14+
if torch.cuda.is_available():
15+
device = "cuda"
16+
else:
17+
device = "cpu"
18+
# The max batch size is less than 16. The following setting can only work in PT2.3.
19+
# We need to turn off the below optimizations to support batch_size = 16,
20+
# which is treated like a special case
21+
# https://github.com/pytorch/pytorch/pull/116152
22+
# torch.backends.mkldnn.set_flags(False)
23+
# torch.backends.nnpack.set_flags(False)
24+
25+
model = model.to(device=device)
26+
example_inputs = (torch.randn(2, 3, 224, 224, device=device),)
27+
28+
batch_dim = torch.export.Dim("batch", min=1, max=MAX_BATCH_SIZE)
29+
torch._C._GLIBCXX_USE_CXX11_ABI = True
30+
so_path = torch._export.aot_compile(
31+
model,
32+
example_inputs,
33+
# Specify the first dimension of the input x as dynamic
34+
dynamic_shapes={"x": {0: batch_dim}},
35+
# Specify the generated shared library path
36+
options={
37+
"aot_inductor.output_path": os.path.join(os.getcwd(), "resnet50_pt2.so"),
38+
"max_autotune": True,
39+
},
40+
)

0 commit comments

Comments
 (0)