fmt

Author: lxning

cpp/CMakeLists.txt

@@ -30,7 +30,7 @@ find_package(folly REQUIRED)
 find_package(fmt REQUIRED)
 find_package(gflags REQUIRED)
 find_package(Torch REQUIRED)
-find_package(yaml-cpp REQUIRED NO_CMAKE_PATH)
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 
 include_directories(${TORCH_INCLUDE_DIRS})

cpp/test/resources/examples/aot_inductor/resnet_handler/MAR-INF/MANIFEST.json

@@ -4,7 +4,8 @@
   "model": {
     "modelName": "resnetcppaot",
     "handler": "libresnet_handler:ResnetCppHandler",
-    "modelVersion": "1.0"
+    "modelVersion": "1.0",
+    "configFile": "model-config.yaml"
   },
   "archiverVersion": "0.9.0"
 }

cpp/test/resources/examples/aot_inductor/resnet_handler/config.json

@@ -1 +1,72 @@
-torch-model-archiver --model-name resnetcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/resnet/resnet_handler/libresnet_handler:ResnetCppHandler --runtime LSP --extra-files config.json,index_to_name.json,resnet50_pt2.so --archive-format no-archive
+This example uses AOTInductor to compile the Resnet50 into an so file which is then executed using libtorch.
+The handler C++ source code for this examples can be found [here](src).
+
+### Setup
+1. Follow the instructions in [README.md](../../../../cpp/README.md) to build the TorchServe C++ backend.
+
+```
+cd serve/cpp
+./builld.sh
+```
+
+The build script will create the necessary artifact for this example.
+To recreate these by hand you can follow the prepare_test_files function of the [build.sh](../../../../cpp/build.sh) script.
+We will need the handler .so file as well as the resne50_pt2.so file containing the model and weights.
+
+2. Create a [model-config.yaml](model-config.yaml)
+
+```yaml
+minWorkers: 1
+maxWorkers: 1
+batchSize: 2
+
+handler:
+  model_so_path: "resnet50_pt2.so"
+  mapping: "index_to_name.json"
+```
+
+### Generate MAR file
+
+Now lets generate the mar file
+
+```bash
+torch-model-archiver --model-name resnetcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/resnet_handler/libresnet_handler:ResnetCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/_build/test/resources/examples/aot_inductor/resnet_handler/resnet50_pt2.so --config-file model-config.yaml --archive-format no-archive
+```
+
+Create model store directory and move the mar file
+
+```
+mkdir model_store
+mv resnetcppaot model_store/
+```
+
+### Inference
+
+Start torchserve using the following command
+
+```
+torchserve --ncs --model-store model_store/
+```
+
+Register the model using the following command
+
+```
+curl -v -X POST "http://localhost:8081/models?initial_workers=1&url=resnetcppaot&batch_size=2&max_batch_delay=5000"
+
+{
+  "status": "Model \"resnetcppaot\" Version: 1.0 registered with 1 initial workers"
+}
+```
+
+Infer the model using the following command
+
+```
+curl http://localhost:8080/predictions/resnetcppaot -T ../../../../cpp/test/resources/examples/aot_inductor/resnet_handler/0_png.pt
+{
+  "lens_cap": 0.0022578993812203407,
+  "lynx": 0.0032067005522549152,
+  "Egyptian_cat": 0.046274684369564056,
+  "tiger_cat": 0.13740436732769012,
+  "tabby": 0.2724998891353607
+}
+```

examples/cpp/aot_inductor/resnet/README.md

@@ -0,0 +1,7 @@
+minWorkers: 1
+maxWorkers: 1
+batchSize: 2
+
+handler:
+  model_so_path: "resnet50_pt2.so"
+  mapping: "index_to_name.json"

examples/cpp/aot_inductor/resnet/config.json

@@ -37,15 +37,19 @@ ResnetCppHandler::LoadModel(
   try {
     auto device = GetTorchDevice(load_model_request);
 
+    const std::string modelConfigYamlFilePath =
+        fmt::format("{}/{}", load_model_request->model_dir, "model-config.yaml");
+    model_config_yaml_ = std::make_unique<YAML::Node>(YAML::LoadFile(modelConfigYamlFilePath));
+
     const std::string mapFilePath =
-        fmt::format("{}/{}", load_model_request->model_dir, "index_to_name.json");
+      fmt::format("{}/{}", load_model_request->model_dir,
+        (*model_config_yaml_)["handler"]["mapping"].as<std::string>());
     mapping_json_ = LoadJsonFile(mapFilePath);
 
-    const std::string configFilePath =
-        fmt::format("{}/{}", load_model_request->model_dir, "config.json");
-    config_json_ = LoadJsonFile(configFilePath);
-
-    std::string model_so_path = fmt::format("{}/{}", load_model_request->model_dir, GetJsonValue(config_json_, "model_so_path").asString());
+    std::string model_so_path =
+      fmt::format("{}/{}", load_model_request->model_dir,
+        (*model_config_yaml_)["handler"]["model_so_path"].as<std::string>());
+    mapping_json_ = LoadJsonFile(mapFilePath);
     c10::InferenceMode mode;
 
     if (device->is_cuda()) {

examples/cpp/aot_inductor/resnet/model-config.yaml

@@ -8,6 +8,7 @@
 #include <torch/torch.h>
 #include <torch/csrc/inductor/aoti_model_container_runner.h>
 #include <torch/csrc/inductor/aoti_model_container_runner_cuda.h>
+#include <yaml-cpp/yaml.h>
 
 #include "src/backends/handler/base_handler.hh"
 
@@ -48,7 +49,7 @@ private:
   const folly::dynamic& GetJsonValue(std::unique_ptr<folly::dynamic>& json, const std::string& key);
   std::string MapClassToLabel(const torch::Tensor& classes, const torch::Tensor& probs);
 
-  std::unique_ptr<folly::dynamic> config_json_;
   std::unique_ptr<folly::dynamic> mapping_json_;
+  std::unique_ptr<YAML::Node> model_config_yaml_;
 };
 }  // namespac