From 798258819cdc3de6d65fe26b80ba4bec0e39f611 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 12 Mar 2024 21:53:01 +0000
Subject: [PATCH 01/19] Move cpp install into cmake file

---
 cpp/CMakeLists.txt                    |  2 ++
 cpp/build.sh                          | 13 -------------
 cpp/install.cmake                     | 14 ++++++++++++++
 examples/cpp/babyllama/CMakeLists.txt |  2 +-
 4 files changed, 17 insertions(+), 14 deletions(-)
 create mode 100644 cpp/install.cmake

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index aaa2c3fa0d..ca804bbcfc 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -71,3 +71,5 @@ add_subdirectory(src/examples)
 add_subdirectory(test)
 
 FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
+
+install(SCRIPT "install.cmake")
diff --git a/cpp/build.sh b/cpp/build.sh
index 2b63b372d5..2c058c96ff 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -95,18 +95,6 @@ function build() {
   fi
 }
 
-function install_torchserve_cpp() {
-  TARGET_DIR=`python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"`
-
-  if [ -d $TARGET_DIR ]; then
-    rm -rf $TARGET_DIR
-  fi
-  mkdir $TARGET_DIR
-  cp -rp $BASE_DIR/_build/bin $TARGET_DIR/bin
-  cp -rp $BASE_DIR/_build/libs $TARGET_DIR/lib
-  cp -rp $BASE_DIR/_build/resources $TARGET_DIR/resources
-}
-
 # Parse args
 JOBS=8
 WITH_QUIC=false
@@ -168,4 +156,3 @@ cd $BASE_DIR
 git submodule update --init --recursive
 
 build
-install_torchserve_cpp
diff --git a/cpp/install.cmake b/cpp/install.cmake
new file mode 100644
index 0000000000..6ed6a6cdf8
--- /dev/null
+++ b/cpp/install.cmake
@@ -0,0 +1,14 @@
+execute_process(COMMAND python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"
+                OUTPUT_VARIABLE TARGET_DIR
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+message("Installing cpp backend into ${TARGET_DIR}")
+
+if(EXISTS ${TARGET_DIR})
+    execute_process(COMMAND rm -rf ${TARGET_DIR})
+endif()
+
+execute_process(COMMAND mkdir ${TARGET_DIR})
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/bin ${TARGET_DIR}/bin)
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/libs ${TARGET_DIR}/lib)
+execute_process(COMMAND cp -rp ${CMAKE_BINARY_DIR}/resources ${TARGET_DIR}/resources)
diff --git a/examples/cpp/babyllama/CMakeLists.txt b/examples/cpp/babyllama/CMakeLists.txt
index 39e4b3b7e9..933285d202 100644
--- a/examples/cpp/babyllama/CMakeLists.txt
+++ b/examples/cpp/babyllama/CMakeLists.txt
@@ -16,7 +16,7 @@ FetchContent_Declare(
 
 FetchContent_MakeAvailable(tokenizer_bin stories15M_bin)
 
-add_library(llama2_c STATIC ../../../cpp/third-party/llama2.c/run.c)
+add_library(llama2_c STATIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../cpp/third-party/llama2.c/run.c)
 target_compile_options(llama2_c PRIVATE -Wall -Wextra -Ofast -fPIC)
 
 add_library(babyllama_handler SHARED src/baby_llama_handler.cc)

From cb2b524935da1e86d313cc58ab3db59168ade7dd Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 12 Mar 2024 21:55:14 +0000
Subject: [PATCH 02/19] add pytest to cpp ci workflow

---
 .github/workflows/ci-cpu-cpp.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index ff7db9cead..1d22ba6ee6 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -55,3 +55,6 @@ jobs:
         run: |
           cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
           ./build.sh
+      - name: Run test
+        run: |
+          cd .. && pytest test/pytest/test_cpp_backend.py

From f19c858565a75e8b611f8cf4c0bb840e943e4ff9 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 00:07:13 +0000
Subject: [PATCH 03/19] Fix cpp ci

---
 .github/workflows/ci-cpu-cpp.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 1d22ba6ee6..d31a4c1407 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -47,7 +47,7 @@ jobs:
           sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
       - name: Install TorchServe
         run: |
-          python ts_scripts/install_from_src.py
+          python ts_scripts/install_from_src.py --environment=dev
       - name: Print Env
         run: |
           python ts_scripts/print_env_info.py
@@ -57,4 +57,4 @@ jobs:
           ./build.sh
       - name: Run test
         run: |
-          cd .. && pytest test/pytest/test_cpp_backend.py
+          pytest test/pytest/test_cpp_backend.py

From ada08dbc9431ea288085369a987c6a1e38675009 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 00:28:37 +0000
Subject: [PATCH 04/19] Add debugging

---
 .github/workflows/ci-cpu-cpp.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index d31a4c1407..cbcc03bcbd 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -55,6 +55,11 @@ jobs:
         run: |
           cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
           ./build.sh
+      - name: Setup upterm session
+        uses: lhotari/action-upterm@v1
+        with:
+          limit-access-to-actor: true
+          limit-access-to-users: mreso
       - name: Run test
         run: |
           pytest test/pytest/test_cpp_backend.py

From 7c09df61afca0f179b400ab50ccea7aad1f2b263 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 03:46:01 +0000
Subject: [PATCH 05/19] Change exec dir for test

---
 .github/workflows/ci-cpu-cpp.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index cbcc03bcbd..40d57ce57c 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -47,7 +47,7 @@ jobs:
           sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
       - name: Install TorchServe
         run: |
-          python ts_scripts/install_from_src.py --environment=dev
+          python ts_scripts/install_from_src.py
       - name: Print Env
         run: |
           python ts_scripts/print_env_info.py
@@ -55,11 +55,6 @@ jobs:
         run: |
           cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
           ./build.sh
-      - name: Setup upterm session
-        uses: lhotari/action-upterm@v1
-        with:
-          limit-access-to-actor: true
-          limit-access-to-users: mreso
       - name: Run test
         run: |
-          pytest test/pytest/test_cpp_backend.py
+          cd test && pytest pytest/test_cpp_backend.py

From 4412629d339e4acb77bbd3c4682d1edb220a7abb Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 05:11:44 +0000
Subject: [PATCH 06/19] Start running of tests to ctest; removed from build.sh
 start fixing unit tests

---
 cpp/CMakeLists.txt                            |  2 +
 cpp/build.sh                                  | 57 +++++--------------
 cpp/test/CMakeLists.txt                       |  2 +-
 .../backends/otf_protocol_and_handler_test.cc | 14 ++---
 .../torch_scripted/torch_scripted_test.cc     |  6 +-
 cpp/test/utils/common.hh                      |  2 +-
 examples/cpp/llamacpp/CMakeLists.txt          |  4 ++
 7 files changed, 32 insertions(+), 55 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ca804bbcfc..af5068df72 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -73,3 +73,5 @@ add_subdirectory(test)
 FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
 
 install(SCRIPT "install.cmake")
+
+enable_testing()
diff --git a/cpp/build.sh b/cpp/build.sh
index 2c058c96ff..eb665ab40d 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -22,15 +22,7 @@ function detect_platform() {
 
 function build() {
   echo -e "${COLOR_GREEN}[ INFO ]Building backend ${COLOR_OFF}"
-  MAYBE_BUILD_QUIC=""
-  if [ "$WITH_QUIC" == true ] ; then
-    setup_mvfst
-    MAYBE_BUILD_QUIC="-DBUILD_QUIC=On"
-  fi
 
-  MAYBE_USE_STATIC_DEPS=""
-  MAYBE_LIB_FUZZING_ENGINE=""
-  MAYBE_BUILD_SHARED_LIBS=""
   MAYBE_BUILD_TESTS="-DBUILD_TESTS=ON"
   if [ "$NO_BUILD_TESTS" == true ] ; then
     MAYBE_BUILD_TESTS="-DBUILD_TESTS=OFF"
@@ -48,51 +40,33 @@ function build() {
   if [ "$PLATFORM" = "Linux" ]; then
     NCCL_PATH=`python3 -c 'import torch;from pathlib import Path;print(Path(torch.__file__).parents[1]/"nvidia"/"nccl"/"lib")'`
     export LD_LIBRARY_PATH=${NCCL_PATH}:${LD_LIBRARY_PATH}
-    cmake                                                                                     \
-    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH"                                                           \
-    -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
-    "$MAYBE_BUILD_QUIC"                                                                       \
-    "$MAYBE_BUILD_TESTS"                                                                      \
-    "$MAYBE_BUILD_SHARED_LIBS"                                                                \
-    "$MAYBE_OVERRIDE_CXX_FLAGS"                                                               \
-    "$MAYBE_USE_STATIC_DEPS"                                                                  \
-    "$MAYBE_LIB_FUZZING_ENGINE"                                                               \
-    ..
-
   elif [ "$PLATFORM" = "Mac" ]; then
     export LIBRARY_PATH=${LIBRARY_PATH}:`brew --prefix icu4c`/lib:`brew --prefix libomp`/lib
+  else
+    # TODO: Windows
+    echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
+    exit 1
+  fi
 
-    cmake                                                                                     \
+  cmake                                                                                       \
     -DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH"                                        \
     -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
-    "$MAYBE_BUILD_QUIC"                                                                       \
     "$MAYBE_BUILD_TESTS"                                                                      \
-    "$MAYBE_BUILD_SHARED_LIBS"                                                                \
     "$MAYBE_OVERRIDE_CXX_FLAGS"                                                               \
-    "$MAYBE_USE_STATIC_DEPS"                                                                  \
-    "$MAYBE_LIB_FUZZING_ENGINE"                                                               \
-    "-DLLAMA_METAL=OFF"                                                                       \
     ..
 
-
-  else
-    # TODO: Windows
-    echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
-    exit 1
-  fi
-
   make -j "$JOBS"
   make install
   echo -e "${COLOR_GREEN}torchserve_cpp build is complete. To run unit test: \
   ./_build/test/torchserve_cpp_test ${COLOR_OFF}"
 
-  cd $DEPS_DIR/../..
-  if [ -f "$DEPS_DIR/../test/torchserve_cpp_test" ]; then
-    $DEPS_DIR/../test/torchserve_cpp_test
-  else
-    echo -e "${COLOR_RED}[ ERROR ] _build/test/torchserve_cpp_test not exist ${COLOR_OFF}"
-    exit 1
-  fi
+  # cd $DEPS_DIR/../..
+  # if [ -f "$DEPS_DIR/../test/torchserve_cpp_test" ]; then
+  #   $DEPS_DIR/../test/torchserve_cpp_test
+  # else
+  #   echo -e "${COLOR_RED}[ ERROR ] _build/test/torchserve_cpp_test not exist ${COLOR_OFF}"
+  #   exit 1
+  # fi
 }
 
 # Parse args
@@ -101,15 +75,12 @@ WITH_QUIC=false
 INSTALL_DEPENDENCIES=false
 PREFIX=""
 COMPILER_FLAGS=""
-USAGE="./build.sh [-j num_jobs] [-q|--with-quic] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags]"
+USAGE="./build.sh [-j num_jobs] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags]"
 while [ "$1" != "" ]; do
   case $1 in
     -j | --jobs ) shift
                   JOBS=$1
                   ;;
-    -q | --with-quic )
-                  WITH_QUIC=true
-                  ;;
     -t | --no-tests )
                   NO_BUILD_TESTS=true
       ;;
diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt
index 838b250bd9..e418ccacb2 100644
--- a/cpp/test/CMakeLists.txt
+++ b/cpp/test/CMakeLists.txt
@@ -6,7 +6,7 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(googletest)
 # run google test
-enable_testing()
+
 
 set(TEST_BINARY ${CMAKE_PROJECT_NAME}_test)
 file(GLOB_RECURSE TEST_SOURCES LIST_DIRECTORIES false *.cc *.hh)
diff --git a/cpp/test/backends/otf_protocol_and_handler_test.cc b/cpp/test/backends/otf_protocol_and_handler_test.cc
index 89e70205a9..fb6823f1c9 100644
--- a/cpp/test/backends/otf_protocol_and_handler_test.cc
+++ b/cpp/test/backends/otf_protocol_and_handler_test.cc
@@ -24,7 +24,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
       // model_name length
       .WillOnce(::testing::Return(5))
       // model_path length
-      .WillOnce(::testing::Return(42))
+      .WillOnce(::testing::Return(45))
       // batch_size
       .WillOnce(::testing::Return(1))
       // handler length
@@ -44,8 +44,8 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
         strncpy(data, "mnist", length);
       }))
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
-        ASSERT_EQ(length, 42);
-        strncpy(data, "test/resources/examples/mnist/base_handler", length);
+        ASSERT_EQ(length, 45);
+        strncpy(data, "../test/resources/examples/mnist/base_handler", length);
       }))
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
         ASSERT_EQ(length, 11);
@@ -59,7 +59,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
   EXPECT_CALL(*client_socket, SendAll(testing::_, testing::_)).Times(1);
   auto load_model_request = OTFMessage::RetrieveLoadMsg(*client_socket);
   ASSERT_EQ(load_model_request->model_dir,
-            "test/resources/examples/mnist/base_handler");
+            "../test/resources/examples/mnist/base_handler");
   ASSERT_EQ(load_model_request->model_name, "mnist");
   ASSERT_EQ(load_model_request->envelope, "");
   ASSERT_EQ(load_model_request->model_name, "mnist");
@@ -68,9 +68,9 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
 
   // initialize backend
   auto backend = std::make_shared<torchserve::Backend>();
-  MetricsRegistry::Initialize("test/resources/metrics/default_config.yaml",
+  MetricsRegistry::Initialize("../test/resources/metrics/default_config.yaml",
                               MetricsContext::BACKEND);
-  backend->Initialize("test/resources/examples/mnist/base_handler");
+  backend->Initialize("../test/resources/examples/mnist/base_handler");
 
   // load the model
   auto load_model_response = backend->LoadModel(load_model_request);
@@ -125,7 +125,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
         ASSERT_EQ(length, 3883);
         // strncpy(data, "valu", length);
-        std::ifstream input("test/resources/examples/mnist/0_png.pt",
+        std::ifstream input("../test/resources/examples/mnist/0_png.pt",
                             std::ios::in | std::ios::binary);
         std::vector<char> image((std::istreambuf_iterator<char>(input)),
                                 (std::istreambuf_iterator<char>()));
diff --git a/cpp/test/torch_scripted/torch_scripted_test.cc b/cpp/test/torch_scripted/torch_scripted_test.cc
index 5f1c986151..9ef6822ce8 100644
--- a/cpp/test/torch_scripted/torch_scripted_test.cc
+++ b/cpp/test/torch_scripted/torch_scripted_test.cc
@@ -9,10 +9,10 @@
 
 TEST_F(ModelPredictTest, TestLoadPredictBaseHandler) {
   this->LoadPredict(std::make_shared<torchserve::LoadModelRequest>(
-                        "_build/test/resources/examples/mnist/mnist_handler",
+                        "../test/resources/examples/mnist/mnist_handler",
                         "mnist_scripted_v2", -1, "", "", 1, false),
-                    "_build/test/resources/examples/mnist/base_handler",
-                    "_build/test/resources/examples/mnist/0_png.pt", "mnist_ts",
+                    "../test/resources/examples/mnist/base_handler",
+                    "../test/resources/examples/mnist/0_png.pt", "mnist_ts",
                     200);
 }
 
diff --git a/cpp/test/utils/common.hh b/cpp/test/utils/common.hh
index 27d548503a..254cef76c7 100644
--- a/cpp/test/utils/common.hh
+++ b/cpp/test/utils/common.hh
@@ -15,7 +15,7 @@ class ModelPredictTest : public ::testing::Test {
       const std::string& inference_request_id_prefix,
       int inference_expect_code) {
     torchserve::MetricsRegistry::Initialize(
-        "test/resources/metrics/default_config.yaml",
+        "../test/resources/metrics/default_config.yaml",
         torchserve::MetricsContext::BACKEND);
     backend_->Initialize(model_dir);
     auto result = backend_->LoadModel(std::move(load_model_request));
diff --git a/examples/cpp/llamacpp/CMakeLists.txt b/examples/cpp/llamacpp/CMakeLists.txt
index 084c667830..c08736ebfc 100644
--- a/examples/cpp/llamacpp/CMakeLists.txt
+++ b/examples/cpp/llamacpp/CMakeLists.txt
@@ -3,6 +3,10 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 add_library(llamacpp_handler SHARED src/llamacpp_handler.cc)
 
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  set(LLAMA_METAL OFF)
+endif()
+
 FetchContent_Declare(
   llama.cpp
   GIT_REPOSITORY https://github.com/ggerganov/llama.cpp

From 93066e2715d6a043fff35ac637604d9f5cbd11ec Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:09:45 +0000
Subject: [PATCH 07/19] fix tests

---
 cpp/test/examples/examples_test.cc            | 12 ++++-----
 .../aot_inductor/llama_handler/config.json    |  4 +--
 .../babyllama/babyllama_handler/config.json   |  4 +--
 cpp/test/resources/logging/log_to_file.yaml   |  2 +-
 cpp/test/resources/metrics/log_to_file.yaml   |  2 +-
 .../torch_scripted/torch_scripted_test.cc     | 26 +++++++++----------
 cpp/test/utils/dl_loader_test.cc              |  4 +--
 cpp/test/utils/json_test.cc                   |  2 +-
 cpp/test/utils/logging_test.cc                |  6 ++---
 cpp/test/utils/metrics/log_metric_test.cc     |  4 +--
 .../utils/metrics/log_metrics_cache_test.cc   |  2 +-
 cpp/test/utils/metrics/registry_test.cc       |  8 +++---
 cpp/test/utils/metrics/yaml_config_test.cc    | 20 +++++++-------
 cpp/test/utils/model_archiver_test.cc         |  2 +-
 14 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/cpp/test/examples/examples_test.cc b/cpp/test/examples/examples_test.cc
index 433a94d7f6..5b832149d7 100644
--- a/cpp/test/examples/examples_test.cc
+++ b/cpp/test/examples/examples_test.cc
@@ -5,7 +5,7 @@
 #include "test/utils/common.hh"
 
 TEST_F(ModelPredictTest, TestLoadPredictBabyLlamaHandler) {
-  std::string base_dir = "_build/test/resources/examples/babyllama/";
+  std::string base_dir = "resources/examples/babyllama/";
   std::string file1 = base_dir + "babyllama_handler/stories15M.bin";
   std::string file2 = base_dir + "babyllama_handler/tokenizer.bin";
 
@@ -24,10 +24,10 @@ TEST_F(ModelPredictTest, TestLoadPredictBabyLlamaHandler) {
 }
 
 TEST_F(ModelPredictTest, TestLoadPredictAotInductorLlamaHandler) {
-  std::string base_dir = "_build/test/resources/examples/aot_inductor/";
+  std::string base_dir = "resources/examples/aot_inductor/";
   std::string file1 = base_dir + "llama_handler/stories15M.so";
   std::string file2 =
-      "_build/test/resources/examples/babyllama/babyllama_handler/"
+      "resources/examples/babyllama/babyllama_handler/"
       "tokenizer.bin";
 
   std::ifstream f1(file1);
@@ -45,7 +45,7 @@ TEST_F(ModelPredictTest, TestLoadPredictAotInductorLlamaHandler) {
 }
 
 TEST_F(ModelPredictTest, TestLoadPredictLlamaCppHandler) {
-  std::string base_dir = "_build/test/resources/examples/llamacpp/";
+  std::string base_dir = "resources/examples/llamacpp/";
   std::string file1 = base_dir + "llamacpp_handler/llama-2-7b-chat.Q5_0.gguf";
   std::ifstream f(file1);
 
@@ -61,7 +61,7 @@ TEST_F(ModelPredictTest, TestLoadPredictLlamaCppHandler) {
 }
 
 TEST_F(ModelPredictTest, TestLoadPredictAotInductorBertHandler) {
-  std::string base_dir = "_build/test/resources/examples/aot_inductor/";
+  std::string base_dir = "resources/examples/aot_inductor/";
   std::string file1 = base_dir + "bert_handler/bert-seq.so";
   std::string file2 = base_dir + "bert_handler/tokenizer.json";
 
@@ -84,7 +84,7 @@ TEST_F(ModelPredictTest, TestLoadPredictAotInductorBertHandler) {
 }
 
 TEST_F(ModelPredictTest, TestLoadPredictAotInductorResnetHandler) {
-  std::string base_dir = "_build/test/resources/examples/aot_inductor/";
+  std::string base_dir = "resources/examples/aot_inductor/";
   std::string file1 = base_dir + "resnet_handler/resnet50_pt2.so";
 
   std::ifstream f1(file1);
diff --git a/cpp/test/resources/examples/aot_inductor/llama_handler/config.json b/cpp/test/resources/examples/aot_inductor/llama_handler/config.json
index 04e1cd48ee..c3d8155f79 100644
--- a/cpp/test/resources/examples/aot_inductor/llama_handler/config.json
+++ b/cpp/test/resources/examples/aot_inductor/llama_handler/config.json
@@ -1,4 +1,4 @@
 {
-"checkpoint_path" : "_build/test/resources/examples/aot_inductor/llama_handler/stories15M.so",
-"tokenizer_path" : "_build/test/resources/examples/babyllama/babyllama_handler/tokenizer.bin"
+"checkpoint_path" : "resources/examples/aot_inductor/llama_handler/stories15M.so",
+"tokenizer_path" : "resources/examples/babyllama/babyllama_handler/tokenizer.bin"
 }
diff --git a/cpp/test/resources/examples/babyllama/babyllama_handler/config.json b/cpp/test/resources/examples/babyllama/babyllama_handler/config.json
index c88e48143b..1e9a5d6410 100644
--- a/cpp/test/resources/examples/babyllama/babyllama_handler/config.json
+++ b/cpp/test/resources/examples/babyllama/babyllama_handler/config.json
@@ -1,4 +1,4 @@
 {
-"checkpoint_path" : "_build/test/resources/examples/babyllama/babyllama_handler/stories15M.bin",
-"tokenizer_path" : "_build/test/resources/examples/babyllama/babyllama_handler/tokenizer.bin"
+"checkpoint_path" : "resources/examples/babyllama/babyllama_handler/stories15M.bin",
+"tokenizer_path" : "resources/examples/babyllama/babyllama_handler/tokenizer.bin"
 }
diff --git a/cpp/test/resources/logging/log_to_file.yaml b/cpp/test/resources/logging/log_to_file.yaml
index a938a256d4..9d4dbceb09 100644
--- a/cpp/test/resources/logging/log_to_file.yaml
+++ b/cpp/test/resources/logging/log_to_file.yaml
@@ -1,4 +1,4 @@
 config:
   loglevel: INFO
-  logfile: "test/resources/logging/test.log"
+  logfile: "resources/logging/test.log"
   async: false
diff --git a/cpp/test/resources/metrics/log_to_file.yaml b/cpp/test/resources/metrics/log_to_file.yaml
index c18935566b..5ea0b437ea 100644
--- a/cpp/test/resources/metrics/log_to_file.yaml
+++ b/cpp/test/resources/metrics/log_to_file.yaml
@@ -1,4 +1,4 @@
 config:
   loglevel: INFO
-  logfile: "test/utils/metrics/metrics_test.log"
+  logfile: "utils/metrics/metrics_test.log"
   async: false
diff --git a/cpp/test/torch_scripted/torch_scripted_test.cc b/cpp/test/torch_scripted/torch_scripted_test.cc
index 9ef6822ce8..51f522a805 100644
--- a/cpp/test/torch_scripted/torch_scripted_test.cc
+++ b/cpp/test/torch_scripted/torch_scripted_test.cc
@@ -9,47 +9,47 @@
 
 TEST_F(ModelPredictTest, TestLoadPredictBaseHandler) {
   this->LoadPredict(std::make_shared<torchserve::LoadModelRequest>(
-                        "../test/resources/examples/mnist/mnist_handler",
+                        "resources/examples/mnist/mnist_handler",
                         "mnist_scripted_v2", -1, "", "", 1, false),
-                    "../test/resources/examples/mnist/base_handler",
-                    "../test/resources/examples/mnist/0_png.pt", "mnist_ts",
+                    "resources/examples/mnist/base_handler",
+                    "resources/examples/mnist/0_png.pt", "mnist_ts",
                     200);
 }
 
 TEST_F(ModelPredictTest, TestLoadPredictMnistHandler) {
   this->LoadPredict(std::make_shared<torchserve::LoadModelRequest>(
-                        "_build/test/resources/examples/mnist/mnist_handler",
+                        "resources/examples/mnist/mnist_handler",
                         "mnist_scripted_v2", -1, "", "", 1, false),
-                    "_build/test/resources/examples/mnist/mnist_handler",
-                    "_build/test/resources/examples/mnist/0_png.pt", "mnist_ts",
+                    "resources/examples/mnist/mnist_handler",
+                    "resources/examples/mnist/0_png.pt", "mnist_ts",
                     200);
 }
 
 TEST_F(ModelPredictTest, TestBackendInitWrongModelDir) {
-  auto result = backend_->Initialize("_build/test/resources/examples/mnist");
+  auto result = backend_->Initialize("resources/examples/mnist");
   ASSERT_EQ(result, false);
 }
 
 TEST_F(ModelPredictTest, TestBackendInitWrongHandler) {
   auto result = backend_->Initialize(
-      "_build/test/resources/examples/mnist/wrong_handler");
+      "resources/examples/mnist/wrong_handler");
   ASSERT_EQ(result, false);
 }
 
 TEST_F(ModelPredictTest, TestLoadModelFailure) {
-  backend_->Initialize("_build/test/resources/examples/mnist/wrong_model");
+  backend_->Initialize("resources/examples/mnist/wrong_model");
   auto result =
       backend_->LoadModel(std::make_shared<torchserve::LoadModelRequest>(
-          "_build/test/resources/examples/mnist/wrong_model",
+          "resources/examples/mnist/wrong_model",
           "mnist_scripted_v2", -1, "", "", 1, false));
   ASSERT_EQ(result->code, 500);
 }
 
 TEST_F(ModelPredictTest, TestLoadPredictMnistHandlerFailure) {
   this->LoadPredict(std::make_shared<torchserve::LoadModelRequest>(
-                        "_build/test/resources/examples/mnist/mnist_handler",
+                        "resources/examples/mnist/mnist_handler",
                         "mnist_scripted_v2", -1, "", "", 1, false),
-                    "_build/test/resources/examples/mnist/mnist_handler",
-                    "_build/test/resources/examples/mnist/0.png", "mnist_ts",
+                    "resources/examples/mnist/mnist_handler",
+                    "resources/examples/mnist/0.png", "mnist_ts",
                     500);
 }
diff --git a/cpp/test/utils/dl_loader_test.cc b/cpp/test/utils/dl_loader_test.cc
index 30b9a30e57..db6c307467 100644
--- a/cpp/test/utils/dl_loader_test.cc
+++ b/cpp/test/utils/dl_loader_test.cc
@@ -13,9 +13,9 @@ class DLLoaderTest : public ::testing::Test {
  protected:
   void SetUp() override {
 #ifdef __APPLE__
-    lib_path_ = "./_build/src/utils/libfoo.dylib";
+    lib_path_ = "../src/utils/libfoo.dylib";
 #elif __linux__
-    lib_path_ = "./_build/src/utils/libfoo.so";
+    lib_path_ = "../src/utils/libfoo.so";
 #endif
 
     dl_loader_ = new torchserve::DLLoader<IFoo>(lib_path_);
diff --git a/cpp/test/utils/json_test.cc b/cpp/test/utils/json_test.cc
index a0167af738..0f95e77b94 100644
--- a/cpp/test/utils/json_test.cc
+++ b/cpp/test/utils/json_test.cc
@@ -5,7 +5,7 @@
 namespace torchserve {
 
 TEST(JsonTest, TestParsingAndGetValue) {
-  std::string json_file = "test/resources/test.json";
+  std::string json_file = "resources/test.json";
   auto data = Json::ParseJsonFile(json_file);
 
   EXPECT_TRUE(data.GetValue("string").AsString().compare("test") == 0);
diff --git a/cpp/test/utils/logging_test.cc b/cpp/test/utils/logging_test.cc
index af63c0387a..0604a62227 100644
--- a/cpp/test/utils/logging_test.cc
+++ b/cpp/test/utils/logging_test.cc
@@ -23,16 +23,16 @@ void Cleanup(const std::string& logfile_path) {
 }
 
 TEST(LoggingTest, TestIncorrectLogInitialization) {
-  std::string logger_config_path_str = "test/resources/logging/invalid.yaml";
+  std::string logger_config_path_str = "resources/logging/invalid.yaml";
   EXPECT_THROW(torchserve::Logger::InitLogger(logger_config_path_str),
                std::invalid_argument);
 }
 
 
 TEST(LoggingTest, TestFileLogInitialization) {
-  std::string logfile_path = "test/resources/logging/test.log";
+  std::string logfile_path = "resources/logging/test.log";
   std::string logger_config_path_str =
-      "test/resources/logging/log_to_file.yaml";
+      "resources/logging/log_to_file.yaml";
   torchserve::Logger::InitLogger(logger_config_path_str);
   std::string log_line("Test");
   TS_LOG(INFO, log_line);
diff --git a/cpp/test/utils/metrics/log_metric_test.cc b/cpp/test/utils/metrics/log_metric_test.cc
index be93fc34db..94e4d28b1c 100644
--- a/cpp/test/utils/metrics/log_metric_test.cc
+++ b/cpp/test/utils/metrics/log_metric_test.cc
@@ -22,9 +22,9 @@ class TestLogger : public Logger {
 
 class TSLogMetricTest : public ::testing::Test {
  protected:
-  const std::string logfile_path{"test/utils/metrics/metrics_test.log"};
+  const std::string logfile_path{"utils/metrics/metrics_test.log"};
   const std::string logger_config_path_str{
-      "test/resources/metrics/log_to_file.yaml"};
+      "resources/metrics/log_to_file.yaml"};
   const std::string metric_name{"test_metric"};
   const std::vector<std::string> metric_dimension_names{"level", "model_name"};
   const std::vector<std::string> metric_dimension_values{"model", "test_model"};
diff --git a/cpp/test/utils/metrics/log_metrics_cache_test.cc b/cpp/test/utils/metrics/log_metrics_cache_test.cc
index c0c7d94e78..14ae27adee 100644
--- a/cpp/test/utils/metrics/log_metrics_cache_test.cc
+++ b/cpp/test/utils/metrics/log_metrics_cache_test.cc
@@ -11,7 +11,7 @@ namespace torchserve {
 class TSLogMetricsCacheTest : public ::testing::Test {
  protected:
   const std::string config_file_path{
-      "test/resources/metrics/valid_config.yaml"};
+      "resources/metrics/valid_config.yaml"};
   MetricsConfigurationHandler* config_handler{nullptr};
   LogMetricsCache* cache{nullptr};
 
diff --git a/cpp/test/utils/metrics/registry_test.cc b/cpp/test/utils/metrics/registry_test.cc
index f5a0fbd4d3..8c0e269056 100644
--- a/cpp/test/utils/metrics/registry_test.cc
+++ b/cpp/test/utils/metrics/registry_test.cc
@@ -7,7 +7,7 @@
 
 namespace torchserve {
 TEST(RegistryTest, TestValidConfigFile) {
-  MetricsRegistry::Initialize("test/resources/metrics/valid_config.yaml",
+  MetricsRegistry::Initialize("resources/metrics/valid_config.yaml",
                               MetricsContext::BACKEND);
   ASSERT_THAT(MetricsRegistry::GetMetricsCacheInstance(),
               ::testing::A<std::shared_ptr<MetricsCache>>());
@@ -16,19 +16,19 @@ TEST(RegistryTest, TestValidConfigFile) {
 TEST(RegistryTest, TestInvalidConfigFile) {
   ASSERT_THROW(
       MetricsRegistry::Initialize(
-          "test/resources/metrics/invalid_config_duplicate_dimension.yaml",
+          "resources/metrics/invalid_config_duplicate_dimension.yaml",
           MetricsContext::BACKEND),
       std::invalid_argument);
   ASSERT_THROW(MetricsRegistry::GetMetricsCacheInstance(), std::runtime_error);
 }
 
 TEST(RegistryTest, TestReInitialize) {
-  MetricsRegistry::Initialize("test/resources/metrics/valid_config.yaml",
+  MetricsRegistry::Initialize("resources/metrics/valid_config.yaml",
                               MetricsContext::BACKEND);
   ASSERT_THAT(MetricsRegistry::GetMetricsCacheInstance(),
               ::testing::A<std::shared_ptr<MetricsCache>>());
 
-  MetricsRegistry::Initialize("test/resources/metrics/default_config.yaml",
+  MetricsRegistry::Initialize("resources/metrics/default_config.yaml",
                               MetricsContext::BACKEND);
   ASSERT_THAT(MetricsRegistry::GetMetricsCacheInstance(),
               ::testing::A<std::shared_ptr<MetricsCache>>());
diff --git a/cpp/test/utils/metrics/yaml_config_test.cc b/cpp/test/utils/metrics/yaml_config_test.cc
index 8acb39827f..92d0ab8514 100644
--- a/cpp/test/utils/metrics/yaml_config_test.cc
+++ b/cpp/test/utils/metrics/yaml_config_test.cc
@@ -16,7 +16,7 @@ TEST(YAMLConfigTest, TestLoadValidConfigFrontendContext) {
   ASSERT_TRUE(config_handler.GetModelMetrics().empty());
 
   const std::string config_file_path =
-      "test/resources/metrics/valid_config.yaml";
+      "resources/metrics/valid_config.yaml";
   config_handler.LoadConfiguration(config_file_path, MetricsContext::FRONTEND);
 
   ASSERT_EQ(config_handler.GetMode(), MetricsMode::PROMETHEUS);
@@ -69,7 +69,7 @@ TEST(YAMLConfigTest, TestLoadValidConfigBackendContext) {
   ASSERT_TRUE(config_handler.GetModelMetrics().empty());
 
   const std::string config_file_path =
-      "test/resources/metrics/valid_config.yaml";
+      "resources/metrics/valid_config.yaml";
   config_handler.LoadConfiguration(config_file_path, MetricsContext::BACKEND);
 
   std::set<std::string> expected_dimension_names = {"model_name", "host_name",
@@ -102,7 +102,7 @@ TEST(YAMLConfigTest, TestLoadMinimalValidConfig) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/minimal_valid_config.yaml";
+      "resources/metrics/minimal_valid_config.yaml";
   config_handler.LoadConfiguration(config_file_path, MetricsContext::FRONTEND);
 
   ASSERT_EQ(config_handler.GetMode(), MetricsMode::LOG);
@@ -119,7 +119,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithDuplicateDimension) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/invalid_config_duplicate_dimension.yaml";
+      "resources/metrics/invalid_config_duplicate_dimension.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
                std::invalid_argument);
@@ -134,7 +134,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithEmptyDimension) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/invalid_config_empty_dimension.yaml";
+      "resources/metrics/invalid_config_empty_dimension.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
                std::invalid_argument);
@@ -149,7 +149,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithUndefinedDimension) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/invalid_config_undefined_dimension.yaml";
+      "resources/metrics/invalid_config_undefined_dimension.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
                std::invalid_argument);
@@ -164,7 +164,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithDuplicateMetricDimension) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/"
+      "resources/metrics/"
       "invalid_config_duplicate_metric_dimension.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
@@ -180,7 +180,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithMissingMetricName) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/"
+      "resources/metrics/"
       "invalid_config_missing_metric_name.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
@@ -196,7 +196,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithEmptyMetricName) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/"
+      "resources/metrics/"
       "invalid_config_empty_metric_name.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
@@ -212,7 +212,7 @@ TEST(YAMLConfigTest, TestLoadInvalidConfigWithDuplicateMetricName) {
   YAMLMetricsConfigurationHandler config_handler =
       YAMLMetricsConfigurationHandler();
   const std::string config_file_path =
-      "test/resources/metrics/"
+      "resources/metrics/"
       "invalid_config_duplicate_metric_name.yaml";
   ASSERT_THROW(config_handler.LoadConfiguration(config_file_path,
                                                 MetricsContext::FRONTEND),
diff --git a/cpp/test/utils/model_archiver_test.cc b/cpp/test/utils/model_archiver_test.cc
index 596048e266..73c3689a17 100644
--- a/cpp/test/utils/model_archiver_test.cc
+++ b/cpp/test/utils/model_archiver_test.cc
@@ -6,7 +6,7 @@ namespace torchserve {
 TEST(ManifestTest, TestInitialize) {
   torchserve::Manifest manifest;
   manifest.Initialize(
-      "test/resources/examples/mnist/base_handler/MAR-INF/"
+      "resources/examples/mnist/base_handler/MAR-INF/"
       "MANIFEST.json");
   ASSERT_EQ(manifest.GetCreatOn(), "28/07/2020 06:32:08");
   ASSERT_EQ(manifest.GetArchiverVersion(), "0.2.0");

From 0d27e5e8875ae304335fd4461aba681a3b82a388 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:39:46 +0000
Subject: [PATCH 08/19] Enable test before adding subdirectory; use make test
 in build.sh

---
 cpp/CMakeLists.txt |  4 ++--
 cpp/build.sh       | 16 ++++++----------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index af5068df72..639f60c305 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR)
 project(torchserve_cpp VERSION 0.1)
 
+enable_testing()
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_CXX_EXTENSIONS OFF)
@@ -73,5 +75,3 @@ add_subdirectory(test)
 FILE(COPY src/resources/logging.yaml DESTINATION "${CMAKE_INSTALL_PREFIX}/resources")
 
 install(SCRIPT "install.cmake")
-
-enable_testing()
diff --git a/cpp/build.sh b/cpp/build.sh
index eb665ab40d..78c2083c52 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -57,16 +57,12 @@ function build() {
 
   make -j "$JOBS"
   make install
-  echo -e "${COLOR_GREEN}torchserve_cpp build is complete. To run unit test: \
-  ./_build/test/torchserve_cpp_test ${COLOR_OFF}"
-
-  # cd $DEPS_DIR/../..
-  # if [ -f "$DEPS_DIR/../test/torchserve_cpp_test" ]; then
-  #   $DEPS_DIR/../test/torchserve_cpp_test
-  # else
-  #   echo -e "${COLOR_RED}[ ERROR ] _build/test/torchserve_cpp_test not exist ${COLOR_OFF}"
-  #   exit 1
-  # fi
+  echo -e "${COLOR_GREEN}torchserve_cpp build is complete. ${COLOR_OFF}"
+
+
+  if [ -z "$NO_BUILD_TESTS" ]; then
+    make test
+  fi
 }
 
 # Parse args

From 94a24bf81e7246ff9041cec6a2dee888034a6994 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 22:11:59 +0000
Subject: [PATCH 09/19] Remove build.sh

---
 .github/workflows/ci-cpu-cpp.yml |   6 +-
 cpp/CMakeLists.txt               |  16 ++++
 cpp/README.md                    |  11 ++-
 cpp/build.sh                     | 125 -------------------------------
 cpp/src/backends/CMakeLists.txt  |   8 +-
 cpp/test/CMakeLists.txt          |   2 +-
 6 files changed, 33 insertions(+), 135 deletions(-)
 delete mode 100755 cpp/build.sh

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 40d57ce57c..de09ad2899 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -53,8 +53,10 @@ jobs:
           python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
-          ./build.sh
+          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir build && sudo mount --bind /mnt/_build build
+          cd build && cmake ..
+          make -j && make install
+          make test
       - name: Run test
         run: |
           cd test && pytest pytest/test_cpp_backend.py
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 639f60c305..85e8a7b925 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -7,6 +7,7 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wall -Wextra -fPIC")
+set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
 
 find_program(CLANG_TIDY_EXE NAMES "clang-tidy" REQUIRED)
 set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXE}")
@@ -22,9 +23,24 @@ if(CLANG_FORMAT_EXE)
 
 endif()
 
+execute_process(COMMAND python -c "import importlib; print(importlib.util.find_spec('nvidia') is not None)"
+                OUTPUT_VARIABLE NVIDIA_AVAILABLE
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(${NVIDIA_AVAILABLE} STREQUAL "True")
+  execute_process(COMMAND python -c "import nvidia;from pathlib import Path;print(Path(nvidia.__file__).parent/'nccl'/'lib')"
+                  OUTPUT_VARIABLE NCCL_PATH
+                  OUTPUT_STRIP_TRAILING_WHITESPACE)
+  list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.2")
+endif()
+
+execute_process(COMMAND python -c "import torch;print(torch.utils.cmake_prefix_path)"
+                OUTPUT_VARIABLE TORCH_CMAKE_PREFIX_PATH
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+list(APPEND CMAKE_PREFIX_PATH ${TORCH_CMAKE_PREFIX_PATH})
 
 find_package(Boost REQUIRED)
 find_package(Torch REQUIRED)
+find_library(NCCL_LIBRARY nccl HINTS ${NCCL_PATH})
 
 include(FetchContent)
 
diff --git a/cpp/README.md b/cpp/README.md
index c3cfdd6a47..9c722ed2e0 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -17,9 +17,9 @@ cd serve/docker
 Start the container and optionally bind mount a build directory into the container to persist build artifacts across container runs
 ```
 # For CPU support
-docker run [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
+docker run [-v /path/to/build/dir:/serve/cpp/build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
 # For GPU support
-docker run --gpus all [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
+docker run --gpus all [-v /path/to/build/dir:/serve/cpp/build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
 ```
 `Warning`: The dev docker container does not install all necessary dependencies or build Torchserve CPP. Please follow the steps below after starting the container.
 
@@ -41,7 +41,12 @@ Then build the backend:
 ```
 ## Dev Build
 cd cpp
-./build.sh
+mkdir build && cd build
+# Optionally, you can skip building the tests by adding: -DBUILD_TESTS=OFF
+cmake ..
+make -j && make install
+## Optionally, you can run the tests with
+make test
 ```
 
 ### Run TorchServe
diff --git a/cpp/build.sh b/cpp/build.sh
deleted file mode 100755
index 78c2083c52..0000000000
--- a/cpp/build.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/env bash
-
-# Obtain the base directory this script resides in.
-BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-echo "BASE_DIR=${BASE_DIR}"
-
-# Useful constants
-COLOR_RED="\033[0;31m"
-COLOR_GREEN="\033[0;32m"
-COLOR_OFF="\033[0m"
-
-function detect_platform() {
-  unameOut="$(uname -s)"
-  case "${unameOut}" in
-      Linux*)     PLATFORM=Linux;;
-      Darwin*)    PLATFORM=Mac;;
-      Windows*)   PLATFORM=Windows;;
-      *)          PLATFORM="UNKNOWN:${unameOut}"
-  esac
-  echo -e "${COLOR_GREEN}Detected platform: $PLATFORM ${COLOR_OFF}"
-}
-
-function build() {
-  echo -e "${COLOR_GREEN}[ INFO ]Building backend ${COLOR_OFF}"
-
-  MAYBE_BUILD_TESTS="-DBUILD_TESTS=ON"
-  if [ "$NO_BUILD_TESTS" == true ] ; then
-    MAYBE_BUILD_TESTS="-DBUILD_TESTS=OFF"
-  fi
-
-  if [ -z "$PREFIX" ]; then
-    PREFIX=$BWD
-  fi
-
-  # Build torchserve_cpp with cmake
-  cd "$BWD" || exit
-
-  CMAKE_PREFIX_PATH=`python3 -c 'import torch;print(torch.utils.cmake_prefix_path)'`
-
-  if [ "$PLATFORM" = "Linux" ]; then
-    NCCL_PATH=`python3 -c 'import torch;from pathlib import Path;print(Path(torch.__file__).parents[1]/"nvidia"/"nccl"/"lib")'`
-    export LD_LIBRARY_PATH=${NCCL_PATH}:${LD_LIBRARY_PATH}
-  elif [ "$PLATFORM" = "Mac" ]; then
-    export LIBRARY_PATH=${LIBRARY_PATH}:`brew --prefix icu4c`/lib:`brew --prefix libomp`/lib
-  else
-    # TODO: Windows
-    echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
-    exit 1
-  fi
-
-  cmake                                                                                       \
-    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$CMAKE_PREFIX_PATH"                                        \
-    -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
-    "$MAYBE_BUILD_TESTS"                                                                      \
-    "$MAYBE_OVERRIDE_CXX_FLAGS"                                                               \
-    ..
-
-  make -j "$JOBS"
-  make install
-  echo -e "${COLOR_GREEN}torchserve_cpp build is complete. ${COLOR_OFF}"
-
-
-  if [ -z "$NO_BUILD_TESTS" ]; then
-    make test
-  fi
-}
-
-# Parse args
-JOBS=8
-WITH_QUIC=false
-INSTALL_DEPENDENCIES=false
-PREFIX=""
-COMPILER_FLAGS=""
-USAGE="./build.sh [-j num_jobs] [-t|--no-tets] [-p|--prefix] [-x|--compiler-flags]"
-while [ "$1" != "" ]; do
-  case $1 in
-    -j | --jobs ) shift
-                  JOBS=$1
-                  ;;
-    -t | --no-tests )
-                  NO_BUILD_TESTS=true
-      ;;
-    -p | --prefix )
-                  shift
-                  PREFIX=$1
-      ;;
-    -x | --compiler-flags )
-                  shift
-                  COMPILER_FLAGS=$1
-      ;;
-    * )           echo $USAGE
-                  exit 1
-esac
-shift
-done
-
-detect_platform
-
-MAYBE_OVERRIDE_CXX_FLAGS=""
-if [ -n "$COMPILER_FLAGS" ] ; then
-  MAYBE_OVERRIDE_CXX_FLAGS="-DCMAKE_CXX_FLAGS=$COMPILER_FLAGS"
-fi
-
-BUILD_DIR=$BASE_DIR/_build
-if [ ! -d "$BUILD_DIR" ] ; then
-  mkdir -p $BUILD_DIR
-fi
-
-set -e nounset
-trap 'cd $BASE_DIR' EXIT
-cd $BUILD_DIR || exit
-BWD=$(pwd)
-DEPS_DIR=$BWD/_deps
-LIBS_DIR=$BWD/libs
-TR_DIR=$BWD/test/resources/
-mkdir -p "$DEPS_DIR"
-mkdir -p "$LIBS_DIR"
-mkdir -p "$TR_DIR"
-
-# Must execute from the directory containing this script
-cd $BASE_DIR
-
-git submodule update --init --recursive
-
-build
diff --git a/cpp/src/backends/CMakeLists.txt b/cpp/src/backends/CMakeLists.txt
index 1b706e74f7..174a8148f9 100644
--- a/cpp/src/backends/CMakeLists.txt
+++ b/cpp/src/backends/CMakeLists.txt
@@ -26,7 +26,7 @@ list(APPEND TS_BACKENDS_PROTOCOL_SOURCE_FILES ${TS_BACKENDS_PROTOCOL_SRC_DIR}/so
 add_library(ts_backends_protocol SHARED ${TS_BACKENDS_PROTOCOL_SOURCE_FILES})
 target_include_directories(ts_backends_protocol PUBLIC ${TS_BACKENDS_PROTOCOL_SRC_DIR})
 target_link_libraries(ts_backends_protocol PRIVATE ts_utils)
-install(TARGETS ts_backends_protocol DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_backends_protocol DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 # build library ts_backend_core
 set(BACKEND_SOURCE_FILES "")
@@ -37,7 +37,7 @@ list(APPEND BACKEND_SOURCE_FILES ${TS_BACKENDS_SRC_DIR}/handler/torch_scripted_h
 add_library(ts_backends_core SHARED ${BACKEND_SOURCE_FILES})
 target_include_directories(ts_backends_core PUBLIC ${TS_BACKENDS_CORE_SRC_DIR})
 target_link_libraries(ts_backends_core PUBLIC ts_utils ts_backends_protocol ${TORCH_LIBRARIES})
-install(TARGETS ts_backends_core DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_backends_core DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 # build exe model_worker_socket
 add_executable(model_worker_socket
@@ -51,5 +51,5 @@ target_include_directories(model_worker_socket PRIVATE
   ${TS_BACKENDS_TORCH_SCRIPTED_SRC_DIR}
 )
 target_link_libraries(model_worker_socket
-  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags)
-install(TARGETS model_worker_socket DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/bin)
+  PRIVATE ts_backends_core ts_backends_protocol ${TORCH_LIBRARIES} gflags ${NCCL_LIBRARY})
+install(TARGETS model_worker_socket DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt
index e418ccacb2..659c3d6ea2 100644
--- a/cpp/test/CMakeLists.txt
+++ b/cpp/test/CMakeLists.txt
@@ -12,7 +12,7 @@ set(TEST_BINARY ${CMAKE_PROJECT_NAME}_test)
 file(GLOB_RECURSE TEST_SOURCES LIST_DIRECTORIES false *.cc *.hh)
 
 add_executable(${TEST_BINARY} ${TEST_SOURCES})
-target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES})
+target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES} ${NCCL_LIBRARY})
 
 include(GoogleTest)
 gtest_discover_tests(${TEST_BINARY})

From cc2f7c78e8c8d0aa314fd921dbc3cf2389d59194 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 22:35:42 +0000
Subject: [PATCH 10/19] Fix import error in python 3.9

---
 cpp/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 85e8a7b925..f352bd1334 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -23,7 +23,7 @@ if(CLANG_FORMAT_EXE)
 
 endif()
 
-execute_process(COMMAND python -c "import importlib; print(importlib.util.find_spec('nvidia') is not None)"
+execute_process(COMMAND python -c "import importlib.util; print(importlib.util.find_spec('nvidia') is not None)"
                 OUTPUT_VARIABLE NVIDIA_AVAILABLE
                 OUTPUT_STRIP_TRAILING_WHITESPACE)
 if(${NVIDIA_AVAILABLE} STREQUAL "True")

From 203e0a66f009fe763bb6a3d9acdef0ed2de952a7 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 22:59:54 +0000
Subject: [PATCH 11/19] Fix examples + halde nccl not found

---
 cpp/CMakeLists.txt                         |  4 ++++
 cpp/src/utils/CMakeLists.txt               |  2 +-
 examples/cpp/aot_inductor/bert/README.md   | 11 +++--------
 examples/cpp/aot_inductor/llama2/README.md | 15 +++++----------
 examples/cpp/aot_inductor/resnet/README.md | 11 +++--------
 5 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f352bd1334..bcb0e946b2 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -40,7 +40,11 @@ list(APPEND CMAKE_PREFIX_PATH ${TORCH_CMAKE_PREFIX_PATH})
 
 find_package(Boost REQUIRED)
 find_package(Torch REQUIRED)
+
 find_library(NCCL_LIBRARY nccl HINTS ${NCCL_PATH})
+if("${NCCL_LIBRARY}" STREQUAL "NOTFOUND")
+  set(NCCL_LIBRARY "")
+endif()
 
 include(FetchContent)
 
diff --git a/cpp/src/utils/CMakeLists.txt b/cpp/src/utils/CMakeLists.txt
index 1e1c2ea270..5f50d53d1b 100644
--- a/cpp/src/utils/CMakeLists.txt
+++ b/cpp/src/utils/CMakeLists.txt
@@ -47,7 +47,7 @@ else()
   target_link_libraries(ts_utils ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} yaml-cpp nlohmann_json fmt)
 endif()
 
-install(TARGETS ts_utils DESTINATION ${torchserve_cpp_SOURCE_DIR}/_build/libs)
+install(TARGETS ts_utils DESTINATION ${CMAKE_INSTALL_PREFIX}/libs)
 
 list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/ifoo.hh)
 list(APPEND FOO_SOURCE_FILES ${TS_UTILS_SRC_DIR}/foo.hh)
diff --git a/examples/cpp/aot_inductor/bert/README.md b/examples/cpp/aot_inductor/bert/README.md
index 5cb76be3d1..c9243b94c4 100644
--- a/examples/cpp/aot_inductor/bert/README.md
+++ b/examples/cpp/aot_inductor/bert/README.md
@@ -7,13 +7,8 @@ Then, this example loads model and runs prediction using libtorch. The handler C
 ### Setup
 1. Follow the instructions in [README.md](../../../../cpp/README.md) to build the TorchServe C++ backend.
 
-```
-cd serve/cpp
-./builld.sh
-```
-
-The build script will create the necessary artifact for this example.
-To recreate these by hand you can follow the prepare_test_files function of the [build.sh](../../../../cpp/build.sh) script.
+The build process will create the necessary artifact for this example.
+To recreate these by hand you can follow the the [CMakeLists.txt](./CMakeLists.txt) file.
 We will need the handler .so file as well as the bert-seq.so and tokenizer.json.
 
 2. Create a [model-config.yaml](model-config.yaml)
@@ -37,7 +32,7 @@ handler:
 ### Generate Model Artifact Folder
 
 ```bash
-torch-model-archiver --model-name bertcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/libbert_handler:BertCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/bert-seq.so,../../../../cpp/_build/test/resources/examples/aot_inductor/bert_handler/tokenizer.json  --config-file model-config.yaml --archive-format no-archive
+torch-model-archiver --model-name bertcppaot --version 1.0 --handler ../../../../cpp/build/test/resources/examples/aot_inductor/bert_handler/libbert_handler:BertCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/build/test/resources/examples/aot_inductor/bert_handler/bert-seq.so,../../../../cpp/build/test/resources/examples/aot_inductor/bert_handler/tokenizer.json  --config-file model-config.yaml --archive-format no-archive
 ```
 
 Create model store directory and move the folder `bertcppaot`
diff --git a/examples/cpp/aot_inductor/llama2/README.md b/examples/cpp/aot_inductor/llama2/README.md
index e3a27c0439..3c789c8c0e 100644
--- a/examples/cpp/aot_inductor/llama2/README.md
+++ b/examples/cpp/aot_inductor/llama2/README.md
@@ -5,20 +5,15 @@ The handler C++ source code for this examples can be found [here](src/).
 ### Setup
 1. Follow the instructions in [README.md](../../../../cpp/README.md) to build the TorchServe C++ backend.
 
-```
-cd serve/cpp
-./builld.sh
-```
-
-The build script will already create the necessary artifact for this example.
-To recreate these by hand you can follow the prepare_test_files function of the [build.sh](../../../../cpp/build.sh) script.
+The build process will create the necessary artifact for this example.
+To recreate these by hand you can follow the the [CMakeLists.txt](./CMakeLists.txt) file.
 We will need the handler .so file as well as the stories15M.so file containing the model and weights.
 
 2. Copy the handler file
 
 ```bash
 cd ~/serve/examples/cpp/aot_inductor/llama2
-cp ../../../../cpp/_build/test/resources/examples/aot_inductor/llama_handler/libllama_so_handler.so ./
+cp ../../../../cpp/build/test/resources/examples/aot_inductor/llama_handler/libllama_so_handler.so ./
 ```
 We will leave the model .so file in place and just use its absolute path in the next step.
 
@@ -26,8 +21,8 @@ We will leave the model .so file in place and just use its absolute path in the
 
 ```bash
 echo '{
-"checkpoint_path" : "/home/ubuntu/serve/cpp/_build/test/resources/examples/aot_inductor/llama_handler/stories15M.so",
-"tokenizer_path" : "/home/ubuntu/serve/cpp/_build/test/resources/examples/babyllama/babyllama_handler/tokenizer.bin"
+"checkpoint_path" : "/home/ubuntu/serve/cpp/build/test/resources/examples/aot_inductor/llama_handler/stories15M.so",
+"tokenizer_path" : "/home/ubuntu/serve/cpp/build/test/resources/examples/babyllama/babyllama_handler/tokenizer.bin"
 }' > config.json
 ```
 
diff --git a/examples/cpp/aot_inductor/resnet/README.md b/examples/cpp/aot_inductor/resnet/README.md
index 0a601f67f9..ba507da7e6 100644
--- a/examples/cpp/aot_inductor/resnet/README.md
+++ b/examples/cpp/aot_inductor/resnet/README.md
@@ -6,13 +6,8 @@ The handler C++ source code for this examples can be found [here](src).
 ### Setup
 1. Follow the instructions in [README.md](../../../../cpp/README.md) to build the TorchServe C++ backend.
 
-```
-cd serve/cpp
-./builld.sh
-```
-
-The build script will create the necessary artifact for this example.
-To recreate these by hand you can follow the prepare_test_files function of the [build.sh](../../../../cpp/build.sh) script.
+The build process will create the necessary artifact for this example.
+To recreate these by hand you can follow the the [CMakeLists.txt](./CMakeLists.txt) file.
 We will need the handler .so file as well as the resnet50_pt2.so file containing the model and weights.
 
 2. Create a [model-config.yaml](model-config.yaml)
@@ -30,7 +25,7 @@ handler:
 ### Generate Model Artifacts Folder
 
 ```bash
-torch-model-archiver --model-name resnetcppaot --version 1.0 --handler ../../../../cpp/_build/test/resources/examples/aot_inductor/resnet_handler/libresnet_handler:ResnetCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/_build/test/resources/examples/aot_inductor/resnet_handler/resnet50_pt2.so --config-file model-config.yaml --archive-format no-archive
+torch-model-archiver --model-name resnetcppaot --version 1.0 --handler ../../../../cpp/build/test/resources/examples/aot_inductor/resnet_handler/libresnet_handler:ResnetCppHandler --runtime LSP --extra-files index_to_name.json,../../../../cpp/build/test/resources/examples/aot_inductor/resnet_handler/resnet50_pt2.so --config-file model-config.yaml --archive-format no-archive
 ```
 
 Create model store directory and move the folder `resnetcppaot`

From 04b7ca32b679394f574d1d9bb3c068c51b560993 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 23:22:40 +0000
Subject: [PATCH 12/19] Fix lib not found

---
 cpp/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index bcb0e946b2..b6233906c6 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -42,7 +42,7 @@ find_package(Boost REQUIRED)
 find_package(Torch REQUIRED)
 
 find_library(NCCL_LIBRARY nccl HINTS ${NCCL_PATH})
-if("${NCCL_LIBRARY}" STREQUAL "NOTFOUND")
+if("${NCCL_LIBRARY}" STREQUAL "NCCL_LIBRARY-NOTFOUND")
   set(NCCL_LIBRARY "")
 endif()
 

From 769dc07a14293895f19b89af1ba596c43d328e42 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Wed, 13 Mar 2024 23:50:11 +0000
Subject: [PATCH 13/19] convert tokenizer-cpp from submodule to fetch_content

---
 .gitmodules                                   |  3 ---
 cpp/third-party/tokenizers-cpp                |  1 -
 examples/cpp/aot_inductor/bert/CMakeLists.txt | 14 +++++++++++---
 3 files changed, 11 insertions(+), 7 deletions(-)
 delete mode 160000 cpp/third-party/tokenizers-cpp

diff --git a/.gitmodules b/.gitmodules
index be2c2a8611..08073f4207 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,9 +10,6 @@
 [submodule "cpp/third-party/yaml-cpp"]
 	path = cpp/third-party/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git
-[submodule "cpp/third-party/tokenizers-cpp"]
-	path = cpp/third-party/tokenizers-cpp
-	url = https://github.com/mlc-ai/tokenizers-cpp.git
 [submodule "cpp/third-party/kineto"]
 	path = cpp/third-party/kineto
 	url = https://github.com/pytorch/kineto.git
diff --git a/cpp/third-party/tokenizers-cpp b/cpp/third-party/tokenizers-cpp
deleted file mode 160000
index 27dbe17d72..0000000000
--- a/cpp/third-party/tokenizers-cpp
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 27dbe17d7268801ec720569167af905c88d3db50
diff --git a/examples/cpp/aot_inductor/bert/CMakeLists.txt b/examples/cpp/aot_inductor/bert/CMakeLists.txt
index cb5dc43549..df4c341628 100644
--- a/examples/cpp/aot_inductor/bert/CMakeLists.txt
+++ b/examples/cpp/aot_inductor/bert/CMakeLists.txt
@@ -1,4 +1,12 @@
 
+FetchContent_Declare(
+  tokenizer_cpp
+  GIT_REPOSITORY https://github.com/mlc-ai/tokenizers-cpp.git
+  GIT_TAG 27dbe17d7268801ec720569167af905c88d3db50
+)
+
+FetchContent_MakeAvailable(tokenizer_cpp)
+
 add_custom_command(
     OUTPUT bert-seq.so
     COMMAND TOKENIZERS_PARALLELISM=false python ${CMAKE_CURRENT_SOURCE_DIR}/aot_compile_export.py
@@ -6,8 +14,8 @@ add_custom_command(
     DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/aot_compile_export.py
 )
 
-set(TOKENZIER_CPP_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../../../cpp/third-party/tokenizers-cpp)
-add_subdirectory(${TOKENZIER_CPP_PATH} tokenizers EXCLUDE_FROM_ALL)
+# set(TOKENZIER_CPP_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../../../cpp/third-party/tokenizers-cpp)
+# add_subdirectory(${TOKENZIER_CPP_PATH} tokenizers EXCLUDE_FROM_ALL)
 add_library(bert_handler SHARED src/bert_handler.cc bert-seq.so)
-target_include_directories(bert_handler PRIVATE ${TOKENZIER_CPP_PATH}/include)
+target_include_directories(bert_handler PRIVATE)
 target_link_libraries(bert_handler PRIVATE ts_backends_core ts_utils ${TORCH_LIBRARIES} tokenizers_cpp)

From fc26591faeaac2f5131cb002e4c028f4ca510c53 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 14 Mar 2024 00:01:40 +0000
Subject: [PATCH 14/19] Change skip condition for cpp pytest

---
 test/pytest/test_cpp_backend.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/pytest/test_cpp_backend.py b/test/pytest/test_cpp_backend.py
index 917a2e21f4..fc461a5d60 100644
--- a/test/pytest/test_cpp_backend.py
+++ b/test/pytest/test_cpp_backend.py
@@ -13,6 +13,8 @@
 from PIL import Image
 from torchvision import transforms
 
+import ts
+
 CURR_FILE_PATH = Path(__file__).parent
 
 TORCH_NCCL_PATH = (Path(torch.__file__).parent / "lib").as_posix()
@@ -40,7 +42,7 @@ def create_mar_file(work_dir, model_archiver, model_name):
 
     mnist_scriptes_pt = (
         CURR_FILE_PATH.parents[1]
-        / "cpp/_build/test/resources/examples/mnist/mnist_handler/mnist_script.pt"
+        / "cpp/build/test/resources/examples/mnist/mnist_handler/mnist_script.pt"
     )
 
     config = ModelArchiverConfig(
@@ -107,9 +109,7 @@ def register_model(mar_file_path, model_store, torchserve):
 
 
 @pytest.mark.skipif(
-    not (
-        CURR_FILE_PATH.parents[1] / "ts" / "cpp" / "bin" / "model_worker_socket"
-    ).exists(),
+    not (Path(ts.__file__).parent / "cpp" / "bin" / "model_worker_socket").exists(),
     reason="CPP backend not found",
 )
 def test_cpp_mnist(model_name_and_stdout):

From 452e40eacc0de9c6c08c9233e615299884d41c4c Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 14 Mar 2024 00:04:43 +0000
Subject: [PATCH 15/19] Remove kineto submodule

---
 .gitmodules            | 3 ---
 cpp/third-party/kineto | 1 -
 2 files changed, 4 deletions(-)
 delete mode 160000 cpp/third-party/kineto

diff --git a/.gitmodules b/.gitmodules
index 08073f4207..43f86eb6a9 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,6 +10,3 @@
 [submodule "cpp/third-party/yaml-cpp"]
 	path = cpp/third-party/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git
-[submodule "cpp/third-party/kineto"]
-	path = cpp/third-party/kineto
-	url = https://github.com/pytorch/kineto.git
diff --git a/cpp/third-party/kineto b/cpp/third-party/kineto
deleted file mode 160000
index 594c63c50d..0000000000
--- a/cpp/third-party/kineto
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 594c63c50dd9684a592ad7670ecdef6dd5e36be7

From 1e2064a622fba85010eeada55686e45966e33014 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 14 Mar 2024 00:05:41 +0000
Subject: [PATCH 16/19] Remove yaml-cpp submodule

---
 .gitmodules              | 3 ---
 cpp/third-party/yaml-cpp | 1 -
 2 files changed, 4 deletions(-)
 delete mode 160000 cpp/third-party/yaml-cpp

diff --git a/.gitmodules b/.gitmodules
index 43f86eb6a9..5099aedbd3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -7,6 +7,3 @@
 [submodule "cpp/third-party/llama2.so"]
 	path = cpp/third-party/llama2.so
 	url = https://github.com/mreso/llama2.so.git
-[submodule "cpp/third-party/yaml-cpp"]
-	path = cpp/third-party/yaml-cpp
-	url = https://github.com/jbeder/yaml-cpp.git
diff --git a/cpp/third-party/yaml-cpp b/cpp/third-party/yaml-cpp
deleted file mode 160000
index f732014112..0000000000
--- a/cpp/third-party/yaml-cpp
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f7320141120f720aecc4c32be25586e7da9eb978

From 749d758c20656565c9d5d0874a1dedc27249ab1a Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 14 Mar 2024 00:33:18 +0000
Subject: [PATCH 17/19] Convert llama2.so to fetch_content

---
 .gitmodules                                     |  3 ---
 cpp/third-party/llama2.so                       |  1 -
 examples/cpp/aot_inductor/llama2/CMakeLists.txt | 13 ++++++++++++-
 3 files changed, 12 insertions(+), 5 deletions(-)
 delete mode 160000 cpp/third-party/llama2.so

diff --git a/.gitmodules b/.gitmodules
index 5099aedbd3..45b3371907 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,6 +4,3 @@
 [submodule "cpp/third-party/llama2.c"]
 	path = cpp/third-party/llama2.c
 	url = https://github.com/karpathy/llama2.c
-[submodule "cpp/third-party/llama2.so"]
-	path = cpp/third-party/llama2.so
-	url = https://github.com/mreso/llama2.so.git
diff --git a/cpp/third-party/llama2.so b/cpp/third-party/llama2.so
deleted file mode 160000
index 370154d8c7..0000000000
--- a/cpp/third-party/llama2.so
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 370154d8c7311e55fb91ebf4f9cd44344d92296e
diff --git a/examples/cpp/aot_inductor/llama2/CMakeLists.txt b/examples/cpp/aot_inductor/llama2/CMakeLists.txt
index 7dbcc71c43..b2484abe4d 100644
--- a/examples/cpp/aot_inductor/llama2/CMakeLists.txt
+++ b/examples/cpp/aot_inductor/llama2/CMakeLists.txt
@@ -1,4 +1,15 @@
 
+FetchContent_Declare(
+  llama2_so
+  GIT_REPOSITORY https://github.com/mreso/llama2.so.git
+  GIT_TAG 370154d
+)
+
+if(NOT llama2_so_POPULATED)
+  message(STATUS "Fetching llama2.so...")
+  FetchContent_Populate(llama2_so)
+endif()
+
 FetchContent_Declare(
   stories15M_pt
   URL      https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true
@@ -16,7 +27,7 @@ add_custom_command(
 )
 
 
-add_library(llama2_so STATIC ../../../../cpp/third-party/llama2.so/run.cpp)
+add_library(llama2_so STATIC ${llama2_so_SOURCE_DIR}/run.cpp)
 target_compile_options(llama2_so PRIVATE -Wall -Wextra -Ofast -fpermissive)
 
 add_library(llama_so_handler SHARED src/llama_handler.cc stories15M.so)

From 9f2cbe31a9f241375b9117ace64282b4b23b6bf5 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 14 Mar 2024 05:33:27 +0000
Subject: [PATCH 18/19] Convert llama2.c submodule into fetch_content

---
 .gitmodules                                     |  3 ---
 cpp/third-party/llama2.c                        |  1 -
 examples/cpp/aot_inductor/bert/CMakeLists.txt   |  2 --
 examples/cpp/aot_inductor/llama2/CMakeLists.txt |  2 +-
 examples/cpp/babyllama/CMakeLists.txt           | 11 ++++++++---
 5 files changed, 9 insertions(+), 10 deletions(-)
 delete mode 160000 cpp/third-party/llama2.c

diff --git a/.gitmodules b/.gitmodules
index 45b3371907..e60c11fbfb 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,3 @@
 [submodule "third_party/google/rpc"]
 	path = third_party/google/rpc
 	url = https://github.com/googleapis/googleapis.git
-[submodule "cpp/third-party/llama2.c"]
-	path = cpp/third-party/llama2.c
-	url = https://github.com/karpathy/llama2.c
diff --git a/cpp/third-party/llama2.c b/cpp/third-party/llama2.c
deleted file mode 160000
index d9862069e7..0000000000
--- a/cpp/third-party/llama2.c
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d9862069e7ef665fe6309e3c17398ded2f121bf5
diff --git a/examples/cpp/aot_inductor/bert/CMakeLists.txt b/examples/cpp/aot_inductor/bert/CMakeLists.txt
index df4c341628..8c8ba59f93 100644
--- a/examples/cpp/aot_inductor/bert/CMakeLists.txt
+++ b/examples/cpp/aot_inductor/bert/CMakeLists.txt
@@ -14,8 +14,6 @@ add_custom_command(
     DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/aot_compile_export.py
 )
 
-# set(TOKENZIER_CPP_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../../../cpp/third-party/tokenizers-cpp)
-# add_subdirectory(${TOKENZIER_CPP_PATH} tokenizers EXCLUDE_FROM_ALL)
 add_library(bert_handler SHARED src/bert_handler.cc bert-seq.so)
 target_include_directories(bert_handler PRIVATE)
 target_link_libraries(bert_handler PRIVATE ts_backends_core ts_utils ${TORCH_LIBRARIES} tokenizers_cpp)
diff --git a/examples/cpp/aot_inductor/llama2/CMakeLists.txt b/examples/cpp/aot_inductor/llama2/CMakeLists.txt
index b2484abe4d..94629360d1 100644
--- a/examples/cpp/aot_inductor/llama2/CMakeLists.txt
+++ b/examples/cpp/aot_inductor/llama2/CMakeLists.txt
@@ -22,7 +22,7 @@ FetchContent_MakeAvailable(stories15M_pt)
 
 add_custom_command(
     OUTPUT stories15M.so
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../../../../cpp/third-party/llama2.so/ python ${CMAKE_CURRENT_SOURCE_DIR}/compile.py --checkpoint ${CMAKE_CURRENT_BINARY_DIR}/\'stories15M.pt?download=true\' ${CMAKE_CURRENT_BINARY_DIR}/stories15M.so
+    COMMAND PYTHONPATH=${llama2_so_SOURCE_DIR} python ${CMAKE_CURRENT_SOURCE_DIR}/compile.py --checkpoint ${CMAKE_CURRENT_BINARY_DIR}/\'stories15M.pt?download=true\' ${CMAKE_CURRENT_BINARY_DIR}/stories15M.so
     DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile.py
 )
 
diff --git a/examples/cpp/babyllama/CMakeLists.txt b/examples/cpp/babyllama/CMakeLists.txt
index 933285d202..872b708ac4 100644
--- a/examples/cpp/babyllama/CMakeLists.txt
+++ b/examples/cpp/babyllama/CMakeLists.txt
@@ -1,4 +1,9 @@
-include(FetchContent)
+
+FetchContent_Declare(
+  llama2_c
+  GIT_REPOSITORY https://github.com/karpathy/llama2.c
+  GIT_TAG d986206
+)
 
 FetchContent_Declare(
     stories15M_bin
@@ -14,9 +19,9 @@ FetchContent_Declare(
   DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR}/
 )
 
-FetchContent_MakeAvailable(tokenizer_bin stories15M_bin)
+FetchContent_MakeAvailable(tokenizer_bin stories15M_bin llama2_c)
 
-add_library(llama2_c STATIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../cpp/third-party/llama2.c/run.c)
+add_library(llama2_c STATIC ${llama2_c_SOURCE_DIR}/run.c)
 target_compile_options(llama2_c PRIVATE -Wall -Wextra -Ofast -fPIC)
 
 add_library(babyllama_handler SHARED src/baby_llama_handler.cc)

From 7a4758bb977c78f4ff9edfe534b4ec9b5a5f8224 Mon Sep 17 00:00:00 2001
From: lxning <lninga@amazon.com>
Date: Wed, 20 Mar 2024 14:53:49 -0700
Subject: [PATCH 19/19] fix relateive path

---
 cpp/src/examples/CMakeLists.txt                    |  3 ---
 cpp/test/CMakeLists.txt                            |  1 +
 cpp/test/backends/otf_protocol_and_handler_test.cc | 14 +++++++-------
 cpp/test/utils/common.hh                           |  2 +-
 4 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/cpp/src/examples/CMakeLists.txt b/cpp/src/examples/CMakeLists.txt
index 6b211a8302..c18a36168e 100644
--- a/cpp/src/examples/CMakeLists.txt
+++ b/cpp/src/examples/CMakeLists.txt
@@ -1,6 +1,3 @@
-
-file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/../../test/resources/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/)
-
 add_subdirectory("../../../examples/cpp/babyllama/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/babyllama/babyllama_handler/")
 
 add_subdirectory("../../../examples/cpp/llamacpp/" "${CMAKE_CURRENT_BINARY_DIR}/../../test/resources/examples/llamacpp/llamacpp_handler/")
diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt
index 659c3d6ea2..e254c4e52d 100644
--- a/cpp/test/CMakeLists.txt
+++ b/cpp/test/CMakeLists.txt
@@ -10,6 +10,7 @@ FetchContent_MakeAvailable(googletest)
 
 set(TEST_BINARY ${CMAKE_PROJECT_NAME}_test)
 file(GLOB_RECURSE TEST_SOURCES LIST_DIRECTORIES false *.cc *.hh)
+file(COPY resources/ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/resources/)
 
 add_executable(${TEST_BINARY} ${TEST_SOURCES})
 target_link_libraries(${TEST_BINARY} gtest_main gmock_main ts_backends_core ts_backends_protocol ts_utils ${TORCH_LIBRARIES} ${NCCL_LIBRARY})
diff --git a/cpp/test/backends/otf_protocol_and_handler_test.cc b/cpp/test/backends/otf_protocol_and_handler_test.cc
index fb6823f1c9..7c09ceb9fb 100644
--- a/cpp/test/backends/otf_protocol_and_handler_test.cc
+++ b/cpp/test/backends/otf_protocol_and_handler_test.cc
@@ -24,7 +24,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
       // model_name length
       .WillOnce(::testing::Return(5))
       // model_path length
-      .WillOnce(::testing::Return(45))
+      .WillOnce(::testing::Return(37))
       // batch_size
       .WillOnce(::testing::Return(1))
       // handler length
@@ -44,8 +44,8 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
         strncpy(data, "mnist", length);
       }))
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
-        ASSERT_EQ(length, 45);
-        strncpy(data, "../test/resources/examples/mnist/base_handler", length);
+        ASSERT_EQ(length, 37);
+        strncpy(data, "resources/examples/mnist/base_handler", length);
       }))
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
         ASSERT_EQ(length, 11);
@@ -59,7 +59,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
   EXPECT_CALL(*client_socket, SendAll(testing::_, testing::_)).Times(1);
   auto load_model_request = OTFMessage::RetrieveLoadMsg(*client_socket);
   ASSERT_EQ(load_model_request->model_dir,
-            "../test/resources/examples/mnist/base_handler");
+            "resources/examples/mnist/base_handler");
   ASSERT_EQ(load_model_request->model_name, "mnist");
   ASSERT_EQ(load_model_request->envelope, "");
   ASSERT_EQ(load_model_request->model_name, "mnist");
@@ -68,9 +68,9 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
 
   // initialize backend
   auto backend = std::make_shared<torchserve::Backend>();
-  MetricsRegistry::Initialize("../test/resources/metrics/default_config.yaml",
+  MetricsRegistry::Initialize("resources/metrics/default_config.yaml",
                               MetricsContext::BACKEND);
-  backend->Initialize("../test/resources/examples/mnist/base_handler");
+  backend->Initialize("resources/examples/mnist/base_handler");
 
   // load the model
   auto load_model_response = backend->LoadModel(load_model_request);
@@ -125,7 +125,7 @@ TEST(BackendIntegTest, TestOTFProtocolAndHandler) {
       .WillOnce(testing::Invoke([=](size_t length, char* data) {
         ASSERT_EQ(length, 3883);
         // strncpy(data, "valu", length);
-        std::ifstream input("../test/resources/examples/mnist/0_png.pt",
+        std::ifstream input("resources/examples/mnist/0_png.pt",
                             std::ios::in | std::ios::binary);
         std::vector<char> image((std::istreambuf_iterator<char>(input)),
                                 (std::istreambuf_iterator<char>()));
diff --git a/cpp/test/utils/common.hh b/cpp/test/utils/common.hh
index 254cef76c7..6a71069f88 100644
--- a/cpp/test/utils/common.hh
+++ b/cpp/test/utils/common.hh
@@ -15,7 +15,7 @@ class ModelPredictTest : public ::testing::Test {
       const std::string& inference_request_id_prefix,
       int inference_expect_code) {
     torchserve::MetricsRegistry::Initialize(
-        "../test/resources/metrics/default_config.yaml",
+        "resources/metrics/default_config.yaml",
         torchserve::MetricsContext::BACKEND);
     backend_->Initialize(model_dir);
     auto result = backend_->LoadModel(std::move(load_model_request));