Merge branch 'master' into m1

Author: agunapal

.gitmodules

@@ -7,9 +7,6 @@
 [submodule "cpp/third-party/llama2.so"]
 	path = cpp/third-party/llama2.so
 	url = https://github.com/mreso/llama2.so.git
-[submodule "cpp/third-party/folly"]
-	path = cpp/third-party/folly
-	url = https://github.com/facebook/folly.git
 [submodule "cpp/third-party/yaml-cpp"]
 	path = cpp/third-party/yaml-cpp
 	url = https://github.com/jbeder/yaml-cpp.git

cpp/README.md

@@ -4,6 +4,24 @@
 * GCC version: gcc-9
 * cmake version: 3.18+
 * Linux
+
+For convenience, a docker container can be used as the development environment to build and install Torchserve CPP
+```
+cd serve/docker
+# For CPU support
+./build_image.sh -bt dev -cpp
+# For GPU support
+./build_image.sh -bt dev -g [-cv cu121|cu118] -cpp
+```
+
+Start the container and optionally bind mount a build directory into the container to persist build artifacts across container runs
+```
+# For CPU support
+docker run [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-cpu /bin/bash
+# For GPU support
+docker run --gpus all [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
+```
+
 ## Installation and Running TorchServe CPP
 This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.
 
@@ -22,7 +40,7 @@ Then build the backend:
 ```
 ## Dev Build
 cd cpp
-./build.sh [-g cu121|cu118]
+./build.sh
 ```
 
 ### Run TorchServe

cpp/third-party/folly

@@ -0,0 +1,91 @@
+# syntax = docker/dockerfile:experimental
+#
+# This file can build images for CPU & GPU with CPP backend support.
+#
+# Following comments have been shamelessly copied from https://github.com/pytorch/pytorch/blob/master/Dockerfile
+#
+# NOTE: To build this you will need a docker version > 18.06 with
+#       experimental enabled and DOCKER_BUILDKIT=1
+#
+#       If you do not use buildkit you are not going to have a good time
+#
+#       For reference:
+#           https://docs.docker.com/develop/develop-images/build_enhancements/
+
+
+ARG BASE_IMAGE=ubuntu:20.04
+ARG PYTHON_VERSION=3.9
+ARG CMAKE_VERSION=3.26.4
+ARG BRANCH_NAME="master"
+ARG USE_CUDA_VERSION=""
+
+FROM ${BASE_IMAGE} AS cpp-dev-image
+ARG BASE_IMAGE
+ARG PYTHON_VERSION
+ARG CMAKE_VERSION
+ARG BRANCH_NAME
+ARG USE_CUDA_VERSION
+ENV PYTHONUNBUFFERED TRUE
+
+RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
+    apt-get update && \
+    apt-get install software-properties-common -y && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+        sudo \
+        vim \
+        git \
+        curl \
+        wget \
+        rsync \
+        gpg \
+        ca-certificates \
+        lsb-release \
+        openjdk-17-jdk \
+        python$PYTHON_VERSION \
+        python$PYTHON_VERSION-dev \
+        python$PYTHON_VERSION-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a virtual environment and "activate" it by adding it first to the path.
+RUN python$PYTHON_VERSION -m venv /home/venv
+ENV PATH="/home/venv/bin:$PATH"
+
+# Enable installation of recent cmake release
+# Ref: https://apt.kitware.com/
+RUN (wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null) \
+    && (echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null) \
+    && apt-get update \
+    && (test -f /usr/share/doc/kitware-archive-keyring/copyright || sudo rm /usr/share/keyrings/kitware-archive-keyring.gpg) \
+    && sudo apt-get install kitware-archive-keyring \
+    && rm -rf /var/lib/apt/lists/*
+
+# Pin cmake and cmake-data version
+# Ref: https://manpages.ubuntu.com/manpages/xenial/man5/apt_preferences.5.html
+RUN echo "Package: cmake\nPin: version $CMAKE_VERSION*\nPin-Priority: 1001" > /etc/apt/preferences.d/cmake
+RUN echo "Package: cmake-data\nPin: version $CMAKE_VERSION*\nPin-Priority: 1001" > /etc/apt/preferences.d/cmake-data
+
+# Install CUDA toolkit to enable "libtorch" build with GPU support
+RUN apt-get update && \
+    if echo "$BASE_IMAGE" | grep -q "cuda:"; then \
+        if [ "$USE_CUDA_VERSION" = "cu121" ]; then \
+            apt-get -y install cuda-toolkit-12-1; \
+        elif [ "$USE_CUDA_VERSION" = "cu118" ]; then \
+            apt-get -y install cuda-toolkit-11-8; \
+        else \
+            echo "Cuda version not supported by CPP backend: $USE_CUDA_VERSION"; \
+            exit 1; \
+        fi; \
+    fi \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git clone --recursive https://github.com/pytorch/serve.git \
+    && cd serve \
+    && git checkout ${BRANCH_NAME}
+
+WORKDIR "serve"
+
+# CPP backend binary install depends on "ts" directory being present in python site-packages
+RUN pip install pygit2 && python ts_scripts/install_from_src.py
+
+EXPOSE 8080 8081 8082 7070 7071

docker/Dockerfile.cpp

@@ -12,6 +12,7 @@ USE_CUSTOM_TAG=false
 CUDA_VERSION=""
 USE_LOCAL_SERVE_FOLDER=false
 BUILD_WITH_IPEX=false
+BUILD_CPP=false
 BUILD_NIGHTLY=false
 PYTHON_VERSION=3.9
 
@@ -29,6 +30,7 @@ do
           echo "-t, --tag specify tag name for docker image"
           echo "-lf, --use-local-serve-folder specify this option for the benchmark image if the current 'serve' folder should be used during automated benchmarks"
           echo "-ipex, --build-with-ipex specify to build with intel_extension_for_pytorch"
+          echo "-cpp, --build-cpp specify to build TorchServe CPP"
           echo "-py, --pythonversion specify to python version to use: Possible values: 3.8 3.9 3.10"
           echo "-n, --nightly specify to build with TorchServe nightly"
           exit 0
@@ -76,6 +78,10 @@ do
           BUILD_WITH_IPEX=true
           shift
           ;;
+        -cpp|--build-cpp)
+          BUILD_CPP=true
+          shift
+          ;;
         -n|--nightly)
           BUILD_NIGHTLY=true
           shift
@@ -139,7 +145,12 @@ fi
 
 if [ "${BUILD_TYPE}" == "dev" ] && ! $USE_CUSTOM_TAG ;
 then
-  DOCKER_TAG="pytorch/torchserve:dev-$MACHINE"
+  if [ "${BUILD_CPP}" == "true" ]
+  then
+    DOCKER_TAG="pytorch/torchserve:cpp-dev-$MACHINE"
+  else
+    DOCKER_TAG="pytorch/torchserve:dev-$MACHINE"
+  fi
 fi
 
 if [ "$USE_CUSTOM_TAG" = true ]
@@ -153,12 +164,35 @@ then
   exit 1
 fi
 
+if [ "$BUILD_CPP" == "true" ];
+then
+  if [ "$BUILD_TYPE" != "dev" ];
+  then
+    echo "Only dev container build is supported for CPP"
+    exit 1
+  fi
+
+  if [[ "${MACHINE}" == "gpu" || "${CUDA_VERSION}" != "" ]];
+  then
+    if [[ "${CUDA_VERSION}" != "cu121" && "${CUDA_VERSION}" != "cu118" ]];
+    then
+      echo "Only cuda versions 12.1 and 11.8 are supported for CPP"
+      exit 1
+    fi
+  fi
+fi
+
 if [ "${BUILD_TYPE}" == "production" ]
 then
   DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" -t "${DOCKER_TAG}" --target production-image  .
 elif [ "${BUILD_TYPE}" == "ci" ]
 then
   DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}"  -t "${DOCKER_TAG}" --target ci-image  .
 else
-  DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg BUILD_WITH_IPEX="${BUILD_WITH_IPEX}"  -t "${DOCKER_TAG}" --target dev-image  .
+  if [ "${BUILD_CPP}" == "true" ]
+  then
+    DOCKER_BUILDKIT=1 docker build --file Dockerfile.cpp --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BRANCH_NAME="${BRANCH_NAME}" -t "${DOCKER_TAG}" --target cpp-dev-image .
+  else
+    DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg USE_CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg BUILD_WITH_IPEX="${BUILD_WITH_IPEX}"  -t "${DOCKER_TAG}" --target dev-image  .
+  fi
 fi

docker/build_image.sh

@@ -0,0 +1,27 @@
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).parents[3]
+
+
+MAR_CONFIG = REPO_ROOT.joinpath("ts_scripts", "mar_config.json")
+
+
+@pytest.fixture(name="gen_models", scope="module")
+def load_gen_models() -> dict:
+    with open(MAR_CONFIG) as f:
+        models = json.load(f)
+    models = {m["model_name"]: m for m in models}
+    return models
+
+
+@pytest.fixture(scope="module")
+def ts_scripts_path():
+    sys.path.append(REPO_ROOT.as_posix())
+
+    yield
+
+    sys.path.pop()

test/pytest/sanity/conftest.py

@@ -0,0 +1,54 @@
+import json
+from pathlib import Path
+
+import pytest
+import test_utils
+
+REPO_ROOT = Path(__file__).parents[3]
+SANITY_MODELS_CONFIG = REPO_ROOT.joinpath("ts_scripts", "configs", "sanity_models.json")
+
+
+def load_resnet18() -> dict:
+    with open(SANITY_MODELS_CONFIG) as f:
+        models = json.load(f)
+    return list(filter(lambda x: x["name"] == "resnet-18", models))[0]
+
+
+@pytest.fixture(name="resnet18")
+def generate_resnet18(model_store, gen_models, ts_scripts_path):
+    model = load_resnet18()
+
+    from ts_scripts.marsgen import generate_model
+
+    generate_model(gen_models[model["name"]], model_store)
+
+    yield model
+
+
+@pytest.fixture(scope="module")
+def torchserve_with_snapshot(model_store):
+    test_utils.torchserve_cleanup()
+
+    test_utils.start_torchserve(
+        model_store=model_store, no_config_snapshots=False, gen_mar=False
+    )
+
+    yield
+
+    test_utils.torchserve_cleanup()
+
+
+def test_config_snapshotting(
+    resnet18, model_store, torchserve_with_snapshot, ts_scripts_path
+):
+    from ts_scripts.sanity_utils import run_rest_test
+
+    run_rest_test(resnet18, unregister_model=False)
+
+    test_utils.stop_torchserve()
+
+    test_utils.start_torchserve(
+        model_store=model_store, no_config_snapshots=False, gen_mar=False
+    )
+
+    run_rest_test(resnet18, register_model=False)

test/pytest/sanity/test_config_snapshot.py

@@ -0,0 +1,55 @@
+import json
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).parents[3]
+SANITY_MODELS_CONFIG = REPO_ROOT.joinpath("ts_scripts", "configs", "sanity_models.json")
+
+
+@pytest.fixture(scope="module")
+def grpc_client_stubs(ts_scripts_path):
+    from ts_scripts.shell_utils import rm_file
+    from ts_scripts.tsutils import generate_grpc_client_stubs
+
+    generate_grpc_client_stubs()
+
+    yield
+
+    rm_file(REPO_ROOT.joinpath("ts_scripts", "*_pb2*.py").as_posix(), True)
+
+
+def load_models() -> dict:
+    with open(SANITY_MODELS_CONFIG) as f:
+        models = json.load(f)
+    return models
+
+
+@pytest.fixture(name="model", params=load_models(), scope="module")
+def models_to_validate(request, model_store, gen_models, ts_scripts_path):
+    model = request.param
+
+    if model["name"] in gen_models:
+        from ts_scripts.marsgen import generate_model
+
+        generate_model(gen_models[model["name"]], model_store)
+
+    yield model
+
+
+def test_models_with_grpc(model, torchserve, ts_scripts_path, grpc_client_stubs):
+    from ts_scripts.sanity_utils import run_grpc_test
+
+    run_grpc_test(model)
+
+
+def test_models_with_rest(model, torchserve, ts_scripts_path):
+    from ts_scripts.sanity_utils import run_rest_test
+
+    run_rest_test(model)
+
+
+def test_gpu_setup(ts_scripts_path):
+    from ts_scripts.sanity_utils import test_gpu_setup
+
+    test_gpu_setup()