Merge branch 'master' into feature/k8s_nightly_test

Author: msaroufim

.github/workflows/regression_tests_cpu_binaries.yml

@@ -3,7 +3,7 @@ name: Run Regression Tests for CPU nightly binaries
 on:
   # run every day at 6:15am
   schedule:
-    - cron:  '15 6 * * *'
+    - cron: '15 6 * * *'
 
 concurrency:
   group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }}
@@ -31,24 +31,39 @@ jobs:
         with:
           submodules: recursive
       - name: Setup conda with Python ${{ matrix.python-version }}
+        if: matrix.os == 'macos-14'
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          channels: anaconda, conda-forge
+          python-version: ${{ matrix.python-version }}
+      - name: Setup conda with Python ${{ matrix.python-version }}
+        if: matrix.os != 'macos-14'
         uses: s-weigand/setup-conda@v1
         with:
           update-conda: true
           python-version: ${{ matrix.python-version }}
           conda-channels: anaconda, conda-forge
-      - run: conda --version
-      - run: python --version
       - name: Setup Java 17
         uses: actions/setup-java@v3
         with:
           distribution: 'zulu'
           java-version: '17'
       - name: Checkout TorchServe
         uses: actions/checkout@v3
+      - name: Run install dependencies and regression test
+        if: matrix.os == 'macos-14'
+        shell: bash -el {0}
+        run: |
+          conda info 
+          python ts_scripts/install_dependencies.py --environment=dev 
+          python test/regression_tests.py --binaries --${{ matrix.binaries }} --nightly
       - name: Install dependencies
+        if: matrix.os != 'macos-14'
         run: |
           python ts_scripts/install_dependencies.py --environment=dev
       - name: Validate Torchserve CPU Regression
+        if: matrix.os != 'macos-14'
         run: |
           python test/regression_tests.py --binaries --${{ matrix.binaries }} --nightly
           

cpp/README.md

@@ -2,10 +2,10 @@
 ## Requirements
 * C++17
 * GCC version: gcc-9
-* cmake version: 3.18+
+* cmake version: 3.26.4+
 * Linux
 
-For convenience, a docker container can be used as the development environment to build and install Torchserve CPP
+For convenience, a [docker container](../docker/README.md#create-torchserve-docker-image) can be used as the development environment to build and install Torchserve CPP
 ```
 cd serve/docker
 # For CPU support
@@ -21,6 +21,7 @@ docker run [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-
 # For GPU support
 docker run --gpus all [-v /path/to/build/dir:/serve/cpp/_build] -it pytorch/torchserve:cpp-dev-gpu /bin/bash
 ```
+`Warning`: The dev docker container does not install all necessary dependencies or build Torchserve CPP. Please follow the steps below after starting the container.
 
 ## Installation and Running TorchServe CPP
 This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.

docker/Dockerfile.cpp

@@ -16,29 +16,34 @@
 ARG BASE_IMAGE=ubuntu:20.04
 ARG PYTHON_VERSION=3.9
 ARG CMAKE_VERSION=3.26.4
+ARG GCC_VERSION=9
 ARG BRANCH_NAME="master"
 ARG USE_CUDA_VERSION=""
 
 FROM ${BASE_IMAGE} AS cpp-dev-image
 ARG BASE_IMAGE
 ARG PYTHON_VERSION
 ARG CMAKE_VERSION
+ARG GCC_VERSION
 ARG BRANCH_NAME
 ARG USE_CUDA_VERSION
+ARG DEBIAN_FRONTEND=noninteractive
 ENV PYTHONUNBUFFERED TRUE
+ENV TZ=Etc/UTC
 
 RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
     apt-get update && \
     apt-get install software-properties-common -y && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
-    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+    apt-get install --no-install-recommends -y \
         sudo \
         vim \
         git \
         curl \
         wget \
         rsync \
         gpg \
+        gcc-$GCC_VERSION \
         ca-certificates \
         lsb-release \
         openjdk-17-jdk \
@@ -51,32 +56,15 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
 RUN python$PYTHON_VERSION -m venv /home/venv
 ENV PATH="/home/venv/bin:$PATH"
 
-# Enable installation of recent cmake release
+# Enable installation of recent cmake release and pin cmake & cmake-data version
 # Ref: https://apt.kitware.com/
 RUN (wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null) \
     && (echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null) \
     && apt-get update \
     && (test -f /usr/share/doc/kitware-archive-keyring/copyright || sudo rm /usr/share/keyrings/kitware-archive-keyring.gpg) \
     && sudo apt-get install kitware-archive-keyring \
-    && rm -rf /var/lib/apt/lists/*
-
-# Pin cmake and cmake-data version
-# Ref: https://manpages.ubuntu.com/manpages/xenial/man5/apt_preferences.5.html
-RUN echo "Package: cmake\nPin: version $CMAKE_VERSION*\nPin-Priority: 1001" > /etc/apt/preferences.d/cmake
-RUN echo "Package: cmake-data\nPin: version $CMAKE_VERSION*\nPin-Priority: 1001" > /etc/apt/preferences.d/cmake-data
-
-# Install CUDA toolkit to enable "libtorch" build with GPU support
-RUN apt-get update && \
-    if echo "$BASE_IMAGE" | grep -q "cuda:"; then \
-        if [ "$USE_CUDA_VERSION" = "cu121" ]; then \
-            apt-get -y install cuda-toolkit-12-1; \
-        elif [ "$USE_CUDA_VERSION" = "cu118" ]; then \
-            apt-get -y install cuda-toolkit-11-8; \
-        else \
-            echo "Cuda version not supported by CPP backend: $USE_CUDA_VERSION"; \
-            exit 1; \
-        fi; \
-    fi \
+    && echo "Package: cmake\nPin: version $CMAKE_VERSION*\nPin-Priority: 1001" > /etc/apt/preferences.d/cmake \
+    && echo "Package: cmake-data\nPin: version $CMAKE_VERSION*\nPin-Priority: 1001" > /etc/apt/preferences.d/cmake-data \
     && rm -rf /var/lib/apt/lists/*
 
 RUN git clone --recursive https://github.com/pytorch/serve.git \

docker/README.md

@@ -41,6 +41,7 @@ Use `build_image.sh` script to build the docker images. The script builds the `p
 |-t, --tag|Tag name for image. If not specified, script uses torchserve default tag names.|
 |-cv, --cudaversion| Specify to cuda version to use. Supported values `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`. `cu121`, Default `cu121`|
 |-ipex, --build-with-ipex| Specify to build with intel_extension_for_pytorch. If not specified, script builds without intel_extension_for_pytorch.|
+|-cpp, --build-cpp specify to build TorchServe CPP|
 |-n, --nightly| Specify to build with TorchServe nightly.|
 |-py, --pythonversion| Specify the python version to use. Supported values `3.8`, `3.9`, `3.10`, `3.11`. Default `3.9`|
 
@@ -147,6 +148,16 @@ Creates a docker image with `torchserve` and `torch-model-archiver` installed fr
 ./build_image.sh -bt dev -ipex -t torchserve-ipex:1.0
 ```
 
+ - For creating image to build Torchserve CPP with CPU support:
+```bash
+./build_image.sh -bt dev -cpp
+```
+
+- For creating image to build Torchserve CPP with GPU support:
+```bash
+./build_image.sh -bt dev -g [-cv cu121|cu118] -cpp
+```
+
 
 ## Start a container with a TorchServe image
 

docker/build_image.sh

@@ -174,9 +174,14 @@ then
 
   if [[ "${MACHINE}" == "gpu" || "${CUDA_VERSION}" != "" ]];
   then
-    if [[ "${CUDA_VERSION}" != "cu121" && "${CUDA_VERSION}" != "cu118" ]];
+    if [ "${CUDA_VERSION}" == "cu121" ];
     then
-      echo "Only cuda versions 12.1 and 11.8 are supported for CPP"
+      BASE_IMAGE="nvidia/cuda:12.1.1-devel-ubuntu20.04"
+    elif [ "${CUDA_VERSION}" == "cu118" ];
+    then
+      BASE_IMAGE="nvidia/cuda:11.8.0-devel-ubuntu20.04"
+    else
+      echo "Cuda version $CUDA_VERSION is not supported for CPP"
       exit 1
     fi
   fi

docker/build_upload_release.py

@@ -36,6 +36,14 @@
         f"./build_image.sh -g -cv cu121 -t {organization}/torchserve:latest-gpu",
         dry_run,
     )
+    try_and_handle(
+        f"./build_image.sh -bt dev -cpp -t {organization}/torchserve:latest-cpp-dev-cpu",
+        dry_run,
+    )
+    try_and_handle(
+        f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/torchserve:latest-cpp-dev-gpu",
+        dry_run,
+    )
     try_and_handle(
         f"docker tag {organization}/torchserve:latest {organization}/torchserve:latest-cpu",
         dry_run,
@@ -48,13 +56,25 @@
         f"docker tag {organization}/torchserve:latest-gpu {organization}/torchserve:{check_ts_version()}-gpu",
         dry_run,
     )
+    try_and_handle(
+        f"docker tag {organization}/torchserve:latest-cpp-dev-cpu {organization}/torchserve:{check_ts_version()}-cpp-dev-cpu",
+        dry_run,
+    )
+    try_and_handle(
+        f"docker tag {organization}/torchserve:latest-cpp-dev-gpu {organization}/torchserve:{check_ts_version()}-cpp-dev-gpu",
+        dry_run,
+    )
 
     for image in [
         f"{organization}/torchserve:latest",
         f"{organization}/torchserve:latest-cpu",
         f"{organization}/torchserve:latest-gpu",
+        f"{organization}/torchserve:latest-cpp-dev-cpu",
+        f"{organization}/torchserve:latest-cpp-dev-gpu",
         f"{organization}/torchserve:{check_ts_version()}-cpu",
         f"{organization}/torchserve:{check_ts_version()}-gpu",
+        f"{organization}/torchserve:{check_ts_version()}-cpp-dev-cpu",
+        f"{organization}/torchserve:{check_ts_version()}-cpp-dev-gpu",
     ]:
         try_and_handle(f"docker push {image}", dry_run)
 

docker/docker_nightly.py

@@ -35,17 +35,29 @@
     project = "torchserve-nightly"
     cpu_version = f"{project}:cpu-{get_nightly_version()}"
     gpu_version = f"{project}:gpu-{get_nightly_version()}"
+    cpp_dev_cpu_version = f"{project}:cpp-dev-cpu-{get_nightly_version()}"
+    cpp_dev_gpu_version = f"{project}:cpp-dev-gpu-{get_nightly_version()}"
 
     # Build Nightly images and append the date in the name
     try_and_handle(f"./build_image.sh -n -t {organization}/{cpu_version}", dry_run)
     try_and_handle(
         f"./build_image.sh -g -cv cu121 -n -t {organization}/{gpu_version}",
         dry_run,
     )
+    try_and_handle(
+        f"./build_image.sh -bt dev -cpp -t {organization}/{cpp_dev_cpu_version}",
+        dry_run,
+    )
+    try_and_handle(
+        f"./build_image.sh -bt dev -g -cv cu121 -cpp -t {organization}/{cpp_dev_gpu_version}",
+        dry_run,
+    )
 
     # Push Nightly images to official PyTorch Dockerhub account
     try_and_handle(f"docker push {organization}/{cpu_version}", dry_run)
     try_and_handle(f"docker push {organization}/{gpu_version}", dry_run)
+    try_and_handle(f"docker push {organization}/{cpp_dev_cpu_version}", dry_run)
+    try_and_handle(f"docker push {organization}/{cpp_dev_gpu_version}", dry_run)
 
     # Tag nightly images with latest
     try_and_handle(
@@ -56,10 +68,20 @@
         f"docker tag {organization}/{gpu_version} {organization}/{project}:latest-gpu",
         dry_run,
     )
+    try_and_handle(
+        f"docker tag {organization}/{cpp_dev_cpu_version} {organization}/{project}:latest-cpp-dev-cpu",
+        dry_run,
+    )
+    try_and_handle(
+        f"docker tag {organization}/{cpp_dev_gpu_version} {organization}/{project}:latest-cpp-dev-gpu",
+        dry_run,
+    )
 
     # Push images with latest tag
     try_and_handle(f"docker push {organization}/{project}:latest-cpu", dry_run)
     try_and_handle(f"docker push {organization}/{project}:latest-gpu", dry_run)
+    try_and_handle(f"docker push {organization}/{project}:latest-cpp-dev-cpu", dry_run)
+    try_and_handle(f"docker push {organization}/{project}:latest-cpp-dev-gpu", dry_run)
 
     # Cleanup built images
     if args.cleanup:

docs/Security.md

@@ -5,6 +5,7 @@
 | Version | Supported          |
 |---------| ------------------ |
 | 0.9.0   | :white_check_mark: |
+| 0.10.0  | :white_check_mark: |
 
 
 ## How we do security
@@ -36,6 +37,7 @@ TorchServe as much as possible relies on automated tools to do security scanning
     2. Using private-key/certificate files
 
     You can find more details in the [configuration guide](https://pytorch.org/serve/configuration.html#enable-ssl)
+6. TorchServe supports token authorization: check [documentation](https://github.com/pytorch/serve/blob/master/docs/token_authorization_api.md) for more information. 
 
 
 

examples/large_models/inferentia2/llama2/continuous_batching/requirements.txt

@@ -1 +1,2 @@
-sentencepiece
+transformers==4.36.2
+sentencepiece==0.1.99

ts_scripts/install_dependencies.py

@@ -209,10 +209,9 @@ def install_numactl(self):
             os.system(f"{self.sudo_cmd}apt-get install -y numactl")
 
     def install_cpp_dependencies(self):
-        if os.system("clang-tidy --version") != 0 or args.force:
-            os.system(
-                f"{self.sudo_cmd}apt-get install -y {' '.join(CPP_LINUX_DEPENDENCIES)}"
-            )
+        os.system(
+            f"{self.sudo_cmd}apt-get install -y {' '.join(CPP_LINUX_DEPENDENCIES)}"
+        )
 
     def install_neuronx_driver(self):
         # Configure Linux for Neuron repository updates