Python: added MacOS and amazonlinux build tests to github actions CI

Author: barshaul

.github/workflows/build-python-wrapper/action.yml

@@ -0,0 +1,73 @@
+name: Build Python wrapper
+
+inputs:
+    os:
+        description: "The current operating system"
+        required: true
+        type: string
+        options:
+          - macOS
+          - ubuntu
+          - amazon-linux
+    release_mode:
+        description: "Enable building the wrapper in release mode"
+        required: false
+        type: boolean
+        default: 'false'
+
+env:
+    CARGO_TERM_COLOR: always
+
+runs:
+    using: "composite"
+    steps:        
+        - name: Install software dependencies for macOS
+          shell: bash
+          if: "${{ inputs.os == 'macOS' }}"
+          run: |
+              brew install git gcc pkgconfig protobuf openssl redis
+        
+        - name: Install software dependencies for Ubuntu
+          shell: bash
+          if: "${{ inputs.os == 'ubuntu' }}"
+          run: |
+              sudo apt update
+              sudo apt install protobuf-compiler
+
+        - name: Install software dependencies for Amazon-Linux
+          shell: bash
+          if: "${{ inputs.os == 'amazon-linux' }}"
+          run: |
+              yum install -y python3 gcc pkgconfig protobuf-compiler openssl openssl-devel which curl redis6 --allowerasing
+
+        - name: Install Python software dependencies
+          shell: bash
+          run: |
+              python3 -m ensurepip --upgrade
+              python3 -m pip install --upgrade pip
+              python3 -m pip install virtualenv mypy-protobuf
+
+        - name: Install Rust
+          shell: bash
+          run: |
+              curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+        - name: Generate protobuf files
+          shell: bash
+          working-directory: .
+          run: |
+              export PATH="$PATH:$HOME/.local/bin"
+              MYPY_PROTOC_PATH=`which protoc-gen-mypy`
+              protoc --plugin=protoc-gen-mypy=${MYPY_PROTOC_PATH} -Iprotobuf=./babushka-core/src/protobuf/ --python_out=./python/python/pybushka --mypy_out=./python/python/pybushka ./babushka-core/src/protobuf/*.proto
+
+        - name: Build Pybushka
+          shell: bash
+          working-directory: ./python
+          run: |
+              source "$HOME/.cargo/env"
+              python3 -m venv .env
+              source .env/bin/activate
+              python3 -m pip install --upgrade pip
+              python3 -m pip install -r requirements.txt
+              RELEASE_FLAG=`if [ ${{ inputs.release_mode }} = 'true'  ]; then echo --release; fi`
+              maturin develop ${RELEASE_FLAG}

.github/workflows/python.yml

@@ -8,18 +8,20 @@ on:
             - babushka-core/src/**
             - submodules/**
             - utils/cluster_manager.py
+            - .github/workflows/python.yml
     pull_request:
         paths:
             - python/**
             - babushka-core/src/**
             - submodules/**
             - utils/cluster_manager.py
+            - .github/workflows/python.yml
 
 permissions:
     contents: read
 
 jobs:
-    build:
+    test-ubuntu-latest:
         runs-on: ubuntu-latest
         timeout-minutes: 15
         strategy:
@@ -39,11 +41,6 @@ jobs:
               with:
                   redis-version: ${{ matrix.redis }}
 
-            - name: Install protoc
-              run: |
-                  sudo apt update
-                  sudo apt install protobuf-compiler
-
             - name: Set up Python 3.10
               uses: actions/setup-python@v3
               with:
@@ -73,23 +70,11 @@ jobs:
               run: |
                   black --target-version py36 --check --diff .
 
-            - name: Start redis server
-              working-directory: ./python
-              run: redis-server &
-
-            - name: Generate protobuf files
-              working-directory: .
-              run: |
-                  MYPY_PROTOC_PATH=`which protoc-gen-mypy`
-                  protoc --plugin=protoc-gen-mypy=${MYPY_PROTOC_PATH} -Iprotobuf=./babushka-core/src/protobuf/ --python_out=./python/python/pybushka --mypy_out=./python/python/pybushka ./babushka-core/src/protobuf/*.proto
-
-            - name: Build Pybushka
-              working-directory: ./python
-              run: |
-                  python -m venv .env
-                  source .env/bin/activate
-                  pip install -r requirements.txt
-                  maturin develop --release
+            - name: Build Python wrapper
+              uses: ./.github/workflows/build-python-wrapper
+              with:
+                  os: 'ubuntu'
+                  release_mode: 'true'
 
             - name: Type check with mypy
               working-directory: ./python
@@ -101,7 +86,7 @@ jobs:
                   # Install the benchmark requirements 
                   pip install -r ../benchmarks/python/requirements.txt
                   python -m mypy ..
-              
+
             - name: Test with pytest
               working-directory: ./python
               run: |
@@ -120,3 +105,60 @@ jobs:
               with:
                   cargo-toml-folder: ./python                  
               name: lint python-rust
+    
+    build-macos-latest:
+        runs-on: macos-latest
+        timeout-minutes: 15
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+            - name: Set up Homebrew
+              uses: Homebrew/actions/setup-homebrew@master
+
+            - name: Build Python wrapper
+              uses: ./.github/workflows/build-python-wrapper
+              with:
+                  os: 'macOS'              
+            - name: Test compatibility with pytest
+              working-directory: ./python
+              run: |
+                  source .env/bin/activate
+                  pytest --asyncio-mode=auto -m smoke_test
+    
+    build-amazonlinux-latest:
+      runs-on: ubuntu-latest
+      container: amazonlinux:latest
+      timeout-minutes: 15
+      steps:
+          - name: Install git
+            run: |
+              yum -y remove git
+              yum -y remove git-*
+              yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
+              yum install -y git 
+              git --version
+
+          - uses: actions/checkout@v3
+
+          - name: Checkout submodules
+            run: |
+              git config --global --add safe.directory "$GITHUB_WORKSPACE"
+              git submodule update --init --recursive
+
+          - name: Build Python wrapper
+            uses: ./.github/workflows/build-python-wrapper
+            with:
+                os: 'amazon-linux'
+
+          - name: Create a symbolic Link for redis6 binaries
+            working-directory: ./python
+            run: |
+                ln -s /usr/bin/redis6-server /usr/bin/redis-server
+                ln -s /usr/bin/redis6-cli /usr/bin/redis-cli
+            
+          - name: Test compatibility with pytest
+            working-directory: ./python
+            run: |
+                source .env/bin/activate
+                pytest --asyncio-mode=auto -m smoke_test

python/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    smoke_test: mark a test as a build verification testing.

python/python/tests/test_async_client.py

@@ -197,6 +197,7 @@ async def test_select_standalone_database_id(self, request):
 
 @pytest.mark.asyncio
 class TestCommands:
+    @pytest.mark.smoke_test
     @pytest.mark.parametrize("cluster_mode", [True, False])
     async def test_socket_set_get(self, redis_client: TRedisClient):
         key = get_random_string(10)