ci(performance): Add performance tests to CI (#9560)

* ci(performance): Add performance tests to CI

* ci(req): Fix requirements

* ci(pre-commit): Apply automatic fixes

* ci(pre-commit): Increase maximum allowed complexity for python

---------

Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: Jan Procházka <[email protected]>

Author: 3 people

.flake8

@@ -1,12 +1,10 @@
 # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-python/.flake8
 # See: https://flake8.pycqa.org/en/latest/user/configuration.html
-# The code style defined in this file is the official standardized style to be used in all Arduino tooling projects and
-# should not be modified.
 
 [flake8]
 doctests = True
 # W503 and W504 are mutually exclusive. PEP 8 recommends line break before.
 ignore = W503,E203
-max-complexity = 10
+max-complexity = 20
 max-line-length = 120
 select = E,W,F,C,N

.github/scripts/sketch_utils.sh

@@ -121,7 +121,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
     fi
 
     if [ -z "$fqbn" ]; then
-        echo "No FQBN passed or unvalid chip: $target"
+        echo "No FQBN passed or invalid chip: $target"
         exit 1
     fi
 
@@ -139,7 +139,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         echo "Skipping $sketchname for target $target"
         exit 0
     fi
-    
+
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
     if [ -n "$ARDUINO_BUILD_DIR" ]; then
         build_dir="$ARDUINO_BUILD_DIR"
@@ -177,7 +177,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 --build-path "$build_dir" \
                 $xtra_opts "${sketchdir}" \
                 > $output_file
-            
+
             exit_status=$?
             if [ $exit_status -ne 0 ]; then
                 echo ""ERROR: Compilation failed with error code $exit_status""
@@ -198,11 +198,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 # Extract the desired substring using sed
                 lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
                 #append json file where key is fqbn, sketch name, sizes -> extracted values
-                echo "{\"name\": \"$lib_sketch_name\", 
+                echo "{\"name\": \"$lib_sketch_name\",
                     \"sizes\": [{
-                            \"flash_bytes\": $flash_bytes, 
-                            \"flash_percentage\": $flash_percentage, 
-                            \"ram_bytes\": $ram_bytes, 
+                            \"flash_bytes\": $flash_bytes,
+                            \"flash_percentage\": $flash_percentage,
+                            \"ram_bytes\": $ram_bytes,
                             \"ram_percentage\": $ram_percentage
                             }]
                     }," >> "$sizes_file"
@@ -365,6 +365,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
         start_index=$(( $chunk_index * $chunk_size ))
         if [ "$sketchcount" -le "$start_index" ]; then
             echo "Skipping job"
+            touch ~/.build_skipped
             return 0
         fi
 
@@ -386,7 +387,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     if [ $log_compilation ]; then
         #echo board,target and start of sketches to sizes_file json
         echo "{ \"board\": \"$fqbn\",
-                \"target\": \"$target\", 
+                \"target\": \"$target\",
                 \"sketches\": [" >> "$sizes_file"
     fi
 

.github/scripts/tests_build.sh

@@ -2,19 +2,20 @@
 
 USAGE="
 USAGE:
-    ${0} -c <chunk_build_opts>
-       Example: ${0} -c -t esp32 -i 0 -m 15
+    ${0} -c -type <test_type> <chunk_build_opts>
+       Example: ${0} -c -type validation -t esp32 -i 0 -m 15
     ${0} -s sketch_name <build_opts>
        Example: ${0} -s hello_world -t esp32
     ${0} -clean
        Remove build and test generated files
 "
 
 function clean(){
-    rm -rf tests/*/build*/
     rm -rf tests/.pytest_cache
-    rm -rf tests/*/__pycache__/
-    rm -rf tests/*/*.xml
+    find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
+    find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
+    find tests/ -name '*.xml' -exec rm -rf "{}" \+
+    find tests/ -name 'result_*.json' -exec rm -rf "{}" \+
 }
 
 SCRIPTS_DIR="./.github/scripts"
@@ -35,6 +36,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     -clean )
         clean
         exit 0
@@ -52,12 +57,25 @@ source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
 args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
 
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_build -eq 1 ]; then
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
-    args+=" -p $PWD/tests"
+    args+=" -p $test_folder"
 else
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
-    args+=" -s $PWD/tests/$sketch"
+    args+=" -s $test_folder/$sketch"
 fi
 
 ${BUILD_CMD} ${args} $*

.github/scripts/tests_run.sh

@@ -15,9 +15,9 @@ function run_test() {
     fi
 
     if [ $len -eq 1 ]; then
-      # build_dir="tests/$sketchname/build"
+      # build_dir="$sketchdir/build"
       build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
-      report_file="tests/$sketchname/$sketchname.xml"
+      report_file="$sketchdir/$sketchname.xml"
     fi
 
     for i in `seq 0 $(($len - 1))`
@@ -28,9 +28,9 @@ function run_test() {
         fi
 
         if [ $len -ne 1 ]; then
-            # build_dir="tests/$sketchname/build$i"
+            # build_dir="$sketchdir/build$i"
             build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
-            report_file="tests/$sketchname/$sketchname$i.xml"
+            report_file="$sketchdir/$sketchname$i.xml"
         fi
 
         pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
@@ -79,6 +79,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     * )
       break
       ;;
@@ -88,21 +92,39 @@ done
 
 source ${SCRIPTS_DIR}/install-arduino-ide.sh
 
+# If sketch is provided and test type is not, test type is inferred from the sketch path
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_run -eq 0 ]; then
-    run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase
+    if [ -z $sketch ]; then
+        echo "ERROR: Sketch name is required for single test run"
+        exit 1
+    fi
+    run_test $target $test_folder/$sketch/$sketch.ino $options $erase
 else
   if [ "$chunk_max" -le 0 ]; then
       echo "ERROR: Chunks count must be positive number"
-      return 1
+      exit 1
   fi
 
   if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
       echo "ERROR: Chunk index must be less than chunks count"
-      return 1
+      exit 1
   fi
 
   set +e
-  ${COUNT_SKETCHES} $PWD/tests $target
+  ${COUNT_SKETCHES} $test_folder $target
   sketchcount=$?
   set -e
   sketches=$(cat sketches.txt)
@@ -123,7 +145,8 @@ else
       start_index=$(( $chunk_index * $chunk_size ))
       if [ "$sketchcount" -le "$start_index" ]; then
           echo "Skipping job"
-          return 0
+          touch ~/.test_skipped
+          exit 0
       fi
 
       end_index=$(( $(( $chunk_index + 1 )) * $chunk_size ))

.github/workflows/hil.yml

@@ -18,11 +18,14 @@ jobs:
   gen_chunks:
     if: |
       contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+      contains(github.event.pull_request.labels.*.name, 'perf_test') ||
       (github.event_name == 'schedule' && github.repository == 'espressif/arduino-esp32')
     name: Generate Chunks matrix
     runs-on: ubuntu-latest
     outputs:
       chunks: ${{ steps.gen-chunks.outputs.chunks }}
+      test_folder: ${{ steps.gen-chunks.outputs.test_folder }}
+      test_type: ${{ steps.gen-chunks.outputs.test_type }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -31,15 +34,29 @@ jobs:
         id: gen-chunks
         run: |
           set +e
-          .github/scripts/sketch_utils.sh count tests
+          if [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "true" ] && \
+             [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "false" ]; then
+            test_folder="tests/validation"
+            test_type="validation"
+          elif [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "false" ] && \
+               [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "true" ]; then
+            test_folder="tests/performance"
+            test_type="performance"
+          else
+            test_folder="tests"
+            test_type="all"
+          fi
+          .github/scripts/sketch_utils.sh count $test_folder
           sketches=$?
           if [[ $sketches -ge ${{env.MAX_CHUNKS}} ]]; then
             $sketches=${{env.MAX_CHUNKS}}
           fi
           set -e
           rm sketches.txt
           CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`)
-          echo "chunks=${CHUNKS}" >>$GITHUB_OUTPUT
+          echo "chunks=${CHUNKS}" >> $GITHUB_OUTPUT
+          echo "test_folder=${test_folder}" >> $GITHUB_OUTPUT
+          echo "test_type=${test_type}" >> $GITHUB_OUTPUT
 
   Build:
     needs: gen_chunks
@@ -52,17 +69,21 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
+
       - name: Build sketches
         run: |
-          bash .github/scripts/tests_build.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
+          bash .github/scripts/tests_build.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}
+
       - name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
         uses: actions/upload-artifact@v4
         with:
           name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
-          path: |
-             ~/.arduino/tests/*/build*.tmp/*.bin
-             ~/.arduino/tests/*/build*.tmp/*.json
           if-no-files-found: error
+          path: |
+            ~/.build_skipped
+            ~/.arduino/tests/**/build*.tmp/*.bin
+            ~/.arduino/tests/**/build*.tmp/*.json
+
   Test:
     needs: [gen_chunks, Build]
     name: ${{matrix.chip}}-Test#${{matrix.chunks}}
@@ -77,36 +98,49 @@ jobs:
       options: --privileged
 
     steps:
-       - name: Checkout repository
-         uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-       - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
-         uses: actions/download-artifact@v4
-         with:
-           name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
-           path: ~/.arduino/tests/
+      - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
+          path: ~/
 
-       - name: Install dependencies
-         run: |
-           pip install -U pip
-           pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
-           apt update && apt install -y -qq jq
+      - name: Install dependencies
+        run: |
+          pip install -U pip
+          pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
+          apt update && apt install -y -qq jq
 
-       - name: Run Tests
-         run: |
-           bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
+      - name: Run Tests
+        run: |
+          bash .github/scripts/tests_run.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e
 
-       - name: Upload test result artifacts
-         uses: actions/upload-artifact@v4
-         if: always()
-         with:
-           name: test_results-${{matrix.chip}}-${{matrix.chunks}}
-           path: tests/*/*.xml
+      - name: Check if tests were skipped
+        id: check-test-skipped
+        run: |
+          if [ -f ~/.test_skipped ]; then
+            echo "skipped=true" >> $GITHUB_OUTPUT
+          else
+            echo "skipped=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload test result artifacts
+        uses: actions/upload-artifact@v4
+        if: ${{ always() && steps.check-test-skipped.outputs.skipped == 'false' }}
+        with:
+          name: test_results-${{matrix.chip}}-${{matrix.chunks}}
+          if-no-files-found: error
+          path: |
+            tests/**/*.xml
+            tests/**/result_*.json
 
   event_file:
     name: "Event File"
     if: |
       contains(github.event.pull_request.labels.*.name, 'hil_test') ||
+      contains(github.event.pull_request.labels.*.name, 'perf_test') ||
       github.event_name == 'schedule'
     needs: Test
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -1,4 +1,8 @@
-exclude: ".github/.*"
+exclude: |
+  (?x)(
+      ^\.github\/|
+      ^tests\/performance\/coremark\/.*\.[ch]$
+  )
 
 default_language_version:
   # force all unspecified python hooks to run python3

tests/.gitignore

@@ -1,3 +1,4 @@
 build*/
 __pycache__/
 *.xml
+result_*.json