ci(json): Add requires_any field and use QIO by default to match IDE (#10472)

lucasssvaz · web-flow · commit 4ca635a3b623 · 2024-10-19T12:00:48.000+03:00
* ci(FQBN): Use QIO as default as DIO can be used on demand now

* fix(indentation): Fix default indentation for bash files

* fix(compilation): Make errors appear on CI fail

* ci(json): Add requires_any field to JSON and fix comparison
diff --git a/.editorconfig b/.editorconfig
@@ -18,7 +18,7 @@ indent_size = 2
 indent_style = space
 
 [*.{bash,sh}]
-indent_size = 2
+indent_size = 4
 indent_style = space
 
 [*.{c,cc,cp,cpp,cxx,h,hh,hpp,hxx,ii,inl,ino,ixx,pde,tpl,tpp,txx}]
diff --git a/.github/scripts/install-platformio-esp32.sh b/.github/scripts/install-platformio-esp32.sh
@@ -96,9 +96,9 @@ function count_sketches(){ # count_sketches <examples-path>
                 continue
             fi
 
-            # Check if the sketch requires any configuration options
+            # Check if the sketch requires any configuration options (AND)
             requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
-            if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+            if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
                 for requirement in $requirements; do
                     requirement=$(echo $requirement | xargs)
                     found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
@@ -107,6 +107,23 @@ function count_sketches(){ # count_sketches <examples-path>
                     fi
                 done
             fi
+
+            # Check if the sketch requires any configuration options (OR)
+            requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
+            if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
+                found=false
+                for requirement in $requirements_or; do
+                    requirement=$(echo $requirement | xargs)
+                    found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
+                    if [[ "$found_line" != "" ]]; then
+                        found=true
+                        break
+                    fi
+                done
+                if [[ "$found" == "false" ]]; then
+                    continue
+                fi
+            fi
         fi
 
         echo $sketch >> sketches.txt
@@ -187,9 +204,9 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
                 continue
             fi
 
-            # Check if the sketch requires any configuration options
+            # Check if the sketch requires any configuration options (AND)
             requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
-            if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+            if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
                 for requirement in $requirements; do
                     requirement=$(echo $requirement | xargs)
                     found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
@@ -198,6 +215,23 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
                     fi
                 done
             fi
+
+            # Check if the sketch requires any configuration options (OR)
+            requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
+            if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
+                found=false
+                for requirement in $requirements_or; do
+                    requirement=$(echo $requirement | xargs)
+                    found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/esp32/sdkconfig")
+                    if [[ "$found_line" != "" ]]; then
+                        found=true
+                        break
+                    fi
+                done
+                if [[ "$found" == "false" ]]; then
+                    continue
+                fi
+            fi
         fi
 
         sketchnum=$(($sketchnum + 1))
diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh
@@ -98,34 +98,42 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
             # Default FQBN options if none were passed in the command line.
 
-            esp32_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}"
-            esp32s2_opts="PSRAM=enabled,FlashMode=dio${fqbn_append:+,$fqbn_append}"
-            esp32s3_opts="PSRAM=opi,USBMode=default,FlashMode=dio${fqbn_append:+,$fqbn_append}"
-            esp32c3_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
-            esp32c6_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
-            esp32h2_opts="FlashMode=dio${fqbn_append:+,$fqbn_append}"
+            esp32_opts="PSRAM=enabled${fqbn_append:+,$fqbn_append}"
+            esp32s2_opts="PSRAM=enabled${fqbn_append:+,$fqbn_append}"
+            esp32s3_opts="PSRAM=opi,USBMode=default${fqbn_append:+,$fqbn_append}"
+            esp32c3_opts="$fqbn_append"
+            esp32c6_opts="$fqbn_append"
+            esp32h2_opts="$fqbn_append"
 
             # Select the common part of the FQBN based on the target.  The rest will be
             # appended depending on the passed options.
 
+            opt=""
+
             case "$target" in
                 "esp32")
-                    fqbn="espressif:esp32:esp32:${options:-$esp32_opts}"
+                    [ -n "${options:-$esp32_opts}" ] && opt=":${options:-$esp32_opts}"
+                    fqbn="espressif:esp32:esp32$opt"
                 ;;
                 "esp32s2")
-                    fqbn="espressif:esp32:esp32s2:${options:-$esp32s2_opts}"
+                    [ -n "${options:-$esp32s2_opts}" ] && opt=":${options:-$esp32s2_opts}"
+                    fqbn="espressif:esp32:esp32s2$opt"
                 ;;
                 "esp32c3")
-                    fqbn="espressif:esp32:esp32c3:${options:-$esp32c3_opts}"
+                    [ -n "${options:-$esp32c3_opts}" ] && opt=":${options:-$esp32c3_opts}"
+                    fqbn="espressif:esp32:esp32c3$opt"
                 ;;
                 "esp32s3")
-                    fqbn="espressif:esp32:esp32s3:${options:-$esp32s3_opts}"
+                    [ -n "${options:-$esp32s3_opts}" ] && opt=":${options:-$esp32s3_opts}"
+                    fqbn="espressif:esp32:esp32s3$opt"
                 ;;
                 "esp32c6")
-                    fqbn="espressif:esp32:esp32c6:${options:-$esp32c6_opts}"
+                    [ -n "${options:-$esp32c6_opts}" ] && opt=":${options:-$esp32c6_opts}"
+                    fqbn="espressif:esp32:esp32c6$opt"
                 ;;
                 "esp32h2")
-                    fqbn="espressif:esp32:esp32h2:${options:-$esp32h2_opts}"
+                    [ -n "${options:-$esp32h2_opts}" ] && opt=":${options:-$esp32h2_opts}"
+                    fqbn="espressif:esp32:esp32h2$opt"
                 ;;
             esac
 
@@ -163,9 +171,9 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
             exit 0
         fi
 
-        # Check if the sketch requires any configuration options
+        # Check if the sketch requires any configuration options (AND)
         requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
-        if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+        if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
             for requirement in $requirements; do
                 requirement=$(echo $requirement | xargs)
                 found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig")
@@ -175,6 +183,24 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 fi
             done
         fi
+
+        # Check if the sketch excludes any configuration options (OR)
+        requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
+        if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
+            found=false
+            for requirement in $requirements_or; do
+                requirement=$(echo $requirement | xargs)
+                found_line=$(grep -E "^$requirement" "$SDKCONFIG_DIR/$target/sdkconfig")
+                if [[ "$found_line" != "" ]]; then
+                    found=true
+                    break
+                fi
+            done
+            if [[ "$found" == "false" ]]; then
+                echo "Target $target meets none of the requirements in requires_any for $sketchname. Skipping."
+                exit 0
+            fi
+        fi
     fi
 
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
@@ -213,9 +239,9 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 --build-cache-path "$ARDUINO_CACHE_DIR" \
                 --build-path "$build_dir" \
                 $xtra_opts "${sketchdir}" \
-                > $output_file
+                2>&1 | tee $output_file
 
-            exit_status=$?
+            exit_status=${PIPESTATUS[0]}
             if [ $exit_status -ne 0 ]; then
                 echo "ERROR: Compilation failed with error code $exit_status"
                 exit $exit_status
@@ -322,9 +348,9 @@ function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requi
             fi
 
             if [ "$ignore_requirements" != "1" ]; then
-                # Check if the sketch requires any configuration options
+                # Check if the sketch requires any configuration options (AND)
                 requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
-                if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+                if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
                     for requirement in $requirements; do
                         requirement=$(echo $requirement | xargs)
                         found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig)
@@ -333,6 +359,23 @@ function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requi
                         fi
                     done
                 fi
+
+                # Check if the sketch excludes any configuration options (OR)
+                requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
+                if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
+                    found=false
+                    for requirement in $requirements_or; do
+                        requirement=$(echo $requirement | xargs)
+                        found_line=$(grep -E "^$requirement" $SDKCONFIG_DIR/$target/sdkconfig)
+                        if [[ "$found_line" != "" ]]; then
+                            found=true
+                            break
+                        fi
+                    done
+                    if [[ "$found" == "false" ]]; then
+                        continue 2
+                    fi
+                fi
             fi
         fi
         echo $sketch >> sketches.txt
diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh
@@ -36,9 +36,9 @@ function run_test() {
             return 0
         fi
 
-        # Check if the sketch requires any configuration options
+        # Check if the sketch requires any configuration options (AND)
         requirements=$(jq -r '.requires[]? // empty' $sketchdir/ci.json)
-        if [[ "$requirements" != "null" ]] || [[ "$requirements" != "" ]]; then
+        if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
             for requirement in $requirements; do
                 requirement=$(echo $requirement | xargs)
                 found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH")
@@ -49,6 +49,25 @@ function run_test() {
                 fi
             done
         fi
+
+        # Check if the sketch requires any configuration options (OR)
+        requirements_or=$(jq -r '.requires_any[]? // empty' $sketchdir/ci.json)
+        if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
+            found=false
+            for requirement in $requirements_or; do
+                requirement=$(echo $requirement | xargs)
+                found_line=$(grep -E "^$requirement" "$SDKCONFIG_PATH")
+                if [[ "$found_line" != "" ]]; then
+                    found=true
+                    break
+                fi
+            done
+            if [[ "$found" == "false" ]]; then
+                printf "\033[93mTarget $target meets none of the requirements in requires_any for $sketchname. Skipping.\033[0m\n"
+                printf "\n\n\n"
+                return 0
+            fi
+        fi
     fi
 
     if [ $len -eq 1 ]; then
diff --git a/docs/en/contributing.rst b/docs/en/contributing.rst
@@ -172,12 +172,12 @@ And in the ``README.md`` file:
 By default, the CI system will use the FQBNs specified in the ``.github/scripts/sketch_utils.sh`` file to compile the sketches.
 Currently, the default FQBNs are:
 
-* ``espressif:esp32:esp32:PSRAM=enabled,FlashMode=dio``
-* ``espressif:esp32:esp32s2:PSRAM=enabled,FlashMode=dio``
-* ``espressif:esp32:esp32s3:PSRAM=opi,USBMode=default,FlashMode=dio``
-* ``espressif:esp32:esp32c3:FlashMode=dio``
-* ``espressif:esp32:esp32c6:FlashMode=dio``
-* ``espressif:esp32:esp32h2:FlashMode=dio``
+* ``espressif:esp32:esp32:PSRAM=enabled``
+* ``espressif:esp32:esp32s2:PSRAM=enabled``
+* ``espressif:esp32:esp32s3:PSRAM=opi,USBMode=default``
+* ``espressif:esp32:esp32c3``
+* ``espressif:esp32:esp32c6``
+* ``espressif:esp32:esp32h2``
 
 There are two ways to alter the FQBNs used to compile the sketches: by using the ``fqbn`` or ``fqbn_append`` fields in the ``ci.json`` file.
 
@@ -408,7 +408,9 @@ CI JSON File
 The ``ci.json`` file is used to specify how the test suite and sketches will handled by the CI system. It can contain the following fields:
 
 * ``requires``: A list of configurations in ``sdkconfig`` that are required to run the test suite. The test suite will only run on the targets
-  that have the required configurations. By default, no configurations are required.
+  that have **ALL** the required configurations. By default, no configurations are required.
+* ``requires_any``: A list of configurations in ``sdkconfig`` that are required to run the test suite. The test suite will only run on the targets
+  that have **ANY** of the required configurations. By default, no configurations are required.
 * ``targets``: A dictionary that specifies the targets for which the tests will be run. The key is the target name and the value is a boolean
   that specifies if the test should be run for that target. By default, all targets are enabled as long as they have the required configurations
   specified in the ``requires`` field. This field is also valid for examples.
