Skip to content

Commit 96f8f5e

Browse files
Ouss4Ouss4
authored
Add initial hardware testing support (espressif#6313)
- Added workflow triggered by cron or label "hil_test" - Added examples with both pytest and unity
1 parent 4da1051 commit 96f8f5e

14 files changed

+372
-7
lines changed

‎.github/scripts/on-push.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ fi
5858

5959
SCRIPTS_DIR="./.github/scripts"
6060
if [ "$BUILD_PIO" -eq 0 ]; then
61-
source ./.github/scripts/install-arduino-ide.sh
61+
source ${SCRIPTS_DIR}/install-arduino-ide.sh
6262
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
6363

6464
FQBN_ESP32="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
@@ -80,7 +80,7 @@ if [ "$BUILD_PIO" -eq 0 ]; then
8080
build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
8181
build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
8282
else
83-
source ./${SCRIPTS_DIR}/install-platformio-esp32.sh
83+
source ${SCRIPTS_DIR}/install-platformio-esp32.sh
8484
# PlatformIO ESP32 Test
8585
BOARD="esp32dev"
8686
OPTIONS="board_build.partitions = huge_app.csv"

‎.github/scripts/sketch_utils.sh

+5-5
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <fqbn> <path-to-i
3232
$win_opts $xtra_opts "$sketch"
3333
}
3434

35-
function count_sketches(){ # count_sketches <path> <target>
35+
function count_sketches(){ # count_sketches <path> [target]
3636
local path=$1
3737
local target=$2
3838

39-
if [ $# -lt 2 ]; then
39+
if [ $# -lt 1 ]; then
4040
echo "ERROR: Illegal number of parameters"
41-
echo "USAGE: ${0} count <path> <target>"
41+
echo "USAGE: ${0} count <path> [target]"
4242
fi
4343

4444
rm -rf sketches.txt
@@ -47,15 +47,15 @@ function count_sketches(){ # count_sketches <path> <target>
4747
return 0
4848
fi
4949

50-
local sketches=$(find $path -name *.ino)
50+
local sketches=$(find $path -name *.ino | sort)
5151
local sketchnum=0
5252
for sketch in $sketches; do
5353
local sketchdir=$(dirname $sketch)
5454
local sketchdirname=$(basename $sketchdir)
5555
local sketchname=$(basename $sketch)
5656
if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
5757
continue
58-
elif [[ -f "$sketchdir/.skip.$target" ]]; then
58+
elif [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
5959
continue
6060
else
6161
echo $sketch >> sketches.txt

‎.github/scripts/tests_build.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
SCRIPTS_DIR="./.github/scripts"
4+
BUILD_CMD=""
5+
6+
if [ $# -eq 3 ]; then
7+
chunk_build=1
8+
elif [ $# -eq 2 ]; then
9+
chunk_build=0
10+
else
11+
echo "ERROR: Illegal number of parameters"
12+
echo "USAGE:
13+
${0} <target> <sketch_dir>
14+
${0} <target> <chunk> <total_chunks>
15+
"
16+
exit 0
17+
fi
18+
19+
target=$1
20+
21+
case "$target" in
22+
"esp32") fqbn="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app"
23+
;;
24+
"esp32s2") fqbn="espressif:esp32:esp32s2:PSRAM=enabled,PartitionScheme=huge_app"
25+
;;
26+
"esp32c3") fqbn="espressif:esp32:esp32c3:PartitionScheme=huge_app"
27+
;;
28+
esac
29+
30+
if [ -z $fqbn ]; then
31+
echo "Unvalid chip $1"
32+
exit 0
33+
fi
34+
35+
source ${SCRIPTS_DIR}/install-arduino-ide.sh
36+
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
37+
38+
args="$ARDUINO_IDE_PATH $ARDUINO_USR_PATH \"$fqbn\""
39+
40+
if [ $chunk_build -eq 1 ]; then
41+
chunk_index=$2
42+
chunk_max=$3
43+
44+
if [ "$chunk_index" -gt "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
45+
chunk_index=$chunk_max
46+
fi
47+
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
48+
args+=" $target $PWD/tests $chunk_index $chunk_max"
49+
else
50+
sketchdir=$2
51+
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
52+
args+=" $PWD/tests/$sketchdir/$sketchdir.ino"
53+
fi
54+
55+
${BUILD_CMD} ${args}
56+

‎.github/scripts/tests_run.sh

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
3+
target=$1
4+
chunk_idex=$2
5+
chunks_num=$3
6+
7+
SCRIPTS_DIR="./.github/scripts"
8+
COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
9+
10+
source ${SCRIPTS_DIR}/install-arduino-ide.sh
11+
12+
if [ "$chunks_num" -le 0 ]; then
13+
echo "ERROR: Chunks count must be positive number"
14+
return 1
15+
fi
16+
if [ "$chunk_idex" -ge "$chunks_num" ] && [ "$chunks_num" -ge 2 ]; then
17+
echo "ERROR: Chunk index must be less than chunks count"
18+
return 1
19+
fi
20+
21+
set +e
22+
${COUNT_SKETCHES} $PWD/tests $target
23+
sketchcount=$?
24+
set -e
25+
sketches=$(cat sketches.txt)
26+
rm -rf sketches.txt
27+
28+
chunk_size=$(( $sketchcount / $chunks_num ))
29+
all_chunks=$(( $chunks_num * $chunk_size ))
30+
if [ "$all_chunks" -lt "$sketchcount" ]; then
31+
chunk_size=$(( $chunk_size + 1 ))
32+
fi
33+
34+
start_index=0
35+
end_index=0
36+
if [ "$chunk_idex" -ge "$chunks_num" ]; then
37+
start_index=$chunk_idex
38+
end_index=$sketchcount
39+
else
40+
start_index=$(( $chunk_idex * $chunk_size ))
41+
if [ "$sketchcount" -le "$start_index" ]; then
42+
echo "Skipping job"
43+
return 0
44+
fi
45+
46+
end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
47+
if [ "$end_index" -gt "$sketchcount" ]; then
48+
end_index=$sketchcount
49+
fi
50+
fi
51+
52+
start_num=$(( $start_index + 1 ))
53+
sketchnum=0
54+
55+
for sketch in $sketches; do
56+
sketchdir=$(dirname $sketch)
57+
sketchdirname=$(basename $sketchdir)
58+
sketchname=$(basename $sketch)
59+
sketchnum=$(($sketchnum + 1))
60+
if [ "$sketchnum" -le "$start_index" ] \
61+
|| [ "$sketchnum" -gt "$end_index" ]; then
62+
continue
63+
fi
64+
echo ""
65+
echo "Test for Sketch Index $(($sketchnum - 1)) - $sketchdirname"
66+
pytest tests -k test_$sketchdirname --junit-xml=tests/$sketchdirname/$sketchdirname.xml
67+
result=$?
68+
if [ $result -ne 0 ]; then
69+
return $result
70+
fi
71+
done

‎.github/workflows/hil.yml

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
name: Run tests in hardware
2+
3+
on:
4+
pull_request:
5+
types: [opened, reopened, synchronize, labeled]
6+
7+
schedule:
8+
- cron: '0 2 * * *'
9+
10+
env:
11+
MAX_CHUNKS: 15
12+
13+
concurrency:
14+
group: hil-${{github.event.pull_request.number || github.ref}}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
gen_chunks:
19+
if: |
20+
contains(github.event.pull_request.labels.*.name, 'hil_test') ||
21+
github.event_name == 'schedule'
22+
name: Generate Chunks matrix
23+
runs-on: ubuntu-latest
24+
outputs:
25+
chunks: ${{ steps.gen-chunks.outputs.chunks }}
26+
steps:
27+
- name: Checkout Repository
28+
uses: actions/checkout@v2
29+
30+
- name: Generate Chunks matrix
31+
id: gen-chunks
32+
run: |
33+
set +e
34+
bash .github/scripts/sketch_utils.sh count tests
35+
sketches=$((? - 1))
36+
if [[ $sketches -gt ${{env.MAX_CHUNKS}} ]]; then
37+
$sketches=${{env.MAX_CHUNKS}}
38+
fi
39+
set -e
40+
rm sketches.txt
41+
CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $sketches`)
42+
echo "::set-output name=chunks::${CHUNKS}"
43+
44+
Build:
45+
needs: gen_chunks
46+
name: ${{matrix.chip}}-Build#${{matrix.chunks}}
47+
runs-on: ubuntu-latest
48+
strategy:
49+
matrix:
50+
chip: ['esp32', 'esp32s2', 'esp32c3']
51+
chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
52+
53+
steps:
54+
- name: Checkout Repository
55+
uses: actions/checkout@v2
56+
57+
- name: Build sketches
58+
run: |
59+
bash .github/scripts/tests_build.sh ${{matrix.chip}} ${{matrix.chunks}} ${{env.MAX_CHUNKS}}
60+
- name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
61+
uses: actions/upload-artifact@v2
62+
with:
63+
name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
64+
path: |
65+
tests/*/build/*.bin
66+
tests/*/build/*.json
67+
Test:
68+
needs: [gen_chunks, Build]
69+
name: ${{matrix.chip}}-Test#${{matrix.chunks}}
70+
runs-on: ESP32
71+
strategy:
72+
fail-fast: false
73+
matrix:
74+
chip: ['esp32', 'esp32s2', 'esp32c3']
75+
chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
76+
container:
77+
image: python:3.10.1-bullseye
78+
options: --privileged
79+
80+
steps:
81+
- name: Checkout repository
82+
uses: actions/checkout@v2
83+
84+
- name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
85+
uses: actions/download-artifact@v2
86+
with:
87+
name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
88+
path: tests/
89+
90+
- name: Check Artifacts
91+
run: |
92+
ls -R tests
93+
cat tests/*/build/build.options.json
94+
95+
- name: Install dependencies
96+
run: |
97+
pip install -U pip
98+
pip install -r tests/requirements.txt
99+
100+
- name: Run Tests
101+
run: |
102+
bash .github/scripts/tests_run.sh ${{matrix.chip}} ${{matrix.chunks}} ${{env.MAX_CHUNKS}}
103+
104+
- name: Upload test result artifacts
105+
uses: actions/upload-artifact@v2
106+
if: always()
107+
with:
108+
name: test_results-${{matrix.chip}}-${{matrix.chunks}}
109+
path: tests/*/*.xml
110+
111+
event_file:
112+
name: "Event File"
113+
needs: Test
114+
runs-on: ubuntu-latest
115+
steps:
116+
- name: Upload
117+
uses: actions/upload-artifact@v2
118+
with:
119+
name: Event File
120+
path: ${{github.event_path}}

‎.github/workflows/publish.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Unit Test Results
2+
3+
on:
4+
workflow_run:
5+
workflows: [Run tests in hardware]
6+
7+
types:
8+
- completed
9+
10+
jobs:
11+
debug:
12+
name: Debug
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- name: Debug Action
17+
uses: hmarr/[email protected]
18+
19+
unit-test-results:
20+
name: Unit Test Results
21+
runs-on: ubuntu-latest
22+
steps:
23+
- name: Download and Extract Artifacts
24+
env:
25+
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
26+
run: |
27+
mkdir -p artifacts && cd artifacts
28+
artifacts_url=${{ github.event.workflow_run.artifacts_url }}
29+
gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
30+
do
31+
IFS=$'\t' read name url <<< "$artifact"
32+
gh api $url > "$name.zip"
33+
unzip -d "$name" "$name.zip"
34+
done
35+
- name: Publish Unit Test Results
36+
uses: EnricoMi/publish-unit-test-result-action@v1
37+
with:
38+
commit: ${{ github.event.workflow

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,6 @@ boards.sloeber.txt
2222
# Ignore docs build (Sphinx)
2323
docs/build
2424
docs/source/_build
25+
26+
# Test log files
27+
*.log

‎tests/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
build/
2+
__pycache__/

‎tests/hello_world/hello_world.ino

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
void setup(){
2+
// Open serial communications and wait for port to open:
3+
Serial.begin(115200);
4+
while (!Serial) {
5+
;
6+
}
7+
8+
Serial.println("Hello Arduino!");
9+
}
10+
11+
void loop(){
12+
}

‎tests/hello_world/test_hello_world.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
def test_hello_world(dut):
2+
dut.expect('Hello Arduino!')

‎tests/pytest.ini

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[pytest]
2+
addopts = --embedded-services esp,arduino
3+
4+
# log related
5+
log_cli = True
6+
log_cli_level = INFO
7+
log_cli_format = %(asctime)s %(levelname)s %(message)s
8+
log_cli_date_format = %Y-%m-%d %H:%M:%S
9+
10+
log_file = test.log
11+
log_file_level = INFO
12+
log_file_format = %(asctime)s %(levelname)s %(message)s
13+
log_file_date_format = %Y-%m-%d %H:%M:%S

‎tests/requirements.txt

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pyserial>=3.0
2+
esptool>=3.1
3+
pytest-cov
4+
cryptography<3.4; platform_machine == "armv7l"
5+
6+
pytest>=6.2.0
7+
pexpect>=4.4
8+
9+
pytest-embedded>=0.5.1
10+
pytest-embedded-serial>=0.5.1
11+
pytest-embedded-serial-esp>=0.5.1
12+
pytest-embedded-arduino>=0.5.1
13+

‎tests/unity/test_unity.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
def test_unity(dut):
2+
dut.expect_unity_test_output(timeout=240)

0 commit comments

Comments
 (0)