Run ISA tests in CI (riscv#134)

* test: Ignore generated XML output

* run_tests: Build RVFI emulators too

Can't run tests with them though as they're built for direct instruction
injection (RVFI-DII) via an instruction stream over a network socket,
not fetching instructions from memory, so this remains just a build
test.

* run_tests/run_fp_tests: Print summary and give meaningful exit code

* run_tests/run_fp_tests: Include tests and failures in top-level XML entity

* run_tests/run_fp_tests: Use failure not error for XML output

The former is the standard tag for normal test failures, the latter is
for catastrophic things like test harness errors.

* Run ISA tests in CI

Author: jrtc27

.github/workflows/compile.yml

@@ -6,10 +6,10 @@ jobs:
   build:
     runs-on: [ubuntu-18.04]
     steps:
-    - name: Install opam2
-      run: |
-        sudo add-apt-repository -y ppa:avsm/ppa
-        sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3
+    - name: Add opam2 PPA
+      run: sudo add-apt-repository -y ppa:avsm/ppa
+    - name: Install packages
+      run: sudo apt install -y opam zlib1g-dev pkg-config libgmp-dev z3 device-tree-compiler
     - name: Init opam
       run: opam init --disable-sandboxing -y
     - name: Install sail
@@ -18,7 +18,17 @@ jobs:
       uses: actions/checkout@HEAD
       with:
         submodules: true
-    - name: Build RV32 simulators
-      run: eval $(opam env) && make ARCH=RV32 -j2 csim rvfi osim
-    - name: Build RV64 simulators
-      run: eval $(opam env) && make ARCH=RV64 -j2 csim rvfi osim
+    - name: Build and test simulators
+      run: eval $(opam env) && test/run_tests.sh
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: tests.xml
+        path: test/tests.xml
+    - name: Upload event payload
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: event.json
+        path: ${{ github.event_path }}

.github/workflows/test-results.yml

@@ -0,0 +1,51 @@
+name: Publish test results
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+    - completed
+
+jobs:
+  publish-test-results:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+    steps:
+    - name: Download artifacts
+      uses: actions/[email protected]
+      with:
+        script: |
+          var fs = require('fs');
+          var artifacts = await github.actions.listWorkflowRunArtifacts({
+             owner: context.repo.owner,
+             repo: context.repo.repo,
+             run_id: ${{github.event.workflow_run.id }},
+          });
+          var matchArtifacts = artifacts.data.artifacts.filter((artifact) => {
+            return artifact.name == 'tests.xml' || artifact.name == 'event.json'
+          });
+          var count = matchArtifacts.length;
+          for (var i = 0; i < count; i++) {
+            var matchArtifact = matchArtifacts[i];
+            var download = await github.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var name = matchArtifact.name;
+            var dest = name + '.zip'
+            fs.writeFileSync('${{github.workspace}}/' + dest, Buffer.from(download.data));
+            console.log("Downloaded", name, "as", dest);
+          }
+    - name: Extract test results
+      run: unzip tests.xml.zip
+    - name: Extract event payload
+      run: unzip event.json.zip
+    - name: Publish test results
+      uses: EnricoMi/publish-unit-test-result-action@v1
+      with:
+        commit: ${{ github.event.workflow_run.head_sha }}
+        event_file: event.json
+        event_name: ${{ github.event.workflow_run.event }}
+        files: tests.xml

test/.gitignore

@@ -0,0 +1 @@
+/tests.xml

test/run_fp_tests.sh

@@ -14,39 +14,41 @@ rm -f $DIR/tests.xml
 
 pass=0
 fail=0
-XML=""
+all_pass=0
+all_fail=0
+SUITE_XML=""
+SUITES_XML=""
 
 function green {
     (( pass += 1 ))
     printf "$1: ${GREEN}$2${NC}\n"
-    XML+="    <testcase name=\"$1\"/>\n"
+    SUITE_XML+="    <testcase name=\"$1\"/>\n"
 }
 
 function yellow {
     (( fail += 1 ))
     printf "$1: ${YELLOW}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function red {
     (( fail += 1 ))
     printf "$1: ${RED}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function finish_suite {
     printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n"
-    XML="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML  </testsuite>\n"
-    printf "$XML" >> $DIR/tests.xml
-    XML=""
+    SUITES_XML+="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML  </testsuite>\n"
+    SUITE_XML=""
+    (( all_pass += pass )) || :
+    (( all_fail += fail )) || :
     pass=0
     fail=0
 }
 
 SAILLIBDIR="$DIR/../../lib/"
 
-printf "<testsuites>\n" >> $DIR/tests.xml
-
 cd $RISCVDIR
 
 # Do 'make clean' to avoid cross-arch pollution.
@@ -85,4 +87,11 @@ for test in $DIR/riscv-tests/rv32u{f,d}*.elf $DIR/riscv-tests/rv32mi-p-csr.elf;
 done
 finish_suite "32-bit RISCV C tests"
 
-printf "</testsuites>\n" >> $DIR/tests.xml
+printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n"
+XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n"
+printf "$XML" > $DIR/tests.xml
+
+if [ $all_fail -gt 0 ]
+then
+    exit 1
+fi

test/run_tests.sh

@@ -14,39 +14,41 @@ rm -f $DIR/tests.xml
 
 pass=0
 fail=0
-XML=""
+all_pass=0
+all_fail=0
+SUITE_XML=""
+SUITES_XML=""
 
 function green {
     (( pass += 1 ))
     printf "$1: ${GREEN}$2${NC}\n"
-    XML+="    <testcase name=\"$1\"/>\n"
+    SUITE_XML+="    <testcase name=\"$1\"/>\n"
 }
 
 function yellow {
     (( fail += 1 ))
     printf "$1: ${YELLOW}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function red {
     (( fail += 1 ))
     printf "$1: ${RED}$2${NC}\n"
-    XML+="    <testcase name=\"$1\">\n      <error message=\"$2\">$2</error>\n    </testcase>\n"
+    SUITE_XML+="    <testcase name=\"$1\">\n      <failure message=\"$2\">$2</failure>\n    </testcase>\n"
 }
 
 function finish_suite {
     printf "$1: Passed ${pass} out of $(( pass + fail ))\n\n"
-    XML="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$XML  </testsuite>\n"
-    printf "$XML" >> $DIR/tests.xml
-    XML=""
+    SUITES_XML+="  <testsuite name=\"$1\" tests=\"$(( pass + fail ))\" failures=\"${fail}\" timestamp=\"$(date)\">\n$SUITE_XML  </testsuite>\n"
+    SUITE_XML=""
+    (( all_pass += pass )) || :
+    (( all_fail += fail )) || :
     pass=0
     fail=0
 }
 
 SAILLIBDIR="$DIR/../../lib/"
 
-printf "<testsuites>\n" >> $DIR/tests.xml
-
 cd $RISCVDIR
 
 # Do 'make clean' to avoid cross-arch pollution.
@@ -141,4 +143,33 @@ for test in $DIR/riscv-tests/rv64*.elf; do
 done
 finish_suite "64-bit RISCV C tests"
 
-printf "</testsuites>\n" >> $DIR/tests.xml
+# Do 'make clean' to avoid cross-arch pollution.
+make clean
+
+if ARCH=RV32 make c_emulator/riscv_rvfi_RV32;
+then
+    green "Building 32-bit RISCV RVFI C emulator" "ok"
+else
+    red "Building 32-bit RISCV RVFI C emulator" "fail"
+fi
+finish_suite "32-bit RISCV RVFI C tests"
+
+# Do 'make clean' to avoid cross-arch pollution.
+make clean
+
+if ARCH=RV64 make c_emulator/riscv_rvfi_RV64;
+then
+    green "Building 64-bit RISCV RVFI C emulator" "ok"
+else
+    red "Building 64-bit RISCV RVFI C emulator" "fail"
+fi
+finish_suite "64-bit RISCV RVFI C tests"
+
+printf "Passed ${all_pass} out of $(( all_pass + all_fail ))\n\n"
+XML="<testsuites tests=\"$(( all_pass + all_fail ))\" failures=\"${all_fail}\">\n$SUITES_XML</testsuites>\n"
+printf "$XML" > $DIR/tests.xml
+
+if [ $all_fail -gt 0 ]
+then
+    exit 1
+fi