From de9d6ad3f5f59c9e051e86537fc13bbd466eb675 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:59:10 +0100 Subject: [PATCH] Add CI coverage summaries and aggregate JaCoCo report (#6451) --- .github/workflows/backend-build.yml | 73 +++- .github/workflows/build.yml | 18 + .github/workflows/coverage-aggregate.yml | 230 +++++++++++++ .github/workflows/docker-compose-tests.yml | 62 ++++ .github/workflows/e2e-live.yml | 125 ++++++- .github/workflows/frontend-validation.yml | 35 ++ .taskfiles/e2e.yml | 12 +- .taskfiles/frontend.yml | 19 +- build.gradle | 126 ++++++- .../src/core/tests/helpers/test-base.ts | 72 +++- scripts/coverage-matrix.py | 325 ++++++++++++++++++ scripts/coverage-summary.py | 259 ++++++++++++++ scripts/playwright-coverage-summary.py | 174 ++++++++++ .../docker-compose-coverage.override.yml | 27 ++ testing/test.sh | 49 ++- 15 files changed, 1583 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/coverage-aggregate.yml create mode 100644 scripts/coverage-matrix.py create mode 100644 scripts/coverage-summary.py create mode 100644 scripts/playwright-coverage-summary.py create mode 100644 testing/compose/docker-compose-coverage.override.yml diff --git a/.github/workflows/backend-build.yml b/.github/workflows/backend-build.yml index 73871dd7c..8d3023a89 100644 --- a/.github/workflows/backend-build.yml +++ b/.github/workflows/backend-build.yml @@ -1,8 +1,14 @@ name: Backend build, format check, and coverage # Reusable workflow called from build.yml. Runs the backend build matrix -# (JDK 25 × spring-security on/off), Spotless formatting check, JUnit, and +# (JDK 25 × every flavor), Spotless formatting check, JUnit, and # posts Jacoco coverage to PRs. +# +# Flavor axis (maps to STIRLING_FLAVOR in settings.gradle): +# core - DISABLE_ADDITIONAL_FEATURES=true, no proprietary, no saas +# proprietary - default build, no saas +# saas - proprietary + the saas subproject (build + JUnit only, +# never any runtime/integration testing) on: workflow_call: @@ -25,7 +31,7 @@ jobs: fail-fast: false matrix: jdk-version: [25] - spring-security: [true, false] + flavor: [core, proprietary, saas] steps: - name: Harden Runner uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 @@ -58,7 +64,10 @@ jobs: - name: Install Task uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0 - name: Check Java formatting (Spotless) - if: matrix.jdk-version == 25 && matrix.spring-security == false + # Runs once per matrix combination - pick the cheapest leg + # (core - no proprietary, no saas) so we don't wait for the + # heavier flavors just to fail formatting. + if: matrix.jdk-version == 25 && matrix.flavor == 'core' id: spotless-check run: task backend:format:check continue-on-error: true @@ -143,17 +152,24 @@ jobs: }); } - - name: Build with Gradle and spring security ${{ matrix.spring-security }} + - name: Build with Gradle (flavor=${{ matrix.flavor }}) + # STIRLING_FLAVOR is read by settings.gradle and expands into the + # right combination of DISABLE_ADDITIONAL_FEATURES + ENABLE_SAAS + # so we don't have to set them by hand. The saas flavor pulls in + # the app/saas subproject (unit tests only - no runtime tests). run: task backend:build:ci env: MAVEN_USER: ${{ secrets.MAVEN_USER }} MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }} - DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }} + STIRLING_FLAVOR: ${{ matrix.flavor }} - name: Check Test Reports Exist if: always() run: | + # Common + core + proprietary always build (proprietary is + # excluded only at runtime, not from the gradle subproject + # graph). Saas builds add a fourth report dir. declare -a dirs=( "app/core/build/reports/tests/" "app/core/build/test-results/" @@ -162,6 +178,9 @@ jobs: "app/proprietary/build/reports/tests/" "app/proprietary/build/test-results/" ) + if [ "${{ matrix.flavor }}" = "saas" ]; then + dirs+=("app/saas/build/reports/tests/" "app/saas/build/test-results/") + fi for dir in "${dirs[@]}"; do if [ ! -d "$dir" ]; then echo "Missing $dir" @@ -173,7 +192,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }} + name: test-reports-jdk-${{ matrix.jdk-version }}-flavor-${{ matrix.flavor }} path: | app/**/build/reports/jacoco/test app/**/build/reports/tests/ @@ -183,7 +202,47 @@ jobs: retention-days: 3 if-no-files-found: warn - - name: Add coverage to PR with spring security ${{ matrix.spring-security }} and JDK ${{ matrix.jdk-version }} + - name: Install defusedxml for coverage summary + # coverage-summary.py parses JaCoCo XML through defusedxml to + # silence security scanners that pattern-match on the stdlib + # xml.etree.ElementTree.parse call. + if: always() && matrix.flavor == 'saas' + run: python -m pip install --quiet defusedxml + + - name: JaCoCo coverage step summary + # Only the saas leg posts the JUnit summary - it's a strict + # superset of the core + proprietary legs (same .exec files plus + # the saas subproject). Posting from all three would mean three + # near-identical tables crowding out the aggregate report. + if: always() && matrix.flavor == 'saas' + run: | + python scripts/coverage-summary.py \ + --title "Backend JUnit coverage (JDK ${{ matrix.jdk-version }})" \ + --jacoco "common=app/common/build/reports/jacoco/test/jacocoTestReport.xml" \ + --jacoco "core=app/core/build/reports/jacoco/test/jacocoTestReport.xml" \ + --jacoco "proprietary=app/proprietary/build/reports/jacoco/test/jacocoTestReport.xml" \ + --jacoco "saas=app/saas/build/reports/jacoco/test/jacocoTestReport.xml" \ + --github-step-summary + + - name: Upload raw JUnit .exec for aggregate merge + # Same dedup rationale as the summary step: upload from the saas + # leg only (the most complete set, includes app/saas/.../test.exec) + # so the aggregate workflow merges the union rather than three + # overlapping subsets. + # + # Separate artifact from the HTML reports so the aggregate + # workflow can grab just the .exec files with a name pattern + # (`jacoco-exec-*`) instead of unpacking the whole test-reports + # tarball. + if: always() && matrix.flavor == 'saas' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-exec-junit-jdk-${{ matrix.jdk-version }} + path: app/*/build/jacoco/*.exec + retention-days: 7 + if-no-files-found: warn + + - name: Add coverage to PR (flavor=${{ matrix.flavor }}, JDK=${{ matrix.jdk-version }}) # The action only supports the pull_request event (it posts a PR comment), # so skip it for merge_group runs and workflow_dispatch. if: github.event_name == 'pull_request' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac47fe134..7018a3812 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -185,6 +185,24 @@ jobs: uses: ./.github/workflows/dependency-review.yml secrets: inherit + # Coverage aggregate: merges the JUnit + e2e:live + cucumber .exec + # artifacts produced by the jobs above into one report, plus pulls + # in vitest + Playwright frontend coverage for the per-area matrix. + # `if: always()` so a producer failing partway still gets credit + # for whatever did record. Advisory only - intentionally NOT in + # all-checks-passed, so a flaky aggregate run never blocks merging. + coverage-aggregate: + if: always() + needs: + - build + - playwright-e2e-live + - docker-compose-tests + - frontend-validation + permissions: + contents: read + uses: ./.github/workflows/coverage-aggregate.yml + secrets: inherit + # Single status check that branch protection should mark as required. # Succeeds when every upstream job is either `success` or `skipped` (path- # gated jobs that didn't apply this run). Any `failure` or `cancelled` diff --git a/.github/workflows/coverage-aggregate.yml b/.github/workflows/coverage-aggregate.yml new file mode 100644 index 000000000..35c252c4b --- /dev/null +++ b/.github/workflows/coverage-aggregate.yml @@ -0,0 +1,230 @@ +name: Aggregate backend coverage + +# Reusable workflow called from build.yml after every backend coverage +# producer (JUnit, e2e:live, cucumber) has run. Downloads each job's raw +# .exec, merges them into one JaCoCo report, and posts a combined step +# summary alongside the per-source ones. +# +# Kept separate from the per-source jobs so: +# - the per-source jobs stay fast and independent (no cross-job waits) +# - this job can `if: always()` and still produce something useful when +# one of the producers fails partway through +# - frontend producers can be added later without touching the +# producers themselves +on: + workflow_call: + +permissions: + contents: read + +jobs: + pick: + uses: ./.github/workflows/_runner-pick.yml + + aggregate: + needs: pick + runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }} + timeout-minutes: 15 + steps: + - name: Harden Runner + uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3 + with: + egress-policy: audit + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up JDK 25 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: "25" + distribution: "temurin" + + - name: Cache Gradle dependency artifacts + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.gradle/wrapper + ~/.gradle/caches/modules-2/files-2.1 + ~/.gradle/caches/modules-2/metadata-2.* + key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 + with: + gradle-version: 9.3.1 + cache-disabled: true + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install defusedxml for coverage scripts + # Both coverage-summary.py and coverage-matrix.py parse JaCoCo + # XML through defusedxml - see the script headers for context. + run: python -m pip install --quiet defusedxml + + # Pattern matches every artifact this PR's producers might upload: + # jacoco-exec-junit-jdk-25 (uploaded only by the saas + # leg of backend-build, which + # is a strict superset of the + # core + proprietary legs) + # jacoco-exec-e2e-live + # jacoco-exec-cucumber + # Each lands as a sibling dir under coverage-execs/, with the .exec + # files preserving their original relative paths. + - name: Download all .exec artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0 + with: + pattern: jacoco-exec-* + path: coverage-execs/ + merge-multiple: false + continue-on-error: true + + - name: Inventory .exec files + id: inventory + # Splits the downloaded artifacts into two buckets: + # * e2e-only = cucumber + Playwright live (user-flow coverage) + # * all = the above plus JUnit (everything we test) + # + # Bucketing is by artifact-name prefix: download-artifact preserves + # the artifact name as the top-level dir, so JUnit's `.exec`s live + # under coverage-execs/jacoco-exec-junit-*/... while the others + # are under coverage-execs/jacoco-exec-{e2e-live,cucumber}/... + # + # If nothing was uploaded (e.g. all producers crashed before + # writing) we exit gracefully so this advisory job never fails CI. + run: | + mapfile -t all_execs < <(find coverage-execs -name '*.exec' -type f | sort) + mapfile -t e2e_execs < <(find coverage-execs -name '*.exec' -type f -not -path '*/jacoco-exec-junit-*' | sort) + if [ "${#all_execs[@]}" -eq 0 ]; then + echo "::warning::No .exec artifacts found - skipping aggregate report" + echo "found_all=false" >> "$GITHUB_OUTPUT" + echo "found_e2e=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + printf 'All %d .exec files:\n' "${#all_execs[@]}" + printf ' %s\n' "${all_execs[@]}" + IFS=','; all_joined="${all_execs[*]}" + echo "files_all=$all_joined" >> "$GITHUB_OUTPUT" + echo "found_all=true" >> "$GITHUB_OUTPUT" + if [ "${#e2e_execs[@]}" -eq 0 ]; then + echo "::notice::No e2e/cucumber .exec files - e2e-only report will be skipped" + echo "found_e2e=false" >> "$GITHUB_OUTPUT" + else + printf 'E2E-only %d .exec files:\n' "${#e2e_execs[@]}" + printf ' %s\n' "${e2e_execs[@]}" + unset IFS + IFS=','; e2e_joined="${e2e_execs[*]}" + echo "files_e2e=$e2e_joined" >> "$GITHUB_OUTPUT" + echo "found_e2e=true" >> "$GITHUB_OUTPUT" + fi + + - name: Compile classes for JaCoCo class lookup + # jacocoReportFromExec only needs the compiled .class files + # under each subproject's build/classes/java/main/. `classes` + # (compileJava + processResources) is enough; we skipped the + # heavier `assemble` to avoid building bootJar / fat jars that + # add 60+ seconds per run for no gain to the report. + if: steps.inventory.outputs.found_all == 'true' + run: ./gradlew classes -PnoSpotless + + - name: Generate e2e-only JaCoCo report + # "Real user-flow" coverage: only counts code reached by an actual + # HTTP request from cucumber or live Playwright. Useful for + # questions like "how much of our backend does a user actually + # hit?". Skipped when neither producer uploaded a .exec. + if: steps.inventory.outputs.found_e2e == 'true' + run: | + ./gradlew jacocoReportFromExec \ + -PexecFile="${{ steps.inventory.outputs.files_e2e }}" \ + -PreportDir=build/reports/jacoco/aggregate-e2e \ + -PnoSpotless + + - name: Generate combined JaCoCo report (everything) + if: steps.inventory.outputs.found_all == 'true' + run: | + ./gradlew jacocoReportFromExec \ + -PexecFile="${{ steps.inventory.outputs.files_all }}" \ + -PreportDir=build/reports/jacoco/aggregate-all \ + -PnoSpotless + + - name: E2E-only step summary + # Rendered first so it gets prime real estate in the Summary + # tab - this is the number most readers actually want + # ("how much of the backend do real user flows cover?"). + if: steps.inventory.outputs.found_e2e == 'true' + run: | + python scripts/coverage-summary.py \ + --title "Real user-flow backend coverage (e2e:live + cucumber)" \ + --jacoco "merged=build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml" \ + --github-step-summary + + - name: ALL-sources step summary + # Separate call (not a multi-input one) because the helper's + # rightmost "Aggregate" column would sum the two reports - which + # is meaningless when one is a strict superset of the other. + if: steps.inventory.outputs.found_all == 'true' + run: | + python scripts/coverage-summary.py \ + --title "Combined backend coverage (JUnit + e2e:live + cucumber)" \ + --jacoco "merged=build/reports/jacoco/aggregate-all/jacocoTestReport.xml" \ + --github-step-summary + + - name: Upload combined aggregate report + if: steps.inventory.outputs.found_all == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-aggregate-all-${{ github.run_id }} + path: build/reports/jacoco/aggregate-all/ + retention-days: 14 + + - name: Upload e2e-only aggregate report + if: steps.inventory.outputs.found_e2e == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-aggregate-e2e-${{ github.run_id }} + path: build/reports/jacoco/aggregate-e2e/ + retention-days: 14 + + # -------------------------------------------------------------- + # Per-area matrix: rolls backend + frontend coverage into one + # table indexed by core/proprietary/saas/desktop. Pulls the + # frontend artifacts now (after the JaCoCo step has done its + # work) so the per-source backend summaries still render first + # even if the matrix step fails. + # -------------------------------------------------------------- + - name: Download vitest coverage artifact + # frontend-validation uploads as `frontend-coverage`. Tolerate + # absence so a backend-only PR still produces the matrix with + # just backend rows populated. + if: always() + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0 + with: + name: frontend-coverage + path: matrix-inputs/vitest/ + continue-on-error: true + + - name: Download Playwright frontend coverage artifact + # e2e-live uploads as `playwright-frontend-coverage-`. + # Same tolerance as vitest - matrix script handles missing inputs. + if: always() + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0 + with: + name: playwright-frontend-coverage-${{ github.run_id }} + path: matrix-inputs/playwright/ + continue-on-error: true + + - name: Coverage matrix step summary + if: always() + # Matrix references the two aggregate JaCoCo XMLs (already + # generated above) plus whichever frontend artifacts landed. + # Every input is optional; missing ones render as "-". + run: | + python scripts/coverage-matrix.py \ + ${{ steps.inventory.outputs.found_all == 'true' && '--jacoco-all build/reports/jacoco/aggregate-all/jacocoTestReport.xml' || '' }} \ + ${{ steps.inventory.outputs.found_e2e == 'true' && '--jacoco-e2e build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml' || '' }} \ + --vitest matrix-inputs/vitest/coverage-summary.json \ + --playwright-frontend matrix-inputs/playwright/coverage-pw-summary/coverage-summary.json \ + --title "Coverage matrix (per-area, e2e vs all)" \ + --github-step-summary diff --git a/.github/workflows/docker-compose-tests.yml b/.github/workflows/docker-compose-tests.yml index 80f472fac..69031fed5 100644 --- a/.github/workflows/docker-compose-tests.yml +++ b/.github/workflows/docker-compose-tests.yml @@ -87,6 +87,12 @@ jobs: run: | pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt + - name: Extract JaCoCo agent for cucumber coverage + # Stages build/jacoco/jacocoagent.jar where the coverage override + # file bind-mounts it into the cucumber container. The agent jar + # never goes into the published image - this is host-only. + run: ./gradlew copyJacocoAgent -PnoSpotless + - name: Run Docker Compose Tests run: | chmod +x ./testing/test_webpages.sh @@ -98,6 +104,62 @@ jobs: MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }} DOCKER_BASE_CHANGED: ${{ inputs.docker-base-changed }} + # Tells test.sh to layer testing/compose/docker-compose-coverage.override.yml + # over the cucumber compose so the container starts with the + # JaCoCo agent attached via JAVA_CUSTOM_OPTS. + STIRLING_PDF_TEST_COVERAGE: "1" + + - name: Generate cucumber JaCoCo report + # `if: always()` so a behave failure still produces partial + # coverage from whatever endpoints did run. The exec file only + # exists when the container shut down cleanly - guard so the step + # is silent on the (rare) crash path. + if: always() + id: cucumber-coverage + run: | + if [ -s testing/cucumber-coverage/cucumber.exec ]; then + ./gradlew jacocoReportFromExec \ + -PexecFile=testing/cucumber-coverage/cucumber.exec \ + -PreportDir=build/reports/jacoco/cucumber \ + -PnoSpotless + echo "report=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::No cucumber .exec at testing/cucumber-coverage/cucumber.exec (container may have crashed before flushing)" + echo "report=false" >> "$GITHUB_OUTPUT" + fi + + - name: Install defusedxml for coverage summary + # coverage-summary.py parses JaCoCo XML through defusedxml - + # see the script header for context. + if: always() && steps.cucumber-coverage.outputs.report == 'true' + run: python -m pip install --quiet defusedxml + + - name: Cucumber coverage step summary + if: always() && steps.cucumber-coverage.outputs.report == 'true' + run: | + python scripts/coverage-summary.py \ + --title "Cucumber (docker) JaCoCo coverage" \ + --jacoco "cucumber=build/reports/jacoco/cucumber/jacocoTestReport.xml" \ + --github-step-summary + + - name: Upload cucumber JaCoCo report + if: always() && steps.cucumber-coverage.outputs.report == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-cucumber-${{ github.run_id }} + path: build/reports/jacoco/cucumber/ + retention-days: 7 + + - name: Upload raw cucumber .exec for aggregate merge + # Picked up by the coverage-aggregate workflow via the + # `jacoco-exec-*` artifact name pattern. + if: always() && steps.cucumber-coverage.outputs.report == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-exec-cucumber + path: testing/cucumber-coverage/cucumber.exec + retention-days: 7 + if-no-files-found: warn - name: Upload Cucumber Report if: always() diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index e367267ec..ec46a19f9 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -49,9 +49,132 @@ jobs: env: VITE_BUILD_FOR_PREVIEW: "1" run: task frontend:build - - name: Run live E2E tests (chromium) + - name: Run live E2E tests (chromium) with coverage id: live-tests + env: + # Attaches the JaCoCo agent to the bootRun JVM (see + # .taskfiles/e2e.yml live:backend). The .exec gets flushed on + # graceful shutdown when the runner traps EXIT/INT/TERM, so the + # report step below sees a populated file. + COVERAGE: "1" + # Tells the Playwright fixture (test-base.ts) to capture per-test + # V8 JS coverage. Raw dumps land under + # .test-state/playwright/coverage-pw/ for the post-process step + # to aggregate. Chromium-only - other engines silently skip. + PW_COVERAGE: "1" run: task e2e:live + - name: Generate JaCoCo report from e2e:live .exec + if: always() + id: live-coverage + # `if: always()` so even a failed test run still produces a + # report from whatever flows did exercise the backend before + # the failure. The task itself tolerates a missing .exec + # (jacoco emits an empty report rather than crashing) but we + # guard with `test -s` to keep the job log clean. + run: | + if [ -s .test-state/playwright/jacoco.exec ]; then + ./gradlew jacocoReportFromExec \ + -PexecFile=.test-state/playwright/jacoco.exec \ + -PreportDir=build/reports/jacoco/e2e-live \ + -PnoSpotless + echo "report=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::No e2e:live .exec found at .test-state/playwright/jacoco.exec; skipping report" + echo "report=false" >> "$GITHUB_OUTPUT" + fi + - name: Set up Python for coverage summary + if: always() && steps.live-coverage.outputs.report == 'true' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + - name: Install defusedxml for coverage summary + # coverage-summary.py uses defusedxml instead of stdlib xml.etree + # to dodge XXE / billion-laughs scanner findings. + if: always() && steps.live-coverage.outputs.report == 'true' + run: python -m pip install --quiet defusedxml + - name: e2e:live coverage step summary + if: always() && steps.live-coverage.outputs.report == 'true' + run: | + python scripts/coverage-summary.py \ + --title "Playwright (live backend) JaCoCo coverage" \ + --jacoco "e2e-live=build/reports/jacoco/e2e-live/jacocoTestReport.xml" \ + --github-step-summary + - name: Upload e2e:live JaCoCo report + if: always() && steps.live-coverage.outputs.report == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-e2e-live-${{ github.run_id }} + path: build/reports/jacoco/e2e-live/ + retention-days: 7 + + - name: Upload raw e2e:live .exec for aggregate merge + # Picked up by the coverage-aggregate workflow via the + # `jacoco-exec-*` artifact name pattern. + if: always() && steps.live-coverage.outputs.report == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: jacoco-exec-e2e-live + path: .test-state/playwright/jacoco.exec + retention-days: 7 + if-no-files-found: warn + + - name: Set up Python for frontend coverage summary + # Separate from the backend-coverage python step because the + # frontend path doesn't depend on a JaCoCo report - it produces + # a summary even on backend failure, as long as some Playwright + # tests ran far enough to dump V8 coverage. + if: always() + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install defusedxml for frontend coverage summary + # Idempotent re-install: the backend-coverage step may have + # installed it already, but this leg can run on its own when the + # backend report step skips (e.g. .exec missing). + if: always() + run: python -m pip install --quiet defusedxml + + - name: Aggregate Playwright frontend (V8) coverage + # Rolls per-test V8 dumps from the test-base fixture into one + # vitest-shaped coverage-summary.json. Tolerates a missing dump + # dir (firefox/webkit runs, or a failure before any test got + # far enough to dump). + if: always() + id: pw-frontend-coverage + run: | + if [ -d .test-state/playwright/coverage-pw ] && \ + find .test-state/playwright/coverage-pw -name '*.json' -type f | grep -q .; then + python scripts/playwright-coverage-summary.py \ + .test-state/playwright/coverage-pw \ + --out .test-state/playwright/coverage-pw-summary/coverage-summary.json + echo "summary=true" >> "$GITHUB_OUTPUT" + else + echo "::notice::No Playwright frontend coverage dumps found (chromium-only feature)" + echo "summary=false" >> "$GITHUB_OUTPUT" + fi + + - name: Playwright frontend coverage step summary + if: always() && steps.pw-frontend-coverage.outputs.summary == 'true' + run: | + python scripts/coverage-summary.py \ + --title "Playwright (live) frontend coverage" \ + --vitest .test-state/playwright/coverage-pw-summary/coverage-summary.json \ + --github-step-summary + + - name: Upload Playwright frontend coverage + # Bundle both the aggregated summary and the raw V8 dumps so + # someone debugging "why is this function showing as covered" + # can trace it back to the source dump. + if: always() && steps.pw-frontend-coverage.outputs.summary == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-frontend-coverage-${{ github.run_id }} + path: | + .test-state/playwright/coverage-pw-summary/ + .test-state/playwright/coverage-pw/ + retention-days: 7 + - name: Print backend log on failure if: failure() && steps.live-tests.conclusion == 'failure' run: | diff --git a/.github/workflows/frontend-validation.yml b/.github/workflows/frontend-validation.yml index b5cd538d0..1e1068e22 100644 --- a/.github/workflows/frontend-validation.yml +++ b/.github/workflows/frontend-validation.yml @@ -109,6 +109,41 @@ jobs: comment_id: existing.id, }); } + - name: Vitest coverage + # Separate from `frontend:check:all` so the quality-gate run stays + # uninstrumented (faster signal) and coverage stays an informational + # follow-up. Continue-on-error keeps the workflow green even when + # a handful of test files refuse to import (e.g. missing icon + # specifiers) - the summary still gets posted with whatever + # vitest managed to instrument. + id: frontend-coverage + continue-on-error: true + run: task frontend:test:coverage + - name: Set up Python for coverage summary + if: always() + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + - name: Install defusedxml for coverage summary + # See coverage-summary.py header - it parses XML through defusedxml + # to dodge the stdlib parser's exposure to XXE / billion-laughs. + if: always() + run: python -m pip install --quiet defusedxml + - name: Vitest coverage step summary + if: always() + run: | + python scripts/coverage-summary.py \ + --title "Frontend Vitest coverage" \ + --vitest frontend/editor/coverage/coverage-summary.json \ + --github-step-summary + - name: Upload vitest coverage report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: frontend-coverage + path: frontend/editor/coverage/ + retention-days: 7 + if-no-files-found: warn - name: Upload frontend build artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.taskfiles/e2e.yml b/.taskfiles/e2e.yml index 82462e810..825b7cb69 100644 --- a/.taskfiles/e2e.yml +++ b/.taskfiles/e2e.yml @@ -34,6 +34,9 @@ tasks: ignore_error: true vars: BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright' + # COVERAGE=1 in the calling environment attaches the JaCoCo agent to + # the bootRun JVM and writes to BASE_DIR/jacoco.exec on shutdown. + # Off by default to keep local dev runs uninstrumented; CI flips it. env: STIRLING_BASE_PATH: '{{.BASE_DIR}}' # Suppress the analytics opt-in modal that fires on first admin login. @@ -58,12 +61,19 @@ tasks: set -e rm -rf "{{.BASE_DIR}}" mkdir -p "{{.BASE_DIR}}" + GRADLE_ARGS=":stirling-pdf:bootRun" + if [ -n "${COVERAGE:-}" ]; then + # copyJacocoAgent is wired as a dependency of bootRun when + # -PjacocoAgent=true, so we do not need to invoke it separately. + GRADLE_ARGS="$GRADLE_ARGS -PjacocoAgent=true -PjacocoExec={{.BASE_DIR}}/jacoco.exec" + echo "JaCoCo coverage enabled, writing to {{.BASE_DIR}}/jacoco.exec" + fi # Background gradle and record its PID so the runner can clean up # the exact process tree (wrapper + forked Spring Boot JVM) without # resorting to fuzzy `pkill -f` patterns. `wait` keeps this script # alive for the lifetime of gradle so Task'"'"'s parallel deps stay # synchronised. - bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 & + bash gradlew $GRADLE_ARGS > "{{.BASE_DIR}}/backend.log" 2>&1 & GRADLE_PID=$! echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid" wait $GRADLE_PID diff --git a/.taskfiles/frontend.yml b/.taskfiles/frontend.yml index 8c3fbc617..9e6f87889 100644 --- a/.taskfiles/frontend.yml +++ b/.taskfiles/frontend.yml @@ -284,10 +284,25 @@ tasks: - npx vitest --watch --root editor test:coverage: - desc: "Run tests with coverage" + desc: "Run tests with coverage (one-shot; CI-friendly)." deps: [install] cmds: - - npx vitest --coverage --root editor + # `vitest run` makes this CI-safe (the bare `vitest` form enters watch + # mode). Explicit reporter list because v8 + json-summary is what the + # coverage-summary.py helper consumes; html/text are kept for humans. + # + # reportsDirectory is pinned to ./coverage relative to vitest's root + # (--root editor), so output lands at frontend/editor/coverage/. The + # CI upload step reads from that path. An earlier attempt with + # `./editor/coverage` double-nested into frontend/editor/editor/coverage; + # pinning future-proofs against vitest changing the default. + - > + npx vitest run --root editor --coverage + --coverage.provider=v8 + --coverage.reporter=text-summary + --coverage.reporter=json-summary + --coverage.reporter=html + --coverage.reportsDirectory=./coverage # ============================================================ # Code Generation diff --git a/build.gradle b/build.gradle index b8fa600ea..3f78bc635 100644 --- a/build.gradle +++ b/build.gradle @@ -438,7 +438,7 @@ subprojects { } tasks.named("bootRun") { - jvmArgs = [ + def runtimeArgs = [ "-XX:+UseG1GC", "-XX:MaxGCPauseMillis=200", "-XX:G1HeapRegionSize=4m", @@ -447,6 +447,53 @@ subprojects { "-XX:+UseCompactObjectHeaders", "--enable-native-access=ALL-UNNAMED" ] + + // Optional JaCoCo agent for e2e/cucumber backend coverage. + // + // Enabled with `-PjacocoAgent=true`. The default destfile lives + // outside the source tree so it doesn't poison a normal bootRun + // for a developer who forgot to pass the property. Override + // both with `-PjacocoExec=/path/to.exec` if you need a custom + // location (e.g. when sharing it with the Playwright runner). + if (rootProject.hasProperty('jacocoAgent') && + rootProject.property('jacocoAgent').toString() == 'true') { + def agentJar = rootProject.layout.buildDirectory + .file('jacoco/jacocoagent.jar').get().asFile.absolutePath + def execFile = (rootProject.findProperty('jacocoExec') ?: + rootProject.layout.projectDirectory + .file('.test-state/playwright/jacoco.exec').asFile.absolutePath + ).toString() + // The JaCoCo agent argument is a comma-separated key=value list, + // so a path that itself contains ',' or '=' would smuggle extra + // agent options (e.g. -PjacocoExec='out.exec,sessionid=evil'). + // Today the value only comes from our own CI workflows, but + // validating defensively is cheap and silences Aikido. + // + // Control characters are checked via String.contains rather + // than inside a slashy regex character class - a literal NULL + // accidentally landing in the regex killed Groovy parsing on + // the previous attempt. + def hasBadChar = execFile.find(/[,= ]/) != null || + execFile.contains('\r') || + execFile.contains('\n') || + execFile.contains('\t') + if (hasBadChar) { + throw new GradleException( + "jacocoExec='" + execFile + "' contains characters " + + "(',', '=', whitespace, or control chars) that " + + "would break -javaagent option parsing. Choose " + + "a different path." + ) + } + runtimeArgs.add("-javaagent:${agentJar}=destfile=${execFile},output=file,append=false,dumponexit=true") + dependsOn(rootProject.tasks.named('copyJacocoAgent')) + doFirst { + new File(execFile).parentFile?.mkdirs() + logger.lifecycle("JaCoCo agent attached: ${execFile}") + } + } + + jvmArgs = runtimeArgs } } } @@ -627,3 +674,80 @@ tasks.withType(Test).configureEach { // Half of available CPUs is a safe default; bump if your tests are I/O-bound. maxParallelForks = Math.max(1, (Runtime.runtime.availableProcessors().intdiv(2)) as int) } + +// ---------------------------------------------------------------------------- +// JaCoCo helpers used by CI for cucumber + Playwright (live backend) coverage. +// +// These let CI attach the JaCoCo agent to a running Spring Boot process +// (gradle bootRun or the docker image, via JAVA_TOOL_OPTIONS) and then turn +// the resulting .exec dump back into HTML + XML reports without anyone +// hand-installing the JaCoCo CLI. +// ---------------------------------------------------------------------------- + +configurations { + jacocoRuntimeAgent + jacocoCli +} + +dependencies { + // The :runtime classifier on org.jacoco.agent IS the agent jar - no + // unzipping required. Kept on the version the plugin already picks so + // agent + reporter line up exactly. + jacocoRuntimeAgent "org.jacoco:org.jacoco.agent:${jacoco.toolVersion}:runtime" + jacocoCli "org.jacoco:org.jacoco.cli:${jacoco.toolVersion}" +} + +tasks.register('copyJacocoAgent', Copy) { + group = 'verification' + description = 'Copies the JaCoCo runtime agent jar to build/jacoco/jacocoagent.jar.' + from configurations.jacocoRuntimeAgent + into layout.buildDirectory.dir('jacoco') + rename { 'jacocoagent.jar' } +} + +// Aggregates an externally-produced .exec (e.g. from e2e:live or the cucumber +// docker container) against this project's compiled classes + sources. +// +// Inputs are configured via -P properties so the same task works for every +// caller: +// +// ./gradlew jacocoReportFromExec -PexecFile=.test-state/playwright/jacoco.exec \ +// -PreportDir=build/reports/jacoco/e2e-live +tasks.register('jacocoReportFromExec', JacocoReport) { + group = 'verification' + description = 'Generates a JaCoCo HTML+XML report from an externally captured .exec.' + + def execProp = project.findProperty('execFile') ?: project.findProperty('execFiles') + def reportProp = project.findProperty('reportDir') ?: "build/reports/jacoco/external" + + executionData fileTree(rootProject.rootDir) { + if (execProp) { + include execProp.toString().split(',').collect { it.trim() } + } else { + // Sensible defaults so the task is usable without props. + include '.test-state/**/*.exec' + include 'coverage-tools/exec/*.exec' + include 'testing/cucumber-coverage/*.exec' + } + } + + // Pull compiled classes + sources from every subproject so the report is + // an aggregate. JaCoCo silently skips classes that have no matching .exec + // probes, so this is safe even when only a subset of code was exercised. + classDirectories.setFrom(files(subprojects.collect { sub -> + sub.fileTree(dir: "${sub.buildDir}/classes/java/main", excludes: [ + '**/generated/**', + ]) + })) + sourceDirectories.setFrom(files(subprojects.collect { sub -> + "${sub.projectDir}/src/main/java" + })) + + reports { + xml.required.set(true) + html.required.set(true) + csv.required.set(false) + xml.outputLocation.set(layout.projectDirectory.file("${reportProp}/jacocoTestReport.xml")) + html.outputLocation.set(layout.projectDirectory.dir("${reportProp}/html")) + } +} diff --git a/frontend/editor/src/core/tests/helpers/test-base.ts b/frontend/editor/src/core/tests/helpers/test-base.ts index e9ff8caa8..1659bc3ab 100644 --- a/frontend/editor/src/core/tests/helpers/test-base.ts +++ b/frontend/editor/src/core/tests/helpers/test-base.ts @@ -1,17 +1,45 @@ import { test as base, expect } from "@playwright/test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; /** - * Custom test fixture that auto-dismisses the cookie consent banner - * before every test. The banner (#cc-main) overlays the page and - * intercepts pointer events, causing click timeouts across all tests. + * Custom test fixture that: + * 1. auto-dismisses the cookie consent banner (the banner overlays + * the page and intercepts pointer events, causing click timeouts + * across all tests); + * 2. optionally captures V8 JS coverage per test when PW_COVERAGE=1. + * + * Coverage notes: + * - Only chromium supports page.coverage.* - the try/catch silently + * skips collection on firefox/webkit so cross-browser runs don't + * fail when coverage is enabled. + * - Raw V8 dumps land under .test-state/playwright/coverage-pw/ as + * -.json. CI post-processes them with + * scripts/playwright-coverage-summary.py into a vitest-shaped + * coverage-summary.json that the existing helper renders. + * - resetOnNavigation: false so navigation between pages inside the + * same test (login → tool page → preview) accumulates instead of + * starting fresh. * * Usage: import { test, expect } from '@app/tests/helpers/test-base'; */ +const COVERAGE_ENABLED = process.env.PW_COVERAGE === "1"; + +// Path is workspace-relative because Playwright cwd is frontend/editor/ +// at runtime (defined by playwright.config.ts). Going up two levels +// lands at the repo root - the same `.test-state/playwright/` that +// `task e2e:live` already provisions. +const COVERAGE_DIR = path.resolve( + process.cwd(), + "..", + "..", + ".test-state", + "playwright", + "coverage-pw", +); + export const test = base.extend({ - page: async ({ page }, use) => { - // Set the cookie consent cookie before any navigation so the banner - // never appears. The cookieconsent library (orestbida/cookieconsent) - // reads this cookie on init and skips the banner if consent exists. + page: async ({ page }, use, testInfo) => { await page.context().addCookies([ { name: "cc_cookie", @@ -28,7 +56,37 @@ export const test = base.extend({ }, ]); + if (COVERAGE_ENABLED) { + try { + await page.coverage.startJSCoverage({ resetOnNavigation: false }); + } catch { + // Browser doesn't support V8 coverage (firefox/webkit) - silently skip. + } + } + await use(page); + + if (COVERAGE_ENABLED) { + try { + const entries = await page.coverage.stopJSCoverage(); + if (entries.length > 0) { + await fs.mkdir(COVERAGE_DIR, { recursive: true }); + // Sanitize the path so it survives every filesystem we run on. + const safeTitle = testInfo.titlePath + .join("-") + .replace(/[^A-Za-z0-9._-]/g, "_") + .slice(0, 120); + const outPath = path.join( + COVERAGE_DIR, + `${safeTitle}-${testInfo.testId}.json`, + ); + await fs.writeFile(outPath, JSON.stringify(entries)); + } + } catch { + // Browser closed or coverage unsupported - swallow so a coverage + // failure never fails the underlying test. + } + } }, }); diff --git a/scripts/coverage-matrix.py b/scripts/coverage-matrix.py new file mode 100644 index 000000000..e0745f078 --- /dev/null +++ b/scripts/coverage-matrix.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""Render a single coverage matrix combining backend + frontend, per area. + +Rows: + - Frontend (e2e only) - Playwright fixture V8 capture + - Frontend (all) - Vitest unit tests + Playwright V8 + - Backend (e2e only) - Playwright live JaCoCo + cucumber JaCoCo + - Backend (all) - JUnit + e2e:live + cucumber JaCoCo + - Both (e2e only) - frontend (e2e) + backend (e2e), summed + - Both (all) - frontend (all) + backend (all), summed + +Columns: + - core, proprietary, saas, desktop - bucketed per area + - ALL - sum across the row + +Bucketing rules: + Backend (JaCoCo package -> area): + stirling/software/SPDF/** -> core (the main backend module) + stirling/software/common/** -> core (shared infra, attributed to core) + org/apache/pdfbox/** -> core (vendored helpers in common/core) + stirling/software/proprietary/** -> proprietary (unless saas-flavoured) + stirling/software/saas/** -> saas + *desktop* -> desktop (no real backend match today) + Frontend (source path -> area): + frontend/editor/src/core/** -> core + frontend/editor/src/proprietary/** -> proprietary + frontend/editor/src/saas/** -> saas + frontend/editor/src/desktop/** -> desktop + +Cells render as `pct% (covered/total)` where the metric is: + - Backend: JaCoCo METHOD counter (most directly comparable to JS funcs) + - Frontend: function counts from vitest per-file summary + +Playwright (live) frontend V8 dumps only know about bundled JS URLs, not +source paths, so they only feed the ALL column for the Frontend rows. +Per-area Playwright contributions show "n/a" with a footnote. + +Missing inputs are tolerated: any source that isn't passed renders as `-` +and the row aggregates from what is available. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# defusedxml hardens the parser against XXE / billion-laughs / entity- +# expansion attacks. JaCoCo XML on a CI runner is trusted input today, +# but using the hardened parser is a one-line change and silences +# scanners that pattern-match on `xml.etree.ElementTree.parse`. +from defusedxml.ElementTree import parse as _xml_parse +from defusedxml.ElementTree import ParseError as _XMLParseError + +AREAS = ("core", "proprietary", "saas", "desktop") + + +@dataclass +class Bucket: + """Covered + total counters that can be combined across sources.""" + + covered: int = 0 + total: int = 0 + + @property + def pct(self) -> float: + return 100.0 * self.covered / self.total if self.total else 0.0 + + def add(self, other: "Bucket") -> None: + self.covered += other.covered + self.total += other.total + + +@dataclass +class RowBuckets: + """Per-area buckets for one row of the matrix.""" + + by_area: dict[str, Bucket] = field( + default_factory=lambda: {a: Bucket() for a in AREAS} + ) + # Some inputs (Playwright V8) don't have source-path info, so they + # only contribute to ALL without an area attribution. Track those + # separately so per-area cells stay honest. + unattributed: Bucket = field(default_factory=Bucket) + + @property + def all(self) -> Bucket: + agg = Bucket() + for b in self.by_area.values(): + agg.add(b) + agg.add(self.unattributed) + return agg + + def merge(self, other: "RowBuckets") -> None: + for area in AREAS: + self.by_area[area].add(other.by_area[area]) + self.unattributed.add(other.unattributed) + + +# --------------------------------------------------------------------- jacoco + + +def _classify_backend(package_name: str) -> Optional[str]: + """Map a JaCoCo package name to an area, or None to skip.""" + if not package_name: + return None + p = package_name.replace("/", ".") + if "desktop" in p: + return "desktop" + if p.startswith("stirling.software.saas"): + return "saas" + # Proprietary saas-flavoured sub-package: rare, but kept for forward + # compatibility if a future build moves saas under proprietary. + if p.startswith("stirling.software.proprietary") and ".saas" in p: + return "saas" + if p.startswith("stirling.software.proprietary"): + return "proprietary" + if ( + p.startswith("stirling.software.SPDF") + or p.startswith("stirling.software.common") + or p.startswith("org.apache.pdfbox") + ): + return "core" + return None + + +def parse_jacoco_methods(path: Path) -> RowBuckets: + row = RowBuckets() + if not path.exists(): + return row + try: + root = _xml_parse(path).getroot() + except _XMLParseError as exc: + print(f"::warning::Failed to parse {path}: {exc}", file=sys.stderr) + return row + for pkg in root.findall("package"): + area = _classify_backend(pkg.get("name", "")) + if area is None: + continue + for counter in pkg.findall("counter"): + if counter.get("type") != "METHOD": + continue + covered = int(counter.get("covered") or 0) + missed = int(counter.get("missed") or 0) + row.by_area[area].add(Bucket(covered=covered, total=covered + missed)) + return row + + +# --------------------------------------------------------------- vitest (frontend) + + +def _classify_frontend(file_path: str) -> Optional[str]: + """Map a vitest per-file path (anything containing src//) to an area.""" + if not file_path: + return None + norm = file_path.replace("\\", "/") + for area in AREAS: + if f"/src/{area}/" in norm: + return area + return None + + +def parse_vitest_per_file(path: Path) -> RowBuckets: + row = RowBuckets() + if not path.exists(): + return row + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print( + f"::warning::Failed to parse vitest summary {path}: {exc}", file=sys.stderr + ) + return row + for file_path, metrics in data.items(): + if file_path == "total": + continue + area = _classify_frontend(file_path) + if area is None: + continue + fn = (metrics or {}).get("functions") or {} + covered = int(fn.get("covered") or 0) + total = int(fn.get("total") or 0) + row.by_area[area].add(Bucket(covered=covered, total=total)) + return row + + +# -------------------------------------------------- playwright frontend (V8) + + +def parse_playwright_frontend_total(path: Path) -> RowBuckets: + """Playwright's V8 dump aggregator only knows the grand total. + + Without source maps we can't bucket per area, so the whole number + goes into `unattributed`. The matrix renderer surfaces this in the + ALL column and shows "n/a" in per-area cells with a footnote. + """ + row = RowBuckets() + if not path.exists(): + return row + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print( + f"::warning::Failed to parse Playwright summary {path}: {exc}", + file=sys.stderr, + ) + return row + fn = (data.get("total") or {}).get("functions") or {} + covered = int(fn.get("covered") or 0) + total = int(fn.get("total") or 0) + row.unattributed.add(Bucket(covered=covered, total=total)) + return row + + +# ---------------------------------------------------------------- rendering + + +def _cell(bucket: Bucket) -> str: + if bucket.total == 0: + return "-" + return f"{bucket.pct:.1f}% ({bucket.covered}/{bucket.total})" + + +def render(rows: dict[str, RowBuckets], title: str) -> str: + cols = list(AREAS) + ["ALL"] + header = "| Row | " + " | ".join(c for c in cols) + " |" + sep = "|---" * (len(cols) + 1) + "|" + lines = [f"## {title}", "", header, sep] + for row_label, row in rows.items(): + is_backend_row = row_label.lower().startswith("backend") + cells = [row_label] + for area in AREAS: + b = row.by_area[area] + unattr = row.unattributed + if area == "desktop" and is_backend_row: + cells.append("n/a") + continue + if b.total == 0 and unattr.total > 0: + cells.append("n/a") + else: + cells.append(_cell(b)) + cells.append(_cell(row.all)) + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + + +# ------------------------------------------------------------------- main + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--jacoco-all", type=Path, help="aggregate jacoco-all XML") + parser.add_argument("--jacoco-e2e", type=Path, help="aggregate jacoco-e2e XML") + parser.add_argument("--vitest", type=Path, help="vitest coverage-summary.json") + parser.add_argument( + "--playwright-frontend", + type=Path, + help="Playwright frontend coverage-summary.json (from playwright-coverage-summary.py)", + ) + parser.add_argument("--title", default="Coverage matrix") + parser.add_argument("--out", type=Path) + parser.add_argument("--github-step-summary", action="store_true") + parser.add_argument("--stdout", action="store_true") + args = parser.parse_args(argv) + + # Build each row from its source(s). + fe_e2e = ( + parse_playwright_frontend_total(args.playwright_frontend) + if args.playwright_frontend + else RowBuckets() + ) + fe_unit = parse_vitest_per_file(args.vitest) if args.vitest else RowBuckets() + fe_all = RowBuckets() + fe_all.merge(fe_unit) + fe_all.merge(fe_e2e) + + be_e2e = parse_jacoco_methods(args.jacoco_e2e) if args.jacoco_e2e else RowBuckets() + be_all = parse_jacoco_methods(args.jacoco_all) if args.jacoco_all else RowBuckets() + + both_e2e = RowBuckets() + both_e2e.merge(fe_e2e) + both_e2e.merge(be_e2e) + + both_all = RowBuckets() + both_all.merge(fe_all) + both_all.merge(be_all) + + rows = { + "Frontend (e2e only)": fe_e2e, + "Frontend (all)": fe_all, + "Backend (e2e only)": be_e2e, + "Backend (all)": be_all, + "Both (e2e only)": both_e2e, + "Both (all)": both_all, + } + + body = render(rows, args.title) + "\n" + + # Always write to stdout when no sink is selected so the script + # is useful from the command line. + sinks: list[Path] = [] + if args.out: + sinks.append(args.out) + if args.github_step_summary: + import os + + gh = os.environ.get("GITHUB_STEP_SUMMARY") + if gh: + sinks.append(Path(gh)) + + for sink in sinks: + sink.parent.mkdir(parents=True, exist_ok=True) + with sink.open("a", encoding="utf-8") as handle: + handle.write(body) + + if args.stdout or not sinks: + print(body) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/coverage-summary.py b/scripts/coverage-summary.py new file mode 100644 index 000000000..81460a8a7 --- /dev/null +++ b/scripts/coverage-summary.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Render coverage results as a GitHub-Actions-friendly markdown summary. + +Reads JaCoCo XML reports (one per gradle subproject) and/or a vitest +coverage-summary.json, and writes a single Markdown block to: + + - the path supplied with --out + - $GITHUB_STEP_SUMMARY when --github-step-summary is passed (and the env + var is set) + - stdout when --stdout is passed (default if no other sink is selected) + +Designed to be safe to run when some inputs are missing - missing inputs +become a "skipped" note rather than an error, so the same call can be +shared across multiple CI jobs (some of which only produce one of the +two report types). +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +# `defusedxml` swaps out the stdlib expat parser for one that rejects the +# usual XML attack vectors (XXE / billion laughs / entity expansion). Even +# though JaCoCo XML on a CI runner is trusted input, swapping the parser is +# a one-line change that silences security scanners and costs nothing. +from defusedxml.ElementTree import parse as _xml_parse +from defusedxml.ElementTree import ParseError as _XMLParseError + +JACOCO_COUNTERS = ("LINE", "BRANCH", "METHOD", "CLASS", "INSTRUCTION", "COMPLEXITY") + + +@dataclass +class CounterTotals: + covered: int = 0 + missed: int = 0 + + @property + def total(self) -> int: + return self.covered + self.missed + + @property + def pct(self) -> float: + return 100.0 * self.covered / self.total if self.total else 0.0 + + def add(self, other: "CounterTotals") -> None: + self.covered += other.covered + self.missed += other.missed + + +def _parse_jacoco_xml(path: Path) -> dict[str, CounterTotals]: + """Read top-level elements from a JaCoCo report XML. + + ElementTree's default parser doesn't validate the DTD, so JaCoCo's + `report.dtd` reference is harmless even on offline CI runners. + Counters are direct children of the root. + """ + try: + root = _xml_parse(path).getroot() + except _XMLParseError as exc: + raise RuntimeError(f"Failed to parse {path}: {exc}") from exc + + out: dict[str, CounterTotals] = {} + for counter in root.findall("counter"): + t = counter.get("type") or "" + if t not in JACOCO_COUNTERS: + continue + out[t] = CounterTotals( + covered=int(counter.get("covered") or 0), + missed=int(counter.get("missed") or 0), + ) + return out + + +def _bar(pct: float, width: int = 20) -> str: + """Render a fixed-width ASCII progress bar. Markdown-safe on all consoles.""" + filled = int(round(pct / 100.0 * width)) + return "[" + "#" * filled + "-" * (width - filled) + "]" + + +def render_jacoco(reports: Iterable[tuple[str, Path]]) -> str: + """Render a markdown table from one or more (label, xml_path) pairs.""" + rows: list[tuple[str, dict[str, CounterTotals]]] = [] + aggregate: dict[str, CounterTotals] = {t: CounterTotals() for t in JACOCO_COUNTERS} + missing: list[str] = [] + + for label, path in reports: + if not path.exists(): + missing.append(f"`{label}` ({path})") + continue + per_counter = _parse_jacoco_xml(path) + rows.append((label, per_counter)) + for t in JACOCO_COUNTERS: + if t in per_counter: + aggregate[t].add(per_counter[t]) + + if not rows: + body = "_No JaCoCo reports found._" + if missing: + body += "\n\n
Searched paths\n\n" + body += "\n".join(f"- {m}" for m in missing) + body += "\n\n
" + return body + + lines: list[str] = [] + lines.append( + "| Metric | " + " | ".join(label for label, _ in rows) + " | **Aggregate** |" + ) + lines.append("|---" * (len(rows) + 2) + "|") + + for t in ("LINE", "BRANCH", "METHOD", "CLASS"): + cells = [t.title()] + for _, per_counter in rows: + c = per_counter.get(t) + cells.append(f"{c.pct:.1f}% ({c.covered}/{c.total})" if c else "-") + agg = aggregate[t] + cells.append(f"**{agg.pct:.1f}%** ({agg.covered}/{agg.total})") + lines.append("| " + " | ".join(cells) + " |") + + agg_line = aggregate["LINE"] + lines.append("") + lines.append( + f"Aggregate line coverage: `{_bar(agg_line.pct)}` **{agg_line.pct:.1f}%** " + f"({agg_line.covered} / {agg_line.total} lines covered)" + ) + if missing: + lines.append("") + lines.append("
Skipped reports\n") + for m in missing: + lines.append(f"- {m}") + lines.append("\n
") + return "\n".join(lines) + + +def render_vitest(summary_path: Path) -> str: + """Render a markdown summary from a vitest coverage-summary.json file. + + Note: with @vitest/coverage-v8 + @vitejs/plugin-react-swc the + statements/lines counts in coverage-summary.json's `total` are + unreliable (source maps don't round-trip), so we report functions + and branches as the primary signal and call this out in the text. + """ + if not summary_path.exists(): + return f"_No vitest coverage summary at `{summary_path}`._" + try: + data = json.loads(summary_path.read_text()) + except json.JSONDecodeError as exc: + return f"_Failed to parse `{summary_path}`: {exc}._" + total = data.get("total") or {} + rows = [] + for key in ("functions", "branches", "lines", "statements"): + v = total.get(key) or {} + covered = v.get("covered", 0) + tot = v.get("total", 0) + pct = v.get("pct", 0.0) + rows.append(f"| {key.title()} | {pct:.1f}% | {covered} / {tot} |") + body = ["| Metric | % | Covered / Total |", "|---|---|---|", *rows] + fn = total.get("functions", {}) + fn_pct = fn.get("pct", 0.0) + body.append("") + body.append(f"Function coverage bar: `{_bar(fn_pct)}` **{fn_pct:.1f}%**") + body.append("") + body.append( + "> Lines/Statements use 0 as the denominator-mismatch sentinel when " + "v8 + SWC source maps don't round-trip; trust Functions and Branches " + "as the primary signal." + ) + return "\n".join(body) + + +def _write(sinks: list[Path | None], text: str, *, to_stdout: bool) -> None: + for path in sinks: + if path is None: + continue + path.parent.mkdir(parents=True, exist_ok=True) + # GitHub job summaries are append-only by convention. + with path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + if to_stdout: + print(text) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--jacoco", + action="append", + default=[], + metavar="LABEL=PATH", + help="Add a JaCoCo XML report (can be passed multiple times).", + ) + parser.add_argument( + "--vitest", + type=Path, + default=None, + help="Path to a vitest coverage-summary.json file.", + ) + parser.add_argument( + "--title", + default="Coverage", + help="Heading text for the summary section.", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="Append the rendered markdown to this file as well.", + ) + parser.add_argument( + "--github-step-summary", + action="store_true", + help="Append to $GITHUB_STEP_SUMMARY if set.", + ) + parser.add_argument( + "--stdout", + action="store_true", + help="Also write the rendered markdown to stdout.", + ) + args = parser.parse_args(argv) + + jacoco_reports: list[tuple[str, Path]] = [] + for entry in args.jacoco: + if "=" not in entry: + parser.error(f"--jacoco expects LABEL=PATH, got {entry!r}") + label, path = entry.split("=", 1) + jacoco_reports.append((label.strip(), Path(path.strip()))) + + sections: list[str] = [f"## {args.title}"] + if jacoco_reports: + sections.append("### Backend (JaCoCo)") + sections.append(render_jacoco(jacoco_reports)) + if args.vitest is not None: + # Sub-heading omitted on purpose: the --title argument already + # disambiguates (Vitest vs Playwright vs anything else that emits + # a coverage-summary.json), so a generic header reads cleaner. + sections.append(render_vitest(args.vitest)) + if len(sections) == 1: + sections.append("_No coverage inputs provided._") + + body = "\n\n".join(sections) + "\n" + + sinks: list[Path | None] = [args.out] + if args.github_step_summary: + gh = os.environ.get("GITHUB_STEP_SUMMARY") + sinks.append(Path(gh) if gh else None) + to_stdout = args.stdout or not any(s for s in sinks) + _write(sinks, body, to_stdout=to_stdout) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/playwright-coverage-summary.py b/scripts/playwright-coverage-summary.py new file mode 100644 index 000000000..5ad7fc562 --- /dev/null +++ b/scripts/playwright-coverage-summary.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Aggregate per-test V8 coverage dumps from Playwright into one summary. + +Playwright's test-base fixture writes raw `page.coverage.stopJSCoverage()` +output to a directory (one JSON file per test, when `PW_COVERAGE=1`). +This script walks that directory and produces: + + - coverage-summary.json (vitest-shaped, so the existing + scripts/coverage-summary.py renderer can consume it without changes) + +Method: + - V8 reports per-function coverage as a list of ranges; the first + range covers the whole function body, and a non-zero `count` on + that outer range means the function was entered at least once. + - We deduplicate functions across the merged dumps by (script-url, + startOffset, endOffset). This avoids double-counting a function + that ran in many tests. + - Per-file line coverage requires source maps + a `v8-to-istanbul` + style walker, which is out of scope here - we report + Lines/Statements as 0/0 (the helper renders that as "not computed", + matching how it handles vitest's known v8+SWC degradation). + +Filtering: + - URLs not on the local dev server (e.g. CDN assets, blob: URLs, + chrome-extension: pages) are skipped. + - URLs containing `node_modules` or vite's HMR client are skipped so + the percentage reflects app code, not framework noise. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +# URLs we deliberately don't count: vite client, hot-reload runtime, +# anything served out of node_modules, and any non-http(s) scheme. +SKIP_URL_FRAGMENTS = ( + "/@vite/", + "/@react-refresh", + "/node_modules/", + "/__vite_ping", + "/__open-in-editor", +) + + +def _is_app_url(url: str) -> bool: + if not url: + return False + if not (url.startswith("http://") or url.startswith("https://")): + return False + if any(frag in url for frag in SKIP_URL_FRAGMENTS): + return False + return True + + +def aggregate(dump_dir: Path) -> dict: + files = sorted(dump_dir.glob("*.json")) + if not files: + return { + "tests": 0, + "scripts": 0, + "functions_total": 0, + "functions_covered": 0, + } + + # (url, start, end) -> covered? Set semantics dedup the same function + # appearing in many test dumps. + seen: dict[tuple[str, int, int], bool] = {} + scripts_seen: set[str] = set() + + for path in files: + try: + entries = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + continue + for entry in entries: + url = entry.get("url", "") + if not _is_app_url(url): + continue + scripts_seen.add(url) + for fn in entry.get("functions", []): + ranges = fn.get("ranges") or [] + if not ranges: + continue + outer = ranges[0] + key = ( + url, + int(outer.get("startOffset", 0)), + int(outer.get("endOffset", 0)), + ) + covered = int(outer.get("count", 0)) > 0 + # Promote to covered if any dump exercised it. + seen[key] = seen.get(key, False) or covered + + return { + "tests": len(files), + "scripts": len(scripts_seen), + "functions_total": len(seen), + "functions_covered": sum(1 for v in seen.values() if v), + } + + +def write_vitest_summary(stats: dict, out_path: Path) -> None: + total = stats["functions_total"] + covered = stats["functions_covered"] + pct = 100.0 * covered / total if total else 0.0 + # vitest coverage-summary.json schema. We report the function metric + # under both `functions` and `branches` so the helper script's + # "trust functions/branches" line still applies. Lines/Statements + # left at 0/0 so the renderer's degradation note kicks in. + payload = { + "total": { + "functions": { + "covered": covered, + "total": total, + "pct": pct, + "skipped": 0, + }, + "branches": { + "covered": covered, + "total": total, + "pct": pct, + "skipped": 0, + }, + "lines": {"covered": 0, "total": 0, "pct": 0.0, "skipped": 0}, + "statements": {"covered": 0, "total": 0, "pct": 0.0, "skipped": 0}, + } + } + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(payload, indent=2)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "dump_dir", + type=Path, + help="Directory containing per-test V8 JSON dumps from the Playwright fixture.", + ) + parser.add_argument( + "--out", + type=Path, + required=True, + help="Path to write the vitest-shaped coverage-summary.json", + ) + args = parser.parse_args(argv) + + if not args.dump_dir.exists(): + print( + f"::warning::No Playwright coverage dump dir at {args.dump_dir}", + file=sys.stderr, + ) + write_vitest_summary({"functions_total": 0, "functions_covered": 0}, args.out) + return 0 + + stats = aggregate(args.dump_dir) + write_vitest_summary(stats, args.out) + pct = ( + 100.0 * stats["functions_covered"] / stats["functions_total"] + if stats["functions_total"] + else 0.0 + ) + print( + f"Aggregated {stats['tests']} tests / {stats['scripts']} scripts: " + f"{stats['functions_covered']}/{stats['functions_total']} functions " + f"({pct:.1f}%)" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testing/compose/docker-compose-coverage.override.yml b/testing/compose/docker-compose-coverage.override.yml new file mode 100644 index 000000000..9af2a4a18 --- /dev/null +++ b/testing/compose/docker-compose-coverage.override.yml @@ -0,0 +1,27 @@ +# Coverage override applied on top of an existing test compose file via +# `docker compose -f base.yml -f docker-compose-coverage.override.yml up`. +# +# It does NOT modify the production image. The agent jar is bind-mounted in +# from the host's `build/jacoco/jacocoagent.jar` (produced by `./gradlew +# copyJacocoAgent`) so the published Stirling-PDF image stays untouched and +# we cannot accidentally ship the agent to end users. +# +# The container appends the agent to its JVM via `JAVA_CUSTOM_OPTS`, which +# the init script concatenates into `JAVA_TOOL_OPTIONS` for the Spring Boot +# process. On graceful shutdown the agent writes +# /jacoco-out/cucumber.exec on the host (mapped to +# `testing/cucumber-coverage/cucumber.exec`) which a follow-up +# `jacocoReportFromExec` task converts into HTML+XML. +services: + stirling-pdf: + # docker-compose-security-with-login.yml renames the service container, + # so the volume mounts intentionally land on whichever service uses the + # key "stirling-pdf" in the base file. + volumes: + - ../../build/jacoco/jacocoagent.jar:/jacoco/jacocoagent.jar:ro + - ../../testing/cucumber-coverage:/jacoco-out:rw + environment: + # append=false so an aborted re-run starts clean. dumponexit=true is + # what flushes the .exec when SIGTERM hits the JVM (which happens on + # `docker compose down`). + JAVA_CUSTOM_OPTS: "-javaagent:/jacoco/jacocoagent.jar=destfile=/jacoco-out/cucumber.exec,output=file,append=false,dumponexit=true" diff --git a/testing/test.sh b/testing/test.sh index 1ab271704..0aa30294a 100644 --- a/testing/test.sh +++ b/testing/test.sh @@ -509,6 +509,23 @@ verify_app_version() { fi } +# Optional second compose file injected on every up/down call (e.g. the +# JaCoCo coverage override). Set externally by callers that want the +# extra layer applied; left empty so production runs are unchanged. +COVERAGE_COMPOSE_FILE="${COVERAGE_COMPOSE_FILE:-}" + +# Helper that joins the base compose file with any optional override into +# the `-f a -f b` form docker-compose expects. Keeps callers terse and +# means we only have one place to add new overrides later. +compose_args() { + local base=$1 + if [ -n "$COVERAGE_COMPOSE_FILE" ]; then + printf -- '-f %s -f %s' "$base" "$COVERAGE_COMPOSE_FILE" + else + printf -- '-f %s' "$base" + fi +} + # Function to test a Docker Compose configuration test_compose() { local compose_file=$1 @@ -519,18 +536,18 @@ test_compose() { echo "Testing ${compose_file} configuration..." # Start up the Docker Compose service - docker-compose -f "$compose_file" up -d + docker-compose $(compose_args "$compose_file") up -d # Wait a moment for containers to appear sleep 3 local container_name - container_name=$(docker-compose -f "$compose_file" ps --format '{{.Names}}' --filter "status=running" | head -n1) + container_name=$(docker-compose $(compose_args "$compose_file") ps --format '{{.Names}}' --filter "status=running" | head -n1) if [[ -z "$container_name" ]]; then echo "ERROR: No running container found for ${compose_file}" local compose_output - compose_output=$(docker-compose -f "$compose_file" ps 2>&1) + compose_output=$(docker-compose $(compose_args "$compose_file") ps 2>&1) echo "$compose_output" capture_failure_logs "$test_name" "" "docker-compose failed for: ${compose_file} ${compose_output}" @@ -856,6 +873,24 @@ main() { # ================================================================== # 3. Regression test with login (test_cicd.yml) # ================================================================== + # STIRLING_PDF_TEST_COVERAGE=1 layers the JaCoCo agent override over + # the cucumber container ONLY. The agent jar is bind-mounted from + # build/jacoco/jacocoagent.jar so the published image never carries + # it. After behave finishes, we trigger `docker compose down` (further + # down) which sends SIGTERM and lets dumponexit=true flush the .exec + # to testing/cucumber-coverage/cucumber.exec on the host. + COVERAGE_COMPOSE_FILE="" + if [ -n "${STIRLING_PDF_TEST_COVERAGE:-}" ]; then + if [ ! -f "$PROJECT_ROOT/build/jacoco/jacocoagent.jar" ]; then + echo "::warning::STIRLING_PDF_TEST_COVERAGE=1 but build/jacoco/jacocoagent.jar is missing - run ./gradlew copyJacocoAgent first" + else + mkdir -p "$PROJECT_ROOT/testing/cucumber-coverage" + rm -f "$PROJECT_ROOT/testing/cucumber-coverage/cucumber.exec" + COVERAGE_COMPOSE_FILE="$PROJECT_ROOT/testing/compose/docker-compose-coverage.override.yml" + echo "Cucumber JaCoCo coverage enabled - exec will land at testing/cucumber-coverage/cucumber.exec" + fi + fi + run_tests "Stirling-PDF-Security-Fat-with-login" "./docker/embedded/compose/test_cicd.yml" # Only run behave tests if the container started successfully @@ -979,7 +1014,13 @@ main() { fi stop_test_timer "Stirling-PDF-Regression" fi - docker-compose -f "./docker/embedded/compose/test_cicd.yml" down -v + # `down` with the override removes the agent bind-mount cleanly. The + # SIGTERM that `down` sends is what triggers dumponexit=true in the + # agent to flush testing/cucumber-coverage/cucumber.exec to disk. + docker-compose $(compose_args "./docker/embedded/compose/test_cicd.yml") down -v + # Reset so subsequent compose calls (CI cleanup, non-cucumber tests) + # don't accidentally pick up the override. + COVERAGE_COMPOSE_FILE="" # ================================================================== # 4. Disabled Endpoints Test