arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-14 10:34:06 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

Add CI coverage summaries and aggregate JaCoCo report (#6451)

Browse Source
This commit is contained in:
Anthony Stirling
2026-06-02 14:59:10 +01:00
committed by GitHub
parent 2c0ebc28a7
commit de9d6ad3f5
15 changed files with 1583 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-build.yml
+66 -7
View File
@@ -1,8 +1,14 @@
name: Backend build, format check, and coverage
# Reusable workflow called from build.yml. Runs the backend build matrix
# (JDK 25 × spring-security on/off), Spotless formatting check, JUnit, and
# (JDK 25 × every flavor), Spotless formatting check, JUnit, and
# posts Jacoco coverage to PRs.
#
# Flavor axis (maps to STIRLING_FLAVOR in settings.gradle):
# core - DISABLE_ADDITIONAL_FEATURES=true, no proprietary, no saas
# proprietary - default build, no saas
# saas - proprietary + the saas subproject (build + JUnit only,
# never any runtime/integration testing)
on:
workflow_call:
@@ -25,7 +31,7 @@ jobs:
fail-fast: false
matrix:
jdk-version: [25]
spring-security: [true, false]
flavor: [core, proprietary, saas]
steps:
- name: Harden Runner
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
@@ -58,7 +64,10 @@ jobs:
- name: Install Task
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
- name: Check Java formatting (Spotless)
if: matrix.jdk-version == 25 && matrix.spring-security == false
# Runs once per matrix combination - pick the cheapest leg
# (core - no proprietary, no saas) so we don't wait for the
# heavier flavors just to fail formatting.
if: matrix.jdk-version == 25 && matrix.flavor == 'core'
id: spotless-check
run: task backend:format:check
continue-on-error: true
@@ -143,17 +152,24 @@ jobs:
});
}
- name: Build with Gradle and spring security ${{ matrix.spring-security }}
- name: Build with Gradle (flavor=${{ matrix.flavor }})
# STIRLING_FLAVOR is read by settings.gradle and expands into the
# right combination of DISABLE_ADDITIONAL_FEATURES + ENABLE_SAAS
# so we don't have to set them by hand. The saas flavor pulls in
# the app/saas subproject (unit tests only - no runtime tests).
run: task backend:build:ci
env:
MAVEN_USER: ${{ secrets.MAVEN_USER }}
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }}
STIRLING_FLAVOR: ${{ matrix.flavor }}
- name: Check Test Reports Exist
if: always()
run: |
# Common + core + proprietary always build (proprietary is
# excluded only at runtime, not from the gradle subproject
# graph). Saas builds add a fourth report dir.
declare -a dirs=(
"app/core/build/reports/tests/"
"app/core/build/test-results/"
@@ -162,6 +178,9 @@ jobs:
"app/proprietary/build/reports/tests/"
"app/proprietary/build/test-results/"
)
if [ "${{ matrix.flavor }}" = "saas" ]; then
dirs+=("app/saas/build/reports/tests/" "app/saas/build/test-results/")
fi
for dir in "${dirs[@]}"; do
if [ ! -d "$dir" ]; then
echo "Missing $dir"
@@ -173,7 +192,7 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }}
name: test-reports-jdk-${{ matrix.jdk-version }}-flavor-${{ matrix.flavor }}
path: |
app/**/build/reports/jacoco/test
app/**/build/reports/tests/
@@ -183,7 +202,47 @@ jobs:
retention-days: 3
if-no-files-found: warn
- name: Add coverage to PR with spring security ${{ matrix.spring-security }} and JDK ${{ matrix.jdk-version }}
- name: Install defusedxml for coverage summary
# coverage-summary.py parses JaCoCo XML through defusedxml to
# silence security scanners that pattern-match on the stdlib
# xml.etree.ElementTree.parse call.
if: always() && matrix.flavor == 'saas'
run: python -m pip install --quiet defusedxml
- name: JaCoCo coverage step summary
# Only the saas leg posts the JUnit summary - it's a strict
# superset of the core + proprietary legs (same .exec files plus
# the saas subproject). Posting from all three would mean three
# near-identical tables crowding out the aggregate report.
if: always() && matrix.flavor == 'saas'
run: |
python scripts/coverage-summary.py \
--title "Backend JUnit coverage (JDK ${{ matrix.jdk-version }})" \
--jacoco "common=app/common/build/reports/jacoco/test/jacocoTestReport.xml" \
--jacoco "core=app/core/build/reports/jacoco/test/jacocoTestReport.xml" \
--jacoco "proprietary=app/proprietary/build/reports/jacoco/test/jacocoTestReport.xml" \
--jacoco "saas=app/saas/build/reports/jacoco/test/jacocoTestReport.xml" \
--github-step-summary
- name: Upload raw JUnit .exec for aggregate merge
# Same dedup rationale as the summary step: upload from the saas
# leg only (the most complete set, includes app/saas/.../test.exec)
# so the aggregate workflow merges the union rather than three
# overlapping subsets.
#
# Separate artifact from the HTML reports so the aggregate
# workflow can grab just the .exec files with a name pattern
# (`jacoco-exec-*`) instead of unpacking the whole test-reports
# tarball.
if: always() && matrix.flavor == 'saas'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-exec-junit-jdk-${{ matrix.jdk-version }}
path: app/*/build/jacoco/*.exec
retention-days: 7
if-no-files-found: warn
- name: Add coverage to PR (flavor=${{ matrix.flavor }}, JDK=${{ matrix.jdk-version }})
# The action only supports the pull_request event (it posts a PR comment),
# so skip it for merge_group runs and workflow_dispatch.
if: github.event_name == 'pull_request'
.github/workflows/build.yml
+18
View File
@@ -185,6 +185,24 @@ jobs:
uses: ./.github/workflows/dependency-review.yml
secrets: inherit
# Coverage aggregate: merges the JUnit + e2e:live + cucumber .exec
# artifacts produced by the jobs above into one report, plus pulls
# in vitest + Playwright frontend coverage for the per-area matrix.
# `if: always()` so a producer failing partway still gets credit
# for whatever did record. Advisory only - intentionally NOT in
# all-checks-passed, so a flaky aggregate run never blocks merging.
coverage-aggregate:
if: always()
needs:
- build
- playwright-e2e-live
- docker-compose-tests
- frontend-validation
permissions:
contents: read
uses: ./.github/workflows/coverage-aggregate.yml
secrets: inherit
# Single status check that branch protection should mark as required.
# Succeeds when every upstream job is either `success` or `skipped` (path-
# gated jobs that didn't apply this run). Any `failure` or `cancelled`
.github/workflows/coverage-aggregate.yml
+230
View File
@@ -0,0 +1,230 @@
name: Aggregate backend coverage
# Reusable workflow called from build.yml after every backend coverage
# producer (JUnit, e2e:live, cucumber) has run. Downloads each job's raw
# .exec, merges them into one JaCoCo report, and posts a combined step
# summary alongside the per-source ones.
#
# Kept separate from the per-source jobs so:
# - the per-source jobs stay fast and independent (no cross-job waits)
# - this job can `if: always()` and still produce something useful when
# one of the producers fails partway through
# - frontend producers can be added later without touching the
# producers themselves
on:
workflow_call:
permissions:
contents: read
jobs:
pick:
uses: ./.github/workflows/_runner-pick.yml
aggregate:
needs: pick
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
timeout-minutes: 15
steps:
- name: Harden Runner
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
with:
egress-policy: audit
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up JDK 25
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
java-version: "25"
distribution: "temurin"
- name: Cache Gradle dependency artifacts
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
~/.gradle/wrapper
~/.gradle/caches/modules-2/files-2.1
~/.gradle/caches/modules-2/metadata-2.*
key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
- name: Setup Gradle
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
with:
gradle-version: 9.3.1
cache-disabled: true
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"
- name: Install defusedxml for coverage scripts
# Both coverage-summary.py and coverage-matrix.py parse JaCoCo
# XML through defusedxml - see the script headers for context.
run: python -m pip install --quiet defusedxml
# Pattern matches every artifact this PR's producers might upload:
# jacoco-exec-junit-jdk-25 (uploaded only by the saas
# leg of backend-build, which
# is a strict superset of the
# core + proprietary legs)
# jacoco-exec-e2e-live
# jacoco-exec-cucumber
# Each lands as a sibling dir under coverage-execs/, with the .exec
# files preserving their original relative paths.
- name: Download all .exec artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
with:
pattern: jacoco-exec-*
path: coverage-execs/
merge-multiple: false
continue-on-error: true
- name: Inventory .exec files
id: inventory
# Splits the downloaded artifacts into two buckets:
# * e2e-only = cucumber + Playwright live (user-flow coverage)
# * all = the above plus JUnit (everything we test)
#
# Bucketing is by artifact-name prefix: download-artifact preserves
# the artifact name as the top-level dir, so JUnit's `.exec`s live
# under coverage-execs/jacoco-exec-junit-*/... while the others
# are under coverage-execs/jacoco-exec-{e2e-live,cucumber}/...
#
# If nothing was uploaded (e.g. all producers crashed before
# writing) we exit gracefully so this advisory job never fails CI.
run: |
mapfile -t all_execs < <(find coverage-execs -name '*.exec' -type f | sort)
mapfile -t e2e_execs < <(find coverage-execs -name '*.exec' -type f -not -path '*/jacoco-exec-junit-*' | sort)
if [ "${#all_execs[@]}" -eq 0 ]; then
echo "::warning::No .exec artifacts found - skipping aggregate report"
echo "found_all=false" >> "$GITHUB_OUTPUT"
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
exit 0
fi
printf 'All %d .exec files:\n' "${#all_execs[@]}"
printf ' %s\n' "${all_execs[@]}"
IFS=','; all_joined="${all_execs[*]}"
echo "files_all=$all_joined" >> "$GITHUB_OUTPUT"
echo "found_all=true" >> "$GITHUB_OUTPUT"
if [ "${#e2e_execs[@]}" -eq 0 ]; then
echo "::notice::No e2e/cucumber .exec files - e2e-only report will be skipped"
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
else
printf 'E2E-only %d .exec files:\n' "${#e2e_execs[@]}"
printf ' %s\n' "${e2e_execs[@]}"
unset IFS
IFS=','; e2e_joined="${e2e_execs[*]}"
echo "files_e2e=$e2e_joined" >> "$GITHUB_OUTPUT"
echo "found_e2e=true" >> "$GITHUB_OUTPUT"
fi
- name: Compile classes for JaCoCo class lookup
# jacocoReportFromExec only needs the compiled .class files
# under each subproject's build/classes/java/main/. `classes`
# (compileJava + processResources) is enough; we skipped the
# heavier `assemble` to avoid building bootJar / fat jars that
# add 60+ seconds per run for no gain to the report.
if: steps.inventory.outputs.found_all == 'true'
run: ./gradlew classes -PnoSpotless
- name: Generate e2e-only JaCoCo report
# "Real user-flow" coverage: only counts code reached by an actual
# HTTP request from cucumber or live Playwright. Useful for
# questions like "how much of our backend does a user actually
# hit?". Skipped when neither producer uploaded a .exec.
if: steps.inventory.outputs.found_e2e == 'true'
run: |
./gradlew jacocoReportFromExec \
-PexecFile="${{ steps.inventory.outputs.files_e2e }}" \
-PreportDir=build/reports/jacoco/aggregate-e2e \
-PnoSpotless
- name: Generate combined JaCoCo report (everything)
if: steps.inventory.outputs.found_all == 'true'
run: |
./gradlew jacocoReportFromExec \
-PexecFile="${{ steps.inventory.outputs.files_all }}" \
-PreportDir=build/reports/jacoco/aggregate-all \
-PnoSpotless
- name: E2E-only step summary
# Rendered first so it gets prime real estate in the Summary
# tab - this is the number most readers actually want
# ("how much of the backend do real user flows cover?").
if: steps.inventory.outputs.found_e2e == 'true'
run: |
python scripts/coverage-summary.py \
--title "Real user-flow backend coverage (e2e:live + cucumber)" \
--jacoco "merged=build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml" \
--github-step-summary
- name: ALL-sources step summary
# Separate call (not a multi-input one) because the helper's
# rightmost "Aggregate" column would sum the two reports - which
# is meaningless when one is a strict superset of the other.
if: steps.inventory.outputs.found_all == 'true'
run: |
python scripts/coverage-summary.py \
--title "Combined backend coverage (JUnit + e2e:live + cucumber)" \
--jacoco "merged=build/reports/jacoco/aggregate-all/jacocoTestReport.xml" \
--github-step-summary
- name: Upload combined aggregate report
if: steps.inventory.outputs.found_all == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-aggregate-all-${{ github.run_id }}
path: build/reports/jacoco/aggregate-all/
retention-days: 14
- name: Upload e2e-only aggregate report
if: steps.inventory.outputs.found_e2e == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-aggregate-e2e-${{ github.run_id }}
path: build/reports/jacoco/aggregate-e2e/
retention-days: 14
# --------------------------------------------------------------
# Per-area matrix: rolls backend + frontend coverage into one
# table indexed by core/proprietary/saas/desktop. Pulls the
# frontend artifacts now (after the JaCoCo step has done its
# work) so the per-source backend summaries still render first
# even if the matrix step fails.
# --------------------------------------------------------------
- name: Download vitest coverage artifact
# frontend-validation uploads as `frontend-coverage`. Tolerate
# absence so a backend-only PR still produces the matrix with
# just backend rows populated.
if: always()
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
with:
name: frontend-coverage
path: matrix-inputs/vitest/
continue-on-error: true
- name: Download Playwright frontend coverage artifact
# e2e-live uploads as `playwright-frontend-coverage-<run_id>`.
# Same tolerance as vitest - matrix script handles missing inputs.
if: always()
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
with:
name: playwright-frontend-coverage-${{ github.run_id }}
path: matrix-inputs/playwright/
continue-on-error: true
- name: Coverage matrix step summary
if: always()
# Matrix references the two aggregate JaCoCo XMLs (already
# generated above) plus whichever frontend artifacts landed.
# Every input is optional; missing ones render as "-".
run: |
python scripts/coverage-matrix.py \
${{ steps.inventory.outputs.found_all == 'true' && '--jacoco-all build/reports/jacoco/aggregate-all/jacocoTestReport.xml' || '' }} \
${{ steps.inventory.outputs.found_e2e == 'true' && '--jacoco-e2e build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml' || '' }} \
--vitest matrix-inputs/vitest/coverage-summary.json \
--playwright-frontend matrix-inputs/playwright/coverage-pw-summary/coverage-summary.json \
--title "Coverage matrix (per-area, e2e vs all)" \
--github-step-summary
.github/workflows/docker-compose-tests.yml
+62
View File
@@ -87,6 +87,12 @@ jobs:
run: |
pip install --require-hashes --only-binary=:all: -r ./testing/cucumber/requirements.txt
- name: Extract JaCoCo agent for cucumber coverage
# Stages build/jacoco/jacocoagent.jar where the coverage override
# file bind-mounts it into the cucumber container. The agent jar
# never goes into the published image - this is host-only.
run: ./gradlew copyJacocoAgent -PnoSpotless
- name: Run Docker Compose Tests
run: |
chmod +x ./testing/test_webpages.sh
@@ -98,6 +104,62 @@ jobs:
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
DOCKER_BASE_CHANGED: ${{ inputs.docker-base-changed }}
# Tells test.sh to layer testing/compose/docker-compose-coverage.override.yml
# over the cucumber compose so the container starts with the
# JaCoCo agent attached via JAVA_CUSTOM_OPTS.
STIRLING_PDF_TEST_COVERAGE: "1"
- name: Generate cucumber JaCoCo report
# `if: always()` so a behave failure still produces partial
# coverage from whatever endpoints did run. The exec file only
# exists when the container shut down cleanly - guard so the step
# is silent on the (rare) crash path.
if: always()
id: cucumber-coverage
run: |
if [ -s testing/cucumber-coverage/cucumber.exec ]; then
./gradlew jacocoReportFromExec \
-PexecFile=testing/cucumber-coverage/cucumber.exec \
-PreportDir=build/reports/jacoco/cucumber \
-PnoSpotless
echo "report=true" >> "$GITHUB_OUTPUT"
else
echo "::warning::No cucumber .exec at testing/cucumber-coverage/cucumber.exec (container may have crashed before flushing)"
echo "report=false" >> "$GITHUB_OUTPUT"
fi
- name: Install defusedxml for coverage summary
# coverage-summary.py parses JaCoCo XML through defusedxml -
# see the script header for context.
if: always() && steps.cucumber-coverage.outputs.report == 'true'
run: python -m pip install --quiet defusedxml
- name: Cucumber coverage step summary
if: always() && steps.cucumber-coverage.outputs.report == 'true'
run: |
python scripts/coverage-summary.py \
--title "Cucumber (docker) JaCoCo coverage" \
--jacoco "cucumber=build/reports/jacoco/cucumber/jacocoTestReport.xml" \
--github-step-summary
- name: Upload cucumber JaCoCo report
if: always() && steps.cucumber-coverage.outputs.report == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-cucumber-${{ github.run_id }}
path: build/reports/jacoco/cucumber/
retention-days: 7
- name: Upload raw cucumber .exec for aggregate merge
# Picked up by the coverage-aggregate workflow via the
# `jacoco-exec-*` artifact name pattern.
if: always() && steps.cucumber-coverage.outputs.report == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-exec-cucumber
path: testing/cucumber-coverage/cucumber.exec
retention-days: 7
if-no-files-found: warn
- name: Upload Cucumber Report
if: always()
.github/workflows/e2e-live.yml
+124 -1
View File
@@ -49,9 +49,132 @@ jobs:
env:
VITE_BUILD_FOR_PREVIEW: "1"
run: task frontend:build
- name: Run live E2E tests (chromium)
- name: Run live E2E tests (chromium) with coverage
id: live-tests
env:
# Attaches the JaCoCo agent to the bootRun JVM (see
# .taskfiles/e2e.yml live:backend). The .exec gets flushed on
# graceful shutdown when the runner traps EXIT/INT/TERM, so the
# report step below sees a populated file.
COVERAGE: "1"
# Tells the Playwright fixture (test-base.ts) to capture per-test
# V8 JS coverage. Raw dumps land under
# .test-state/playwright/coverage-pw/ for the post-process step
# to aggregate. Chromium-only - other engines silently skip.
PW_COVERAGE: "1"
run: task e2e:live
- name: Generate JaCoCo report from e2e:live .exec
if: always()
id: live-coverage
# `if: always()` so even a failed test run still produces a
# report from whatever flows did exercise the backend before
# the failure. The task itself tolerates a missing .exec
# (jacoco emits an empty report rather than crashing) but we
# guard with `test -s` to keep the job log clean.
run: |
if [ -s .test-state/playwright/jacoco.exec ]; then
./gradlew jacocoReportFromExec \
-PexecFile=.test-state/playwright/jacoco.exec \
-PreportDir=build/reports/jacoco/e2e-live \
-PnoSpotless
echo "report=true" >> "$GITHUB_OUTPUT"
else
echo "::warning::No e2e:live .exec found at .test-state/playwright/jacoco.exec; skipping report"
echo "report=false" >> "$GITHUB_OUTPUT"
fi
- name: Set up Python for coverage summary
if: always() && steps.live-coverage.outputs.report == 'true'
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"
- name: Install defusedxml for coverage summary
# coverage-summary.py uses defusedxml instead of stdlib xml.etree
# to dodge XXE / billion-laughs scanner findings.
if: always() && steps.live-coverage.outputs.report == 'true'
run: python -m pip install --quiet defusedxml
- name: e2e:live coverage step summary
if: always() && steps.live-coverage.outputs.report == 'true'
run: |
python scripts/coverage-summary.py \
--title "Playwright (live backend) JaCoCo coverage" \
--jacoco "e2e-live=build/reports/jacoco/e2e-live/jacocoTestReport.xml" \
--github-step-summary
- name: Upload e2e:live JaCoCo report
if: always() && steps.live-coverage.outputs.report == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-e2e-live-${{ github.run_id }}
path: build/reports/jacoco/e2e-live/
retention-days: 7
- name: Upload raw e2e:live .exec for aggregate merge
# Picked up by the coverage-aggregate workflow via the
# `jacoco-exec-*` artifact name pattern.
if: always() && steps.live-coverage.outputs.report == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: jacoco-exec-e2e-live
path: .test-state/playwright/jacoco.exec
retention-days: 7
if-no-files-found: warn
- name: Set up Python for frontend coverage summary
# Separate from the backend-coverage python step because the
# frontend path doesn't depend on a JaCoCo report - it produces
# a summary even on backend failure, as long as some Playwright
# tests ran far enough to dump V8 coverage.
if: always()
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"
- name: Install defusedxml for frontend coverage summary
# Idempotent re-install: the backend-coverage step may have
# installed it already, but this leg can run on its own when the
# backend report step skips (e.g. .exec missing).
if: always()
run: python -m pip install --quiet defusedxml
- name: Aggregate Playwright frontend (V8) coverage
# Rolls per-test V8 dumps from the test-base fixture into one
# vitest-shaped coverage-summary.json. Tolerates a missing dump
# dir (firefox/webkit runs, or a failure before any test got
# far enough to dump).
if: always()
id: pw-frontend-coverage
run: |
if [ -d .test-state/playwright/coverage-pw ] && \
find .test-state/playwright/coverage-pw -name '*.json' -type f | grep -q .; then
python scripts/playwright-coverage-summary.py \
.test-state/playwright/coverage-pw \
--out .test-state/playwright/coverage-pw-summary/coverage-summary.json
echo "summary=true" >> "$GITHUB_OUTPUT"
else
echo "::notice::No Playwright frontend coverage dumps found (chromium-only feature)"
echo "summary=false" >> "$GITHUB_OUTPUT"
fi
- name: Playwright frontend coverage step summary
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
run: |
python scripts/coverage-summary.py \
--title "Playwright (live) frontend coverage" \
--vitest .test-state/playwright/coverage-pw-summary/coverage-summary.json \
--github-step-summary
- name: Upload Playwright frontend coverage
# Bundle both the aggregated summary and the raw V8 dumps so
# someone debugging "why is this function showing as covered"
# can trace it back to the source dump.
if: always() && steps.pw-frontend-coverage.outputs.summary == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: playwright-frontend-coverage-${{ github.run_id }}
path: |
.test-state/playwright/coverage-pw-summary/
.test-state/playwright/coverage-pw/
retention-days: 7
- name: Print backend log on failure
if: failure() && steps.live-tests.conclusion == 'failure'
run: |
.github/workflows/frontend-validation.yml
+35
View File
@@ -109,6 +109,41 @@ jobs:
comment_id: existing.id,
});
}
- name: Vitest coverage
# Separate from `frontend:check:all` so the quality-gate run stays
# uninstrumented (faster signal) and coverage stays an informational
# follow-up. Continue-on-error keeps the workflow green even when
# a handful of test files refuse to import (e.g. missing icon
# specifiers) - the summary still gets posted with whatever
# vitest managed to instrument.
id: frontend-coverage
continue-on-error: true
run: task frontend:test:coverage
- name: Set up Python for coverage summary
if: always()
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"
- name: Install defusedxml for coverage summary
# See coverage-summary.py header - it parses XML through defusedxml
# to dodge the stdlib parser's exposure to XXE / billion-laughs.
if: always()
run: python -m pip install --quiet defusedxml
- name: Vitest coverage step summary
if: always()
run: |
python scripts/coverage-summary.py \
--title "Frontend Vitest coverage" \
--vitest frontend/editor/coverage/coverage-summary.json \
--github-step-summary
- name: Upload vitest coverage report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: frontend-coverage
path: frontend/editor/coverage/
retention-days: 7
if-no-files-found: warn
- name: Upload frontend build artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
.taskfiles/e2e.yml
+11 -1
View File
@@ -34,6 +34,9 @@ tasks:
ignore_error: true
vars:
BASE_DIR: '{{.ROOT_DIR}}/.test-state/playwright'
# COVERAGE=1 in the calling environment attaches the JaCoCo agent to
# the bootRun JVM and writes to BASE_DIR/jacoco.exec on shutdown.
# Off by default to keep local dev runs uninstrumented; CI flips it.
env:
STIRLING_BASE_PATH: '{{.BASE_DIR}}'
# Suppress the analytics opt-in modal that fires on first admin login.
@@ -58,12 +61,19 @@ tasks:
set -e
rm -rf "{{.BASE_DIR}}"
mkdir -p "{{.BASE_DIR}}"
GRADLE_ARGS=":stirling-pdf:bootRun"
if [ -n "${COVERAGE:-}" ]; then
# copyJacocoAgent is wired as a dependency of bootRun when
# -PjacocoAgent=true, so we do not need to invoke it separately.
GRADLE_ARGS="$GRADLE_ARGS -PjacocoAgent=true -PjacocoExec={{.BASE_DIR}}/jacoco.exec"
echo "JaCoCo coverage enabled, writing to {{.BASE_DIR}}/jacoco.exec"
fi
# Background gradle and record its PID so the runner can clean up
# the exact process tree (wrapper + forked Spring Boot JVM) without
# resorting to fuzzy `pkill -f` patterns. `wait` keeps this script
# alive for the lifetime of gradle so Task'"'"'s parallel deps stay
# synchronised.
bash gradlew :stirling-pdf:bootRun > "{{.BASE_DIR}}/backend.log" 2>&1 &
bash gradlew $GRADLE_ARGS > "{{.BASE_DIR}}/backend.log" 2>&1 &
GRADLE_PID=$!
echo $GRADLE_PID > "{{.BASE_DIR}}/backend.pid"
wait $GRADLE_PID
.taskfiles/frontend.yml
+17 -2
View File
@@ -284,10 +284,25 @@ tasks:
- npx vitest --watch --root editor
test:coverage:
desc: "Run tests with coverage"
desc: "Run tests with coverage (one-shot; CI-friendly)."
deps: [install]
cmds:
- npx vitest --coverage --root editor
# `vitest run` makes this CI-safe (the bare `vitest` form enters watch
# mode). Explicit reporter list because v8 + json-summary is what the
# coverage-summary.py helper consumes; html/text are kept for humans.
#
# reportsDirectory is pinned to ./coverage relative to vitest's root
# (--root editor), so output lands at frontend/editor/coverage/. The
# CI upload step reads from that path. An earlier attempt with
# `./editor/coverage` double-nested into frontend/editor/editor/coverage;
# pinning future-proofs against vitest changing the default.
- >
npx vitest run --root editor --coverage
--coverage.provider=v8
--coverage.reporter=text-summary
--coverage.reporter=json-summary
--coverage.reporter=html
--coverage.reportsDirectory=./coverage
# ============================================================
# Code Generation
build.gradle
+125 -1
View File
@@ -438,7 +438,7 @@ subprojects {
}
tasks.named("bootRun") {
jvmArgs = [
def runtimeArgs = [
"-XX:+UseG1GC",
"-XX:MaxGCPauseMillis=200",
"-XX:G1HeapRegionSize=4m",
@@ -447,6 +447,53 @@ subprojects {
"-XX:+UseCompactObjectHeaders",
"--enable-native-access=ALL-UNNAMED"
]
// Optional JaCoCo agent for e2e/cucumber backend coverage.
//
// Enabled with `-PjacocoAgent=true`. The default destfile lives
// outside the source tree so it doesn't poison a normal bootRun
// for a developer who forgot to pass the property. Override
// both with `-PjacocoExec=/path/to.exec` if you need a custom
// location (e.g. when sharing it with the Playwright runner).
if (rootProject.hasProperty('jacocoAgent') &&
rootProject.property('jacocoAgent').toString() == 'true') {
def agentJar = rootProject.layout.buildDirectory
.file('jacoco/jacocoagent.jar').get().asFile.absolutePath
def execFile = (rootProject.findProperty('jacocoExec') ?:
rootProject.layout.projectDirectory
.file('.test-state/playwright/jacoco.exec').asFile.absolutePath
).toString()
// The JaCoCo agent argument is a comma-separated key=value list,
// so a path that itself contains ',' or '=' would smuggle extra
// agent options (e.g. -PjacocoExec='out.exec,sessionid=evil').
// Today the value only comes from our own CI workflows, but
// validating defensively is cheap and silences Aikido.
//
// Control characters are checked via String.contains rather
// than inside a slashy regex character class - a literal NULL
// accidentally landing in the regex killed Groovy parsing on
// the previous attempt.
def hasBadChar = execFile.find(/[,= ]/) != null ||
execFile.contains('\r') ||
execFile.contains('\n') ||
execFile.contains('\t')
if (hasBadChar) {
throw new GradleException(
"jacocoExec='" + execFile + "' contains characters " +
"(',', '=', whitespace, or control chars) that " +
"would break -javaagent option parsing. Choose " +
"a different path."
)
}
runtimeArgs.add("-javaagent:${agentJar}=destfile=${execFile},output=file,append=false,dumponexit=true")
dependsOn(rootProject.tasks.named('copyJacocoAgent'))
doFirst {
new File(execFile).parentFile?.mkdirs()
logger.lifecycle("JaCoCo agent attached: ${execFile}")
}
}
jvmArgs = runtimeArgs
}
}
}
@@ -627,3 +674,80 @@ tasks.withType(Test).configureEach {
// Half of available CPUs is a safe default; bump if your tests are I/O-bound.
maxParallelForks = Math.max(1, (Runtime.runtime.availableProcessors().intdiv(2)) as int)
}
// ----------------------------------------------------------------------------
// JaCoCo helpers used by CI for cucumber + Playwright (live backend) coverage.
//
// These let CI attach the JaCoCo agent to a running Spring Boot process
// (gradle bootRun or the docker image, via JAVA_TOOL_OPTIONS) and then turn
// the resulting .exec dump back into HTML + XML reports without anyone
// hand-installing the JaCoCo CLI.
// ----------------------------------------------------------------------------
configurations {
jacocoRuntimeAgent
jacocoCli
}
dependencies {
// The :runtime classifier on org.jacoco.agent IS the agent jar - no
// unzipping required. Kept on the version the plugin already picks so
// agent + reporter line up exactly.
jacocoRuntimeAgent "org.jacoco:org.jacoco.agent:${jacoco.toolVersion}:runtime"
jacocoCli "org.jacoco:org.jacoco.cli:${jacoco.toolVersion}"
}
tasks.register('copyJacocoAgent', Copy) {
group = 'verification'
description = 'Copies the JaCoCo runtime agent jar to build/jacoco/jacocoagent.jar.'
from configurations.jacocoRuntimeAgent
into layout.buildDirectory.dir('jacoco')
rename { 'jacocoagent.jar' }
}
// Aggregates an externally-produced .exec (e.g. from e2e:live or the cucumber
// docker container) against this project's compiled classes + sources.
//
// Inputs are configured via -P properties so the same task works for every
// caller:
//
// ./gradlew jacocoReportFromExec -PexecFile=.test-state/playwright/jacoco.exec \
// -PreportDir=build/reports/jacoco/e2e-live
tasks.register('jacocoReportFromExec', JacocoReport) {
group = 'verification'
description = 'Generates a JaCoCo HTML+XML report from an externally captured .exec.'
def execProp = project.findProperty('execFile') ?: project.findProperty('execFiles')
def reportProp = project.findProperty('reportDir') ?: "build/reports/jacoco/external"
executionData fileTree(rootProject.rootDir) {
if (execProp) {
include execProp.toString().split(',').collect { it.trim() }
} else {
// Sensible defaults so the task is usable without props.
include '.test-state/**/*.exec'
include 'coverage-tools/exec/*.exec'
include 'testing/cucumber-coverage/*.exec'
}
}
// Pull compiled classes + sources from every subproject so the report is
// an aggregate. JaCoCo silently skips classes that have no matching .exec
// probes, so this is safe even when only a subset of code was exercised.
classDirectories.setFrom(files(subprojects.collect { sub ->
sub.fileTree(dir: "${sub.buildDir}/classes/java/main", excludes: [
'**/generated/**',
])
}))
sourceDirectories.setFrom(files(subprojects.collect { sub ->
"${sub.projectDir}/src/main/java"
}))
reports {
xml.required.set(true)
html.required.set(true)
csv.required.set(false)
xml.outputLocation.set(layout.projectDirectory.file("${reportProp}/jacocoTestReport.xml"))
html.outputLocation.set(layout.projectDirectory.dir("${reportProp}/html"))
}
}
frontend/editor/src/core/tests/helpers/test-base.ts
+65 -7
View File
@@ -1,17 +1,45 @@
import { test as base, expect } from "@playwright/test";
import * as fs from "node:fs/promises";
import * as path from "node:path";
/**
* Custom test fixture that auto-dismisses the cookie consent banner
* before every test. The banner (#cc-main) overlays the page and
* intercepts pointer events, causing click timeouts across all tests.
* Custom test fixture that:
* 1. auto-dismisses the cookie consent banner (the banner overlays
* the page and intercepts pointer events, causing click timeouts
* across all tests);
* 2. optionally captures V8 JS coverage per test when PW_COVERAGE=1.
*
* Coverage notes:
* - Only chromium supports page.coverage.* - the try/catch silently
* skips collection on firefox/webkit so cross-browser runs don't
* fail when coverage is enabled.
* - Raw V8 dumps land under .test-state/playwright/coverage-pw/ as
* <test-title>-<testId>.json. CI post-processes them with
* scripts/playwright-coverage-summary.py into a vitest-shaped
* coverage-summary.json that the existing helper renders.
* - resetOnNavigation: false so navigation between pages inside the
* same test (login → tool page → preview) accumulates instead of
* starting fresh.
*
* Usage: import { test, expect } from '@app/tests/helpers/test-base';
*/
const COVERAGE_ENABLED = process.env.PW_COVERAGE === "1";
// Path is workspace-relative because Playwright cwd is frontend/editor/
// at runtime (defined by playwright.config.ts). Going up two levels
// lands at the repo root - the same `.test-state/playwright/` that
// `task e2e:live` already provisions.
const COVERAGE_DIR = path.resolve(
process.cwd(),
"..",
"..",
".test-state",
"playwright",
"coverage-pw",
);
export const test = base.extend({
page: async ({ page }, use) => {
// Set the cookie consent cookie before any navigation so the banner
// never appears. The cookieconsent library (orestbida/cookieconsent)
// reads this cookie on init and skips the banner if consent exists.
page: async ({ page }, use, testInfo) => {
await page.context().addCookies([
{
name: "cc_cookie",
@@ -28,7 +56,37 @@ export const test = base.extend({
},
]);
if (COVERAGE_ENABLED) {
try {
await page.coverage.startJSCoverage({ resetOnNavigation: false });
} catch {
// Browser doesn't support V8 coverage (firefox/webkit) - silently skip.
}
}
await use(page);
if (COVERAGE_ENABLED) {
try {
const entries = await page.coverage.stopJSCoverage();
if (entries.length > 0) {
await fs.mkdir(COVERAGE_DIR, { recursive: true });
// Sanitize the path so it survives every filesystem we run on.
const safeTitle = testInfo.titlePath
.join("-")
.replace(/[^A-Za-z0-9._-]/g, "_")
.slice(0, 120);
const outPath = path.join(
COVERAGE_DIR,
`${safeTitle}-${testInfo.testId}.json`,
);
await fs.writeFile(outPath, JSON.stringify(entries));
}
} catch {
// Browser closed or coverage unsupported - swallow so a coverage
// failure never fails the underlying test.
}
}
},
});
scripts/coverage-matrix.py
+325
View File
@@ -0,0 +1,325 @@
#!/usr/bin/env python3
"""Render a single coverage matrix combining backend + frontend, per area.
Rows:
- Frontend (e2e only) - Playwright fixture V8 capture
- Frontend (all) - Vitest unit tests + Playwright V8
- Backend (e2e only) - Playwright live JaCoCo + cucumber JaCoCo
- Backend (all) - JUnit + e2e:live + cucumber JaCoCo
- Both (e2e only) - frontend (e2e) + backend (e2e), summed
- Both (all) - frontend (all) + backend (all), summed
Columns:
- core, proprietary, saas, desktop - bucketed per area
- ALL - sum across the row
Bucketing rules:
Backend (JaCoCo package -> area):
stirling/software/SPDF/** -> core (the main backend module)
stirling/software/common/** -> core (shared infra, attributed to core)
org/apache/pdfbox/** -> core (vendored helpers in common/core)
stirling/software/proprietary/** -> proprietary (unless saas-flavoured)
stirling/software/saas/** -> saas
*desktop* -> desktop (no real backend match today)
Frontend (source path -> area):
frontend/editor/src/core/** -> core
frontend/editor/src/proprietary/** -> proprietary
frontend/editor/src/saas/** -> saas
frontend/editor/src/desktop/** -> desktop
Cells render as `pct% (covered/total)` where the metric is:
- Backend: JaCoCo METHOD counter (most directly comparable to JS funcs)
- Frontend: function counts from vitest per-file summary
Playwright (live) frontend V8 dumps only know about bundled JS URLs, not
source paths, so they only feed the ALL column for the Frontend rows.
Per-area Playwright contributions show "n/a" with a footnote.
Missing inputs are tolerated: any source that isn't passed renders as `-`
and the row aggregates from what is available.
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
# defusedxml hardens the parser against XXE / billion-laughs / entity-
# expansion attacks. JaCoCo XML on a CI runner is trusted input today,
# but using the hardened parser is a one-line change and silences
# scanners that pattern-match on `xml.etree.ElementTree.parse`.
from defusedxml.ElementTree import parse as _xml_parse
from defusedxml.ElementTree import ParseError as _XMLParseError
AREAS = ("core", "proprietary", "saas", "desktop")
@dataclass
class Bucket:
"""Covered + total counters that can be combined across sources."""
covered: int = 0
total: int = 0
@property
def pct(self) -> float:
return 100.0 * self.covered / self.total if self.total else 0.0
def add(self, other: "Bucket") -> None:
self.covered += other.covered
self.total += other.total
@dataclass
class RowBuckets:
"""Per-area buckets for one row of the matrix."""
by_area: dict[str, Bucket] = field(
default_factory=lambda: {a: Bucket() for a in AREAS}
)
# Some inputs (Playwright V8) don't have source-path info, so they
# only contribute to ALL without an area attribution. Track those
# separately so per-area cells stay honest.
unattributed: Bucket = field(default_factory=Bucket)
@property
def all(self) -> Bucket:
agg = Bucket()
for b in self.by_area.values():
agg.add(b)
agg.add(self.unattributed)
return agg
def merge(self, other: "RowBuckets") -> None:
for area in AREAS:
self.by_area[area].add(other.by_area[area])
self.unattributed.add(other.unattributed)
# --------------------------------------------------------------------- jacoco
def _classify_backend(package_name: str) -> Optional[str]:
"""Map a JaCoCo package name to an area, or None to skip."""
if not package_name:
return None
p = package_name.replace("/", ".")
if "desktop" in p:
return "desktop"
if p.startswith("stirling.software.saas"):
return "saas"
# Proprietary saas-flavoured sub-package: rare, but kept for forward
# compatibility if a future build moves saas under proprietary.
if p.startswith("stirling.software.proprietary") and ".saas" in p:
return "saas"
if p.startswith("stirling.software.proprietary"):
return "proprietary"
if (
p.startswith("stirling.software.SPDF")
or p.startswith("stirling.software.common")
or p.startswith("org.apache.pdfbox")
):
return "core"
return None
def parse_jacoco_methods(path: Path) -> RowBuckets:
row = RowBuckets()
if not path.exists():
return row
try:
root = _xml_parse(path).getroot()
except _XMLParseError as exc:
print(f"::warning::Failed to parse {path}: {exc}", file=sys.stderr)
return row
for pkg in root.findall("package"):
area = _classify_backend(pkg.get("name", ""))
if area is None:
continue
for counter in pkg.findall("counter"):
if counter.get("type") != "METHOD":
continue
covered = int(counter.get("covered") or 0)
missed = int(counter.get("missed") or 0)
row.by_area[area].add(Bucket(covered=covered, total=covered + missed))
return row
# --------------------------------------------------------------- vitest (frontend)
def _classify_frontend(file_path: str) -> Optional[str]:
"""Map a vitest per-file path (anything containing src/<area>/) to an area."""
if not file_path:
return None
norm = file_path.replace("\\", "/")
for area in AREAS:
if f"/src/{area}/" in norm:
return area
return None
def parse_vitest_per_file(path: Path) -> RowBuckets:
row = RowBuckets()
if not path.exists():
return row
try:
data = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
print(
f"::warning::Failed to parse vitest summary {path}: {exc}", file=sys.stderr
)
return row
for file_path, metrics in data.items():
if file_path == "total":
continue
area = _classify_frontend(file_path)
if area is None:
continue
fn = (metrics or {}).get("functions") or {}
covered = int(fn.get("covered") or 0)
total = int(fn.get("total") or 0)
row.by_area[area].add(Bucket(covered=covered, total=total))
return row
# -------------------------------------------------- playwright frontend (V8)
def parse_playwright_frontend_total(path: Path) -> RowBuckets:
"""Playwright's V8 dump aggregator only knows the grand total.
Without source maps we can't bucket per area, so the whole number
goes into `unattributed`. The matrix renderer surfaces this in the
ALL column and shows "n/a" in per-area cells with a footnote.
"""
row = RowBuckets()
if not path.exists():
return row
try:
data = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
print(
f"::warning::Failed to parse Playwright summary {path}: {exc}",
file=sys.stderr,
)
return row
fn = (data.get("total") or {}).get("functions") or {}
covered = int(fn.get("covered") or 0)
total = int(fn.get("total") or 0)
row.unattributed.add(Bucket(covered=covered, total=total))
return row
# ---------------------------------------------------------------- rendering
def _cell(bucket: Bucket) -> str:
if bucket.total == 0:
return "-"
return f"{bucket.pct:.1f}% ({bucket.covered}/{bucket.total})"
def render(rows: dict[str, RowBuckets], title: str) -> str:
cols = list(AREAS) + ["ALL"]
header = "| Row | " + " | ".join(c for c in cols) + " |"
sep = "|---" * (len(cols) + 1) + "|"
lines = [f"## {title}", "", header, sep]
for row_label, row in rows.items():
is_backend_row = row_label.lower().startswith("backend")
cells = [row_label]
for area in AREAS:
b = row.by_area[area]
unattr = row.unattributed
if area == "desktop" and is_backend_row:
cells.append("n/a")
continue
if b.total == 0 and unattr.total > 0:
cells.append("n/a")
else:
cells.append(_cell(b))
cells.append(_cell(row.all))
lines.append("| " + " | ".join(cells) + " |")
return "\n".join(lines)
# ------------------------------------------------------------------- main
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--jacoco-all", type=Path, help="aggregate jacoco-all XML")
parser.add_argument("--jacoco-e2e", type=Path, help="aggregate jacoco-e2e XML")
parser.add_argument("--vitest", type=Path, help="vitest coverage-summary.json")
parser.add_argument(
"--playwright-frontend",
type=Path,
help="Playwright frontend coverage-summary.json (from playwright-coverage-summary.py)",
)
parser.add_argument("--title", default="Coverage matrix")
parser.add_argument("--out", type=Path)
parser.add_argument("--github-step-summary", action="store_true")
parser.add_argument("--stdout", action="store_true")
args = parser.parse_args(argv)
# Build each row from its source(s).
fe_e2e = (
parse_playwright_frontend_total(args.playwright_frontend)
if args.playwright_frontend
else RowBuckets()
)
fe_unit = parse_vitest_per_file(args.vitest) if args.vitest else RowBuckets()
fe_all = RowBuckets()
fe_all.merge(fe_unit)
fe_all.merge(fe_e2e)
be_e2e = parse_jacoco_methods(args.jacoco_e2e) if args.jacoco_e2e else RowBuckets()
be_all = parse_jacoco_methods(args.jacoco_all) if args.jacoco_all else RowBuckets()
both_e2e = RowBuckets()
both_e2e.merge(fe_e2e)
both_e2e.merge(be_e2e)
both_all = RowBuckets()
both_all.merge(fe_all)
both_all.merge(be_all)
rows = {
"Frontend (e2e only)": fe_e2e,
"Frontend (all)": fe_all,
"Backend (e2e only)": be_e2e,
"Backend (all)": be_all,
"Both (e2e only)": both_e2e,
"Both (all)": both_all,
}
body = render(rows, args.title) + "\n"
# Always write to stdout when no sink is selected so the script
# is useful from the command line.
sinks: list[Path] = []
if args.out:
sinks.append(args.out)
if args.github_step_summary:
import os
gh = os.environ.get("GITHUB_STEP_SUMMARY")
if gh:
sinks.append(Path(gh))
for sink in sinks:
sink.parent.mkdir(parents=True, exist_ok=True)
with sink.open("a", encoding="utf-8") as handle:
handle.write(body)
if args.stdout or not sinks:
print(body)
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/coverage-summary.py
+259
View File
@@ -0,0 +1,259 @@
#!/usr/bin/env python3
"""Render coverage results as a GitHub-Actions-friendly markdown summary.
Reads JaCoCo XML reports (one per gradle subproject) and/or a vitest
coverage-summary.json, and writes a single Markdown block to:
- the path supplied with --out
- $GITHUB_STEP_SUMMARY when --github-step-summary is passed (and the env
var is set)
- stdout when --stdout is passed (default if no other sink is selected)
Designed to be safe to run when some inputs are missing - missing inputs
become a "skipped" note rather than an error, so the same call can be
shared across multiple CI jobs (some of which only produce one of the
two report types).
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable
# `defusedxml` swaps out the stdlib expat parser for one that rejects the
# usual XML attack vectors (XXE / billion laughs / entity expansion). Even
# though JaCoCo XML on a CI runner is trusted input, swapping the parser is
# a one-line change that silences security scanners and costs nothing.
from defusedxml.ElementTree import parse as _xml_parse
from defusedxml.ElementTree import ParseError as _XMLParseError
JACOCO_COUNTERS = ("LINE", "BRANCH", "METHOD", "CLASS", "INSTRUCTION", "COMPLEXITY")
@dataclass
class CounterTotals:
covered: int = 0
missed: int = 0
@property
def total(self) -> int:
return self.covered + self.missed
@property
def pct(self) -> float:
return 100.0 * self.covered / self.total if self.total else 0.0
def add(self, other: "CounterTotals") -> None:
self.covered += other.covered
self.missed += other.missed
def _parse_jacoco_xml(path: Path) -> dict[str, CounterTotals]:
"""Read top-level <counter> elements from a JaCoCo report XML.
ElementTree's default parser doesn't validate the DTD, so JaCoCo's
`report.dtd` reference is harmless even on offline CI runners.
Counters are direct children of the <report> root.
"""
try:
root = _xml_parse(path).getroot()
except _XMLParseError as exc:
raise RuntimeError(f"Failed to parse {path}: {exc}") from exc
out: dict[str, CounterTotals] = {}
for counter in root.findall("counter"):
t = counter.get("type") or ""
if t not in JACOCO_COUNTERS:
continue
out[t] = CounterTotals(
covered=int(counter.get("covered") or 0),
missed=int(counter.get("missed") or 0),
)
return out
def _bar(pct: float, width: int = 20) -> str:
"""Render a fixed-width ASCII progress bar. Markdown-safe on all consoles."""
filled = int(round(pct / 100.0 * width))
return "[" + "#" * filled + "-" * (width - filled) + "]"
def render_jacoco(reports: Iterable[tuple[str, Path]]) -> str:
"""Render a markdown table from one or more (label, xml_path) pairs."""
rows: list[tuple[str, dict[str, CounterTotals]]] = []
aggregate: dict[str, CounterTotals] = {t: CounterTotals() for t in JACOCO_COUNTERS}
missing: list[str] = []
for label, path in reports:
if not path.exists():
missing.append(f"`{label}` ({path})")
continue
per_counter = _parse_jacoco_xml(path)
rows.append((label, per_counter))
for t in JACOCO_COUNTERS:
if t in per_counter:
aggregate[t].add(per_counter[t])
if not rows:
body = "_No JaCoCo reports found._"
if missing:
body += "\n\n<details><summary>Searched paths</summary>\n\n"
body += "\n".join(f"- {m}" for m in missing)
body += "\n\n</details>"
return body
lines: list[str] = []
lines.append(
"| Metric | " + " | ".join(label for label, _ in rows) + " | **Aggregate** |"
)
lines.append("|---" * (len(rows) + 2) + "|")
for t in ("LINE", "BRANCH", "METHOD", "CLASS"):
cells = [t.title()]
for _, per_counter in rows:
c = per_counter.get(t)
cells.append(f"{c.pct:.1f}% ({c.covered}/{c.total})" if c else "-")
agg = aggregate[t]
cells.append(f"**{agg.pct:.1f}%** ({agg.covered}/{agg.total})")
lines.append("| " + " | ".join(cells) + " |")
agg_line = aggregate["LINE"]
lines.append("")
lines.append(
f"Aggregate line coverage: `{_bar(agg_line.pct)}` **{agg_line.pct:.1f}%** "
f"({agg_line.covered} / {agg_line.total} lines covered)"
)
if missing:
lines.append("")
lines.append("<details><summary>Skipped reports</summary>\n")
for m in missing:
lines.append(f"- {m}")
lines.append("\n</details>")
return "\n".join(lines)
def render_vitest(summary_path: Path) -> str:
"""Render a markdown summary from a vitest coverage-summary.json file.
Note: with @vitest/coverage-v8 + @vitejs/plugin-react-swc the
statements/lines counts in coverage-summary.json's `total` are
unreliable (source maps don't round-trip), so we report functions
and branches as the primary signal and call this out in the text.
"""
if not summary_path.exists():
return f"_No vitest coverage summary at `{summary_path}`._"
try:
data = json.loads(summary_path.read_text())
except json.JSONDecodeError as exc:
return f"_Failed to parse `{summary_path}`: {exc}._"
total = data.get("total") or {}
rows = []
for key in ("functions", "branches", "lines", "statements"):
v = total.get(key) or {}
covered = v.get("covered", 0)
tot = v.get("total", 0)
pct = v.get("pct", 0.0)
rows.append(f"| {key.title()} | {pct:.1f}% | {covered} / {tot} |")
body = ["| Metric | % | Covered / Total |", "|---|---|---|", *rows]
fn = total.get("functions", {})
fn_pct = fn.get("pct", 0.0)
body.append("")
body.append(f"Function coverage bar: `{_bar(fn_pct)}` **{fn_pct:.1f}%**")
body.append("")
body.append(
"> Lines/Statements use 0 as the denominator-mismatch sentinel when "
"v8 + SWC source maps don't round-trip; trust Functions and Branches "
"as the primary signal."
)
return "\n".join(body)
def _write(sinks: list[Path | None], text: str, *, to_stdout: bool) -> None:
for path in sinks:
if path is None:
continue
path.parent.mkdir(parents=True, exist_ok=True)
# GitHub job summaries are append-only by convention.
with path.open("a", encoding="utf-8") as handle:
handle.write(text)
if not text.endswith("\n"):
handle.write("\n")
if to_stdout:
print(text)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--jacoco",
action="append",
default=[],
metavar="LABEL=PATH",
help="Add a JaCoCo XML report (can be passed multiple times).",
)
parser.add_argument(
"--vitest",
type=Path,
default=None,
help="Path to a vitest coverage-summary.json file.",
)
parser.add_argument(
"--title",
default="Coverage",
help="Heading text for the summary section.",
)
parser.add_argument(
"--out",
type=Path,
default=None,
help="Append the rendered markdown to this file as well.",
)
parser.add_argument(
"--github-step-summary",
action="store_true",
help="Append to $GITHUB_STEP_SUMMARY if set.",
)
parser.add_argument(
"--stdout",
action="store_true",
help="Also write the rendered markdown to stdout.",
)
args = parser.parse_args(argv)
jacoco_reports: list[tuple[str, Path]] = []
for entry in args.jacoco:
if "=" not in entry:
parser.error(f"--jacoco expects LABEL=PATH, got {entry!r}")
label, path = entry.split("=", 1)
jacoco_reports.append((label.strip(), Path(path.strip())))
sections: list[str] = [f"## {args.title}"]
if jacoco_reports:
sections.append("### Backend (JaCoCo)")
sections.append(render_jacoco(jacoco_reports))
if args.vitest is not None:
# Sub-heading omitted on purpose: the --title argument already
# disambiguates (Vitest vs Playwright vs anything else that emits
# a coverage-summary.json), so a generic header reads cleaner.
sections.append(render_vitest(args.vitest))
if len(sections) == 1:
sections.append("_No coverage inputs provided._")
body = "\n\n".join(sections) + "\n"
sinks: list[Path | None] = [args.out]
if args.github_step_summary:
gh = os.environ.get("GITHUB_STEP_SUMMARY")
sinks.append(Path(gh) if gh else None)
to_stdout = args.stdout or not any(s for s in sinks)
_write(sinks, body, to_stdout=to_stdout)
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/playwright-coverage-summary.py
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Aggregate per-test V8 coverage dumps from Playwright into one summary.
Playwright's test-base fixture writes raw `page.coverage.stopJSCoverage()`
output to a directory (one JSON file per test, when `PW_COVERAGE=1`).
This script walks that directory and produces:
- coverage-summary.json (vitest-shaped, so the existing
scripts/coverage-summary.py renderer can consume it without changes)
Method:
- V8 reports per-function coverage as a list of ranges; the first
range covers the whole function body, and a non-zero `count` on
that outer range means the function was entered at least once.
- We deduplicate functions across the merged dumps by (script-url,
startOffset, endOffset). This avoids double-counting a function
that ran in many tests.
- Per-file line coverage requires source maps + a `v8-to-istanbul`
style walker, which is out of scope here - we report
Lines/Statements as 0/0 (the helper renders that as "not computed",
matching how it handles vitest's known v8+SWC degradation).
Filtering:
- URLs not on the local dev server (e.g. CDN assets, blob: URLs,
chrome-extension: pages) are skipped.
- URLs containing `node_modules` or vite's HMR client are skipped so
the percentage reflects app code, not framework noise.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
# URLs we deliberately don't count: vite client, hot-reload runtime,
# anything served out of node_modules, and any non-http(s) scheme.
SKIP_URL_FRAGMENTS = (
"/@vite/",
"/@react-refresh",
"/node_modules/",
"/__vite_ping",
"/__open-in-editor",
)
def _is_app_url(url: str) -> bool:
if not url:
return False
if not (url.startswith("http://") or url.startswith("https://")):
return False
if any(frag in url for frag in SKIP_URL_FRAGMENTS):
return False
return True
def aggregate(dump_dir: Path) -> dict:
files = sorted(dump_dir.glob("*.json"))
if not files:
return {
"tests": 0,
"scripts": 0,
"functions_total": 0,
"functions_covered": 0,
}
# (url, start, end) -> covered? Set semantics dedup the same function
# appearing in many test dumps.
seen: dict[tuple[str, int, int], bool] = {}
scripts_seen: set[str] = set()
for path in files:
try:
entries = json.loads(path.read_text())
except (OSError, json.JSONDecodeError):
continue
for entry in entries:
url = entry.get("url", "")
if not _is_app_url(url):
continue
scripts_seen.add(url)
for fn in entry.get("functions", []):
ranges = fn.get("ranges") or []
if not ranges:
continue
outer = ranges[0]
key = (
url,
int(outer.get("startOffset", 0)),
int(outer.get("endOffset", 0)),
)
covered = int(outer.get("count", 0)) > 0
# Promote to covered if any dump exercised it.
seen[key] = seen.get(key, False) or covered
return {
"tests": len(files),
"scripts": len(scripts_seen),
"functions_total": len(seen),
"functions_covered": sum(1 for v in seen.values() if v),
}
def write_vitest_summary(stats: dict, out_path: Path) -> None:
total = stats["functions_total"]
covered = stats["functions_covered"]
pct = 100.0 * covered / total if total else 0.0
# vitest coverage-summary.json schema. We report the function metric
# under both `functions` and `branches` so the helper script's
# "trust functions/branches" line still applies. Lines/Statements
# left at 0/0 so the renderer's degradation note kicks in.
payload = {
"total": {
"functions": {
"covered": covered,
"total": total,
"pct": pct,
"skipped": 0,
},
"branches": {
"covered": covered,
"total": total,
"pct": pct,
"skipped": 0,
},
"lines": {"covered": 0, "total": 0, "pct": 0.0, "skipped": 0},
"statements": {"covered": 0, "total": 0, "pct": 0.0, "skipped": 0},
}
}
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(payload, indent=2))
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"dump_dir",
type=Path,
help="Directory containing per-test V8 JSON dumps from the Playwright fixture.",
)
parser.add_argument(
"--out",
type=Path,
required=True,
help="Path to write the vitest-shaped coverage-summary.json",
)
args = parser.parse_args(argv)
if not args.dump_dir.exists():
print(
f"::warning::No Playwright coverage dump dir at {args.dump_dir}",
file=sys.stderr,
)
write_vitest_summary({"functions_total": 0, "functions_covered": 0}, args.out)
return 0
stats = aggregate(args.dump_dir)
write_vitest_summary(stats, args.out)
pct = (
100.0 * stats["functions_covered"] / stats["functions_total"]
if stats["functions_total"]
else 0.0
)
print(
f"Aggregated {stats['tests']} tests / {stats['scripts']} scripts: "
f"{stats['functions_covered']}/{stats['functions_total']} functions "
f"({pct:.1f}%)"
)
return 0
if __name__ == "__main__":
sys.exit(main())
testing/compose/docker-compose-coverage.override.yml
+27
View File
@@ -0,0 +1,27 @@
# Coverage override applied on top of an existing test compose file via
# `docker compose -f base.yml -f docker-compose-coverage.override.yml up`.
#
# It does NOT modify the production image. The agent jar is bind-mounted in
# from the host's `build/jacoco/jacocoagent.jar` (produced by `./gradlew
# copyJacocoAgent`) so the published Stirling-PDF image stays untouched and
# we cannot accidentally ship the agent to end users.
#
# The container appends the agent to its JVM via `JAVA_CUSTOM_OPTS`, which
# the init script concatenates into `JAVA_TOOL_OPTIONS` for the Spring Boot
# process. On graceful shutdown the agent writes
# /jacoco-out/cucumber.exec on the host (mapped to
# `testing/cucumber-coverage/cucumber.exec`) which a follow-up
# `jacocoReportFromExec` task converts into HTML+XML.
services:
stirling-pdf:
# docker-compose-security-with-login.yml renames the service container,
# so the volume mounts intentionally land on whichever service uses the
# key "stirling-pdf" in the base file.
volumes:
- ../../build/jacoco/jacocoagent.jar:/jacoco/jacocoagent.jar:ro
- ../../testing/cucumber-coverage:/jacoco-out:rw
environment:
# append=false so an aborted re-run starts clean. dumponexit=true is
# what flushes the .exec when SIGTERM hits the JVM (which happens on
# `docker compose down`).
JAVA_CUSTOM_OPTS: "-javaagent:/jacoco/jacocoagent.jar=destfile=/jacoco-out/cucumber.exec,output=file,append=false,dumponexit=true"
testing/test.sh
+45 -4
View File
@@ -509,6 +509,23 @@ verify_app_version() {
fi
}
# Optional second compose file injected on every up/down call (e.g. the
# JaCoCo coverage override). Set externally by callers that want the
# extra layer applied; left empty so production runs are unchanged.
COVERAGE_COMPOSE_FILE="${COVERAGE_COMPOSE_FILE:-}"
# Helper that joins the base compose file with any optional override into
# the `-f a -f b` form docker-compose expects. Keeps callers terse and
# means we only have one place to add new overrides later.
compose_args() {
local base=$1
if [ -n "$COVERAGE_COMPOSE_FILE" ]; then
printf -- '-f %s -f %s' "$base" "$COVERAGE_COMPOSE_FILE"
else
printf -- '-f %s' "$base"
fi
}
# Function to test a Docker Compose configuration
test_compose() {
local compose_file=$1
@@ -519,18 +536,18 @@ test_compose() {
echo "Testing ${compose_file} configuration..."
# Start up the Docker Compose service
docker-compose -f "$compose_file" up -d
docker-compose $(compose_args "$compose_file") up -d
# Wait a moment for containers to appear
sleep 3
local container_name
container_name=$(docker-compose -f "$compose_file" ps --format '{{.Names}}' --filter "status=running" | head -n1)
container_name=$(docker-compose $(compose_args "$compose_file") ps --format '{{.Names}}' --filter "status=running" | head -n1)
if [[ -z "$container_name" ]]; then
echo "ERROR: No running container found for ${compose_file}"
local compose_output
compose_output=$(docker-compose -f "$compose_file" ps 2>&1)
compose_output=$(docker-compose $(compose_args "$compose_file") ps 2>&1)
echo "$compose_output"
capture_failure_logs "$test_name" "" "docker-compose failed for: ${compose_file}
${compose_output}"
@@ -856,6 +873,24 @@ main() {
# ==================================================================
# 3. Regression test with login (test_cicd.yml)
# ==================================================================
# STIRLING_PDF_TEST_COVERAGE=1 layers the JaCoCo agent override over
# the cucumber container ONLY. The agent jar is bind-mounted from
# build/jacoco/jacocoagent.jar so the published image never carries
# it. After behave finishes, we trigger `docker compose down` (further
# down) which sends SIGTERM and lets dumponexit=true flush the .exec
# to testing/cucumber-coverage/cucumber.exec on the host.
COVERAGE_COMPOSE_FILE=""
if [ -n "${STIRLING_PDF_TEST_COVERAGE:-}" ]; then
if [ ! -f "$PROJECT_ROOT/build/jacoco/jacocoagent.jar" ]; then
echo "::warning::STIRLING_PDF_TEST_COVERAGE=1 but build/jacoco/jacocoagent.jar is missing - run ./gradlew copyJacocoAgent first"
else
mkdir -p "$PROJECT_ROOT/testing/cucumber-coverage"
rm -f "$PROJECT_ROOT/testing/cucumber-coverage/cucumber.exec"
COVERAGE_COMPOSE_FILE="$PROJECT_ROOT/testing/compose/docker-compose-coverage.override.yml"
echo "Cucumber JaCoCo coverage enabled - exec will land at testing/cucumber-coverage/cucumber.exec"
fi
fi
run_tests "Stirling-PDF-Security-Fat-with-login" "./docker/embedded/compose/test_cicd.yml"
# Only run behave tests if the container started successfully
@@ -979,7 +1014,13 @@ main() {
fi
stop_test_timer "Stirling-PDF-Regression"
fi
docker-compose -f "./docker/embedded/compose/test_cicd.yml" down -v
# `down` with the override removes the agent bind-mount cleanly. The
# SIGTERM that `down` sends is what triggers dumponexit=true in the
# agent to flush testing/cucumber-coverage/cucumber.exec to disk.
docker-compose $(compose_args "./docker/embedded/compose/test_cicd.yml") down -v
# Reset so subsequent compose calls (CI cleanup, non-cucumber tests)
# don't accidentally pick up the override.
COVERAGE_COMPOSE_FILE=""
# ==================================================================
# 4. Disabled Endpoints Test