Add CI coverage summaries and aggregate JaCoCo report (#6451)

This commit is contained in:
Anthony Stirling
2026-06-02 14:59:10 +01:00
committed by GitHub
parent 2c0ebc28a7
commit de9d6ad3f5
15 changed files with 1583 additions and 23 deletions
+325
View File
@@ -0,0 +1,325 @@
#!/usr/bin/env python3
"""Render a single coverage matrix combining backend + frontend, per area.
Rows:
- Frontend (e2e only) - Playwright fixture V8 capture
- Frontend (all) - Vitest unit tests + Playwright V8
- Backend (e2e only) - Playwright live JaCoCo + cucumber JaCoCo
- Backend (all) - JUnit + e2e:live + cucumber JaCoCo
- Both (e2e only) - frontend (e2e) + backend (e2e), summed
- Both (all) - frontend (all) + backend (all), summed
Columns:
- core, proprietary, saas, desktop - bucketed per area
- ALL - sum across the row
Bucketing rules:
Backend (JaCoCo package -> area):
stirling/software/SPDF/** -> core (the main backend module)
stirling/software/common/** -> core (shared infra, attributed to core)
org/apache/pdfbox/** -> core (vendored helpers in common/core)
stirling/software/proprietary/** -> proprietary (unless saas-flavoured)
stirling/software/saas/** -> saas
*desktop* -> desktop (no real backend match today)
Frontend (source path -> area):
frontend/editor/src/core/** -> core
frontend/editor/src/proprietary/** -> proprietary
frontend/editor/src/saas/** -> saas
frontend/editor/src/desktop/** -> desktop
Cells render as `pct% (covered/total)` where the metric is:
- Backend: JaCoCo METHOD counter (most directly comparable to JS funcs)
- Frontend: function counts from vitest per-file summary
Playwright (live) frontend V8 dumps only know about bundled JS URLs, not
source paths, so they only feed the ALL column for the Frontend rows.
Per-area Playwright contributions show "n/a" with a footnote.
Missing inputs are tolerated: any source that isn't passed renders as `-`
and the row aggregates from what is available.
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
# defusedxml hardens the parser against XXE / billion-laughs / entity-
# expansion attacks. JaCoCo XML on a CI runner is trusted input today,
# but using the hardened parser is a one-line change and silences
# scanners that pattern-match on `xml.etree.ElementTree.parse`.
from defusedxml.ElementTree import parse as _xml_parse
from defusedxml.ElementTree import ParseError as _XMLParseError
AREAS = ("core", "proprietary", "saas", "desktop")
@dataclass
class Bucket:
"""Covered + total counters that can be combined across sources."""
covered: int = 0
total: int = 0
@property
def pct(self) -> float:
return 100.0 * self.covered / self.total if self.total else 0.0
def add(self, other: "Bucket") -> None:
self.covered += other.covered
self.total += other.total
@dataclass
class RowBuckets:
"""Per-area buckets for one row of the matrix."""
by_area: dict[str, Bucket] = field(
default_factory=lambda: {a: Bucket() for a in AREAS}
)
# Some inputs (Playwright V8) don't have source-path info, so they
# only contribute to ALL without an area attribution. Track those
# separately so per-area cells stay honest.
unattributed: Bucket = field(default_factory=Bucket)
@property
def all(self) -> Bucket:
agg = Bucket()
for b in self.by_area.values():
agg.add(b)
agg.add(self.unattributed)
return agg
def merge(self, other: "RowBuckets") -> None:
for area in AREAS:
self.by_area[area].add(other.by_area[area])
self.unattributed.add(other.unattributed)
# --------------------------------------------------------------------- jacoco
def _classify_backend(package_name: str) -> Optional[str]:
"""Map a JaCoCo package name to an area, or None to skip."""
if not package_name:
return None
p = package_name.replace("/", ".")
if "desktop" in p:
return "desktop"
if p.startswith("stirling.software.saas"):
return "saas"
# Proprietary saas-flavoured sub-package: rare, but kept for forward
# compatibility if a future build moves saas under proprietary.
if p.startswith("stirling.software.proprietary") and ".saas" in p:
return "saas"
if p.startswith("stirling.software.proprietary"):
return "proprietary"
if (
p.startswith("stirling.software.SPDF")
or p.startswith("stirling.software.common")
or p.startswith("org.apache.pdfbox")
):
return "core"
return None
def parse_jacoco_methods(path: Path) -> RowBuckets:
row = RowBuckets()
if not path.exists():
return row
try:
root = _xml_parse(path).getroot()
except _XMLParseError as exc:
print(f"::warning::Failed to parse {path}: {exc}", file=sys.stderr)
return row
for pkg in root.findall("package"):
area = _classify_backend(pkg.get("name", ""))
if area is None:
continue
for counter in pkg.findall("counter"):
if counter.get("type") != "METHOD":
continue
covered = int(counter.get("covered") or 0)
missed = int(counter.get("missed") or 0)
row.by_area[area].add(Bucket(covered=covered, total=covered + missed))
return row
# --------------------------------------------------------------- vitest (frontend)
def _classify_frontend(file_path: str) -> Optional[str]:
"""Map a vitest per-file path (anything containing src/<area>/) to an area."""
if not file_path:
return None
norm = file_path.replace("\\", "/")
for area in AREAS:
if f"/src/{area}/" in norm:
return area
return None
def parse_vitest_per_file(path: Path) -> RowBuckets:
row = RowBuckets()
if not path.exists():
return row
try:
data = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
print(
f"::warning::Failed to parse vitest summary {path}: {exc}", file=sys.stderr
)
return row
for file_path, metrics in data.items():
if file_path == "total":
continue
area = _classify_frontend(file_path)
if area is None:
continue
fn = (metrics or {}).get("functions") or {}
covered = int(fn.get("covered") or 0)
total = int(fn.get("total") or 0)
row.by_area[area].add(Bucket(covered=covered, total=total))
return row
# -------------------------------------------------- playwright frontend (V8)
def parse_playwright_frontend_total(path: Path) -> RowBuckets:
"""Playwright's V8 dump aggregator only knows the grand total.
Without source maps we can't bucket per area, so the whole number
goes into `unattributed`. The matrix renderer surfaces this in the
ALL column and shows "n/a" in per-area cells with a footnote.
"""
row = RowBuckets()
if not path.exists():
return row
try:
data = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
print(
f"::warning::Failed to parse Playwright summary {path}: {exc}",
file=sys.stderr,
)
return row
fn = (data.get("total") or {}).get("functions") or {}
covered = int(fn.get("covered") or 0)
total = int(fn.get("total") or 0)
row.unattributed.add(Bucket(covered=covered, total=total))
return row
# ---------------------------------------------------------------- rendering
def _cell(bucket: Bucket) -> str:
if bucket.total == 0:
return "-"
return f"{bucket.pct:.1f}% ({bucket.covered}/{bucket.total})"
def render(rows: dict[str, RowBuckets], title: str) -> str:
cols = list(AREAS) + ["ALL"]
header = "| Row | " + " | ".join(c for c in cols) + " |"
sep = "|---" * (len(cols) + 1) + "|"
lines = [f"## {title}", "", header, sep]
for row_label, row in rows.items():
is_backend_row = row_label.lower().startswith("backend")
cells = [row_label]
for area in AREAS:
b = row.by_area[area]
unattr = row.unattributed
if area == "desktop" and is_backend_row:
cells.append("n/a")
continue
if b.total == 0 and unattr.total > 0:
cells.append("n/a")
else:
cells.append(_cell(b))
cells.append(_cell(row.all))
lines.append("| " + " | ".join(cells) + " |")
return "\n".join(lines)
# ------------------------------------------------------------------- main
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--jacoco-all", type=Path, help="aggregate jacoco-all XML")
parser.add_argument("--jacoco-e2e", type=Path, help="aggregate jacoco-e2e XML")
parser.add_argument("--vitest", type=Path, help="vitest coverage-summary.json")
parser.add_argument(
"--playwright-frontend",
type=Path,
help="Playwright frontend coverage-summary.json (from playwright-coverage-summary.py)",
)
parser.add_argument("--title", default="Coverage matrix")
parser.add_argument("--out", type=Path)
parser.add_argument("--github-step-summary", action="store_true")
parser.add_argument("--stdout", action="store_true")
args = parser.parse_args(argv)
# Build each row from its source(s).
fe_e2e = (
parse_playwright_frontend_total(args.playwright_frontend)
if args.playwright_frontend
else RowBuckets()
)
fe_unit = parse_vitest_per_file(args.vitest) if args.vitest else RowBuckets()
fe_all = RowBuckets()
fe_all.merge(fe_unit)
fe_all.merge(fe_e2e)
be_e2e = parse_jacoco_methods(args.jacoco_e2e) if args.jacoco_e2e else RowBuckets()
be_all = parse_jacoco_methods(args.jacoco_all) if args.jacoco_all else RowBuckets()
both_e2e = RowBuckets()
both_e2e.merge(fe_e2e)
both_e2e.merge(be_e2e)
both_all = RowBuckets()
both_all.merge(fe_all)
both_all.merge(be_all)
rows = {
"Frontend (e2e only)": fe_e2e,
"Frontend (all)": fe_all,
"Backend (e2e only)": be_e2e,
"Backend (all)": be_all,
"Both (e2e only)": both_e2e,
"Both (all)": both_all,
}
body = render(rows, args.title) + "\n"
# Always write to stdout when no sink is selected so the script
# is useful from the command line.
sinks: list[Path] = []
if args.out:
sinks.append(args.out)
if args.github_step_summary:
import os
gh = os.environ.get("GITHUB_STEP_SUMMARY")
if gh:
sinks.append(Path(gh))
for sink in sinks:
sink.parent.mkdir(parents=True, exist_ok=True)
with sink.open("a", encoding="utf-8") as handle:
handle.write(body)
if args.stdout or not sinks:
print(body)
return 0
if __name__ == "__main__":
sys.exit(main())
+259
View File
@@ -0,0 +1,259 @@
#!/usr/bin/env python3
"""Render coverage results as a GitHub-Actions-friendly markdown summary.
Reads JaCoCo XML reports (one per gradle subproject) and/or a vitest
coverage-summary.json, and writes a single Markdown block to:
- the path supplied with --out
- $GITHUB_STEP_SUMMARY when --github-step-summary is passed (and the env
var is set)
- stdout when --stdout is passed (default if no other sink is selected)
Designed to be safe to run when some inputs are missing - missing inputs
become a "skipped" note rather than an error, so the same call can be
shared across multiple CI jobs (some of which only produce one of the
two report types).
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable
# `defusedxml` swaps out the stdlib expat parser for one that rejects the
# usual XML attack vectors (XXE / billion laughs / entity expansion). Even
# though JaCoCo XML on a CI runner is trusted input, swapping the parser is
# a one-line change that silences security scanners and costs nothing.
from defusedxml.ElementTree import parse as _xml_parse
from defusedxml.ElementTree import ParseError as _XMLParseError
JACOCO_COUNTERS = ("LINE", "BRANCH", "METHOD", "CLASS", "INSTRUCTION", "COMPLEXITY")
@dataclass
class CounterTotals:
covered: int = 0
missed: int = 0
@property
def total(self) -> int:
return self.covered + self.missed
@property
def pct(self) -> float:
return 100.0 * self.covered / self.total if self.total else 0.0
def add(self, other: "CounterTotals") -> None:
self.covered += other.covered
self.missed += other.missed
def _parse_jacoco_xml(path: Path) -> dict[str, CounterTotals]:
"""Read top-level <counter> elements from a JaCoCo report XML.
ElementTree's default parser doesn't validate the DTD, so JaCoCo's
`report.dtd` reference is harmless even on offline CI runners.
Counters are direct children of the <report> root.
"""
try:
root = _xml_parse(path).getroot()
except _XMLParseError as exc:
raise RuntimeError(f"Failed to parse {path}: {exc}") from exc
out: dict[str, CounterTotals] = {}
for counter in root.findall("counter"):
t = counter.get("type") or ""
if t not in JACOCO_COUNTERS:
continue
out[t] = CounterTotals(
covered=int(counter.get("covered") or 0),
missed=int(counter.get("missed") or 0),
)
return out
def _bar(pct: float, width: int = 20) -> str:
"""Render a fixed-width ASCII progress bar. Markdown-safe on all consoles."""
filled = int(round(pct / 100.0 * width))
return "[" + "#" * filled + "-" * (width - filled) + "]"
def render_jacoco(reports: Iterable[tuple[str, Path]]) -> str:
"""Render a markdown table from one or more (label, xml_path) pairs."""
rows: list[tuple[str, dict[str, CounterTotals]]] = []
aggregate: dict[str, CounterTotals] = {t: CounterTotals() for t in JACOCO_COUNTERS}
missing: list[str] = []
for label, path in reports:
if not path.exists():
missing.append(f"`{label}` ({path})")
continue
per_counter = _parse_jacoco_xml(path)
rows.append((label, per_counter))
for t in JACOCO_COUNTERS:
if t in per_counter:
aggregate[t].add(per_counter[t])
if not rows:
body = "_No JaCoCo reports found._"
if missing:
body += "\n\n<details><summary>Searched paths</summary>\n\n"
body += "\n".join(f"- {m}" for m in missing)
body += "\n\n</details>"
return body
lines: list[str] = []
lines.append(
"| Metric | " + " | ".join(label for label, _ in rows) + " | **Aggregate** |"
)
lines.append("|---" * (len(rows) + 2) + "|")
for t in ("LINE", "BRANCH", "METHOD", "CLASS"):
cells = [t.title()]
for _, per_counter in rows:
c = per_counter.get(t)
cells.append(f"{c.pct:.1f}% ({c.covered}/{c.total})" if c else "-")
agg = aggregate[t]
cells.append(f"**{agg.pct:.1f}%** ({agg.covered}/{agg.total})")
lines.append("| " + " | ".join(cells) + " |")
agg_line = aggregate["LINE"]
lines.append("")
lines.append(
f"Aggregate line coverage: `{_bar(agg_line.pct)}` **{agg_line.pct:.1f}%** "
f"({agg_line.covered} / {agg_line.total} lines covered)"
)
if missing:
lines.append("")
lines.append("<details><summary>Skipped reports</summary>\n")
for m in missing:
lines.append(f"- {m}")
lines.append("\n</details>")
return "\n".join(lines)
def render_vitest(summary_path: Path) -> str:
"""Render a markdown summary from a vitest coverage-summary.json file.
Note: with @vitest/coverage-v8 + @vitejs/plugin-react-swc the
statements/lines counts in coverage-summary.json's `total` are
unreliable (source maps don't round-trip), so we report functions
and branches as the primary signal and call this out in the text.
"""
if not summary_path.exists():
return f"_No vitest coverage summary at `{summary_path}`._"
try:
data = json.loads(summary_path.read_text())
except json.JSONDecodeError as exc:
return f"_Failed to parse `{summary_path}`: {exc}._"
total = data.get("total") or {}
rows = []
for key in ("functions", "branches", "lines", "statements"):
v = total.get(key) or {}
covered = v.get("covered", 0)
tot = v.get("total", 0)
pct = v.get("pct", 0.0)
rows.append(f"| {key.title()} | {pct:.1f}% | {covered} / {tot} |")
body = ["| Metric | % | Covered / Total |", "|---|---|---|", *rows]
fn = total.get("functions", {})
fn_pct = fn.get("pct", 0.0)
body.append("")
body.append(f"Function coverage bar: `{_bar(fn_pct)}` **{fn_pct:.1f}%**")
body.append("")
body.append(
"> Lines/Statements use 0 as the denominator-mismatch sentinel when "
"v8 + SWC source maps don't round-trip; trust Functions and Branches "
"as the primary signal."
)
return "\n".join(body)
def _write(sinks: list[Path | None], text: str, *, to_stdout: bool) -> None:
for path in sinks:
if path is None:
continue
path.parent.mkdir(parents=True, exist_ok=True)
# GitHub job summaries are append-only by convention.
with path.open("a", encoding="utf-8") as handle:
handle.write(text)
if not text.endswith("\n"):
handle.write("\n")
if to_stdout:
print(text)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--jacoco",
action="append",
default=[],
metavar="LABEL=PATH",
help="Add a JaCoCo XML report (can be passed multiple times).",
)
parser.add_argument(
"--vitest",
type=Path,
default=None,
help="Path to a vitest coverage-summary.json file.",
)
parser.add_argument(
"--title",
default="Coverage",
help="Heading text for the summary section.",
)
parser.add_argument(
"--out",
type=Path,
default=None,
help="Append the rendered markdown to this file as well.",
)
parser.add_argument(
"--github-step-summary",
action="store_true",
help="Append to $GITHUB_STEP_SUMMARY if set.",
)
parser.add_argument(
"--stdout",
action="store_true",
help="Also write the rendered markdown to stdout.",
)
args = parser.parse_args(argv)
jacoco_reports: list[tuple[str, Path]] = []
for entry in args.jacoco:
if "=" not in entry:
parser.error(f"--jacoco expects LABEL=PATH, got {entry!r}")
label, path = entry.split("=", 1)
jacoco_reports.append((label.strip(), Path(path.strip())))
sections: list[str] = [f"## {args.title}"]
if jacoco_reports:
sections.append("### Backend (JaCoCo)")
sections.append(render_jacoco(jacoco_reports))
if args.vitest is not None:
# Sub-heading omitted on purpose: the --title argument already
# disambiguates (Vitest vs Playwright vs anything else that emits
# a coverage-summary.json), so a generic header reads cleaner.
sections.append(render_vitest(args.vitest))
if len(sections) == 1:
sections.append("_No coverage inputs provided._")
body = "\n\n".join(sections) + "\n"
sinks: list[Path | None] = [args.out]
if args.github_step_summary:
gh = os.environ.get("GITHUB_STEP_SUMMARY")
sinks.append(Path(gh) if gh else None)
to_stdout = args.stdout or not any(s for s in sinks)
_write(sinks, body, to_stdout=to_stdout)
return 0
if __name__ == "__main__":
sys.exit(main())
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Aggregate per-test V8 coverage dumps from Playwright into one summary.
Playwright's test-base fixture writes raw `page.coverage.stopJSCoverage()`
output to a directory (one JSON file per test, when `PW_COVERAGE=1`).
This script walks that directory and produces:
- coverage-summary.json (vitest-shaped, so the existing
scripts/coverage-summary.py renderer can consume it without changes)
Method:
- V8 reports per-function coverage as a list of ranges; the first
range covers the whole function body, and a non-zero `count` on
that outer range means the function was entered at least once.
- We deduplicate functions across the merged dumps by (script-url,
startOffset, endOffset). This avoids double-counting a function
that ran in many tests.
- Per-file line coverage requires source maps + a `v8-to-istanbul`
style walker, which is out of scope here - we report
Lines/Statements as 0/0 (the helper renders that as "not computed",
matching how it handles vitest's known v8+SWC degradation).
Filtering:
- URLs not on the local dev server (e.g. CDN assets, blob: URLs,
chrome-extension: pages) are skipped.
- URLs containing `node_modules` or vite's HMR client are skipped so
the percentage reflects app code, not framework noise.
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
# URLs we deliberately don't count: vite client, hot-reload runtime,
# anything served out of node_modules, and any non-http(s) scheme.
SKIP_URL_FRAGMENTS = (
"/@vite/",
"/@react-refresh",
"/node_modules/",
"/__vite_ping",
"/__open-in-editor",
)
def _is_app_url(url: str) -> bool:
if not url:
return False
if not (url.startswith("http://") or url.startswith("https://")):
return False
if any(frag in url for frag in SKIP_URL_FRAGMENTS):
return False
return True
def aggregate(dump_dir: Path) -> dict:
files = sorted(dump_dir.glob("*.json"))
if not files:
return {
"tests": 0,
"scripts": 0,
"functions_total": 0,
"functions_covered": 0,
}
# (url, start, end) -> covered? Set semantics dedup the same function
# appearing in many test dumps.
seen: dict[tuple[str, int, int], bool] = {}
scripts_seen: set[str] = set()
for path in files:
try:
entries = json.loads(path.read_text())
except (OSError, json.JSONDecodeError):
continue
for entry in entries:
url = entry.get("url", "")
if not _is_app_url(url):
continue
scripts_seen.add(url)
for fn in entry.get("functions", []):
ranges = fn.get("ranges") or []
if not ranges:
continue
outer = ranges[0]
key = (
url,
int(outer.get("startOffset", 0)),
int(outer.get("endOffset", 0)),
)
covered = int(outer.get("count", 0)) > 0
# Promote to covered if any dump exercised it.
seen[key] = seen.get(key, False) or covered
return {
"tests": len(files),
"scripts": len(scripts_seen),
"functions_total": len(seen),
"functions_covered": sum(1 for v in seen.values() if v),
}
def write_vitest_summary(stats: dict, out_path: Path) -> None:
total = stats["functions_total"]
covered = stats["functions_covered"]
pct = 100.0 * covered / total if total else 0.0
# vitest coverage-summary.json schema. We report the function metric
# under both `functions` and `branches` so the helper script's
# "trust functions/branches" line still applies. Lines/Statements
# left at 0/0 so the renderer's degradation note kicks in.
payload = {
"total": {
"functions": {
"covered": covered,
"total": total,
"pct": pct,
"skipped": 0,
},
"branches": {
"covered": covered,
"total": total,
"pct": pct,
"skipped": 0,
},
"lines": {"covered": 0, "total": 0, "pct": 0.0, "skipped": 0},
"statements": {"covered": 0, "total": 0, "pct": 0.0, "skipped": 0},
}
}
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(payload, indent=2))
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"dump_dir",
type=Path,
help="Directory containing per-test V8 JSON dumps from the Playwright fixture.",
)
parser.add_argument(
"--out",
type=Path,
required=True,
help="Path to write the vitest-shaped coverage-summary.json",
)
args = parser.parse_args(argv)
if not args.dump_dir.exists():
print(
f"::warning::No Playwright coverage dump dir at {args.dump_dir}",
file=sys.stderr,
)
write_vitest_summary({"functions_total": 0, "functions_covered": 0}, args.out)
return 0
stats = aggregate(args.dump_dir)
write_vitest_summary(stats, args.out)
pct = (
100.0 * stats["functions_covered"] / stats["functions_total"]
if stats["functions_total"]
else 0.0
)
print(
f"Aggregated {stats['tests']} tests / {stats['scripts']} scripts: "
f"{stats['functions_covered']}/{stats['functions_total']} functions "
f"({pct:.1f}%)"
)
return 0
if __name__ == "__main__":
sys.exit(main())