Pdf comment agent (#6196)

Co-authored-by: James Brunton <[email protected]>

engine/tests/agents/test_math_presentation.py

@@ -0,0 +1,106 @@
+"""Tests for ``stirling.agents.math_presentation``.
+
+Only one helper lives in this module now: Verdict-artifact extraction
+on the resume turn. Math intent itself is decided by the orchestrator's
+top-level LLM and passed in as a flag, so there's no English regex to
+test here. Verdict → prose / sticky-note text are the consumer agents'
+responsibility — those projections are tested with each consumer.
+"""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from stirling.agents.math_presentation import extract_math_verdict
+from stirling.contracts import (
+    ExtractedFileText,
+    ExtractedTextArtifact,
+    MathAuditorToolReportArtifact,
+    OrchestratorRequest,
+    WorkflowArtifact,
+)
+from stirling.contracts.ledger import Discrepancy, DiscrepancyKind, Severity, Verdict
+
+
+def _make_verdict(discrepancies: list[Discrepancy]) -> Verdict:
+    return Verdict(
+        session_id="s1",
+        discrepancies=discrepancies,
+        pages_examined=[d.page for d in discrepancies] or [0],
+        rounds_taken=1,
+        summary="Test verdict.",
+        clean=not discrepancies,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Resume-turn round-trip — ToolReportArtifact → Verdict
+# ---------------------------------------------------------------------------
+
+
+def _orchestrator_request_with_artifacts(artifacts: list[WorkflowArtifact]) -> OrchestratorRequest:
+    return OrchestratorRequest(
+        user_message="review the math",
+        file_names=["report.pdf"],
+        artifacts=artifacts,
+    )
+
+
+def test_extract_math_verdict_roundtrips_a_math_auditor_report() -> None:
+    """When the math auditor has already run, Java re-enters the orchestrator with
+    a ToolReportArtifact carrying the serialised Verdict; the meta-agent's first
+    job on the resume turn is to hydrate that back into a Verdict."""
+    original = _make_verdict(
+        [
+            Discrepancy(
+                page=0,
+                kind=DiscrepancyKind.TALLY,
+                severity=Severity.ERROR,
+                description="Total mismatch.",
+                stated="$215,000",
+                expected="$215,500",
+                context="Revenue row",
+            )
+        ]
+    )
+    artifact = MathAuditorToolReportArtifact(report=original)
+    request = _orchestrator_request_with_artifacts([artifact])
+
+    verdict = extract_math_verdict(request)
+
+    assert verdict is not None
+    assert len(verdict.discrepancies) == 1
+    assert verdict.discrepancies[0].stated == "$215,000"
+    assert verdict.discrepancies[0].expected == "$215,500"
+
+
+def test_extract_math_verdict_returns_none_when_no_artifacts_present() -> None:
+    """First turn — the plan has not yet run, so artifacts is empty."""
+    request = _orchestrator_request_with_artifacts([])
+    assert extract_math_verdict(request) is None
+
+
+def test_extract_math_verdict_ignores_other_artifact_kinds() -> None:
+    """Only MathAuditorToolReportArtifact counts. Other artifact kinds (e.g.
+    extracted page text from a NeedContent round-trip) must be ignored here so
+    meta-agents don't misinterpret them as math reports."""
+    unrelated = ExtractedTextArtifact(
+        files=[ExtractedFileText(file_name="report.pdf", pages=[])],
+    )
+    request = _orchestrator_request_with_artifacts([unrelated])
+    assert extract_math_verdict(request) is None
+
+
+def test_malformed_math_auditor_report_is_rejected_at_validation_time() -> None:
+    """The discriminated-union contract validates the report payload as a
+    :class:`Verdict` on receipt — a corrupt body raises at construction time
+    rather than silently surviving until the meta-agent tries to read it."""
+    with pytest.raises(ValidationError):
+        MathAuditorToolReportArtifact.model_validate(
+            {
+                "kind": "tool_report",
+                "source_tool": "math_auditor_agent",
+                "report": {"not_a_verdict_field": "garbage"},
+            }
+        )

engine/tests/agents/test_orchestrator_pdf_comment.py

@@ -0,0 +1,60 @@
+"""
+Orchestrator ``delegate_pdf_review`` contract test.
+
+The real orchestrator delegates PDF-review requests via a pydantic-ai tool
+output. Exercising the full ``agent.run(...)`` call would hit the LLM and
+requires building a real ``RunContext`` — so instead this test invokes
+``delegate_pdf_review`` directly with a minimal ``deps`` stand-in. That's
+enough to verify the wire contract the orchestrator produces:
+
+* it returns an ``EditPlanResponse``;
+* with exactly one step;
+* whose ``tool`` is ``AgentToolId.PDF_COMMENT_AGENT`` (the composed AI tool
+  under ``/api/v1/ai/tools/pdf-comment-agent``);
+* whose ``parameters.prompt`` echoes the user's request.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from stirling.agents import OrchestratorAgent
+from stirling.contracts import OrchestratorRequest
+from stirling.contracts.pdf_edit import EditPlanResponse
+from stirling.models.agent_tool_models import AgentToolId, PdfCommentAgentParams
+from stirling.services.runtime import AppRuntime
+
+
+@dataclass(frozen=True)
+class _FakeDeps:
+    request: OrchestratorRequest
+
+
+@pytest.mark.anyio
+async def test_delegate_pdf_review_wires_prompt_to_tool_step(runtime: AppRuntime) -> None:
+    orchestrator = OrchestratorAgent(runtime)
+    request = OrchestratorRequest(
+        user_message="please add review comments flagging ambiguous dates",
+        file_names=["contract.pdf"],
+    )
+    ctx = SimpleNamespace(deps=_FakeDeps(request=request))
+
+    # PdfReviewAgent now classifies math intent locally via a tiny LLM. Stub it
+    # to false so this test stays focused on the prose-review wire contract.
+    with patch(
+        "stirling.agents.pdf_review.MathIntentClassifier.classify",
+        new=AsyncMock(return_value=False),
+    ):
+        response = await orchestrator.delegate_pdf_review(ctx)  # type: ignore[arg-type]
+
+    assert isinstance(response, EditPlanResponse)
+    assert len(response.steps) == 1
+    step = response.steps[0]
+    assert step.tool == AgentToolId.PDF_COMMENT_AGENT
+    assert step.tool.value == "/api/v1/ai/tools/pdf-comment-agent"
+    assert isinstance(step.parameters, PdfCommentAgentParams)
+    assert step.parameters.prompt == request.user_message

engine/tests/agents/test_pdf_questions_orchestrate.py

@@ -0,0 +1,101 @@
+"""Tests for ``PdfQuestionAgent.orchestrate`` — classifier-driven first-turn
+routing and prompt pinning. The legacy text-grounded ``handle`` path is
+covered separately in ``tests/test_pdf_question_agent.py``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from stirling.agents.pdf_questions import _MATH_SYNTH_SYSTEM_PROMPT, PdfQuestionAgent
+from stirling.contracts import (
+    MathAuditorToolReportArtifact,
+    OrchestratorRequest,
+    PdfQuestionAnswerResponse,
+    SupportedCapability,
+)
+from stirling.contracts.ledger import Discrepancy, DiscrepancyKind, Severity, Verdict
+from stirling.models.agent_tool_models import AgentToolId
+from stirling.services.runtime import AppRuntime
+
+
+@dataclass
+class _StubResult:
+    output: str
+
+
+def _make_verdict() -> Verdict:
+    return Verdict(
+        session_id="s1",
+        discrepancies=[
+            Discrepancy(
+                page=0,
+                kind=DiscrepancyKind.TALLY,
+                severity=Severity.ERROR,
+                description="Total mismatch.",
+                stated="$215,000",
+                expected="$215,500",
+                context="Revenue row",
+            )
+        ],
+        pages_examined=[0],
+        rounds_taken=1,
+        summary="One discrepancy.",
+        clean=False,
+    )
+
+
+@pytest.mark.anyio
+async def test_orchestrate_classifier_true_embeds_plan_in_answer(runtime: AppRuntime) -> None:
+    """First turn — classifier says math; the response is a PdfQuestionAnswerResponse
+    with the math-auditor plan attached as a nullable ``edit_plan`` field. The
+    answer is empty on this turn; the caller runs the embedded plan and resumes."""
+    agent = PdfQuestionAgent(runtime)
+    request = OrchestratorRequest(
+        user_message="ist die mathematik korrekt?",
+        file_names=["report.pdf"],
+    )
+
+    with patch.object(agent._math_intent_classifier, "classify", AsyncMock(return_value=True)):
+        response = await agent.orchestrate(request)
+
+    assert isinstance(response, PdfQuestionAnswerResponse)
+    assert response.answer == ""
+    assert response.edit_plan is not None
+    assert response.edit_plan.resume_with == SupportedCapability.PDF_QUESTION
+    assert len(response.edit_plan.steps) == 1
+    assert response.edit_plan.steps[0].tool == AgentToolId.MATH_AUDITOR_AGENT
+
+
+@pytest.mark.anyio
+async def test_orchestrate_resume_synthesises_answer_without_calling_classifier(
+    runtime: AppRuntime,
+) -> None:
+    """Resume turn — Verdict in artifacts. The math-synth LLM is mocked; we
+    verify the answer is plumbed through and that the classifier is short-
+    circuited (no point asking 'is this math?' when we already have a Verdict)."""
+    agent = PdfQuestionAgent(runtime)
+    verdict = _make_verdict()
+    request = OrchestratorRequest(
+        user_message="ist die mathematik korrekt?",
+        file_names=["report.pdf"],
+        artifacts=[MathAuditorToolReportArtifact(report=verdict)],
+    )
+    canned_answer = "Die Summe stimmt nicht: angegeben $215,000, erwartet $215,500."
+    classifier_mock = AsyncMock(return_value=False)
+    with patch.object(agent._math_synth_agent, "run", return_value=_StubResult(output=canned_answer)):
+        with patch.object(agent._math_intent_classifier, "classify", classifier_mock):
+            response = await agent.orchestrate(request)
+
+    assert isinstance(response, PdfQuestionAnswerResponse)
+    assert response.answer == canned_answer
+    classifier_mock.assert_not_called()
+
+
+def test_math_synth_prompt_requires_verbatim_quoting() -> None:
+    """If this prompt is rephrased and drops the verbatim rule, the LLM may
+    paraphrase numeric values from the Verdict."""
+    assert "verbatim" in _MATH_SYNTH_SYSTEM_PROMPT.lower()

engine/tests/agents/test_pdf_review.py

@@ -0,0 +1,216 @@
+"""Tests for ``PdfReviewAgent``.
+
+LLM-localised text is the consumer's responsibility (verified by mocking
+the localiser agent), but the deterministic placement geometry —
+anchor-text selection, per-page stacking, fallback right-margin — is pure
+Python and worth pinning here.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from stirling.agents.pdf_review import (
+    _LOCALISER_SYSTEM_PROMPT,
+    PdfReviewAgent,
+    _LocalisedComment,
+    _LocalisedVerdict,
+)
+from stirling.contracts import EditPlanResponse, OrchestratorRequest, SupportedCapability
+from stirling.contracts.ledger import Discrepancy, DiscrepancyKind, Severity, Verdict
+from stirling.models import ToolEndpoint
+from stirling.models.agent_tool_models import AgentToolId, PdfCommentAgentParams
+from stirling.services.runtime import AppRuntime
+
+
+@dataclass
+class _StubResult:
+    output: _LocalisedVerdict
+
+
+def _make_verdict(discrepancies: list[Discrepancy]) -> Verdict:
+    return Verdict(
+        session_id="s1",
+        discrepancies=discrepancies,
+        pages_examined=[d.page for d in discrepancies] or [0],
+        rounds_taken=1,
+        summary="Test verdict.",
+        clean=not discrepancies,
+    )
+
+
+def _discrepancy(page: int = 0, stated: str = "$215,000", context: str = "Total row") -> Discrepancy:
+    return Discrepancy(
+        page=page,
+        kind=DiscrepancyKind.TALLY,
+        severity=Severity.ERROR,
+        description="Column total is wrong.",
+        stated=stated,
+        expected="$215,500",
+        context=context,
+    )
+
+
+def test_specs_prefer_stated_as_anchor_text() -> None:
+    verdict = _make_verdict([_discrepancy(stated="$215,000")])
+    localised = [_LocalisedComment(discrepancy_index=0, subject="Total mismatch", text="Off by $500.")]
+    specs = PdfReviewAgent._build_comment_specs(verdict, localised)
+    assert len(specs) == 1
+    assert specs[0].anchor_text == "$215,000"
+
+
+def test_specs_fall_back_to_context_when_stated_missing() -> None:
+    verdict = _make_verdict(
+        [
+            _discrepancy(stated="", context="We grew 15% this year"),
+        ]
+    )
+    localised = [_LocalisedComment(discrepancy_index=0, subject="Claim", text="Unverified.")]
+    specs = PdfReviewAgent._build_comment_specs(verdict, localised)
+    assert specs[0].anchor_text == "We grew 15% this year"
+
+
+def test_specs_anchor_text_none_when_no_hints() -> None:
+    verdict = _make_verdict([_discrepancy(stated="", context="")])
+    localised = [_LocalisedComment(discrepancy_index=0, subject="Total wrong", text="Off by ten.")]
+    specs = PdfReviewAgent._build_comment_specs(verdict, localised)
+    assert specs[0].anchor_text is None
+
+
+def test_specs_drop_out_of_range_indices() -> None:
+    verdict = _make_verdict([_discrepancy(page=0)])  # only one discrepancy, valid index is 0
+    localised = [
+        _LocalisedComment(discrepancy_index=0, subject="Real", text="Real comment."),
+        _LocalisedComment(discrepancy_index=99, subject="Hallucinated", text="Should be dropped."),
+    ]
+    specs = PdfReviewAgent._build_comment_specs(verdict, localised)
+    assert len(specs) == 1
+    assert specs[0].text == "Real comment."
+
+
+def test_specs_stack_per_page() -> None:
+    """Multiple discrepancies on the same page should be vertically stacked
+    in the right margin (decreasing y) rather than overlapping."""
+    verdict = _make_verdict(
+        [
+            _discrepancy(page=0, stated="A"),
+            _discrepancy(page=0, stated="B"),
+            _discrepancy(page=1, stated="C"),
+        ]
+    )
+    localised = [
+        _LocalisedComment(discrepancy_index=0, subject="s", text="t"),
+        _LocalisedComment(discrepancy_index=1, subject="s", text="t"),
+        _LocalisedComment(discrepancy_index=2, subject="s", text="t"),
+    ]
+    specs = PdfReviewAgent._build_comment_specs(verdict, localised)
+    page0 = [s for s in specs if s.page_index == 0]
+    assert len(page0) == 2
+    assert page0[0].y > page0[1].y  # stacked downward
+    page1 = [s for s in specs if s.page_index == 1]
+    assert page1[0].y == page0[0].y  # first on a new page resets the stack
+
+
+@pytest.mark.anyio
+async def test_payload_serialises_anchor_text_as_camel_case(runtime: AppRuntime) -> None:
+    """Java deserialises the comments JSON via record-component names, so the
+    keys must be camelCase (anchorText, pageIndex)."""
+    agent = PdfReviewAgent(runtime)
+    verdict = _make_verdict([_discrepancy(page=2, stated="110", context="Line 3")])
+    canned = _LocalisedVerdict(
+        comments=[_LocalisedComment(discrepancy_index=0, subject="Off by ten", text="Subtotal wrong.")],
+    )
+    with patch.object(agent._localiser_agent, "run", return_value=_StubResult(output=canned)):
+        payload_json = await agent._build_localised_comments_payload("flag math errors", verdict)
+
+    payload = json.loads(payload_json)
+    assert len(payload) == 1
+    assert payload[0]["anchorText"] == "110"
+    assert payload[0]["pageIndex"] == 2
+    assert payload[0]["text"] == "Subtotal wrong."
+
+
+# ---------------------------------------------------------------------------
+# orchestrate() — classifier-driven first-turn routing
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.anyio
+async def test_orchestrate_classifier_true_emits_math_audit_plan(runtime: AppRuntime) -> None:
+    """First turn — when the math-intent classifier says yes, emit a one-step plan
+    calling the math auditor with resume_with=PDF_REVIEW."""
+    agent = PdfReviewAgent(runtime)
+    request = OrchestratorRequest(user_message="vérifie les totaux", file_names=["report.pdf"])
+
+    with patch.object(agent._math_intent_classifier, "classify", AsyncMock(return_value=True)):
+        response = await agent.orchestrate(request)
+
+    assert isinstance(response, EditPlanResponse)
+    assert response.resume_with == SupportedCapability.PDF_REVIEW
+    assert len(response.steps) == 1
+    assert response.steps[0].tool == AgentToolId.MATH_AUDITOR_AGENT
+
+
+@pytest.mark.anyio
+async def test_orchestrate_classifier_false_routes_to_pdf_comment_agent(runtime: AppRuntime) -> None:
+    """When the classifier says no math, delegate to pdf-comment-agent for prose review."""
+    agent = PdfReviewAgent(runtime)
+    request = OrchestratorRequest(
+        user_message="review the invoices for ambiguous wording",
+        file_names=["contract.pdf"],
+    )
+
+    with patch.object(agent._math_intent_classifier, "classify", AsyncMock(return_value=False)):
+        response = await agent.orchestrate(request)
+
+    assert isinstance(response, EditPlanResponse)
+    assert response.resume_with is None
+    assert len(response.steps) == 1
+    assert response.steps[0].tool == AgentToolId.PDF_COMMENT_AGENT
+    assert isinstance(response.steps[0].parameters, PdfCommentAgentParams)
+    assert response.steps[0].parameters.prompt == request.user_message
+
+
+@pytest.mark.anyio
+async def test_orchestrate_resume_uses_verdict_without_calling_classifier(
+    runtime: AppRuntime,
+) -> None:
+    """Resume turns are detected by Verdict-artifact presence and bypass the
+    classifier entirely (saves an LLM call when we already know the answer)."""
+    from stirling.contracts import MathAuditorToolReportArtifact
+
+    agent = PdfReviewAgent(runtime)
+    verdict = _make_verdict([_discrepancy(page=0, stated="$100")])
+    request = OrchestratorRequest(
+        user_message="flag math errors",
+        file_names=["report.pdf"],
+        artifacts=[MathAuditorToolReportArtifact(report=verdict)],
+    )
+    canned = _LocalisedVerdict(
+        comments=[_LocalisedComment(discrepancy_index=0, subject="Wrong", text="Off.")],
+    )
+    classifier_mock = AsyncMock(return_value=False)
+    with patch.object(agent._localiser_agent, "run", return_value=_StubResult(output=canned)):
+        with patch.object(agent._math_intent_classifier, "classify", classifier_mock):
+            response = await agent.orchestrate(request)
+
+    assert isinstance(response, EditPlanResponse)
+    assert response.resume_with is None
+    assert len(response.steps) == 1
+    assert response.steps[0].tool == ToolEndpoint.ADD_COMMENTS
+    classifier_mock.assert_not_called()  # short-circuit on Verdict
+
+
+# ---------------------------------------------------------------------------
+# Prompt pinning — guard against accidental drift
+# ---------------------------------------------------------------------------
+
+
+def test_localiser_prompt_requires_verbatim_quoting() -> None:
+    """If this prompt is rephrased and drops the verbatim rule, the LLM may
+    paraphrase numeric values like ``$215,000`` as 'about $215k'."""
+    assert "verbatim" in _LOCALISER_SYSTEM_PROMPT.lower()