Feature/pdf to markdown agent (#6271)

Co-authored-by: James Brunton <[email protected]>

engine/src/stirling/agents/__init__.py

@@ -5,6 +5,7 @@ from .orchestrator import OrchestratorAgent
 from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection
 from .pdf_questions import PdfQuestionAgent
 from .pdf_review import PdfReviewAgent
+from .pdf_to_markdown import PdfToMarkdownAgent
 from .user_spec import UserSpecAgent
 
 __all__ = [
@@ -15,5 +16,6 @@ __all__ = [
     "PdfEditPlanSelection",
     "PdfQuestionAgent",
     "PdfReviewAgent",
+    "PdfToMarkdownAgent",
     "UserSpecAgent",
 ]

engine/src/stirling/agents/orchestrator.py

@@ -11,12 +11,14 @@ from pydantic_ai.tools import RunContext
 from stirling.agents.pdf_edit import PdfEditAgent
 from stirling.agents.pdf_questions import PdfQuestionAgent
 from stirling.agents.pdf_review import PdfReviewAgent
+from stirling.agents.pdf_to_markdown import PdfToMarkdownAgent
 from stirling.agents.user_spec import UserSpecAgent
 from stirling.contracts import (
     AgentDraftWorkflowResponse,
     ExtractedTextArtifact,
     OrchestratorRequest,
     OrchestratorResponse,
+    PageLayoutArtifact,
     PdfEditResponse,
     PdfQuestionOrchestrateResponse,
     SupportedCapability,
@@ -25,6 +27,7 @@ from stirling.contracts import (
     format_file_names,
 )
 from stirling.contracts.pdf_edit import EditPlanResponse
+from stirling.contracts.pdf_to_markdown import PdfToMarkdownOrchestrateResponse
 from stirling.services import AppRuntime
 
 logger = logging.getLogger(__name__)
@@ -68,6 +71,11 @@ class OrchestratorAgent:
                         " feedback')."
                     ),
                 ),
+                ToolOutput(
+                    self.delegate_pdf_to_markdown,
+                    name="delegate_pdf_to_markdown",
+                    description=("Delegate requests to reconstruct a PDF as a Markdown document."),
+                ),
                 ToolOutput(
                     self.unsupported_capability,
                     name="unsupported_capability",
@@ -84,6 +92,8 @@ class OrchestratorAgent:
                 "Use delegate_pdf_review when the user wants the PDF returned with review"
                 " comments attached — anything like 'review this', 'annotate with comments',"
                 " 'leave feedback on the PDF'. "
+                "Use delegate_pdf_to_markdown for any request to convert a PDF to Markdown "
+                "or reconstruct its content as readable text. "
                 "Use unsupported_capability when the user asks about the assistant itself "
                 "or when none of the other outputs fit; supply a helpful message."
             ),
@@ -123,6 +133,8 @@ class OrchestratorAgent:
                 return await self._run_pdf_edit(request)
             case SupportedCapability.AGENT_DRAFT:
                 return await self._run_agent_draft(request)
+            case SupportedCapability.PDF_TO_MARKDOWN:
+                return await self._run_pdf_to_markdown(request)
             case (
                 SupportedCapability.ORCHESTRATE
                 | SupportedCapability.AGENT_REVISE
@@ -151,6 +163,12 @@ class OrchestratorAgent:
     async def _run_agent_draft(self, request: OrchestratorRequest) -> AgentDraftWorkflowResponse:
         return await UserSpecAgent(self.runtime).orchestrate(request)
 
+    async def delegate_pdf_to_markdown(self, ctx: RunContext[OrchestratorDeps]) -> PdfToMarkdownOrchestrateResponse:
+        return await self._run_pdf_to_markdown(ctx.deps.request)
+
+    async def _run_pdf_to_markdown(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse:
+        return await PdfToMarkdownAgent(self.runtime).orchestrate(request)
+
     async def delegate_pdf_review(self, ctx: RunContext[OrchestratorDeps]) -> EditPlanResponse:
         return await self._run_pdf_review(ctx.deps.request)
 
@@ -186,5 +204,10 @@ class OrchestratorAgent:
                 file_names = [f.file_name for f in artifact.files]
                 descriptions.append(f"- extracted_text: {total_pages} pages from {file_names}")
                 continue
+            if isinstance(artifact, PageLayoutArtifact):
+                total_pages = sum(len(f.pages) for f in artifact.files)
+                file_names = [f.file_name for f in artifact.files]
+                descriptions.append(f"- page_layout: {total_pages} pages from {file_names}")
+                continue
             descriptions.append("- unknown artifact")
         return "\n".join(descriptions)

engine/src/stirling/agents/pdf_to_markdown/__init__.py

@@ -0,0 +1,3 @@
+from .agent import PdfToMarkdownAgent
+
+__all__ = ["PdfToMarkdownAgent"]

engine/src/stirling/agents/pdf_to_markdown/agent.py

@@ -0,0 +1,435 @@
+"""PDF to Markdown Agent.
+
+Converts a parsed PDF document into a single clean Markdown document, preserving
+headings, paragraphs, and tables in reading order.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import time
+
+from pydantic import BaseModel, Field
+from pydantic_ai import Agent
+from pydantic_ai.output import NativeOutput
+
+from stirling.contracts import (
+    EditCannotDoResponse,
+    GenerateFileResponse,
+    NeedContentFileRequest,
+    NeedContentResponse,
+    OrchestratorRequest,
+    PdfContentType,
+    SupportedCapability,
+    format_conversation_history,
+)
+from stirling.contracts.pdf_to_markdown import (
+    PageLayout,
+    PageLayoutArtifact,
+    PdfToMarkdownCannotDoResponse,
+    PdfToMarkdownOrchestrateResponse,
+    PdfToMarkdownRequest,
+    PdfToMarkdownResponse,
+    PdfToMarkdownSuccessResponse,
+)
+from stirling.services import AppRuntime
+
+logger = logging.getLogger(__name__)
+
+
+# Warn when output tokens are close to the typical model output limit (~8192 for most
+# configurations). The actual limit is model-specific; this threshold catches likely truncation.
+_OUTPUT_TOKEN_TRUNCATION_THRESHOLD = 7500
+
+# Chunking limits — keep each LLM call to a manageable payload size.
+# Fragment count is the primary driver of JSON payload size (each fragment carries x/y/width/
+# fontSize/bold metadata beyond its text). Page cap prevents low-text pages accumulating.
+_MAX_CHUNK_FRAGMENTS = 1_000
+_MAX_CHUNK_PAGES = 10
+
+# Max concurrent LLM calls — limits API rate pressure on large documents.
+_MAX_PARALLEL_CHUNKS = 3
+
+# ── LLM output model ────────────────────────────────────────────────────────────────────────────
+
+
+class _ReconstructionOutput(BaseModel):
+    markdown: str = Field(description="Full document reconstructed as clean Markdown.")
+
+
+# ── Agent ────────────────────────────────────────────────────────────────────────────────────────
+
+
+class PdfToMarkdownAgent:
+    def __init__(self, runtime: AppRuntime) -> None:
+        self.runtime = runtime
+        self._sem = asyncio.Semaphore(_MAX_PARALLEL_CHUNKS)
+        self._reconstruct_agent = Agent(
+            model=runtime.smart_model,
+            output_type=NativeOutput(_ReconstructionOutput),
+            system_prompt=(
+                "You reconstruct PDF pages into clean Markdown from spatial fragment data.\n"
+                "Input: PAGE LAYOUT — per-fragment x/y/font data for structural analysis.\n\n"
+                "COLUMN DETECTION (for tables in page_layout):\n"
+                "- Look at the x-positions of fragments across 3+ consecutive lines.\n"
+                "- If fragments cluster at the same x-positions across multiple lines, those are table columns.\n"
+                "- Each distinct x-cluster is one column."
+                " Name them from the header row (the first line in the cluster).\n"
+                "- Do NOT merge values from different x-columns into one cell.\n\n"
+                "ROW DETECTION:\n"
+                "- Each unique y-coordinate (or group within 3pt) is one table row.\n"
+                "- Every line of layout data is its own row — do not merge rows.\n"
+                "- If a column has no fragment on a given y-row, that cell is empty.\n\n"
+                "TABLE RENDERING:\n"
+                "- Render as: | col1 | col2 | col3 |\n"
+                "             | --- | --- | --- |\n"
+                "             | val | val | val |\n"
+                "- One source row = one table row. Never collapse multiple rows into one.\n"
+                "- Preserve numeric values exactly (no rounding, no formatting changes).\n"
+                "- Bold cells: wrap with ** in the Markdown cell.\n"
+                "- CRITICAL: the separator row `| --- | --- |` appears EXACTLY ONCE per table, immediately\n"
+                "  after the header row. NEVER put `| --- |` after a data row or between data rows.\n"
+                "  NEVER put a blank line inside a table. All rows (header + data) must be consecutive.\n"
+                "- Do NOT produce a header-only table followed by a second table with the data rows.\n"
+                "  One logical table = one markdown table block, with header, one separator, then all data.\n\n"
+                "GROUP HEADERS (label-only rows inside a table):\n"
+                "- A row is a group header when: the first column has text AND every numeric column is empty.\n"
+                "- Do NOT render group headers as table rows with empty cells.\n"
+                "- Break the table, emit the label as **bold text** on its own line,"
+                " then start a new table for the rows that follow.\n"
+                "- Example labels: 'Policy functions', 'Non-current assets'.\n\n"
+                "TOTAL AND SUBTOTAL ROWS:\n"
+                "- Detect rows whose first cell contains (case-insensitive):"
+                " total, subtotal, surplus, balance, net, sum.\n"
+                "- These rows have numeric content — they are NOT group headers.\n"
+                "- Render the entire row in bold: | **Total income** | **1,234** | **5,678** |\n"
+                "- Keep total rows attached to the group they summarise.\n\n"
+                "MULTI-LEVEL TABLES (year or period as a row label):\n"
+                "- Detect when a row contains only a single label (a year like '2010' or period like 'Q1 2023')"
+                " with no numeric content, followed by repeated metric rows.\n"
+                "- Do NOT render the year as a table row.\n"
+                "- Normalise: add 'Year' as the first column, 'Metric' as the second,"
+                " and repeat the year value on each metric row.\n\n"
+                "PROSE REGIONS:\n"
+                "- Lines where x-positions vary across lines (not repeating columns) are prose.\n"
+                "- Merge lines at the same x-level into paragraphs. Separate indented lines.\n\n"
+                "HEADINGS:\n"
+                "- A line is a heading when it is bold OR font_size ≥2pt above body.\n"
+                "  CRITICAL EXCEPTION: a bold fragment is a TABLE HEADER CELL, not a document heading, when\n"
+                "  the same y-row in page_layout contains other fragments at different x-positions.\n"
+                "  Only classify a bold line as a document heading when it is the SOLE fragment on its y-row.\n"
+                "  Example: 'Non-current assets' at y=120 with '2010'@x=350, '2009'@x=420, '2008'@x=490\n"
+                "  → this is a table header row, NOT a heading. Render it as the first cell of the table.\n"
+                "- Use ## for section headings, ### for sub-headings. Use # only for the document title.\n\n"
+                "ORDERING:\n"
+                "- Process content top-to-bottom as it appears on the page.\n"
+                "- Interleave prose blocks and table blocks in page order.\n"
+                "- Do not move text that appears before a table to after it, or vice versa.\n\n"
+                "FIDELITY:\n"
+                "- Do NOT invent, summarise, or omit any content.\n"
+                "- Do NOT add commentary, metadata, or JSON — output Markdown only."
+            ),
+            model_settings={
+                **runtime.smart_model_settings,
+                "temperature": 0.0,
+                "max_tokens": _OUTPUT_TOKEN_TRUNCATION_THRESHOLD,
+            },
+        )
+
+    async def orchestrate(self, request: OrchestratorRequest) -> PdfToMarkdownOrchestrateResponse:
+        """Entry point for the orchestrator delegate.
+
+        First turn: requests PAGE_LAYOUT extraction from Java via NeedContentResponse.
+        Resume turn: runs the LLM reconstruction and returns a write-file plan step.
+        """
+        layout_artifact = next(
+            (a for a in request.artifacts if isinstance(a, PageLayoutArtifact)),
+            None,
+        )
+        if layout_artifact is None:
+            return NeedContentResponse(
+                resume_with=SupportedCapability.PDF_TO_MARKDOWN,
+                reason="Page layout data is required to reconstruct the document.",
+                files=[
+                    NeedContentFileRequest(file=f, content_types=[PdfContentType.PAGE_LAYOUT]) for f in request.files
+                ],
+                max_pages=self.runtime.settings.max_pages,
+                max_characters=self.runtime.settings.max_characters,
+            )
+
+        page_layout = [page for entry in layout_artifact.files for page in entry.pages]
+        file_names = [f.name for f in request.files]
+        result = await self.handle(
+            PdfToMarkdownRequest(
+                user_message=request.user_message,
+                file_names=file_names,
+                conversation_history=request.conversation_history,
+                page_layout=page_layout,
+            )
+        )
+        if isinstance(result, PdfToMarkdownCannotDoResponse):
+            return EditCannotDoResponse(reason=result.reason)
+
+        base = file_names[0].rsplit(".", 1)[0] if file_names else "document"
+        return GenerateFileResponse(
+            content=result.markdown,
+            filename=f"{base}-reconstruction.md",
+            summary="Reconstructed the document as a Markdown file.",
+        )
+
+    async def handle(self, request: PdfToMarkdownRequest) -> PdfToMarkdownResponse:
+        total_fragments = sum(len(line.fragments) for page in request.page_layout for line in page.lines)
+        logger.info(
+            "[pdf-to-markdown] received layout-pages=%d fragments=%d",
+            len(request.page_layout),
+            total_fragments,
+        )
+
+        if not request.page_layout:
+            logger.warning("[pdf-to-markdown] no content extracted from document; returning cannot_do")
+            return PdfToMarkdownCannotDoResponse(
+                reason=(
+                    "No content was extracted from the document. "
+                    "The file may be a scanned image PDF with no readable text. "
+                    "Try running OCR on the document first."
+                )
+            )
+
+        chunks = _build_page_chunks(request.page_layout)
+        logger.info("[pdf-to-markdown] chunks=%d (max %d in parallel)", len(chunks), _MAX_PARALLEL_CHUNKS)
+
+        if len(chunks) == 1:
+            return await self._reconstruct_chunk(request, chunks[0], chunk_num=1, total_chunks=1)
+
+        total = len(chunks)
+        results = await asyncio.gather(
+            *(
+                self._reconstruct_chunk(request, chunk, chunk_num=i + 1, total_chunks=total)
+                for i, chunk in enumerate(chunks)
+            )
+        )
+
+        markdown_parts: list[str] = []
+        for result in results:
+            if isinstance(result, PdfToMarkdownSuccessResponse) and result.markdown:
+                markdown_parts.append(result.markdown)
+            elif isinstance(result, PdfToMarkdownCannotDoResponse):
+                logger.warning("[pdf-to-markdown] chunk dropped: %s", result.reason)
+
+        if not markdown_parts:
+            return PdfToMarkdownCannotDoResponse(reason="The document could not be reconstructed. All chunks failed.")
+
+        logger.info("[pdf-to-markdown] assembly: %d/%d chunks produced output", len(markdown_parts), len(chunks))
+        return PdfToMarkdownSuccessResponse(markdown="\n\n".join(markdown_parts))
+
+    async def _reconstruct_chunk(
+        self,
+        request: PdfToMarkdownRequest,
+        pages: list[PageLayout],
+        chunk_num: int,
+        total_chunks: int,
+    ) -> PdfToMarkdownResponse:
+        chunk_request = PdfToMarkdownRequest(
+            user_message=request.user_message,
+            file_names=request.file_names,
+            conversation_history=request.conversation_history,
+            page_layout=pages,
+        )
+        try:
+            async with self._sem:
+                return await self._reconstruct_document(chunk_request, chunk_num, total_chunks)
+        except Exception as e:
+            logger.error("[pdf-to-markdown] chunk %d/%d failed: %s", chunk_num, total_chunks, e, exc_info=True)
+            return PdfToMarkdownCannotDoResponse(
+                reason="The document could not be reconstructed. The AI model failed to process it."
+            )
+
+    async def _reconstruct_document(
+        self, request: PdfToMarkdownRequest, chunk_num: int = 1, total_chunks: int = 1
+    ) -> PdfToMarkdownSuccessResponse:
+        content = _build_reconstruction_prompt(request)
+        logger.info("[timing] chunk %d/%d llm-call prompt-chars=%d", chunk_num, total_chunks, len(content))
+        t0 = time.monotonic()
+        result = await self._reconstruct_agent.run([content])
+        llm_ms = int((time.monotonic() - t0) * 1000)
+        output: _ReconstructionOutput = result.output
+        usage = result.usage()
+        logger.info(
+            "[timing] chunk %d/%d llm-done ms=%d input-tokens=%s output-tokens=%s markdown-chars=%d",
+            chunk_num,
+            total_chunks,
+            llm_ms,
+            usage.input_tokens,
+            usage.output_tokens,
+            len(output.markdown),
+        )
+        if usage.output_tokens and usage.output_tokens >= _OUTPUT_TOKEN_TRUNCATION_THRESHOLD:
+            logger.warning(
+                "[timing] chunk %d/%d output likely truncated (output-tokens=%d)",
+                chunk_num,
+                total_chunks,
+                usage.output_tokens,
+            )
+        markdown = _remove_extra_separators(_fix_markdown_tables(_merge_orphaned_table_rows(output.markdown)))
+        return PdfToMarkdownSuccessResponse(markdown=markdown)
+
+
+# ── Chunking ────────────────────────────────────────────────────────────────────────────────────
+
+
+def _build_page_chunks(pages: list[PageLayout]) -> list[list[PageLayout]]:
+    chunks: list[list[PageLayout]] = []
+    current: list[PageLayout] = []
+    current_fragments = 0
+    for page in pages:
+        page_fragments = sum(len(line.fragments) for line in page.lines)
+        fragment_full = current and current_fragments + page_fragments > _MAX_CHUNK_FRAGMENTS
+        page_full = len(current) >= _MAX_CHUNK_PAGES
+        if fragment_full or page_full:
+            chunks.append(current)
+            current = []
+            current_fragments = 0
+        current.append(page)
+        current_fragments += page_fragments
+    if current:
+        chunks.append(current)
+    return chunks
+
+
+# ── Prompt builders (module-level, no state) ────────────────────────────────────────────────────
+
+
+def _build_reconstruction_prompt(request: PdfToMarkdownRequest) -> str:
+    history = format_conversation_history(request.conversation_history)
+    file_names = ", ".join(request.file_names) if request.file_names else "Unknown files"
+    layout_section = _format_layout(request.page_layout)
+
+    return (
+        f"Files: {file_names}\n\n"
+        f"User request: {request.user_message}\n\n"
+        f"Conversation history:\n{history}\n\n"
+        "PAGE LAYOUT (structural source — x/y fragment positions):\n"
+        "Each line is: y=NNN | text@(x,y) fs=N  text@(x,y) fs=N ...\n"
+        "- y=NNN is the vertical position (row). Lines close in y are the same visual row.\n"
+        "- x=NNN is the horizontal position (column). Consistent x across rows = a column.\n"
+        "- fs=N is font size. Larger = likely a heading.\n"
+        "- **bold** markers indicate bold text.\n\n"
+        f"{layout_section}"
+    )
+
+
+# ── LLM output post-processing ──────────────────────────────────────────────────────────────────
+
+
+def _fix_markdown_tables(markdown: str) -> str:
+    """Remove blank lines between table rows produced by the LLM."""
+    lines = markdown.split("\n")
+    result: list[str] = []
+    i = 0
+    while i < len(lines):
+        result.append(lines[i])
+        if lines[i].strip().startswith("|"):
+            j = i + 1
+            while j < len(lines) and lines[j].strip() == "":
+                j += 1
+            if j < len(lines) and lines[j].strip().startswith("|"):
+                i = j
+                continue
+        i += 1
+    return "\n".join(result)
+
+
+_SEP_CELL = re.compile(r"^:?-+:?$")
+
+
+def _is_sep_row(line: str) -> bool:
+    """Return True when a pipe row is a Markdown table separator (| --- | --- |)."""
+    stripped = line.strip()
+    if not stripped.startswith("|"):
+        return False
+    cells = [c.strip() for c in stripped.split("|") if c.strip()]
+    return bool(cells) and all(_SEP_CELL.match(c) for c in cells)
+
+
+def _merge_orphaned_table_rows(markdown: str) -> str:
+    """Merge pipe-row blocks that lack a separator into the preceding table.
+
+    When the LLM incorrectly breaks a table (e.g. on a false group-header), it emits
+    orphaned pipe rows with no header or separator. These are invalid markdown and get
+    merged back into the preceding table, discarding the intervening non-table content.
+    """
+    lines = markdown.split("\n")
+
+    segments: list[tuple[str, list[str]]] = []
+    i = 0
+    while i < len(lines):
+        if lines[i].strip().startswith("|"):
+            block: list[str] = []
+            while i < len(lines) and lines[i].strip().startswith("|"):
+                block.append(lines[i])
+                i += 1
+            has_sep = any(_is_sep_row(row) for row in block)
+            segments.append(("table" if has_sep else "orphan", block))
+        else:
+            block = []
+            while i < len(lines) and not lines[i].strip().startswith("|"):
+                block.append(lines[i])
+                i += 1
+            segments.append(("prose", block))
+
+    result: list[tuple[str, list[str]]] = []
+    last_table_idx: int | None = None
+    for seg_type, seg_lines in segments:
+        if seg_type == "orphan":
+            if last_table_idx is not None:
+                result = result[: last_table_idx + 1]
+                result[-1] = ("table", result[-1][1] + seg_lines)
+            else:
+                result.append((seg_type, seg_lines))
+        else:
+            if seg_type == "table":
+                last_table_idx = len(result)
+            result.append((seg_type, seg_lines))
+
+    return "\n".join(line for _, seg_lines in result for line in seg_lines)
+
+
+def _remove_extra_separators(markdown: str) -> str:
+    """Within each contiguous table block, keep only the first separator row."""
+    lines = markdown.split("\n")
+    result: list[str] = []
+    seen_sep = False
+
+    for line in lines:
+        if not line.strip().startswith("|"):
+            seen_sep = False
+            result.append(line)
+            continue
+        if _is_sep_row(line):
+            if seen_sep:
+                continue
+            seen_sep = True
+        result.append(line)
+
+    return "\n".join(result)
+
+
+# ── Formatting helpers (module-level, no state) ──────────────────────────────────────────────────
+
+
+def _format_layout(pages: list[PageLayout]) -> str:
+    if not pages:
+        return "None"
+    parts: list[str] = []
+    for page in pages:
+        line_strs: list[str] = []
+        for line in page.lines:
+            frags = " ".join(
+                f"{'**' if f.bold else ''}{f.text}{'**' if f.bold else ''}@({f.x:.0f},{f.y:.0f}) fs={f.font_size:.0f}"
+                for f in line.fragments
+            )
+            line_strs.append(f"y={line.y:.0f} | {frags}")
+        parts.append(f"--- Page {page.page_number} ---\n" + "\n".join(line_strs))
+    return "\n\n".join(parts)

engine/src/stirling/contracts/__init__.py

@@ -14,6 +14,7 @@ from .common import (
     ArtifactKind,
     ConversationMessage,
     ExtractedFileText,
+    GenerateFileResponse,
     MathAuditorToolReportArtifact,
     NeedContentFileRequest,
     NeedContentResponse,
@@ -87,6 +88,17 @@ from .pdf_questions import (
     PdfQuestionResponse,
     PdfQuestionTerminalResponse,
 )
+from .pdf_to_markdown import (
+    LayoutFragment,
+    LayoutLine,
+    PageLayout,
+    PageLayoutArtifact,
+    PageLayoutFileEntry,
+    PdfToMarkdownCannotDoResponse,
+    PdfToMarkdownOrchestrateResponse,
+    PdfToMarkdownRequest,
+    PdfToMarkdownResponse,
+)
 from .progress import (
     ProgressEvent,
     WholeDocCompressionRound,
@@ -114,6 +126,10 @@ __all__ = [
     "CompletedExecutionAction",
     "ConversationMessage",
     "DeleteDocumentResponse",
+    "PdfToMarkdownCannotDoResponse",
+    "PdfToMarkdownOrchestrateResponse",
+    "PdfToMarkdownRequest",
+    "PdfToMarkdownResponse",
     "Discrepancy",
     "DiscrepancyKind",
     "EditCannotDoResponse",
@@ -129,6 +145,7 @@ __all__ = [
     "FolioType",
     "format_conversation_history",
     "format_file_names",
+    "GenerateFileResponse",
     "HealthResponse",
     "IngestDocumentRequest",
     "IngestDocumentResponse",
@@ -139,7 +156,12 @@ __all__ = [
     "NextExecutionAction",
     "OrchestratorRequest",
     "OrchestratorResponse",
+    "LayoutFragment",
+    "LayoutLine",
     "Page",
+    "PageLayout",
+    "PageLayoutArtifact",
+    "PageLayoutFileEntry",
     "PageRange",
     "PageText",
     "PdfCommentInstruction",

engine/src/stirling/contracts/common.py

@@ -61,6 +61,7 @@ class WorkflowOutcome(StrEnum):
     COMPLETED = "completed"
     CANNOT_CONTINUE = "cannot_continue"
     UNSUPPORTED_CAPABILITY = "unsupported_capability"
+    GENERATE_FILE = "generate_file"
 
 
 class ArtifactKind(StrEnum):
@@ -70,6 +71,7 @@ class ArtifactKind(StrEnum):
     """
 
     EXTRACTED_TEXT = "extracted_text"
+    PAGE_LAYOUT = "page_layout"
     TOOL_REPORT = "tool_report"
 
 
@@ -89,6 +91,7 @@ class SupportedCapability(StrEnum):
     AGENT_REVISE = "agent_revise"
     AGENT_NEXT_ACTION = "agent_next_action"
     MATH_AUDITOR_AGENT = "math_auditor_agent"
+    PDF_TO_MARKDOWN = "pdf_to_markdown"
 
 
 class ConversationMessage(ApiModel):
@@ -200,6 +203,19 @@ class ToolOperationStep(ApiModel):
         return self
 
 
+class GenerateFileResponse(ApiModel):
+    """Return generated text content directly to Java for file packaging.
+
+    Java converts the content string to bytes and stores it as a result file,
+    avoiding a round-trip through a write-file tool endpoint.
+    """
+
+    outcome: Literal[WorkflowOutcome.GENERATE_FILE] = WorkflowOutcome.GENERATE_FILE
+    content: str
+    filename: str = Field(pattern=r"^[^/\\]+$", description="Output filename; no path separators.")
+    summary: str | None = None
+
+
 def drop_unknown_tool_endpoints(value: Iterable[str | ToolEndpoint]) -> list[ToolEndpoint]:
     """Coerce inbound endpoint identifiers into `ToolEndpoint` members, dropping unknowns.
 

engine/src/stirling/contracts/orchestrator.py

@@ -12,6 +12,7 @@ from .common import (
     ArtifactKind,
     ConversationMessage,
     ExtractedFileText,
+    GenerateFileResponse,
     NeedContentResponse,
     NeedIngestResponse,
     SupportedCapability,
@@ -22,6 +23,7 @@ from .common import (
 from .execution import NextExecutionAction
 from .pdf_edit import PdfEditTerminalResponse
 from .pdf_questions import PdfQuestionTerminalResponse
+from .pdf_to_markdown import PageLayoutArtifact
 
 
 class ExtractedTextArtifact(ApiModel):
@@ -29,7 +31,10 @@ class ExtractedTextArtifact(ApiModel):
     files: list[ExtractedFileText] = Field(default_factory=list)
 
 
-WorkflowArtifact = Annotated[ExtractedTextArtifact | ToolReportArtifact, Field(discriminator="kind")]
+WorkflowArtifact = Annotated[
+    ExtractedTextArtifact | PageLayoutArtifact | ToolReportArtifact,
+    Field(discriminator="kind"),
+]
 
 
 class OrchestratorRequest(ApiModel):
@@ -53,6 +58,7 @@ class UnsupportedCapabilityResponse(ApiModel):
 type OrchestratorResponse = Annotated[
     PdfEditTerminalResponse
     | PdfQuestionTerminalResponse
+    | GenerateFileResponse
     | NeedContentResponse
     | NeedIngestResponse
     | AgentDraftResponse

engine/src/stirling/contracts/pdf_to_markdown.py

@@ -0,0 +1,105 @@
+"""Contracts for the PDF to Markdown Agent.
+
+The agent accepts a parsed document and returns a single Markdown document that
+faithfully reconstructs the PDF content — headings, paragraphs, and tables in
+reading order, using page_layout as the primary source of truth for structure.
+
+Java extracts page layout via PdfIngester and returns it as a PageLayoutArtifact
+through the orchestrator resume_with pattern.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import Field
+
+from stirling.models import ApiModel
+
+from .common import ArtifactKind, ConversationMessage, GenerateFileResponse, NeedContentResponse
+from .pdf_edit import EditCannotDoResponse
+
+# ── Input: layout models (mirror Java's RawLine / TextFragment geometry) ────────────────────────
+
+
+class LayoutFragment(ApiModel):
+    """One text fragment with its bounding-box geometry and font properties."""
+
+    text: str
+    x: float
+    y: float
+    width: float
+    font_size: float
+    bold: bool
+
+
+class LayoutLine(ApiModel):
+    """A visual line on the page: one y-coordinate and all fragments on that line."""
+
+    y: float
+    fragments: list[LayoutFragment]
+
+
+class PageLayout(ApiModel):
+    """All layout lines for a single page, in top-to-bottom order."""
+
+    page_number: int
+    lines: list[LayoutLine]
+
+
+# ── Artifact: page layout (produced by Java, consumed by orchestrate()) ──────────────────────────
+
+
+class PageLayoutFileEntry(ApiModel):
+    """Page layout data for one file, as extracted by Java's PdfIngester."""
+
+    file_name: str
+    pages: list[PageLayout] = Field(default_factory=list)
+
+
+class PageLayoutArtifact(ApiModel):
+    """Artifact carrying full spatial page layout for all input files."""
+
+    kind: Literal[ArtifactKind.PAGE_LAYOUT] = ArtifactKind.PAGE_LAYOUT
+    files: list[PageLayoutFileEntry] = Field(default_factory=list)
+
+
+# ── Input: full request ──────────────────────────────────────────────────────────────────────────
+
+
+class PdfToMarkdownRequest(ApiModel):
+    """Request sent by Java after PdfIngester has parsed the document.
+
+    page_layout: per-fragment positional data from the original (y-sorted) line order.
+        Each fragment carries its x/y position, width, font size, and bold flag.
+        This is the primary source of truth for column detection and heading hierarchy.
+    """
+
+    user_message: str
+    file_names: list[str] = Field(default_factory=list)
+    conversation_history: list[ConversationMessage] = Field(default_factory=list)
+    page_layout: list[PageLayout] = Field(default_factory=list)
+
+
+# ── Output: response variants ────────────────────────────────────────────────────────────────────
+
+
+class PdfToMarkdownSuccessResponse(ApiModel):
+    outcome: Literal["document_reconstructed"] = "document_reconstructed"
+    markdown: str
+
+
+class PdfToMarkdownCannotDoResponse(ApiModel):
+    outcome: Literal["cannot_do"] = "cannot_do"
+    reason: str
+
+
+type PdfToMarkdownResponse = Annotated[
+    PdfToMarkdownSuccessResponse | PdfToMarkdownCannotDoResponse,
+    Field(discriminator="outcome"),
+]
+
+type PdfToMarkdownOrchestrateResponse = Annotated[
+    GenerateFileResponse | EditCannotDoResponse | NeedContentResponse,
+    Field(discriminator="outcome"),
+]

engine/tests/test_pdf_to_markdown.py

@@ -0,0 +1,138 @@
+"""Tests for PDF to Markdown agent.
+
+Two cases:
+1. Narrative-only page: request validates and routes to reconstruction.
+2. Mixed text + table page: layout with table region validates correctly.
+"""
+
+from __future__ import annotations
+
+from stirling.contracts.pdf_to_markdown import (
+    LayoutFragment,
+    LayoutLine,
+    PageLayout,
+    PageLayoutArtifact,
+    PdfToMarkdownRequest,
+    PdfToMarkdownSuccessResponse,
+)
+
+
+def _frag(text: str, x: float, y: float, font_size: float = 10.0, bold: bool = False) -> LayoutFragment:
+    return LayoutFragment(text=text, x=x, y=y, width=float(len(text) * 6), font_size=font_size, bold=bold)
+
+
+def _line(y: float, *frags: LayoutFragment) -> LayoutLine:
+    return LayoutLine(y=y, fragments=list(frags))
+
+
+# ── Test 1: Narrative-only reconstruction ────────────────────────────────────────────────────────
+
+
+# ── Contract test: Java serialization ↔ Python deserialization ──────────────────────────────────
+# This JSON is also asserted field-by-field in PageLayoutArtifactContractTest.java.
+# If either side renames a field, one of these tests fails.
+_CONTRACT_JSON = (
+    '{"kind":"page_layout","files":[{"fileName":"test.pdf","pages":'
+    '[{"pageNumber":1,"lines":[{"y":10.0,"fragments":'
+    '[{"text":"Hello","x":1.0,"y":2.0,"width":30.0,"fontSize":12.0,"bold":true}]}]}]}]}'
+)
+
+
+def test_page_layout_artifact_deserialises_java_json() -> None:
+    artifact = PageLayoutArtifact.model_validate_json(_CONTRACT_JSON)
+
+    assert artifact.kind == "page_layout"
+    assert artifact.files[0].file_name == "test.pdf"
+    page = artifact.files[0].pages[0]
+    assert page.page_number == 1
+    line = page.lines[0]
+    assert line.y == 10.0
+    frag = line.fragments[0]
+    assert frag.text == "Hello"
+    assert frag.x == 1.0
+    assert frag.y == 2.0
+    assert frag.width == 30.0
+    assert frag.font_size == 12.0
+    assert frag.bold is True
+
+
+def test_narrative_reconstruction_request_validates() -> None:
+    """A prose-only page with no tables produces a valid PdfToMarkdownRequest."""
+    layout = PageLayout(
+        page_number=1,
+        lines=[
+            _line(72.0, _frag("Annual Report 2023", x=72.0, y=72.0, font_size=18.0, bold=True)),
+            _line(100.0, _frag("Our revenue grew significantly", x=72.0, y=100.0)),
+            _line(114.0, _frag("during the fiscal year ended", x=72.0, y=114.0)),
+            _line(128.0, _frag("December 31, 2023.", x=72.0, y=128.0)),
+        ],
+    )
+    request = PdfToMarkdownRequest(
+        user_message="reconstruct this document",
+        page_layout=[layout],
+    )
+
+    assert len(request.page_layout) == 1
+    assert len(request.page_layout[0].lines) == 4
+    assert request.page_layout[0].lines[0].fragments[0].bold is True
+    assert request.page_layout[0].lines[0].fragments[0].font_size == 18.0
+
+
+def test_narrative_reconstruction_response_validates() -> None:
+    """PdfToMarkdownSuccessResponse accepts markdown and returns document_reconstructed outcome."""
+    response = PdfToMarkdownSuccessResponse(
+        markdown="# Annual Report 2023\n\nOur revenue grew significantly during the fiscal year.",
+    )
+
+    assert response.outcome == "document_reconstructed"
+    assert response.markdown.startswith("#")
+
+
+# ── Test 2: Mixed text + table reconstruction ─────────────────────────────────────────────────────
+
+
+def test_mixed_page_layout_validates() -> None:
+    """A page with both prose lines and a table region produces a valid request."""
+    layout = PageLayout(
+        page_number=1,
+        lines=[
+            # Prose heading
+            _line(50.0, _frag("Projects in Development", x=72.0, y=50.0, font_size=14.0, bold=True)),
+            # Table header row
+            _line(
+                80.0,
+                _frag("Project Name", x=72.0, y=80.0, bold=True),
+                _frag("Location", x=200.0, y=80.0, bold=True),
+                _frag("Size (MW)", x=290.0, y=80.0, bold=True),
+            ),
+            # Table data rows
+            _line(
+                95.0,
+                _frag("Chaplin Wind 1", x=72.0, y=95.0),
+                _frag("Saskatchewan", x=200.0, y=95.0),
+                _frag("177", x=290.0, y=95.0),
+            ),
+            _line(
+                110.0,
+                _frag("Amherst Island 2", x=72.0, y=110.0),
+                _frag("Ontario", x=200.0, y=110.0),
+                _frag("75", x=290.0, y=110.0),
+            ),
+            # Prose after table
+            _line(140.0, _frag("Notes:", x=72.0, y=140.0, bold=True)),
+            _line(154.0, _frag("1 PPA signed", x=85.0, y=154.0)),
+        ],
+    )
+    request = PdfToMarkdownRequest(
+        user_message="markdown",
+        page_layout=[layout],
+    )
+
+    assert len(request.page_layout[0].lines) == 6
+    # Header line has 3 fragments at distinct x-positions (column detection)
+    header_line = request.page_layout[0].lines[1]
+    xs = [f.x for f in header_line.fragments]
+    assert xs == [72.0, 200.0, 290.0]
+    # Data rows have matching x-positions
+    data_row = request.page_layout[0].lines[2]
+    assert [f.x for f in data_row.fragments] == [72.0, 200.0, 290.0]