Add ability for Stirling engine to reason across large documents (#6314)

# Description of Changes
Adds storage in the database for full document content alongside the RAG
content (and changes the service to `DocumentService` instead of
`RagService`). Then adds a generic capability that should be usable by
any agent (currently just used by the Question Agent) which allows the
agent to pull out the full contents of the doc, chunks it into various
sections that will fit in the context window, and then processes them in
parallel to create an intermediate result, and then processes the
intermediate result into a final answer. It will re-chunk as many times
as necessary to get the content small enough for the actual answer to be
analysed (I've tested on PDFs ~3500 pages long, which is well above the
context limit and requires maybe 3 rounds of compression to get an
answer).

The new full doc analysis stuff is heavier than the RAG lookup so both
remain. The agents should use RAG for targeted info and the chunked
reasoner for info that requires reading the full doc.

engine/src/stirling/contracts/__init__.py

@@ -28,6 +28,14 @@ from .common import (
     format_conversation_history,
     format_file_names,
 )
+from .documents import (
+    DeleteDocumentResponse,
+    IngestDocumentRequest,
+    IngestDocumentResponse,
+    Page,
+    PageRange,
+    PageText,
+)
 from .execution import (
     AgentExecutionRequest,
     CannotContinueExecutionAction,
@@ -79,11 +87,12 @@ from .pdf_questions import (
     PdfQuestionResponse,
     PdfQuestionTerminalResponse,
 )
-from .rag import (
-    DeleteDocumentResponse,
-    IngestDocumentRequest,
-    IngestDocumentResponse,
-    IngestedPageText,
+from .progress import (
+    ProgressEvent,
+    WholeDocCompressionRound,
+    WholeDocReadDone,
+    WholeDocReadStarted,
+    WholeDocSliceDone,
 )
 
 __all__ = [
@@ -123,7 +132,6 @@ __all__ = [
     "HealthResponse",
     "IngestDocumentRequest",
     "IngestDocumentResponse",
-    "IngestedPageText",
     "MathAuditorToolReportArtifact",
     "NeedContentFileRequest",
     "NeedContentResponse",
@@ -131,6 +139,9 @@ __all__ = [
     "NextExecutionAction",
     "OrchestratorRequest",
     "OrchestratorResponse",
+    "Page",
+    "PageRange",
+    "PageText",
     "PdfCommentInstruction",
     "PdfCommentReport",
     "PdfCommentRequest",
@@ -146,6 +157,7 @@ __all__ = [
     "PdfQuestionResponse",
     "PdfQuestionTerminalResponse",
     "PdfTextSelection",
+    "ProgressEvent",
     "Requisition",
     "Severity",
     "StepKind",
@@ -156,6 +168,10 @@ __all__ = [
     "ToolReportArtifact",
     "UnsupportedCapabilityResponse",
     "Verdict",
+    "WholeDocCompressionRound",
+    "WholeDocReadDone",
+    "WholeDocReadStarted",
+    "WholeDocSliceDone",
     "WorkflowArtifact",
     "WorkflowOutcome",
 ]

engine/src/stirling/contracts/common.py

@@ -167,10 +167,10 @@ ToolReportArtifact = MathAuditorToolReportArtifact
 
 
 class NeedIngestResponse(ApiModel):
-    """Signal that the listed files must be ingested into RAG before the agent can continue.
+    """Signal that the listed files must be ingested before the agent can continue.
 
     Java's handling: for each file, extract the requested content types, POST to
-    ``/api/v1/rag/documents`` keyed by ``file.id``, then retry the original request.
+    ``/api/v1/documents`` keyed by ``file.id``, then retry the original request.
     """
 
     outcome: Literal[WorkflowOutcome.NEED_INGEST] = WorkflowOutcome.NEED_INGEST

engine/src/stirling/contracts/documents.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from pydantic import Field
+
+from stirling.models import ApiModel
+
+from .common import FileId
+
+
+class PageText(ApiModel):
+    """A single page of extracted text on the ingest wire."""
+
+    page_number: int = Field(ge=1)
+    text: str
+
+
+class Page(ApiModel):
+    """A single page of a document, retrieved from storage.
+
+    ``char_count`` is precomputed at ingest time and reported here so callers
+    can budget how much content they want to read without first concatenating
+    the text of every page.
+    """
+
+    page_number: int = Field(ge=1)
+    text: str
+    char_count: int = Field(ge=0)
+
+
+class PageRange(ApiModel):
+    """Inclusive page range for partial reads. Both bounds are 1-indexed."""
+
+    start: int = Field(ge=1)
+    end: int = Field(ge=1)
+
+
+class IngestDocumentRequest(ApiModel):
+    """Replace-ingest a document's content under the given ``document_id``.
+
+    Each call wipes any previously-stored content for the document and writes
+    both the vector-chunk and ordered-page representations from the supplied
+    pages.
+
+    ``source`` is a human-readable label (typically the original filename)
+    that flows into chunk metadata so search results are readable when
+    ``document_id`` is a hash.
+    """
+
+    document_id: FileId = Field(min_length=1)
+    source: str = Field(min_length=1)
+    page_text: list[PageText] | None = None
+
+
+class IngestDocumentResponse(ApiModel):
+    document_id: FileId
+    chunks_indexed: int
+
+
+class DeleteDocumentResponse(ApiModel):
+    document_id: FileId
+    deleted: bool

engine/src/stirling/contracts/progress.py

@@ -0,0 +1,70 @@
+"""Progress events emitted by deep callees during a streaming orchestrator run.
+
+Each subclass models one engine-side phase. The Java side forwards the JSON
+verbatim into ``AiWorkflowProgressEvent.engineDetail``; the frontend switches
+on ``phase`` and renders the typed fields.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import Field
+
+from stirling.models import ApiModel
+
+
+class WholeDocReadStarted(ApiModel):
+    phase: Literal["whole_doc_read_started"] = "whole_doc_read_started"
+    question: str
+    pages: int
+    slices: int
+
+
+class WholeDocSliceDone(ApiModel):
+    """Emitted as each chunked-reasoner worker completes.
+
+    ``completed`` is a monotonically increasing counter (1..total) reflecting
+    the order in which workers finished, NOT the slice's position in the
+    document. Callers showing "Read X of Y" should use this directly so X
+    increments by one with each event.
+    """
+
+    phase: Literal["whole_doc_slice_done"] = "whole_doc_slice_done"
+    completed: int
+    total: int
+    pages: str
+    duration_ms: int
+    excerpts: int
+    facts: int
+
+
+class WholeDocCompressionRound(ApiModel):
+    """Emitted when the gathered slice notes exceed the synthesis context
+    budget and the reasoner consolidates them with a fast-model fold pass.
+
+    Long documents (a 3000-page novel produces ~900k chars of raw notes)
+    would otherwise overflow the smart-model's prompt. ``notes_in`` is the
+    count entering the round; ``groups`` is the number of fold calls fired
+    (each producing one consolidated note). One or two rounds usually fit;
+    the event fires per round so callers can render "Consolidating notes
+    (round N)..." rather than going silent through the fold.
+    """
+
+    phase: Literal["whole_doc_compression_round"] = "whole_doc_compression_round"
+    round_number: int
+    notes_in: int
+    groups: int
+
+
+class WholeDocReadDone(ApiModel):
+    phase: Literal["whole_doc_read_done"] = "whole_doc_read_done"
+    completed: int
+    slices: int
+    duration_seconds: float
+
+
+type ProgressEvent = Annotated[
+    WholeDocReadStarted | WholeDocSliceDone | WholeDocCompressionRound | WholeDocReadDone,
+    Field(discriminator="phase"),
+]

engine/src/stirling/contracts/rag.py

@@ -1,39 +0,0 @@
-from __future__ import annotations
-
-from pydantic import Field
-
-from stirling.models import ApiModel
-
-from .common import FileId
-
-
-class IngestedPageText(ApiModel):
-    page_number: int = Field(ge=1)
-    text: str
-
-
-class IngestDocumentRequest(ApiModel):
-    """Replace-ingest a document's content into RAG under the given document_id.
-
-    Each content-type field is optional; the endpoint replaces the document's entire
-    stored content with whatever is provided. To add a content type later, call again
-    with all content types the document should have (incremental-add-without-replace
-    will be a separate endpoint if/when we need it).
-
-    ``source`` is a human-readable label (typically the original filename) that flows
-    into chunk metadata so search results are readable when document_id is a hash.
-    """
-
-    document_id: FileId = Field(min_length=1)
-    source: str = Field(min_length=1)
-    page_text: list[IngestedPageText] | None = None
-
-
-class IngestDocumentResponse(ApiModel):
-    document_id: FileId
-    chunks_indexed: int
-
-
-class DeleteDocumentResponse(ApiModel):
-    document_id: FileId
-    deleted: bool