Flesh out RAG system (#6197)

# Description of Changes
Flesh out the RAG system and connect it to the PDF Question Agent so it
can respond to questions about PDFs of an extremely large size.

I'd expect lots more work will need to be done to finish off the RAG
system to really be what we need, but this should be a reasonable start
which will let us connect it to tools and have the ingestion mostly
handled automatically. I'm leaving file deletion and proper file ID
management to be done in a future PR. We also need to consider whether
all tools should retrieve content exclusively via RAG, or whether it's
beneficial to have tools sometimes fetch the direct content and other
times fetch it from RAG.

A diagram of the expected interaction is as follows:

```mermaid
sequenceDiagram
    autonumber
    actor U as User
    participant FE as Frontend<br/>(ChatPanel)
    participant J as Java<br/>(AiWorkflowService)
    participant O as Engine:<br/>OrchestratorAgent
    participant QA as Engine:<br/>PdfQuestionAgent
    participant RAG as Engine:<br/>RagService + SqliteVecStore
    participant V as VoyageAI<br/>(embeddings)
    participant L as LLM<br/>(Claude / etc.)

    U->>FE: types "Summarise this PDF"<br/>(PDF already uploaded)
    FE->>J: POST /api/v1/ai/orchestrate/stream<br/>multipart: fileInputs[], userMessage
    Note over J: ByteHashFileIdStrategy<br/>id = sha256(bytes)[:16]
    J->>O: POST /api/v1/orchestrator<br/>{ files:[{id,name}], userMessage }

    O->>L: route via fast model
    L-->>O: delegate_pdf_question
    O->>QA: PdfQuestionRequest

    loop for each file
        QA->>RAG: has_collection(file.id)
        RAG-->>QA: false
    end
    QA-->>O: NeedIngestResponse(files_to_ingest)
    O-->>J: { outcome:"need_ingest", filesToIngest:[...] }

    Note over J: onNeedIngest
    loop per file
        J->>J: PDFBox: extract page text
        J->>O: POST /api/v1/rag/documents<br/>(long-running timeout)
        O->>RAG: chunk + stage documents
        O->>V: embed_documents (batches of 256)
        V-->>O: embeddings
        O->>RAG: add_documents
        O-->>J: { chunks_indexed: N }
    end

    Note over J: retry with resumeWith=pdf_question
    J->>O: POST /api/v1/orchestrator
    Note over O: fast-path to PdfQuestionAgent

    O->>QA: PdfQuestionRequest
    Note over QA: build RagCapability<br/>pinned to file IDs
    QA->>L: run(prompt) with search_knowledge tool

    loop up to max_searches
        L->>QA: search_knowledge(query)
        QA->>V: embed_query
        V-->>QA: query vector
        QA->>RAG: search(vector, collections=[file.id])
        RAG-->>QA: top-k chunks
        QA-->>L: formatted chunks
    end

    Note over QA: once budget spent,<br/>prepare() hides the tool
    L-->>QA: PdfQuestionAnswerResponse
    QA-->>O: answer
    O-->>J: { outcome:"answer", answer, evidence }
    J-->>FE: SSE "result"
    FE->>U: assistant bubble
```

engine/src/stirling/contracts/__init__.py

@@ -10,12 +10,14 @@ from .agent_drafts import (
 from .agent_specs import AgentSpec, AgentSpecStep, AiToolAgentStep
 from .comments import CommentSpec
 from .common import (
+    AiFile,
     ArtifactKind,
     ConversationMessage,
     ExtractedFileText,
     MathAuditorToolReportArtifact,
     NeedContentFileRequest,
     NeedContentResponse,
+    NeedIngestResponse,
     PdfContentType,
     PdfTextSelection,
     StepKind,
@@ -24,6 +26,7 @@ from .common import (
     ToolReportArtifact,
     WorkflowOutcome,
     format_conversation_history,
+    format_file_names,
 )
 from .execution import (
     AgentExecutionRequest,
@@ -71,24 +74,20 @@ from .pdf_edit import (
 from .pdf_questions import (
     PdfQuestionAnswerResponse,
     PdfQuestionNotFoundResponse,
+    PdfQuestionOrchestrateResponse,
     PdfQuestionRequest,
     PdfQuestionResponse,
     PdfQuestionTerminalResponse,
 )
 from .rag import (
-    MAX_INDEX_TEXT_LENGTH,
-    RagCollectionsResponse,
-    RagDeleteCollectionResponse,
-    RagIndexRequest,
-    RagIndexResponse,
-    RagSearchRequest,
-    RagSearchResponse,
-    RagSearchResultItem,
-    RagStatusResponse,
+    DeleteDocumentResponse,
+    IngestDocumentRequest,
+    IngestDocumentResponse,
+    IngestedPageText,
 )
 
 __all__ = [
-    "MAX_INDEX_TEXT_LENGTH",
+    "AiFile",
     "AgentDraft",
     "AgentDraftRequest",
     "AgentDraftResponse",
@@ -105,6 +104,7 @@ __all__ = [
     "CommentSpec",
     "CompletedExecutionAction",
     "ConversationMessage",
+    "DeleteDocumentResponse",
     "Discrepancy",
     "DiscrepancyKind",
     "EditCannotDoResponse",
@@ -119,10 +119,15 @@ __all__ = [
     "FolioManifest",
     "FolioType",
     "format_conversation_history",
+    "format_file_names",
     "HealthResponse",
+    "IngestDocumentRequest",
+    "IngestDocumentResponse",
+    "IngestedPageText",
     "MathAuditorToolReportArtifact",
     "NeedContentFileRequest",
     "NeedContentResponse",
+    "NeedIngestResponse",
     "NextExecutionAction",
     "OrchestratorRequest",
     "OrchestratorResponse",
@@ -136,18 +141,11 @@ __all__ = [
     "PdfEditTerminalResponse",
     "PdfQuestionAnswerResponse",
     "PdfQuestionNotFoundResponse",
+    "PdfQuestionOrchestrateResponse",
     "PdfQuestionRequest",
     "PdfQuestionResponse",
     "PdfQuestionTerminalResponse",
     "PdfTextSelection",
-    "RagCollectionsResponse",
-    "RagDeleteCollectionResponse",
-    "RagIndexRequest",
-    "RagIndexResponse",
-    "RagSearchRequest",
-    "RagSearchResponse",
-    "RagSearchResultItem",
-    "RagStatusResponse",
     "Requisition",
     "Severity",
     "StepKind",

engine/src/stirling/contracts/common.py

@@ -6,7 +6,7 @@ from typing import Literal, assert_never
 from pydantic import Field, model_validator
 
 from stirling.contracts.ledger import Verdict
-from stirling.models import OPERATIONS, ApiModel, ToolEndpoint
+from stirling.models import OPERATIONS, ApiModel, FileId, ToolEndpoint
 from stirling.models.agent_tool_models import AGENT_OPERATIONS, AgentToolId, AnyParamModel, AnyToolId
 
 
@@ -50,6 +50,7 @@ class WorkflowOutcome(StrEnum):
 
     ANSWER = "answer"
     NEED_CONTENT = "need_content"
+    NEED_INGEST = "need_ingest"
     NOT_FOUND = "not_found"
     PLAN = "plan"
     NEED_CLARIFICATION = "need_clarification"
@@ -94,12 +95,30 @@ class ConversationMessage(ApiModel):
     content: str
 
 
+class AiFile(ApiModel):
+    """A file the user has supplied, identified by both a stable id and a display name.
+
+    The id is opaque to the engine: Java generates it (content hash, file path, UUID, etc.)
+    and the engine uses it as the RAG collection key for any agent that indexes content.
+    The name is used in user-facing prompts and responses.
+    """
+
+    id: FileId = Field(min_length=1)
+    name: str = Field(min_length=1)
+
+
 def format_conversation_history(conversation_history: list[ConversationMessage]) -> str:
     if not conversation_history:
         return "None"
     return "\n".join(f"- {message.role}: {message.content}" for message in conversation_history)
 
 
+def format_file_names(files: list[AiFile]) -> str:
+    if not files:
+        return "No file names were provided."
+    return ", ".join(file.name for file in files)
+
+
 class PdfTextSelection(ApiModel):
     page_number: int | None = None
     text: str
@@ -111,7 +130,7 @@ class ExtractedFileText(ApiModel):
 
 
 class NeedContentFileRequest(ApiModel):
-    file_name: str
+    file: AiFile
     page_numbers: list[int] = Field(default_factory=list)
     content_types: list[PdfContentType]
 
@@ -146,6 +165,20 @@ class MathAuditorToolReportArtifact(ApiModel):
 ToolReportArtifact = MathAuditorToolReportArtifact
 
 
+class NeedIngestResponse(ApiModel):
+    """Signal that the listed files must be ingested into RAG before the agent can continue.
+
+    Java's handling: for each file, extract the requested content types, POST to
+    ``/api/v1/rag/documents`` keyed by ``file.id``, then retry the original request.
+    """
+
+    outcome: Literal[WorkflowOutcome.NEED_INGEST] = WorkflowOutcome.NEED_INGEST
+    resume_with: SupportedCapability
+    reason: str
+    files_to_ingest: list[AiFile]
+    content_types: list[PdfContentType] = Field(default_factory=list)
+
+
 class ToolOperationStep(ApiModel):
     kind: Literal[StepKind.TOOL] = StepKind.TOOL
     tool: AnyToolId

engine/src/stirling/contracts/orchestrator.py

@@ -8,10 +8,12 @@ from stirling.models import ApiModel
 
 from .agent_drafts import AgentDraftResponse
 from .common import (
+    AiFile,
     ArtifactKind,
     ConversationMessage,
     ExtractedFileText,
     NeedContentResponse,
+    NeedIngestResponse,
     SupportedCapability,
     ToolReportArtifact,
     WorkflowOutcome,
@@ -31,7 +33,7 @@ WorkflowArtifact = Annotated[ExtractedTextArtifact | ToolReportArtifact, Field(d
 
 class OrchestratorRequest(ApiModel):
     user_message: str
-    file_names: list[str]
+    files: list[AiFile] = Field(default_factory=list)
     conversation_history: list[ConversationMessage] = Field(default_factory=list)
     artifacts: list[WorkflowArtifact] = Field(default_factory=list)
     resume_with: SupportedCapability | None = None
@@ -47,6 +49,7 @@ type OrchestratorResponse = Annotated[
     PdfEditTerminalResponse
     | PdfQuestionTerminalResponse
     | NeedContentResponse
+    | NeedIngestResponse
     | AgentDraftResponse
     | NextExecutionAction
     | UnsupportedCapabilityResponse,

engine/src/stirling/contracts/pdf_edit.py

@@ -7,6 +7,7 @@ from pydantic import Field
 from stirling.models import ApiModel
 
 from .common import (
+    AiFile,
     ConversationMessage,
     ExtractedFileText,
     NeedContentResponse,
@@ -18,7 +19,7 @@ from .common import (
 
 class PdfEditRequest(ApiModel):
     user_message: str
-    file_names: list[str] = Field(default_factory=list)
+    files: list[AiFile] = Field(default_factory=list)
     conversation_history: list[ConversationMessage] = Field(default_factory=list)
     page_text: list[ExtractedFileText] = Field(default_factory=list)
 

engine/src/stirling/contracts/pdf_questions.py

@@ -7,9 +7,10 @@ from pydantic import Field
 from stirling.models import ApiModel
 
 from .common import (
+    AiFile,
     ConversationMessage,
     ExtractedFileText,
-    NeedContentResponse,
+    NeedIngestResponse,
     WorkflowOutcome,
 )
 from .pdf_edit import EditPlanResponse
@@ -17,8 +18,7 @@ from .pdf_edit import EditPlanResponse
 
 class PdfQuestionRequest(ApiModel):
     question: str
-    page_text: list[ExtractedFileText] = Field(default_factory=list)
-    file_names: list[str]
+    files: list[AiFile] = Field(default_factory=list)
     conversation_history: list[ConversationMessage] = Field(default_factory=list)
 
 
@@ -26,15 +26,6 @@ class PdfQuestionAnswerResponse(ApiModel):
     outcome: Literal[WorkflowOutcome.ANSWER] = WorkflowOutcome.ANSWER
     answer: str
     evidence: list[ExtractedFileText] = Field(default_factory=list)
-    edit_plan: EditPlanResponse | None = Field(
-        default=None,
-        description=(
-            "Optional plan the caller must run before the answer is final. When"
-            " populated, ``answer`` is empty on this turn — the caller executes"
-            " the plan and re-invokes the orchestrator with ``resume_with`` set"
-            " to PDF_QUESTION; the real answer arrives on the resume turn."
-        ),
-    )
 
 
 class PdfQuestionNotFoundResponse(ApiModel):
@@ -44,6 +35,14 @@ class PdfQuestionNotFoundResponse(ApiModel):
 
 type PdfQuestionTerminalResponse = PdfQuestionAnswerResponse | PdfQuestionNotFoundResponse
 type PdfQuestionResponse = Annotated[
-    PdfQuestionTerminalResponse | NeedContentResponse,
+    PdfQuestionTerminalResponse | NeedIngestResponse,
     Field(discriminator="outcome"),
 ]
+
+
+# ``orchestrate`` may also emit an ``EditPlanResponse`` on the math-routing
+# first turn (``outcome=PLAN`` with ``resume_with=PDF_QUESTION``). It's not in
+# ``PdfQuestionTerminalResponse`` because that alias would otherwise duplicate
+# the PLAN branch already provided by ``PdfEditTerminalResponse`` in the
+# top-level :class:`OrchestratorResponse` discriminated union.
+type PdfQuestionOrchestrateResponse = PdfQuestionResponse | EditPlanResponse

engine/src/stirling/contracts/rag.py

@@ -4,48 +4,36 @@ from pydantic import Field
 
 from stirling.models import ApiModel
 
-MAX_INDEX_TEXT_LENGTH = 1_000_000  # 1MB text limit per index request
+from .common import FileId
 
 
-class RagStatusResponse(ApiModel):
-    embedding_model: str
-    collections: list[str]
+class IngestedPageText(ApiModel):
+    page_number: int = Field(ge=1)
+    text: str
 
 
-class RagIndexRequest(ApiModel):
-    collection: str = Field(min_length=1)
-    text: str = Field(max_length=MAX_INDEX_TEXT_LENGTH)
-    source: str = ""
-    metadata: dict[str, str] = Field(default_factory=dict)
+class IngestDocumentRequest(ApiModel):
+    """Replace-ingest a document's content into RAG under the given document_id.
+
+    Each content-type field is optional; the endpoint replaces the document's entire
+    stored content with whatever is provided. To add a content type later, call again
+    with all content types the document should have (incremental-add-without-replace
+    will be a separate endpoint if/when we need it).
+
+    ``source`` is a human-readable label (typically the original filename) that flows
+    into chunk metadata so search results are readable when document_id is a hash.
+    """
+
+    document_id: FileId = Field(min_length=1)
+    source: str = Field(min_length=1)
+    page_text: list[IngestedPageText] | None = None
 
 
-class RagIndexResponse(ApiModel):
-    collection: str
+class IngestDocumentResponse(ApiModel):
+    document_id: FileId
     chunks_indexed: int
 
 
-class RagSearchRequest(ApiModel):
-    query: str
-    collection: str | None = Field(default=None, min_length=1)
-    top_k: int = 5
-
-
-class RagSearchResultItem(ApiModel):
-    text: str
-    source: str
-    chunk_id: str
-    score: float
-
-
-class RagSearchResponse(ApiModel):
-    query: str
-    results: list[RagSearchResultItem]
-
-
-class RagCollectionsResponse(ApiModel):
-    collections: list[str]
-
-
-class RagDeleteCollectionResponse(ApiModel):
-    status: str
-    collection: str
+class DeleteDocumentResponse(ApiModel):
+    document_id: FileId
+    deleted: bool