Flesh out RAG system (#6197)

# Description of Changes
Flesh out the RAG system and connect it to the PDF Question Agent so it
can respond to questions about PDFs of an extremely large size.

I'd expect lots more work will need to be done to finish off the RAG
system to really be what we need, but this should be a reasonable start
which will let us connect it to tools and have the ingestion mostly
handled automatically. I'm leaving file deletion and proper file ID
management to be done in a future PR. We also need to consider whether
all tools should retrieve content exclusively via RAG, or whether it's
beneficial to have tools sometimes fetch the direct content and other
times fetch it from RAG.

A diagram of the expected interaction is as follows:

```mermaid
sequenceDiagram
    autonumber
    actor U as User
    participant FE as Frontend<br/>(ChatPanel)
    participant J as Java<br/>(AiWorkflowService)
    participant O as Engine:<br/>OrchestratorAgent
    participant QA as Engine:<br/>PdfQuestionAgent
    participant RAG as Engine:<br/>RagService + SqliteVecStore
    participant V as VoyageAI<br/>(embeddings)
    participant L as LLM<br/>(Claude / etc.)

    U->>FE: types "Summarise this PDF"<br/>(PDF already uploaded)
    FE->>J: POST /api/v1/ai/orchestrate/stream<br/>multipart: fileInputs[], userMessage
    Note over J: ByteHashFileIdStrategy<br/>id = sha256(bytes)[:16]
    J->>O: POST /api/v1/orchestrator<br/>{ files:[{id,name}], userMessage }

    O->>L: route via fast model
    L-->>O: delegate_pdf_question
    O->>QA: PdfQuestionRequest

    loop for each file
        QA->>RAG: has_collection(file.id)
        RAG-->>QA: false
    end
    QA-->>O: NeedIngestResponse(files_to_ingest)
    O-->>J: { outcome:"need_ingest", filesToIngest:[...] }

    Note over J: onNeedIngest
    loop per file
        J->>J: PDFBox: extract page text
        J->>O: POST /api/v1/rag/documents<br/>(long-running timeout)
        O->>RAG: chunk + stage documents
        O->>V: embed_documents (batches of 256)
        V-->>O: embeddings
        O->>RAG: add_documents
        O-->>J: { chunks_indexed: N }
    end

    Note over J: retry with resumeWith=pdf_question
    J->>O: POST /api/v1/orchestrator
    Note over O: fast-path to PdfQuestionAgent

    O->>QA: PdfQuestionRequest
    Note over QA: build RagCapability<br/>pinned to file IDs
    QA->>L: run(prompt) with search_knowledge tool

    loop up to max_searches
        L->>QA: search_knowledge(query)
        QA->>V: embed_query
        V-->>QA: query vector
        QA->>RAG: search(vector, collections=[file.id])
        RAG-->>QA: top-k chunks
        QA-->>L: formatted chunks
    end

    Note over QA: once budget spent,<br/>prepare() hides the tool
    L-->>QA: PdfQuestionAnswerResponse
    QA-->>O: answer
    O-->>J: { outcome:"answer", answer, evidence }
    J-->>FE: SSE "result"
    FE->>U: assistant bubble
```

engine/src/stirling/api/app.py

@@ -7,7 +7,13 @@ from fastapi import Depends, FastAPI
 from pydantic_ai import Agent
 from pydantic_ai.models.instrumented import InstrumentationSettings
 
-from stirling.agents import ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, PdfQuestionAgent, UserSpecAgent
+from stirling.agents import (
+    ExecutionPlanningAgent,
+    OrchestratorAgent,
+    PdfEditAgent,
+    PdfQuestionAgent,
+    UserSpecAgent,
+)
 from stirling.agents.ledger import MathAuditorAgent
 from stirling.agents.pdf_comment import PdfCommentAgent
 from stirling.api.middleware import UserIdMiddleware
@@ -51,6 +57,7 @@ async def lifespan(fast_api: FastAPI):
     if tracer_provider:
         Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider))
     yield
+    await runtime.rag_service.close()
     if tracer_provider:
         tracer_provider.shutdown()
 

engine/src/stirling/api/dependencies.py

@@ -2,7 +2,13 @@ from __future__ import annotations
 
 from fastapi import Request
 
-from stirling.agents import ExecutionPlanningAgent, OrchestratorAgent, PdfEditAgent, PdfQuestionAgent, UserSpecAgent
+from stirling.agents import (
+    ExecutionPlanningAgent,
+    OrchestratorAgent,
+    PdfEditAgent,
+    PdfQuestionAgent,
+    UserSpecAgent,
+)
 from stirling.agents.ledger import MathAuditorAgent
 from stirling.agents.pdf_comment import PdfCommentAgent
 from stirling.rag import RagService
@@ -37,10 +43,6 @@ def get_rag_service(request: Request) -> RagService:
     return request.app.state.runtime.rag_service
 
 
-def get_rag_embedding_model(request: Request) -> str:
-    return request.app.state.runtime.settings.rag_embedding_model
-
-
 def get_math_auditor_agent(request: Request) -> MathAuditorAgent:
     return request.app.state.math_auditor_agent
 

engine/src/stirling/api/routes/rag.py

@@ -4,75 +4,60 @@ from typing import Annotated
 
 from fastapi import APIRouter, Depends
 
-from stirling.api.dependencies import get_rag_embedding_model, get_rag_service
+from stirling.api.dependencies import get_rag_service
 from stirling.contracts import (
-    RagCollectionsResponse,
-    RagDeleteCollectionResponse,
-    RagIndexRequest,
-    RagIndexResponse,
-    RagSearchRequest,
-    RagSearchResponse,
-    RagSearchResultItem,
-    RagStatusResponse,
+    DeleteDocumentResponse,
+    IngestDocumentRequest,
+    IngestDocumentResponse,
+    PdfContentType,
 )
-from stirling.rag import RagService
+from stirling.models import FileId
+from stirling.rag import Document, RagService
 
 router = APIRouter(prefix="/api/v1/rag", tags=["rag"])
 
 
-@router.get("/status", response_model=RagStatusResponse)
-async def rag_status(
+@router.post("/documents", response_model=IngestDocumentResponse)
+async def ingest_document(
+    request: IngestDocumentRequest,
     rag: Annotated[RagService, Depends(get_rag_service)],
-    embedding_model: Annotated[str, Depends(get_rag_embedding_model)],
-) -> RagStatusResponse:
-    collections = await rag.list_collections()
-    return RagStatusResponse(embedding_model=embedding_model, collections=collections)
+) -> IngestDocumentResponse:
+    """Replace-ingest a document's content under ``document_id``.
+
+    Any previously-stored content for this document is removed and the
+    provided content replaces it wholesale. All pages are chunked up front
+    and then embedded in a single batched call so large documents (e.g. a
+    500-page book) don't fan out into hundreds of embedding requests.
+    """
+    await rag.delete_collection(request.document_id)
+
+    chunks: list[Document] = []
+    if request.page_text:
+        for page in request.page_text:
+            if not page.text.strip():
+                continue
+            chunks.extend(
+                rag.chunk_text(
+                    text=page.text,
+                    source=f"{request.source}:page:{page.page_number}",
+                    base_metadata={
+                        "page_number": str(page.page_number),
+                        "content_type": PdfContentType.PAGE_TEXT.value,
+                    },
+                )
+            )
+
+    indexed = await rag.index_documents(request.document_id, chunks) if chunks else 0
+    return IngestDocumentResponse(document_id=request.document_id, chunks_indexed=indexed)
 
 
-@router.post("/index", response_model=RagIndexResponse)
-async def rag_index(
-    request: RagIndexRequest,
+@router.delete("/documents/{document_id}", response_model=DeleteDocumentResponse)
+async def delete_document(
+    document_id: FileId,
     rag: Annotated[RagService, Depends(get_rag_service)],
-) -> RagIndexResponse:
-    count = await rag.index_text(
-        collection=request.collection,
-        text=request.text,
-        source=request.source,
-        metadata=request.metadata,
-    )
-    return RagIndexResponse(collection=request.collection, chunks_indexed=count)
-
-
-@router.post("/search", response_model=RagSearchResponse)
-async def rag_search(
-    request: RagSearchRequest,
-    rag: Annotated[RagService, Depends(get_rag_service)],
-) -> RagSearchResponse:
-    results = await rag.search(query=request.query, collection=request.collection, top_k=request.top_k)
-    items = [
-        RagSearchResultItem(
-            text=r.document.text,
-            source=r.document.metadata.get("source", ""),
-            chunk_id=r.document.metadata.get("chunk_index", ""),
-            score=r.score,
-        )
-        for r in results
-    ]
-    return RagSearchResponse(query=request.query, results=items)
-
-
-@router.get("/collections", response_model=RagCollectionsResponse)
-async def rag_collections(
-    rag: Annotated[RagService, Depends(get_rag_service)],
-) -> RagCollectionsResponse:
-    collections = await rag.list_collections()
-    return RagCollectionsResponse(collections=collections)
-
-
-@router.delete("/collections/{name}", response_model=RagDeleteCollectionResponse)
-async def rag_delete_collection(
-    name: str,
-    rag: Annotated[RagService, Depends(get_rag_service)],
-) -> RagDeleteCollectionResponse:
-    await rag.delete_collection(name)
-    return RagDeleteCollectionResponse(status="deleted", collection=name)
+) -> DeleteDocumentResponse:
+    """Remove a document's content from RAG. Idempotent."""
+    existed = await rag.has_collection(document_id)
+    if existed:
+        await rag.delete_collection(document_id)
+    return DeleteDocumentResponse(document_id=document_id, deleted=existed)