Add ability for Stirling engine to reason across large documents (#6314)

# Description of Changes
Adds storage in the database for full document content alongside the RAG
content (and changes the service to `DocumentService` instead of
`RagService`). Then adds a generic capability that should be usable by
any agent (currently just used by the Question Agent) which allows the
agent to pull out the full contents of the doc, chunks it into various
sections that will fit in the context window, and then processes them in
parallel to create an intermediate result, and then processes the
intermediate result into a final answer. It will re-chunk as many times
as necessary to get the content small enough for the actual answer to be
analysed (I've tested on PDFs ~3500 pages long, which is well above the
context limit and requires maybe 3 rounds of compression to get an
answer).

The new full doc analysis stuff is heavier than the RAG lookup so both
remain. The agents should use RAG for targeted info and the chunked
reasoner for info that requires reading the full doc.

engine/src/stirling/services/__init__.py

@@ -1,11 +1,21 @@
 """Shared services used by the Stirling AI runtime."""
 
+from .progress import (
+    ProgressEmitter,
+    emit_progress,
+    reset_progress_emitter,
+    set_progress_emitter,
+)
 from .runtime import AppRuntime, build_model_settings, build_runtime
 from .tracking import setup_posthog_tracking
 
 __all__ = [
     "AppRuntime",
+    "ProgressEmitter",
     "build_model_settings",
     "build_runtime",
+    "emit_progress",
+    "reset_progress_emitter",
+    "set_progress_emitter",
     "setup_posthog_tracking",
 ]

engine/src/stirling/services/progress.py

@@ -0,0 +1,47 @@
+"""Per-request progress emission, plumbed via a ContextVar so deep call stacks
+can publish typed events to the streaming orchestrator endpoint without every
+intermediate layer knowing about it.
+
+Outside a streaming request no emitter is bound and ``emit_progress`` is a
+no-op, so callers in agents/services can emit unconditionally.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from contextvars import ContextVar, Token
+
+from stirling.contracts import ProgressEvent
+
+logger = logging.getLogger(__name__)
+
+type ProgressEmitter = Callable[[ProgressEvent], Awaitable[None]]
+
+_emitter: ContextVar[ProgressEmitter | None] = ContextVar("stirling_progress_emitter", default=None)
+
+
+def set_progress_emitter(emitter: ProgressEmitter | None) -> Token[ProgressEmitter | None]:
+    return _emitter.set(emitter)
+
+
+def reset_progress_emitter(token: Token[ProgressEmitter | None]) -> None:
+    _emitter.reset(token)
+
+
+async def emit_progress(event: ProgressEvent) -> None:
+    """Publish ``event`` to the current request's emitter, if any.
+
+    Failures inside the emitter are logged and swallowed so progress emission
+    can never break the work it's reporting on.
+    """
+    emitter = _emitter.get()
+    if emitter is None:
+        return
+    try:
+        await emitter(event)
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        logger.exception("progress emitter raised; dropping event %r", event.phase)

engine/src/stirling/services/runtime.py

@@ -4,28 +4,49 @@ import logging
 from dataclasses import dataclass
 from typing import assert_never
 
+import httpx
 from pydantic_ai.models import Model, infer_model
+from pydantic_ai.models.anthropic import AnthropicModel
+from pydantic_ai.providers.anthropic import AnthropicProvider
 from pydantic_ai.settings import ModelSettings
 
 from stirling.config import ENGINE_ROOT, AppSettings, RagBackend
-from stirling.rag import (
+from stirling.documents import (
+    DocumentService,
+    DocumentStore,
     EmbeddingService,
     PgVectorStore,
     RagCapability,
-    RagService,
     SqliteVecStore,
-    VectorStore,
 )
 
 logger = logging.getLogger(__name__)
 
 
+def _build_anthropic_http_client() -> httpx.AsyncClient:
+    """Build the httpx client used for Anthropic API calls.
+
+    We disable connection-pool keepalive so every request opens a fresh
+    TCP+TLS connection. The default HTTP/1.1 pool reuses connections that
+    Anthropic's front door (Cloudflare) sometimes closes silently between
+    requests; the next request that picks up a stale connection sends its
+    body into a black hole and never gets a response, hanging until our
+    chunked-reasoner timeout fires.
+
+    A fresh handshake costs ~150ms — rounding error against a 5-15s LLM
+    call. The trade is determinism: we never reuse a connection that might
+    have died in the pool. See ``STIRLING_HTTP_DEBUG`` traces of slice 6
+    on 2026-05-06 for the concrete failure mode this addresses.
+    """
+    return httpx.AsyncClient(limits=httpx.Limits(max_keepalive_connections=0))
+
+
 @dataclass(frozen=True)
 class AppRuntime:
     settings: AppSettings
     fast_model: Model
     smart_model: Model
-    rag_service: RagService
+    documents: DocumentService
     rag_capability: RagCapability
 
     @property
@@ -53,47 +74,62 @@ def validate_structured_output_support(model: Model, model_name: str) -> None:
         raise ValueError(f"Unsupported model {model_name}. This model does not support structured outputs.")
 
 
-def _build_vector_store(settings: AppSettings) -> VectorStore:
-    """Build the configured vector store backend."""
+def _build_document_store(settings: AppSettings) -> DocumentStore:
+    """Build the configured document store backend."""
     if settings.rag_backend == RagBackend.SQLITE:
         store_path = settings.rag_store_path
         # Treat ":memory:" as a special in-process token; otherwise resolve against the engine root.
         if str(store_path) != ":memory:" and not store_path.is_absolute():
             store_path = ENGINE_ROOT / store_path
-        logger.info("RAG backend=sqlite, db_path=%s", store_path)
+        logger.info("Document store backend=sqlite, db_path=%s", store_path)
         return SqliteVecStore(db_path=store_path)
     if settings.rag_backend == RagBackend.PGVECTOR:
-        logger.info("RAG backend=pgvector, dsn=<configured>")
+        logger.info("Document store backend=pgvector, dsn=<configured>")
         return PgVectorStore(dsn=settings.rag_pgvector_dsn)
     assert_never(settings.rag_backend)
 
 
-def _build_rag(settings: AppSettings) -> tuple[RagService, RagCapability]:
-    """Build the RAG service and capability."""
-    logger.info("RAG: embedding_model=%s", settings.rag_embedding_model)
+def _build_documents(settings: AppSettings) -> tuple[DocumentService, RagCapability]:
+    """Build the document service and the RAG-search capability that wraps it."""
+    logger.info("Documents: embedding_model=%s", settings.rag_embedding_model)
     embedder = EmbeddingService(
         model_name=settings.rag_embedding_model,
         chunk_size=settings.rag_chunk_size,
         chunk_overlap=settings.rag_chunk_overlap,
     )
-    store = _build_vector_store(settings)
-    service = RagService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k)
-    capability = RagCapability(rag_service=service, top_k=settings.rag_default_top_k)
+    store = _build_document_store(settings)
+    service = DocumentService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k)
+    capability = RagCapability(documents=service, top_k=settings.rag_default_top_k)
     return service, capability
 
 
 def build_runtime(settings: AppSettings) -> AppRuntime:
-    fast_model = infer_model(settings.fast_model_name)
-    smart_model = infer_model(settings.smart_model_name)
+    fast_model = _build_model(settings.fast_model_name)
+    smart_model = _build_model(settings.smart_model_name)
     validate_structured_output_support(fast_model, settings.fast_model_name)
     validate_structured_output_support(smart_model, settings.smart_model_name)
 
-    rag_service, rag_capability = _build_rag(settings)
+    documents, rag_capability = _build_documents(settings)
 
     return AppRuntime(
         settings=settings,
         fast_model=fast_model,
         smart_model=smart_model,
-        rag_service=rag_service,
+        documents=documents,
         rag_capability=rag_capability,
     )
+
+
+def _build_model(model_name: str) -> Model:
+    """Construct a model, injecting our keepalive-free httpx client for
+    Anthropic models so workers don't pick up stale pooled connections.
+
+    Other providers fall back to ``infer_model`` defaults; the stale-pool
+    issue is specific to the Cloudflare-fronted Anthropic API in our
+    observations and the fix doesn't necessarily apply elsewhere.
+    """
+    if model_name.startswith("anthropic:"):
+        bare_name = model_name.removeprefix("anthropic:")
+        provider = AnthropicProvider(http_client=_build_anthropic_http_client())
+        return AnthropicModel(bare_name, provider=provider)
+    return infer_model(model_name)