Set up document management for Stirling Engine (#6476)

# Description of Changes
Change Stirling Engine to support deleting documents automatically. This
happens both on user logout and after an amount of time specified by the
Java when ingesting a document (allowing for personal documents to have
short lifetimes but org documents to be left in the db with no expiry
date). Also sets up an [ACL
policy](https://en.wikipedia.org/wiki/Access-control_list) for the
documents so the database knows which users have access to which
documents. This is not fully implemented in the Java, so currently all
docs are treated as having a single owner, the uploader, but
theoretically when we need to support org storage, we shouldn't need to
change the db schema.

engine/src/stirling/services/__init__.py

@@ -7,14 +7,16 @@ from .progress import (
     set_progress_emitter,
 )
 from .runtime import AppRuntime, build_model_settings, build_runtime
-from .tracking import setup_posthog_tracking
+from .tracking import current_user_id, require_current_user_id, setup_posthog_tracking
 
 __all__ = [
     "AppRuntime",
     "ProgressEmitter",
     "build_model_settings",
     "build_runtime",
+    "current_user_id",
     "emit_progress",
+    "require_current_user_id",
     "reset_progress_emitter",
     "set_progress_emitter",
     "setup_posthog_tracking",

engine/src/stirling/services/runtime.py

@@ -16,7 +16,6 @@ from stirling.documents import (
     DocumentStore,
     EmbeddingService,
     PgVectorStore,
-    RagCapability,
     SqliteVecStore,
 )
 
@@ -47,7 +46,6 @@ class AppRuntime:
     fast_model: Model
     smart_model: Model
     documents: DocumentService
-    rag_capability: RagCapability
 
     @property
     def fast_model_settings(self) -> ModelSettings:
@@ -89,8 +87,8 @@ def _build_document_store(settings: AppSettings) -> DocumentStore:
     assert_never(settings.rag_backend)
 
 
-def _build_documents(settings: AppSettings) -> tuple[DocumentService, RagCapability]:
-    """Build the document service and the RAG-search capability that wraps it."""
+def _build_documents(settings: AppSettings) -> DocumentService:
+    """Build the document service used by per-request RAG capabilities."""
     logger.info("Documents: embedding_model=%s", settings.rag_embedding_model)
     embedder = EmbeddingService(
         model_name=settings.rag_embedding_model,
@@ -98,9 +96,7 @@ def _build_documents(settings: AppSettings) -> tuple[DocumentService, RagCapabil
         chunk_overlap=settings.rag_chunk_overlap,
     )
     store = _build_document_store(settings)
-    service = DocumentService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k)
-    capability = RagCapability(documents=service, top_k=settings.rag_default_top_k)
-    return service, capability
+    return DocumentService(embedder=embedder, store=store, default_top_k=settings.rag_default_top_k)
 
 
 def build_runtime(settings: AppSettings) -> AppRuntime:
@@ -109,14 +105,11 @@ def build_runtime(settings: AppSettings) -> AppRuntime:
     validate_structured_output_support(fast_model, settings.fast_model_name)
     validate_structured_output_support(smart_model, settings.smart_model_name)
 
-    documents, rag_capability = _build_documents(settings)
-
     return AppRuntime(
         settings=settings,
         fast_model=fast_model,
         smart_model=smart_model,
-        documents=documents,
-        rag_capability=rag_capability,
+        documents=_build_documents(settings),
     )
 
 

engine/src/stirling/services/tracking.py

@@ -27,10 +27,25 @@ from opentelemetry.trace import Span
 from posthog.client import Client as PostHogClient
 
 from stirling.config import AppSettings
+from stirling.models import UserId
 
 # Per-request user ID, set by middleware from the X-User-Id header.
 # When not set, PostHog generates a random ID and marks the event as personless.
-current_user_id: ContextVar[str | None] = ContextVar("current_user_id", default=None)
+current_user_id: ContextVar[UserId | None] = ContextVar("current_user_id", default=None)
+
+
+def require_current_user_id() -> UserId:
+    """Return the request's user ID or raise if the X-User-Id header was missing.
+
+    Use at the boundary of any code path that touches per-user document
+    storage (vector chunks, page text, ACL rows). Routes prefer the FastAPI dependency form
+    (``Depends(require_user_id)``); agent internals that don't have a request
+    object in scope call this helper directly to fail closed.
+    """
+    user_id = current_user_id.get()
+    if user_id is None:
+        raise RuntimeError("X-User-Id is required for this operation but was not set on the request")
+    return user_id
 
 
 class LRUSet: