Set up document management for Stirling Engine (#6476)

# Description of Changes
Change Stirling Engine to support deleting documents automatically. This
happens both on user logout and after an amount of time specified by the
Java when ingesting a document (allowing for personal documents to have
short lifetimes but org documents to be left in the db with no expiry
date). Also sets up an [ACL
policy](https://en.wikipedia.org/wiki/Access-control_list) for the
documents so the database knows which users have access to which
documents. This is not fully implemented in the Java, so currently all
docs are treated as having a single owner, the uploader, but
theoretically when we need to support org storage, we shouldn't need to
change the db schema.

engine/src/stirling/api/app.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import asyncio
+import logging
 from contextlib import asynccontextmanager
 from typing import Annotated
 
@@ -29,8 +31,45 @@ from stirling.api.routes import (
 )
 from stirling.config import AppSettings, load_settings
 from stirling.contracts import HealthResponse
+from stirling.documents import DocumentService
 from stirling.services import build_runtime, setup_posthog_tracking
 
+logger = logging.getLogger(__name__)
+
+
+async def _run_expired_doc_reaper(
+    documents: DocumentService,
+    interval_seconds: int,
+) -> None:
+    """Periodically delete documents whose ``expires_at`` has passed.
+
+    A reaped collection drops everything rooted at that document. Backstop
+    for the explicit logout purge: catches sessions that ended without a
+    clean logout (tab close, JWT expiry, engine restart). Persistent rows
+    (``expires_at`` null, the shape we use for org-shared docs) are never
+    touched. Runs until cancelled by the lifespan teardown.
+    """
+    await _reap(documents)
+    while True:
+        await asyncio.sleep(interval_seconds)
+        await _reap(documents)
+
+
+async def _reap(documents: DocumentService) -> None:
+    """One reaper iteration. Logs the deleted count on success and the full
+    exception with traceback on failure; never re-raises non-cancel errors so
+    a bad iteration doesn't kill the loop. ``asyncio.CancelledError`` is
+    re-raised so the lifespan teardown can cancel the task cleanly.
+    """
+    try:
+        deleted = await documents.reap_expired()
+        if deleted:
+            logger.info("Reaped %d expired document collection(s)", deleted)
+    except asyncio.CancelledError:
+        raise
+    except Exception:
+        logger.exception("Document reaper iteration failed; will retry on next interval")
+
 
 def _load_startup_settings(fast_api: FastAPI) -> AppSettings:
     override = fast_api.dependency_overrides.get(load_settings)
@@ -56,7 +95,19 @@ async def lifespan(fast_api: FastAPI):
     tracer_provider = setup_posthog_tracking(settings)
     if tracer_provider:
         Agent.instrument_all(InstrumentationSettings(tracer_provider=tracer_provider))
+    reaper_task = asyncio.create_task(
+        _run_expired_doc_reaper(
+            runtime.documents,
+            interval_seconds=settings.documents_reaper_interval_seconds,
+        ),
+        name="expired-document-reaper",
+    )
     yield
+    reaper_task.cancel()
+    try:
+        await reaper_task
+    except asyncio.CancelledError:
+        pass
     await runtime.documents.close()
     if tracer_provider:
         tracer_provider.shutdown()

engine/src/stirling/api/dependencies.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from fastapi import Request
+from fastapi import HTTPException, Request, status
 
 from stirling.agents import (
     ExecutionPlanningAgent,
@@ -12,7 +12,8 @@ from stirling.agents import (
 from stirling.agents.ledger import MathAuditorAgent
 from stirling.agents.pdf_comment import PdfCommentAgent
 from stirling.documents import DocumentService
-from stirling.services import AppRuntime
+from stirling.models import UserId
+from stirling.services import AppRuntime, current_user_id
 
 
 def get_runtime(request: Request) -> AppRuntime:
@@ -49,3 +50,20 @@ def get_math_auditor_agent(request: Request) -> MathAuditorAgent:
 
 def get_pdf_comment_agent(request: Request) -> PdfCommentAgent:
     return request.app.state.pdf_comment_agent
+
+
+def require_user_id() -> UserId:
+    """FastAPI dependency for routes that touch per-user storage.
+
+    Reads ``X-User-Id`` (already extracted into a ContextVar by ``UserIdMiddleware``)
+    and returns it. Returns HTTP 401 if the caller didn't supply the header. Apply
+    to any route that ingests, searches, reads, or deletes document content so
+    the tenancy gate is enforced at the API boundary.
+    """
+    user_id = current_user_id.get()
+    if user_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="X-User-Id header is required",
+        )
+    return user_id

engine/src/stirling/api/middleware.py

@@ -4,6 +4,7 @@ from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoin
 from starlette.requests import Request
 from starlette.responses import Response
 
+from stirling.models import UserId
 from stirling.services.tracking import current_user_id
 
 _USER_ID_HEADER = "X-User-Id"
@@ -15,7 +16,7 @@ class UserIdMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
         user_id = request.headers.get(_USER_ID_HEADER)
         if user_id:
-            token = current_user_id.set(user_id)
+            token = current_user_id.set(UserId(user_id))
             try:
                 return await call_next(request)
             finally:

engine/src/stirling/api/routes/documents.py

@@ -1,49 +1,83 @@
 from __future__ import annotations
 
+import logging
 from typing import Annotated
 
 from fastapi import APIRouter, Depends
 
-from stirling.api.dependencies import get_document_service
+from stirling.api.dependencies import get_document_service, require_user_id
 from stirling.contracts import (
     DeleteDocumentResponse,
     IngestDocumentRequest,
     IngestDocumentResponse,
 )
+from stirling.contracts.documents import PurgeOwnerResponse
 from stirling.documents import DocumentService
-from stirling.models import FileId
+from stirling.models import FileId, OwnerId, PrincipalId, UserId
 
 router = APIRouter(prefix="/api/v1/documents", tags=["documents"])
+logger = logging.getLogger(__name__)
 
 
 @router.post("", response_model=IngestDocumentResponse)
 async def ingest_document(
     request: IngestDocumentRequest,
     documents: Annotated[DocumentService, Depends(get_document_service)],
+    _user_id: Annotated[UserId, Depends(require_user_id)],
 ) -> IngestDocumentResponse:
-    """Replace-ingest a document's content under ``document_id``.
+    """Replace-ingest a document's content under ``(owner_id, read_principals)``.
 
-    Stores both representations in one shot:
-      * embedded chunks for RAG search,
-      * ordered page text for whole-document reading.
-    Any previously-stored content for this document is removed first.
+    ``owner_id`` and ``read_principals`` are required on the request body and
+    are stored verbatim.
+
+    ``X-User-Id`` is still required so the caller is authenticated and
+    PostHog tracks the right user, but the value is not used to constrain
+    the body.
     """
-    pages = request.page_text or []
     chunks_indexed = await documents.ingest(
         collection=request.document_id,
-        pages=pages,
+        pages=request.page_text or [],
         source=request.source,
+        owner_id=request.owner_id,
+        read_principals=request.read_principals,
+        expires_at=request.expires_at,
     )
     return IngestDocumentResponse(document_id=request.document_id, chunks_indexed=chunks_indexed)
 
 
-@router.delete("/{document_id}", response_model=DeleteDocumentResponse)
+@router.delete("/by-id/{document_id}", response_model=DeleteDocumentResponse)
 async def delete_document(
     document_id: FileId,
     documents: Annotated[DocumentService, Depends(get_document_service)],
+    user_id: Annotated[UserId, Depends(require_user_id)],
 ) -> DeleteDocumentResponse:
-    """Remove a document's content. Idempotent."""
-    existed = await documents.has_collection(document_id)
+    """Remove the caller's copy of this document.
+
+    Owner is inferred from the caller - only personal-doc deletes go through
+    here. Org-doc deletes will need an explicit owner_id once we add the
+    admin endpoints for them; for now this route can't reach docs owned by
+    a different principal.
+    """
+    owner_id = OwnerId(user_id)
+    principals = [PrincipalId(user_id)]
+    existed = await documents.has_collection(document_id, principals=principals)
     if existed:
-        await documents.delete_collection(document_id)
+        await documents.delete_collection(document_id, owner_id=owner_id)
     return DeleteDocumentResponse(document_id=document_id, deleted=existed)
+
+
+@router.delete("/by-owner", response_model=PurgeOwnerResponse)
+async def purge_caller_documents(
+    documents: Annotated[DocumentService, Depends(get_document_service)],
+    user_id: Annotated[UserId, Depends(require_user_id)],
+) -> PurgeOwnerResponse:
+    """Delete every personal-doc collection owned by the caller.
+
+    Called by Java on logout so a user's document content disappears as soon as
+    the session ends. Org-owned docs (where the caller is a reader but not
+    the owner) are not touched - only collections whose ``owner_id`` matches
+    the calling user are removed.
+    """
+    deleted = await documents.purge_owner(OwnerId(user_id))
+    logger.info("Purged %d collection(s) for owner=%s", deleted, user_id)
+    return PurgeOwnerResponse(owner_id=OwnerId(user_id), deleted=deleted)