Set up document management for Stirling Engine (#6476)

# Description of Changes
Change Stirling Engine to support deleting documents automatically. This
happens both on user logout and after an amount of time specified by the
Java when ingesting a document (allowing for personal documents to have
short lifetimes but org documents to be left in the db with no expiry
date). Also sets up an [ACL
policy](https://en.wikipedia.org/wiki/Access-control_list) for the
documents so the database knows which users have access to which
documents. This is not fully implemented in the Java, so currently all
docs are treated as having a single owner, the uploader, but
theoretically when we need to support org storage, we shouldn't need to
change the db schema.

engine/src/stirling/agents/contradiction/capability.py

@@ -21,6 +21,7 @@ from pydantic_ai.toolsets import AbstractToolset
 from stirling.agents.contradiction.detector import ContradictionDetector
 from stirling.contracts import AiFile
 from stirling.contracts.contradiction import Claim, ContradictionReport
+from stirling.models import PrincipalId
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +53,7 @@ class ContradictionCapability:
         self,
         detector: ContradictionDetector,
         files: list[AiFile],
+        principals: list[PrincipalId],
         *,
         max_audits: int = DEFAULT_MAX_AUDITS,
     ) -> None:
@@ -59,6 +61,7 @@ class ContradictionCapability:
             raise ValueError("max_audits must be >= 1")
         self._detector = detector
         self._files = files
+        self._principals = principals
         self._max_audits = max_audits
         self._audit_count = 0
         toolset: FunctionToolset[None] = FunctionToolset()
@@ -121,7 +124,7 @@ class ContradictionCapability:
         if not self._files:
             return "No documents attached to audit."
 
-        report = await self._detector.detect(self._files, query=query)
+        report = await self._detector.detect(self._files, principals=self._principals, query=query)
         formatted = self.format_report(report)
         logger.info(
             "[contradiction-capability] audit query=%r files=%d -> %d findings, %d chars",

engine/src/stirling/agents/contradiction/detector.py

@@ -41,6 +41,7 @@ from stirling.contracts.contradiction import (
     ContradictionSeverity,
 )
 from stirling.contracts.documents import Page
+from stirling.models import PrincipalId
 from stirling.services import AppRuntime
 
 logger = logging.getLogger(__name__)
@@ -205,8 +206,13 @@ class ContradictionDetector:
     # Public entry point
     # ------------------------------------------------------------------
 
-    async def detect(self, files: list[AiFile], query: str | None = None) -> ContradictionReport:
-        """Run the full pipeline over the supplied files.
+    async def detect(
+        self,
+        files: list[AiFile],
+        principals: list[PrincipalId],
+        query: str | None = None,
+    ) -> ContradictionReport:
+        """Run the full pipeline over the supplied files for ``principals``.
 
         ``files`` must have already been ingested (the caller is
         responsible for the ``has_collection`` precheck — the question
@@ -234,7 +240,7 @@ class ContradictionDetector:
         effective_query = query or "extract claims"
 
         per_file_results = await asyncio.gather(
-            *(self._extract_claims_for_file(file, effective_query) for file in files),
+            *(self._extract_claims_for_file(file, effective_query, principals) for file in files),
             return_exceptions=True,
         )
 
@@ -312,6 +318,7 @@ class ContradictionDetector:
         self,
         file: AiFile,
         query: str,
+        principals: list[PrincipalId],
     ) -> _FileExtractionResult:
         """Run the per-chunk extractor over one file's pages.
 
@@ -324,7 +331,7 @@ class ContradictionDetector:
         ``asyncio.gather`` and the mapper's internal semaphore — this
         helper itself awaits each step sequentially within one file.
         """
-        file_pages = await self._runtime.documents.read_pages(file.id)
+        file_pages = await self._runtime.documents.read_pages(file.id, principals=principals)
         if not file_pages:
             logger.info(
                 "[contradiction] no stored pages for %s (id=%s); skipping",

engine/src/stirling/agents/pdf_questions.py

@@ -27,8 +27,9 @@ from stirling.contracts import (
     format_file_names,
 )
 from stirling.documents import RagCapability
+from stirling.models import PrincipalId
 from stirling.models.agent_tool_models import AgentToolId, MathAuditorAgentParams
-from stirling.services import AppRuntime
+from stirling.services import AppRuntime, require_current_user_id
 
 logger = logging.getLogger(__name__)
 
@@ -161,9 +162,10 @@ class PdfQuestionAgent:
         )
 
     async def _find_missing_files(self, files: list[AiFile]) -> list[AiFile]:
+        principals = [PrincipalId(require_current_user_id())]
         missing: list[AiFile] = []
         for file in files:
-            if not await self.runtime.documents.has_collection(file.id):
+            if not await self.runtime.documents.has_collection(file.id, principals=principals):
                 missing.append(file)
         return missing
 
@@ -175,8 +177,10 @@ class PdfQuestionAgent:
         upstream classifier keeps that judgement in the same call that writes
         the answer, and lets the agent mix tools when the question warrants it.
         """
+        principals = [PrincipalId(require_current_user_id())]
         rag = RagCapability(
             documents=self.runtime.documents,
+            principals=principals,
             collections=[file.id for file in request.files],
             top_k=self.runtime.settings.rag_default_top_k,
             max_searches=self.runtime.settings.rag_max_searches,
@@ -184,11 +188,13 @@ class PdfQuestionAgent:
         whole_doc = WholeDocReaderCapability(
             runtime=self.runtime,
             files=request.files,
+            principals=principals,
             reasoner=self._chunked_reasoner,
         )
         contradiction = ContradictionCapability(
             detector=self._contradiction_detector,
             files=request.files,
+            principals=principals,
         )
         agent = Agent(
             model=self.runtime.smart_model,

engine/src/stirling/agents/pdf_review.py

@@ -49,14 +49,14 @@ from stirling.contracts import (
     Verdict,
 )
 from stirling.contracts.ledger import Discrepancy
-from stirling.models import ApiModel, ToolEndpoint
+from stirling.models import ApiModel, PrincipalId, ToolEndpoint
 from stirling.models.agent_tool_models import (
     AgentToolId,
     MathAuditorAgentParams,
     PdfCommentAgentParams,
 )
 from stirling.models.tool_models import AddCommentsParams
-from stirling.services import AppRuntime
+from stirling.services import AppRuntime, require_current_user_id
 
 # Fallback right-margin placement used when a finding has no usable
 # anchor text. A4/Letter portrait assumed.
@@ -158,7 +158,11 @@ class PdfReviewAgent:
                     files_to_ingest=missing,
                     content_types=[PdfContentType.PAGE_TEXT],
                 )
-            report = await self._contradiction_detector.detect(request.files, query=request.user_message)
+            report = await self._contradiction_detector.detect(
+                request.files,
+                principals=[PrincipalId(require_current_user_id())],
+                query=request.user_message,
+            )
             comments_json = await self._build_contradiction_comments_payload(request.user_message, report)
             return EditPlanResponse(
                 summary="",
@@ -193,9 +197,10 @@ class PdfReviewAgent:
         )
 
     async def _find_missing_files(self, files: list[AiFile]) -> list[AiFile]:
+        principals = [PrincipalId(require_current_user_id())]
         missing: list[AiFile] = []
         for file in files:
-            if not await self.runtime.documents.has_collection(file.id):
+            if not await self.runtime.documents.has_collection(file.id, principals=principals):
                 missing.append(file)
         return missing
 

engine/src/stirling/agents/shared/whole_doc_reader.py

@@ -19,6 +19,7 @@ from pydantic_ai.toolsets import AbstractToolset
 
 from stirling.agents.shared.chunked_reasoner import ChunkedReasoner
 from stirling.contracts import AiFile
+from stirling.models import PrincipalId
 from stirling.services import AppRuntime
 
 logger = logging.getLogger(__name__)
@@ -47,12 +48,14 @@ class WholeDocReaderCapability:
         self,
         runtime: AppRuntime,
         files: list[AiFile],
+        principals: list[PrincipalId],
         *,
         reasoner: ChunkedReasoner | None = None,
         max_reads: int = DEFAULT_MAX_READS,
     ) -> None:
         self._runtime = runtime
         self._files = files
+        self._principals = principals
         self._reasoner = reasoner if reasoner is not None else ChunkedReasoner(runtime)
         self._max_reads = max_reads
         self._read_count = 0
@@ -115,7 +118,7 @@ class WholeDocReaderCapability:
 
         sections: list[str] = []
         for file in self._files:
-            pages = await self._runtime.documents.read_pages(file.id)
+            pages = await self._runtime.documents.read_pages(file.id, principals=self._principals)
             if not pages:
                 logger.info(
                     "[whole-doc-reader] no stored pages for %s (id=%s); skipping",