Add ability for Stirling engine to reason across large documents (#6314)

# Description of Changes
Adds storage in the database for full document content alongside the RAG
content (and changes the service to `DocumentService` instead of
`RagService`). Then adds a generic capability that should be usable by
any agent (currently just used by the Question Agent) which allows the
agent to pull out the full contents of the doc, chunks it into various
sections that will fit in the context window, and then processes them in
parallel to create an intermediate result, and then processes the
intermediate result into a final answer. It will re-chunk as many times
as necessary to get the content small enough for the actual answer to be
analysed (I've tested on PDFs ~3500 pages long, which is well above the
context limit and requires maybe 3 rounds of compression to get an
answer).

The new full doc analysis stuff is heavier than the RAG lookup so both
remain. The agents should use RAG for targeted info and the chunked
reasoner for info that requires reading the full doc.

frontend/public/locales/en-GB/translation.toml

@@ -2682,6 +2682,10 @@ executing_tool_step = "Running {{tool}} (step {{step}} of {{total}})..."
 extracting_content = "Extracting content from your documents..."
 processing = "Processing extracted content..."
 thinking = "Thinking..."
+whole_doc_compression_round = "Consolidating notes..."
+whole_doc_read_done = "Finished reading the document..."
+whole_doc_read_started = "Reading the document..."
+whole_doc_slice_done = "Reading the document... ({{percent}}% complete)"
 
 [chat.toolsUsed]
 summary = "Ran {{count}} tools"

frontend/src/prototypes/components/chat/ChatContext.tsx

@@ -37,6 +37,79 @@ export enum AiWorkflowPhase {
   EXTRACTING_CONTENT = "extracting_content",
   EXECUTING_TOOL = "executing_tool",
   PROCESSING = "processing",
+  ENGINE_PROGRESS = "engine_progress",
+}
+
+/**
+ * Engine-side progress detail for ENGINE_PROGRESS events. Mirrors the Python
+ * {@code ProgressEvent} discriminated union (engine/src/stirling/contracts/progress.py)
+ * and the Java {@code AiEngineProgressDetail} sealed interface; the {@code phase}
+ * string is the discriminator. Field names are camelCase because the engine
+ * serialises by alias.
+ */
+export interface WholeDocReadStartedDetail {
+  phase: "whole_doc_read_started";
+  question: string;
+  pages: number;
+  slices: number;
+}
+
+export interface WholeDocSliceDoneDetail {
+  phase: "whole_doc_slice_done";
+  completed: number;
+  total: number;
+  /** Page-range label, e.g. "pages=1-5". */
+  pages: string;
+  durationMs: number;
+  excerpts: number;
+  facts: number;
+}
+
+export interface WholeDocCompressionRoundDetail {
+  phase: "whole_doc_compression_round";
+  roundNumber: number;
+  notesIn: number;
+  groups: number;
+}
+
+export interface WholeDocReadDoneDetail {
+  phase: "whole_doc_read_done";
+  completed: number;
+  slices: number;
+  durationSeconds: number;
+}
+
+export type EngineProgressDetail =
+  | WholeDocReadStartedDetail
+  | WholeDocSliceDoneDetail
+  | WholeDocCompressionRoundDetail
+  | WholeDocReadDoneDetail;
+
+/**
+ * What we actually carry across the wire boundary: a known typed variant, or a
+ * forward-compat shape with just the discriminator string. The "unknown" arm
+ * exists so a new engine-side phase rolling out before a frontend update keeps
+ * rendering the generic processing message instead of crashing the union.
+ */
+export interface UnknownEngineProgressDetail {
+  phase: string;
+}
+
+export type AnyEngineProgressDetail =
+  | EngineProgressDetail
+  | UnknownEngineProgressDetail;
+
+const KNOWN_ENGINE_PHASES = new Set<string>([
+  "whole_doc_read_started",
+  "whole_doc_slice_done",
+  "whole_doc_compression_round",
+  "whole_doc_read_done",
+]);
+
+export function isKnownEngineProgressDetail(
+  detail: AnyEngineProgressDetail,
+): detail is EngineProgressDetail {
+  return KNOWN_ENGINE_PHASES.has(detail.phase);
 }
 
 export interface AiWorkflowProgress {
@@ -47,6 +120,11 @@ export interface AiWorkflowProgress {
   stepIndex?: number;
   /** Total number of plan steps, for EXECUTING_TOOL events. */
   stepCount?: number;
+  /**
+   * Engine-side event payload, for ENGINE_PROGRESS events. Typed sub-phase
+   * record (e.g. {@link WholeDocSliceDoneDetail}) the UI can render in detail.
+   */
+  engineDetail?: AnyEngineProgressDetail;
 }
 
 type AiWorkflowOutcome =
@@ -159,6 +237,7 @@ interface ProgressEvent {
   tool?: string;
   stepIndex?: number;
   stepCount?: number;
+  engineDetail?: AnyEngineProgressDetail;
 }
 
 async function consumeSSEStream(
@@ -378,6 +457,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
                 tool: data.tool,
                 stepIndex: data.stepIndex,
                 stepCount: data.stepCount,
+                engineDetail: data.engineDetail,
               },
             });
           },

frontend/src/prototypes/components/chat/ChatPanel.tsx

@@ -29,7 +29,9 @@ import ExpandLessIcon from "@mui/icons-material/ExpandLess";
 import {
   useChat,
   AiWorkflowPhase,
+  isKnownEngineProgressDetail,
   type AiWorkflowProgress,
+  type AnyEngineProgressDetail,
 } from "@app/components/chat/ChatContext";
 import { useTranslatedToolCatalog } from "@app/data/useTranslatedToolRegistry";
 import "@app/components/chat/ChatPanel.css";
@@ -90,9 +92,41 @@ function formatProgress(
         })
       : t("chat.progress.executing_tool_generic");
   }
+  if (progress.phase === AiWorkflowPhase.ENGINE_PROGRESS) {
+    return formatEngineProgress(progress.engineDetail, t);
+  }
   return t(`chat.progress.${progress.phase}`);
 }
 
+/**
+ * Render an engine-side progress event (e.g. chunked-reasoner slice progress) into a user-facing
+ * message. Falls through to the generic processing label for unknown sub-phases so adding new
+ * engine events doesn't break the UI before the frontend learns about them.
+ */
+function formatEngineProgress(
+  detail: AnyEngineProgressDetail | undefined,
+  t: TranslateFn,
+): string {
+  if (!detail || !isKnownEngineProgressDetail(detail)) {
+    return t("chat.progress.processing");
+  }
+  switch (detail.phase) {
+    case "whole_doc_read_started":
+      return t("chat.progress.whole_doc_read_started");
+    case "whole_doc_slice_done": {
+      const percent =
+        detail.total > 0
+          ? Math.round((detail.completed / detail.total) * 100)
+          : 0;
+      return t("chat.progress.whole_doc_slice_done", { percent });
+    }
+    case "whole_doc_compression_round":
+      return t("chat.progress.whole_doc_compression_round");
+    case "whole_doc_read_done":
+      return t("chat.progress.whole_doc_read_done");
+  }
+}
+
 function ToolsUsedBlock({
   tools,
   resolveToolName,