Add ability for Stirling engine to reason across large documents (#6314)

# Description of Changes
Adds storage in the database for full document content alongside the RAG
content (and changes the service to `DocumentService` instead of
`RagService`). Then adds a generic capability that should be usable by
any agent (currently just used by the Question Agent) which allows the
agent to pull out the full contents of the doc, chunks it into various
sections that will fit in the context window, and then processes them in
parallel to create an intermediate result, and then processes the
intermediate result into a final answer. It will re-chunk as many times
as necessary to get the content small enough for the actual answer to be
analysed (I've tested on PDFs ~3500 pages long, which is well above the
context limit and requires maybe 3 rounds of compression to get an
answer).

The new full doc analysis stuff is heavier than the RAG lookup so both
remain. The agents should use RAG for targeted info and the chunked
reasoner for info that requires reading the full doc.

engine/src/stirling/config/settings.py

@@ -38,18 +38,38 @@ class AppSettings(BaseSettings):
     rag_default_top_k: int = Field(validation_alias="STIRLING_RAG_TOP_K")
     rag_max_searches: int = Field(validation_alias="STIRLING_RAG_MAX_SEARCHES")
 
+    # Chunked reasoner settings (whole-document map-reduce).
+    chunked_reasoner_chars_per_slice: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_CHARS_PER_SLICE")
+    chunked_reasoner_concurrency: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_CONCURRENCY")
+    chunked_reasoner_worker_timeout_seconds: float = Field(
+        validation_alias="STIRLING_CHUNKED_REASONER_WORKER_TIMEOUT_SECONDS"
+    )
+    # Maximum size, in characters, of the rendered notes block before the
+    # reasoner folds slice notes hierarchically. The Anthropic context limit
+    # is 200k tokens (~880k chars); we leave a generous margin for the
+    # downstream agent's system prompt, history, tool definitions, and
+    # response budget.
+    chunked_reasoner_notes_char_budget: int = Field(validation_alias="STIRLING_CHUNKED_REASONER_NOTES_CHAR_BUDGET")
+
     max_pages: int = Field(validation_alias="STIRLING_MAX_PAGES")
     max_characters: int = Field(validation_alias="STIRLING_MAX_CHARACTERS")
 
     log_level: str = Field(default="INFO", validation_alias="STIRLING_LOG_LEVEL")
     log_file: str = Field(default="", validation_alias="STIRLING_LOG_FILE")
+    # When true, raises httpx + httpcore logger levels so every outgoing
+    # model SDK call is logged with timing. Use to diagnose worker stalls:
+    # a hung request shows the "Request: POST ..." line with no matching
+    # response line, confirming the hang is transport-layer (not in our
+    # code or the Anthropic SDK itself). Off by default — DEBUG-level
+    # output is high-volume.
+    http_debug: bool = Field(default=False, validation_alias="STIRLING_HTTP_DEBUG")
 
     posthog_enabled: bool = Field(validation_alias="STIRLING_POSTHOG_ENABLED")
     posthog_api_key: str = Field(validation_alias="STIRLING_POSTHOG_API_KEY")
     posthog_host: str = Field(validation_alias="STIRLING_POSTHOG_HOST")
 
 
-def _configure_logging(level_name: str, log_file: str) -> None:
+def _configure_logging(level_name: str, log_file: str, http_debug: bool) -> None:
     """Configure the ``stirling`` logger hierarchy."""
     level = logging.getLevelNamesMapping().get(level_name.upper())
     if level is None:
@@ -83,11 +103,43 @@ def _configure_logging(level_name: str, log_file: str) -> None:
         fh.setLevel(level)
         root.addHandler(fh)
 
+    if http_debug:
+        _enable_http_debug(formatter)
+
+
+def _enable_http_debug(formatter: logging.Formatter) -> None:
+    """Surface every httpx/httpcore call against the Anthropic API.
+
+    httpx emits one INFO line per request with the URL and final status,
+    which is the most useful signal for diagnosing hung worker calls: a
+    successful call shows "Request" then "Response" within a second or two;
+    a hung one shows "Request" with no matching response until it's
+    cancelled. httpcore at DEBUG drills down to TCP / HTTP/2 stream events
+    if the user wants to see exactly where bytes stop flowing.
+
+    The ``stirling`` console handler is scoped to its own logger tree, so
+    we attach a dedicated stream handler here. Without it, httpx records
+    propagate to the root logger which has no handler in our setup and the
+    output is silently dropped.
+    """
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    handler.setLevel(logging.DEBUG)
+
+    for name, level in (("httpx", logging.INFO), ("httpcore", logging.DEBUG)):
+        lg = logging.getLogger(name)
+        lg.setLevel(level)
+        # Idempotent: avoid stacking handlers on settings reload.
+        if not any(getattr(h, "_stirling_http_debug", False) for h in lg.handlers):
+            handler._stirling_http_debug = True  # type: ignore[attr-defined]
+            lg.addHandler(handler)
+        lg.propagate = False
+
 
 @lru_cache(maxsize=1)
 def load_settings() -> AppSettings:
     load_dotenv(ENV_FILE)
     load_dotenv(ENV_LOCAL_FILE, override=True)
     settings = AppSettings.model_validate({})
-    _configure_logging(settings.log_level, settings.log_file)
+    _configure_logging(settings.log_level, settings.log_file, settings.http_debug)
     return settings