Improvements to Stirling Engine to prepare for SaaS release (#6603)

# Description of Changes
- Use pool for postgres connections
- Add ability to require user ID to be set on API calls to the engine
- Add process-wide concurrency cap on AI access (in addition to existing
user caps)
- Allow number of workers (threads) to be specified for stirling engine
- Update env var names to reflect that the DB is not just for RAG

engine/src/stirling/api/app.py

@@ -18,6 +18,7 @@ from stirling.agents import (
 )
 from stirling.agents.ledger import MathAuditorAgent
 from stirling.agents.pdf_comment import PdfCommentAgent
+from stirling.api.dependencies import enforce_required_user_id
 from stirling.api.engine_auth import EngineSharedSecretMiddleware
 from stirling.api.middleware import UserIdMiddleware
 from stirling.api.routes import (
@@ -118,15 +119,18 @@ async def lifespan(fast_api: FastAPI):
 app = FastAPI(title="Stirling AI Engine", lifespan=lifespan, version="0.1.0")
 app.add_middleware(UserIdMiddleware)
 app.add_middleware(EngineSharedSecretMiddleware)
-app.include_router(orchestrator_router)
-app.include_router(pdf_edit_router)
-app.include_router(pdf_question_router)
-app.include_router(agent_draft_router)
-app.include_router(execution_router)
-app.include_router(document_router)
-app.include_router(ledger_router)
-app.include_router(pdf_comments_router)
-app.include_router(agent_capabilities_router)
+# Every router gets the same configurable identity gate; /health stays open
+# for liveness probes. See enforce_required_user_id for the policy.
+_user_gate = [Depends(enforce_required_user_id)]
+app.include_router(orchestrator_router, dependencies=_user_gate)
+app.include_router(pdf_edit_router, dependencies=_user_gate)
+app.include_router(pdf_question_router, dependencies=_user_gate)
+app.include_router(agent_draft_router, dependencies=_user_gate)
+app.include_router(execution_router, dependencies=_user_gate)
+app.include_router(document_router, dependencies=_user_gate)
+app.include_router(ledger_router, dependencies=_user_gate)
+app.include_router(pdf_comments_router, dependencies=_user_gate)
+app.include_router(agent_capabilities_router, dependencies=_user_gate)
 
 
 @app.get("/health", response_model=HealthResponse)

engine/src/stirling/api/dependencies.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-from fastapi import HTTPException, Request, status
+from typing import Annotated
+
+from fastapi import Depends, HTTPException, Request, status
 
 from stirling.agents import (
     ExecutionPlanningAgent,
@@ -11,6 +13,7 @@ from stirling.agents import (
 )
 from stirling.agents.ledger import MathAuditorAgent
 from stirling.agents.pdf_comment import PdfCommentAgent
+from stirling.config import AppSettings, load_settings
 from stirling.documents import DocumentService
 from stirling.models import UserId
 from stirling.services import AppRuntime, current_user_id
@@ -67,3 +70,16 @@ def require_user_id() -> UserId:
             detail="X-User-Id header is required",
         )
     return user_id
+
+
+def enforce_required_user_id(
+    settings: Annotated[AppSettings, Depends(load_settings)],
+) -> None:
+    """Router-level boundary gate, applied uniformly to every router."""
+    if not settings.require_user_id:
+        return
+    if current_user_id.get() is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="X-User-Id header is required",
+        )