Change AI engine to execute tools in Java instead of on frontend (#6116)

# Description of Changes
Redesign AI engine so that it autogenerates the `tool_models.py` file
from the OpenAPI spec so the Python has access to the Java API
parameters and the full list of Java tools that it can run. CI ensures
that whenever someone modifies a tool endpoint that the AI enigne tool
models get updated as well (the dev gets told to run `task
engine:tool-models`).

There's loads of advantages to having the Java be the one that actually
executes the tools, rather than the frontend as it was previously set up
to theoretically use:
- The AI gets much better descriptions of the params from the API docs
- It'll be usable headless in the future so a Java daemon could run to
execute ops on files in a folder without the need for the UI to run
- The Java already has all the logic it needs to execute the tools 
- We don't need to parse the TypeScript to find the API (which is hard
because the TS wasn't designed to be computer-read to extract the API)

I've also hooked up the prototype frontend to ensure it's working
properly, and have built it in a way that all the tool names can be
translated properly, which was always an issue with previous prototypes
of this.

---------

Co-authored-by: Anthony Stirling <[email protected]>
Co-authored-by: EthanHealy01 <[email protected]>

engine/src/stirling/agents/orchestrator.py

@@ -74,7 +74,7 @@ class OrchestratorAgent:
             system_prompt=(
                 "You are the top-level orchestrator. "
                 "Choose exactly one output function that best handles the request. "
-                "Use delegate_pdf_edit for requested PDF modifications. "
+                "Use delegate_pdf_edit for requested modifications of single or multiple PDFs. "
                 "Use delegate_pdf_question for questions about PDF contents. "
                 "Use delegate_user_spec for requests to create or define an agent spec. "
                 "Use math_auditor_agent for requests to check arithmetic, validate "
@@ -116,7 +116,9 @@ class OrchestratorAgent:
         return await self._run_pdf_edit(ctx.deps.request)
 
     async def _run_pdf_edit(self, request: OrchestratorRequest) -> PdfEditResponse:
-        return await PdfEditAgent(self.runtime).handle(PdfEditRequest(user_message=request.user_message))
+        return await PdfEditAgent(self.runtime).handle(
+            PdfEditRequest(user_message=request.user_message, file_names=request.file_names)
+        )
 
     async def delegate_pdf_question(self, ctx: RunContext[OrchestratorDeps]) -> PdfQuestionResponse:
         return await self._run_pdf_question(ctx.deps.request)

engine/src/stirling/agents/pdf_edit.py

@@ -14,13 +14,13 @@ from stirling.contracts import (
     PdfEditResponse,
     ToolOperationStep,
 )
-from stirling.models import OPERATIONS, ApiModel, OperationId, ParamToolModel
+from stirling.models import OPERATIONS, ApiModel, ParamToolModel, ToolEndpoint
 from stirling.services import AppRuntime
 
 
 class PdfEditPlanSelection(ApiModel):
     outcome: Literal["plan"] = "plan"
-    operations: list[OperationId] = Field(min_length=1)
+    operations: list[ToolEndpoint] = Field(min_length=1)
     summary: str
     rationale: str | None = None
 
@@ -41,7 +41,7 @@ class PdfEditParameterSelector:
     async def select(
         self,
         request: PdfEditRequest,
-        operation_plan: list[OperationId],
+        operation_plan: list[ToolEndpoint],
         operation_index: int,
         generated_steps: list[ToolOperationStep],
     ) -> ParamToolModel:
@@ -51,7 +51,7 @@ class PdfEditParameterSelector:
             self._build_parameter_prompt(request, operation_plan, operation_index, generated_steps),
             output_type=NativeOutput(parameter_model),
             instructions=(
-                f"Generate only the parameters for the PDF operation `{operation_id.value}`. "
+                f"Generate only the parameters for the PDF operation `{operation_id.name}`. "
                 "Do not include fields from any other operation."
             ),
         )
@@ -60,12 +60,12 @@ class PdfEditParameterSelector:
     def _build_parameter_prompt(
         self,
         request: PdfEditRequest,
-        operation_plan: list[OperationId],
+        operation_plan: list[ToolEndpoint],
         operation_index: int,
         generated_steps: list[ToolOperationStep],
     ) -> str:
         operation_id = operation_plan[operation_index]
-        operation_list = ", ".join(operation.value for operation in operation_plan)
+        operation_list = ", ".join(operation.name for operation in operation_plan)
         file_names = ", ".join(request.file_names) if request.file_names else "No file names were provided."
         generated_steps_text = (
             "\n".join(
@@ -79,7 +79,7 @@ class PdfEditParameterSelector:
             f"Files: {file_names}\n"
             f"Operation plan: {operation_list}\n"
             f"Selected operation index: {operation_index + 1} of {len(operation_plan)}\n"
-            f"Selected operation: {operation_id.value}\n"
+            f"Selected operation: {operation_id.name}\n"
             f"Already generated steps:\n{generated_steps_text}\n"
             "Return only the parameter object for the selected operation."
         )
@@ -153,4 +153,4 @@ class PdfEditAgent:
         )
 
     def _supported_operations_prompt(self) -> str:
-        return ", ".join(operation_id.value for operation_id in self.supported_operations)
+        return ", ".join(f"{op.name} ({op.value})" for op in self.supported_operations)

engine/src/stirling/contracts/agent_specs.py

@@ -4,7 +4,7 @@ from typing import Annotated, Literal
 
 from pydantic import Field
 
-from stirling.models import ApiModel, OperationId
+from stirling.models import ApiModel, ToolEndpoint
 
 from .common import StepKind, ToolOperationStep
 
@@ -13,7 +13,7 @@ class AiToolAgentStep(ApiModel):
     kind: Literal[StepKind.AI_TOOL] = StepKind.AI_TOOL
     title: str
     description: str
-    tool: OperationId
+    tool: ToolEndpoint
     instruction: str
 
 

engine/src/stirling/contracts/common.py

@@ -5,7 +5,7 @@ from typing import Literal, assert_never
 
 from pydantic import Field, model_validator
 
-from stirling.models import OPERATIONS, ApiModel, OperationId
+from stirling.models import OPERATIONS, ApiModel, ToolEndpoint
 from stirling.models.agent_tool_models import AGENT_OPERATIONS, AgentToolId, AnyParamModel, AnyToolId
 
 
@@ -110,7 +110,7 @@ class ToolOperationStep(ApiModel):
     def validate_tool_parameter_pairing(self) -> ToolOperationStep:
         if isinstance(self.tool, AgentToolId):
             expected_type = AGENT_OPERATIONS[self.tool]
-        elif isinstance(self.tool, OperationId):
+        elif isinstance(self.tool, ToolEndpoint):
             expected_type = OPERATIONS[self.tool]
         else:
             assert_never(self.tool)

engine/src/stirling/contracts/execution.py

@@ -4,7 +4,7 @@ from typing import Annotated, Any, Literal
 
 from pydantic import Field
 
-from stirling.models import ApiModel, OperationId, ParamToolModel
+from stirling.models import ApiModel, ParamToolModel, ToolEndpoint
 
 from .agent_specs import AgentSpec
 from .common import WorkflowOutcome
@@ -12,7 +12,7 @@ from .common import WorkflowOutcome
 
 class ExecutionStepResult(ApiModel):
     step_index: int
-    tool: OperationId | None = None
+    tool: ToolEndpoint | None = None
     success: bool
     output_summary: str | None = None
     output_data: dict[str, Any] = Field(default_factory=dict)
@@ -33,7 +33,7 @@ class AgentExecutionRequest(ApiModel):
 
 class ToolCallExecutionAction(ApiModel):
     outcome: Literal[WorkflowOutcome.TOOL_CALL] = WorkflowOutcome.TOOL_CALL
-    tool: OperationId
+    tool: ToolEndpoint
     parameters: ParamToolModel
     rationale: str | None = None
 

engine/src/stirling/models/__init__.py

@@ -1,11 +1,11 @@
 from . import tool_models
 from .base import ApiModel
-from .tool_models import OPERATIONS, OperationId, ParamToolModel
+from .tool_models import OPERATIONS, ParamToolModel, ToolEndpoint
 
 __all__ = [
     "ApiModel",
     "OPERATIONS",
-    "OperationId",
     "ParamToolModel",
+    "ToolEndpoint",
     "tool_models",
 ]

engine/src/stirling/models/agent_tool_models.py

@@ -1,6 +1,6 @@
 """Agent tool IDs, parameter models, and registry.
 
-tool_models.py is auto-generated from the frontend. This file is its
+tool_models.py is auto-generated from the Java OpenAPI spec. This file is its
 manually-maintained counterpart for tools backed by AI agent pipelines.
 """
 
@@ -9,7 +9,7 @@ from __future__ import annotations
 from enum import StrEnum
 
 from stirling.models.base import ApiModel
-from stirling.models.tool_models import OperationId, ParamToolModel
+from stirling.models.tool_models import ParamToolModel, ToolEndpoint
 
 
 class AgentToolId(StrEnum):
@@ -22,7 +22,7 @@ class MathAuditorAgentParams(ApiModel):
 
 type AgentParamModel = MathAuditorAgentParams
 
-type AnyToolId = OperationId | AgentToolId
+type AnyToolId = ToolEndpoint | AgentToolId
 type AnyParamModel = ParamToolModel | AgentParamModel
 
 AGENT_OPERATIONS: dict[AgentToolId, type[AgentParamModel]] = {