mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-14 10:34:06 +02:00
Improve edit agent's knowledge of tools (#6356)
# Description of Changes Give Edit Agent access to descriptions of the request from the Java API. This opens the door to us better documenting our Java APIs to give the stirling engine better knowledge of what the various tools are and how to use them. Also improves the tool selection sub-agent to get the tool parameters and descriptions so it can more intelligently decide which operations should be used to fulfil the user's request. Also provides it more encouragement to string together multiple operations if necessary.
This commit is contained in:
@@ -86,8 +86,9 @@ tasks:
|
|||||||
fix:
|
fix:
|
||||||
desc: "Auto-fix lint + format"
|
desc: "Auto-fix lint + format"
|
||||||
cmds:
|
cmds:
|
||||||
|
- task: format # Can auto-fix some things that `lint:fix` can't like line length violations
|
||||||
- task: lint:fix
|
- task: lint:fix
|
||||||
- task: format
|
- task: format # Ensure that after lint fixing that the code is still formatted correctly
|
||||||
|
|
||||||
check:
|
check:
|
||||||
desc: "Full engine quality gate"
|
desc: "Full engine quality gate"
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import lombok.NoArgsConstructor;
|
|||||||
@Data
|
@Data
|
||||||
@NoArgsConstructor
|
@NoArgsConstructor
|
||||||
@EqualsAndHashCode
|
@EqualsAndHashCode
|
||||||
@Schema(description = "PDF file input - either upload a file or provide a server-side file ID")
|
|
||||||
public class PDFFile {
|
public class PDFFile {
|
||||||
|
|
||||||
@Schema(
|
@Schema(
|
||||||
|
|||||||
+1
-2
@@ -13,8 +13,7 @@ public class RotatePDFRequest extends PDFFile {
|
|||||||
|
|
||||||
@Schema(
|
@Schema(
|
||||||
description =
|
description =
|
||||||
"The clockwise angle by which to rotate the PDF file. Must be a multiple of"
|
"The clockwise angle by which to rotate all pages in the PDF file. Must be a multiple of 90.",
|
||||||
+ " 90.",
|
|
||||||
type = "integer",
|
type = "integer",
|
||||||
requiredMode = Schema.RequiredMode.REQUIRED,
|
requiredMode = Schema.RequiredMode.REQUIRED,
|
||||||
allowableValues = {"0", "90", "180", "270"})
|
allowableValues = {"0", "90", "180", "270"})
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from datamodel_code_generator.format import Formatter
|
|||||||
from referencing import Registry, Resource
|
from referencing import Registry, Resource
|
||||||
from referencing.jsonschema import DRAFT202012
|
from referencing.jsonschema import DRAFT202012
|
||||||
|
|
||||||
# Fields inherited from PDFFile base class — not tool parameters.
|
# Fields inherited from PDFFile base class - not tool parameters.
|
||||||
BASE_CLASS_FIELDS = frozenset({"fileInput", "fileId"})
|
BASE_CLASS_FIELDS = frozenset({"fileInput", "fileId"})
|
||||||
|
|
||||||
_ENGINE_ROOT = Path(__file__).resolve().parents[1]
|
_ENGINE_ROOT = Path(__file__).resolve().parents[1]
|
||||||
@@ -73,8 +73,9 @@ class ToolDiscovery:
|
|||||||
for path, path_item in sorted(self.spec.get("paths", {}).items()):
|
for path, path_item in sorted(self.spec.get("paths", {}).items()):
|
||||||
if "{" in path or not any(path.startswith(p) for p in self.ALLOWED_PATH_PREFIXES):
|
if "{" in path or not any(path.startswith(p) for p in self.ALLOWED_PATH_PREFIXES):
|
||||||
continue
|
continue
|
||||||
body_props = self._get_request_properties(path_item) or {}
|
body_schema = self._get_request_body_schema(path_item) or {}
|
||||||
query_props = self._get_query_parameters(path_item)
|
query_props = self._get_query_parameters(path_item)
|
||||||
|
body_props = body_schema.get("properties") or {}
|
||||||
# Body properties win on name collision — body is the canonical param source
|
# Body properties win on name collision — body is the canonical param source
|
||||||
# for the existing tools; query params are additive.
|
# for the existing tools; query params are additive.
|
||||||
properties = {**query_props, **body_props}
|
properties = {**query_props, **body_props}
|
||||||
@@ -87,7 +88,21 @@ class ToolDiscovery:
|
|||||||
enum_name = _deduplicate(_path_to_enum_name(path), used_enum)
|
enum_name = _deduplicate(_path_to_enum_name(path), used_enum)
|
||||||
class_name = _deduplicate(_path_to_class_name(path), used_class)
|
class_name = _deduplicate(_path_to_class_name(path), used_class)
|
||||||
|
|
||||||
defs[class_name] = {"type": "object", "properties": clean_props}
|
entry: dict[str, Any] = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": clean_props,
|
||||||
|
"description": body_schema.get("description"),
|
||||||
|
}
|
||||||
|
# Calculate which fields are actually required (many are marked as required,
|
||||||
|
# but have a default set, so they're not really required)
|
||||||
|
required = [
|
||||||
|
name
|
||||||
|
for name in body_schema.get("required") or []
|
||||||
|
if name in clean_props and "default" not in (clean_props[name] or {})
|
||||||
|
]
|
||||||
|
if required:
|
||||||
|
entry["required"] = required
|
||||||
|
defs[class_name] = entry
|
||||||
tools.append(ToolSpec(path, enum_name, class_name))
|
tools.append(ToolSpec(path, enum_name, class_name))
|
||||||
|
|
||||||
self._inline_component_refs(defs)
|
self._inline_component_refs(defs)
|
||||||
@@ -119,7 +134,7 @@ class ToolDiscovery:
|
|||||||
return self.resolver.lookup(schema["$ref"]).contents
|
return self.resolver.lookup(schema["$ref"]).contents
|
||||||
return schema
|
return schema
|
||||||
|
|
||||||
def _get_request_properties(self, path_item: dict[str, Any]) -> dict[str, Any] | None:
|
def _get_request_body_schema(self, path_item: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
post = path_item.get("post")
|
post = path_item.get("post")
|
||||||
if not post:
|
if not post:
|
||||||
return None
|
return None
|
||||||
@@ -128,7 +143,7 @@ class ToolDiscovery:
|
|||||||
if media_type in content:
|
if media_type in content:
|
||||||
schema = content[media_type].get("schema")
|
schema = content[media_type].get("schema")
|
||||||
if schema:
|
if schema:
|
||||||
return self._resolve_ref(schema).get("properties")
|
return self._resolve_ref(schema)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _get_query_parameters(self, path_item: dict[str, Any]) -> dict[str, Any]:
|
def _get_query_parameters(self, path_item: dict[str, Any]) -> dict[str, Any]:
|
||||||
@@ -227,6 +242,8 @@ def generate_models_code(combined_schema: dict[str, Any]) -> str:
|
|||||||
field_constraints=True,
|
field_constraints=True,
|
||||||
no_alias=True,
|
no_alias=True,
|
||||||
set_default_enum_member=True,
|
set_default_enum_member=True,
|
||||||
|
strict_nullable=True,
|
||||||
|
use_schema_description=True,
|
||||||
additional_imports=["enum.StrEnum"],
|
additional_imports=["enum.StrEnum"],
|
||||||
enable_version_header=False,
|
enable_version_header=False,
|
||||||
custom_file_header=_FILE_HEADER,
|
custom_file_header=_FILE_HEADER,
|
||||||
|
|||||||
@@ -34,9 +34,9 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
class PdfEditPlanSelection(ApiModel):
|
class PdfEditPlanSelection(ApiModel):
|
||||||
outcome: Literal["plan"] = "plan"
|
outcome: Literal["plan"] = "plan"
|
||||||
|
rationale: str
|
||||||
operations: list[ToolEndpoint] = Field(min_length=1)
|
operations: list[ToolEndpoint] = Field(min_length=1)
|
||||||
summary: str
|
summary: str
|
||||||
rationale: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
type PdfEditPlanOutput = PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | NeedContentResponse
|
type PdfEditPlanOutput = PdfEditPlanSelection | EditClarificationRequest | EditCannotDoResponse | NeedContentResponse
|
||||||
@@ -254,10 +254,16 @@ class PdfEditAgent:
|
|||||||
"Plan PDF edit requests. "
|
"Plan PDF edit requests. "
|
||||||
f"Supported operations are: {self._get_operations_prompt(supported_operations)}."
|
f"Supported operations are: {self._get_operations_prompt(supported_operations)}."
|
||||||
f"{unavailable_clause} "
|
f"{unavailable_clause} "
|
||||||
|
"Each operation in the user-facing prompt is listed with its full set of parameters. "
|
||||||
|
"Treat that list as authoritative: an operation can ONLY do what its listed parameters "
|
||||||
|
"and description allow. "
|
||||||
"Return an ordered list of one or more supported operations for the plan. "
|
"Return an ordered list of one or more supported operations for the plan. "
|
||||||
|
"Chain multiple operations together whenever the request needs effects that no single "
|
||||||
|
"supported operation provides on its own (for example, splitting then rotating then "
|
||||||
|
"merging, or extracting pages then re-inserting them). "
|
||||||
|
"Only return cannot_do when no sequence of the supported operations could achieve the request. "
|
||||||
"Do not produce operation parameters in this stage. "
|
"Do not produce operation parameters in this stage. "
|
||||||
"Return need_clarification when the request is genuinely ambiguous. "
|
"Return need_clarification when the request is genuinely ambiguous. "
|
||||||
"Return cannot_do when the request is outside the supported operations. "
|
|
||||||
"Return plan when a reasonable multi-step plan can be created. "
|
"Return plan when a reasonable multi-step plan can be created. "
|
||||||
"Never return partial plans."
|
"Never return partial plans."
|
||||||
),
|
),
|
||||||
@@ -280,7 +286,7 @@ class PdfEditAgent:
|
|||||||
f"Conversation history:\n{format_conversation_history(request.conversation_history)}\n"
|
f"Conversation history:\n{format_conversation_history(request.conversation_history)}\n"
|
||||||
f"User request: {request.user_message}\n"
|
f"User request: {request.user_message}\n"
|
||||||
f"Files: {format_file_names(request.files)}\n"
|
f"Files: {format_file_names(request.files)}\n"
|
||||||
f"Supported operations: {self._get_operations_prompt(supported_operations)}\n"
|
f"Supported operations:\n{self._get_supported_operations_prompt(supported_operations)}\n"
|
||||||
f"{unavailable_line}"
|
f"{unavailable_line}"
|
||||||
f"Extracted page text:\n{format_page_text(request.page_text)}"
|
f"Extracted page text:\n{format_page_text(request.page_text)}"
|
||||||
)
|
)
|
||||||
@@ -297,6 +303,29 @@ class PdfEditAgent:
|
|||||||
def _get_operations_prompt(operations: Iterable[ToolEndpoint]) -> str:
|
def _get_operations_prompt(operations: Iterable[ToolEndpoint]) -> str:
|
||||||
return ", ".join(f"{op.name} ({op.value})" for op in operations)
|
return ", ".join(f"{op.name} ({op.value})" for op in operations)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_supported_operations_prompt(operations: Iterable[ToolEndpoint]) -> str:
|
||||||
|
"""Render each operation with its description and a flat list of param descriptions.
|
||||||
|
|
||||||
|
The selection step decides which tool fits the user's request, so it just needs
|
||||||
|
what each tool does and what knobs it has, not the full schema.
|
||||||
|
"""
|
||||||
|
lines: list[str] = []
|
||||||
|
for op in operations:
|
||||||
|
schema = OPERATIONS[op].model_json_schema()
|
||||||
|
head = f"- {op.name} ({op.value})"
|
||||||
|
description = (schema.get("description") or "").strip()
|
||||||
|
if description:
|
||||||
|
head += f": {description}"
|
||||||
|
lines.append(head)
|
||||||
|
for name, prop in (schema.get("properties") or {}).items():
|
||||||
|
param_description = (prop.get("description") or "").strip()
|
||||||
|
if param_description:
|
||||||
|
lines.append(f" {name}: {param_description}")
|
||||||
|
else:
|
||||||
|
lines.append(f" {name}")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
def _fill_need_content_defaults(
|
def _fill_need_content_defaults(
|
||||||
self,
|
self,
|
||||||
selection: NeedContentResponse,
|
selection: NeedContentResponse,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -136,6 +136,7 @@ async def test_pdf_edit_agent_passes_previous_steps_to_parameter_selector(runtim
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.ROTATE_PDF, ToolEndpoint.FLATTEN],
|
operations=[ToolEndpoint.ROTATE_PDF, ToolEndpoint.FLATTEN],
|
||||||
summary="Rotate the PDF, then compress it.",
|
summary="Rotate the PDF, then compress it.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
@@ -294,6 +295,7 @@ async def test_pdf_edit_agent_passes_page_text_to_parameter_selector(runtime: Ap
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.ROTATE_PDF],
|
operations=[ToolEndpoint.ROTATE_PDF],
|
||||||
summary="Rotate the PDF.",
|
summary="Rotate the PDF.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
@@ -390,6 +392,7 @@ async def test_pdf_edit_agent_rejects_plan_referencing_unavailable_operations(
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.COMPRESS_PDF],
|
operations=[ToolEndpoint.COMPRESS_PDF],
|
||||||
summary="Compress.",
|
summary="Compress.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
@@ -420,6 +423,7 @@ async def test_pdf_edit_agent_supports_literal_find_replace(runtime: AppRuntime)
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.EDIT_TEXT],
|
operations=[ToolEndpoint.EDIT_TEXT],
|
||||||
summary="Replace 2025 with 2026 throughout the document.",
|
summary="Replace 2025 with 2026 throughout the document.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
@@ -466,6 +470,7 @@ async def test_pdf_edit_agent_supports_copy_edit_using_page_text(runtime: AppRun
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.EDIT_TEXT],
|
operations=[ToolEndpoint.EDIT_TEXT],
|
||||||
summary="Fix typos on page 3.",
|
summary="Fix typos on page 3.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
@@ -515,6 +520,7 @@ async def test_pdf_edit_agent_supports_natural_language_directed_edit(runtime: A
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.EDIT_TEXT],
|
operations=[ToolEndpoint.EDIT_TEXT],
|
||||||
summary="Update the company name to Acme Corp.",
|
summary="Update the company name to Acme Corp.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
@@ -551,6 +557,7 @@ async def test_pdf_edit_agent_composes_edit_text_with_other_operations(runtime:
|
|||||||
PdfEditPlanSelection(
|
PdfEditPlanSelection(
|
||||||
operations=[ToolEndpoint.EDIT_TEXT, ToolEndpoint.ROTATE_PDF],
|
operations=[ToolEndpoint.EDIT_TEXT, ToolEndpoint.ROTATE_PDF],
|
||||||
summary="Remove DRAFT marker, then rotate.",
|
summary="Remove DRAFT marker, then rotate.",
|
||||||
|
rationale="test rationale",
|
||||||
),
|
),
|
||||||
parameter_selector=parameter_selector,
|
parameter_selector=parameter_selector,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user