arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-16 19:33:11 +02:00
Code Issues Actions 3 Packages Projects Releases 1 Wiki Activity

Change AI engine to execute tools in Java instead of on frontend (#6116)

Browse Source 
# Description of Changes
Redesign AI engine so that it autogenerates the `tool_models.py` file
from the OpenAPI spec so the Python has access to the Java API
parameters and the full list of Java tools that it can run. CI ensures
that whenever someone modifies a tool endpoint that the AI enigne tool
models get updated as well (the dev gets told to run `task
engine:tool-models`).

There's loads of advantages to having the Java be the one that actually
executes the tools, rather than the frontend as it was previously set up
to theoretically use:
- The AI gets much better descriptions of the params from the API docs
- It'll be usable headless in the future so a Java daemon could run to
execute ops on files in a folder without the need for the UI to run
- The Java already has all the logic it needs to execute the tools 
- We don't need to parse the TypeScript to find the API (which is hard
because the TS wasn't designed to be computer-read to extract the API)

I've also hooked up the prototype frontend to ensure it's working
properly, and have built it in a way that all the tool names can be
translated properly, which was always an issue with previous prototypes
of this.

---------

Co-authored-by: Anthony Stirling <[email protected]>
Co-authored-by: EthanHealy01 <[email protected]>
This commit is contained in:
James Brunton
2026-04-20 15:57:11 +01:00
committed by GitHub
co-authored by Anthony Stirling EthanHealy01
parent cc9650e7a3
commit e5767ed58b
45 changed files with 3565 additions and 1285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/prototypes/components/chat/ChatContext.tsx
+162 -15
View File
@@ -6,23 +6,49 @@ import {
useRef,
type ReactNode,
} from "react";
import { useAllFiles } from "@app/contexts/FileContext";
import { useAllFiles, useFileActions } from "@app/contexts/FileContext";
import apiClient from "@app/services/apiClient";
import { getAuthHeaders } from "@app/services/apiClientSetup";
import { createChildStub } from "@app/contexts/file/fileActions";
import {
createNewStirlingFileStub,
createStirlingFile,
type StirlingFileStub,
} from "@app/types/fileContext";
import type { ToolOperation } from "@app/types/file";
export interface ChatMessage {
id: string;
role: "user" | "assistant";
content: string;
timestamp: number;
/**
* Tool endpoint paths executed during this assistant turn (e.g.
* {@code /api/v1/general/rotate-pdf}). Populated for assistant messages when the workflow
* ran one or more tools, in execution order. Undefined for user messages and for assistant
* turns that answered without running any tool.
*/
toolsUsed?: string[];
}
export enum AiWorkflowPhase {
ANALYZING = "analyzing",
CALLING_ENGINE = "calling_engine",
EXTRACTING_CONTENT = "extracting_content",
EXECUTING_TOOL = "executing_tool",
PROCESSING = "processing",
}
export interface AiWorkflowProgress {
phase: AiWorkflowPhase;
/** Tool endpoint path currently executing, for EXECUTING_TOOL events. */
tool?: string;
/** 1-based step index, for EXECUTING_TOOL events. */
stepIndex?: number;
/** Total number of plan steps, for EXECUTING_TOOL events. */
stepCount?: number;
}
type AiWorkflowOutcome =
| "answer"
| "not_found"
@@ -35,6 +61,13 @@ type AiWorkflowOutcome =
| "unsupported_capability"
| "cannot_continue";
interface AiWorkflowResultFile {
/** Stirling file ID — download with /api/v1/general/files/{fileId}. */
fileId: string;
fileName: string;
contentType: string;
}
interface AiWorkflowResponse {
outcome: AiWorkflowOutcome;
answer?: string;
@@ -46,19 +79,25 @@ interface AiWorkflowResponse {
message?: string;
evidence?: Array<{ pageNumber: number; text: string }>;
steps?: Array<Record<string, unknown>>;
/** Every file produced by the workflow (empty if the outcome has no files). */
resultFiles?: AiWorkflowResultFile[];
// Legacy single-file fields — mirror the first entry of resultFiles.
fileId?: string;
fileName?: string;
contentType?: string;
}
interface ChatState {
messages: ChatMessage[];
isOpen: boolean;
isLoading: boolean;
progressPhase: AiWorkflowPhase | null;
progress: AiWorkflowProgress | null;
}
type ChatAction =
| { type: "ADD_MESSAGE"; message: ChatMessage }
| { type: "SET_LOADING"; loading: boolean }
| { type: "SET_PROGRESS"; phase: AiWorkflowPhase | null }
| { type: "SET_PROGRESS"; progress: AiWorkflowProgress | null }
| { type: "TOGGLE_OPEN" }
| { type: "SET_OPEN"; open: boolean };
@@ -69,7 +108,7 @@ function chatReducer(state: ChatState, action: ChatAction): ChatState {
case "SET_LOADING":
return { ...state, isLoading: action.loading };
case "SET_PROGRESS":
return { ...state, progressPhase: action.phase };
return { ...state, progress: action.progress };
case "TOGGLE_OPEN":
return { ...state, isOpen: !state.isOpen };
case "SET_OPEN":
@@ -114,10 +153,18 @@ function formatWorkflowResponse(data: AiWorkflowResponse): string {
/**
* Parses an SSE text stream and invokes callbacks for each named event.
*/
interface ProgressEvent {
phase: string;
timestamp: number;
tool?: string;
stepIndex?: number;
stepCount?: number;
}
async function consumeSSEStream(
response: Response,
handlers: {
onProgress: (data: { phase: string; timestamp: number }) => void;
onProgress: (data: ProgressEvent) => void;
onResult: (data: AiWorkflowResponse) => void;
onError: (data: { message: string }) => void;
},
@@ -171,7 +218,7 @@ interface ChatContextValue {
messages: ChatMessage[];
isOpen: boolean;
isLoading: boolean;
progressPhase: AiWorkflowPhase | null;
progress: AiWorkflowProgress | null;
toggleOpen: () => void;
setOpen: (open: boolean) => void;
sendMessage: (content: string) => Promise<void>;
@@ -183,14 +230,86 @@ const initialState: ChatState = {
messages: [],
isOpen: false,
isLoading: false,
progressPhase: null,
progress: null,
};
export function ChatProvider({ children }: { children: ReactNode }) {
const [state, dispatch] = useReducer(chatReducer, initialState);
const { files: activeFiles } = useAllFiles();
const { files: activeFiles, fileStubs: activeFileStubs } = useAllFiles();
const { actions: fileActions } = useFileActions();
const abortRef = useRef<AbortController | null>(null);
// Download a File from the Stirling files endpoint.
const downloadFile = useCallback(
async (descriptor: AiWorkflowResultFile): Promise<File> => {
const response = await apiClient.get<Blob>(
`/api/v1/general/files/${descriptor.fileId}`,
{ responseType: "blob" },
);
return new File([response.data], descriptor.fileName, {
type: descriptor.contentType ?? response.data.type,
});
},
[],
);
// Import the files produced by an AI workflow result into FileContext.
//
// If the workflow produced the same number of outputs as inputs, map each output to its
// corresponding input as a new version in the same chain. Otherwise (merge, split, etc.)
// add the outputs as new root files.
const importResultFile = useCallback(
async (
result: AiWorkflowResponse,
sourceStubs: StirlingFileStub[],
): Promise<void> => {
const descriptors = result.resultFiles?.length
? result.resultFiles
: result.fileId && result.fileName && result.contentType
? [
{
fileId: result.fileId,
fileName: result.fileName,
contentType: result.contentType,
} satisfies AiWorkflowResultFile,
]
: [];
if (descriptors.length === 0) return;
const files = await Promise.all(descriptors.map(downloadFile));
const operation: ToolOperation = {
toolId: "ai-workflow",
timestamp: Date.now(),
};
const isVersionMapping =
sourceStubs.length > 0 && files.length === sourceStubs.length;
const stubs = files.map((file, i) =>
isVersionMapping
? createChildStub(sourceStubs[i], operation, file)
: createNewStirlingFileStub(file),
);
const stirlingFiles = files.map((file, i) =>
createStirlingFile(file, stubs[i].id),
);
if (sourceStubs.length > 0) {
// Always consume the inputs so merge/split inputs are removed from the workbench.
// For 1:1 operations (rotate, compress) the outputs carry the version chain; for
// merge/split they're fresh roots.
await fileActions.consumeFiles(
sourceStubs.map((s) => s.id),
stirlingFiles,
stubs,
);
} else {
// No inputs were provided (unlikely for completed workflows, but handle it safely).
await fileActions.addFiles(files, { selectFiles: true });
}
},
[fileActions, downloadFile],
);
const toggleOpen = useCallback(() => dispatch({ type: "TOGGLE_OPEN" }), []);
const setOpen = useCallback(
(open: boolean) => dispatch({ type: "SET_OPEN", open }),
@@ -212,7 +331,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
};
dispatch({ type: "ADD_MESSAGE", message: userMessage });
dispatch({ type: "SET_LOADING", loading: true });
dispatch({ type: "SET_PROGRESS", phase: null });
dispatch({ type: "SET_PROGRESS", progress: null });
try {
const formData = new FormData();
@@ -234,17 +353,29 @@ export function ChatProvider({ children }: { children: ReactNode }) {
}
let receivedResult = false;
const toolsUsed: string[] = [];
await consumeSSEStream(response, {
onProgress: (data) => {
if (
data.phase === AiWorkflowPhase.EXECUTING_TOOL &&
typeof data.tool === "string"
) {
toolsUsed.push(data.tool);
}
dispatch({
type: "SET_PROGRESS",
phase: data.phase as AiWorkflowPhase,
progress: {
phase: data.phase as AiWorkflowPhase,
tool: data.tool,
stepIndex: data.stepIndex,
stepCount: data.stepCount,
},
});
},
onResult: (data) => {
receivedResult = true;
dispatch({ type: "SET_PROGRESS", phase: null });
dispatch({ type: "SET_PROGRESS", progress: null });
const replyContent = formatWorkflowResponse(data);
dispatch({
type: "ADD_MESSAGE",
@@ -253,12 +384,28 @@ export function ChatProvider({ children }: { children: ReactNode }) {
role: "assistant",
content: replyContent,
timestamp: Date.now(),
toolsUsed: toolsUsed.length > 0 ? toolsUsed : undefined,
},
});
if (data.fileId || data.resultFiles?.length) {
importResultFile(data, activeFileStubs).catch((err) => {
console.error("Failed to import AI result file", err);
dispatch({
type: "ADD_MESSAGE",
message: {
id: crypto.randomUUID(),
role: "assistant",
content:
"The file was processed but I couldn't download it.",
timestamp: Date.now(),
},
});
});
}
},
onError: (data) => {
receivedResult = true;
dispatch({ type: "SET_PROGRESS", phase: null });
dispatch({ type: "SET_PROGRESS", progress: null });
dispatch({
type: "ADD_MESSAGE",
message: {
@@ -276,7 +423,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
}
} catch (e) {
if ((e as Error).name === "AbortError") return;
dispatch({ type: "SET_PROGRESS", phase: null });
dispatch({ type: "SET_PROGRESS", progress: null });
dispatch({
type: "ADD_MESSAGE",
message: {
@@ -294,7 +441,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
}
}
},
[activeFiles],
[activeFiles, activeFileStubs, importResultFile],
);
return (
@@ -303,7 +450,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
messages: state.messages,
isOpen: state.isOpen,
isLoading: state.isLoading,
progressPhase: state.progressPhase,
progress: state.progress,
toggleOpen,
setOpen,
sendMessage,
frontend/src/prototypes/components/chat/ChatPanel.tsx
+146 -12
View File
@@ -1,4 +1,10 @@
import { useRef, useEffect, useState, type KeyboardEvent } from "react";
import {
useMemo,
useRef,
useEffect,
useState,
type KeyboardEvent,
} from "react";
import { useTranslation } from "react-i18next";
import {
ActionIcon,
@@ -11,19 +17,142 @@ import {
Transition,
Loader,
Group,
Collapse,
UnstyledButton,
List,
} from "@mantine/core";
import SendIcon from "@mui/icons-material/Send";
import ChatBubbleOutlineIcon from "@mui/icons-material/ChatBubbleOutline";
import CloseIcon from "@mui/icons-material/Close";
import { useChat } from "@app/components/chat/ChatContext";
import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
import ExpandLessIcon from "@mui/icons-material/ExpandLess";
import {
useChat,
AiWorkflowPhase,
type AiWorkflowProgress,
} from "@app/components/chat/ChatContext";
import { useTranslatedToolCatalog } from "@app/data/useTranslatedToolRegistry";
import "@app/components/chat/ChatPanel.css";
type TranslateFn = (key: string, options?: Record<string, unknown>) => string;
/** Resolver mapping a tool endpoint path to its translated display name. */
type ToolNameResolver = (endpoint: string) => string | null;
/**
* Look up a tool's translated name from the tool catalog. The catalog's {@code operationConfig}
* exposes the full API endpoint path for each tool, so we key the lookup on the exact path that
* arrives in SSE progress events — no string parsing.
*/
function useToolNameResolver(): ToolNameResolver {
const { allTools } = useTranslatedToolCatalog();
return useMemo(() => {
const nameByEndpoint = new Map<string, string>();
Object.values(allTools).forEach((tool) => {
const endpoint = tool.operationConfig?.endpoint;
// Only register tools with a static endpoint. Tools whose endpoint is a function
// (dynamic routing, e.g. Convert / Split) need runtime params to resolve, so they fall
// through to the generic progress message rather than mis-matching.
if (typeof endpoint === "string") {
nameByEndpoint.set(endpoint, tool.name);
}
});
return (endpoint: string) => nameByEndpoint.get(endpoint) ?? null;
}, [allTools]);
}
function formatProgress(
progress: AiWorkflowProgress,
t: TranslateFn,
resolveToolName: ToolNameResolver,
): string {
if (progress.phase === AiWorkflowPhase.EXECUTING_TOOL && progress.tool) {
const tool = resolveToolName(progress.tool);
const hasSteps =
progress.stepIndex != null &&
progress.stepCount != null &&
progress.stepCount > 1;
if (tool) {
return hasSteps
? t("chat.progress.executing_tool_step", {
tool,
step: progress.stepIndex,
total: progress.stepCount,
})
: t("chat.progress.executing_tool_single", { tool });
}
// Unknown tool — fall back to a generic translated message rather than
// prettifying the endpoint path by hand.
return hasSteps
? t("chat.progress.executing_tool_generic_step", {
step: progress.stepIndex,
total: progress.stepCount,
})
: t("chat.progress.executing_tool_generic");
}
return t(`chat.progress.${progress.phase}`);
}
function ToolsUsedBlock({
tools,
resolveToolName,
t,
}: {
tools: string[];
resolveToolName: ToolNameResolver;
t: TranslateFn;
}) {
const [expanded, setExpanded] = useState(false);
const names = tools.map(
(endpoint) => resolveToolName(endpoint) ?? t("chat.toolsUsed.unknownTool"),
);
const label = t("chat.toolsUsed.summary", { count: tools.length });
return (
<Box mt={6}>
<UnstyledButton
onClick={() => setExpanded((v) => !v)}
aria-expanded={expanded}
>
<Group gap={4} wrap="nowrap">
{expanded ? (
<ExpandLessIcon sx={{ fontSize: 14 }} />
) : (
<ExpandMoreIcon sx={{ fontSize: 14 }} />
)}
<Text size="xs" c="dimmed">
{label}
</Text>
</Group>
</UnstyledButton>
<Collapse in={expanded}>
<List
type="ordered"
size="xs"
mt={4}
pl="lg"
styles={{ itemWrapper: { lineHeight: 1.4 } }}
>
{names.map((name, i) => (
<List.Item key={i}>{name}</List.Item>
))}
</List>
</Collapse>
</Box>
);
}
function ChatMessageBubble({
role,
content,
toolsUsed,
resolveToolName,
t,
}: {
role: "user" | "assistant";
content: string;
toolsUsed?: string[];
resolveToolName: ToolNameResolver;
t: TranslateFn;
}) {
return (
<div className={`chat-message chat-message-${role}`}>
@@ -31,6 +160,13 @@ function ChatMessageBubble({
<Text size="sm" style={{ whiteSpace: "pre-wrap" }}>
{content}
</Text>
{toolsUsed && toolsUsed.length > 0 && (
<ToolsUsedBlock
tools={toolsUsed}
resolveToolName={resolveToolName}
t={t}
/>
)}
</Paper>
</div>
);
@@ -38,14 +174,9 @@ function ChatMessageBubble({
export function ChatPanel() {
const { t } = useTranslation();
const {
messages,
isOpen,
isLoading,
progressPhase,
toggleOpen,
sendMessage,
} = useChat();
const { messages, isOpen, isLoading, progress, toggleOpen, sendMessage } =
useChat();
const resolveToolName = useToolNameResolver();
const [input, setInput] = useState("");
const scrollRef = useRef<HTMLDivElement>(null);
const inputRef = useRef<HTMLInputElement>(null);
@@ -129,6 +260,9 @@ export function ChatPanel() {
key={msg.id}
role={msg.role}
content={msg.content}
toolsUsed={msg.toolsUsed}
resolveToolName={resolveToolName}
t={t}
/>
))}
{isLoading && (
@@ -141,8 +275,8 @@ export function ChatPanel() {
<Group gap="xs" wrap="nowrap">
<Loader size="xs" type="dots" />
<Text size="sm" c="dimmed">
{progressPhase
? t(`chat.progress.${progressPhase}`)
{progress
? formatProgress(progress, t, resolveToolName)
: t("chat.progress.thinking")}
</Text>
</Group>