Merge pull request #6636 from Stirling-Tools/SaaS-update

Update SaaS to latest main
This commit is contained in:
James Brunton
2026-06-12 10:19:35 +01:00
committed by GitHub
59 changed files with 10405 additions and 227 deletions
+3 -3
View File
@@ -13,7 +13,7 @@ Usage:
""" """
# Sample for Windows: # Sample for Windows:
# python .github/scripts/check_language_toml.py --reference-file frontend/editor/public/locales/en-GB/translation.toml --branch "" --files frontend/editor/public/locales/de-DE/translation.toml frontend/editor/public/locales/fr-FR/translation.toml # python .github/scripts/check_language_toml.py --reference-file frontend/editor/public/locales/en-US/translation.toml --branch "" --files frontend/editor/public/locales/de-DE/translation.toml frontend/editor/public/locales/fr-FR/translation.toml
import argparse import argparse
import glob import glob
@@ -211,7 +211,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
) )
continue continue
if basename_current_file == basename_reference_file and locale_dir == "en-GB": if basename_current_file == basename_reference_file and locale_dir == "en-US":
continue continue
if ( if (
@@ -308,7 +308,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
report.append("## ❌ Overall Check Status: **_Failed_**") report.append("## ❌ Overall Check Status: **_Failed_**")
report.append("") report.append("")
report.append( report.append(
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-GB/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/editor/public/locales/en-GB/translation.toml)" f"@{actor} please check your translation if it conforms to the standard. Follow the format of [en-US/translation.toml](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/frontend/editor/public/locales/en-US/translation.toml)"
) )
else: else:
report.append("## ✅ Overall Check Status: **_Success_**") report.append("## ✅ Overall Check Status: **_Success_**")
+1 -1
View File
@@ -293,7 +293,7 @@ jobs:
SECURITY_ENABLELOGIN: "true" SECURITY_ENABLELOGIN: "true"
SECURITY_INITIALLOGIN_USERNAME: "${{ secrets.TEST_LOGIN_USERNAME }}" SECURITY_INITIALLOGIN_USERNAME: "${{ secrets.TEST_LOGIN_USERNAME }}"
SECURITY_INITIALLOGIN_PASSWORD: "${{ secrets.TEST_LOGIN_PASSWORD }}" SECURITY_INITIALLOGIN_PASSWORD: "${{ secrets.TEST_LOGIN_PASSWORD }}"
SYSTEM_DEFAULTLOCALE: en-GB SYSTEM_DEFAULTLOCALE: en-US
UI_APPNAME: "Stirling-PDF V2 PR#${{ needs.check-pr.outputs.pr_number }}" UI_APPNAME: "Stirling-PDF V2 PR#${{ needs.check-pr.outputs.pr_number }}"
UI_HOMEDESCRIPTION: "V2 PR#${{ needs.check-pr.outputs.pr_number }} - Embedded Architecture" UI_HOMEDESCRIPTION: "V2 PR#${{ needs.check-pr.outputs.pr_number }} - Embedded Architecture"
UI_APPNAMENAVBAR: "V2 PR#${{ needs.check-pr.outputs.pr_number }}" UI_APPNAMENAVBAR: "V2 PR#${{ needs.check-pr.outputs.pr_number }}"
@@ -388,7 +388,7 @@ jobs:
environment: environment:
DISABLE_ADDITIONAL_FEATURES: "${DISABLE_ADDITIONAL_FEATURES}" DISABLE_ADDITIONAL_FEATURES: "${DISABLE_ADDITIONAL_FEATURES}"
SECURITY_ENABLELOGIN: "${LOGIN_SECURITY}" SECURITY_ENABLELOGIN: "${LOGIN_SECURITY}"
SYSTEM_DEFAULTLOCALE: en-GB SYSTEM_DEFAULTLOCALE: en-US
UI_APPNAME: "Stirling-PDF PR#${PR_NUMBER}" UI_APPNAME: "Stirling-PDF PR#${PR_NUMBER}"
UI_HOMEDESCRIPTION: "PR#${PR_NUMBER} for Stirling-PDF Latest" UI_HOMEDESCRIPTION: "PR#${PR_NUMBER} for Stirling-PDF Latest"
UI_APPNAMENAVBAR: "PR#${PR_NUMBER}" UI_APPNAMENAVBAR: "PR#${PR_NUMBER}"
+6 -6
View File
@@ -166,16 +166,16 @@ jobs:
// Determine reference file // Determine reference file
let referenceFilePath; let referenceFilePath;
if (changedFiles.includes("frontend/editor/public/locales/en-GB/translation.toml")) { if (changedFiles.includes("frontend/editor/public/locales/en-US/translation.toml")) {
console.log("Using PR branch reference file."); console.log("Using PR branch reference file.");
const { data: fileContent } = await github.rest.repos.getContent({ const { data: fileContent } = await github.rest.repos.getContent({
owner: prRepoOwner, owner: prRepoOwner,
repo: prRepoName, repo: prRepoName,
path: "frontend/editor/public/locales/en-GB/translation.toml", path: "frontend/editor/public/locales/en-US/translation.toml",
ref: branch, ref: branch,
}); });
referenceFilePath = "pr-branch-translation-en-GB.toml"; referenceFilePath = "pr-branch-translation-en-US.toml";
const content = Buffer.from(fileContent.content, "base64").toString("utf-8"); const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
fs.writeFileSync(referenceFilePath, content); fs.writeFileSync(referenceFilePath, content);
} else { } else {
@@ -183,11 +183,11 @@ jobs:
const { data: fileContent } = await github.rest.repos.getContent({ const { data: fileContent } = await github.rest.repos.getContent({
owner: repoOwner, owner: repoOwner,
repo: repoName, repo: repoName,
path: "frontend/editor/public/locales/en-GB/translation.toml", path: "frontend/editor/public/locales/en-US/translation.toml",
ref: "main", ref: "main",
}); });
referenceFilePath = "main-branch-translation-en-GB.toml"; referenceFilePath = "main-branch-translation-en-US.toml";
const content = Buffer.from(fileContent.content, "base64").toString("utf-8"); const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
fs.writeFileSync(referenceFilePath, content); fs.writeFileSync(referenceFilePath, content);
} }
@@ -293,6 +293,6 @@ jobs:
run: | run: |
echo "Cleaning up temporary files..." echo "Cleaning up temporary files..."
rm -rf pr-branch rm -rf pr-branch
rm -f pr-branch-translation-en-GB.toml main-branch-translation-en-GB.toml changed_files.txt result.txt rm -f pr-branch-translation-en-US.toml main-branch-translation-en-US.toml changed_files.txt result.txt
echo "Cleanup complete." echo "Cleanup complete."
continue-on-error: true # Ensure cleanup runs even if previous steps fail continue-on-error: true # Ensure cleanup runs even if previous steps fail
+1 -1
View File
@@ -188,7 +188,7 @@ jobs:
environment: environment:
DISABLE_ADDITIONAL_FEATURES: "true" DISABLE_ADDITIONAL_FEATURES: "true"
SECURITY_ENABLELOGIN: "false" SECURITY_ENABLELOGIN: "false"
SYSTEM_DEFAULTLOCALE: en-GB SYSTEM_DEFAULTLOCALE: en-US
UI_APPNAME: "Stirling-PDF V2" UI_APPNAME: "Stirling-PDF V2"
UI_HOMEDESCRIPTION: "V2 Frontend/Backend Split" UI_HOMEDESCRIPTION: "V2 Frontend/Backend Split"
UI_APPNAMENAVBAR: "V2 Deployment" UI_APPNAMENAVBAR: "V2 Deployment"
+2 -2
View File
@@ -62,7 +62,7 @@ jobs:
- name: Sync translation TOML files - name: Sync translation TOML files
run: | run: |
python .github/scripts/check_language_toml.py --reference-file "frontend/editor/public/locales/en-GB/translation.toml" --branch main python .github/scripts/check_language_toml.py --reference-file "frontend/editor/public/locales/en-US/translation.toml" --branch main
- name: pre-commit run - name: pre-commit run
run: | run: |
@@ -100,7 +100,7 @@ jobs:
This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made: This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made:
#### **1. Synchronization of Translation Files** #### **1. Synchronization of Translation Files**
- Updated translation files (`frontend/editor/public/locales/*/translation.toml`) to reflect changes in the reference file `en-GB/translation.toml`. - Updated translation files (`frontend/editor/public/locales/*/translation.toml`) to reflect changes in the reference file `en-US/translation.toml`.
- Ensured consistency and synchronization across all supported language files. - Ensured consistency and synchronization across all supported language files.
- Highlighted any missing or incomplete translations. - Highlighted any missing or incomplete translations.
- **Format**: TOML - **Format**: TOML
+1 -1
View File
@@ -129,7 +129,7 @@ jobs:
environment: environment:
DISABLE_ADDITIONAL_FEATURES: "true" DISABLE_ADDITIONAL_FEATURES: "true"
SECURITY_ENABLELOGIN: "false" SECURITY_ENABLELOGIN: "false"
SYSTEM_DEFAULTLOCALE: en-GB SYSTEM_DEFAULTLOCALE: en-US
UI_APPNAME: "Stirling-PDF Test" UI_APPNAME: "Stirling-PDF Test"
UI_HOMEDESCRIPTION: "Test Deployment" UI_HOMEDESCRIPTION: "Test Deployment"
UI_APPNAMENAVBAR: "Test" UI_APPNAMENAVBAR: "Test"
+3 -3
View File
@@ -200,9 +200,9 @@ const [ToolName] = (props: BaseToolProps) => {
``` ```
## 5. Add Translations ## 5. Add Translations
Update translation files. **Important: Only update `en-GB` files** - other languages are handled separately. Update translation files. **Important: Only update `en-US` files** - other languages are handled separately.
**File to update:** `frontend/editor/public/locales/en-GB/translation.toml` **File to update:** `frontend/editor/public/locales/en-US/translation.toml`
**Required Translation Keys**: **Required Translation Keys**:
```toml ```toml
@@ -251,7 +251,7 @@ Update translation files. **Important: Only update `en-GB` files** - other langu
``` ```
**Translation Notes:** **Translation Notes:**
- **Only update `en-GB/translation.toml`** - other locale files are managed separately - **Only update `en-US/translation.toml`** - other locale files are managed separately
- Use descriptive keys that match your component's `t()` calls - Use descriptive keys that match your component's `t()` calls
- Include tooltip translations if you created tooltip hooks - Include tooltip translations if you created tooltip hooks
- Add `options.*` keys if your tool has settings with descriptions - Add `options.*` keys if your tool has settings with descriptions
+1 -1
View File
@@ -426,7 +426,7 @@ The frontend is organized with a clear separation of concerns:
## Translation Rules ## Translation Rules
- **CRITICAL**: Always update translations in `en-GB` only, never `en-US` - **CRITICAL**: Always update translations in `en-US` only - all other languages (including `en-GB`) are handled separately
- Translation files are located in `frontend/editor/public/locales/` - Translation files are located in `frontend/editor/public/locales/`
## Important Notes ## Important Notes
+12 -1
View File
@@ -52,6 +52,17 @@ This guide focuses on developing for Stirling 2.0, including both the React fron
- Rust and Cargo (required for Tauri desktop app development) - Rust and Cargo (required for Tauri desktop app development)
- Tauri CLI (install with `cargo install tauri-cli`) - Tauri CLI (install with `cargo install tauri-cli`)
### Optional System Dependencies
These are not required to run the app but enable specific features. The app detects them at startup and disables the relevant features if they are missing.
| Dependency | Feature | Install |
|---|---|---|
| LibreOffice | File-to-PDF conversions | `brew install libreoffice` / `apt install libreoffice` |
| Tesseract | OCR | `brew install tesseract` / `apt install tesseract-ocr` |
| WeasyPrint | AI document creation | `brew install weasyprint` / `apt install weasyprint` |
| qpdf | PDF optimisation | `brew install qpdf` / `apt install qpdf` |
### Setup Steps ### Setup Steps
1. Clone the repository: 1. Clone the repository:
@@ -576,7 +587,7 @@ When adding a new feature or modifying existing ones in Stirling-PDF, you'll nee
Find the existing `messages.properties` files in the `stirling-pdf/src/main/resources` directory. You'll see files like: Find the existing `messages.properties` files in the `stirling-pdf/src/main/resources` directory. You'll see files like:
- `messages.properties` (default, usually English) - `messages.properties` (default, usually English)
- `messages_en_GB.properties` - `messages_en_US.properties`
- `messages_fr_FR.properties` - `messages_fr_FR.properties`
- `messages_de_DE.properties` - `messages_de_DE.properties`
- etc. - etc.
@@ -112,6 +112,17 @@ public class InternalApiClient {
// every caller of this dispatcher is an automation surface by design. // every caller of this dispatcher is an automation surface by design.
headers.add(AUTOMATION_HEADER, "true"); headers.add(AUTOMATION_HEADER, "true");
// A no-file ai/tools call (e.g. create-pdf-from-html-agent) sends only string params, so
// without this RestTemplate would use urlencoded instead of the multipart the controller
// expects. File-bearing calls get the right multipart content-type from RestTemplate.
boolean isAiTool = endpointPath.startsWith("/api/v1/ai/tools/");
boolean hasFilePart =
body.values().stream()
.flatMap(java.util.List::stream)
.anyMatch(v -> v instanceof Resource);
if (isAiTool && !hasFilePart) {
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
}
HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers); HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<>(body, headers);
RequestCallback requestCallback = restTemplate.httpEntityCallback(entity, Resource.class); RequestCallback requestCallback = restTemplate.httpEntityCallback(entity, Resource.class);
@@ -37,8 +37,8 @@ public class LocaleConfiguration implements WebMvcConfigurer {
public LocaleResolver localeResolver() { public LocaleResolver localeResolver() {
SessionLocaleResolver slr = new SessionLocaleResolver(); SessionLocaleResolver slr = new SessionLocaleResolver();
String appLocaleEnv = applicationProperties.getSystem().getDefaultLocale(); String appLocaleEnv = applicationProperties.getSystem().getDefaultLocale();
Locale defaultLocale = // Fallback to UK locale if environment variable is not set Locale defaultLocale = // Fallback to US locale if environment variable is not set
Locale.UK; Locale.US;
if (appLocaleEnv != null && !appLocaleEnv.isEmpty()) { if (appLocaleEnv != null && !appLocaleEnv.isEmpty()) {
Locale tempLocale = Locale.forLanguageTag(appLocaleEnv); Locale tempLocale = Locale.forLanguageTag(appLocaleEnv);
String tempLanguageTag = tempLocale.toLanguageTag(); String tempLanguageTag = tempLocale.toLanguageTag();
@@ -51,7 +51,7 @@ public class LocaleConfiguration implements WebMvcConfigurer {
defaultLocale = tempLocale; defaultLocale = tempLocale;
} else { } else {
System.err.println( System.err.println(
"Invalid SYSTEM_DEFAULTLOCALE environment variable value. Falling back to default en-GB."); "Invalid SYSTEM_DEFAULTLOCALE environment variable value. Falling back to default en-US.");
} }
} }
} }
@@ -48,7 +48,7 @@ public class AdditionalLanguageJsController {
} }
} }
// Fallback // Fallback
return "en_GB"; return "en_US";
} }
"""); """);
writer.flush(); writer.flush();
@@ -167,7 +167,7 @@ legal:
impressum: "" # URL to the impressum of your application (e.g. https://example.com/impressum). Empty string to disable or filename to load from local file in static folder impressum: "" # URL to the impressum of your application (e.g. https://example.com/impressum). Empty string to disable or filename to load from local file in static folder
system: system:
defaultLocale: en-US # set the default language (e.g. 'de-DE', 'fr-FR', etc) defaultLocale: "" # force a default language for new users (e.g. 'en-US', 'de-DE'). Empty string auto-detects from the browser, falling back to en-US
googlevisibility: false # 'true' to allow Google visibility (via robots.txt), 'false' to disallow googlevisibility: false # 'true' to allow Google visibility (via robots.txt), 'false' to disallow
enableAlphaFunctionality: false # set to enable functionality which might need more testing before it fully goes live (this feature might make no changes) enableAlphaFunctionality: false # set to enable functionality which might need more testing before it fully goes live (this feature might make no changes)
showUpdate: false # see when a new update is available showUpdate: false # see when a new update is available
@@ -23,7 +23,7 @@ class AdditionalLanguageJsControllerTest {
LanguageService lang = mock(LanguageService.class); LanguageService lang = mock(LanguageService.class);
// LinkedHashSet for deterministic order in the array // LinkedHashSet for deterministic order in the array
when(lang.getSupportedLanguages()) when(lang.getSupportedLanguages())
.thenReturn(new LinkedHashSet<>(List.of("de_DE", "en_GB"))); .thenReturn(new LinkedHashSet<>(List.of("de_DE", "en_US")));
MockMvc mvc = MockMvc mvc =
MockMvcBuilders.standaloneSetup(new AdditionalLanguageJsController(lang)).build(); MockMvcBuilders.standaloneSetup(new AdditionalLanguageJsController(lang)).build();
@@ -36,9 +36,9 @@ class AdditionalLanguageJsControllerTest {
.string( .string(
containsString( containsString(
"const supportedLanguages =" "const supportedLanguages ="
+ " [\"de_DE\",\"en_GB\"];"))) + " [\"de_DE\",\"en_US\"];")))
.andExpect(content().string(containsString("function getDetailedLanguageCode()"))) .andExpect(content().string(containsString("function getDetailedLanguageCode()")))
.andExpect(content().string(containsString("return \"en_GB\";"))); .andExpect(content().string(containsString("return \"en_US\";")));
verify(lang, times(1)).getSupportedLanguages(); verify(lang, times(1)).getSupportedLanguages();
} }
@@ -0,0 +1,144 @@
package stirling.software.proprietary.controller.api;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.springframework.core.io.Resource;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import stirling.software.common.configuration.RuntimePathConfig;
import stirling.software.common.service.CustomPDFDocumentFactory;
import stirling.software.common.util.ProcessExecutor;
import stirling.software.common.util.TempFile;
import stirling.software.common.util.TempFileManager;
import stirling.software.common.util.WebResponseUtils;
/**
* Dispatchable tool that converts an AI-generated HTML string to a PDF via WeasyPrint.
*
* <p>Called by {@link stirling.software.proprietary.service.AiWorkflowService} when the engine
* emits a {@code CREATE_PDF_FROM_HTML_AGENT} plan step. The HTML comes from a trusted Jinja
* template so sanitization is intentionally skipped.
*/
@Slf4j
@Hidden
@RestController
@RequestMapping("/api/v1/ai/tools")
@RequiredArgsConstructor
@Tag(name = "AI Tools", description = "Dispatchable AI-backed tools.")
public class CreatePdfAgentController {
private final TempFileManager tempFileManager;
private final CustomPDFDocumentFactory pdfDocumentFactory;
private final RuntimePathConfig runtimePathConfig;
/**
* Returns true only when WeasyPrint is definitively unavailable — either the binary could not
* be launched at all, or it launched but immediately failed to load a required system library.
* Other conversion failures (bad HTML, output errors, etc.) return false so they surface as
* real errors rather than a misleading "dependency missing" message.
*/
private static boolean isMissingDependencyError(IOException e) {
String msg = e.getMessage();
if (msg == null) return false;
// OS could not start the process — binary not on PATH or not at the configured path.
if (msg.contains("Cannot run program")) return true;
// Process started but crashed immediately loading a shared library.
// "cannot load library" — Python/cffi error (Linux and macOS via pip)
// "Library not loaded" / "image not found" — macOS dyld error (Homebrew installs)
String lower = msg.toLowerCase();
if (lower.contains("cannot load library")) return true;
if (lower.contains("library not loaded")) return true;
if (lower.contains("image not found")) return true;
return false;
}
@PostMapping(
value = "/create-pdf-from-html-agent",
consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@Operation(
summary = "Convert AI-generated HTML to a PDF",
description =
"Accepts an HTML document as a plain-text parameter and returns a PDF."
+ " This endpoint is dispatched by the AI workflow orchestrator as a"
+ " plan step; it is not intended for direct client use.")
public ResponseEntity<Resource> createPdfFromHtml(
@RequestParam("htmlContent") String htmlContent,
@RequestParam("filename") String filename)
throws Exception {
log.info(
"[create-pdf-agent] converting HTML to PDF via WeasyPrint — html_bytes={}",
htmlContent.length());
try (TempFile htmlFile = tempFileManager.createManagedTempFile(".html");
TempFile pdfFile = tempFileManager.createManagedTempFile(".pdf")) {
Files.writeString(htmlFile.getPath(), htmlContent, StandardCharsets.UTF_8);
List<String> command = new ArrayList<>();
command.add(runtimePathConfig.getWeasyPrintPath());
command.add("-e");
command.add("utf-8");
command.add("-v");
// SSRF: the HTML is self-contained and the engine validates style colours, so no
// external url() reaches WeasyPrint. For full isolation, run it network-isolated.
command.add(htmlFile.getAbsolutePath());
command.add(pdfFile.getAbsolutePath());
try {
ProcessExecutor.getInstance(ProcessExecutor.Processes.WEASYPRINT)
.runCommandWithOutputHandling(command);
} catch (IOException e) {
if (isMissingDependencyError(e)) {
throw new IOException(
"AI document creation is not available on this server because a required"
+ " system dependency is not installed. Please contact your"
+ " system administrator.");
}
throw e;
}
String safeFilename = Filenames.toSimpleFileName(filename);
if (safeFilename == null || safeFilename.isBlank() || !safeFilename.endsWith(".pdf")) {
safeFilename = "generated-document.pdf";
}
// Stamp the standard Stirling metadata onto the WeasyPrint output and write the result
// straight to the response temp file. Loading from the file and saving to the file
// avoids materialising the whole document as a byte[] twice (read-all + re-serialise),
// which matters for large generated documents.
TempFile tempOut = tempFileManager.createManagedTempFile(".pdf");
try (PDDocument document = pdfDocumentFactory.load(pdfFile.getPath())) {
document.save(tempOut.getPath().toFile());
} catch (Exception e) {
tempOut.close();
throw e;
}
log.info(
"[create-pdf-agent] PDF ready — filename={} bytes={}",
safeFilename,
Files.size(tempOut.getPath()));
return WebResponseUtils.pdfFileToWebResponse(tempOut, safeFilename);
}
}
}
@@ -17,6 +17,7 @@ import org.springframework.core.io.Resource;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.MediaTypeFactory; import org.springframework.http.MediaTypeFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import io.github.pixee.security.Filenames; import io.github.pixee.security.Filenames;
@@ -175,7 +176,6 @@ public class AiWorkflowService {
initialRequest.setFiles(files); initialRequest.setFiles(files);
initialRequest.setConversationHistory(new ArrayList<>(request.getConversationHistory())); initialRequest.setConversationHistory(new ArrayList<>(request.getConversationHistory()));
initialRequest.setEnabledEndpoints(endpointResolver.getEnabledEndpointUrls()); initialRequest.setEnabledEndpoints(endpointResolver.getEnabledEndpointUrls());
listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.ANALYZING)); listener.onProgress(AiWorkflowProgressEvent.of(AiWorkflowPhase.ANALYZING));
WorkflowState state = new WorkflowState.Pending(initialRequest); WorkflowState state = new WorkflowState.Pending(initialRequest);
@@ -580,6 +580,10 @@ public class AiWorkflowService {
log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage()); log.error("Plan step on tool {} timed out: {}", e.getEndpointPath(), e.getMessage());
return new WorkflowState.Terminal( return new WorkflowState.Terminal(
cannotContinue(toolTimeoutMessage(e.getEndpointPath(), e))); cannotContinue(toolTimeoutMessage(e.getEndpointPath(), e)));
} catch (HttpServerErrorException e) {
String reason = extractDetailFromHttpError(e);
log.error("Plan step failed (HTTP {}): {}", e.getStatusCode(), reason);
return new WorkflowState.Terminal(cannotContinue(reason));
} catch (Exception e) { } catch (Exception e) {
log.error("Failed to execute plan: {}", e.getMessage(), e); log.error("Failed to execute plan: {}", e.getMessage(), e);
return new WorkflowState.Terminal( return new WorkflowState.Terminal(
@@ -605,6 +609,27 @@ public class AiWorkflowService {
return String.format("The %s tool failed: %s", endpointPath, reason); return String.format("The %s tool failed: %s", endpointPath, reason);
} }
/**
* Extracts the {@code detail} field from an HTTP error response body if it is valid JSON,
* otherwise falls back to the exception message. This lets controller-level error messages
* (e.g. missing system dependency) surface cleanly in the chat response.
*/
private String extractDetailFromHttpError(HttpServerErrorException e) {
try {
String body = e.getResponseBodyAsString();
if (body != null && !body.isBlank()) {
JsonNode node = objectMapper.readTree(body);
JsonNode detail = node.get("detail");
if (detail != null && detail.isTextual() && !detail.asText().isBlank()) {
return detail.asText();
}
}
} catch (Exception ignored) {
// fall through to generic message
}
return "The request could not be completed. Please try again or contact your system administrator.";
}
/** /**
* Adapt the AI workflow's {@link ProgressListener} to the engine's {@link * Adapt the AI workflow's {@link ProgressListener} to the engine's {@link
* PolicyProgressListener}: each step start maps to an {@code EXECUTING_TOOL} progress event * PolicyProgressListener}: each step start maps to an {@code EXECUTING_TOOL} progress event
+3 -3
View File
@@ -16,7 +16,7 @@ Java forms the core of Stirling-PDF. When adding new features or handling errors
2. **Use `try-with-resources`** when working with streams or other closable resources to ensure clean-up even on failure. 2. **Use `try-with-resources`** when working with streams or other closable resources to ensure clean-up even on failure.
3. **Return meaningful HTTP status codes** in controllers by throwing `ResponseStatusException` or using `@ExceptionHandler` methods. 3. **Return meaningful HTTP status codes** in controllers by throwing `ResponseStatusException` or using `@ExceptionHandler` methods.
4. **Log with context** using the projects logging framework. Include identifiers or IDs that help trace the issue. 4. **Log with context** using the projects logging framework. Include identifiers or IDs that help trace the issue.
5. **Internationalise messages** by placing user-facing text in `messages_en_GB.properties` and referencing them with message keys. 5. **Internationalise messages** by placing user-facing text in `messages_en_US.properties` and referencing them with message keys.
## JavaScript ## JavaScript
@@ -55,11 +55,11 @@ except Exception as err:
## Internationalisation (i18n) ## Internationalisation (i18n)
All user-visible error strings should be defined in the main translation file (`messages_en_GB.properties`). Other language files will use the same keys. Refer to messages in code rather than hard-coding text. All user-visible error strings should be defined in the main translation file (`messages_en_US.properties`). Other language files will use the same keys. Refer to messages in code rather than hard-coding text.
When creating new messages: When creating new messages:
1. Add the English phrase to `messages_en_GB.properties`. 1. Add the English phrase to `messages_en_US.properties`.
2. Reference the message key in your Java, JavaScript, or Python code. 2. Reference the message key in your Java, JavaScript, or Python code.
3. Update other localisation files as needed. 3. Update other localisation files as needed.
+4 -4
View File
@@ -16,7 +16,7 @@ Fork Stirling-PDF and create a new branch out of `main`.
- Use hyphenated format: `pl-PL` (not underscore) - Use hyphenated format: `pl-PL` (not underscore)
2. Copy the reference translation file: 2. Copy the reference translation file:
- Source: `frontend/editor/public/locales/en-GB/translation.toml` - Source: `frontend/editor/public/locales/en-US/translation.toml`
- Destination: `frontend/editor/public/locales/pl-PL/translation.toml` - Destination: `frontend/editor/public/locales/pl-PL/translation.toml`
3. Translate all entries in the TOML file 3. Translate all entries in the TOML file
@@ -47,10 +47,10 @@ ignore = [
## Add New Translation Tags ## Add New Translation Tags
> [!IMPORTANT] > [!IMPORTANT]
> If you add any new translation tags, they must first be added to the `en-GB/translation.toml` file. This ensures consistency across all language files. > If you add any new translation tags, they must first be added to the `en-US/translation.toml` file. This ensures consistency across all language files.
- New translation tags **must be added** to `frontend/editor/public/locales/en-GB/translation.toml` to maintain a reference for other languages. - New translation tags **must be added** to `frontend/editor/public/locales/en-US/translation.toml` to maintain a reference for other languages.
- After adding the new tags to `en-GB/translation.toml`, add and translate them in the respective language file (e.g., `pl-PL/translation.toml`). - After adding the new tags to `en-US/translation.toml`, add and translate them in the respective language file (e.g., `pl-PL/translation.toml`).
- Use the scripts in `scripts/translations/` to validate and manage translations (see `scripts/translations/README.md`) - Use the scripts in `scripts/translations/` to validate and manage translations (see `scripts/translations/README.md`)
Make sure to place the entry under the correct language section. This helps maintain the accuracy of translation progress statistics and ensures that the translation tool or scripts do not misinterpret the completion rate. Make sure to place the entry under the correct language section. This helps maintain the accuracy of translation progress statistics and ensures that the translation tool or scripts do not misinterpret the completion rate.
+1 -1
View File
@@ -3,7 +3,7 @@
## Overview ## Overview
The script [`scripts/counter_translation.py`](../scripts/counter_translation.py) checks the translation progress of the property files in the directory `app/core/src/main/resources/`. The script [`scripts/counter_translation.py`](../scripts/counter_translation.py) checks the translation progress of the property files in the directory `app/core/src/main/resources/`.
It compares each `messages_*.properties` file with the English reference file `messages_en_GB.properties` and calculates a percentage of completion for each language. It compares each `messages_*.properties` file with the English reference file `messages_en_US.properties` and calculates a percentage of completion for each language.
In addition to console output, the script automatically updates the progress badges in the projects `README.md` and maintains the configuration file [`scripts/ignore_translation.toml`](../scripts/ignore_translation.toml), which lists translation keys to be ignored for each language. In addition to console output, the script automatically updates the progress badges in the projects `README.md` and maintains the configuration file [`scripts/ignore_translation.toml`](../scripts/ignore_translation.toml), which lists translation keys to be ignored for each language.
+1
View File
@@ -5,6 +5,7 @@ description = "AI Document Engine"
requires-python = ">=3.13" requires-python = ">=3.13"
dependencies = [ dependencies = [
"fastapi>=0.116.0", "fastapi>=0.116.0",
"jinja2>=3.1.0",
"pgvector>=0.3.6", "pgvector>=0.3.6",
"psycopg[binary,pool]>=3.2", "psycopg[binary,pool]>=3.2",
"pydantic>=2.0.0", "pydantic>=2.0.0",
+2
View File
@@ -2,6 +2,7 @@
from .execution import ExecutionPlanningAgent from .execution import ExecutionPlanningAgent
from .orchestrator import OrchestratorAgent from .orchestrator import OrchestratorAgent
from .pdf_create import PdfCreateAgent
from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection from .pdf_edit import PdfEditAgent, PdfEditParameterSelector, PdfEditPlanSelection
from .pdf_questions import PdfQuestionAgent from .pdf_questions import PdfQuestionAgent
from .pdf_review import PdfReviewAgent from .pdf_review import PdfReviewAgent
@@ -10,6 +11,7 @@ from .user_spec import UserSpecAgent
__all__ = [ __all__ = [
"ExecutionPlanningAgent", "ExecutionPlanningAgent",
"OrchestratorAgent", "OrchestratorAgent",
"PdfCreateAgent",
"PdfEditAgent", "PdfEditAgent",
"PdfEditParameterSelector", "PdfEditParameterSelector",
"PdfEditPlanSelection", "PdfEditPlanSelection",
+22 -1
View File
@@ -8,6 +8,7 @@ from pydantic_ai import Agent
from pydantic_ai.output import ToolOutput from pydantic_ai.output import ToolOutput
from pydantic_ai.tools import RunContext from pydantic_ai.tools import RunContext
from stirling.agents.pdf_create import PdfCreateAgent
from stirling.agents.pdf_edit import PdfEditAgent from stirling.agents.pdf_edit import PdfEditAgent
from stirling.agents.pdf_questions import PdfQuestionAgent from stirling.agents.pdf_questions import PdfQuestionAgent
from stirling.agents.pdf_review import PdfReviewAgent from stirling.agents.pdf_review import PdfReviewAgent
@@ -26,6 +27,7 @@ from stirling.contracts import (
format_conversation_history, format_conversation_history,
format_file_names, format_file_names,
) )
from stirling.contracts.pdf_create import PdfCreateOrchestrateResponse
from stirling.services import AppRuntime from stirling.services import AppRuntime
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -76,6 +78,16 @@ class OrchestratorAgent:
"Delegate requests to convert a PDF to Markdown or extract its content as readable text." "Delegate requests to convert a PDF to Markdown or extract its content as readable text."
), ),
), ),
ToolOutput(
self.delegate_pdf_create,
name="delegate_pdf_create",
description=(
"Delegate requests to create a new PDF document from scratch based on a"
" description. Use this when the user wants to generate a new document"
" (e.g. 'create an invoice', 'write a report', 'make a contract',"
" 'draft a letter'). No input file is required."
),
),
ToolOutput( ToolOutput(
self.unsupported_capability, self.unsupported_capability,
name="unsupported_capability", name="unsupported_capability",
@@ -92,6 +104,8 @@ class OrchestratorAgent:
"Use delegate_pdf_review when the user wants the PDF returned with review" "Use delegate_pdf_review when the user wants the PDF returned with review"
" comments attached — anything like 'review this', 'annotate with comments'," " comments attached — anything like 'review this', 'annotate with comments',"
" 'leave feedback on the PDF'. " " 'leave feedback on the PDF'. "
"Use delegate_pdf_create when the user wants to generate a new document from"
" scratch with no input file — invoices, reports, letters, contracts, etc. "
"Use delegate_pdf_ingest for any request to convert a PDF to Markdown " "Use delegate_pdf_ingest for any request to convert a PDF to Markdown "
"or extract its content as readable text. " "or extract its content as readable text. "
"Use unsupported_capability when the user asks about the assistant itself " "Use unsupported_capability when the user asks about the assistant itself "
@@ -133,12 +147,13 @@ class OrchestratorAgent:
return await self._run_pdf_edit(request) return await self._run_pdf_edit(request)
case SupportedCapability.AGENT_DRAFT: case SupportedCapability.AGENT_DRAFT:
return await self._run_agent_draft(request) return await self._run_agent_draft(request)
case SupportedCapability.PDF_CREATE:
return await self._run_pdf_create(request)
case ( case (
SupportedCapability.ORCHESTRATE SupportedCapability.ORCHESTRATE
| SupportedCapability.AGENT_REVISE | SupportedCapability.AGENT_REVISE
| SupportedCapability.AGENT_NEXT_ACTION | SupportedCapability.AGENT_NEXT_ACTION
| SupportedCapability.MATH_AUDITOR_AGENT | SupportedCapability.MATH_AUDITOR_AGENT
| SupportedCapability.PDF_TO_MARKDOWN
): ):
raise ValueError(f"Cannot resume orchestrator with capability: {capability}") raise ValueError(f"Cannot resume orchestrator with capability: {capability}")
case _ as unreachable: case _ as unreachable:
@@ -175,6 +190,12 @@ class OrchestratorAgent:
async def _run_pdf_review(self, request: OrchestratorRequest) -> PdfReviewOrchestrateResponse: async def _run_pdf_review(self, request: OrchestratorRequest) -> PdfReviewOrchestrateResponse:
return await PdfReviewAgent(self.runtime).orchestrate(request) return await PdfReviewAgent(self.runtime).orchestrate(request)
async def delegate_pdf_create(self, ctx: RunContext[OrchestratorDeps]) -> PdfCreateOrchestrateResponse:
return await self._run_pdf_create(ctx.deps.request)
async def _run_pdf_create(self, request: OrchestratorRequest) -> PdfCreateOrchestrateResponse:
return await PdfCreateAgent(self.runtime).orchestrate(request)
async def unsupported_capability( async def unsupported_capability(
self, self,
ctx: RunContext[OrchestratorDeps], ctx: RunContext[OrchestratorDeps],
@@ -0,0 +1,3 @@
from .agent import PdfCreateAgent
__all__ = ["PdfCreateAgent"]
@@ -0,0 +1,443 @@
"""PDF Create Agent — chunked multi-agent pipeline.
Flow:
1. MetaPlannerAgent (smart_model) analyses the request and produces DocumentMeta:
title, tone, shared terms, style, and cannot_do_reason. No sections yet.
2. SectionPlannerAgent (smart_model) reads the meta and produces DocumentSections:
ordered list of PlannedSection with heading, type, depth, and key_points.
3. Python assembles DocumentPlan from meta + sections, then groups sections into
chunks, each staying under the output-token ceiling.
4. SectionWriterAgents (smart_model) run in parallel via asyncio.gather.
Each returns a WrittenSections with fully populated DocumentSection objects.
5. The assembler collects sections in plan order → GeneratedDocument.
6. Jinja renders the document to HTML. The LLM never writes HTML.
The planner is split into two calls (meta then sections) so each LLM output schema
stays small enough for grammar compilation on all model tiers including Haiku.
"""
from __future__ import annotations
import asyncio
import logging
import re
from dataclasses import dataclass
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
from pydantic_ai import Agent
from pydantic_ai.output import NativeOutput
from stirling.contracts import (
EditCannotDoResponse,
EditPlanResponse,
OrchestratorRequest,
ToolOperationStep,
format_conversation_history,
)
from stirling.contracts.pdf_create import (
DocumentMeta,
DocumentPlan,
DocumentSection,
DocumentSections,
GeneratedDocument,
PdfCreateOrchestrateResponse,
PlannedSection,
SectionDepth,
WrittenSections,
)
from stirling.models.agent_tool_models import AgentToolId, CreatePdfFromHtmlAgentParams
from stirling.services import AppRuntime
logger = logging.getLogger(__name__)
_TEMPLATES_DIR = Path(__file__).parent / "templates"
# ── Token budget ──────────────────────────────────────────────────────────────────────────────────
# Conservative per-section token estimates mapped from planner-assigned depth.
_DEPTH_TOKENS: dict[SectionDepth, int] = {
SectionDepth.BRIEF: 250,
SectionDepth.STANDARD: 550,
SectionDepth.DETAILED: 1200,
}
# Maximum output tokens per writer call. Stays well below the quality cliff (~4k).
_CHUNK_CEILING = 3000
# Cap on simultaneous writer calls so a large document doesn't open a burst of LLM
# connections and trip provider rate limits.
_MAX_PARALLEL_WRITERS = 10
# ── Chunk dataclass ───────────────────────────────────────────────────────────────────────────────
@dataclass(frozen=True)
class _Chunk:
index: int
sections: list[PlannedSection]
# Descriptions of neighbouring chunks from the plan — passed to writers as
# read-only context so they can open/close their sections naturally.
context_before: str | None
context_after: str | None
# ── Chunking logic ────────────────────────────────────────────────────────────────────────────────
def _describe_sections(sections: list[PlannedSection]) -> str:
"""One-line summary of a chunk used as neighbour context."""
return "; ".join(f'"{s.heading}" ({s.type.value})' for s in sections)
def _make_chunks(sections: list[PlannedSection]) -> list[_Chunk]:
"""Group planned sections into chunks, each under _CHUNK_CEILING output tokens.
Section boundaries are atomic — a section is never split across chunks.
A single section whose estimated cost exceeds the ceiling gets its own chunk.
asyncio.gather preserves insertion order so chunk index is only used for logging.
"""
if not sections:
return []
groups: list[list[PlannedSection]] = []
current: list[PlannedSection] = []
current_tokens = 0
for section in sections:
cost = _DEPTH_TOKENS[section.depth]
if current and current_tokens + cost > _CHUNK_CEILING:
groups.append(current)
current = [section]
current_tokens = cost
else:
current.append(section)
current_tokens += cost
if current:
groups.append(current)
chunks: list[_Chunk] = []
for i, group in enumerate(groups):
context_before = _describe_sections(groups[i - 1]) if i > 0 else None
context_after = _describe_sections(groups[i + 1]) if i < len(groups) - 1 else None
chunks.append(
_Chunk(
index=i,
sections=group,
context_before=context_before,
context_after=context_after,
)
)
return chunks
# ── Prompts ───────────────────────────────────────────────────────────────────────────────────────
_META_PLANNER_SYSTEM_PROMPT = """\
You are a document planner. Your job is Step 1 of 2: produce the document header — NOT the
section list (that comes in Step 2) and NOT any body text (section writers handle that).
Analyse the user's request and produce a DocumentMeta with:
- title, subtitle (if appropriate), reference_number (only if the user supplies one explicitly)
- tone_brief: one sentence describing register and style
(e.g. "Formal legal language, third person, present tense." or
"Professional business tone, active voice.")
- shared_terms: consistent names for key entities AND ground-truth facts used throughout
the document. Two rules:
1. Capture EVERY value the user states explicitly that could be referenced in more than
one section. This includes — but is not limited to:
· Named parties, organisations, products, or systems
· Numeric values: amounts, quantities, percentages, durations, limits
· Identifiers: version numbers, reference codes, model names
· Dates and time periods
· Units of measure or currency
2. For any fact that will appear in two or more sections and that the user did NOT specify
(e.g. a default time, a standard rate, a typical threshold), assign ONE specific value
here. Do NOT let multiple writers independently invent the same fact.
Examples: {"the Agreement": "this Non-Disclosure Agreement", "the Client": "Acme Corp",
"contract value": "£120,000", "notice period": "30 days"}
- document_context: a single sentence anchoring the temporal or versioning context of the
document, if the user provides one. Leave empty if the user provides no such context.
- style_primary_color: accent and heading colour. Set ONLY when the user explicitly names a
colour or colour scheme (e.g. "make it red", "use navy blue"). Use CSS named colours
(e.g. "magenta", "navy", "crimson") or hex values. Leave null if no colour is stated.
- style_background_color: page background colour. Set only if explicitly requested.
- style_body_text_color: body text colour. Set only if explicitly requested.
- cannot_do_reason: set this ONLY when the request is not asking to create a document at all
(e.g. a question, a greeting, an edit request to an existing document). Never set it
because the document is large, complex, or technically detailed. Leave null otherwise.
RULES:
1. Extract ALL information the user provides. Do not invent content.
2. Do not produce any sections — that is Step 2.
"""
_SECTIONS_PLANNER_SYSTEM_PROMPT = """\
You are a document planner. Your job is Step 2 of 2: produce the ordered section list for
a document whose header has already been decided. Do NOT write any body text.
You will be given:
- The document meta (title, tone, shared terms, etc.) produced in Step 1
- The original user request
Produce a DocumentSections with an ordered list of PlannedSection objects.
For each section choose:
type — the most appropriate section type:
text — prose paragraphs (narrative, obligations, terms, descriptions)
key_value — labelled fields (parties, dates, metadata, identifiers)
line_items — tables with column headers (expenses, schedules, item lists)
bullet_list — unordered items (requirements, responsibilities, definitions)
signature — sign-off blocks for named parties or roles
depth — honest estimate of content volume:
brief (~250 tokens) — 1-2 items, a short paragraph, or a small table
standard (~550 tokens) — a few paragraphs, a medium table, or a moderate list
detailed (~1200 tokens) — long clauses, complex multi-row tables, or dense content
key_points — specific points this section MUST cover, taken directly from the user's input.
These are instructions to the writer, not summaries. Be precise and complete.
Every fact, name, date, amount, and requirement the user provides must appear somewhere.
For large documents, include enough key_points that the writer can produce substantial
content.
RULES:
1. Extract ALL information the user provides. Do not invent content.
2. Assign depth honestly — for a long detailed document most sections will be detailed.
3. For large documents, produce as many sections as needed — there is no section count limit.
4. Use the shared_terms from the meta exactly when writing key_points.
"""
_WRITER_SYSTEM_PROMPT = """\
You are a section writer for a structured document.
Write ONLY the sections assigned to you — no extras, no merging, no skipping.
SECTION TYPES — produce sections of exactly the requested type:
text — prose paragraphs. Use \\n\\n between paragraphs.
key_value — list of (label, value) pairs. Labels ≤ 5 words. Values verbatim from the data.
line_items — table. Every row must have exactly as many cells as there are columns.
bullet_list — flat list of items.
signature — list of signatory names/roles.
RULES:
1. Write ONLY the sections in your assignment list, in the order given.
2. Cover every key_point listed for each section. Do not omit any.
3. Use the shared_terms exactly — no paraphrasing or substituting alternatives.
Shared terms are ground truth. If your general knowledge or a common default would
produce a different value (e.g. a different duration, amount, date, or version number),
the shared term takes precedence. This applies everywhere in the document, including
boilerplate, FAQ, and summary sections.
4. Match the depth for each section: brief = concise, standard = moderate, \
detailed = thorough.
5. Maintain the document's tone throughout.
6. Do not reference other sections by number (e.g. "as defined in Section 3").
7. If a document_context is provided, use it to anchor any dates, versions, or time
references you generate. Do not invent a different temporal or versioning context.
"""
def _build_sections_prompt(meta: DocumentMeta, user_request: str, history: str) -> str:
lines: list[str] = [
"Document meta from Step 1:",
f" Title: {meta.title}",
f" Tone: {meta.tone_brief}",
]
if meta.subtitle:
lines.append(f" Subtitle: {meta.subtitle}")
if meta.document_context:
lines.append(f" Document context: {meta.document_context}")
if meta.shared_terms:
lines.append(" Shared terms:")
for term, referent in meta.shared_terms.items():
lines.append(f" {term}{referent}")
lines.append(f"\nConversation history:\n{history}")
lines.append(f"\nUser request: {user_request}")
return "\n".join(lines)
def _build_writer_prompt(plan: DocumentPlan, chunk: _Chunk) -> str:
lines: list[str] = [
f"Document: {plan.title}",
f"Tone: {plan.tone_brief}",
]
if plan.document_context:
lines.append(f"Document context: {plan.document_context}")
if plan.shared_terms:
lines.append("Ground-truth facts and shared terms (use exactly — these override defaults):")
for term, referent in plan.shared_terms.items():
lines.append(f" {term}{referent}")
if chunk.context_before:
lines.append(f"\nThe sections BEFORE yours cover: {chunk.context_before}")
if chunk.context_after:
lines.append(f"The sections AFTER yours cover: {chunk.context_after}")
lines.append(f"\nWrite these {len(chunk.sections)} section(s) in order:")
for i, s in enumerate(chunk.sections, 1):
lines.append(f"\n--- Section {i} ---")
lines.append(f"Heading: {s.heading}")
lines.append(f"Type: {s.type.value}")
lines.append(f"Depth: {s.depth.value}")
lines.append("Key points to cover:")
for point in s.key_points:
lines.append(f" - {point}")
return "\n".join(lines)
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────────
def _build_jinja_env() -> Environment:
return Environment(
loader=FileSystemLoader(str(_TEMPLATES_DIR)),
autoescape=True,
trim_blocks=True,
lstrip_blocks=True,
)
def _safe_filename(title: str) -> str:
slug = re.sub(r"[^\w\s-]", "", title.lower())
slug = re.sub(r"[\s_-]+", "-", slug).strip("-")
return (slug[:60] or "document") + ".pdf"
# ── Agent ─────────────────────────────────────────────────────────────────────────────────────────
class PdfCreateAgent:
def __init__(self, runtime: AppRuntime) -> None:
self.runtime = runtime
self._jinja_env = _build_jinja_env()
self._meta_planner: Agent[None, DocumentMeta] = Agent(
model=runtime.smart_model,
output_type=NativeOutput(DocumentMeta),
system_prompt=_META_PLANNER_SYSTEM_PROMPT,
model_settings={**runtime.smart_model_settings, "temperature": 0.1},
)
self._sections_planner: Agent[None, DocumentSections] = Agent(
model=runtime.smart_model,
output_type=NativeOutput(DocumentSections),
system_prompt=_SECTIONS_PLANNER_SYSTEM_PROMPT,
model_settings={**runtime.smart_model_settings, "temperature": 0.1},
)
self._writer: Agent[None, WrittenSections] = Agent(
model=runtime.smart_model,
output_type=NativeOutput(WrittenSections),
system_prompt=_WRITER_SYSTEM_PROMPT,
model_settings={**runtime.smart_model_settings, "temperature": 0.3},
)
async def orchestrate(self, request: OrchestratorRequest) -> PdfCreateOrchestrateResponse:
history = format_conversation_history(request.conversation_history)
# ── Phase 1: plan meta ─────────────────────────────────────────────────
logger.info("[pdf-create] phase 1/6: planning document meta")
meta_prompt = f"Conversation history:\n{history}\n\nUser request: {request.user_message}"
meta_result = await self._meta_planner.run(meta_prompt)
meta = meta_result.output
if meta.cannot_do_reason:
logger.info("[pdf-create] cannot_do: %s", meta.cannot_do_reason)
return EditCannotDoResponse(reason=meta.cannot_do_reason)
logger.info("[pdf-create] meta: title=%r tone=%r", meta.title, meta.tone_brief)
# ── Phase 2: plan sections ─────────────────────────────────────────────
logger.info("[pdf-create] phase 2/6: planning sections")
sections_prompt = _build_sections_prompt(meta, request.user_message, history)
sections_result = await self._sections_planner.run(sections_prompt)
planned_sections = sections_result.output
if not planned_sections.sections:
logger.info("[pdf-create] sections planner returned empty sections")
return EditCannotDoResponse(reason="No document sections could be planned from the request.")
plan = DocumentPlan.assemble(meta, planned_sections)
# ── Phase 3: chunk ─────────────────────────────────────────────────────
chunks = _make_chunks(plan.sections)
logger.info(
"[pdf-create] phase 3/6: chunked — sections=%d chunks=%d",
len(plan.sections),
len(chunks),
)
# ── Phase 4: write in parallel, bounded ────────────────────────────────
logger.info("[pdf-create] phase 4/6: writing %d chunk(s) in parallel", len(chunks))
total_chunks = len(chunks)
semaphore = asyncio.Semaphore(_MAX_PARALLEL_WRITERS)
written_chunks: list[WrittenSections] = await asyncio.gather(
*[self._write_chunk(plan, chunk, total_chunks, semaphore) for chunk in chunks]
)
# ── Phase 5: assemble in plan order (gather preserves insertion order) ──
all_sections: list[DocumentSection] = []
for written in written_chunks:
all_sections.extend(written.sections)
logger.info("[pdf-create] phase 5/6: assembled %d sections", len(all_sections))
doc = GeneratedDocument(
title=plan.title,
subtitle=plan.subtitle,
reference_number=plan.reference_number,
style=plan.style,
sections=all_sections,
)
# ── Phase 6: render ────────────────────────────────────────────────────
logger.info("[pdf-create] phase 6/6: rendering HTML")
html = self._render(doc)
filename = _safe_filename(plan.title)
logger.info(
"[pdf-create] done — filename=%r html_bytes=%d",
filename,
len(html),
)
return EditPlanResponse(
summary=f"Created {plan.title}",
steps=[
ToolOperationStep(
tool=AgentToolId.CREATE_PDF_FROM_HTML_AGENT,
parameters=CreatePdfFromHtmlAgentParams(
html_content=html,
filename=filename,
),
)
],
)
async def _write_chunk(
self, plan: DocumentPlan, chunk: _Chunk, total_chunks: int, semaphore: asyncio.Semaphore
) -> WrittenSections:
async with semaphore:
prompt = _build_writer_prompt(plan, chunk)
result = await self._writer.run(prompt)
logger.info(
"[pdf-create] chunk %d/%d wrote %d sections",
chunk.index + 1,
total_chunks,
len(result.output.sections),
)
return result.output
def _render(self, doc: GeneratedDocument) -> str:
template = self._jinja_env.get_template("document.html.jinja2")
return template.render(doc=doc)
@@ -0,0 +1,301 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<style>
:root {
--color-bg: #ffffff;
--color-primary: #1e3a5f;
--color-subtitle: #475569;
--color-ref: #6b7280;
--color-label: #374151;
--color-body: #1a1a1a;
--color-border-light: #e2e8f0;
--color-border-heading: #cbd5e1;
--font-body: "Helvetica Neue", Arial, sans-serif;
--font-size-base: 10pt;
}
@page {
size: A4;
margin: 20mm;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: var(--font-body);
font-size: var(--font-size-base);
line-height: 1.5;
color: var(--color-body);
background: var(--color-bg);
}
/* ── Header ── */
.doc-header {
margin-bottom: 20pt;
padding-bottom: 10pt;
border-bottom: 2pt solid var(--color-primary);
}
.doc-title {
font-size: 20pt;
font-weight: 700;
color: var(--color-primary);
line-height: 1.2;
}
.doc-subtitle {
font-size: 11pt;
color: var(--color-subtitle);
margin-top: 3pt;
}
.doc-reference {
font-size: 9pt;
color: var(--color-ref);
margin-top: 4pt;
}
/* ── Sections ── */
section {
margin-bottom: 16pt;
page-break-inside: avoid;
break-inside: avoid;
}
section.line-items-section {
page-break-inside: auto;
break-inside: auto;
}
section h2 {
font-size: 11pt;
font-weight: 700;
color: var(--color-primary);
border-bottom: 0.5pt solid var(--color-border-heading);
padding-bottom: 3pt;
margin-bottom: 8pt;
}
/* ── Text ── */
.text-body p {
margin-bottom: 6pt;
}
.text-body p:last-child {
margin-bottom: 0;
}
/* ── Key-value ── */
.kv-table {
width: 100%;
border-collapse: collapse;
}
.kv-table td {
padding: 3pt 0;
vertical-align: top;
}
.kv-table td.kv-label {
font-weight: 600;
color: var(--color-label);
width: 36%;
padding-right: 10pt;
}
.kv-table td.kv-value {
color: var(--color-body);
}
/* ── Line items ── */
.line-items-table {
width: 100%;
border-collapse: collapse;
font-size: 9.5pt;
}
.line-items-table thead tr {
background-color: var(--color-primary);
color: #ffffff;
}
.line-items-table thead th {
padding: 5pt 8pt;
text-align: left;
font-weight: 600;
}
.line-items-table thead th:not(:first-child) {
text-align: right;
}
.line-items-table tbody td {
padding: 4pt 8pt;
border-bottom: 0.5pt solid var(--color-border-light);
vertical-align: top;
}
.line-items-table tbody td:not(:first-child) {
text-align: right;
}
.line-items-table tbody tr:last-child td {
border-bottom: none;
}
.line-items-table tbody tr {
page-break-inside: avoid;
break-inside: avoid;
}
.line-items-table tr.total-row td {
padding: 5pt 8pt;
font-weight: 700;
border-top: 1pt solid var(--color-primary);
}
.line-items-table tr.total-row td:not(:first-child) {
text-align: right;
}
/* ── Bullet list ── */
.bullet-list {
padding-left: 14pt;
}
.bullet-list li {
margin-bottom: 3pt;
}
.bullet-list li:last-child {
margin-bottom: 0;
}
/* ── Signature ── */
.signature-grid {
width: 100%;
margin-top: 8pt;
}
.signatory {
display: inline-block;
width: 44%;
margin-right: 5%;
margin-bottom: 8pt;
vertical-align: top;
}
.sig-line {
border-bottom: 1pt solid var(--color-label);
height: 28pt;
margin-bottom: 4pt;
}
.sig-name {
font-size: 9pt;
color: var(--color-label);
}
</style>
{%- if doc.style %}
<style>
:root {
{%- if doc.style.primary_color %}
--color-primary: {{ doc.style.primary_color }};
{%- endif %}
{%- if doc.style.background_color %}
--color-bg: {{ doc.style.background_color }};
{%- endif %}
{%- if doc.style.body_text_color %}
--color-body: {{ doc.style.body_text_color }};
--color-label: {{ doc.style.body_text_color }};
{%- endif %}
}
</style>
{%- endif %}
</head>
<body>
<div class="doc-header">
<div class="doc-title">{{ doc.title }}</div>
{%- if doc.subtitle %}
<div class="doc-subtitle">{{ doc.subtitle }}</div>
{%- endif %}
{%- if doc.reference_number %}
<div class="doc-reference">{{ doc.reference_number }}</div>
{%- endif %}
</div>
{%- for section in doc.sections %}
{%- if section.type == "text" %}
<section>
{%- if section.heading %}
<h2>{{ section.heading }}</h2>
{%- endif %}
<div class="text-body">
{%- for para in section.body.split('\n\n') %}
<p>{{ para | replace('\n', ' ') }}</p>
{%- endfor %}
</div>
</section>
{%- elif section.type == "key_value" %}
<section>
{%- if section.heading %}
<h2>{{ section.heading }}</h2>
{%- endif %}
<table class="kv-table">
<tbody>
{%- for label, value in section.pairs %}
<tr>
<td class="kv-label">{{ label }}</td>
<td class="kv-value">{{ value }}</td>
</tr>
{%- endfor %}
</tbody>
</table>
</section>
{%- elif section.type == "line_items" %}
<section class="line-items-section">
{%- if section.heading %}
<h2>{{ section.heading }}</h2>
{%- endif %}
<table class="line-items-table">
<thead>
<tr>
{%- for col in section.columns %}
<th>{{ col }}</th>
{%- endfor %}
</tr>
</thead>
<tbody>
{%- for row in section.rows %}
<tr>
{%- for cell in row %}
<td>{{ cell }}</td>
{%- endfor %}
</tr>
{%- endfor %}
{%- if section.total_row %}
<tr class="total-row">
{%- for cell in section.total_row %}
<td>{{ cell }}</td>
{%- endfor %}
</tr>
{%- endif %}
</tbody>
</table>
</section>
{%- elif section.type == "bullet_list" %}
<section>
{%- if section.heading %}
<h2>{{ section.heading }}</h2>
{%- endif %}
<ul class="bullet-list">
{%- for item in section.items %}
<li>{{ item }}</li>
{%- endfor %}
</ul>
</section>
{%- elif section.type == "signature" %}
<section>
{%- if section.heading %}
<h2>{{ section.heading }}</h2>
{%- endif %}
<div class="signature-grid">
{%- for signatory in section.signatories %}
<div class="signatory">
<div class="sig-line"></div>
<div class="sig-name">{{ signatory }}</div>
</div>
{%- endfor %}
</div>
</section>
{%- endif %}
{%- endfor %}
</body>
</html>
+16
View File
@@ -80,6 +80,15 @@ from .pdf_comments import (
PdfCommentResponse, PdfCommentResponse,
TextChunk, TextChunk,
) )
from .pdf_create import (
DocumentMeta,
DocumentSections,
PdfCreateCannotDoResponse,
PdfCreateOrchestrateResponse,
PdfCreateRequest,
PdfCreateResponse,
PdfCreateSuccessResponse,
)
from .pdf_edit import ( from .pdf_edit import (
EditCannotDoResponse, EditCannotDoResponse,
EditClarificationRequest, EditClarificationRequest,
@@ -130,6 +139,8 @@ __all__ = [
"DeleteDocumentResponse", "DeleteDocumentResponse",
"PurgeOwnerResponse", "PurgeOwnerResponse",
"Discrepancy", "Discrepancy",
"DocumentMeta",
"DocumentSections",
"DiscrepancyKind", "DiscrepancyKind",
"EditCannotDoResponse", "EditCannotDoResponse",
"EditClarificationRequest", "EditClarificationRequest",
@@ -164,6 +175,11 @@ __all__ = [
"PdfCommentRequest", "PdfCommentRequest",
"PdfCommentResponse", "PdfCommentResponse",
"PdfContentType", "PdfContentType",
"PdfCreateCannotDoResponse",
"PdfCreateOrchestrateResponse",
"PdfCreateRequest",
"PdfCreateResponse",
"PdfCreateSuccessResponse",
"PdfEditRequest", "PdfEditRequest",
"PdfEditResponse", "PdfEditResponse",
"PdfEditTerminalResponse", "PdfEditTerminalResponse",
+1 -1
View File
@@ -88,11 +88,11 @@ class SupportedCapability(StrEnum):
PDF_EDIT = "pdf_edit" PDF_EDIT = "pdf_edit"
PDF_QUESTION = "pdf_question" PDF_QUESTION = "pdf_question"
PDF_REVIEW = "pdf_review" PDF_REVIEW = "pdf_review"
PDF_CREATE = "pdf_create"
AGENT_DRAFT = "agent_draft" AGENT_DRAFT = "agent_draft"
AGENT_REVISE = "agent_revise" AGENT_REVISE = "agent_revise"
AGENT_NEXT_ACTION = "agent_next_action" AGENT_NEXT_ACTION = "agent_next_action"
MATH_AUDITOR_AGENT = "math_auditor_agent" MATH_AUDITOR_AGENT = "math_auditor_agent"
PDF_TO_MARKDOWN = "pdf_to_markdown"
class ConversationMessage(ApiModel): class ConversationMessage(ApiModel):
+226
View File
@@ -0,0 +1,226 @@
"""Contracts for the PDF Create Agent.
The agent accepts a natural-language prompt and returns a single
CREATE_PDF_FROM_HTML_AGENT plan step carrying the rendered HTML.
Pipeline:
1. PlannerAgent (smart_model) → DocumentPlan: structured skeleton, no body text.
2. Python chunks the plan by token budget.
3. SectionWriterAgents (smart_model, parallel) → WrittenSections per chunk.
4. Assembler collects sections in plan order → GeneratedDocument.
5. Jinja renders GeneratedDocument → HTML. The LLM never writes HTML.
"""
from __future__ import annotations
import re
from enum import StrEnum
from typing import Annotated, Literal
from pydantic import Field, field_validator
from stirling.models import ApiModel
from .common import ConversationMessage
from .pdf_edit import EditCannotDoResponse, EditPlanResponse
class SectionType(StrEnum):
TEXT = "text"
KEY_VALUE = "key_value"
LINE_ITEMS = "line_items"
BULLET_LIST = "bullet_list"
SIGNATURE = "signature"
class TextSection(ApiModel):
"""One or more prose paragraphs. Use for introductions, summaries, and narrative content."""
type: Literal[SectionType.TEXT] = SectionType.TEXT
heading: str | None = None
body: str = Field(description="Paragraph text. Use \\n\\n to separate paragraphs.")
class KeyValueSection(ApiModel):
"""Labelled fields. Use for contact info, dates, invoice details, and metadata."""
type: Literal[SectionType.KEY_VALUE] = SectionType.KEY_VALUE
heading: str | None = None
pairs: list[tuple[str, str]] = Field(description="List of (label, value) pairs.")
class LineItemsSection(ApiModel):
"""A table with column headers and data rows. Use for invoices, expenses, schedules."""
type: Literal[SectionType.LINE_ITEMS] = SectionType.LINE_ITEMS
heading: str | None = None
columns: list[str] = Field(description="Column header names.")
rows: list[list[str]] = Field(description="Data rows; each row must match columns in length.")
total_row: list[str] | None = None
class BulletListSection(ApiModel):
"""An unordered list. Use for requirements, responsibilities, or any enumerated items."""
type: Literal[SectionType.BULLET_LIST] = SectionType.BULLET_LIST
heading: str | None = None
items: list[str]
class SignatureSection(ApiModel):
"""Signature blocks. Use when the document requires sign-off from named parties."""
type: Literal[SectionType.SIGNATURE] = SectionType.SIGNATURE
heading: str | None = None
signatories: list[str] = Field(description="Names or roles to sign, e.g. 'John Smith, CEO'.")
type DocumentSection = Annotated[
TextSection | KeyValueSection | LineItemsSection | BulletListSection | SignatureSection,
Field(discriminator="type"),
]
# Named colour or hex only — anything else is dropped so a colour can't inject CSS into the
# <style> block (which would let WeasyPrint fetch an attacker-controlled url() → SSRF).
_SAFE_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{3,8}$|^[a-zA-Z]{1,30}$")
class DocumentStyle(ApiModel):
"""Document colours, inferred by the meta planner and rendered into the engine's Jinja
template (never sent to Java). Unsafe colours are dropped to ``None``."""
primary_color: str | None = Field(default=None)
background_color: str | None = Field(default=None)
body_text_color: str | None = Field(default=None)
@field_validator("primary_color", "background_color", "body_text_color", mode="after")
@classmethod
def _drop_unsafe_color(cls, value: str | None) -> str | None:
if value is None:
return None
return value if _SAFE_COLOR_RE.fullmatch(value) else None
class GeneratedDocument(ApiModel):
"""The full document model passed to Jinja for HTML rendering."""
title: str
subtitle: str | None = None
reference_number: str | None = None
style: DocumentStyle | None = None
sections: list[DocumentSection]
# ── Planner models ────────────────────────────────────────────────────────────────────────────────
class SectionDepth(StrEnum):
BRIEF = "brief"
STANDARD = "standard"
DETAILED = "detailed"
class PlannedSection(ApiModel):
"""One section in the document plan. Contains structure and intent — no body text."""
heading: str
type: SectionType
depth: SectionDepth
key_points: list[str]
class DocumentMeta(ApiModel):
"""Document header fields produced by the first planner call.
Contains everything except the section list. Kept deliberately flat so the
JSON schema stays small enough for grammar compilation on all model tiers.
Style is expressed as three flat optional strings rather than a nested object
for the same reason; assemble() reconstructs DocumentStyle from them.
"""
cannot_do_reason: str | None = None
title: str = ""
subtitle: str | None = None
reference_number: str | None = None
tone_brief: str = ""
shared_terms: dict[str, str] = Field(default_factory=dict)
document_context: str = ""
style_primary_color: str | None = None
style_background_color: str | None = None
style_body_text_color: str | None = None
class DocumentSections(ApiModel):
"""Section list produced by the second planner call."""
sections: list[PlannedSection] = Field(default_factory=list)
class DocumentPlan(ApiModel):
"""Assembled plan: meta + sections. Not used as a direct LLM output schema."""
cannot_do_reason: str | None = None
title: str = ""
subtitle: str | None = None
reference_number: str | None = None
tone_brief: str = ""
shared_terms: dict[str, str] = Field(default_factory=dict)
document_context: str = ""
style: DocumentStyle | None = None
sections: list[PlannedSection] = Field(default_factory=list)
@classmethod
def assemble(cls, meta: DocumentMeta, sections: DocumentSections) -> DocumentPlan:
style_fields = {
"primary_color": meta.style_primary_color,
"background_color": meta.style_background_color,
"body_text_color": meta.style_body_text_color,
}
inferred_style = DocumentStyle(**style_fields) if any(style_fields.values()) else None
return cls(
cannot_do_reason=meta.cannot_do_reason,
title=meta.title,
subtitle=meta.subtitle,
reference_number=meta.reference_number,
tone_brief=meta.tone_brief,
shared_terms=meta.shared_terms,
document_context=meta.document_context,
style=inferred_style,
sections=sections.sections,
)
class WrittenSections(ApiModel):
"""Sections produced by one section-writer agent for one chunk."""
sections: list[DocumentSection]
# ── Request/response contracts ───────────────────────────────────────────────────────────────────
class PdfCreateRequest(ApiModel):
user_message: str
conversation_history: list[ConversationMessage] = Field(default_factory=list)
class PdfCreateSuccessResponse(ApiModel):
outcome: Literal["document_created"] = "document_created"
document: GeneratedDocument
class PdfCreateCannotDoResponse(ApiModel):
outcome: Literal["cannot_do"] = "cannot_do"
reason: str
type PdfCreateResponse = Annotated[
PdfCreateSuccessResponse | PdfCreateCannotDoResponse,
Field(discriminator="outcome"),
]
type PdfCreateOrchestrateResponse = Annotated[
EditPlanResponse | EditCannotDoResponse,
Field(discriminator="outcome"),
]
@@ -1,13 +1,15 @@
"""Agent tool IDs, parameter models, and registry. """Agent tool IDs, parameter models, and registry.
tool_models.py is auto-generated from the Java OpenAPI spec. This file is its Hand-maintained counterpart to the generated tool_models.py, for engine-emitted tools
manually-maintained counterpart for tools backed by AI agent pipelines. hidden from the OpenAPI spec: AI-backed agents and deterministic conversions like HTML-to-PDF.
""" """
from __future__ import annotations from __future__ import annotations
from enum import StrEnum from enum import StrEnum
from pydantic import Field
from stirling.models.base import ApiModel from stirling.models.base import ApiModel
from stirling.models.tool_models import ParamToolModel, ToolEndpoint from stirling.models.tool_models import ParamToolModel, ToolEndpoint
@@ -15,6 +17,7 @@ from stirling.models.tool_models import ParamToolModel, ToolEndpoint
class AgentToolId(StrEnum): class AgentToolId(StrEnum):
MATH_AUDITOR_AGENT = "/api/v1/ai/tools/math-auditor-agent" MATH_AUDITOR_AGENT = "/api/v1/ai/tools/math-auditor-agent"
PDF_COMMENT_AGENT = "/api/v1/ai/tools/pdf-comment-agent" PDF_COMMENT_AGENT = "/api/v1/ai/tools/pdf-comment-agent"
CREATE_PDF_FROM_HTML_AGENT = "/api/v1/ai/tools/create-pdf-from-html-agent"
class MathAuditorAgentParams(ApiModel): class MathAuditorAgentParams(ApiModel):
@@ -25,7 +28,12 @@ class PdfCommentAgentParams(ApiModel):
prompt: str | None = None prompt: str | None = None
type AgentParamModel = MathAuditorAgentParams | PdfCommentAgentParams class CreatePdfFromHtmlAgentParams(ApiModel):
html_content: str
filename: str = Field(pattern=r"^.+\.pdf$")
type AgentParamModel = MathAuditorAgentParams | PdfCommentAgentParams | CreatePdfFromHtmlAgentParams
type AnyToolId = ToolEndpoint | AgentToolId type AnyToolId = ToolEndpoint | AgentToolId
type AnyParamModel = ParamToolModel | AgentParamModel type AnyParamModel = ParamToolModel | AgentParamModel
@@ -33,4 +41,5 @@ type AnyParamModel = ParamToolModel | AgentParamModel
AGENT_OPERATIONS: dict[AgentToolId, type[AgentParamModel]] = { AGENT_OPERATIONS: dict[AgentToolId, type[AgentParamModel]] = {
AgentToolId.MATH_AUDITOR_AGENT: MathAuditorAgentParams, AgentToolId.MATH_AUDITOR_AGENT: MathAuditorAgentParams,
AgentToolId.PDF_COMMENT_AGENT: PdfCommentAgentParams, AgentToolId.PDF_COMMENT_AGENT: PdfCommentAgentParams,
AgentToolId.CREATE_PDF_FROM_HTML_AGENT: CreatePdfFromHtmlAgentParams,
} }
+4 -1
View File
@@ -43,7 +43,10 @@ def _build_anthropic_http_client() -> httpx.AsyncClient:
have died in the pool. See ``STIRLING_HTTP_DEBUG`` traces of slice 6 have died in the pool. See ``STIRLING_HTTP_DEBUG`` traces of slice 6
on 2026-05-06 for the concrete failure mode this addresses. on 2026-05-06 for the concrete failure mode this addresses.
""" """
return httpx.AsyncClient(limits=httpx.Limits(max_keepalive_connections=0)) return httpx.AsyncClient(
limits=httpx.Limits(max_keepalive_connections=0),
timeout=httpx.Timeout(connect=30.0, read=300.0, write=30.0, pool=5.0),
)
class ConcurrencyLimitedModel(WrapperModel): class ConcurrencyLimitedModel(WrapperModel):
+564
View File
@@ -0,0 +1,564 @@
"""Tests for the chunked PdfCreateAgent pipeline.
Coverage:
1. Section model validation (each section type round-trips correctly)
2. Jinja rendering (_render produces valid HTML for each section type)
3. _safe_filename produces clean slugs
4. _make_chunks groups sections correctly by token budget
5. orchestrate() produces the correct EditPlanResponse via planner + writer mocks
6. orchestrate() returns EditCannotDoResponse when meta planner signals cannot_do
7. orchestrate() returns EditCannotDoResponse when sections planner returns empty list
"""
from __future__ import annotations
import pytest
from conftest import build_app_settings
from pydantic_ai.models.test import TestModel
from pydantic_ai.profiles import ModelProfile
from stirling.agents.pdf_create.agent import (
PdfCreateAgent,
_make_chunks,
_safe_filename,
)
from stirling.contracts import (
EditCannotDoResponse,
EditPlanResponse,
OrchestratorRequest,
)
from stirling.contracts.pdf_create import (
BulletListSection,
DocumentMeta,
DocumentSections,
DocumentStyle,
GeneratedDocument,
KeyValueSection,
LineItemsSection,
PlannedSection,
SectionDepth,
SectionType,
SignatureSection,
TextSection,
WrittenSections,
)
from stirling.models.agent_tool_models import AgentToolId, CreatePdfFromHtmlAgentParams
from stirling.services import build_runtime
from stirling.services.runtime import AppRuntime
_NATIVE_PROFILE = ModelProfile(supports_json_schema_output=True)
# ── Fixtures ──────────────────────────────────────────────────────────────────────────────────────
@pytest.fixture
def runtime() -> AppRuntime:
return build_runtime(build_app_settings())
@pytest.fixture
def agent(runtime: AppRuntime) -> PdfCreateAgent:
return PdfCreateAgent(runtime)
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────────
def _invoice_doc() -> GeneratedDocument:
return GeneratedDocument(
title="Invoice",
subtitle="Acme Corp",
reference_number="Invoice #INV-001",
sections=[
KeyValueSection(
heading="Details",
pairs=[("Date", "2026-05-06"), ("Due", "2026-06-06"), ("Currency", "USD")],
),
LineItemsSection(
heading="Line Items",
columns=["Description", "Qty", "Unit Price", "Total"],
rows=[
["Consulting services", "10", "$500.00", "$5,000.00"],
["Expenses", "1", "$200.00", "$200.00"],
],
total_row=["Total", "", "", "$5,200.00"],
),
TextSection(
heading="Payment Terms",
body="Payment is due within 30 days.\n\nPlease reference the invoice number.",
),
SignatureSection(
heading="Authorised By",
signatories=["Jane Smith, CEO", "Bob Jones, CFO"],
),
],
)
def _simple_meta() -> DocumentMeta:
return DocumentMeta(
title="Invoice",
subtitle="Acme Corp",
tone_brief="Professional business tone.",
shared_terms={"the Client": "Acme Corp"},
)
def _simple_sections() -> DocumentSections:
return DocumentSections(
sections=[
PlannedSection(
heading="Details",
type=SectionType.KEY_VALUE,
depth=SectionDepth.BRIEF,
key_points=["Date: 2026-05-06", "Due: 2026-06-06"],
),
PlannedSection(
heading="Line Items",
type=SectionType.LINE_ITEMS,
depth=SectionDepth.STANDARD,
key_points=["Consulting services, 10h, $500/h", "Expenses, $200"],
),
]
)
def _written_sections() -> WrittenSections:
return WrittenSections(
sections=[
KeyValueSection(
heading="Details",
pairs=[("Date", "2026-05-06"), ("Due", "2026-06-06")],
),
LineItemsSection(
heading="Line Items",
columns=["Description", "Qty", "Unit Price", "Total"],
rows=[["Consulting services", "10", "$500.00", "$5,000.00"]],
total_row=["Total", "", "", "$5,000.00"],
),
]
)
def _orchestrator_request(message: str = "Create an invoice for Acme Corp") -> OrchestratorRequest:
return OrchestratorRequest(
user_message=message,
files=[],
conversation_history=[],
artifacts=[],
enabled_endpoints=[],
)
# ── Section model validation ──────────────────────────────────────────────────────────────────────
def test_text_section_round_trips() -> None:
s = TextSection(heading="Summary", body="Hello\n\nWorld")
assert s.type == "text"
assert s.heading == "Summary"
assert "World" in s.body
def test_key_value_section_round_trips() -> None:
s = KeyValueSection(pairs=[("Name", "Alice"), ("Role", "Engineer")])
assert s.type == "key_value"
assert s.pairs[0] == ("Name", "Alice")
def test_line_items_section_with_total_row() -> None:
s = LineItemsSection(
columns=["Item", "Amount"],
rows=[["Widget", "$10"]],
total_row=["Total", "$10"],
)
assert s.total_row is not None
assert s.total_row[1] == "$10"
def test_line_items_section_optional_total_row() -> None:
s = LineItemsSection(columns=["Item", "Amount"], rows=[["Widget", "$10"]])
assert s.total_row is None
def test_bullet_list_section_round_trips() -> None:
s = BulletListSection(items=["Alpha", "Beta", "Gamma"])
assert s.type == "bullet_list"
assert len(s.items) == 3
def test_signature_section_round_trips() -> None:
s = SignatureSection(signatories=["Alice", "Bob"])
assert s.type == "signature"
assert "Alice" in s.signatories
def test_generated_document_optional_fields() -> None:
doc = GeneratedDocument(title="Simple Doc", sections=[TextSection(body="Hello")])
assert doc.subtitle is None
assert doc.reference_number is None
# ── Jinja rendering ───────────────────────────────────────────────────────────────────────────────
def test_render_produces_html(agent: PdfCreateAgent) -> None:
doc = _invoice_doc()
html = agent._render(doc)
assert "<!DOCTYPE html>" in html
assert "Invoice" in html
assert "INV-001" in html
def test_render_includes_all_section_types(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="All Sections",
sections=[
TextSection(body="Some prose text."),
KeyValueSection(pairs=[("Key", "Value")]),
LineItemsSection(columns=["A", "B"], rows=[["1", "2"]]),
BulletListSection(items=["item one"]),
SignatureSection(signatories=["Alice"]),
],
)
html = agent._render(doc)
assert "Some prose text." in html
assert "Key" in html and "Value" in html
assert "<th>" in html
assert "item one" in html
assert "Alice" in html
def test_render_escapes_html_in_content(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="XSS Test",
sections=[TextSection(body="<script>alert('xss')</script>")],
)
html = agent._render(doc)
assert "<script>" not in html
assert "&lt;script&gt;" in html
def test_render_total_row_present(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="Table",
sections=[
LineItemsSection(
columns=["Item", "Total"],
rows=[["Widget", "$10"]],
total_row=["Total", "$10"],
)
],
)
html = agent._render(doc)
assert "total-row" in html
def test_render_no_total_row_skips_tfoot(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="Table",
sections=[LineItemsSection(columns=["Item"], rows=[["Widget"]])],
)
html = agent._render(doc)
assert "<tfoot>" not in html
def test_render_subtitle_and_reference(agent: PdfCreateAgent) -> None:
doc = GeneratedDocument(
title="My Doc",
subtitle="Subtitle Here",
reference_number="REF-42",
sections=[TextSection(body="Content.")],
)
html = agent._render(doc)
assert "Subtitle Here" in html
assert "REF-42" in html
# ── _safe_filename ────────────────────────────────────────────────────────────────────────────────
def test_safe_filename_basic() -> None:
assert _safe_filename("My Invoice") == "my-invoice.pdf"
def test_safe_filename_strips_special_chars() -> None:
assert _safe_filename("Report: Q1/2026!") == "report-q12026.pdf"
def test_safe_filename_empty_title() -> None:
assert _safe_filename("!!!") == "document.pdf"
# ── _make_chunks ──────────────────────────────────────────────────────────────────────────────────
def _planned(heading: str, depth: SectionDepth) -> PlannedSection:
return PlannedSection(
heading=heading,
type=SectionType.TEXT,
depth=depth,
key_points=["placeholder"],
)
def test_make_chunks_empty_returns_empty() -> None:
assert _make_chunks([]) == []
def test_make_chunks_single_section_is_one_chunk() -> None:
chunks = _make_chunks([_planned("Intro", SectionDepth.STANDARD)])
assert len(chunks) == 1
assert chunks[0].index == 0
assert len(chunks[0].sections) == 1
assert chunks[0].context_before is None
assert chunks[0].context_after is None
def test_make_chunks_groups_under_ceiling() -> None:
# 5 × STANDARD (550 each) = 2750 — fits in one chunk under ceiling of 3000
sections = [_planned(f"Section {i}", SectionDepth.STANDARD) for i in range(5)]
chunks = _make_chunks(sections)
assert len(chunks) == 1
assert len(chunks[0].sections) == 5
def test_make_chunks_splits_when_over_ceiling() -> None:
# 6 × STANDARD (550 each) = 3300 — must split (3000 ceiling)
# First chunk: 5 sections (2750), second: 1 section (550)
sections = [_planned(f"Section {i}", SectionDepth.STANDARD) for i in range(6)]
chunks = _make_chunks(sections)
assert len(chunks) == 2
assert len(chunks[0].sections) == 5
assert len(chunks[1].sections) == 1
def test_make_chunks_oversized_section_gets_own_chunk() -> None:
# DETAILED (1200) + 3×STANDARD (1650) = 2850; adding a 4th STANDARD (550) = 3400 > 3000.
# So first chunk holds DETAILED + 3 STANDARDs; last STANDARD spills to chunk 2.
sections = [
_planned("Big Table", SectionDepth.DETAILED),
_planned("Terms", SectionDepth.STANDARD),
_planned("Notes", SectionDepth.STANDARD),
_planned("Extra", SectionDepth.STANDARD),
_planned("Appendix", SectionDepth.STANDARD),
]
chunks = _make_chunks(sections)
assert len(chunks) == 2
assert chunks[0].sections[0].heading == "Big Table"
assert len(chunks[0].sections) == 4
assert chunks[1].sections[0].heading == "Appendix"
def test_make_chunks_neighbour_context() -> None:
# 6 × STANDARD (550 each) = 3300 → splits into chunk of 5 (2750) + chunk of 1 (550)
many = [_planned(f"S{i}", SectionDepth.STANDARD) for i in range(5)]
many.append(_planned("Last", SectionDepth.STANDARD))
chunks = _make_chunks(many)
assert len(chunks) == 2
assert chunks[0].context_after is not None
assert chunks[1].context_before is not None
assert chunks[0].context_before is None
assert chunks[1].context_after is None
def test_make_chunks_preserves_section_order() -> None:
headings = [f"Section {i}" for i in range(8)]
sections = [_planned(h, SectionDepth.STANDARD) for h in headings]
chunks = _make_chunks(sections)
reassembled = [s.heading for chunk in chunks for s in chunk.sections]
assert reassembled == headings
# ── orchestrate() ─────────────────────────────────────────────────────────────────────────────────
@pytest.mark.anyio
async def test_orchestrate_returns_plan_step(agent: PdfCreateAgent) -> None:
meta = _simple_meta()
sections = _simple_sections()
written = _written_sections()
with (
agent._meta_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=meta.model_dump_json())
),
agent._sections_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=sections.model_dump_json())
),
agent._writer.override(model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=written.model_dump_json())),
):
result = await agent.orchestrate(_orchestrator_request())
assert isinstance(result, EditPlanResponse)
assert len(result.steps) == 1
step = result.steps[0]
assert step.tool == AgentToolId.CREATE_PDF_FROM_HTML_AGENT
assert isinstance(step.parameters, CreatePdfFromHtmlAgentParams)
assert step.parameters.filename.endswith(".pdf")
assert "<!DOCTYPE html>" in step.parameters.html_content
assert "Invoice" in step.parameters.html_content
@pytest.mark.anyio
async def test_orchestrate_cannot_do_from_planner(agent: PdfCreateAgent) -> None:
cannot_do_meta = DocumentMeta(cannot_do_reason="This is not a document creation request.")
with agent._meta_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=cannot_do_meta.model_dump_json())
):
result = await agent.orchestrate(_orchestrator_request("what is 2+2?"))
assert isinstance(result, EditCannotDoResponse)
assert "not a document" in result.reason
@pytest.mark.anyio
async def test_orchestrate_empty_sections_returns_cannot_do(agent: PdfCreateAgent) -> None:
meta = DocumentMeta(title="Empty", tone_brief=".")
empty_sections = DocumentSections(sections=[])
with (
agent._meta_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=meta.model_dump_json())
),
agent._sections_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=empty_sections.model_dump_json())
),
):
result = await agent.orchestrate(_orchestrator_request("do the thing"))
assert isinstance(result, EditCannotDoResponse)
@pytest.mark.anyio
async def test_orchestrate_assembles_multiple_chunks(agent: PdfCreateAgent) -> None:
"""Two chunks of written sections are assembled in order into one document."""
meta = DocumentMeta(title="Multi-Chunk Doc", tone_brief="Formal.")
sections = DocumentSections(
sections=[
PlannedSection(heading="Intro", type=SectionType.TEXT, depth=SectionDepth.BRIEF, key_points=["x"]),
PlannedSection(
heading="Details",
type=SectionType.KEY_VALUE,
depth=SectionDepth.BRIEF,
key_points=["y"],
),
]
)
# The writer override is shared across all parallel calls, so use a combined
# WrittenSections that contains all sections — both chunks will return the same
# payload and we verify the final HTML contains the expected content.
combined = WrittenSections(
sections=[
TextSection(heading="Intro", body="Introduction text."),
KeyValueSection(heading="Details", pairs=[("Key", "Value")]),
]
)
with (
agent._meta_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=meta.model_dump_json())
),
agent._sections_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=sections.model_dump_json())
),
agent._writer.override(model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=combined.model_dump_json())),
):
result = await agent.orchestrate(_orchestrator_request("Create a multi-chunk doc"))
assert isinstance(result, EditPlanResponse)
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
assert "Introduction text." in html
assert "Details" in html
# ── Style inference ───────────────────────────────────────────────────────────────────────────────
@pytest.mark.anyio
async def test_orchestrate_applies_planner_inferred_style(agent: PdfCreateAgent) -> None:
"""Style extracted by the meta planner is applied to the rendered HTML."""
meta = DocumentMeta(
title="Styled Doc",
tone_brief="Professional.",
style_primary_color="magenta",
)
sections = _simple_sections()
written = _written_sections()
with (
agent._meta_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=meta.model_dump_json())
),
agent._sections_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=sections.model_dump_json())
),
agent._writer.override(model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=written.model_dump_json())),
):
result = await agent.orchestrate(_orchestrator_request("Make an invoice, magenta styling"))
assert isinstance(result, EditPlanResponse)
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
assert "magenta" in html
def test_render_applies_style(agent: PdfCreateAgent) -> None:
"""DocumentStyle fields are injected as CSS custom properties in the rendered HTML."""
doc = GeneratedDocument(
title="Styled",
sections=[TextSection(body="Content.")],
style=DocumentStyle(primary_color="magenta", background_color="#111111"),
)
html = agent._render(doc)
assert "--color-primary: magenta" in html
assert "--color-bg: #111111" in html
def test_document_style_drops_unsafe_colors() -> None:
"""Unsafe colours (not named/hex) are dropped, closing the <style> url() injection."""
safe = DocumentStyle(primary_color="navy", background_color="#1e3a5f", body_text_color="#fff")
assert (safe.primary_color, safe.background_color, safe.body_text_color) == (
"navy",
"#1e3a5f",
"#fff",
)
unsafe = DocumentStyle(
primary_color="red; background: url(http://evil.test/steal)",
background_color="expression(alert(1))",
body_text_color="navy; }",
)
assert unsafe.primary_color is None
assert unsafe.background_color is None
assert unsafe.body_text_color is None
# A trailing newline must not slip a value through (fullmatch, not $-before-newline).
assert DocumentStyle(primary_color="navy\n").primary_color is None
@pytest.mark.anyio
async def test_orchestrate_drops_unsafe_planner_color(agent: PdfCreateAgent) -> None:
"""An unsafe colour inferred by the meta planner never reaches the rendered HTML."""
meta = DocumentMeta(
title="Doc",
tone_brief="Professional.",
style_primary_color="blue; background: url(http://evil.test/)",
)
sections = _simple_sections()
written = _written_sections()
with (
agent._meta_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=meta.model_dump_json())
),
agent._sections_planner.override(
model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=sections.model_dump_json())
),
agent._writer.override(model=TestModel(profile=_NATIVE_PROFILE, custom_output_text=written.model_dump_json())),
):
result = await agent.orchestrate(_orchestrator_request("make it blue"))
assert isinstance(result, EditPlanResponse)
html = result.steps[0].parameters.html_content # type: ignore[union-attr]
assert "evil.test" not in html
assert "url(" not in html
+2
View File
@@ -604,6 +604,7 @@ version = "0.1.0"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "fastapi" }, { name = "fastapi" },
{ name = "jinja2" },
{ name = "opentelemetry-sdk" }, { name = "opentelemetry-sdk" },
{ name = "pgvector" }, { name = "pgvector" },
{ name = "posthog" }, { name = "posthog" },
@@ -630,6 +631,7 @@ dev = [
[package.metadata] [package.metadata]
requires-dist = [ requires-dist = [
{ name = "fastapi", specifier = ">=0.116.0" }, { name = "fastapi", specifier = ">=0.116.0" },
{ name = "jinja2", specifier = ">=3.1.0" },
{ name = "opentelemetry-sdk", specifier = ">=1.39.0" }, { name = "opentelemetry-sdk", specifier = ">=1.39.0" },
{ name = "pgvector", specifier = ">=0.3.6" }, { name = "pgvector", specifier = ">=0.3.6" },
{ name = "posthog", specifier = ">=3.0.0" }, { name = "posthog", specifier = ">=3.0.0" },
+1 -1
View File
@@ -1,5 +1,5 @@
<!doctype html> <!doctype html>
<html lang="en-GB"> <html lang="en-US">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<base href="%BASE_URL%" /> <base href="%BASE_URL%" />
File diff suppressed because it is too large Load Diff
@@ -11,6 +11,7 @@ export function useChat() {
progress: null, progress: null,
progressLog: [] as never[], progressLog: [] as never[],
sendMessage: async (_content: string) => {}, sendMessage: async (_content: string) => {},
cancelMessage: () => {},
clearChat: () => {}, clearChat: () => {},
}; };
} }
@@ -243,7 +243,7 @@ const LanguageSelector: React.FC<LanguageSelectorProps> = ({
const currentLanguage = const currentLanguage =
supportedLanguages[i18n.language as keyof typeof supportedLanguages] || supportedLanguages[i18n.language as keyof typeof supportedLanguages] ||
supportedLanguages["en-GB"] || supportedLanguages["en-US"] ||
"English"; // Fallback if supportedLanguages lookup fails "English"; // Fallback if supportedLanguages lookup fails
// Hide the language selector if there's only one language option // Hide the language selector if there's only one language option
+10 -9
View File
@@ -5,7 +5,8 @@ import TomlBackend from "@app/i18n/tomlBackend";
// Define supported languages (based on your existing translations) // Define supported languages (based on your existing translations)
export const supportedLanguages = { export const supportedLanguages = {
"en-GB": "English", "en-US": "English (US)",
"en-GB": "English (UK)",
"ar-AR": "العربية", "ar-AR": "العربية",
"az-AZ": "Azərbaycan Dili", "az-AZ": "Azərbaycan Dili",
"bg-BG": "Български", "bg-BG": "Български",
@@ -72,7 +73,7 @@ i18n
.use(LanguageDetector) .use(LanguageDetector)
.use(initReactI18next) .use(initReactI18next)
.init({ .init({
fallbackLng: "en-GB", fallbackLng: "en-US",
supportedLngs: Object.keys(supportedLanguages), supportedLngs: Object.keys(supportedLanguages),
load: "currentOnly", load: "currentOnly",
nonExplicitSupportedLngs: false, nonExplicitSupportedLngs: false,
@@ -100,8 +101,8 @@ i18n
order: ["localStorage", "navigator", "htmlTag"], order: ["localStorage", "navigator", "htmlTag"],
caches: [], // Don't cache auto-detected language - only cache when user manually selects caches: [], // Don't cache auto-detected language - only cache when user manually selects
convertDetectedLanguage: (lng: string) => { convertDetectedLanguage: (lng: string) => {
// Map en and en-US to en-GB // Map bare en to en-US
if (lng === "en" || lng === "en-US") return "en-GB"; if (lng === "en") return "en-US";
return lng; return lng;
}, },
}, },
@@ -154,7 +155,7 @@ export function normalizeLanguageCode(languageCode: string): string {
} }
/** /**
* Convert language codes to underscore format (e.g., en-GB → en_GB) * Convert language codes to underscore format (e.g., en-US → en_US)
* Used for backend API communication which expects underscore format * Used for backend API communication which expects underscore format
*/ */
export function toUnderscoreFormat(languageCode: string): string { export function toUnderscoreFormat(languageCode: string): string {
@@ -212,7 +213,7 @@ export function setUserLanguage(language: string): void {
* Updates the supported languages list dynamically based on config * Updates the supported languages list dynamically based on config
* If configLanguages is null/empty, all languages remain available * If configLanguages is null/empty, all languages remain available
* Otherwise, only the specified languages are enabled with the first valid * Otherwise, only the specified languages are enabled with the first valid
* option (preferring en-GB when present) used as the fallback language. * option (preferring en-US when present) used as the fallback language.
* *
* @param configLanguages - Optional array of language codes from server config (ui.languages) * @param configLanguages - Optional array of language codes from server config (ui.languages)
* @param defaultLocale - Optional default language for new users (system.defaultLocale) * @param defaultLocale - Optional default language for new users (system.defaultLocale)
@@ -248,12 +249,12 @@ export function updateSupportedLanguages(
return; return;
} }
// Determine fallback: prefer validDefault if in the list, then en-GB, then first valid language // Determine fallback: prefer validDefault if in the list, then en-US, then first valid language
const fallback = const fallback =
validDefault && validLanguages.includes(validDefault) validDefault && validLanguages.includes(validDefault)
? validDefault ? validDefault
: validLanguages.includes("en-GB") : validLanguages.includes("en-US")
? "en-GB" ? "en-US"
: validLanguages[0]; : validLanguages[0];
i18n.options.supportedLngs = validLanguages; i18n.options.supportedLngs = validLanguages;
@@ -154,8 +154,8 @@ export async function mockAppApis(
email: "[email protected]", email: "[email protected]",
roles: ["ROLE_USER"], roles: ["ROLE_USER"],
}, },
languages = ["en-GB"], languages = ["en-US"],
defaultLocale = "en-GB", defaultLocale = "en-US",
endpointsAvailability = {}, endpointsAvailability = {},
backendStatus = "UP", backendStatus = "UP",
} = opts; } = opts;
@@ -6,9 +6,9 @@ import { parse } from "smol-toml";
const REPO_ROOT = path.join(__dirname, "../../../.."); const REPO_ROOT = path.join(__dirname, "../../../..");
const SRC_ROOT = path.join(__dirname, "../.."); const SRC_ROOT = path.join(__dirname, "../..");
const EN_GB_FILE = path.join( const EN_US_FILE = path.join(
__dirname, __dirname,
"../../../public/locales/en-GB/translation.toml", "../../../public/locales/en-US/translation.toml",
); );
const IGNORED_DIRS = new Set(["tests", "__mocks__"]); const IGNORED_DIRS = new Set(["tests", "__mocks__"]);
@@ -174,14 +174,14 @@ const extractKeys = (file: string): FoundKey[] => {
describe("Missing translation coverage", () => { describe("Missing translation coverage", () => {
test( test(
"fails if any en-GB translation key used in source is missing", "fails if any en-US translation key used in source is missing",
{ timeout: 10000 }, { timeout: 10000 },
() => { () => {
expect(fs.existsSync(EN_GB_FILE)).toBe(true); expect(fs.existsSync(EN_US_FILE)).toBe(true);
const localeContent = fs.readFileSync(EN_GB_FILE, "utf8"); const localeContent = fs.readFileSync(EN_US_FILE, "utf8");
const enGb = parse(localeContent); const enUs = parse(localeContent);
const availableKeys = flattenKeys(enGb); const availableKeys = flattenKeys(enUs);
const usedKeys = listSourceFiles() const usedKeys = listSourceFiles()
.flatMap(extractKeys) .flatMap(extractKeys)
@@ -211,7 +211,7 @@ describe("Missing translation coverage", () => {
// Output errors in GitHub Annotations format so they appear tagged in the code in CI // Output errors in GitHub Annotations format so they appear tagged in the code in CI
for (const { key, fallback, file, line, column } of annotations) { for (const { key, fallback, file, line, column } of annotations) {
process.stderr.write( process.stderr.write(
`::error file=${file},line=${line},col=${column}::Missing en-GB translation for ${key} (${fallback})\n`, `::error file=${file},line=${line},col=${column}::Missing en-US translation for ${key} (${fallback})\n`,
); );
} }
@@ -28,8 +28,8 @@ test.describe("OAuth/SSO login buttons", () => {
route.fulfill({ route.fulfill({
json: { json: {
enableLogin: true, enableLogin: true,
languages: ["en-GB"], languages: ["en-US"],
defaultLocale: "en-GB", defaultLocale: "en-US",
oauth2: { oauth2: {
enabled: true, enabled: true,
providers: [ providers: [
@@ -6,9 +6,9 @@ import { parse } from "smol-toml";
const REPO_ROOT = path.join(__dirname, "../../../.."); const REPO_ROOT = path.join(__dirname, "../../../..");
const SRC_ROOT = path.join(__dirname, "../.."); const SRC_ROOT = path.join(__dirname, "../..");
const EN_GB_FILE = path.join( const EN_US_FILE = path.join(
__dirname, __dirname,
"../../../public/locales/en-GB/translation.toml", "../../../public/locales/en-US/translation.toml",
); );
const IGNORED_DIRS = new Set(["tests", "__mocks__"]); const IGNORED_DIRS = new Set(["tests", "__mocks__"]);
@@ -151,13 +151,13 @@ const isIgnored = (key: string): boolean => {
describe("Unused translation coverage", () => { describe("Unused translation coverage", () => {
test( test(
"fails if any en-GB translation key has no source references", "fails if any en-US translation key has no source references",
{ timeout: 30_000 }, { timeout: 30_000 },
() => { () => {
expect(fs.existsSync(EN_GB_FILE)).toBe(true); expect(fs.existsSync(EN_US_FILE)).toBe(true);
const enGb = parse(fs.readFileSync(EN_GB_FILE, "utf8")); const enUs = parse(fs.readFileSync(EN_US_FILE, "utf8"));
const availableKeys = Array.from(flattenKeys(enGb)); const availableKeys = Array.from(flattenKeys(enUs));
expect(availableKeys.length).toBeGreaterThan(100); // sanity check expect(availableKeys.length).toBeGreaterThan(100); // sanity check
const sourceFiles = listSourceFiles(); const sourceFiles = listSourceFiles();
@@ -184,20 +184,20 @@ describe("Unused translation coverage", () => {
}); });
const localeRelative = path const localeRelative = path
.relative(REPO_ROOT, EN_GB_FILE) .relative(REPO_ROOT, EN_US_FILE)
.replace(/\\/g, "/"); .replace(/\\/g, "/");
// GitHub Annotations format so unused keys show up tagged on the // GitHub Annotations format so unused keys show up tagged on the
// translation file in CI. // translation file in CI.
for (const key of unused) { for (const key of unused) {
process.stderr.write( process.stderr.write(
`::error file=${localeRelative}::Unused en-GB translation: ${key}\n`, `::error file=${localeRelative}::Unused en-US translation: ${key}\n`,
); );
} }
expect( expect(
unused, unused,
`Found ${unused.length} unused en-GB translation key(s). ` + `Found ${unused.length} unused en-US translation key(s). ` +
`Remove them from ${localeRelative}, or (if the usage is too ` + `Remove them from ${localeRelative}, or (if the usage is too ` +
`dynamic for the heuristic to spot) add to IGNORED_KEY_PATTERNS ` + `dynamic for the heuristic to spot) add to IGNORED_KEY_PATTERNS ` +
`in this test.`, `in this test.`,
@@ -13,7 +13,16 @@ const languageDefinitions: LanguageDefinition[] = [
{ {
ocrCode: "eng", ocrCode: "eng",
displayName: "English", displayName: "English",
browserCodes: ["en", "en-GB", "en-AU", "en-CA", "en-IE", "en-NZ", "en-ZA"], browserCodes: [
"en",
"en-US",
"en-GB",
"en-AU",
"en-CA",
"en-IE",
"en-NZ",
"en-ZA",
],
}, },
// Spanish // Spanish
@@ -908,12 +917,12 @@ languageDefinitions.forEach((lang) => {
* Maps a browser language code to an OCR language code * Maps a browser language code to an OCR language code
* Handles exact matches and similar language fallbacks * Handles exact matches and similar language fallbacks
* *
* @param browserLanguage - The browser language code (e.g., 'en-GB', 'fr-FR') * @param browserLanguage - The browser language code (e.g., 'en-US', 'fr-FR')
* @returns OCR language code if found, null if no match * @returns OCR language code if found, null if no match
* *
* @example * @example
* mapBrowserLanguageToOcr('de-DE') // Returns 'deu' * mapBrowserLanguageToOcr('de-DE') // Returns 'deu'
* mapBrowserLanguageToOcr('en-GB') // Returns 'eng' * mapBrowserLanguageToOcr('en-US') // Returns 'eng'
* mapBrowserLanguageToOcr('zh-CN') // Returns 'chi_sim' * mapBrowserLanguageToOcr('zh-CN') // Returns 'chi_sim'
*/ */
export function mapBrowserLanguageToOcr( export function mapBrowserLanguageToOcr(
@@ -940,7 +949,7 @@ export function mapBrowserLanguageToOcr(
if (match) return match; if (match) return match;
} }
// Try base language code (e.g., 'en' from 'en-GB') // Try base language code (e.g., 'en' from 'en-US')
const baseLanguage = normalizedInput.split("-")[0]; const baseLanguage = normalizedInput.split("-")[0];
const baseMatch = browserToOcrMap.get(baseLanguage); const baseMatch = browserToOcrMap.get(baseLanguage);
if (baseMatch) return baseMatch; if (baseMatch) return baseMatch;
@@ -1002,7 +1011,7 @@ export function getBrowserLanguagesForOcr(ocrCode: string): string[] {
* *
* @example * @example
* getAutoOcrLanguage('de-DE') // Returns ['deu'] * getAutoOcrLanguage('de-DE') // Returns ['deu']
* getAutoOcrLanguage('en-GB') // Returns ['eng'] * getAutoOcrLanguage('en-US') // Returns ['eng']
* getAutoOcrLanguage('unknown') // Returns [] * getAutoOcrLanguage('unknown') // Returns []
*/ */
export function getAutoOcrLanguage(currentLanguage: string): string[] { export function getAutoOcrLanguage(currentLanguage: string): string[] {
@@ -361,6 +361,7 @@ interface ChatContextValue {
/** Ordered log of every progress event for the current in-flight request. */ /** Ordered log of every progress event for the current in-flight request. */
progressLog: AiWorkflowProgress[]; progressLog: AiWorkflowProgress[];
sendMessage: (content: string) => Promise<void>; sendMessage: (content: string) => Promise<void>;
cancelMessage: () => void;
/** Abort any in-flight request and reset the chat to an empty conversation. */ /** Abort any in-flight request and reset the chat to an empty conversation. */
clearChat: () => void; clearChat: () => void;
} }
@@ -422,38 +423,42 @@ export function ChatProvider({ children }: { children: ReactNode }) {
const files = await Promise.all(descriptors.map(downloadFile)); const files = await Promise.all(descriptors.map(downloadFile));
const operation: ToolOperation = {
toolId: "ai-workflow",
timestamp: Date.now(),
};
const isVersionMapping =
sourceStubs.length > 0 && files.length === sourceStubs.length;
const stubs = files.map((file, i) =>
isVersionMapping
? createChildStub(sourceStubs[i], operation, file)
: createNewStirlingFileStub(file),
);
const stirlingFiles = files.map((file, i) =>
createStirlingFile(file, stubs[i].id),
);
if (sourceStubs.length > 0) { if (sourceStubs.length > 0) {
// Always consume the inputs so merge/split inputs are removed from the workbench. // Always consume the inputs so merge/split inputs are removed from the workbench.
// For 1:1 operations (rotate, compress) the outputs carry the version chain; for // For 1:1 operations (rotate, compress) the outputs carry the version chain; for
// merge/split they're fresh roots. // merge/split they're fresh roots.
const operation: ToolOperation = {
toolId: "ai-workflow",
timestamp: Date.now(),
};
const isVersionMapping = files.length === sourceStubs.length;
const stubs = files.map((file, i) =>
isVersionMapping
? createChildStub(sourceStubs[i], operation, file)
: createNewStirlingFileStub(file),
);
const stirlingFiles = files.map((file, i) =>
createStirlingFile(file, stubs[i].id),
);
await fileActions.consumeFiles( await fileActions.consumeFiles(
sourceStubs.map((s) => s.id), sourceStubs.map((s) => s.id),
stirlingFiles, stirlingFiles,
stubs, stubs,
); );
} else { } else {
// No inputs were provided (unlikely for completed workflows, but handle it safely). // No inputs: pass raw files so addFiles assigns consistent IDs. Pre-assigning stub IDs
// here would cause a fileId mismatch in filesRef, making getFiles() clone the file
// on every render and breaking useFileWithUrl's memoisation (continuous PDF reloads).
await fileActions.addFiles(files, { selectFiles: true }); await fileActions.addFiles(files, { selectFiles: true });
} }
}, },
[fileActions, downloadFile], [fileActions, downloadFile],
); );
const cancelMessage = useCallback(() => {
abortRef.current?.abort();
}, []);
const clearChat = useCallback(() => { const clearChat = useCallback(() => {
abortRef.current?.abort(); abortRef.current?.abort();
abortRef.current = null; abortRef.current = null;
@@ -494,7 +499,6 @@ export function ChatProvider({ children }: { children: ReactNode }) {
formData.append(`conversationHistory[${i}].role`, message.role); formData.append(`conversationHistory[${i}].role`, message.role);
formData.append(`conversationHistory[${i}].content`, message.content); formData.append(`conversationHistory[${i}].content`, message.content);
}); });
const response = await fetch( const response = await fetch(
`${getApiBaseUrl()}/api/v1/ai/orchestrate/stream`, `${getApiBaseUrl()}/api/v1/ai/orchestrate/stream`,
{ {
@@ -637,6 +641,7 @@ export function ChatProvider({ children }: { children: ReactNode }) {
progress: state.progress, progress: state.progress,
progressLog: state.progressLog, progressLog: state.progressLog,
sendMessage, sendMessage,
cancelMessage,
clearChat, clearChat,
}} }}
> >
@@ -625,7 +625,7 @@ export default function AdminGeneralSection() {
data={defaultLocaleOptions} data={defaultLocaleOptions}
searchable searchable
clearable clearable
placeholder="en_GB" placeholder="en_US"
comboboxProps={{ zIndex: Z_INDEX_CONFIG_MODAL }} comboboxProps={{ zIndex: Z_INDEX_CONFIG_MODAL }}
disabled={!loginEnabled} disabled={!loginEnabled}
/> />
@@ -26,10 +26,10 @@ vi.mock("react-i18next", () => ({
// Mock i18n module to avoid initialization // Mock i18n module to avoid initialization
vi.mock("@app/i18n", () => ({ vi.mock("@app/i18n", () => ({
updateSupportedLanguages: vi.fn(), updateSupportedLanguages: vi.fn(),
supportedLanguages: { "en-GB": "English" }, supportedLanguages: { "en-US": "English (US)" },
rtlLanguages: [], rtlLanguages: [],
default: { default: {
language: "en-GB", language: "en-US",
changeLanguage: vi.fn(), changeLanguage: vi.fn(),
options: {}, options: {},
}, },
@@ -366,32 +366,32 @@ export function ChatProvider({ children }: { children: ReactNode }) {
const files = await Promise.all(descriptors.map(downloadFile)); const files = await Promise.all(descriptors.map(downloadFile));
const operation: ToolOperation = {
toolId: "ai-workflow",
timestamp: Date.now(),
};
const isVersionMapping =
sourceStubs.length > 0 && files.length === sourceStubs.length;
const stubs = files.map((file, i) =>
isVersionMapping
? createChildStub(sourceStubs[i], operation, file)
: createNewStirlingFileStub(file),
);
const stirlingFiles = files.map((file, i) =>
createStirlingFile(file, stubs[i].id),
);
if (sourceStubs.length > 0) { if (sourceStubs.length > 0) {
// Always consume the inputs so merge/split inputs are removed from the workbench. // Always consume the inputs so merge/split inputs are removed from the workbench.
// For 1:1 operations (rotate, compress) the outputs carry the version chain; for // For 1:1 operations (rotate, compress) the outputs carry the version chain; for
// merge/split they're fresh roots. // merge/split they're fresh roots.
const operation: ToolOperation = {
toolId: "ai-workflow",
timestamp: Date.now(),
};
const isVersionMapping = files.length === sourceStubs.length;
const stubs = files.map((file, i) =>
isVersionMapping
? createChildStub(sourceStubs[i], operation, file)
: createNewStirlingFileStub(file),
);
const stirlingFiles = files.map((file, i) =>
createStirlingFile(file, stubs[i].id),
);
await fileActions.consumeFiles( await fileActions.consumeFiles(
sourceStubs.map((s) => s.id), sourceStubs.map((s) => s.id),
stirlingFiles, stirlingFiles,
stubs, stubs,
); );
} else { } else {
// No inputs were provided (unlikely for completed workflows, but handle it safely). // No inputs: pass raw files so addFiles assigns consistent IDs. Pre-assigning stub IDs
// here would cause a fileId mismatch in filesRef, making getFiles() clone the file
// on every render and breaking useFileWithUrl's memoisation (continuous PDF reloads).
await fileActions.addFiles(files, { selectFiles: true }); await fileActions.addFiles(files, { selectFiles: true });
} }
}, },
+1 -1
View File
@@ -1,5 +1,5 @@
<!doctype html> <!doctype html>
<html lang="en-GB"> <html lang="en-US">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+2 -2
View File
@@ -3,7 +3,7 @@ A script to update language progress status in README.md based on
frontend locale TOML file comparisons. frontend locale TOML file comparisons.
This script compares the default (reference) TOML file, This script compares the default (reference) TOML file,
`frontend/editor/public/locales/en-GB/translation.toml`, with other translation `frontend/editor/public/locales/en-US/translation.toml`, with other translation
files in `frontend/editor/public/locales/*/translation.toml`. files in `frontend/editor/public/locales/*/translation.toml`.
It determines how many keys are fully translated and automatically updates It determines how many keys are fully translated and automatically updates
progress badges in the `README.md`. progress badges in the `README.md`.
@@ -346,7 +346,7 @@ def main() -> None:
# Project layout assumptions # Project layout assumptions
cwd = os.getcwd() cwd = os.getcwd()
locales_dir = os.path.join(cwd, "frontend", "editor", "public", "locales") locales_dir = os.path.join(cwd, "frontend", "editor", "public", "locales")
reference_file = os.path.join(locales_dir, "en-GB", "translation.toml") reference_file = os.path.join(locales_dir, "en-US", "translation.toml")
scripts_directory = os.path.join(cwd, "scripts") scripts_directory = os.path.join(cwd, "scripts")
translation_state_file = os.path.join(scripts_directory, "ignore_translation.toml") translation_state_file = os.path.join(scripts_directory, "ignore_translation.toml")
+21 -21
View File
@@ -1,6 +1,6 @@
# Translation Management Scripts # Translation Management Scripts
This directory contains Python scripts for managing frontend translations in Stirling PDF. These tools help analyze, merge, validate, and manage translations against the en-GB golden truth file. This directory contains Python scripts for managing frontend translations in Stirling PDF. These tools help analyze, merge, validate, and manage translations against the en-US golden truth file.
## Current Format: TOML ## Current Format: TOML
@@ -33,7 +33,7 @@ python3 scripts/translations/auto_translate.py es-ES --no-cleanup
4. Validates placeholders are preserved 4. Validates placeholders are preserved
5. Merges translated batches 5. Merges translated batches
6. Applies translations to language file 6. Applies translations to language file
7. Beautifies structure to match en-GB 7. Beautifies structure to match en-US
8. Cleans up temporary files 8. Cleans up temporary files
9. Reports final completion percentage 9. Reports final completion percentage
@@ -86,7 +86,7 @@ python scripts/translations/json_validator.py --all-batches ar_AR --quiet
- Trailing commas before closing braces - Trailing commas before closing braces
#### `validate_placeholders.py` #### `validate_placeholders.py`
Validates that translation files have correct placeholders matching en-GB (source of truth). Validates that translation files have correct placeholders matching en-US (source of truth).
**Usage:** **Usage:**
```bash ```bash
@@ -105,12 +105,12 @@ python scripts/translations/validate_placeholders.py --json
**Features:** **Features:**
- Detects missing placeholders (e.g., {n}, {total}, {filename}) - Detects missing placeholders (e.g., {n}, {total}, {filename})
- Detects extra placeholders not in en-GB - Detects extra placeholders not in en-US
- Shows exact keys and text where issues occur - Shows exact keys and text where issues occur
- Exit code 1 if issues found (good for CI/CD) - Exit code 1 if issues found (good for CI/CD)
#### `validate_json_structure.py` #### `validate_json_structure.py`
Validates JSON structure and key consistency with en-GB. Validates JSON structure and key consistency with en-US.
**Usage:** **Usage:**
```bash ```bash
@@ -130,7 +130,7 @@ python scripts/translations/validate_json_structure.py --json
**Features:** **Features:**
- Validates JSON syntax - Validates JSON syntax
- Detects missing keys (not translated yet) - Detects missing keys (not translated yet)
- Detects extra keys (not in en-GB, should be removed) - Detects extra keys (not in en-US, should be removed)
- Reports key counts and structure differences - Reports key counts and structure differences
- Exit code 1 if issues found (good for CI/CD) - Exit code 1 if issues found (good for CI/CD)
@@ -160,21 +160,21 @@ python scripts/translations/translation_analyzer.py --format json
**Features:** **Features:**
- Finds missing translation keys - Finds missing translation keys
- Identifies untranslated entries (identical to en-GB and [UNTRANSLATED] markers) - Identifies untranslated entries (identical to en-US and [UNTRANSLATED] markers)
- Shows accurate completion percentages using ignore patterns - Shows accurate completion percentages using ignore patterns
- Identifies extra keys not in en-GB - Identifies extra keys not in en-US
- Supports JSON and text output formats - Supports JSON and text output formats
- Uses `scripts/ignore_translation.toml` for language-specific exclusions - Uses `scripts/ignore_translation.toml` for language-specific exclusions
### 2. `translation_merger.py` ### 2. `translation_merger.py`
Merges missing translations from en-GB into target language files and manages translation workflows. Merges missing translations from en-US into target language files and manages translation workflows.
**Usage:** **Usage:**
```bash ```bash
# Operate on all locales (except en-GB) when language is omitted # Operate on all locales (except en-US) when language is omitted
python scripts/translations/translation_merger.py add-missing python scripts/translations/translation_merger.py add-missing
# Add missing translations from en-GB to French # Add missing translations from en-US to French
python scripts/translations/translation_merger.py fr-FR add-missing python scripts/translations/translation_merger.py fr-FR add-missing
# Create backups before modifying files # Create backups before modifying files
@@ -192,19 +192,19 @@ python scripts/translations/translation_merger.py fr-FR apply-translations --tra
# Override default paths if needed # Override default paths if needed
python scripts/translations/translation_merger.py fr-FR add-missing --locales-dir ./frontend/editor/public/locales --ignore-file ./scripts/ignore_translation.toml python scripts/translations/translation_merger.py fr-FR add-missing --locales-dir ./frontend/editor/public/locales --ignore-file ./scripts/ignore_translation.toml
# Remove unused translations not present in en-GB # Remove unused translations not present in en-US
python scripts/translations/translation_merger.py fr-FR remove-unused python scripts/translations/translation_merger.py fr-FR remove-unused
``` ```
**Features:** **Features:**
- Adds missing keys from en-GB (copies English text directly) - Adds missing keys from en-US (copies English text directly)
- Runs across all locales for add-missing/remove-unused when language is omitted - Runs across all locales for add-missing/remove-unused when language is omitted
- Extracts untranslated entries for external translation - Extracts untranslated entries for external translation
- Creates structured templates for AI translation - Creates structured templates for AI translation
- Applies translated content back to language files (template format or plain JSON) - Applies translated content back to language files (template format or plain JSON)
- Supports `--backup` on mutating commands - Supports `--backup` on mutating commands
- Automatic backup creation - Automatic backup creation
- Removes unused translations not present in en-GB - Removes unused translations not present in en-US
### 3. `ai_translation_helper.py` ### 3. `ai_translation_helper.py`
Specialized tool for AI-assisted translation workflows with batch processing and validation. Specialized tool for AI-assisted translation workflows with batch processing and validation.
@@ -290,7 +290,7 @@ python3 scripts/translations/auto_translate.py es-ES --skip-verification
4. **Validate**: Ensures placeholders are preserved 4. **Validate**: Ensures placeholders are preserved
5. **Merge**: Combines all translated batches 5. **Merge**: Combines all translated batches
6. **Apply**: Updates the language file 6. **Apply**: Updates the language file
7. **Beautify**: Restructures to match en-GB format 7. **Beautify**: Restructures to match en-US format
8. **Cleanup**: Removes temporary files 8. **Cleanup**: Removes temporary files
9. **Verify**: Reports final completion percentage 9. **Verify**: Reports final completion percentage
@@ -337,11 +337,11 @@ python3 scripts/translations/batch_translator.py batch.json --language it-IT --s
- `gpt-5-nano`: Fastest, most economical - `gpt-5-nano`: Fastest, most economical
### 7. `json_beautifier.py` ### 7. `json_beautifier.py`
Restructures and beautifies translation JSON files to match en-GB structure exactly. Restructures and beautifies translation JSON files to match en-US structure exactly.
**Usage:** **Usage:**
```bash ```bash
# Restructure single language to match en-GB structure # Restructure single language to match en-US structure
python scripts/translations/json_beautifier.py --language de-DE python scripts/translations/json_beautifier.py --language de-DE
# Restructure all languages # Restructure all languages
@@ -355,7 +355,7 @@ python scripts/translations/json_beautifier.py --language de-DE --no-backup
``` ```
**Features:** **Features:**
- Restructures JSON to match en-GB nested structure exactly - Restructures JSON to match en-US nested structure exactly
- Preserves key ordering for line-by-line comparison - Preserves key ordering for line-by-line comparison
- Creates automatic backups before modification - Creates automatic backups before modification
- Validates structure and key ordering - Validates structure and key ordering
@@ -585,7 +585,7 @@ python scripts/translations/json_validator.py --all-batches ar_AR
#### JSON Structure Mismatches #### JSON Structure Mismatches
**Problem**: Flattened dot-notation keys instead of proper nested objects **Problem**: Flattened dot-notation keys instead of proper nested objects
**Solution**: Use `json_beautifier.py` to restructure files to match en-GB exactly **Solution**: Use `json_beautifier.py` to restructure files to match en-US exactly
## Real-World Examples ## Real-World Examples
@@ -629,7 +629,7 @@ EOF
python scripts/translations/translation_merger.py ar-AR apply-translations --translations-file ar_AR_merged.json python scripts/translations/translation_merger.py ar-AR apply-translations --translations-file ar_AR_merged.json
# Result: Applied 1088 translations # Result: Applied 1088 translations
# Beautify to match en-GB structure # Beautify to match en-US structure
python scripts/translations/json_beautifier.py --language ar-AR python scripts/translations/json_beautifier.py --language ar-AR
# Check final progress # Check final progress
@@ -711,4 +711,4 @@ These scripts integrate with the existing translation system:
3. **Updating Existing Language**: Use analyzer to find gaps, then compact or batch method 3. **Updating Existing Language**: Use analyzer to find gaps, then compact or batch method
4. **Quality Assurance**: Use analyzer with `--summary` for completion metrics and issue detection 4. **Quality Assurance**: Use analyzer with `--summary` for completion metrics and issue detection
5. **External Translation Services**: Use export functionality to generate CSV files for translators 5. **External Translation Services**: Use export functionality to generate CSV files for translators
6. **Structure Maintenance**: Use json_beautifier to keep files aligned with en-GB structure 6. **Structure Maintenance**: Use json_beautifier to keep files aligned with en-US structure
@@ -20,7 +20,7 @@ import tomli_w
class AITranslationHelper: class AITranslationHelper:
def __init__(self, locales_dir: str = "frontend/editor/public/locales"): def __init__(self, locales_dir: str = "frontend/editor/public/locales"):
self.locales_dir = Path(locales_dir) self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
def _load_translation_file(self, file_path: Path) -> Dict: def _load_translation_file(self, file_path: Path) -> Dict:
"""Load TOML translation file.""" """Load TOML translation file."""
@@ -47,7 +47,7 @@ class AITranslationHelper:
batch_data = { batch_data = {
"metadata": { "metadata": {
"created_at": datetime.now().isoformat(), "created_at": datetime.now().isoformat(),
"source_language": "en-GB", "source_language": "en-US",
"target_languages": languages, "target_languages": languages,
"max_entries_per_language": max_entries_per_language, "max_entries_per_language": max_entries_per_language,
"instructions": { "instructions": {
@@ -320,7 +320,7 @@ class AITranslationHelper:
) )
with open(output_file, "w", newline="", encoding="utf-8") as csvfile: with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
fieldnames = ["key", "context", "en_GB"] + languages fieldnames = ["key", "context", "en_US"] + languages
writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader() writer.writeheader()
@@ -331,7 +331,7 @@ class AITranslationHelper:
row = { row = {
"key": key, "key": key,
"context": self._get_key_context(key), "context": self._get_key_context(key),
"en_GB": en_value, "en_US": en_value,
} }
for lang in languages: for lang in languages:
@@ -363,7 +363,7 @@ class AITranslationHelper:
continue continue
export_data["translations"][key] = { export_data["translations"][key] = {
"en_GB": en_value, "en_US": en_value,
"context": self._get_key_context(key), "context": self._get_key_context(key),
} }
+3 -3
View File
@@ -55,14 +55,14 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
print(f"\n🔍 Extracting {mode} entries for {language_code}...") print(f"\n🔍 Extracting {mode} entries for {language_code}...")
# Load files # Load files
golden_path = find_translation_file(Path("frontend/editor/public/locales/en-GB")) golden_path = find_translation_file(Path("frontend/editor/public/locales/en-US"))
lang_path = find_translation_file( lang_path = find_translation_file(
Path(f"frontend/editor/public/locales/{language_code}") Path(f"frontend/editor/public/locales/{language_code}")
) )
if not golden_path: if not golden_path:
print( print(
"Error: Golden truth file not found in frontend/editor/public/locales/en-GB" "Error: Golden truth file not found in frontend/editor/public/locales/en-US"
) )
return None return None
@@ -222,7 +222,7 @@ def apply_translations(merged_file, language_code):
def beautify_translations(language_code): def beautify_translations(language_code):
"""Beautify translation file to match en-GB structure.""" """Beautify translation file to match en-US structure."""
print(f"\n✨ Beautifying {language_code} translation file...") print(f"\n✨ Beautifying {language_code} translation file...")
cmd = f"python3 scripts/translations/toml_beautifier.py --language {language_code}" cmd = f"python3 scripts/translations/toml_beautifier.py --language {language_code}"
+11 -11
View File
@@ -37,7 +37,7 @@ def get_all_languages(locales_dir: Path) -> List[str]:
return [] return []
for lang_dir in sorted(locales_dir.iterdir()): for lang_dir in sorted(locales_dir.iterdir()):
if lang_dir.is_dir() and lang_dir.name != "en-GB": if lang_dir.is_dir() and lang_dir.name != "en-US":
toml_file = lang_dir / "translation.toml" toml_file = lang_dir / "translation.toml"
if toml_file.exists(): if toml_file.exists():
languages.append(lang_dir.name) languages.append(lang_dir.name)
@@ -57,10 +57,10 @@ def get_language_completion(locales_dir: Path, language: str) -> Optional[float]
with open(toml_file, "rb") as f: with open(toml_file, "rb") as f:
target_data = tomllib.load(f) target_data = tomllib.load(f)
# Load en-GB reference # Load en-US reference
en_gb_file = locales_dir / "en-GB" / "translation.toml" en_us_file = locales_dir / "en-US" / "translation.toml"
with open(en_gb_file, "rb") as f: with open(en_us_file, "rb") as f:
en_gb_data = tomllib.load(f) en_us_data = tomllib.load(f)
# Flatten and count # Flatten and count
def flatten(d, parent=""): def flatten(d, parent=""):
@@ -73,16 +73,16 @@ def get_language_completion(locales_dir: Path, language: str) -> Optional[float]
items[key] = v items[key] = v
return items return items
en_gb_flat = flatten(en_gb_data) en_us_flat = flatten(en_us_data)
target_flat = flatten(target_data) target_flat = flatten(target_data)
# Count translated (not equal to en-GB) # Count translated (not equal to en-US)
translated = sum( translated = sum(
1 1
for k in en_gb_flat for k in en_us_flat
if k in target_flat and target_flat[k] != en_gb_flat[k] if k in target_flat and target_flat[k] != en_us_flat[k]
) )
total = len(en_gb_flat) total = len(en_us_flat)
return (translated / total * 100) if total > 0 else 0.0 return (translated / total * 100) if total > 0 else 0.0
@@ -246,7 +246,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
print(f"Translating specified languages: {', '.join(languages)}") print(f"Translating specified languages: {', '.join(languages)}")
else: else:
languages = get_all_languages(locales_dir) languages = get_all_languages(locales_dir)
print(f"Found {len(languages)} languages (excluding en-GB)") print(f"Found {len(languages)} languages (excluding en-US)")
if not languages: if not languages:
print("No languages to translate!") print("No languages to translate!")
+3 -3
View File
@@ -19,10 +19,10 @@ class CompactTranslationExtractor:
ignore_file: str = "scripts/ignore_translation.toml", ignore_file: str = "scripts/ignore_translation.toml",
): ):
self.locales_dir = Path(locales_dir) self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
if not self.golden_truth_file.exists(): if not self.golden_truth_file.exists():
print( print(
f"Error: en-GB translation file not found at {self.golden_truth_file}", f"Error: en-US translation file not found at {self.golden_truth_file}",
file=sys.stderr, file=sys.stderr,
) )
sys.exit(1) sys.exit(1)
@@ -95,7 +95,7 @@ class CompactTranslationExtractor:
# Find missing translations # Find missing translations
missing_keys = set(golden_flat.keys()) - set(target_flat.keys()) - ignore_set missing_keys = set(golden_flat.keys()) - set(target_flat.keys()) - ignore_set
# Find untranslated entries (identical to en-GB or marked [UNTRANSLATED]) # Find untranslated entries (identical to en-US or marked [UNTRANSLATED])
untranslated_keys = set() untranslated_keys = set()
for key in target_flat: for key in target_flat:
if key in golden_flat and key not in ignore_set: if key in golden_flat and key not in ignore_set:
+5 -5
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
TOML Beautifier and Structure Fixer for Stirling PDF Frontend TOML Beautifier and Structure Fixer for Stirling PDF Frontend
Restructures translation TOML files to match en-GB structure and key order exactly. Restructures translation TOML files to match en-US structure and key order exactly.
""" """
import sys import sys
@@ -17,7 +17,7 @@ import tomli_w
class TOMLBeautifier: class TOMLBeautifier:
def __init__(self, locales_dir: str = "frontend/editor/public/locales"): def __init__(self, locales_dir: str = "frontend/editor/public/locales"):
self.locales_dir = Path(locales_dir) self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
self.golden_structure = self._load_toml(self.golden_truth_file) self.golden_structure = self._load_toml(self.golden_truth_file)
def _load_toml(self, file_path: Path) -> Dict: def _load_toml(self, file_path: Path) -> Dict:
@@ -95,7 +95,7 @@ class TOMLBeautifier:
return build_recursive(reference_structure) or OrderedDict() return build_recursive(reference_structure) or OrderedDict()
def restructure_translation_file(self, target_file: Path) -> Dict[str, Any]: def restructure_translation_file(self, target_file: Path) -> Dict[str, Any]:
"""Restructure a translation file to match en-GB structure exactly.""" """Restructure a translation file to match en-US structure exactly."""
if not target_file.exists(): if not target_file.exists():
print(f"Error: Target file does not exist: {target_file}") print(f"Error: Target file does not exist: {target_file}")
return {} return {}
@@ -177,7 +177,7 @@ class TOMLBeautifier:
} }
def validate_key_order(self, target_file: Path) -> Dict[str, Any]: def validate_key_order(self, target_file: Path) -> Dict[str, Any]:
"""Validate that keys appear in the same order as en-GB.""" """Validate that keys appear in the same order as en-US."""
target_data = self._load_toml(target_file) target_data = self._load_toml(target_file)
def get_key_order(obj: Dict, path: str = "") -> List[str]: def get_key_order(obj: Dict, path: str = "") -> List[str]:
@@ -276,7 +276,7 @@ def main():
elif args.all_languages: elif args.all_languages:
results = [] results = []
for lang_dir in Path(args.locales_dir).iterdir(): for lang_dir in Path(args.locales_dir).iterdir():
if lang_dir.is_dir() and lang_dir.name != "en-GB": if lang_dir.is_dir() and lang_dir.name != "en-US":
translation_file = lang_dir / "translation.toml" translation_file = lang_dir / "translation.toml"
if translation_file.exists(): if translation_file.exists():
if args.validate_only: if args.validate_only:
+12 -12
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Translation Analyzer for Stirling PDF Frontend Translation Analyzer for Stirling PDF Frontend
Compares language files against en-GB golden truth file. Compares language files against en-US golden truth file.
""" """
import json import json
@@ -19,7 +19,7 @@ class TranslationAnalyzer:
ignore_file: str = "scripts/ignore_translation.toml", ignore_file: str = "scripts/ignore_translation.toml",
): ):
self.locales_dir = Path(locales_dir) self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
self.golden_truth = self._load_translation_file(self.golden_truth_file) self.golden_truth = self._load_translation_file(self.golden_truth_file)
self.ignore_file = Path(ignore_file) self.ignore_file = Path(ignore_file)
self.ignore_patterns = self._load_ignore_patterns() self.ignore_patterns = self._load_ignore_patterns()
@@ -70,17 +70,17 @@ class TranslationAnalyzer:
return dict(items) return dict(items)
def get_all_language_files(self) -> List[Path]: def get_all_language_files(self) -> List[Path]:
"""Get all translation files except en-GB.""" """Get all translation files except en-US."""
files = [] files = []
for lang_dir in self.locales_dir.iterdir(): for lang_dir in self.locales_dir.iterdir():
if lang_dir.is_dir() and lang_dir.name != "en-GB": if lang_dir.is_dir() and lang_dir.name != "en-US":
toml_file = lang_dir / "translation.toml" toml_file = lang_dir / "translation.toml"
if toml_file.exists(): if toml_file.exists():
files.append(toml_file) files.append(toml_file)
return sorted(files) return sorted(files)
def find_missing_translations(self, target_file: Path) -> Set[str]: def find_missing_translations(self, target_file: Path) -> Set[str]:
"""Find keys that exist in en-GB but missing in target file.""" """Find keys that exist in en-US but missing in target file."""
target_data = self._load_translation_file(target_file) target_data = self._load_translation_file(target_file)
golden_flat = self._flatten_dict(self.golden_truth) golden_flat = self._flatten_dict(self.golden_truth)
@@ -94,7 +94,7 @@ class TranslationAnalyzer:
return missing - ignore_set return missing - ignore_set
def find_untranslated_entries(self, target_file: Path) -> Set[str]: def find_untranslated_entries(self, target_file: Path) -> Set[str]:
"""Find entries that appear to be untranslated (identical to en-GB).""" """Find entries that appear to be untranslated (identical to en-US)."""
target_data = self._load_translation_file(target_file) target_data = self._load_translation_file(target_file)
golden_flat = self._flatten_dict(self.golden_truth) golden_flat = self._flatten_dict(self.golden_truth)
@@ -109,7 +109,7 @@ class TranslationAnalyzer:
target_value = target_flat[key] target_value = target_flat[key]
golden_value = golden_flat[key] golden_value = golden_flat[key]
# Check if marked as [UNTRANSLATED] or identical to en-GB # Check if marked as [UNTRANSLATED] or identical to en-US
if ( if (
isinstance(target_value, str) isinstance(target_value, str)
and target_value.startswith("[UNTRANSLATED]") and target_value.startswith("[UNTRANSLATED]")
@@ -139,7 +139,7 @@ class TranslationAnalyzer:
return False return False
def find_extra_translations(self, target_file: Path) -> Set[str]: def find_extra_translations(self, target_file: Path) -> Set[str]:
"""Find keys that exist in target file but not in en-GB.""" """Find keys that exist in target file but not in en-US."""
target_data = self._load_translation_file(target_file) target_data = self._load_translation_file(target_file)
golden_flat = self._flatten_dict(self.golden_truth) golden_flat = self._flatten_dict(self.golden_truth)
@@ -174,7 +174,7 @@ class TranslationAnalyzer:
if not (isinstance(value, str) and value.startswith("[UNTRANSLATED]")): if not (isinstance(value, str) and value.startswith("[UNTRANSLATED]")):
if ( if (
key not in untranslated key not in untranslated
): # Not identical to en-GB (unless expected) ): # Not identical to en-US (unless expected)
properly_translated += 1 properly_translated += 1
completion_rate = ( completion_rate = (
@@ -204,7 +204,7 @@ class TranslationAnalyzer:
def main(): def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Analyze translation files against en-GB golden truth" description="Analyze translation files against en-US golden truth"
) )
parser.add_argument( parser.add_argument(
"--locales-dir", "--locales-dir",
@@ -260,7 +260,7 @@ def main():
print(f"Language: {lang}") print(f"Language: {lang}")
print(f"File: {result['file']}") print(f"File: {result['file']}")
print(f"Completion Rate: {result['completion_rate']:.1f}%") print(f"Completion Rate: {result['completion_rate']:.1f}%")
print(f"Total Keys in en-GB: {result['total_keys']}") print(f"Total Keys in en-US: {result['total_keys']}")
if not args.summary: if not args.summary:
if not args.untranslated_only: if not args.untranslated_only:
@@ -278,7 +278,7 @@ def main():
print(f" ... and {len(result['untranslated_keys']) - 10} more") print(f" ... and {len(result['untranslated_keys']) - 10} more")
if result["extra_count"] > 0: if result["extra_count"] > 0:
print(f"\nExtra Keys Not in en-GB ({result['extra_count']}):") print(f"\nExtra Keys Not in en-US ({result['extra_count']}):")
for key in result["extra_keys"][:5]: for key in result["extra_keys"][:5]:
print(f" - {key}") print(f" - {key}")
if len(result["extra_keys"]) > 5: if len(result["extra_keys"]) > 5:
+10 -10
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Translation Merger for Stirling PDF Frontend Translation Merger for Stirling PDF Frontend
Merges missing translations from en-GB into target language files. Merges missing translations from en-US into target language files.
Useful for AI-assisted translation workflows. Useful for AI-assisted translation workflows.
TOML format only. TOML format only.
""" """
@@ -30,7 +30,7 @@ class TranslationMerger:
), ),
): ):
self.locales_dir = Path(locales_dir) self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
self.golden_truth = self._load_translation_file(self.golden_truth_file) self.golden_truth = self._load_translation_file(self.golden_truth_file)
self.ignore_file = Path(ignore_file) self.ignore_file = Path(ignore_file)
self.ignore_patterns = self._load_ignore_patterns() self.ignore_patterns = self._load_ignore_patterns()
@@ -178,7 +178,7 @@ class TranslationMerger:
save: bool = True, save: bool = True,
backup: bool = False, backup: bool = False,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Add missing translations from en-GB to target file and optionally save.""" """Add missing translations from en-US to target file and optionally save."""
if not target_file.parent.exists(): if not target_file.parent.exists():
target_file.parent.mkdir(parents=True, exist_ok=True) target_file.parent.mkdir(parents=True, exist_ok=True)
target_data = {} target_data = {}
@@ -210,7 +210,7 @@ class TranslationMerger:
def extract_untranslated_entries( def extract_untranslated_entries(
self, target_file: Path, output_file: Path | None = None self, target_file: Path, output_file: Path | None = None
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Extract entries marked as untranslated or identical to en-GB for AI translation.""" """Extract entries marked as untranslated or identical to en-US for AI translation."""
if not target_file.exists(): if not target_file.exists():
print(f"Error: Target file does not exist: {target_file}") print(f"Error: Target file does not exist: {target_file}")
return {} return {}
@@ -335,7 +335,7 @@ class TranslationMerger:
template = { template = {
"metadata": { "metadata": {
"source_language": "en-GB", "source_language": "en-US",
"target_language": target_file.parent.name, "target_language": target_file.parent.name,
"total_entries": len(untranslated), "total_entries": len(untranslated),
"created_at": datetime.now().isoformat(), "created_at": datetime.now().isoformat(),
@@ -384,14 +384,14 @@ def main():
parser.add_argument( parser.add_argument(
"language", "language",
nargs="?", nargs="?",
help="Target language code (e.g., fr-FR). If omitted, add-missing and remove-unused run for all locales except en-GB.", help="Target language code (e.g., fr-FR). If omitted, add-missing and remove-unused run for all locales except en-US.",
) )
subparsers = parser.add_subparsers(dest="command", help="Available commands") subparsers = parser.add_subparsers(dest="command", help="Available commands")
# Add missing command # Add missing command
add_parser = subparsers.add_parser( add_parser = subparsers.add_parser(
"add-missing", help="Add missing translations from en-GB" "add-missing", help="Add missing translations from en-US"
) )
add_parser.add_argument( add_parser.add_argument(
"--backup", action="store_true", help="Create backup before modifying files" "--backup", action="store_true", help="Create backup before modifying files"
@@ -424,7 +424,7 @@ def main():
# Remove unused translations command # Remove unused translations command
remove_parser = subparsers.add_parser( remove_parser = subparsers.add_parser(
"remove-unused", help="Remove unused translations not present in en-GB" "remove-unused", help="Remove unused translations not present in en-US"
) )
remove_parser.add_argument( remove_parser.add_argument(
"--backup", action="store_true", help="Create backup before modifying files" "--backup", action="store_true", help="Create backup before modifying files"
@@ -449,7 +449,7 @@ def main():
else: else:
total_added = 0 total_added = 0
for lang_dir in sorted(Path(args.locales_dir).iterdir()): for lang_dir in sorted(Path(args.locales_dir).iterdir()):
if not lang_dir.is_dir() or lang_dir.name == "en-GB": if not lang_dir.is_dir() or lang_dir.name == "en-US":
continue continue
target_file = lang_dir / "translation.toml" target_file = lang_dir / "translation.toml"
print(f"Processing {lang_dir.name}...") print(f"Processing {lang_dir.name}...")
@@ -471,7 +471,7 @@ def main():
else: else:
total_removed = 0 total_removed = 0
for lang_dir in sorted(Path(args.locales_dir).iterdir()): for lang_dir in sorted(Path(args.locales_dir).iterdir()):
if not lang_dir.is_dir() or lang_dir.name == "en-GB": if not lang_dir.is_dir() or lang_dir.name == "en-US":
continue continue
target_file = lang_dir / "translation.toml" target_file = lang_dir / "translation.toml"
print(f"Processing {lang_dir.name}...") print(f"Processing {lang_dir.name}...")
+22 -22
View File
@@ -4,9 +4,9 @@ Validate TOML structure and formatting of translation files.
Checks for: Checks for:
- Valid TOML syntax - Valid TOML syntax
- Consistent key structure with en-GB - Consistent key structure with en-US
- Missing keys - Missing keys
- Extra keys not in en-GB - Extra keys not in en-US
- Malformed entries - Malformed entries
Usage: Usage:
@@ -43,18 +43,18 @@ def validate_translation_file(file_path: Path) -> tuple[bool, str]:
def validate_structure( def validate_structure(
en_gb_keys: Set[str], lang_keys: Set[str], lang_code: str en_us_keys: Set[str], lang_keys: Set[str], lang_code: str
) -> Dict: ) -> Dict:
"""Compare structure between en-GB and target language.""" """Compare structure between en-US and target language."""
missing_keys = en_gb_keys - lang_keys missing_keys = en_us_keys - lang_keys
extra_keys = lang_keys - en_gb_keys extra_keys = lang_keys - en_us_keys
return { return {
"language": lang_code, "language": lang_code,
"missing_keys": sorted(missing_keys), "missing_keys": sorted(missing_keys),
"extra_keys": sorted(extra_keys), "extra_keys": sorted(extra_keys),
"total_keys": len(lang_keys), "total_keys": len(lang_keys),
"expected_keys": len(en_gb_keys), "expected_keys": len(en_us_keys),
"missing_count": len(missing_keys), "missing_count": len(missing_keys),
"extra_count": len(extra_keys), "extra_count": len(extra_keys),
} }
@@ -68,12 +68,12 @@ def print_validation_result(result: Dict, verbose: bool = False):
print(f"Language: {lang}") print(f"Language: {lang}")
print(f"{'=' * 100}") print(f"{'=' * 100}")
print(f" Total keys: {result['total_keys']}") print(f" Total keys: {result['total_keys']}")
print(f" Expected keys (en-GB): {result['expected_keys']}") print(f" Expected keys (en-US): {result['expected_keys']}")
print(f" Missing keys: {result['missing_count']}") print(f" Missing keys: {result['missing_count']}")
print(f" Extra keys: {result['extra_count']}") print(f" Extra keys: {result['extra_count']}")
if result["missing_count"] == 0 and result["extra_count"] == 0: if result["missing_count"] == 0 and result["extra_count"] == 0:
print(" ✅ Structure matches en-GB perfectly!") print(" ✅ Structure matches en-US perfectly!")
else: else:
if result["missing_count"] > 0: if result["missing_count"] > 0:
print(f"\n ⚠️ Missing {result['missing_count']} key(s):") print(f"\n ⚠️ Missing {result['missing_count']} key(s):")
@@ -86,7 +86,7 @@ def print_validation_result(result: Dict, verbose: bool = False):
print(" (use --verbose to see all)") print(" (use --verbose to see all)")
if result["extra_count"] > 0: if result["extra_count"] > 0:
print(f"\n ⚠️ Extra {result['extra_count']} key(s) not in en-GB:") print(f"\n ⚠️ Extra {result['extra_count']} key(s) not in en-US:")
if verbose or result["extra_count"] <= 20: if verbose or result["extra_count"] <= 20:
for key in result["extra_keys"][:50]: for key in result["extra_keys"][:50]:
print(f" - {key}") print(f" - {key}")
@@ -120,31 +120,31 @@ def main():
# Define paths # Define paths
locales_dir = Path("frontend/editor/public/locales") locales_dir = Path("frontend/editor/public/locales")
en_gb_path = locales_dir / "en-GB" / "translation.toml" en_us_path = locales_dir / "en-US" / "translation.toml"
if not en_gb_path.exists(): if not en_us_path.exists():
print(f"❌ Error: en-GB translation file not found at {en_gb_path}") print(f"❌ Error: en-US translation file not found at {en_us_path}")
sys.exit(1) sys.exit(1)
# Validate en-GB itself # Validate en-US itself
is_valid, message = validate_translation_file(en_gb_path) is_valid, message = validate_translation_file(en_us_path)
if not is_valid: if not is_valid:
print(f"❌ Error in en-GB file: {message}") print(f"❌ Error in en-US file: {message}")
sys.exit(1) sys.exit(1)
# Load en-GB structure # Load en-US structure
en_gb = load_translation_file(en_gb_path) en_us = load_translation_file(en_us_path)
en_gb_keys = get_all_keys(en_gb) en_us_keys = get_all_keys(en_us)
# Get list of languages to validate # Get list of languages to validate
if args.language: if args.language:
languages = [args.language] languages = [args.language]
else: else:
# Validate all languages except en-GB # Validate all languages except en-US
languages = [] languages = []
for d in locales_dir.iterdir(): for d in locales_dir.iterdir():
if d.is_dir() and d.name != "en-GB": if d.is_dir() and d.name != "en-US":
if (d / "translation.toml").exists(): if (d / "translation.toml").exists():
languages.append(d.name) languages.append(d.name)
@@ -171,7 +171,7 @@ def main():
lang_data = load_translation_file(lang_path) lang_data = load_translation_file(lang_path)
lang_keys = get_all_keys(lang_data) lang_keys = get_all_keys(lang_data)
result = validate_structure(en_gb_keys, lang_keys, lang_code) result = validate_structure(en_us_keys, lang_keys, lang_code)
results.append(result) results.append(result)
# Output results # Output results
+16 -16
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Validate that translation files have the same placeholders as en-GB (source of truth). Validate that translation files have the same placeholders as en-US (source of truth).
Usage: Usage:
python scripts/translations/validate_placeholders.py [--language LANG] [--fix] python scripts/translations/validate_placeholders.py [--language LANG] [--fix]
@@ -38,16 +38,16 @@ def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> Dict[str, str
def validate_language( def validate_language(
en_gb_flat: Dict[str, str], lang_flat: Dict[str, str], lang_code: str en_us_flat: Dict[str, str], lang_flat: Dict[str, str], lang_code: str
) -> List[Dict]: ) -> List[Dict]:
"""Validate placeholders for a language against en-GB.""" """Validate placeholders for a language against en-US."""
issues = [] issues = []
for key in en_gb_flat: for key in en_us_flat:
if key not in lang_flat: if key not in lang_flat:
continue continue
en_placeholders = find_placeholders(en_gb_flat[key]) en_placeholders = find_placeholders(en_us_flat[key])
lang_placeholders = find_placeholders(lang_flat[key]) lang_placeholders = find_placeholders(lang_flat[key])
if en_placeholders != lang_placeholders: if en_placeholders != lang_placeholders:
@@ -59,7 +59,7 @@ def validate_language(
"key": key, "key": key,
"missing": missing, "missing": missing,
"extra": extra, "extra": extra,
"en_text": en_gb_flat[key], "en_text": en_us_flat[key],
"lang_text": lang_flat[key], "lang_text": lang_flat[key],
} }
issues.append(issue) issues.append(issue)
@@ -113,26 +113,26 @@ def main():
# Define paths # Define paths
locales_dir = Path("frontend/editor/public/locales") locales_dir = Path("frontend/editor/public/locales")
en_gb_path = locales_dir / "en-GB" / "translation.toml" en_us_path = locales_dir / "en-US" / "translation.toml"
if not en_gb_path.exists(): if not en_us_path.exists():
print(f"❌ Error: en-GB translation file not found at {en_gb_path}") print(f"❌ Error: en-US translation file not found at {en_us_path}")
sys.exit(1) sys.exit(1)
# Load en-GB (source of truth) # Load en-US (source of truth)
with open(en_gb_path, "rb") as f: with open(en_us_path, "rb") as f:
en_gb = tomllib.load(f) en_us = tomllib.load(f)
en_gb_flat = flatten_dict(en_gb) en_us_flat = flatten_dict(en_us)
# Get list of languages to validate # Get list of languages to validate
if args.language: if args.language:
languages = [args.language] languages = [args.language]
else: else:
# Validate all languages except en-GB # Validate all languages except en-US
languages = [] languages = []
for d in locales_dir.iterdir(): for d in locales_dir.iterdir():
if d.is_dir() and d.name != "en-GB": if d.is_dir() and d.name != "en-US":
if (d / "translation.toml").exists(): if (d / "translation.toml").exists():
languages.append(d.name) languages.append(d.name)
@@ -151,7 +151,7 @@ def main():
lang_data = tomllib.load(f) lang_data = tomllib.load(f)
lang_flat = flatten_dict(lang_data) lang_flat = flatten_dict(lang_data)
issues = validate_language(en_gb_flat, lang_flat, lang_code) issues = validate_language(en_us_flat, lang_flat, lang_code)
all_issues.extend(issues) all_issues.extend(issues)
# Output results # Output results