arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-16 19:33:11 +02:00
Code Issues Actions 3 Packages Projects Releases 1 Wiki Activity

🤖 format everything with pre-commit by stirlingbot (#5144)

Browse Source 
Auto-generated by [create-pull-request][1] with **stirlingbot**

[1]: https://github.com/peter-evans/create-pull-request

Signed-off-by: stirlingbot[bot] <stirlingbot[bot]@users.noreply.github.com>
Co-authored-by: stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
This commit is contained in:
stirlingbot[bot]
2025-12-22 15:44:38 +00:00
committed by GitHub
co-authored by stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
parent e6d3f20c36
commit c990ab3216
26 changed files with 1239 additions and 822 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/translations/ai_translation_helper.py
+196 -146
View File
@@ -7,10 +7,8 @@ TOML format only.
"""
import json
import os
import sys
from pathlib import Path
from typing import Dict, List, Set, Tuple, Any, Optional
from typing import Dict, List, Any
import argparse
import re
from datetime import datetime
@@ -27,7 +25,7 @@ class AITranslationHelper:
def _load_translation_file(self, file_path: Path) -> Dict:
"""Load TOML translation file."""
try:
with open(file_path, 'rb') as f:
with open(file_path, "rb") as f:
return tomllib.load(f)
except (FileNotFoundError, Exception) as e:
print(f"Error loading {file_path}: {e}")
@@ -35,27 +33,31 @@ class AITranslationHelper:
def _save_translation_file(self, data: Dict, file_path: Path) -> None:
"""Save TOML translation file."""
with open(file_path, 'wb') as f:
with open(file_path, "wb") as f:
tomli_w.dump(data, f)
def create_ai_batch_file(self, languages: List[str], output_file: Path,
max_entries_per_language: int = 50) -> None:
def create_ai_batch_file(
self,
languages: List[str],
output_file: Path,
max_entries_per_language: int = 50,
) -> None:
"""Create a batch file for AI translation with multiple languages."""
golden_truth = self._load_translation_file(self.golden_truth_file)
batch_data = {
'metadata': {
'created_at': datetime.now().isoformat(),
'source_language': 'en-GB',
'target_languages': languages,
'max_entries_per_language': max_entries_per_language,
'instructions': {
'format': 'Translate each entry maintaining JSON structure and placeholder variables like {n}, {total}, {filename}',
'context': 'This is for a PDF manipulation tool. Keep technical terms consistent.',
'placeholders': 'Preserve all placeholders: {n}, {total}, {filename}, etc.',
'style': 'Keep translations concise and user-friendly'
}
"metadata": {
"created_at": datetime.now().isoformat(),
"source_language": "en-GB",
"target_languages": languages,
"max_entries_per_language": max_entries_per_language,
"instructions": {
"format": "Translate each entry maintaining JSON structure and placeholder variables like {n}, {total}, {filename}",
"context": "This is for a PDF manipulation tool. Keep technical terms consistent.",
"placeholders": "Preserve all placeholders: {n}, {total}, {filename}, etc.",
"style": "Keep translations concise and user-friendly",
},
},
'translations': {}
"translations": {},
}
for lang in languages:
@@ -72,41 +74,57 @@ class AITranslationHelper:
untranslated = self._find_untranslated_entries(golden_truth, lang_data)
# Limit entries if specified
if max_entries_per_language and len(untranslated) > max_entries_per_language:
if (
max_entries_per_language
and len(untranslated) > max_entries_per_language
):
# Prioritize by key importance
untranslated = self._prioritize_translation_keys(untranslated, max_entries_per_language)
untranslated = self._prioritize_translation_keys(
untranslated, max_entries_per_language
)
batch_data['translations'][lang] = {}
batch_data["translations"][lang] = {}
for key, value in untranslated.items():
batch_data['translations'][lang][key] = {
'original': value,
'translated': '', # AI fills this
'context': self._get_key_context(key)
batch_data["translations"][lang][key] = {
"original": value,
"translated": "", # AI fills this
"context": self._get_key_context(key),
}
# Always save batch files as JSON for compatibility
with open(output_file, 'w', encoding='utf-8') as f:
with open(output_file, "w", encoding="utf-8") as f:
json.dump(batch_data, f, indent=2, ensure_ascii=False)
total_entries = sum(len(lang_data) for lang_data in batch_data['translations'].values())
total_entries = sum(
len(lang_data) for lang_data in batch_data["translations"].values()
)
print(f"Created AI batch file: {output_file}")
print(f"Total entries to translate: {total_entries}")
def _find_untranslated_entries(self, golden_truth: Dict, lang_data: Dict) -> Dict[str, str]:
def _find_untranslated_entries(
self, golden_truth: Dict, lang_data: Dict
) -> Dict[str, str]:
"""Find entries that need translation."""
golden_flat = self._flatten_dict(golden_truth)
lang_flat = self._flatten_dict(lang_data)
untranslated = {}
for key, value in golden_flat.items():
if (key not in lang_flat or
lang_flat[key] == value or
(isinstance(lang_flat[key], str) and lang_flat[key].startswith("[UNTRANSLATED]"))):
if (
key not in lang_flat
or lang_flat[key] == value
or (
isinstance(lang_flat[key], str)
and lang_flat[key].startswith("[UNTRANSLATED]")
)
):
if not self._is_expected_identical(key, value):
untranslated[key] = value
return untranslated
def _flatten_dict(self, d: Dict, parent_key: str = '', separator: str = '.') -> Dict[str, Any]:
def _flatten_dict(
self, d: Dict, parent_key: str = "", separator: str = "."
) -> Dict[str, Any]:
"""Flatten nested dictionary."""
items = []
for k, v in d.items():
@@ -119,25 +137,27 @@ class AITranslationHelper:
def _is_expected_identical(self, key: str, value: str) -> bool:
"""Check if key should be identical across languages."""
if str(value).strip() in ['ltr', 'rtl', 'True', 'False', 'true', 'false']:
if str(value).strip() in ["ltr", "rtl", "True", "False", "true", "false"]:
return True
return 'language.direction' in key.lower()
return "language.direction" in key.lower()
def _prioritize_translation_keys(self, untranslated: Dict[str, str], max_count: int) -> Dict[str, str]:
def _prioritize_translation_keys(
self, untranslated: Dict[str, str], max_count: int
) -> Dict[str, str]:
"""Prioritize which keys to translate first based on importance."""
# Define priority order (higher score = higher priority)
priority_patterns = [
('title', 10),
('header', 9),
('submit', 8),
('selectText', 7),
('prompt', 6),
('desc', 5),
('error', 8),
('warning', 7),
('save', 8),
('download', 8),
('upload', 7),
("title", 10),
("header", 9),
("submit", 8),
("selectText", 7),
("prompt", 6),
("desc", 5),
("error", 8),
("warning", 7),
("save", 8),
("download", 8),
("upload", 7),
]
scored_keys = []
@@ -154,89 +174,99 @@ class AITranslationHelper:
def _get_key_context(self, key: str) -> str:
"""Get contextual information for a translation key."""
parts = key.split('.')
parts = key.split(".")
contexts = {
'addPageNumbers': 'Feature for adding page numbers to PDFs',
'compress': 'PDF compression functionality',
'merge': 'PDF merging functionality',
'split': 'PDF splitting functionality',
'rotate': 'PDF rotation functionality',
'convert': 'File conversion functionality',
'security': 'PDF security and permissions',
'metadata': 'PDF metadata editing',
'watermark': 'Adding watermarks to PDFs',
'overlay': 'PDF overlay functionality',
'extract': 'Extracting content from PDFs'
"addPageNumbers": "Feature for adding page numbers to PDFs",
"compress": "PDF compression functionality",
"merge": "PDF merging functionality",
"split": "PDF splitting functionality",
"rotate": "PDF rotation functionality",
"convert": "File conversion functionality",
"security": "PDF security and permissions",
"metadata": "PDF metadata editing",
"watermark": "Adding watermarks to PDFs",
"overlay": "PDF overlay functionality",
"extract": "Extracting content from PDFs",
}
if len(parts) > 0:
main_section = parts[0]
context = contexts.get(main_section, f'Part of {main_section} functionality')
context = contexts.get(
main_section, f"Part of {main_section} functionality"
)
if len(parts) > 1:
context += f', specifically for {parts[-1]}'
context += f", specifically for {parts[-1]}"
return context
return 'General application text'
return "General application text"
def validate_ai_translations(self, batch_file: Path) -> Dict[str, List[str]]:
"""Validate AI translations for common issues."""
# Batch files are always JSON
with open(batch_file, 'r', encoding='utf-8') as f:
with open(batch_file, "r", encoding="utf-8") as f:
batch_data = json.load(f)
issues = {'errors': [], 'warnings': []}
issues = {"errors": [], "warnings": []}
for lang, translations in batch_data.get('translations', {}).items():
for lang, translations in batch_data.get("translations", {}).items():
for key, translation_data in translations.items():
original = translation_data.get('original', '')
translated = translation_data.get('translated', '')
original = translation_data.get("original", "")
translated = translation_data.get("translated", "")
if not translated:
issues['errors'].append(f"{lang}.{key}: Missing translation")
issues["errors"].append(f"{lang}.{key}: Missing translation")
continue
# Check for placeholder preservation
original_placeholders = re.findall(r'\{[^}]+\}', original)
translated_placeholders = re.findall(r'\{[^}]+\}', translated)
original_placeholders = re.findall(r"\{[^}]+\}", original)
translated_placeholders = re.findall(r"\{[^}]+\}", translated)
if set(original_placeholders) != set(translated_placeholders):
issues['warnings'].append(
issues["warnings"].append(
f"{lang}.{key}: Placeholder mismatch - Original: {original_placeholders}, "
f"Translated: {translated_placeholders}"
)
# Check if translation is identical to original (might be untranslated)
if translated == original and not self._is_expected_identical(key, original):
issues['warnings'].append(f"{lang}.{key}: Translation identical to original")
if translated == original and not self._is_expected_identical(
key, original
):
issues["warnings"].append(
f"{lang}.{key}: Translation identical to original"
)
# Check for common AI translation artifacts
artifacts = ['[TRANSLATE]', '[TODO]', 'UNTRANSLATED', '{{', '}}']
artifacts = ["[TRANSLATE]", "[TODO]", "UNTRANSLATED", "{{", "}}"]
for artifact in artifacts:
if artifact in translated:
issues['errors'].append(f"{lang}.{key}: Contains translation artifact: {artifact}")
issues["errors"].append(
f"{lang}.{key}: Contains translation artifact: {artifact}"
)
return issues
def apply_ai_batch_translations(self, batch_file: Path, validate: bool = True) -> Dict[str, Any]:
def apply_ai_batch_translations(
self, batch_file: Path, validate: bool = True
) -> Dict[str, Any]:
"""Apply translations from AI batch file to individual language files."""
# Batch files are always JSON
with open(batch_file, 'r', encoding='utf-8') as f:
with open(batch_file, "r", encoding="utf-8") as f:
batch_data = json.load(f)
results = {'applied': {}, 'errors': [], 'warnings': []}
results = {"applied": {}, "errors": [], "warnings": []}
if validate:
validation_issues = self.validate_ai_translations(batch_file)
if validation_issues['errors']:
if validation_issues["errors"]:
print("Validation errors found. Fix these before applying:")
for error in validation_issues['errors']:
for error in validation_issues["errors"]:
print(f" ERROR: {error}")
return results
if validation_issues['warnings']:
if validation_issues["warnings"]:
print("Validation warnings (review recommended):")
for warning in validation_issues['warnings'][:10]:
for warning in validation_issues["warnings"][:10]:
print(f" WARNING: {warning}")
for lang, translations in batch_data.get('translations', {}).items():
for lang, translations in batch_data.get("translations", {}).items():
lang_dir = self.locales_dir / lang
toml_file = lang_dir / "translation.toml"
@@ -249,42 +279,48 @@ class AITranslationHelper:
applied_count = 0
for key, translation_data in translations.items():
translated = translation_data.get('translated', '').strip()
if translated and translated != translation_data.get('original', ''):
translated = translation_data.get("translated", "").strip()
if translated and translated != translation_data.get("original", ""):
self._set_nested_value(lang_data, key, translated)
applied_count += 1
if applied_count > 0:
self._save_translation_file(lang_data, toml_file)
results['applied'][lang] = applied_count
results["applied"][lang] = applied_count
print(f"Applied {applied_count} translations to {lang}")
return results
def _set_nested_value(self, data: Dict, key_path: str, value: Any) -> None:
"""Set value in nested dict using dot notation."""
keys = key_path.split('.')
keys = key_path.split(".")
current = data
for key in keys[:-1]:
if key not in current:
current[key] = {}
elif not isinstance(current[key], dict):
# If the current value is not a dict, we can't nest into it
print(f"Warning: Converting non-dict value at '{key}' to dict to allow nesting")
print(
f"Warning: Converting non-dict value at '{key}' to dict to allow nesting"
)
current[key] = {}
current = current[key]
current[keys[-1]] = value
def export_for_external_translation(self, languages: List[str], output_format: str = 'csv') -> None:
def export_for_external_translation(
self, languages: List[str], output_format: str = "csv"
) -> None:
"""Export translations for external translation services."""
golden_truth = self._load_translation_file(self.golden_truth_file)
golden_flat = self._flatten_dict(golden_truth)
if output_format == 'csv':
output_file = Path(f'translations_export_{datetime.now().strftime("%Y%m%d")}.csv')
if output_format == "csv":
output_file = Path(
f"translations_export_{datetime.now().strftime('%Y%m%d')}.csv"
)
with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
fieldnames = ['key', 'context', 'en_GB'] + languages
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
fieldnames = ["key", "context", "en_GB"] + languages
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
@@ -293,9 +329,9 @@ class AITranslationHelper:
continue
row = {
'key': key,
'context': self._get_key_context(key),
'en_GB': en_value
"key": key,
"context": self._get_key_context(key),
"en_GB": en_value,
}
for lang in languages:
@@ -305,28 +341,30 @@ class AITranslationHelper:
if toml_file.exists():
lang_data = self._load_translation_file(toml_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
value = lang_flat.get(key, "")
if value.startswith("[UNTRANSLATED]"):
value = ""
row[lang] = value
else:
row[lang] = ''
row[lang] = ""
writer.writerow(row)
print(f"Exported to {output_file}")
elif output_format == 'json':
output_file = Path(f'translations_export_{datetime.now().strftime("%Y%m%d")}.json')
export_data = {'languages': languages, 'translations': {}}
elif output_format == "json":
output_file = Path(
f"translations_export_{datetime.now().strftime('%Y%m%d')}.json"
)
export_data = {"languages": languages, "translations": {}}
for key, en_value in golden_flat.items():
if self._is_expected_identical(key, en_value):
continue
export_data['translations'][key] = {
'en_GB': en_value,
'context': self._get_key_context(key)
export_data["translations"][key] = {
"en_GB": en_value,
"context": self._get_key_context(key),
}
for lang in languages:
@@ -336,51 +374,64 @@ class AITranslationHelper:
if toml_file.exists():
lang_data = self._load_translation_file(toml_file)
lang_flat = self._flatten_dict(lang_data)
value = lang_flat.get(key, '')
if value.startswith('[UNTRANSLATED]'):
value = ''
export_data['translations'][key][lang] = value
value = lang_flat.get(key, "")
if value.startswith("[UNTRANSLATED]"):
value = ""
export_data["translations"][key][lang] = value
# Export files are always JSON
with open(output_file, 'w', encoding='utf-8') as f:
with open(output_file, "w", encoding="utf-8") as f:
json.dump(export_data, f, indent=2, ensure_ascii=False)
print(f"Exported to {output_file}")
def main():
parser = argparse.ArgumentParser(
description='AI Translation Helper',
epilog='Works with TOML translation files.'
description="AI Translation Helper", epilog="Works with TOML translation files."
)
parser.add_argument(
"--locales-dir",
default="frontend/public/locales",
help="Path to locales directory",
)
parser.add_argument('--locales-dir', default='frontend/public/locales',
help='Path to locales directory')
subparsers = parser.add_subparsers(dest='command', help='Available commands')
subparsers = parser.add_subparsers(dest="command", help="Available commands")
# Create batch command
batch_parser = subparsers.add_parser('create-batch', help='Create AI translation batch file')
batch_parser.add_argument('--languages', nargs='+', required=True,
help='Language codes to include')
batch_parser.add_argument('--output', required=True, help='Output batch file')
batch_parser.add_argument('--max-entries', type=int, default=100,
help='Max entries per language')
batch_parser = subparsers.add_parser(
"create-batch", help="Create AI translation batch file"
)
batch_parser.add_argument(
"--languages", nargs="+", required=True, help="Language codes to include"
)
batch_parser.add_argument("--output", required=True, help="Output batch file")
batch_parser.add_argument(
"--max-entries", type=int, default=100, help="Max entries per language"
)
# Validate command
validate_parser = subparsers.add_parser('validate', help='Validate AI translations')
validate_parser.add_argument('batch_file', help='Batch file to validate')
validate_parser = subparsers.add_parser("validate", help="Validate AI translations")
validate_parser.add_argument("batch_file", help="Batch file to validate")
# Apply command
apply_parser = subparsers.add_parser('apply-batch', help='Apply AI batch translations')
apply_parser.add_argument('batch_file', help='Batch file with translations')
apply_parser.add_argument('--skip-validation', action='store_true',
help='Skip validation before applying')
apply_parser = subparsers.add_parser(
"apply-batch", help="Apply AI batch translations"
)
apply_parser.add_argument("batch_file", help="Batch file with translations")
apply_parser.add_argument(
"--skip-validation", action="store_true", help="Skip validation before applying"
)
# Export command
export_parser = subparsers.add_parser('export', help='Export for external translation')
export_parser.add_argument('--languages', nargs='+', required=True,
help='Language codes to export')
export_parser.add_argument('--format', choices=['csv', 'json'], default='csv',
help='Export format')
export_parser = subparsers.add_parser(
"export", help="Export for external translation"
)
export_parser.add_argument(
"--languages", nargs="+", required=True, help="Language codes to export"
)
export_parser.add_argument(
"--format", choices=["csv", "json"], default="csv", help="Export format"
)
args = parser.parse_args()
@@ -390,40 +441,39 @@ def main():
helper = AITranslationHelper(args.locales_dir)
if args.command == 'create-batch':
if args.command == "create-batch":
output_file = Path(args.output)
helper.create_ai_batch_file(args.languages, output_file, args.max_entries)
elif args.command == 'validate':
elif args.command == "validate":
batch_file = Path(args.batch_file)
issues = helper.validate_ai_translations(batch_file)
if issues['errors']:
if issues["errors"]:
print("ERRORS:")
for error in issues['errors']:
for error in issues["errors"]:
print(f" - {error}")
if issues['warnings']:
if issues["warnings"]:
print("WARNINGS:")
for warning in issues['warnings']:
for warning in issues["warnings"]:
print(f" - {warning}")
if not issues['errors'] and not issues['warnings']:
if not issues["errors"] and not issues["warnings"]:
print("No validation issues found!")
elif args.command == 'apply-batch':
elif args.command == "apply-batch":
batch_file = Path(args.batch_file)
results = helper.apply_ai_batch_translations(
batch_file,
validate=not args.skip_validation
batch_file, validate=not args.skip_validation
)
total_applied = sum(results['applied'].values())
total_applied = sum(results["applied"].values())
print(f"Total translations applied: {total_applied}")
elif args.command == 'export':
elif args.command == "export":
helper.export_for_external_translation(args.languages, args.format)
if __name__ == "__main__":
main()
main()