arsvendg/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/arsvendg/Stirling-PDF.git synced 2026-07-16 19:33:11 +02:00
Code Issues Actions 4 Packages Projects Releases 1 Wiki Activity

🤖 format everything with pre-commit by stirlingbot (#5144)

Browse Source 
Auto-generated by [create-pull-request][1] with **stirlingbot**

[1]: https://github.com/peter-evans/create-pull-request

Signed-off-by: stirlingbot[bot] <stirlingbot[bot]@users.noreply.github.com>
Co-authored-by: stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
This commit is contained in:
stirlingbot[bot]
2025-12-22 15:44:38 +00:00
committed by GitHub
co-authored by stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
parent e6d3f20c36
commit c990ab3216
26 changed files with 1239 additions and 822 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/translations/bulk_auto_translate.py
+109 -61
View File
@@ -54,16 +54,16 @@ def get_language_completion(locales_dir: Path, language: str) -> Optional[float]
return None
try:
with open(toml_file, 'rb') as f:
with open(toml_file, "rb") as f:
target_data = tomllib.load(f)
# Load en-GB reference
en_gb_file = locales_dir / 'en-GB' / 'translation.toml'
with open(en_gb_file, 'rb') as f:
en_gb_file = locales_dir / "en-GB" / "translation.toml"
with open(en_gb_file, "rb") as f:
en_gb_data = tomllib.load(f)
# Flatten and count
def flatten(d, parent=''):
def flatten(d, parent=""):
items = {}
for k, v in d.items():
key = f"{parent}.{k}" if parent else k
@@ -77,7 +77,11 @@ def get_language_completion(locales_dir: Path, language: str) -> Optional[float]
target_flat = flatten(target_data)
# Count translated (not equal to en-GB)
translated = sum(1 for k in en_gb_flat if k in target_flat and target_flat[k] != en_gb_flat[k])
translated = sum(
1
for k in en_gb_flat
if k in target_flat and target_flat[k] != en_gb_flat[k]
)
total = len(en_gb_flat)
return (translated / total * 100) if total > 0 else 0.0
@@ -87,7 +91,14 @@ def get_language_completion(locales_dir: Path, language: str) -> Optional[float]
return None
def translate_language(language: str, api_key: str, batch_size: int, timeout: int, skip_verification: bool, include_existing: bool) -> Tuple[str, bool, str]:
def translate_language(
language: str,
api_key: str,
batch_size: int,
timeout: int,
skip_verification: bool,
include_existing: bool,
) -> Tuple[str, bool, str]:
"""
Translate a single language.
Returns: (language_code, success, message)
@@ -95,25 +106,29 @@ def translate_language(language: str, api_key: str, batch_size: int, timeout: in
safe_print(f"[{language}] Starting translation...")
cmd = [
'python3', 'scripts/translations/auto_translate.py',
"python3",
"scripts/translations/auto_translate.py",
language,
'--api-key', api_key,
'--batch-size', str(batch_size),
'--timeout', str(timeout)
"--api-key",
api_key,
"--batch-size",
str(batch_size),
"--timeout",
str(timeout),
]
if skip_verification:
cmd.append('--skip-verification')
cmd.append("--skip-verification")
if include_existing:
cmd.append('--include-existing')
cmd.append("--include-existing")
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=timeout * 5 # Overall timeout = 5x per-batch timeout
timeout=timeout * 5, # Overall timeout = 5x per-batch timeout
)
if result.returncode == 0:
@@ -124,7 +139,9 @@ def translate_language(language: str, api_key: str, batch_size: int, timeout: in
safe_print(f"[{language}] ✓ Success")
return (language, True, "Success")
else:
error_msg = result.stderr.strip() or result.stdout.strip() or "Unknown error"
error_msg = (
result.stderr.strip() or result.stdout.strip() or "Unknown error"
)
safe_print(f"[{language}] ✗ Failed: {error_msg[:100]}")
return (language, False, error_msg[:200]) # Truncate long errors
@@ -138,7 +155,7 @@ def translate_language(language: str, api_key: str, batch_size: int, timeout: in
def main():
parser = argparse.ArgumentParser(
description='Bulk auto-translate all languages using OpenAI API',
description="Bulk auto-translate all languages using OpenAI API",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
@@ -155,35 +172,70 @@ Examples:
python3 bulk_auto_translate.py --dry-run
Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
"""
""",
)
parser.add_argument('--api-key', help='OpenAI API key (or set OPENAI_API_KEY env var)')
parser.add_argument('--parallel', type=int, default=1,
help='Number of parallel translation threads (default: 1)')
parser.add_argument('--batch-size', type=int, default=500,
help='Entries per batch for translation (default: 500)')
parser.add_argument('--timeout', type=int, default=600,
help='Timeout per batch in seconds (default: 600)')
parser.add_argument('--threshold', type=float, default=0.0,
help='Only translate languages below this completion %% (default: 0 = all)')
parser.add_argument('--languages', nargs='+',
help='Translate only specific languages (e.g., de-DE fr-FR)')
parser.add_argument('--locales-dir', default='frontend/public/locales',
help='Path to locales directory')
parser.add_argument('--skip-verification', action='store_true',
help='Skip final completion verification for each language')
parser.add_argument('--include-existing', action='store_true',
help='Also retranslate existing keys that match English (default: only translate missing keys)')
parser.add_argument('--dry-run', action='store_true',
help='Show what would be translated without actually translating')
parser.add_argument(
"--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)"
)
parser.add_argument(
"--parallel",
type=int,
default=1,
help="Number of parallel translation threads (default: 1)",
)
parser.add_argument(
"--batch-size",
type=int,
default=500,
help="Entries per batch for translation (default: 500)",
)
parser.add_argument(
"--timeout",
type=int,
default=600,
help="Timeout per batch in seconds (default: 600)",
)
parser.add_argument(
"--threshold",
type=float,
default=0.0,
help="Only translate languages below this completion %% (default: 0 = all)",
)
parser.add_argument(
"--languages",
nargs="+",
help="Translate only specific languages (e.g., de-DE fr-FR)",
)
parser.add_argument(
"--locales-dir",
default="frontend/public/locales",
help="Path to locales directory",
)
parser.add_argument(
"--skip-verification",
action="store_true",
help="Skip final completion verification for each language",
)
parser.add_argument(
"--include-existing",
action="store_true",
help="Also retranslate existing keys that match English (default: only translate missing keys)",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be translated without actually translating",
)
args = parser.parse_args()
# Verify API key (unless dry run)
api_key = args.api_key or os.environ.get('OPENAI_API_KEY')
api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
if not args.dry_run and not api_key:
print("Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable")
print(
"Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable"
)
sys.exit(1)
locales_dir = Path(args.locales_dir)
@@ -221,16 +273,16 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
print("\nNo languages below threshold!")
sys.exit(0)
print(f"\n{'='*60}")
print(f"Bulk Translation Configuration")
print(f"{'='*60}")
print(f"\n{'=' * 60}")
print("Bulk Translation Configuration")
print(f"{'=' * 60}")
print(f"Languages to translate: {len(languages)}")
print(f"Parallel threads: {args.parallel}")
print(f"Batch size: {args.batch_size}")
print(f"Timeout per batch: {args.timeout}s")
if args.threshold > 0:
print(f"Completion threshold: {args.threshold}%")
print(f"{'='*60}\n")
print(f"{'=' * 60}\n")
if args.dry_run:
print("DRY RUN - Languages that would be translated:")
@@ -244,11 +296,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
start_time = time.time()
# Translate in parallel
results = {
'success': [],
'failed': [],
'already_complete': []
}
results = {"success": [], "failed": [], "already_complete": []}
with ThreadPoolExecutor(max_workers=args.parallel) as executor:
futures = {
@@ -259,7 +307,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
args.batch_size,
args.timeout,
args.skip_verification,
args.include_existing
args.include_existing,
): lang
for lang in languages
}
@@ -269,43 +317,43 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
if success:
if message == "Already complete":
results['already_complete'].append(language)
results["already_complete"].append(language)
else:
results['success'].append(language)
results["success"].append(language)
else:
results['failed'].append((language, message))
results["failed"].append((language, message))
elapsed = time.time() - start_time
# Print summary
print("\n" + "="*60)
print("\n" + "=" * 60)
print("Bulk Translation Summary")
print("="*60)
print("=" * 60)
print(f"Total languages: {len(languages)}")
print(f"Successful: {len(results['success'])}")
print(f"Already complete: {len(results['already_complete'])}")
print(f"Failed: {len(results['failed'])}")
print(f"Time elapsed: {elapsed:.1f} seconds ({elapsed/60:.1f} minutes)")
print("="*60)
print(f"Time elapsed: {elapsed:.1f} seconds ({elapsed / 60:.1f} minutes)")
print("=" * 60)
if results['success']:
if results["success"]:
print(f"\n✅ Successfully translated ({len(results['success'])}):")
for lang in sorted(results['success']):
for lang in sorted(results["success"]):
print(f" - {lang}")
if results['already_complete']:
if results["already_complete"]:
print(f"\n✓ Already complete ({len(results['already_complete'])}):")
for lang in sorted(results['already_complete']):
for lang in sorted(results["already_complete"]):
print(f" - {lang}")
if results['failed']:
if results["failed"]:
print(f"\n❌ Failed ({len(results['failed'])}):")
for lang, msg in sorted(results['failed']):
for lang, msg in sorted(results["failed"]):
print(f" - {lang}: {msg}")
sys.exit(1)
print("\n✅ Bulk translation completed successfully!")
if __name__ == '__main__':
if __name__ == "__main__":
main()