mirror of
https://github.com/arsvendg/Stirling-PDF.git
synced 2026-07-16 19:33:11 +02:00
🤖 format everything with pre-commit by stirlingbot (#5144)
Auto-generated by [create-pull-request][1] with **stirlingbot** [1]: https://github.com/peter-evans/create-pull-request Signed-off-by: stirlingbot[bot] <stirlingbot[bot]@users.noreply.github.com> Co-authored-by: stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
This commit is contained in:
stirlingbot[bot]
co-authored by
stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
stirlingbot[bot] <195170888+stirlingbot[bot]@users.noreply.github.com>
parent
e6d3f20c36
commit
c990ab3216
@@ -19,9 +19,9 @@ import tomllib
|
||||
def run_command(cmd, description=""):
|
||||
"""Run a shell command and return success status."""
|
||||
if description:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Step: {description}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
|
||||
@@ -40,29 +40,35 @@ def find_translation_file(lang_dir):
|
||||
return toml_file
|
||||
return None
|
||||
|
||||
|
||||
def load_translation_file(file_path):
|
||||
"""Load TOML translation file."""
|
||||
with open(file_path, 'rb') as f:
|
||||
with open(file_path, "rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
def extract_untranslated(language_code, batch_size=500, include_existing=False):
|
||||
"""Extract untranslated entries and split into batches."""
|
||||
mode = "all untranslated (including existing)" if include_existing else "new (missing)"
|
||||
mode = (
|
||||
"all untranslated (including existing)" if include_existing else "new (missing)"
|
||||
)
|
||||
print(f"\n🔍 Extracting {mode} entries for {language_code}...")
|
||||
|
||||
# Load files
|
||||
golden_path = find_translation_file(Path('frontend/public/locales/en-GB'))
|
||||
lang_path = find_translation_file(Path(f'frontend/public/locales/{language_code}'))
|
||||
golden_path = find_translation_file(Path("frontend/public/locales/en-GB"))
|
||||
lang_path = find_translation_file(Path(f"frontend/public/locales/{language_code}"))
|
||||
|
||||
if not golden_path:
|
||||
print(f"Error: Golden truth file not found in frontend/public/locales/en-GB")
|
||||
print("Error: Golden truth file not found in frontend/public/locales/en-GB")
|
||||
return None
|
||||
|
||||
if not lang_path:
|
||||
print(f"Error: Language file not found in frontend/public/locales/{language_code}")
|
||||
print(
|
||||
f"Error: Language file not found in frontend/public/locales/{language_code}"
|
||||
)
|
||||
return None
|
||||
|
||||
def flatten_dict(d, parent_key='', separator='.'):
|
||||
def flatten_dict(d, parent_key="", separator="."):
|
||||
items = []
|
||||
for k, v in d.items():
|
||||
new_key = f"{parent_key}{separator}{k}" if parent_key else k
|
||||
@@ -76,7 +82,7 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
|
||||
lang_data = load_translation_file(lang_path)
|
||||
|
||||
if not golden or not lang_data:
|
||||
print(f"Error: Failed to load translation files")
|
||||
print("Error: Failed to load translation files")
|
||||
return None
|
||||
|
||||
golden_flat = flatten_dict(golden)
|
||||
@@ -87,9 +93,14 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
|
||||
for key, value in golden_flat.items():
|
||||
if include_existing:
|
||||
# Include missing keys, keys with English values, and [UNTRANSLATED] keys
|
||||
if (key not in lang_flat or
|
||||
lang_flat.get(key) == value or
|
||||
(isinstance(lang_flat.get(key), str) and lang_flat.get(key).startswith("[UNTRANSLATED]"))):
|
||||
if (
|
||||
key not in lang_flat
|
||||
or lang_flat.get(key) == value
|
||||
or (
|
||||
isinstance(lang_flat.get(key), str)
|
||||
and lang_flat.get(key).startswith("[UNTRANSLATED]")
|
||||
)
|
||||
):
|
||||
untranslated[key] = value
|
||||
else:
|
||||
# Only include missing keys (not in target file at all)
|
||||
@@ -108,16 +119,16 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
|
||||
num_batches = (total + batch_size - 1) // batch_size
|
||||
|
||||
batch_files = []
|
||||
lang_code_safe = language_code.replace('-', '_')
|
||||
lang_code_safe = language_code.replace("-", "_")
|
||||
|
||||
for i in range(num_batches):
|
||||
start = i * batch_size
|
||||
end = min((i + 1) * batch_size, total)
|
||||
batch = dict(entries[start:end])
|
||||
|
||||
filename = f'{lang_code_safe}_batch_{i+1}_of_{num_batches}.json'
|
||||
with open(filename, 'w', encoding='utf-8') as f:
|
||||
json.dump(batch, f, ensure_ascii=False, separators=(',', ':'))
|
||||
filename = f"{lang_code_safe}_batch_{i + 1}_of_{num_batches}.json"
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
json.dump(batch, f, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
batch_files.append(filename)
|
||||
print(f" Created {filename} with {len(batch)} entries")
|
||||
@@ -131,7 +142,7 @@ def translate_batches(batch_files, language_code, api_key, timeout=600):
|
||||
return []
|
||||
|
||||
print(f"\n🤖 Translating {len(batch_files)} batches using GPT-5...")
|
||||
print(f"Timeout: {timeout}s ({timeout//60} minutes) per batch")
|
||||
print(f"Timeout: {timeout}s ({timeout // 60} minutes) per batch")
|
||||
|
||||
translated_files = []
|
||||
|
||||
@@ -142,7 +153,9 @@ def translate_batches(batch_files, language_code, api_key, timeout=600):
|
||||
cmd = f'python3 scripts/translations/batch_translator.py "{batch_file}" --language {language_code} --api-key "{api_key}"'
|
||||
|
||||
# Run with timeout
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
||||
result = subprocess.run(
|
||||
cmd, shell=True, capture_output=True, text=True, timeout=timeout
|
||||
)
|
||||
|
||||
if result.stdout:
|
||||
print(result.stdout)
|
||||
@@ -153,7 +166,7 @@ def translate_batches(batch_files, language_code, api_key, timeout=600):
|
||||
print(f"✗ Failed to translate {batch_file}")
|
||||
return None
|
||||
|
||||
translated_file = batch_file.replace('.json', '_translated.json')
|
||||
translated_file = batch_file.replace(".json", "_translated.json")
|
||||
translated_files.append(translated_file)
|
||||
|
||||
# Small delay between batches
|
||||
@@ -177,14 +190,14 @@ def merge_translations(translated_files, language_code):
|
||||
print(f"Error: Translated file not found: {filename}")
|
||||
return None
|
||||
|
||||
with open(filename, 'r', encoding='utf-8') as f:
|
||||
with open(filename, "r", encoding="utf-8") as f:
|
||||
merged.update(json.load(f))
|
||||
|
||||
lang_code_safe = language_code.replace('-', '_')
|
||||
merged_file = f'{lang_code_safe}_merged.json'
|
||||
lang_code_safe = language_code.replace("-", "_")
|
||||
merged_file = f"{lang_code_safe}_merged.json"
|
||||
|
||||
with open(merged_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(merged, f, ensure_ascii=False, separators=(',', ':'))
|
||||
with open(merged_file, "w", encoding="utf-8") as f:
|
||||
json.dump(merged, f, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
print(f"✓ Merged {len(merged)} translations into {merged_file}")
|
||||
return merged_file
|
||||
@@ -194,13 +207,13 @@ def apply_translations(merged_file, language_code):
|
||||
"""Apply merged translations to the language file."""
|
||||
print(f"\n📝 Applying translations to {language_code}...")
|
||||
|
||||
cmd = f'python3 scripts/translations/translation_merger.py {language_code} apply-translations --translations-file {merged_file}'
|
||||
cmd = f"python3 scripts/translations/translation_merger.py {language_code} apply-translations --translations-file {merged_file}"
|
||||
|
||||
if not run_command(cmd):
|
||||
print(f"✗ Failed to apply translations")
|
||||
print("✗ Failed to apply translations")
|
||||
return False
|
||||
|
||||
print(f"✓ Translations applied successfully")
|
||||
print("✓ Translations applied successfully")
|
||||
return True
|
||||
|
||||
|
||||
@@ -208,27 +221,25 @@ def beautify_translations(language_code):
|
||||
"""Beautify translation file to match en-GB structure."""
|
||||
print(f"\n✨ Beautifying {language_code} translation file...")
|
||||
|
||||
cmd = f'python3 scripts/translations/toml_beautifier.py --language {language_code}'
|
||||
cmd = f"python3 scripts/translations/toml_beautifier.py --language {language_code}"
|
||||
|
||||
if not run_command(cmd):
|
||||
print(f"✗ Failed to beautify translations")
|
||||
print("✗ Failed to beautify translations")
|
||||
return False
|
||||
|
||||
print(f"✓ Translation file beautified")
|
||||
print("✓ Translation file beautified")
|
||||
return True
|
||||
|
||||
|
||||
def cleanup_temp_files(language_code):
|
||||
"""Remove temporary batch files."""
|
||||
print(f"\n🧹 Cleaning up temporary files...")
|
||||
print("\n🧹 Cleaning up temporary files...")
|
||||
|
||||
lang_code_safe = language_code.replace('-', '_')
|
||||
patterns = [
|
||||
f'{lang_code_safe}_batch_*.json',
|
||||
f'{lang_code_safe}_merged.json'
|
||||
]
|
||||
lang_code_safe = language_code.replace("-", "_")
|
||||
patterns = [f"{lang_code_safe}_batch_*.json", f"{lang_code_safe}_merged.json"]
|
||||
|
||||
import glob
|
||||
|
||||
removed = 0
|
||||
for pattern in patterns:
|
||||
for file in glob.glob(pattern):
|
||||
@@ -240,15 +251,15 @@ def cleanup_temp_files(language_code):
|
||||
|
||||
def verify_completion(language_code):
|
||||
"""Check final completion percentage."""
|
||||
print(f"\n📊 Verifying completion...")
|
||||
print("\n📊 Verifying completion...")
|
||||
|
||||
cmd = f'python3 scripts/translations/translation_analyzer.py --language {language_code} --summary'
|
||||
cmd = f"python3 scripts/translations/translation_analyzer.py --language {language_code} --summary"
|
||||
run_command(cmd)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Automated translation pipeline for Stirling PDF',
|
||||
description="Automated translation pipeline for Stirling PDF",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Note: This script works with TOML translation files.
|
||||
@@ -266,36 +277,57 @@ Examples:
|
||||
|
||||
# Skip cleanup (keep temporary files for inspection)
|
||||
python3 scripts/translations/auto_translate.py fr-FR --no-cleanup
|
||||
"""
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument('language', help='Language code (e.g., es-ES, de-DE, zh-CN)')
|
||||
parser.add_argument('--api-key', help='OpenAI API key (or set OPENAI_API_KEY env var)')
|
||||
parser.add_argument('--batch-size', type=int, default=500, help='Entries per batch (default: 500)')
|
||||
parser.add_argument('--no-cleanup', action='store_true', help='Keep temporary batch files')
|
||||
parser.add_argument('--skip-verification', action='store_true', help='Skip final completion check')
|
||||
parser.add_argument('--timeout', type=int, default=600, help='Timeout per batch in seconds (default: 600 = 10 minutes)')
|
||||
parser.add_argument('--include-existing', action='store_true', help='Also retranslate existing keys that match English (default: only translate missing keys)')
|
||||
parser.add_argument("language", help="Language code (e.g., es-ES, de-DE, zh-CN)")
|
||||
parser.add_argument(
|
||||
"--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size", type=int, default=500, help="Entries per batch (default: 500)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-cleanup", action="store_true", help="Keep temporary batch files"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-verification", action="store_true", help="Skip final completion check"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=600,
|
||||
help="Timeout per batch in seconds (default: 600 = 10 minutes)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-existing",
|
||||
action="store_true",
|
||||
help="Also retranslate existing keys that match English (default: only translate missing keys)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Verify API key
|
||||
api_key = args.api_key or os.environ.get('OPENAI_API_KEY')
|
||||
api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable")
|
||||
print(
|
||||
"Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
print("="*60)
|
||||
print(f"Automated Translation Pipeline")
|
||||
print("=" * 60)
|
||||
print("Automated Translation Pipeline")
|
||||
print(f"Language: {args.language}")
|
||||
print(f"Batch Size: {args.batch_size} entries")
|
||||
print("="*60)
|
||||
print("=" * 60)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Step 1: Extract and split
|
||||
batch_files = extract_untranslated(args.language, args.batch_size, args.include_existing)
|
||||
batch_files = extract_untranslated(
|
||||
args.language, args.batch_size, args.include_existing
|
||||
)
|
||||
if batch_files is None:
|
||||
sys.exit(1)
|
||||
|
||||
@@ -304,7 +336,9 @@ Examples:
|
||||
sys.exit(0)
|
||||
|
||||
# Step 2: Translate all batches
|
||||
translated_files = translate_batches(batch_files, args.language, api_key, args.timeout)
|
||||
translated_files = translate_batches(
|
||||
batch_files, args.language, api_key, args.timeout
|
||||
)
|
||||
if translated_files is None:
|
||||
sys.exit(1)
|
||||
|
||||
@@ -330,10 +364,10 @@ Examples:
|
||||
verify_completion(args.language)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print("\n" + "="*60)
|
||||
print(f"✅ Translation pipeline completed successfully!")
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ Translation pipeline completed successfully!")
|
||||
print(f"Time elapsed: {elapsed:.1f} seconds")
|
||||
print("="*60)
|
||||
print("=" * 60)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠ Translation interrupted by user")
|
||||
@@ -341,6 +375,7 @@ Examples:
|
||||
except Exception as e:
|
||||
print(f"\n\n✗ Error: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user