#!/usr/bin/env python import os import sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import json import re import polib from babel.messages.frontend import main as pybabel from flask_swagger import swagger from translatehtml import translate_html from libretranslate.app import create_app, get_version from libretranslate.language import improve_translation_formatting, load_languages from libretranslate.locales import get_available_locale_codes, swag_eval from libretranslate.main import get_args # Update strings if __name__ == "__main__": print("Loading languages") languages = load_languages() en_lang = next((l for l in languages if l.code == 'en'), None) if en_lang is None: print("Error: English model not found. You need it to run this script.") exit(1) locales_dir = os.path.join("libretranslate", "locales") if not os.path.isdir(locales_dir): os.makedirs(locales_dir) # Dump language list so it gets picked up by pybabel langs_file = os.path.join(locales_dir, ".langs.py") lang_names = {} with open(langs_file, 'w') as f: for l in languages: f.write("_(%s)\n" % json.dumps(l.name)) lang_names[l.name] = True # Additional names that might be missing for l in ["Serbian", "Ukrainian", "Vietnamese"]: if not l in lang_names: f.write("_(%s)\n" % json.dumps(l)) lang_names[l] = True print("Wrote %s" % langs_file) # Dump swagger strings args = get_args() app = create_app(args) swag = swagger(app) swag_strings = [] def add_swag_string(s): if not s in swag_strings: swag_strings.append(s) swag_eval(swag, add_swag_string) swag_file = os.path.join(locales_dir, ".swag.py") with open(swag_file, 'w') as f: for ss in swag_strings: f.write("_(%s)\n" % json.dumps(ss)) print("Wrote %s" % swag_file) messagespot = os.path.join(locales_dir, "messages.pot") print("Updating %s" % messagespot) sys.argv = ["", "extract", "-F", "babel.cfg", "-k", "_e _h", "--copyright-holder", "LibreTranslate Authors", "--project", "LibreTranslate", "--version", get_version(), "-o", messagespot, "libretranslate"] pybabel() lang_map = { 'zt': 'zh_Hant' } lang_codes = [lang_map.get(l.code, l.code) for l in languages if l.code != "en"] all_folders = [d for d in os.listdir(locales_dir) if os.path.isdir(os.path.join(locales_dir, d))] review_map = {} # Init/update for l in all_folders: cmd = "init" if os.path.isdir(os.path.join(locales_dir, l, "LC_MESSAGES")): cmd = "update" sys.argv = ["", cmd, "-i", messagespot, "-d", locales_dir, "-l", l] + (["--no-fuzzy-matching"] if cmd == "update" else []) pybabel() meta_file = os.path.join(locales_dir, l, "meta.json") reviewed = False if not os.path.isfile(meta_file): with open(meta_file, 'w') as f: f.write(json.dumps({ 'name': l if l not in lang_codes else next(lang.name for lang in languages if lang_map.get(lang.code, lang.code) == l), 'reviewed': False }, indent=4)) print("Wrote %s" % meta_file) else: with open(meta_file) as f: reviewed = json.loads(f.read()).get('reviewed', False) review_map[l] = reviewed # Automatically translate strings with libretranslate # when a language model is available and a string is empty locales = get_available_locale_codes(only_reviewed=False) for locale in locales: if locale == 'en': continue if review_map.get(locale): # Don't automatically translate reviewed languages continue tgt_lang = next((l for l in languages if l.code == locale), None) if tgt_lang is None: # We cannot translate continue translator = en_lang.get_translation(tgt_lang) messages_file = os.path.join(locales_dir, locale, "LC_MESSAGES", 'messages.po') if os.path.isfile(messages_file): print("Translating '%s'" % locale) pofile = polib.pofile(messages_file) c = 0 for entry in pofile.untranslated_entries(): text = entry.msgid # Extract placeholders placeholders = re.findall(r'%\(?[^\)]*\)?s', text) for p in range(0, len(placeholders)): text = text.replace(placeholders[p], "%s" % p) if len(placeholders) > 0: translated = str(translate_html(translator, text)) else: translated = improve_translation_formatting(text, translator.translate(text)) # Restore placeholders for p in range(0, len(placeholders)): tag = "%s" % p if tag in translated: translated = translated.replace(tag, placeholders[p]) else: # Meh, append translated += " " + placeholders[p] print(entry.msgid, " --> ", translated) entry.msgstr = translated c += 1 if c > 0: pofile.save(messages_file) print("Saved %s" % messages_file)