mirror of
https://github.com/LibreTranslate/LibreTranslate.git
synced 2024-05-23 19:48:13 +00:00
Merge 1de8d84817 into a403ba7e3a
This commit is contained in:
Pit
GitHub
commit
1b53abbce6
|
|
@ -47,7 +47,8 @@ def get_version():
|
|||
|
||||
|
||||
def get_upload_dir():
|
||||
upload_dir = os.path.join(tempfile.gettempdir(), "libretranslate-files-translate")
|
||||
upload_dir = os.path.join(tempfile.gettempdir(),
|
||||
"libretranslate-files-translate")
|
||||
|
||||
if not os.path.isdir(upload_dir):
|
||||
os.mkdir(upload_dir)
|
||||
|
|
@ -64,6 +65,7 @@ def get_req_api_key():
|
|||
|
||||
return ak
|
||||
|
||||
|
||||
def get_req_secret():
|
||||
if request.is_json:
|
||||
json = get_json_dict(request)
|
||||
|
|
@ -149,6 +151,12 @@ def get_routes_limits(args, api_keys_db):
|
|||
return res
|
||||
|
||||
|
||||
def clean_text(text):
|
||||
pattern = re.compile(r'(?<=[^.!?])\n')
|
||||
cleaned_text = re.sub(pattern, ' ', text)
|
||||
return cleaned_text
|
||||
|
||||
|
||||
def create_app(args):
|
||||
from libretranslate.init import boot
|
||||
|
||||
|
|
@ -168,7 +176,8 @@ def create_app(args):
|
|||
languages = load_languages()
|
||||
language_pairs = {}
|
||||
for lang in languages:
|
||||
language_pairs[lang.code] = sorted([l.to_lang.code for l in lang.translations_from])
|
||||
language_pairs[lang.code] = sorted(
|
||||
[l.to_lang.code for l in lang.translations_from])
|
||||
|
||||
# Map userdefined frontend languages to argos language object.
|
||||
if args.frontend_language_source == "auto":
|
||||
|
|
@ -177,14 +186,15 @@ def create_app(args):
|
|||
)
|
||||
else:
|
||||
frontend_argos_language_source = next(
|
||||
iter([l for l in languages if l.code == args.frontend_language_source]),
|
||||
iter([l for l in languages if l.code ==
|
||||
args.frontend_language_source]),
|
||||
None,
|
||||
)
|
||||
if frontend_argos_language_source is None:
|
||||
frontend_argos_language_source = languages[0]
|
||||
|
||||
|
||||
language_target_fallback = languages[1] if len(languages) >= 2 else languages[0]
|
||||
language_target_fallback = languages[1] if len(
|
||||
languages) >= 2 else languages[0]
|
||||
|
||||
if args.frontend_language_target == "locale":
|
||||
def resolve_language_locale():
|
||||
|
|
@ -199,11 +209,13 @@ def create_app(args):
|
|||
frontend_argos_language_target = resolve_language_locale
|
||||
else:
|
||||
language_target = next(
|
||||
iter([l for l in languages if l.code == args.frontend_language_target]), None
|
||||
iter([l for l in languages if l.code ==
|
||||
args.frontend_language_target]), None
|
||||
)
|
||||
if language_target is None:
|
||||
language_target = language_target_fallback
|
||||
frontend_argos_language_target = lambda: language_target
|
||||
|
||||
def frontend_argos_language_target(): return language_target
|
||||
|
||||
frontend_argos_supported_files_format = []
|
||||
|
||||
|
|
@ -216,7 +228,8 @@ def create_app(args):
|
|||
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0 or args.hourly_req_limit > 0:
|
||||
api_keys_db = None
|
||||
if args.api_keys:
|
||||
api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path)
|
||||
api_keys_db = RemoteDatabase(
|
||||
args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path)
|
||||
|
||||
from flask_limiter import Limiter
|
||||
|
||||
|
|
@ -233,7 +246,8 @@ def create_app(args):
|
|||
args, api_keys_db
|
||||
),
|
||||
storage_uri=args.req_limit_storage,
|
||||
default_limits_deduct_when=lambda req: True, # Force cost to be called after the request
|
||||
# Force cost to be called after the request
|
||||
default_limits_deduct_when=lambda req: True,
|
||||
default_limits_cost=limits_cost
|
||||
)
|
||||
else:
|
||||
|
|
@ -271,10 +285,12 @@ def create_app(args):
|
|||
multiprocess.MultiProcessCollector(registry)
|
||||
return Response(generate_latest(registry), mimetype=CONTENT_TYPE_LATEST)
|
||||
|
||||
measure_request = Summary('libretranslate_http_request_duration_seconds', 'Time spent on request', ['endpoint', 'status', 'request_ip', 'api_key'])
|
||||
measure_request = Summary('libretranslate_http_request_duration_seconds', 'Time spent on request', [
|
||||
'endpoint', 'status', 'request_ip', 'api_key'])
|
||||
measure_request.labels('/translate', 200, '127.0.0.1', '')
|
||||
|
||||
gauge_request = Gauge('libretranslate_http_requests_in_flight', 'Active requests', ['endpoint', 'request_ip', 'api_key'], multiprocess_mode='livesum')
|
||||
gauge_request = Gauge('libretranslate_http_requests_in_flight', 'Active requests', [
|
||||
'endpoint', 'request_ip', 'api_key'], multiprocess_mode='livesum')
|
||||
gauge_request.labels('/translate', '127.0.0.1', '')
|
||||
|
||||
def access_check(f):
|
||||
|
|
@ -309,9 +325,11 @@ def create_app(args):
|
|||
need_key = True
|
||||
|
||||
if need_key:
|
||||
description = _("Please contact the server operator to get an API key")
|
||||
description = _(
|
||||
"Please contact the server operator to get an API key")
|
||||
if args.get_api_key_link:
|
||||
description = _("Visit %(url)s to get an API key", url=args.get_api_key_link)
|
||||
description = _(
|
||||
"Visit %(url)s to get an API key", url=args.get_api_key_link)
|
||||
abort(
|
||||
400,
|
||||
description=description,
|
||||
|
|
@ -334,7 +352,8 @@ def create_app(args):
|
|||
raise e
|
||||
finally:
|
||||
request.duration = max(default_timer() - start_t, 0)
|
||||
measure_request.labels(request.path, status, ip, ak).observe(request.duration)
|
||||
measure_request.labels(
|
||||
request.path, status, ip, ak).observe(request.duration)
|
||||
g.dec()
|
||||
return measure_func
|
||||
else:
|
||||
|
|
@ -383,7 +402,8 @@ def create_app(args):
|
|||
web_version=os.environ.get("LT_WEB") is not None,
|
||||
version=get_version(),
|
||||
swagger_url=swagger_url,
|
||||
available_locales=[{'code': l['code'], 'name': _lazy(l['name'])} for l in get_available_locales(not args.debug)],
|
||||
available_locales=[{'code': l['code'], 'name': _lazy(
|
||||
l['name'])} for l in get_available_locales(not args.debug)],
|
||||
current_locale=get_locale(),
|
||||
alternate_locales=get_alternate_locale_links()
|
||||
)
|
||||
|
|
@ -565,17 +585,17 @@ def create_app(args):
|
|||
text_format = request.values.get("format")
|
||||
|
||||
if not q:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='q'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='q'))
|
||||
if not source_lang:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='source'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='source'))
|
||||
if not target_lang:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='target'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='target'))
|
||||
|
||||
if not request.is_json:
|
||||
# Normalize line endings to UNIX style (LF) only so we can consistently
|
||||
# enforce character limits.
|
||||
# https://www.rfc-editor.org/rfc/rfc2046#section-4.1.1
|
||||
q = "\n".join(q.splitlines())
|
||||
q = clean_text(q)
|
||||
|
||||
char_limit = get_char_limit(args.char_limit, api_keys_db)
|
||||
|
||||
|
|
@ -586,7 +606,8 @@ def create_app(args):
|
|||
if args.batch_limit < batch_size:
|
||||
abort(
|
||||
400,
|
||||
description=_("Invalid request: request (%(size)s) exceeds text limit (%(limit)s)", size=batch_size, limit=args.batch_limit),
|
||||
description=_("Invalid request: request (%(size)s) exceeds text limit (%(limit)s)",
|
||||
size=batch_size, limit=args.batch_limit),
|
||||
)
|
||||
|
||||
src_texts = q if batch else [q]
|
||||
|
|
@ -596,7 +617,8 @@ def create_app(args):
|
|||
if len(text) > char_limit:
|
||||
abort(
|
||||
400,
|
||||
description=_("Invalid request: request (%(size)s) exceeds text limit (%(limit)s)", size=len(text), limit=char_limit),
|
||||
description=_("Invalid request: request (%(size)s) exceeds text limit (%(limit)s)", size=len(
|
||||
text), limit=char_limit),
|
||||
)
|
||||
|
||||
if batch:
|
||||
|
|
@ -608,12 +630,14 @@ def create_app(args):
|
|||
else:
|
||||
detected_src_lang = {"confidence": 100.0, "language": source_lang}
|
||||
|
||||
src_lang = next(iter([l for l in languages if l.code == detected_src_lang["language"]]), None)
|
||||
src_lang = next(
|
||||
iter([l for l in languages if l.code == detected_src_lang["language"]]), None)
|
||||
|
||||
if src_lang is None:
|
||||
abort(400, description=_("%(lang)s is not supported", lang=source_lang))
|
||||
|
||||
tgt_lang = next(iter([l for l in languages if l.code == target_lang]), None)
|
||||
tgt_lang = next(
|
||||
iter([l for l in languages if l.code == target_lang]), None)
|
||||
|
||||
if tgt_lang is None:
|
||||
abort(400, description=_("%(lang)s is not supported", lang=target_lang))
|
||||
|
|
@ -622,7 +646,8 @@ def create_app(args):
|
|||
text_format = "text"
|
||||
|
||||
if text_format not in ["text", "html"]:
|
||||
abort(400, description=_("%(format)s format is not supported", format=text_format))
|
||||
abort(400, description=_(
|
||||
"%(format)s format is not supported", format=text_format))
|
||||
|
||||
try:
|
||||
if batch:
|
||||
|
|
@ -630,12 +655,14 @@ def create_app(args):
|
|||
for text in q:
|
||||
translator = src_lang.get_translation(tgt_lang)
|
||||
if translator is None:
|
||||
abort(400, description=_("%(tname)s (%(tcode)s) is not available as a target language from %(sname)s (%(scode)s)", tname=_lazy(tgt_lang.name), tcode=tgt_lang.code, sname=_lazy(src_lang.name), scode=src_lang.code))
|
||||
abort(400, description=_("%(tname)s (%(tcode)s) is not available as a target language from %(sname)s (%(scode)s)", tname=_lazy(
|
||||
tgt_lang.name), tcode=tgt_lang.code, sname=_lazy(src_lang.name), scode=src_lang.code))
|
||||
|
||||
if text_format == "html":
|
||||
translated_text = str(translate_html(translator, text))
|
||||
else:
|
||||
translated_text = improve_translation_formatting(text, translator.translate(text))
|
||||
translated_text = improve_translation_formatting(
|
||||
text, translator.translate(text))
|
||||
|
||||
results.append(unescape(translated_text))
|
||||
if source_lang == "auto":
|
||||
|
|
@ -654,12 +681,14 @@ def create_app(args):
|
|||
else:
|
||||
translator = src_lang.get_translation(tgt_lang)
|
||||
if translator is None:
|
||||
abort(400, description=_("%(tname)s (%(tcode)s) is not available as a target language from %(sname)s (%(scode)s)", tname=_lazy(tgt_lang.name), tcode=tgt_lang.code, sname=_lazy(src_lang.name), scode=src_lang.code))
|
||||
abort(400, description=_("%(tname)s (%(tcode)s) is not available as a target language from %(sname)s (%(scode)s)", tname=_lazy(
|
||||
tgt_lang.name), tcode=tgt_lang.code, sname=_lazy(src_lang.name), scode=src_lang.code))
|
||||
|
||||
if text_format == "html":
|
||||
translated_text = str(translate_html(translator, q))
|
||||
else:
|
||||
translated_text = improve_translation_formatting(q, translator.translate(q))
|
||||
translated_text = improve_translation_formatting(
|
||||
q, translator.translate(q))
|
||||
|
||||
if source_lang == "auto":
|
||||
return jsonify(
|
||||
|
|
@ -676,7 +705,8 @@ def create_app(args):
|
|||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
abort(500, description=_("Cannot translate text: %(text)s", text=str(e)))
|
||||
abort(500, description=_(
|
||||
"Cannot translate text: %(text)s", text=str(e)))
|
||||
|
||||
@bp.post("/translate_file")
|
||||
@access_check
|
||||
|
|
@ -763,7 +793,8 @@ def create_app(args):
|
|||
description: Error message
|
||||
"""
|
||||
if args.disable_files_translation:
|
||||
abort(403, description=_("Files translation are disabled on this server."))
|
||||
abort(403, description=_(
|
||||
"Files translation are disabled on this server."))
|
||||
|
||||
source_lang = request.form.get("source")
|
||||
target_lang = request.form.get("target")
|
||||
|
|
@ -771,11 +802,14 @@ def create_app(args):
|
|||
char_limit = get_char_limit(args.char_limit, api_keys_db)
|
||||
|
||||
if not file:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='file'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='file'))
|
||||
if not source_lang:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='source'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='source'))
|
||||
if not target_lang:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='target'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='target'))
|
||||
|
||||
if file.filename == '':
|
||||
abort(400, description=_("Invalid request: empty file"))
|
||||
|
|
@ -783,12 +817,14 @@ def create_app(args):
|
|||
if os.path.splitext(file.filename)[1] not in frontend_argos_supported_files_format:
|
||||
abort(400, description=_("Invalid request: file format not supported"))
|
||||
|
||||
src_lang = next(iter([l for l in languages if l.code == source_lang]), None)
|
||||
src_lang = next(
|
||||
iter([l for l in languages if l.code == source_lang]), None)
|
||||
|
||||
if src_lang is None:
|
||||
abort(400, description=_("%(lang)s is not supported", lang=source_lang))
|
||||
|
||||
tgt_lang = next(iter([l for l in languages if l.code == target_lang]), None)
|
||||
tgt_lang = next(
|
||||
iter([l for l in languages if l.code == target_lang]), None)
|
||||
|
||||
if tgt_lang is None:
|
||||
abort(400, description=_("%(lang)s is not supported", lang=target_lang))
|
||||
|
|
@ -805,9 +841,11 @@ def create_app(args):
|
|||
# roughly equivalent to a batch process of N batches assuming
|
||||
# each batch uses all available limits
|
||||
if char_limit > 0:
|
||||
request.req_cost = max(1, int(os.path.getsize(filepath) / char_limit))
|
||||
request.req_cost = max(
|
||||
1, int(os.path.getsize(filepath) / char_limit))
|
||||
|
||||
translated_file_path = argostranslatefiles.translate_file(src_lang.get_translation(tgt_lang), filepath)
|
||||
translated_file_path = argostranslatefiles.translate_file(
|
||||
src_lang.get_translation(tgt_lang), filepath)
|
||||
translated_filename = os.path.basename(translated_file_path)
|
||||
|
||||
return jsonify(
|
||||
|
|
@ -824,11 +862,13 @@ def create_app(args):
|
|||
Download a translated file
|
||||
"""
|
||||
if args.disable_files_translation:
|
||||
abort(400, description=_("Files translation are disabled on this server."))
|
||||
abort(400, description=_(
|
||||
"Files translation are disabled on this server."))
|
||||
|
||||
filepath = os.path.join(get_upload_dir(), filename)
|
||||
try:
|
||||
checked_filepath = security.path_traversal_check(filepath, get_upload_dir())
|
||||
checked_filepath = security.path_traversal_check(
|
||||
filepath, get_upload_dir())
|
||||
if os.path.isfile(checked_filepath):
|
||||
filepath = checked_filepath
|
||||
except security.SuspiciousFileOperationError:
|
||||
|
|
@ -932,7 +972,8 @@ def create_app(args):
|
|||
q = request.values.get("q")
|
||||
|
||||
if not q:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='q'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='q'))
|
||||
|
||||
return jsonify(detect_languages(q))
|
||||
|
||||
|
|
@ -1089,13 +1130,17 @@ def create_app(args):
|
|||
target_lang = request.values.get("target")
|
||||
|
||||
if not q:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='q'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='q'))
|
||||
if not s:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='s'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='s'))
|
||||
if not source_lang:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='source'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='source'))
|
||||
if not target_lang:
|
||||
abort(400, description=_("Invalid request: missing %(name)s parameter", name='target'))
|
||||
abort(400, description=_(
|
||||
"Invalid request: missing %(name)s parameter", name='target'))
|
||||
|
||||
SuggestionsDatabase().add(q, s, source_lang, target_lang)
|
||||
return jsonify({"success": True})
|
||||
|
|
|
|||
Loading…
Reference in a new issue