2021-10-24 17:14:09 +00:00
import io
2021-02-15 18:30:28 +00:00
import os
2021-10-24 16:27:37 +00:00
import tempfile
2023-01-23 21:15:23 +00:00
import re
2021-10-24 16:27:37 +00:00
import uuid
2021-05-18 03:51:33 +00:00
from functools import wraps
2022-02-20 08:09:02 +00:00
from html import unescape
2022-12-26 21:10:43 +00:00
from timeit import default_timer
2023-03-09 18:59:25 +00:00
from datetime import datetime
2021-05-18 03:51:33 +00:00
2021-10-24 16:27:37 +00:00
import argostranslatefiles
from argostranslatefiles import get_supported_formats
2022-12-31 04:23:50 +00:00
from flask import ( abort , Blueprint , Flask , jsonify , render_template , request ,
2023-01-06 16:50:51 +00:00
Response , send_file , url_for , session )
2020-12-20 20:05:22 +00:00
from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint
2023-01-06 16:50:51 +00:00
from flask_session import Session
2021-10-24 16:27:37 +00:00
from translatehtml import translate_html
from werkzeug . utils import secure_filename
2022-12-26 21:10:43 +00:00
from werkzeug . exceptions import HTTPException
2023-03-09 19:05:09 +00:00
from werkzeug . http import http_date
2023-01-05 18:12:35 +00:00
from flask_babel import Babel
2021-05-18 03:51:33 +00:00
2023-03-10 04:07:12 +00:00
from libretranslate import scheduler , flood , secret , remove_translated_files , security , storage
2022-12-29 23:44:53 +00:00
from libretranslate . language import detect_languages , improve_translation_formatting
2023-01-06 15:27:39 +00:00
from libretranslate . locales import ( _ , _lazy , get_available_locales , get_available_locale_codes , gettext_escaped ,
2023-01-05 19:36:50 +00:00
gettext_html , lazy_swag , get_alternate_locale_links )
2022-02-20 08:09:02 +00:00
2022-06-21 18:57:32 +00:00
from . api_keys import Database , RemoteDatabase
2021-10-09 09:44:00 +00:00
from . suggestions import Database as SuggestionsDatabase
2021-02-15 18:30:28 +00:00
2021-10-26 19:32:06 +00:00
def get_version ( ) :
try :
with open ( " VERSION " ) as f :
return f . read ( ) . strip ( )
except :
return " ? "
2021-11-05 13:55:45 +00:00
2021-10-25 09:06:39 +00:00
def get_upload_dir ( ) :
2021-10-25 09:46:49 +00:00
upload_dir = os . path . join ( tempfile . gettempdir ( ) , " libretranslate-files-translate " )
if not os . path . isdir ( upload_dir ) :
os . mkdir ( upload_dir )
return upload_dir
2021-11-05 13:55:45 +00:00
2021-10-26 21:13:51 +00:00
def get_req_api_key ( ) :
if request . is_json :
json = get_json_dict ( request )
ak = json . get ( " api_key " )
else :
ak = request . values . get ( " api_key " )
return ak
2021-10-25 09:46:49 +00:00
2023-03-09 18:59:25 +00:00
def get_req_secret ( ) :
if request . is_json :
json = get_json_dict ( request )
ak = json . get ( " secret " )
else :
ak = request . values . get ( " secret " )
return ak
2021-11-05 13:55:45 +00:00
2021-03-07 22:23:25 +00:00
def get_json_dict ( request ) :
d = request . get_json ( )
if not isinstance ( d , dict ) :
2023-01-04 17:15:18 +00:00
abort ( 400 , description = _ ( " Invalid JSON format " ) )
2021-03-07 22:23:25 +00:00
return d
2021-05-18 03:41:02 +00:00
2020-12-21 16:16:49 +00:00
def get_remote_address ( ) :
if request . headers . getlist ( " X-Forwarded-For " ) :
2021-09-08 19:41:12 +00:00
ip = request . headers . getlist ( " X-Forwarded-For " ) [ 0 ] . split ( " , " ) [ 0 ]
2020-12-21 16:16:49 +00:00
else :
2021-05-18 03:41:02 +00:00
ip = request . remote_addr or " 127.0.0.1 "
2020-12-21 16:16:49 +00:00
return ip
2020-12-20 20:05:22 +00:00
2021-06-03 14:36:25 +00:00
2021-10-24 16:27:37 +00:00
def get_req_limits ( default_limit , api_keys_db , multiplier = 1 ) :
2021-06-03 14:36:25 +00:00
req_limit = default_limit
if api_keys_db :
2021-10-26 21:13:51 +00:00
api_key = get_req_api_key ( )
2021-06-03 14:36:25 +00:00
if api_key :
db_req_limit = api_keys_db . lookup ( api_key )
if db_req_limit is not None :
2021-07-04 16:36:13 +00:00
req_limit = db_req_limit * multiplier
2021-10-24 16:27:37 +00:00
2021-06-03 14:36:25 +00:00
return req_limit
2021-05-16 14:57:19 +00:00
def get_routes_limits ( default_req_limit , daily_req_limit , api_keys_db ) :
2021-02-15 18:30:28 +00:00
if default_req_limit == - 1 :
# TODO: better way?
default_req_limit = 9999999999999
2021-06-03 14:36:25 +00:00
def minute_limits ( ) :
return " %s per minute " % get_req_limits ( default_req_limit , api_keys_db )
2021-02-15 18:30:28 +00:00
2021-06-03 14:36:25 +00:00
def daily_limits ( ) :
return " %s per day " % get_req_limits ( daily_req_limit , api_keys_db , 1440 )
2021-03-31 15:57:02 +00:00
2021-06-03 14:36:25 +00:00
res = [ minute_limits ]
2021-05-16 14:57:19 +00:00
if daily_req_limit > 0 :
2021-06-03 14:36:25 +00:00
res . append ( daily_limits )
2021-05-16 14:57:19 +00:00
return res
2021-02-15 18:30:28 +00:00
2021-05-17 15:41:15 +00:00
2021-02-15 18:30:28 +00:00
def create_app ( args ) :
2022-12-29 23:44:53 +00:00
from libretranslate . init import boot
2021-05-18 03:41:02 +00:00
2022-12-20 16:13:56 +00:00
boot ( args . load_only , args . update_models )
2021-01-19 16:51:10 +00:00
2022-12-29 23:44:53 +00:00
from libretranslate . language import load_languages
2021-05-18 03:41:02 +00:00
2022-12-31 21:44:25 +00:00
SWAGGER_URL = args . url_prefix + " /docs " # Swagger UI (w/o trailing '/')
API_URL = args . url_prefix + " /spec "
2022-12-31 04:23:50 +00:00
bp = Blueprint ( ' Main app ' , __name__ )
2020-12-20 20:05:22 +00:00
2023-03-09 21:09:04 +00:00
storage . setup ( args . shared_storage )
2021-10-25 15:12:09 +00:00
if not args . disable_files_translation :
remove_translated_files . setup ( get_upload_dir ( ) )
2022-03-04 08:23:11 +00:00
languages = load_languages ( )
2022-12-09 21:36:12 +00:00
language_pairs = { }
for lang in languages :
2022-12-10 05:03:21 +00:00
language_pairs [ lang . code ] = sorted ( [ l . to_lang . code for l in lang . translations_from ] )
2021-10-25 15:12:09 +00:00
2021-01-10 08:07:56 +00:00
# Map userdefined frontend languages to argos language object.
2021-02-15 18:30:28 +00:00
if args . frontend_language_source == " auto " :
2021-05-18 03:41:02 +00:00
frontend_argos_language_source = type (
2023-01-04 17:15:18 +00:00
" obj " , ( object , ) , { " code " : " auto " , " name " : _ ( " Auto Detect " ) }
2021-05-18 03:41:02 +00:00
)
2021-01-15 17:01:16 +00:00
else :
2021-05-18 03:41:02 +00:00
frontend_argos_language_source = next (
iter ( [ l for l in languages if l . code == args . frontend_language_source ] ) ,
None ,
)
2023-01-15 17:07:51 +00:00
if frontend_argos_language_source is None :
frontend_argos_language_source = languages [ 0 ]
2021-01-19 16:51:10 +00:00
2023-01-15 17:07:51 +00:00
if len ( languages ) > = 2 :
language_target_fallback = languages [ 1 ]
else :
language_target_fallback = languages [ 0 ]
if args . frontend_language_target == " locale " :
def resolve_language_locale ( ) :
loc = get_locale ( )
language_target = next (
iter ( [ l for l in languages if l . code == loc ] ) , None
)
if language_target is None :
language_target = language_target_fallback
return language_target
frontend_argos_language_target = resolve_language_locale
else :
language_target = next (
iter ( [ l for l in languages if l . code == args . frontend_language_target ] ) , None
)
if language_target is None :
language_target = language_target_fallback
frontend_argos_language_target = lambda : language_target
2021-01-19 16:51:10 +00:00
2021-10-24 14:57:45 +00:00
frontend_argos_supported_files_format = [ ]
for file_format in get_supported_formats ( ) :
for ff in file_format . supported_file_extensions :
frontend_argos_supported_files_format . append ( ff )
2021-05-17 15:41:15 +00:00
api_keys_db = None
2021-01-10 08:07:56 +00:00
2021-05-16 14:57:19 +00:00
if args . req_limit > 0 or args . api_keys or args . daily_req_limit > 0 :
2022-06-21 18:57:32 +00:00
api_keys_db = None
if args . api_keys :
if args . api_keys_remote :
api_keys_db = RemoteDatabase ( args . api_keys_remote )
else :
2022-07-15 11:22:04 +00:00
api_keys_db = Database ( args . api_keys_db_path )
2021-05-17 15:41:15 +00:00
2020-12-20 20:05:22 +00:00
from flask_limiter import Limiter
2021-05-18 03:41:02 +00:00
2020-12-20 20:05:22 +00:00
limiter = Limiter (
key_func = get_remote_address ,
2021-05-18 03:41:02 +00:00
default_limits = get_routes_limits (
args . req_limit , args . daily_req_limit , api_keys_db
) ,
2022-10-14 17:27:34 +00:00
storage_uri = args . req_limit_storage ,
2020-12-20 20:05:22 +00:00
)
2021-02-26 14:58:29 +00:00
else :
2021-05-18 03:41:02 +00:00
from . no_limiter import Limiter
limiter = Limiter ( )
2023-03-10 04:07:12 +00:00
if not " gunicorn " in os . environ . get ( " SERVER_SOFTWARE " , " " ) :
# Gunicorn starts the scheduler in the master process
scheduler . setup ( args )
flood . setup ( args )
secret . setup ( args )
2020-12-20 20:05:22 +00:00
2022-12-26 21:10:43 +00:00
measure_request = None
gauge_request = None
if args . metrics :
2023-02-05 23:37:23 +00:00
if os . environ . get ( " PROMETHEUS_MULTIPROC_DIR " ) is None :
default_mp_dir = os . path . abspath ( os . path . join ( " db " , " prometheus " ) )
if not os . path . isdir ( default_mp_dir ) :
os . mkdir ( default_mp_dir )
os . environ [ " PROMETHEUS_MULTIPROC_DIR " ] = default_mp_dir
2022-12-26 21:10:43 +00:00
from prometheus_client import CONTENT_TYPE_LATEST , Summary , Gauge , CollectorRegistry , multiprocess , generate_latest
2022-12-31 04:23:50 +00:00
@bp.route ( " /metrics " )
2022-12-27 01:00:55 +00:00
@limiter.exempt
2022-12-26 21:10:43 +00:00
def prometheus_metrics ( ) :
if args . metrics_auth_token :
authorization = request . headers . get ( ' Authorization ' )
if authorization != " Bearer " + args . metrics_auth_token :
2023-01-05 18:12:35 +00:00
abort ( 401 , description = _ ( " Unauthorized " ) )
2023-02-05 23:39:06 +00:00
2022-12-26 21:10:43 +00:00
registry = CollectorRegistry ( )
multiprocess . MultiProcessCollector ( registry )
return Response ( generate_latest ( registry ) , mimetype = CONTENT_TYPE_LATEST )
2022-12-26 21:49:04 +00:00
measure_request = Summary ( ' libretranslate_http_request_duration_seconds ' , ' Time spent on request ' , [ ' endpoint ' , ' status ' , ' request_ip ' , ' api_key ' ] )
2022-12-26 21:10:43 +00:00
measure_request . labels ( ' /translate ' , 200 , ' 127.0.0.1 ' , ' ' )
2022-12-26 21:49:04 +00:00
gauge_request = Gauge ( ' libretranslate_http_requests_in_flight ' , ' Active requests ' , [ ' endpoint ' , ' request_ip ' , ' api_key ' ] , multiprocess_mode = ' livesum ' )
2022-12-26 21:10:43 +00:00
gauge_request . labels ( ' /translate ' , ' 127.0.0.1 ' , ' ' )
2021-05-17 15:41:15 +00:00
def access_check ( f ) :
@wraps ( f )
def func ( * a , * * kw ) :
2021-11-05 13:55:45 +00:00
ip = get_remote_address ( )
if flood . is_banned ( ip ) :
2023-01-05 18:12:35 +00:00
abort ( 403 , description = _ ( " Too many request limits violations " ) )
2021-05-18 03:41:02 +00:00
2022-02-20 08:06:29 +00:00
if args . api_keys :
2021-10-26 21:13:51 +00:00
ak = get_req_api_key ( )
2023-03-09 18:59:25 +00:00
if ak and api_keys_db . lookup ( ak ) is None :
2022-02-20 08:06:29 +00:00
abort (
403 ,
2023-01-05 18:12:35 +00:00
description = _ ( " Invalid API key " ) ,
2022-02-20 08:06:29 +00:00
)
2023-03-09 18:59:25 +00:00
else :
need_key = False
key_missing = api_keys_db . lookup ( ak ) is None
if ( args . require_api_key_origin
and key_missing
and not re . match ( args . require_api_key_origin , request . headers . get ( " Origin " , " " ) )
) :
need_key = True
if ( args . require_api_key_secret
and key_missing
2023-03-09 21:09:04 +00:00
and not secret . secret_match ( get_req_secret ( ) )
2023-03-09 18:59:25 +00:00
) :
need_key = True
2023-03-09 21:09:04 +00:00
2023-03-09 18:59:25 +00:00
if need_key :
2023-01-05 18:12:35 +00:00
description = _ ( " Please contact the server operator to get an API key " )
2022-06-21 20:39:08 +00:00
if args . get_api_key_link :
2023-01-05 18:12:35 +00:00
description = _ ( " Visit %(url)s to get an API key " , url = args . get_api_key_link )
2021-05-18 03:41:02 +00:00
abort (
2023-03-10 03:00:27 +00:00
400 ,
2022-06-21 20:39:08 +00:00
description = description ,
2021-05-18 03:41:02 +00:00
)
2021-05-17 15:41:15 +00:00
return f ( * a , * * kw )
2022-12-26 21:10:43 +00:00
if args . metrics :
@wraps ( func )
def measure_func ( * a , * * kw ) :
start_t = default_timer ( )
status = 200
ip = get_remote_address ( )
ak = get_req_api_key ( ) or ' '
g = gauge_request . labels ( request . path , ip , ak )
try :
g . inc ( )
return func ( * a , * * kw )
except HTTPException as e :
status = e . code
raise e
finally :
duration = max ( default_timer ( ) - start_t , 0 )
measure_request . labels ( request . path , status , ip , ak ) . observe ( duration )
g . dec ( )
return measure_func
else :
return func
2022-12-31 04:23:50 +00:00
@bp.errorhandler ( 400 )
2020-12-20 20:05:22 +00:00
def invalid_api ( e ) :
return jsonify ( { " error " : str ( e . description ) } ) , 400
2022-12-31 04:23:50 +00:00
@bp.errorhandler ( 500 )
2020-12-20 20:05:22 +00:00
def server_error ( e ) :
return jsonify ( { " error " : str ( e . description ) } ) , 500
2022-12-31 04:23:50 +00:00
@bp.errorhandler ( 429 )
2020-12-20 20:05:22 +00:00
def slow_down_error ( e ) :
2021-05-16 15:50:22 +00:00
flood . report ( get_remote_address ( ) )
2023-01-05 18:12:35 +00:00
return jsonify ( { " error " : _ ( " Slowdown: " ) + " " + str ( e . description ) } ) , 429
2020-12-20 20:05:22 +00:00
2022-12-31 04:23:50 +00:00
@bp.errorhandler ( 403 )
2021-05-16 15:50:22 +00:00
def denied ( e ) :
return jsonify ( { " error " : str ( e . description ) } ) , 403
2022-12-31 04:23:50 +00:00
@bp.route ( " / " )
2021-02-15 18:30:28 +00:00
@limiter.exempt
2020-12-20 20:05:22 +00:00
def index ( ) :
2022-02-07 10:02:32 +00:00
if args . disable_web_ui :
abort ( 404 )
2023-01-06 16:50:51 +00:00
langcode = request . args . get ( ' lang ' )
if langcode and langcode in get_available_locale_codes ( not args . debug ) :
session . update ( preferred_lang = langcode )
2021-05-18 03:41:02 +00:00
return render_template (
" index.html " ,
gaId = args . ga_id ,
frontendTimeout = args . frontend_timeout ,
api_keys = args . api_keys ,
2022-06-21 19:17:42 +00:00
get_api_key_link = args . get_api_key_link ,
2021-05-18 03:41:02 +00:00
web_version = os . environ . get ( " LT_WEB " ) is not None ,
2022-12-31 21:44:25 +00:00
version = get_version ( ) ,
2022-12-31 22:15:51 +00:00
swagger_url = SWAGGER_URL ,
2023-01-06 16:50:51 +00:00
available_locales = [ { ' code ' : l [ ' code ' ] , ' name ' : _lazy ( l [ ' name ' ] ) } for l in get_available_locales ( not args . debug ) ] ,
2023-01-05 19:36:50 +00:00
current_locale = get_locale ( ) ,
alternate_locales = get_alternate_locale_links ( )
2021-05-18 03:41:02 +00:00
)
2020-12-20 20:05:22 +00:00
2023-01-06 16:50:51 +00:00
@bp.route ( " /js/app.js " )
2023-01-04 17:15:18 +00:00
@limiter.exempt
def appjs ( ) :
if args . disable_web_ui :
abort ( 404 )
2023-03-09 18:59:25 +00:00
response = Response ( render_template ( " app.js.template " ,
2023-01-06 16:50:51 +00:00
url_prefix = args . url_prefix ,
2023-03-09 18:59:25 +00:00
get_api_key_link = args . get_api_key_link ,
2023-03-09 21:09:04 +00:00
api_secret = secret . get_current_secret ( ) if args . require_api_key_secret else " " ) , content_type = ' application/javascript; charset=utf-8 ' )
2023-03-09 18:59:25 +00:00
if args . require_api_key_secret :
2023-03-09 19:05:09 +00:00
response . headers [ ' Last-Modified ' ] = http_date ( datetime . now ( ) )
2023-03-09 18:59:25 +00:00
response . headers [ ' Cache-Control ' ] = ' no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0 '
response . headers [ ' Pragma ' ] = ' no-cache '
response . headers [ ' Expires ' ] = ' -1 '
return response
2023-01-04 17:15:18 +00:00
2022-12-31 04:23:50 +00:00
@bp.get ( " /languages " )
2021-02-15 18:30:28 +00:00
@limiter.exempt
2020-12-20 20:05:22 +00:00
def langs ( ) :
"""
Retrieve list of supported languages
- - -
tags :
- translate
responses :
200 :
description : List of languages
2021-01-10 08:38:11 +00:00
schema :
id : languages
type : array
items :
type : object
properties :
code :
type : string
description : Language code
name :
type : string
description : Human - readable language name ( in English )
2022-12-10 05:21:38 +00:00
targets :
type : array
items :
type : string
description : Supported target language codes
2020-12-20 20:05:22 +00:00
"""
2023-01-05 18:12:35 +00:00
return jsonify ( [ { " code " : l . code , " name " : _lazy ( l . name ) , " targets " : language_pairs . get ( l . code , [ ] ) } for l in languages ] )
2020-12-20 20:05:22 +00:00
# Add cors
2022-12-31 04:23:50 +00:00
@bp.after_request
2020-12-20 20:05:22 +00:00
def after_request ( response ) :
2021-05-18 03:41:02 +00:00
response . headers . add ( " Access-Control-Allow-Origin " , " * " )
response . headers . add (
" Access-Control-Allow-Headers " , " Authorization, Content-Type "
)
response . headers . add ( " Access-Control-Expose-Headers " , " Authorization " )
response . headers . add ( " Access-Control-Allow-Methods " , " GET, POST " )
response . headers . add ( " Access-Control-Allow-Credentials " , " true " )
response . headers . add ( " Access-Control-Max-Age " , 60 * 60 * 24 * 20 )
2020-12-20 20:05:22 +00:00
return response
2022-12-31 04:23:50 +00:00
@bp.post ( " /translate " )
2021-05-17 15:41:15 +00:00
@access_check
2020-12-20 20:05:22 +00:00
def translate ( ) :
"""
Translate text from a language to another
- - -
tags :
- translate
parameters :
- in : formData
name : q
schema :
2021-01-19 16:51:10 +00:00
oneOf :
- type : string
example : Hello world !
- type : array
example : [ ' Hello world! ' ]
2020-12-20 20:05:22 +00:00
required : true
2021-01-19 16:51:10 +00:00
description : Text ( s ) to translate
2020-12-20 20:05:22 +00:00
- in : formData
name : source
schema :
type : string
example : en
required : true
2021-01-19 16:51:10 +00:00
description : Source language code
2020-12-20 20:05:22 +00:00
- in : formData
name : target
schema :
type : string
example : es
required : true
description : Target language code
2021-09-11 13:08:57 +00:00
- in : formData
name : format
schema :
type : string
2021-09-11 20:02:10 +00:00
enum : [ text , html ]
default : text
example : text
2021-09-11 13:08:57 +00:00
required : false
2021-09-11 20:02:10 +00:00
description : >
Format of source text :
* ` text ` - Plain text
* ` html ` - HTML markup
2021-02-15 18:30:28 +00:00
- in : formData
name : api_key
schema :
type : string
example : xxxxxxxx - xxxx - xxxx - xxxx - xxxxxxxxxxxx
required : false
description : API key
2020-12-20 20:05:22 +00:00
responses :
200 :
description : Translated text
2021-01-10 08:38:11 +00:00
schema :
id : translate
type : object
properties :
translatedText :
2021-01-19 16:51:10 +00:00
oneOf :
- type : string
- type : array
description : Translated text ( s )
2020-12-20 20:05:22 +00:00
400 :
description : Invalid request
2021-01-10 08:38:11 +00:00
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2020-12-20 20:05:22 +00:00
500 :
description : Translation error
2021-01-10 08:38:11 +00:00
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2020-12-20 20:05:22 +00:00
429 :
description : Slow down
2021-01-10 08:38:11 +00:00
schema :
id : error - slow - down
type : object
properties :
error :
type : string
description : Reason for slow down
2021-05-16 15:50:22 +00:00
403 :
description : Banned
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2020-12-20 20:05:22 +00:00
"""
if request . is_json :
2021-03-07 22:23:25 +00:00
json = get_json_dict ( request )
2021-05-18 03:41:02 +00:00
q = json . get ( " q " )
source_lang = json . get ( " source " )
target_lang = json . get ( " target " )
2021-09-11 13:08:57 +00:00
text_format = json . get ( " format " )
2020-12-20 20:05:22 +00:00
else :
q = request . values . get ( " q " )
source_lang = request . values . get ( " source " )
target_lang = request . values . get ( " target " )
2021-09-11 13:08:57 +00:00
text_format = request . values . get ( " format " )
2020-12-20 20:05:22 +00:00
if not q :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' q ' ) )
2020-12-20 20:05:22 +00:00
if not source_lang :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' source ' ) )
2020-12-20 20:05:22 +00:00
if not target_lang :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' target ' ) )
2020-12-20 20:05:22 +00:00
2021-01-19 16:51:10 +00:00
batch = isinstance ( q , list )
2021-02-15 18:30:28 +00:00
if batch and args . batch_limit != - 1 :
2021-05-18 03:41:02 +00:00
batch_size = len ( q )
if args . batch_limit < batch_size :
abort (
400 ,
2023-01-05 18:12:35 +00:00
description = _ ( " Invalid request: request ( %(size)s ) exceeds text limit ( %(limit)s ) " , size = batch_size , limit = args . batch_limit ) ,
2021-05-18 03:41:02 +00:00
)
2021-01-19 17:53:53 +00:00
2021-02-15 18:30:28 +00:00
if args . char_limit != - 1 :
2021-01-19 16:51:10 +00:00
if batch :
2021-05-18 03:41:02 +00:00
chars = sum ( [ len ( text ) for text in q ] )
2021-01-19 16:51:10 +00:00
else :
2021-05-18 03:41:02 +00:00
chars = len ( q )
2021-01-19 16:51:10 +00:00
2021-02-15 18:30:28 +00:00
if args . char_limit < chars :
2021-05-18 03:41:02 +00:00
abort (
400 ,
2023-01-05 18:12:35 +00:00
description = _ ( " Invalid request: request ( %(size)s ) exceeds text limit ( %(limit)s ) " , size = chars , limit = args . char_limit ) ,
2021-05-18 03:41:02 +00:00
)
2020-12-20 20:05:22 +00:00
2021-05-18 03:41:02 +00:00
if source_lang == " auto " :
2021-08-02 05:06:56 +00:00
source_langs = [ ]
if batch :
2021-10-24 16:27:37 +00:00
auto_detect_texts = q
2021-08-02 05:06:56 +00:00
else :
2021-10-24 16:27:37 +00:00
auto_detect_texts = [ q ]
2021-08-02 05:06:56 +00:00
overall_candidates = detect_languages ( q )
2021-10-24 16:27:37 +00:00
2021-08-02 05:06:56 +00:00
for text_to_check in auto_detect_texts :
2021-10-24 16:27:37 +00:00
if len ( text_to_check ) > 40 :
candidate_langs = detect_languages ( text_to_check )
else :
# Unable to accurately detect languages for short texts
candidate_langs = overall_candidates
2022-03-25 06:19:56 +00:00
source_langs . append ( candidate_langs [ 0 ] )
2021-10-24 16:27:37 +00:00
if args . debug :
print ( text_to_check , candidate_langs )
print ( " Auto detected: %s " % candidate_langs [ 0 ] [ " language " ] )
2021-08-02 05:06:56 +00:00
else :
2021-10-24 16:27:37 +00:00
if batch :
2022-03-25 06:19:56 +00:00
source_langs = [ { " confidence " : 100.0 , " language " : source_lang } for text in q ]
2021-10-24 16:27:37 +00:00
else :
2022-03-25 06:19:56 +00:00
source_langs = [ { " confidence " : 100.0 , " language " : source_lang } ]
2021-01-19 16:51:10 +00:00
2022-03-25 06:19:56 +00:00
src_langs = [ next ( iter ( [ l for l in languages if l . code == source_lang [ " language " ] ] ) , None ) for source_lang in source_langs ]
2021-10-24 16:27:37 +00:00
2021-08-02 05:06:56 +00:00
for idx , lang in enumerate ( src_langs ) :
2021-10-24 16:27:37 +00:00
if lang is None :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = source_langs [ idx ] ) )
2021-01-13 14:33:58 +00:00
2020-12-20 20:05:22 +00:00
tgt_lang = next ( iter ( [ l for l in languages if l . code == target_lang ] ) , None )
2021-01-19 16:51:10 +00:00
2020-12-20 20:05:22 +00:00
if tgt_lang is None :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = target_lang ) )
2020-12-20 20:05:22 +00:00
2021-09-11 13:08:57 +00:00
if not text_format :
text_format = " text "
if text_format not in [ " text " , " html " ] :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(format)s format is not supported " , format = text_format ) )
2021-09-11 13:08:57 +00:00
2020-12-20 20:05:22 +00:00
try :
2021-05-18 03:41:02 +00:00
if batch :
2021-08-02 05:06:56 +00:00
results = [ ]
for idx , text in enumerate ( q ) :
2021-10-24 16:27:37 +00:00
translator = src_langs [ idx ] . get_translation ( tgt_lang )
2022-12-10 05:03:21 +00:00
if translator is None :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(tname)s ( %(tcode)s ) is not available as a target language from %(sname)s ( %(scode)s ) " , tname = _lazy ( tgt_lang . name ) , tcode = tgt_lang . code , sname = _lazy ( src_langs [ idx ] . name ) , scode = src_langs [ idx ] . code ) )
2022-12-10 05:03:21 +00:00
2021-10-24 16:27:37 +00:00
if text_format == " html " :
translated_text = str ( translate_html ( translator , text ) )
else :
2022-11-16 14:31:25 +00:00
translated_text = improve_translation_formatting ( text , translator . translate ( text ) )
2021-09-11 13:08:57 +00:00
2022-02-06 13:54:52 +00:00
results . append ( unescape ( translated_text ) )
2022-03-25 06:19:56 +00:00
if source_lang == " auto " :
return jsonify (
{
" translatedText " : results ,
" detectedLanguage " : source_langs
}
)
else :
return jsonify (
2022-12-09 19:35:39 +00:00
{
2022-03-25 06:19:56 +00:00
" translatedText " : results
2022-12-09 19:35:39 +00:00
}
2022-03-25 06:19:56 +00:00
)
2021-05-18 03:41:02 +00:00
else :
2021-08-02 05:06:56 +00:00
translator = src_langs [ 0 ] . get_translation ( tgt_lang )
2022-12-10 05:03:21 +00:00
if translator is None :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(tname)s ( %(tcode)s ) is not available as a target language from %(sname)s ( %(scode)s ) " , tname = _lazy ( tgt_lang . name ) , tcode = tgt_lang . code , sname = _lazy ( src_langs [ 0 ] . name ) , scode = src_langs [ 0 ] . code ) )
2021-09-11 13:08:57 +00:00
if text_format == " html " :
2021-09-24 15:30:19 +00:00
translated_text = str ( translate_html ( translator , q ) )
2021-09-11 13:08:57 +00:00
else :
2022-11-16 14:31:25 +00:00
translated_text = improve_translation_formatting ( q , translator . translate ( q ) )
2022-03-25 06:19:56 +00:00
if source_lang == " auto " :
return jsonify (
{
" translatedText " : unescape ( translated_text ) ,
2022-12-09 19:29:44 +00:00
" detectedLanguage " : source_langs [ 0 ]
2022-03-25 06:19:56 +00:00
}
)
else :
return jsonify (
{
" translatedText " : unescape ( translated_text )
}
)
2020-12-20 20:05:22 +00:00
except Exception as e :
2023-01-05 18:12:35 +00:00
abort ( 500 , description = _ ( " Cannot translate text: %(text)s " , text = str ( e ) ) )
2020-12-20 20:05:22 +00:00
2022-12-31 04:23:50 +00:00
@bp.post ( " /translate_file " )
2021-10-24 16:27:37 +00:00
@access_check
def translate_file ( ) :
"""
Translate file from a language to another
- - -
tags :
- translate
consumes :
- multipart / form - data
parameters :
- in : formData
name : file
type : file
required : true
2021-10-26 20:04:50 +00:00
description : File to translate
2021-10-24 16:27:37 +00:00
- in : formData
name : source
schema :
type : string
example : en
required : true
description : Source language code
- in : formData
name : target
schema :
type : string
example : es
required : true
description : Target language code
- in : formData
name : api_key
schema :
type : string
example : xxxxxxxx - xxxx - xxxx - xxxx - xxxxxxxxxxxx
required : false
description : API key
responses :
200 :
description : Translated file
schema :
2022-10-18 15:41:36 +00:00
id : translate - file
2021-10-24 16:27:37 +00:00
type : object
properties :
2021-10-25 08:50:55 +00:00
translatedFileUrl :
type : string
description : Translated file url
2021-10-24 16:27:37 +00:00
400 :
description : Invalid request
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
500 :
description : Translation error
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
429 :
description : Slow down
schema :
id : error - slow - down
type : object
properties :
error :
type : string
description : Reason for slow down
403 :
description : Banned
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
"""
2021-10-25 15:09:23 +00:00
if args . disable_files_translation :
2023-01-05 18:12:35 +00:00
abort ( 403 , description = _ ( " Files translation are disabled on this server. " ) )
2021-10-25 15:09:23 +00:00
2021-10-24 17:14:09 +00:00
source_lang = request . form . get ( " source " )
target_lang = request . form . get ( " target " )
2021-10-24 16:27:37 +00:00
file = request . files [ ' file ' ]
if not file :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' file ' ) )
2021-10-24 16:27:37 +00:00
if not source_lang :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' source ' ) )
2021-10-24 16:27:37 +00:00
if not target_lang :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' target ' ) )
2021-10-24 16:27:37 +00:00
if file . filename == ' ' :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: empty file " ) )
2021-10-24 16:27:37 +00:00
if os . path . splitext ( file . filename ) [ 1 ] not in frontend_argos_supported_files_format :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: file format not supported " ) )
2021-10-24 16:27:37 +00:00
source_langs = [ source_lang ]
src_langs = [ next ( iter ( [ l for l in languages if l . code == source_lang ] ) , None ) for source_lang in source_langs ]
for idx , lang in enumerate ( src_langs ) :
if lang is None :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = source_langs [ idx ] ) )
2021-10-24 16:27:37 +00:00
tgt_lang = next ( iter ( [ l for l in languages if l . code == target_lang ] ) , None )
if tgt_lang is None :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " %(lang)s is not supported " , lang = target_lang ) )
2021-10-24 16:27:37 +00:00
try :
filename = str ( uuid . uuid4 ( ) ) + ' . ' + secure_filename ( file . filename )
2021-10-25 09:06:39 +00:00
filepath = os . path . join ( get_upload_dir ( ) , filename )
2021-10-24 16:27:37 +00:00
file . save ( filepath )
translated_file_path = argostranslatefiles . translate_file ( src_langs [ 0 ] . get_translation ( tgt_lang ) , filepath )
2021-10-24 16:38:35 +00:00
translated_filename = os . path . basename ( translated_file_path )
2021-10-25 08:56:17 +00:00
2021-10-24 16:27:37 +00:00
return jsonify (
{
2023-01-03 12:57:29 +00:00
" translatedFileUrl " : url_for ( ' Main app.download_file ' , filename = translated_filename , _external = True )
2021-10-24 16:27:37 +00:00
}
)
except Exception as e :
abort ( 500 , description = e )
2022-12-31 04:23:50 +00:00
@bp.get ( " /download_file/<string:filename> " )
2021-10-24 16:44:35 +00:00
def download_file ( filename : str ) :
2021-10-24 16:38:35 +00:00
"""
Download a translated file
"""
2021-10-25 15:09:23 +00:00
if args . disable_files_translation :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Files translation are disabled on this server. " ) )
2021-11-05 13:55:45 +00:00
2021-10-25 09:06:39 +00:00
filepath = os . path . join ( get_upload_dir ( ) , filename )
2021-10-26 19:41:14 +00:00
try :
checked_filepath = security . path_traversal_check ( filepath , get_upload_dir ( ) )
if os . path . isfile ( checked_filepath ) :
filepath = checked_filepath
except security . SuspiciousFileOperation :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid filename " ) )
2021-10-24 17:14:09 +00:00
return_data = io . BytesIO ( )
with open ( filepath , ' rb ' ) as fo :
return_data . write ( fo . read ( ) )
return_data . seek ( 0 )
2021-10-25 10:05:39 +00:00
download_filename = filename . split ( ' . ' )
download_filename . pop ( 0 )
download_filename = ' . ' . join ( download_filename )
2021-10-24 16:44:35 +00:00
2022-04-07 13:56:57 +00:00
return send_file ( return_data , as_attachment = True , download_name = download_filename )
2021-10-24 16:38:35 +00:00
2022-12-31 04:23:50 +00:00
@bp.post ( " /detect " )
2021-05-17 15:41:15 +00:00
@access_check
2021-02-10 15:51:17 +00:00
def detect ( ) :
"""
Detect the language of a single text
- - -
tags :
- translate
parameters :
- in : formData
name : q
schema :
type : string
example : Hello world !
required : true
description : Text to detect
2021-02-15 18:30:28 +00:00
- in : formData
name : api_key
schema :
type : string
example : xxxxxxxx - xxxx - xxxx - xxxx - xxxxxxxxxxxx
required : false
description : API key
2021-02-10 15:51:17 +00:00
responses :
200 :
description : Detections
schema :
id : detections
type : array
items :
type : object
properties :
confidence :
type : number
format : float
minimum : 0
maximum : 1
description : Confidence value
example : 0.6
language :
type : string
description : Language code
example : en
400 :
description : Invalid request
schema :
id : error - response
type : object
properties :
error :
type : string
2021-05-18 03:41:02 +00:00
description : Error message
2021-02-10 15:51:17 +00:00
500 :
description : Detection error
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
429 :
description : Slow down
schema :
id : error - slow - down
type : object
properties :
error :
type : string
description : Reason for slow down
2021-05-16 15:50:22 +00:00
403 :
description : Banned
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
2021-02-10 15:51:17 +00:00
"""
if request . is_json :
2021-03-07 22:23:25 +00:00
json = get_json_dict ( request )
2021-05-18 03:41:02 +00:00
q = json . get ( " q " )
2021-02-10 15:51:17 +00:00
else :
q = request . values . get ( " q " )
if not q :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' q ' ) )
2021-02-10 15:51:17 +00:00
2021-03-11 09:01:12 +00:00
return jsonify ( detect_languages ( q ) )
2021-02-10 15:51:17 +00:00
2022-12-31 04:23:50 +00:00
@bp.route ( " /frontend/settings " )
2021-02-15 18:30:28 +00:00
@limiter.exempt
2021-01-10 08:07:56 +00:00
def frontend_settings ( ) :
"""
Retrieve frontend specific settings
- - -
tags :
- frontend
responses :
200 :
description : frontend settings
schema :
id : frontend - settings
type : object
properties :
2021-01-10 09:24:42 +00:00
charLimit :
type : integer
description : Character input limit for this language ( - 1 indicates no limit )
2021-01-28 16:16:55 +00:00
frontendTimeout :
type : integer
description : Frontend translation timeout
2021-11-03 14:39:59 +00:00
apiKeys :
type : boolean
description : Whether the API key database is enabled .
2021-10-30 15:31:50 +00:00
keyRequired :
type : boolean
description : Whether an API key is required .
2021-10-09 14:04:16 +00:00
suggestions :
type : boolean
description : Whether submitting suggestions is enabled .
2021-10-24 14:57:45 +00:00
supportedFilesFormat :
type : array
items :
type : string
description : Supported files format
2021-01-10 08:07:56 +00:00
language :
type : object
properties :
source :
type : object
properties :
code :
type : string
description : Language code
name :
type : string
description : Human - readable language name ( in English )
target :
type : object
properties :
code :
type : string
description : Language code
name :
type : string
description : Human - readable language name ( in English )
"""
2023-01-15 17:07:51 +00:00
target_lang = frontend_argos_language_target ( )
2021-05-18 03:41:02 +00:00
return jsonify (
{
" charLimit " : args . char_limit ,
" frontendTimeout " : args . frontend_timeout ,
2021-11-03 14:39:59 +00:00
" apiKeys " : args . api_keys ,
2021-10-30 15:31:50 +00:00
" keyRequired " : bool ( args . api_keys and args . require_api_key_origin ) ,
2021-10-09 13:45:58 +00:00
" suggestions " : args . suggestions ,
2021-10-25 15:09:23 +00:00
" filesTranslation " : not args . disable_files_translation ,
" supportedFilesFormat " : [ ] if args . disable_files_translation else frontend_argos_supported_files_format ,
2021-05-18 03:41:02 +00:00
" language " : {
" source " : {
" code " : frontend_argos_language_source . code ,
2023-01-05 18:12:35 +00:00
" name " : _lazy ( frontend_argos_language_source . name ) ,
2021-05-18 03:41:02 +00:00
} ,
" target " : {
2023-01-15 17:07:51 +00:00
" code " : target_lang . code ,
" name " : _lazy ( target_lang . name ) ,
2021-05-18 03:41:02 +00:00
} ,
} ,
}
)
2020-12-20 20:05:22 +00:00
2022-12-31 04:23:50 +00:00
@bp.post ( " /suggest " )
2021-10-31 05:53:25 +00:00
@access_check
2021-10-09 09:25:56 +00:00
def suggest ( ) :
2021-10-09 14:04:16 +00:00
"""
Submit a suggestion to improve a translation
- - -
tags :
- feedback
parameters :
- in : formData
name : q
schema :
type : string
example : Hello world !
required : true
description : Original text
- in : formData
name : s
schema :
type : string
example : ¡ Hola mundo !
required : true
description : Suggested translation
- in : formData
name : source
schema :
type : string
example : en
required : true
description : Language of original text
- in : formData
name : target
schema :
type : string
example : es
required : true
description : Language of suggested translation
responses :
200 :
description : Success
schema :
id : suggest - response
type : object
properties :
success :
type : boolean
description : Whether submission was successful
403 :
description : Not authorized
schema :
id : error - response
type : object
properties :
error :
type : string
description : Error message
"""
if not args . suggestions :
2023-01-05 18:12:35 +00:00
abort ( 403 , description = _ ( " Suggestions are disabled on this server. " ) )
2021-10-09 13:45:58 +00:00
2021-10-09 09:44:00 +00:00
q = request . values . get ( " q " )
s = request . values . get ( " s " )
source_lang = request . values . get ( " source " )
target_lang = request . values . get ( " target " )
2022-02-20 17:22:12 +00:00
if not q :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' q ' ) )
2022-02-20 17:22:12 +00:00
if not s :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' s ' ) )
2022-02-20 17:22:12 +00:00
if not source_lang :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' source ' ) )
2022-02-20 17:22:12 +00:00
if not target_lang :
2023-01-05 18:12:35 +00:00
abort ( 400 , description = _ ( " Invalid request: missing %(name)s parameter " , name = ' target ' ) )
2022-02-20 17:22:12 +00:00
2021-10-09 09:44:00 +00:00
SuggestionsDatabase ( ) . add ( q , s , source_lang , target_lang )
2021-10-09 09:25:56 +00:00
return jsonify ( { " success " : True } )
2022-12-31 04:23:50 +00:00
app = Flask ( __name__ )
2023-01-06 16:50:51 +00:00
app . config [ " SESSION_TYPE " ] = " filesystem "
app . config [ " SESSION_FILE_DIR " ] = os . path . join ( " db " , " sessions " )
2023-05-29 20:47:32 +00:00
app . config [ " JSON_AS_ASCII " ] = False
2023-01-06 16:50:51 +00:00
Session ( app )
2022-12-31 04:23:50 +00:00
if args . debug :
app . config [ " TEMPLATES_AUTO_RELOAD " ] = True
if args . url_prefix :
app . register_blueprint ( bp , url_prefix = args . url_prefix )
else :
app . register_blueprint ( bp )
2023-01-01 18:18:00 +00:00
limiter . init_app ( app )
2022-12-31 04:23:50 +00:00
2021-10-09 14:04:16 +00:00
swag = swagger ( app )
2022-12-31 21:44:25 +00:00
swag [ " info " ] [ " version " ] = get_version ( )
2021-10-09 14:04:16 +00:00
swag [ " info " ] [ " title " ] = " LibreTranslate "
2022-12-31 21:44:25 +00:00
@app.route ( API_URL )
2021-10-09 14:04:16 +00:00
@limiter.exempt
def spec ( ) :
2023-01-05 19:07:39 +00:00
return jsonify ( lazy_swag ( swag ) )
2021-10-09 14:04:16 +00:00
2023-01-04 22:54:07 +00:00
app . config [ " BABEL_TRANSLATION_DIRECTORIES " ] = ' locales '
2023-04-27 16:21:04 +00:00
2023-01-04 17:15:18 +00:00
def get_locale ( ) :
2023-01-06 16:50:51 +00:00
override_lang = request . headers . get ( ' X-Override-Accept-Language ' )
if override_lang and override_lang in get_available_locale_codes ( ) :
return override_lang
return session . get ( ' preferred_lang ' , request . accept_languages . best_match ( get_available_locale_codes ( ) ) )
2023-01-04 17:15:18 +00:00
2023-04-27 16:21:04 +00:00
babel = Babel ( app , locale_selector = get_locale )
2023-01-04 22:54:07 +00:00
app . jinja_env . globals . update ( _e = gettext_escaped , _h = gettext_html )
2023-01-04 17:15:18 +00:00
2020-12-20 20:05:22 +00:00
# Call factory function to create our blueprint
2021-05-18 03:41:02 +00:00
swaggerui_blueprint = get_swaggerui_blueprint ( SWAGGER_URL , API_URL )
2022-12-31 04:23:50 +00:00
if args . url_prefix :
2022-12-31 21:44:25 +00:00
app . register_blueprint ( swaggerui_blueprint , url_prefix = SWAGGER_URL )
2022-12-31 04:23:50 +00:00
else :
app . register_blueprint ( swaggerui_blueprint )
2020-12-20 20:05:22 +00:00
2020-12-20 23:17:06 +00:00
return app