mirrors/LibreTranslate
1
1
Fork 1
mirror of https://github.com/LibreTranslate/LibreTranslate.git synced 2024-11-21 23:41:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge main

Browse source
This commit is contained in:
Piero Toffanin 2021-06-03 10:39:41 -04:00
commit ca1049f0d3
13 changed files with 376 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -115,6 +115,7 @@ docker-compose up -d --build
| --frontend-language-target | Set frontend default language - target | `es` | | --frontend-language-target | Set frontend default language - target | `es` |
| --frontend-timeout | Set frontend translation timeout | `500` | | --frontend-timeout | Set frontend translation timeout | `500` |
| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | | --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` |
| --require-api-key-origin | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` |
| --load-only | Set available languages | `all from argostranslate` | | --load-only | Set available languages | `all from argostranslate` |
## Manage API Keys ## Manage API Keys
@ -151,6 +152,7 @@ You can use the LibreTranslate API using the following bindings:
- Node.js: https://github.com/franciscop/translate - Node.js: https://github.com/franciscop/translate
- .Net: https://github.com/sigaloid/LibreTranslate.Net - .Net: https://github.com/sigaloid/LibreTranslate.Net
- Go: https://github.com/SnakeSel/libretranslate - Go: https://github.com/SnakeSel/libretranslate
- Python: https://github.com/argosopentech/LibreTranslate-py
More coming soon! More coming soon!

7
app/__init__.py
View file

@ -1,9 +1,12 @@
import os import os
from appdirs import user_data_dir
# override polyglot path # override polyglot path
import polyglot import polyglot
polyglot.polyglot_path = os.path.join(user_data_dir(appname="LibreTranslate", appauthor="uav4geo"), "polyglot_data") from appdirs import user_data_dir
polyglot.polyglot_path = os.path.join(
user_data_dir(appname="LibreTranslate", appauthor="uav4geo"), "polyglot_data"
)
from .main import main from .main import main

33
app/api_keys.py
View file

@ -1,27 +1,33 @@
import sqlite3 import sqlite3
import uuid import uuid
from expiringdict import ExpiringDict from expiringdict import ExpiringDict
DEFAULT_DB_PATH = "api_keys.db" DEFAULT_DB_PATH = "api_keys.db"
class Database: class Database:
def __init__(self, db_path = DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
self.db_path = db_path self.db_path = db_path
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
# Make sure to do data synchronization on writes! # Make sure to do data synchronization on writes!
self.c = sqlite3.connect(db_path, check_same_thread=False) self.c = sqlite3.connect(db_path, check_same_thread=False)
self.c.execute('''CREATE TABLE IF NOT EXISTS api_keys ( self.c.execute(
"""CREATE TABLE IF NOT EXISTS api_keys (
"api_key" TEXT NOT NULL, "api_key" TEXT NOT NULL,
"req_limit" INTEGER NOT NULL, "req_limit" INTEGER NOT NULL,
PRIMARY KEY("api_key") PRIMARY KEY("api_key")
);''') );"""
)
def lookup(self, api_key): def lookup(self, api_key):
req_limit = self.cache.get(api_key) req_limit = self.cache.get(api_key)
if req_limit is None: if req_limit is None:
# DB Lookup # DB Lookup
stmt = self.c.execute('SELECT req_limit FROM api_keys WHERE api_key = ?', (api_key, )) stmt = self.c.execute(
"SELECT req_limit FROM api_keys WHERE api_key = ?", (api_key,)
)
row = stmt.fetchone() row = stmt.fetchone()
if row is not None: if row is not None:
self.cache[api_key] = row[0] self.cache[api_key] = row[0]
@ -29,26 +35,29 @@ class Database:
else: else:
self.cache[api_key] = False self.cache[api_key] = False
req_limit = False req_limit = False
if isinstance(req_limit, bool): if isinstance(req_limit, bool):
req_limit = None req_limit = None
return req_limit return req_limit
def add(self, req_limit, api_key = "auto"): def add(self, req_limit, api_key="auto"):
if api_key == "auto": if api_key == "auto":
api_key = str(uuid.uuid4()) api_key = str(uuid.uuid4())
self.remove(api_key) self.remove(api_key)
self.c.execute("INSERT INTO api_keys (api_key, req_limit) VALUES (?, ?)", (api_key, req_limit)) self.c.execute(
"INSERT INTO api_keys (api_key, req_limit) VALUES (?, ?)",
(api_key, req_limit),
)
self.c.commit() self.c.commit()
return (api_key, req_limit) return (api_key, req_limit)
def remove(self, api_key): def remove(self, api_key):
self.c.execute('DELETE FROM api_keys WHERE api_key = ?', (api_key, )) self.c.execute("DELETE FROM api_keys WHERE api_key = ?", (api_key,))
self.c.commit() self.c.commit()
return api_key return api_key
def all(self): def all(self):
row = self.c.execute("SELECT api_key, req_limit FROM api_keys") row = self.c.execute("SELECT api_key, req_limit FROM api_keys")
return row.fetchall() return row.fetchall()

217
app/app.py
View file

@ -1,11 +1,15 @@
import os import os
from flask import Flask, render_template, jsonify, request, abort, send_from_directory from functools import wraps
from flask import Flask, abort, jsonify, render_template, request
from flask_swagger import swagger from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint from flask_swagger_ui import get_swaggerui_blueprint
from pkg_resources import resource_filename
from .api_keys import Database
from app.language import detect_languages, transliterate
from app import flood from app import flood
from app.language import detect_languages, transliterate
from .api_keys import Database
def get_json_dict(request): def get_json_dict(request):
d = request.get_json() d = request.get_json()
@ -13,11 +17,12 @@ def get_json_dict(request):
abort(400, description="Invalid JSON format") abort(400, description="Invalid JSON format")
return d return d
def get_remote_address(): def get_remote_address():
if request.headers.getlist("X-Forwarded-For"): if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0] ip = request.headers.getlist("X-Forwarded-For")[0]
else: else:
ip = request.remote_addr or '127.0.0.1' ip = request.remote_addr or "127.0.0.1"
return ip return ip
@ -58,47 +63,91 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db):
return res return res
def create_app(args): def create_app(args):
from app.init import boot from app.init import boot
boot(args.load_only) boot(args.load_only)
from app.language import languages from app.language import languages
app = Flask(__name__) app = Flask(__name__)
if args.debug: if args.debug:
app.config['TEMPLATES_AUTO_RELOAD'] = True app.config["TEMPLATES_AUTO_RELOAD"] = True
# Map userdefined frontend languages to argos language object. # Map userdefined frontend languages to argos language object.
if args.frontend_language_source == "auto": if args.frontend_language_source == "auto":
frontend_argos_language_source = type('obj', (object,), { frontend_argos_language_source = type(
'code': 'auto', "obj", (object,), {"code": "auto", "name": "Auto Detect"}
'name': 'Auto Detect' )
})
else: else:
frontend_argos_language_source = next(iter([l for l in languages if l.code == args.frontend_language_source]), None) frontend_argos_language_source = next(
iter([l for l in languages if l.code == args.frontend_language_source]),
None,
)
frontend_argos_language_target = next(iter([l for l in languages if l.code == args.frontend_language_target]), None) frontend_argos_language_target = next(
iter([l for l in languages if l.code == args.frontend_language_target]), None
)
# Raise AttributeError to prevent app startup if user input is not valid. # Raise AttributeError to prevent app startup if user input is not valid.
if frontend_argos_language_source is None: if frontend_argos_language_source is None:
raise AttributeError(f"{args.frontend_language_source} as frontend source language is not supported.") raise AttributeError(
f"{args.frontend_language_source} as frontend source language is not supported."
)
if frontend_argos_language_target is None: if frontend_argos_language_target is None:
raise AttributeError(f"{args.frontend_language_target} as frontend target language is not supported.") raise AttributeError(
f"{args.frontend_language_target} as frontend target language is not supported."
)
api_keys_db = None
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
api_keys_db = Database() if args.api_keys else None
from flask_limiter import Limiter from flask_limiter import Limiter
limiter = Limiter( limiter = Limiter(
app, app,
key_func=get_remote_address, key_func=get_remote_address,
default_limits=get_routes_limits(args.req_limit, args.daily_req_limit, Database() if args.api_keys else None) default_limits=get_routes_limits(
args.req_limit, args.daily_req_limit, api_keys_db
),
) )
else: else:
from .no_limiter import Limiter from .no_limiter import Limiter
limiter = Limiter()
limiter = Limiter()
if args.req_flood_threshold > 0: if args.req_flood_threshold > 0:
flood.setup(args.req_flood_threshold) flood.setup(args.req_flood_threshold)
def access_check(f):
@wraps(f)
def func(*a, **kw):
if flood.is_banned(get_remote_address()):
abort(403, description="Too many request limits violations")
if args.api_keys and args.require_api_key_origin:
if request.is_json:
json = get_json_dict(request)
ak = json.get("api_key")
else:
ak = request.values.get("api_key")
if (
api_keys_db.lookup(ak) is None and request.headers.get("Origin") != args.require_api_key_origin
):
abort(
403,
description="Please contact the server operator to obtain an API key",
)
return f(*a, **kw)
return func
@app.errorhandler(400) @app.errorhandler(400)
def invalid_api(e): def invalid_api(e):
return jsonify({"error": str(e.description)}), 400 return jsonify({"error": str(e.description)}), 400
@ -116,18 +165,23 @@ def create_app(args):
def denied(e): def denied(e):
return jsonify({"error": str(e.description)}), 403 return jsonify({"error": str(e.description)}), 403
@app.route("/") @app.route("/")
@limiter.exempt @limiter.exempt
def index(): def index():
return render_template('index.html', gaId=args.ga_id, frontendTimeout=args.frontend_timeout, api_keys=args.api_keys, web_version=os.environ.get('LT_WEB') is not None) return render_template(
"index.html",
gaId=args.ga_id,
frontendTimeout=args.frontend_timeout,
api_keys=args.api_keys,
web_version=os.environ.get("LT_WEB") is not None,
)
@app.route("/javascript-licenses", methods=['GET']) @app.route("/javascript-licenses", methods=["GET"])
@limiter.exempt @limiter.exempt
def javascript_licenses(): def javascript_licenses():
return render_template('javascript-licenses.html') return render_template("javascript-licenses.html")
@app.route("/languages", methods=['GET', 'POST']) @app.route("/languages", methods=["GET", "POST"])
@limiter.exempt @limiter.exempt
def langs(): def langs():
""" """
@ -160,21 +214,23 @@ def create_app(args):
type: string type: string
description: Reason for slow down description: Reason for slow down
""" """
return jsonify([{'code': l.code, 'name': l.name} for l in languages]) return jsonify([{"code": l.code, "name": l.name} for l in languages])
# Add cors # Add cors
@app.after_request @app.after_request
def after_request(response): def after_request(response):
response.headers.add('Access-Control-Allow-Origin','*') response.headers.add("Access-Control-Allow-Origin", "*")
response.headers.add('Access-Control-Allow-Headers', "Authorization, Content-Type") response.headers.add(
response.headers.add('Access-Control-Expose-Headers', "Authorization") "Access-Control-Allow-Headers", "Authorization, Content-Type"
response.headers.add('Access-Control-Allow-Methods', "GET, POST") )
response.headers.add('Access-Control-Allow-Credentials', "true") response.headers.add("Access-Control-Expose-Headers", "Authorization")
response.headers.add('Access-Control-Max-Age', 60 * 60 * 24 * 20) response.headers.add("Access-Control-Allow-Methods", "GET, POST")
response.headers.add("Access-Control-Allow-Credentials", "true")
response.headers.add("Access-Control-Max-Age", 60 * 60 * 24 * 20)
return response return response
@app.route("/translate", methods=["POST"])
@app.route("/translate", methods=['POST']) @access_check
def translate(): def translate():
""" """
Translate text from a language to another Translate text from a language to another
@ -262,14 +318,11 @@ def create_app(args):
type: string type: string
description: Error message description: Error message
""" """
if flood.is_banned(get_remote_address()):
abort(403, description="Too many request limits violations")
if request.is_json: if request.is_json:
json = get_json_dict(request) json = get_json_dict(request)
q = json.get('q') q = json.get("q")
source_lang = json.get('source') source_lang = json.get("source")
target_lang = json.get('target') target_lang = json.get("target")
else: else:
q = request.values.get("q") q = request.values.get("q")
source_lang = request.values.get("source") source_lang = request.values.get("source")
@ -285,20 +338,28 @@ def create_app(args):
batch = isinstance(q, list) batch = isinstance(q, list)
if batch and args.batch_limit != -1: if batch and args.batch_limit != -1:
batch_size = len(q) batch_size = len(q)
if args.batch_limit < batch_size: if args.batch_limit < batch_size:
abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, args.batch_limit)) abort(
400,
description="Invalid request: Request (%d) exceeds text limit (%d)"
% (batch_size, args.batch_limit),
)
if args.char_limit != -1: if args.char_limit != -1:
if batch: if batch:
chars = sum([len(text) for text in q]) chars = sum([len(text) for text in q])
else: else:
chars = len(q) chars = len(q)
if args.char_limit < chars: if args.char_limit < chars:
abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, args.char_limit)) abort(
400,
description="Invalid request: Request (%d) exceeds character limit (%d)"
% (chars, args.char_limit),
)
if source_lang == 'auto': if source_lang == "auto":
candidate_langs = detect_languages(q) candidate_langs = detect_languages(q)
if args.debug: if args.debug:
@ -320,14 +381,30 @@ def create_app(args):
translator = src_lang.get_translation(tgt_lang) translator = src_lang.get_translation(tgt_lang)
try: try:
if batch: if batch:
return jsonify({"translatedText": [translator.translate(transliterate(text, target_lang=source_lang)) for text in q] }) return jsonify(
else: {
return jsonify({"translatedText": translator.translate(transliterate(q, target_lang=source_lang)) }) "translatedText": [
translator.translate(
transliterate(text, target_lang=source_lang)
)
for text in q
]
}
)
else:
return jsonify(
{
"translatedText": translator.translate(
transliterate(q, target_lang=source_lang)
)
}
)
except Exception as e: except Exception as e:
abort(500, description="Cannot translate text: %s" % str(e)) abort(500, description="Cannot translate text: %s" % str(e))
@app.route("/detect", methods=['POST']) @app.route("/detect", methods=["POST"])
@access_check
def detect(): def detect():
""" """
Detect the language of a single text Detect the language of a single text
@ -377,7 +454,7 @@ def create_app(args):
properties: properties:
error: error:
type: string type: string
description: Error message description: Error message
500: 500:
description: Detection error description: Detection error
schema: schema:
@ -411,7 +488,7 @@ def create_app(args):
if request.is_json: if request.is_json:
json = get_json_dict(request) json = get_json_dict(request)
q = json.get('q') q = json.get("q")
else: else:
q = request.values.get("q") q = request.values.get("q")
@ -420,7 +497,6 @@ def create_app(args):
return jsonify(detect_languages(q)) return jsonify(detect_languages(q))
@app.route("/frontend/settings") @app.route("/frontend/settings")
@limiter.exempt @limiter.exempt
def frontend_settings(): def frontend_settings():
@ -464,30 +540,37 @@ def create_app(args):
type: string type: string
description: Human-readable language name (in English) description: Human-readable language name (in English)
""" """
return jsonify({'charLimit': args.char_limit, return jsonify(
'frontendTimeout': args.frontend_timeout, {
'language': { "charLimit": args.char_limit,
'source': {'code': frontend_argos_language_source.code, 'name': frontend_argos_language_source.name}, "frontendTimeout": args.frontend_timeout,
'target': {'code': frontend_argos_language_target.code, 'name': frontend_argos_language_target.name}} "language": {
}) "source": {
"code": frontend_argos_language_source.code,
"name": frontend_argos_language_source.name,
},
"target": {
"code": frontend_argos_language_target.code,
"name": frontend_argos_language_target.name,
},
},
}
)
swag = swagger(app) swag = swagger(app)
swag['info']['version'] = "1.2" swag["info"]["version"] = "1.2"
swag['info']['title'] = "LibreTranslate" swag["info"]["title"] = "LibreTranslate"
@app.route("/spec") @app.route("/spec")
@limiter.exempt @limiter.exempt
def spec(): def spec():
return jsonify(swag) return jsonify(swag)
SWAGGER_URL = '/docs' # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = "/docs" # URL for exposing Swagger UI (without trailing '/')
API_URL = '/spec' API_URL = "/spec"
# Call factory function to create our blueprint # Call factory function to create our blueprint
swaggerui_blueprint = get_swaggerui_blueprint( swaggerui_blueprint = get_swaggerui_blueprint(SWAGGER_URL, API_URL)
SWAGGER_URL,
API_URL
)
app.register_blueprint(swaggerui_blueprint) app.register_blueprint(swaggerui_blueprint)

6
app/flood.py
View file

@ -1,4 +1,3 @@
import time
import atexit import atexit
from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.schedulers.background import BackgroundScheduler
@ -7,11 +6,13 @@ banned = {}
active = False active = False
threshold = -1 threshold = -1
def clear_banned(): def clear_banned():
global banned global banned
banned = {} banned = {}
def setup(violations_threshold = 100):
def setup(violations_threshold=100):
global active global active
global threshold global threshold
@ -31,6 +32,7 @@ def report(request_ip):
banned[request_ip] = banned.get(request_ip, 0) banned[request_ip] = banned.get(request_ip, 0)
banned[request_ip] += 1 banned[request_ip] += 1
def is_banned(request_ip): def is_banned(request_ip):
# More than X offences? # More than X offences?
return active and banned.get(request_ip, 0) >= threshold return active and banned.get(request_ip, 0) >= threshold

43
app/init.py
View file

@ -1,9 +1,10 @@
import os
from pathlib import Path from pathlib import Path
from argostranslate import settings, package, translate
import os, glob, shutil, zipfile
import app.language
import polyglot import polyglot
from argostranslate import package, translate
import app.language
def boot(load_only=None): def boot(load_only=None):
try: try:
@ -12,6 +13,7 @@ def boot(load_only=None):
except Exception as e: except Exception as e:
print("Cannot update models (normal if you're offline): %s" % str(e)) print("Cannot update models (normal if you're offline): %s" % str(e))
def check_and_install_models(force=False, load_only_lang_codes=None): def check_and_install_models(force=False, load_only_lang_codes=None):
if len(package.get_installed_packages()) < 2 or force: if len(package.get_installed_packages()) < 2 or force:
# Update package definitions from remote # Update package definitions from remote
@ -29,48 +31,63 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
for pack in available_packages: for pack in available_packages:
unavailable_lang_codes -= {pack.from_code, pack.to_code} unavailable_lang_codes -= {pack.from_code, pack.to_code}
if unavailable_lang_codes: if unavailable_lang_codes:
raise ValueError('Unavailable language codes: %s.' % ','.join(sorted(unavailable_lang_codes))) raise ValueError(
"Unavailable language codes: %s."
% ",".join(sorted(unavailable_lang_codes))
)
# Keep only the packages that have both from_code and to_code in our list. # Keep only the packages that have both from_code and to_code in our list.
available_packages = [ available_packages = [
pack pack
for pack in available_packages for pack in available_packages
if pack.from_code in load_only_lang_codes if pack.from_code in load_only_lang_codes and pack.to_code in load_only_lang_codes
and pack.to_code in load_only_lang_codes
] ]
if not available_packages: if not available_packages:
raise ValueError('no available package') raise ValueError("no available package")
print("Keep %s models" % len(available_packages)) print("Keep %s models" % len(available_packages))
# Download and install all available packages # Download and install all available packages
for available_package in available_packages: for available_package in available_packages:
print("Downloading %s (%s) ..." % (available_package, available_package.package_version)) print(
"Downloading %s (%s) ..."
% (available_package, available_package.package_version)
)
download_path = available_package.download() download_path = available_package.download()
package.install_from_path(download_path) package.install_from_path(download_path)
# reload installed languages # reload installed languages
app.language.languages = translate.load_installed_languages() app.language.languages = translate.load_installed_languages()
print("Loaded support for %s languages (%s models total)!" % (len(translate.load_installed_languages()), len(available_packages))) print(
"Loaded support for %s languages (%s models total)!"
% (len(translate.load_installed_languages()), len(available_packages))
)
def check_and_install_transliteration(force=False): def check_and_install_transliteration(force=False):
# 'en' is not a supported transliteration language # 'en' is not a supported transliteration language
transliteration_languages = [l.code for l in app.language.languages if l.code != "en"] transliteration_languages = [
l.code for l in app.language.languages if l.code != "en"
]
# check installed # check installed
install_needed = [] install_needed = []
if not force: if not force:
t_packages_path = Path(polyglot.polyglot_path) / "transliteration2" t_packages_path = Path(polyglot.polyglot_path) / "transliteration2"
for lang in transliteration_languages: for lang in transliteration_languages:
if not (t_packages_path / lang / f"transliteration.{lang}.tar.bz2").exists(): if not (
t_packages_path / lang / f"transliteration.{lang}.tar.bz2"
).exists():
install_needed.append(lang) install_needed.append(lang)
else: else:
install_needed = transliteration_languages install_needed = transliteration_languages
# install the needed transliteration packages # install the needed transliteration packages
if install_needed: if install_needed:
print(f"Installing transliteration models for the following languages: {', '.join(install_needed)}") print(
f"Installing transliteration models for the following languages: {', '.join(install_needed)}"
)
from polyglot.downloader import Downloader from polyglot.downloader import Downloader
downloader = Downloader() downloader = Downloader()
for lang in install_needed: for lang in install_needed:

30
app/language.py
View file

@ -4,7 +4,6 @@ from argostranslate import translate
from polyglot.detect.base import Detector, UnknownLanguage from polyglot.detect.base import Detector, UnknownLanguage
from polyglot.transliteration.base import Transliterator from polyglot.transliteration.base import Transliterator
languages = translate.load_installed_languages() languages = translate.load_installed_languages()
@ -24,24 +23,21 @@ def detect_languages(text):
for t in text: for t in text:
try: try:
candidates.extend(Detector(t).languages) candidates.extend(Detector(t).languages)
except UnknownLanguage as e: except UnknownLanguage:
pass pass
# total read bytes of the provided text # total read bytes of the provided text
read_bytes_total = sum(c.read_bytes for c in candidates) read_bytes_total = sum(c.read_bytes for c in candidates)
# only use candidates that are supported by argostranslate # only use candidates that are supported by argostranslate
candidate_langs = list(filter(lambda l: l.read_bytes != 0 and l.code in __lang_codes, candidates)) candidate_langs = list(
filter(lambda l: l.read_bytes != 0 and l.code in __lang_codes, candidates)
)
# this happens if no language could be detected # this happens if no language could be detected
if not candidate_langs: if not candidate_langs:
# use language "en" by default but with zero confidence # use language "en" by default but with zero confidence
return [ return [{"confidence": 0.0, "language": "en"}]
{
'confidence': 0.0,
'language': "en"
}
]
# for multiple occurrences of the same language (can happen on batch detection) # for multiple occurrences of the same language (can happen on batch detection)
# calculate the average confidence for each language # calculate the average confidence for each language
@ -65,15 +61,11 @@ def detect_languages(text):
candidate_langs = temp_average_list candidate_langs = temp_average_list
# sort the candidates descending based on the detected confidence # sort the candidates descending based on the detected confidence
candidate_langs.sort(key=lambda l: (l.confidence * l.read_bytes) / read_bytes_total, reverse=True) candidate_langs.sort(
key=lambda l: (l.confidence * l.read_bytes) / read_bytes_total, reverse=True
)
return [ return [{"confidence": l.confidence, "language": l.code} for l in candidate_langs]
{
'confidence': l.confidence,
'language': l.code
}
for l in candidate_langs
]
def __transliterate_line(transliterator, line_text): def __transliterate_line(transliterator, line_text):
@ -97,9 +89,9 @@ def __transliterate_line(transliterator, line_text):
else: else:
# add back any stripped punctuation # add back any stripped punctuation
if r_diff: if r_diff:
t_word = t_word + ''.join(r_diff) t_word = t_word + "".join(r_diff)
if l_diff: if l_diff:
t_word = ''.join(l_diff) + t_word t_word = "".join(l_diff) + t_word
new_text.append(t_word) new_text.append(t_word)

138
app/main.py
View file

@ -1,41 +1,104 @@
import argparse import argparse
import operator import operator
from app.app import create_app from app.app import create_app
def main(): def main():
parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API') parser = argparse.ArgumentParser(
parser.add_argument('--host', type=str, description="LibreTranslate - Free and Open Source Translation API"
help='Hostname (%(default)s)', default="127.0.0.1") )
parser.add_argument('--port', type=int, parser.add_argument(
help='Port (%(default)s)', default=5000) "--host", type=str, help="Hostname (%(default)s)", default="127.0.0.1"
parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>", )
help='Set character limit (%(default)s)') parser.add_argument("--port", type=int, help="Port (%(default)s)", default=5000)
parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>", parser.add_argument(
help='Set the default maximum number of requests per minute per client (%(default)s)') "--char-limit",
parser.add_argument('--daily-req-limit', default=-1, type=int, metavar="<number>", default=-1,
help='Set the default maximum number of requests per day per client, in addition to req-limit. (%(default)s)') type=int,
parser.add_argument('--req-flood-threshold', default=-1, type=int, metavar="<number>", metavar="<number of characters>",
help='Set the maximum number of request limit offences per 4 weeks that a client can exceed before being banned. (%(default)s)') help="Set character limit (%(default)s)",
parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>", )
help='Set maximum number of texts to translate in a batch request (%(default)s)') parser.add_argument(
parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>", "--req-limit",
help='Enable Google Analytics on the API client page by providing an ID (%(default)s)') default=-1,
parser.add_argument('--debug', default=False, action="store_true", type=int,
help="Enable debug environment") metavar="<number>",
parser.add_argument('--ssl', default=None, action="store_true", help="Set the default maximum number of requests per minute per client (%(default)s)",
help="Whether to enable SSL") )
parser.add_argument('--frontend-language-source', type=str, default="en", metavar="<language code>", parser.add_argument(
help='Set frontend default language - source (%(default)s)') "--daily-req-limit",
parser.add_argument('--frontend-language-target', type=str, default="es", metavar="<language code>", default=-1,
help='Set frontend default language - target (%(default)s)') type=int,
parser.add_argument('--frontend-timeout', type=int, default=500, metavar="<milliseconds>", metavar="<number>",
help='Set frontend translation timeout (%(default)s)') help="Set the default maximum number of requests per day per client, in addition to req-limit. (%(default)s)",
parser.add_argument('--api-keys', default=False, action="store_true", )
help="Enable API keys database for per-user rate limits lookup") parser.add_argument(
parser.add_argument('--load-only', type=operator.methodcaller('split', ','), "--req-flood-threshold",
metavar='<comma-separated language codes>', default=-1,
help='Set available languages (ar,de,en,es,fr,ga,hi,it,ja,ko,pt,ru,zh)') type=int,
metavar="<number>",
help="Set the maximum number of request limit offences per 4 weeks that a client can exceed before being banned. (%(default)s)",
)
parser.add_argument(
"--batch-limit",
default=-1,
type=int,
metavar="<number of texts>",
help="Set maximum number of texts to translate in a batch request (%(default)s)",
)
parser.add_argument(
"--ga-id",
type=str,
default=None,
metavar="<GA ID>",
help="Enable Google Analytics on the API client page by providing an ID (%(default)s)",
)
parser.add_argument(
"--debug", default=False, action="store_true", help="Enable debug environment"
)
parser.add_argument(
"--ssl", default=None, action="store_true", help="Whether to enable SSL"
)
parser.add_argument(
"--frontend-language-source",
type=str,
default="en",
metavar="<language code>",
help="Set frontend default language - source (%(default)s)",
)
parser.add_argument(
"--frontend-language-target",
type=str,
default="es",
metavar="<language code>",
help="Set frontend default language - target (%(default)s)",
)
parser.add_argument(
"--frontend-timeout",
type=int,
default=500,
metavar="<milliseconds>",
help="Set frontend translation timeout (%(default)s)",
)
parser.add_argument(
"--api-keys",
default=False,
action="store_true",
help="Enable API keys database for per-user rate limits lookup",
)
parser.add_argument(
"--require-api-key-origin",
type=str,
default="",
help="Require use of an API key for programmatic access to the API, unless the request origin matches this domain",
)
parser.add_argument(
"--load-only",
type=operator.methodcaller("split", ","),
metavar="<comma-separated language codes>",
help="Set available languages (ar,de,en,es,fr,ga,hi,it,ja,ko,pt,ru,zh)",
)
args = parser.parse_args() args = parser.parse_args()
app = create_app(args) app = create_app(args)
@ -44,7 +107,14 @@ def main():
app.run(host=args.host, port=args.port) app.run(host=args.host, port=args.port)
else: else:
from waitress import serve from waitress import serve
serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http')
serve(
app,
host=args.host,
port=args.port,
url_scheme="https" if args.ssl else "http",
)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

48
app/manage.py
View file

@ -1,31 +1,35 @@
import argparse import argparse
from app.api_keys import Database from app.api_keys import Database
def manage(): def manage():
parser = argparse.ArgumentParser(description='LibreTranslate Manage Tools') parser = argparse.ArgumentParser(description="LibreTranslate Manage Tools")
subparsers = parser.add_subparsers(help='', dest='command', required=True, title="Command List") subparsers = parser.add_subparsers(
help="", dest="command", required=True, title="Command List"
)
keys_parser = subparsers.add_parser('keys', help='Manage API keys database') keys_parser = subparsers.add_parser("keys", help="Manage API keys database")
keys_subparser = keys_parser.add_subparsers(help='', dest='sub_command', title="Command List") keys_subparser = keys_parser.add_subparsers(
help="", dest="sub_command", title="Command List"
)
keys_add_parser = keys_subparser.add_parser('add', help='Add API keys to database') keys_add_parser = keys_subparser.add_parser("add", help="Add API keys to database")
keys_add_parser.add_argument('req_limit', keys_add_parser.add_argument(
type=int, "req_limit", type=int, help="Request Limits (per second)"
help='Request Limits (per second)') )
keys_add_parser.add_argument('--key', keys_add_parser.add_argument(
type=str, "--key", type=str, default="auto", required=False, help="API Key"
default="auto", )
required=False,
help='API Key')
keys_remove_parser = keys_subparser.add_parser('remove', help='Remove API keys to database') keys_remove_parser = keys_subparser.add_parser(
keys_remove_parser.add_argument('key', "remove", help="Remove API keys to database"
type=str, )
help='API Key') keys_remove_parser.add_argument("key", type=str, help="API Key")
args = parser.parse_args() args = parser.parse_args()
if args.command == 'keys': if args.command == "keys":
db = Database() db = Database()
if args.sub_command is None: if args.sub_command is None:
# Print keys # Print keys
@ -36,10 +40,10 @@ def manage():
for item in keys: for item in keys:
print("%s: %s" % item) print("%s: %s" % item)
elif args.sub_command == 'add': elif args.sub_command == "add":
print(db.add(args.req_limit, args.key)[0]) print(db.add(args.req_limit, args.key)[0])
elif args.sub_command == 'remove': elif args.sub_command == "remove":
print(db.remove(args.key)) print(db.remove(args.key))
else: else:
parser.print_help() parser.print_help()
exit(1) exit(1)

12
app/no_limiter.py
View file

@ -1,8 +1,10 @@
from functools import wraps from functools import wraps
class Limiter: class Limiter:
def exempt(self, f): def exempt(self, f):
@wraps(f) @wraps(f)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
return f(*args, **kwargs) return f(*args, **kwargs)
return wrapper
return wrapper

4
install_models.py
View file

@ -3,5 +3,5 @@
from app.init import check_and_install_models, check_and_install_transliteration from app.init import check_and_install_models, check_and_install_transliteration
if __name__ == "__main__": if __name__ == "__main__":
check_and_install_models(force=True) check_and_install_models(force=True)
check_and_install_transliteration(force=True) check_and_install_transliteration(force=True)

12
setup.cfg Normal file
View file

@ -0,0 +1,12 @@
[flake8]
exclude = .git,
.vscode,
.gitignore,
README.md,
venv,
test,
setup.py,
app/__init__.py
max-line-length = 136
ignore = E741

4
tests/test_init.py
View file

@ -1,9 +1,9 @@
import pytest
from app.init import boot from app.init import boot
from argostranslate import package from argostranslate import package
def test_boot_argos(): def test_boot_argos():
"""Test Argos translate models initialization""" """Test Argos translate models initialization"""
boot() boot()
assert len(package.get_installed_packages()) > 2 assert len(package.get_installed_packages()) > 2