diff --git a/Dockerfile b/Dockerfile index 9aae39e..f6de520 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,21 +11,23 @@ RUN apt-get update -qq \ && apt-get clean \ && rm -rf /var/lib/apt +RUN apt-get update && apt-get upgrade --assume-yes + RUN pip install --upgrade pip COPY . . RUN if [ "$with_models" = "true" ]; then \ - # install only the dependencies first - pip install -e .; \ - # initialize the language models - if [ ! -z "$models" ]; then \ - ./install_models.py --load_only_lang_codes "$models"; \ - else \ - ./install_models.py; \ - fi \ - fi + # install only the dependencies first + pip install -e .; \ + # initialize the language models + if [ ! -z "$models" ]; then \ + ./install_models.py --load_only_lang_codes "$models"; \ + else \ + ./install_models.py; \ + fi \ + fi # Install package from source code RUN pip install . \ && pip cache purge diff --git a/README.md b/README.md index c3eb804..27f1d75 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,18 @@ docker-compose up -d --build > Feel free to change the [`docker-compose.yml`](https://github.com/LibreTranslate/LibreTranslate/blob/main/docker-compose.yml) file to adapt it to your deployment needs, or use an extra `docker-compose.prod.yml` file for your deployment configuration. +> The models are stored inside the container under `/root/.local/share` and `/root/.local/cache`. Feel free to use volumes if you do not want to redownload the models when the container is destroyed. Be aware that this will prevent the models from being updated! + +### CUDA + +You can use hardware acceleration to speed up translations on a GPU machine with CUDA 11.2 and [nvidia-docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) installed. + +Run this version with: + +```bash +docker-compose -f docker-compose.cuda.yml up -d --build +``` + ## Arguments | Argument | Description | Default | Env. name | @@ -180,8 +192,12 @@ docker-compose up -d --build | --frontend-language-target | Set frontend default language - target | `es` | LT_FRONTEND_LANGUAGE_TARGET | | --frontend-timeout | Set frontend translation timeout | `500` | LT_FRONTEND_TIMEOUT | | --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | LT_API_KEYS | +| --api-keys-db-path | Use a specific path inside the container for the local database. Can be absolute or relative | `api_keys.db` | LT_API_KEYS_DB_PATH | +| --api-keys-remote | Use this remote endpoint to query for valid API keys instead of using the local database | `Use local API key database` | LT_API_KEYS_REMOTE | +| --get-api-key-link | Show a link in the UI where to direct users to get an API key | `Don't show a link` | LT_GET_API_KEY_LINK | | --require-api-key-origin | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` | LT_REQUIRE_API_KEY_ORIGIN | | --load-only | Set available languages | `all from argostranslate` | LT_LOAD_ONLY | +| --threads | Set number of threads | `4` | LT_THREADS | | --suggestions | Allow user suggestions | `false` | LT_SUGGESTIONS | | --disable-files-translation | Disable files translation | `false` | LT_DISABLE_FILES_TRANSLATION | | --disable-web-ui | Disable web ui | `false` | LT_DISABLE_WEB_UI | @@ -210,7 +226,7 @@ See ["LibreTranslate: your own translation service on Kubernetes" by JM Robles]( LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits. -To use API keys simply start LibreTranslate with the `--api-keys` option. +To use API keys simply start LibreTranslate with the `--api-keys` option. If you modified the API keys database path with the option `--api-keys-db-path`, you must specify the path with the same argument flag when using the `ltmanage keys` command. ### Add New Keys @@ -220,6 +236,11 @@ To issue a new API key with 120 requests per minute limits: ltmanage keys add 120 ``` +If you changed the API keys database path: +```bash +ltmanage keys --api-keys-db-path path/to/db/dbName.db add 120 +``` + ### Remove Keys ```bash @@ -282,9 +303,9 @@ URL |API Key Required|Payment Link|Cost [libretranslate.de](https://libretranslate.de)|-|- [translate.argosopentech.com](https://translate.argosopentech.com/)|-|- [translate.api.skitzen.com](https://translate.api.skitzen.com/)|-|- -[libretranslate.pussthecat.org](https://libretranslate.pussthecat.org/)|-|- [translate.fortytwo-it.com](https://translate.fortytwo-it.com/)|-|- [translate.terraprint.co](https://translate.terraprint.co/)|-|- +[lt.vern.cc](https://lt.vern.cc)|-|- ## Adding New Languages diff --git a/app/api_keys.py b/app/api_keys.py index 905047b..b60b87b 100644 --- a/app/api_keys.py +++ b/app/api_keys.py @@ -1,13 +1,18 @@ +import os import sqlite3 import uuid - +import requests from expiringdict import ExpiringDict +from app.default_values import DEFAULT_ARGUMENTS as DEFARGS -DEFAULT_DB_PATH = "api_keys.db" +DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH'] class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): + db_dir = os.path.dirname(db_path) + if not db_dir == "" and not os.path.exists(db_dir): + os.makedirs(db_dir) self.db_path = db_path self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) @@ -61,3 +66,27 @@ class Database: def all(self): row = self.c.execute("SELECT api_key, req_limit FROM api_keys") return row.fetchall() + + +class RemoteDatabase: + def __init__(self, url, max_cache_len=1000, max_cache_age=600): + self.url = url + self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) + + def lookup(self, api_key): + req_limit = self.cache.get(api_key) + if req_limit is None: + try: + r = requests.post(self.url, data={'api_key': api_key}) + res = r.json() + except Exception as e: + print("Cannot authenticate API key: " + str(e)) + return None + + if res.get('error', None) is None: + req_limit = res.get('req_limit', None) + else: + req_limit = None + self.cache[api_key] = req_limit + + return req_limit diff --git a/app/app.py b/app/app.py index f3eccfc..6986344 100644 --- a/app/app.py +++ b/app/app.py @@ -17,7 +17,7 @@ from werkzeug.utils import secure_filename from app import flood, remove_translated_files, security from app.language import detect_languages, transliterate -from .api_keys import Database +from .api_keys import Database, RemoteDatabase from .suggestions import Database as SuggestionsDatabase @@ -146,7 +146,12 @@ def create_app(args): api_keys_db = None if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: - api_keys_db = Database() if args.api_keys else None + api_keys_db = None + if args.api_keys: + if args.api_keys_remote: + api_keys_db = RemoteDatabase(args.api_keys_remote) + else: + api_keys_db = Database(args.api_keys_db_path) from flask_limiter import Limiter @@ -190,9 +195,12 @@ def create_app(args): and api_keys_db.lookup(ak) is None and request.headers.get("Origin") != args.require_api_key_origin ): + description = "Please contact the server operator to get an API key" + if args.get_api_key_link: + description = "Visit %s to get an API key" % args.get_api_key_link abort( 403, - description="Please contact the server operator to obtain an API key", + description=description, ) return f(*a, **kw) @@ -227,6 +235,7 @@ def create_app(args): gaId=args.ga_id, frontendTimeout=args.frontend_timeout, api_keys=args.api_keys, + get_api_key_link=args.get_api_key_link, web_version=os.environ.get("LT_WEB") is not None, version=get_version() ) @@ -475,6 +484,21 @@ def create_app(args): abort(400, description="%s format is not supported" % text_format) def improve_translation(source, translation): + source = source.strip() + + source_last_char = source[len(source) - 1] + translation_last_char = translation[len(translation) - 1] + + punctuation_chars = ['!', '?', '.', ',', ';'] + if source_last_char in punctuation_chars: + if translation_last_char != source_last_char: + if translation_last_char in punctuation_chars: + translation = translation[:-1] + + translation += source_last_char + elif translation_last_char in punctuation_chars: + translation = translation[:-1] + if source.islower(): return translation.lower() diff --git a/app/default_values.py b/app/default_values.py index 9161aa6..88a32ef 100644 --- a/app/default_values.py +++ b/app/default_values.py @@ -106,6 +106,21 @@ _default_options_objects = [ 'default_value': False, 'value_type': 'bool' }, + { + 'name': 'API_KEYS_DB_PATH', + 'default_value': 'api_keys.db', + 'value_type': 'str' + }, + { + 'name': 'API_KEYS_REMOTE', + 'default_value': '', + 'value_type': 'str' + }, + { + 'name': 'GET_API_KEY_LINK', + 'default_value': '', + 'value_type': 'str' + }, { 'name': 'REQUIRE_API_KEY_ORIGIN', 'default_value': '', @@ -116,6 +131,11 @@ _default_options_objects = [ 'default_value': None, 'value_type': 'str' }, + { + 'name': 'THREADS', + 'default_value': 4, + 'value_type': 'int' + }, { 'name': 'SUGGESTIONS', 'default_value': False, diff --git a/app/flood.py b/app/flood.py index bc7fc1d..088cf64 100644 --- a/app/flood.py +++ b/app/flood.py @@ -31,7 +31,7 @@ def setup(violations_threshold=100): threshold = violations_threshold scheduler = BackgroundScheduler() - scheduler.add_job(func=forgive_banned, trigger="interval", minutes=480) + scheduler.add_job(func=forgive_banned, trigger="interval", minutes=30) scheduler.start() # Shut down the scheduler when exiting the app diff --git a/app/init.py b/app/init.py index b1c6997..476676c 100644 --- a/app/init.py +++ b/app/init.py @@ -65,7 +65,7 @@ def check_and_install_models(force=False, load_only_lang_codes=None): def check_and_install_transliteration(force=False): # 'en' is not a supported transliteration language transliteration_languages = [ - l.code for l in app.language.languages if l.code != "en" + l.code for l in app.language.load_languages() if l.code != "en" ] # check installed diff --git a/app/main.py b/app/main.py index 1545a77..0df79fb 100644 --- a/app/main.py +++ b/app/main.py @@ -89,6 +89,24 @@ def get_args(): action="store_true", help="Enable API keys database for per-user rate limits lookup", ) + parser.add_argument( + "--api-keys-db-path", + default=DEFARGS['API_KEYS_DB_PATH'], + type=str, + help="Use a specific path inside the container for the local database. Can be absolute or relative (%(default)s)", + ) + parser.add_argument( + "--api-keys-remote", + default=DEFARGS['API_KEYS_REMOTE'], + type=str, + help="Use this remote endpoint to query for valid API keys instead of using the local database", + ) + parser.add_argument( + "--get-api-key-link", + default=DEFARGS['GET_API_KEY_LINK'], + type=str, + help="Show a link in the UI where to direct users to get an API key", + ) parser.add_argument( "--require-api-key-origin", type=str, @@ -102,6 +120,13 @@ def get_args(): metavar="", help="Set available languages (ar,de,en,es,fr,ga,hi,it,ja,ko,pt,ru,zh)", ) + parser.add_argument( + "--threads", + default=DEFARGS['THREADS'], + type=int, + metavar="", + help="Set number of threads (%(default)s)", + ) parser.add_argument( "--suggestions", default=DEFARGS['SUGGESTIONS'], action="store_true", help="Allow user suggestions" ) @@ -128,11 +153,15 @@ def main(): else: from waitress import serve + url_scheme = "https" if args.ssl else "http" + print("Running on %s://%s:%s" % (url_scheme, args.host, args.port)) + serve( app, host=args.host, port=args.port, - url_scheme="https" if args.ssl else "http", + url_scheme=url_scheme, + threads=args.threads ) diff --git a/app/manage.py b/app/manage.py index be548f7..2385e46 100644 --- a/app/manage.py +++ b/app/manage.py @@ -1,6 +1,8 @@ import argparse +import os from app.api_keys import Database +from app.default_values import DEFAULT_ARGUMENTS as DEFARGS def manage(): @@ -10,6 +12,12 @@ def manage(): ) keys_parser = subparsers.add_parser("keys", help="Manage API keys database") + keys_parser.add_argument( + "--api-keys-db-path", + default=DEFARGS['API_KEYS_DB_PATH'], + type=str, + help="Use a specific path inside the container for the local database", + ) keys_subparser = keys_parser.add_subparsers( help="", dest="sub_command", title="Command List" ) @@ -30,7 +38,10 @@ def manage(): args = parser.parse_args() if args.command == "keys": - db = Database() + if not os.path.exists(args.api_keys_db_path): + print("No such database: %s" % args.api_keys_db_path) + exit(1) + db = Database(args.api_keys_db_path) if args.sub_command is None: # Print keys keys = db.all() diff --git a/app/static/css/dark-theme.css b/app/static/css/dark-theme.css index d89ff08..12d8549 100644 --- a/app/static/css/dark-theme.css +++ b/app/static/css/dark-theme.css @@ -7,12 +7,12 @@ .blue.darken-3 { background-color: #1E5DA6 !important; } - - /*" like in btn-delete-text */ - .btn-flat { - color: #666; - } + /* like in btn-delete-text */ + .btn-flat { + color: #666; + } + .btn-switch-type { background-color: #333; color: #5CA8FF; @@ -30,18 +30,19 @@ color: #fff; } + /* like in textarea */ .card-content { - border: 1px solid #444 !important; - background-color: #222 !important; - color: #fff; + border: 1px solid #444 !important; + background-color: #222 !important; + color: #fff; } - + .file-dropzone { background: #222; - border: 1px solid #444; - margin-top: 1rem; + border: 1px solid #444; + margin-top: 1rem; } - + select { color: #fff; background: #111; @@ -57,10 +58,11 @@ background-color: #222 !important; color: #fff; } + /* like in file dropzone */ .textarea-container { - margin-top: 1rem; + margin-top: 1rem; } - + .code { border: 1px solid #444; background: #222; diff --git a/app/static/js/app.js b/app/static/js/app.js index f81cdb2..6605f8e 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -39,14 +39,17 @@ document.addEventListener('DOMContentLoaded', function(){ filesTranslation: true, frontendTimeout: 500 }, - mounted: function(){ - var self = this; - var requestSettings = new XMLHttpRequest(); - requestSettings.open('GET', BaseUrl + '/frontend/settings', true); + mounted: function() { + const self = this; - requestSettings.onload = function() { + const settingsRequest = new XMLHttpRequest(); + settingsRequest.open("GET", BaseUrl + "/frontend/settings", true); + + const langsRequest = new XMLHttpRequest(); + langsRequest.open("GET", BaseUrl + "/languages", true); + + settingsRequest.onload = function() { if (this.status >= 200 && this.status < 400) { - // Success! self.settings = JSON.parse(this.response); self.sourceLang = self.settings.language.source.code; self.targetLang = self.settings.language.target.code; @@ -55,74 +58,42 @@ document.addEventListener('DOMContentLoaded', function(){ self.supportedFilesFormat = self.settings.supportedFilesFormat; self.filesTranslation = self.settings.filesTranslation; self.frontendTimeout = self.settings.frontendTimeout; - }else { + + if (langsRequest.response) { + handleLangsResponse(self, langsRequest); + } else { + langsRequest.onload = function() { + handleLangsResponse(self, this); + } + } + } else { self.error = "Cannot load /frontend/settings"; self.loading = false; } }; - requestSettings.onerror = function() { + settingsRequest.onerror = function() { self.error = "Error while calling /frontend/settings"; self.loading = false; }; - requestSettings.send(); - - var requestLanguages = new XMLHttpRequest(); - requestLanguages.open('GET', BaseUrl + '/languages', true); - - requestLanguages.onload = function() { - if (this.status >= 200 && this.status < 400) { - // Success! - self.langs = JSON.parse(this.response); - self.langs.push({ name: 'Auto Detect (Experimental)', code: 'auto' }) - if (self.langs.length === 0){ - self.loading = false; - self.error = "No languages available. Did you install the models correctly?" - return; - } - - const sourceLanguage = self.langs.find(l => l.code === self.getQueryParam('source')) - const isSourceAuto = !sourceLanguage && self.getQueryParam('source') === "auto" - const targetLanguage = self.langs.find(l => l.code === self.getQueryParam('target')) - - if (sourceLanguage || isSourceAuto) { - self.sourceLang = isSourceAuto ? "auto" : sourceLanguage.code - } - - if (targetLanguage) { - self.targetLang = targetLanguage.code - } - - const defaultText = self.getQueryParam('q') - - if(defaultText) { - self.inputText = decodeURI(defaultText) - } - - self.loading = false; - } else { - self.error = "Cannot load /languages"; - self.loading = false; - } - }; - - requestLanguages.onerror = function() { + langsRequest.onerror = function() { self.error = "Error while calling /languages"; self.loading = false; }; - requestLanguages.send(); + settingsRequest.send(); + langsRequest.send(); }, updated: function(){ M.FormSelect.init(this.$refs.sourceLangDropdown); M.FormSelect.init(this.$refs.targetLangDropdown); - + if (this.$refs.inputTextarea){ if (this.inputText === ""){ this.$refs.inputTextarea.style.height = this.inputTextareaHeight + "px"; this.$refs.translatedTextarea.style.height = this.inputTextareaHeight + "px"; - }else{ + } else{ this.$refs.inputTextarea.style.height = this.$refs.translatedTextarea.style.height = "1px"; this.$refs.inputTextarea.style.height = Math.max(this.inputTextareaHeight, this.$refs.inputTextarea.scrollHeight + 32) + "px"; this.$refs.translatedTextarea.style.height = Math.max(this.inputTextareaHeight, this.$refs.translatedTextarea.scrollHeight + 32) + "px"; @@ -136,28 +107,12 @@ document.addEventListener('DOMContentLoaded', function(){ // Update "selected" attribute (to overcome a vue.js limitation) // but properly display checkmarks on supported browsers. // Also change the