From d6ef04ba3e528acc219a61b815b45b1d3076de1f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 May 2022 16:17:41 -0400 Subject: [PATCH 01/21] Extract JSONL from suggestions.db script --- suggestions-to-jsonl.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 suggestions-to-jsonl.py diff --git a/suggestions-to-jsonl.py b/suggestions-to-jsonl.py new file mode 100755 index 0000000..7878d52 --- /dev/null +++ b/suggestions-to-jsonl.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +import argparse +import time +import sqlite3 +import json + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db") + parser.add_argument( + "--db", + type=str, + nargs=1, + help="Path to suggestions.db file", + default='suggestions.db' + ) + parser.add_argument( + "--clear", + action='store_true', + help="Clear suggestions.db after generation", + default=False + ) + args = parser.parse_args() + + output_file = str(int(time.time())) + ".jsonl" + + con = sqlite3.connect(args.db, check_same_thread=False) + cur = con.cursor() + + with open(output_file, 'w', encoding="utf-8") as f: + for row in cur.execute('SELECT q, s, source, target FROM suggestions WHERE source != "auto" ORDER BY source'): + q, s, source, target = row + obj = { + 'q': q, + 's': s, + 'source': source, + 'target': target + } + json.dump(obj, f, ensure_ascii=False) + f.write('\n') + + print("Wrote %s" % output_file) + + if args.clear: + cur.execute("DELETE FROM suggestions") + con.commit() + print("Cleared " + args.db) \ No newline at end of file From e81d119603c29fd3483bf78941526d015dd1151e Mon Sep 17 00:00:00 2001 From: Dingedi <64697405+dingedi@users.noreply.github.com> Date: Fri, 27 May 2022 10:41:27 +0200 Subject: [PATCH 02/21] Upgrade argos-translate-files Add support for .epub and .html --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 94f6a34..4f605d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,6 @@ polyglot==16.7.4 appdirs==1.4.4 APScheduler==3.9.1 translatehtml==1.5.2 -argos-translate-files==1.0.5 +argos-translate-files==1.1.0 itsdangerous==2.1.2 Werkzeug==2.1.2 From 9831ba88a605ef5d197b0df4a86b950cec846631 Mon Sep 17 00:00:00 2001 From: dingedi Date: Mon, 30 May 2022 09:14:45 +0200 Subject: [PATCH 03/21] improve translation of punctuation --- app/app.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/app.py b/app/app.py index f3eccfc..935d7a0 100644 --- a/app/app.py +++ b/app/app.py @@ -475,6 +475,21 @@ def create_app(args): abort(400, description="%s format is not supported" % text_format) def improve_translation(source, translation): + source = source.strip() + + source_last_char = source[len(source) - 1] + translation_last_char = translation[len(translation) - 1] + + punctuation_chars = ['!', '?', '.', ',', ';'] + if source_last_char in punctuation_chars: + if translation_last_char != source_last_char: + if translation_last_char in punctuation_chars: + translation = translation[:-1] + + translation += source_last_char + elif translation_last_char in punctuation_chars: + translation = translation[:-1] + if source.islower(): return translation.lower() From 967077748466c5d252d2978d2d11233b66446e79 Mon Sep 17 00:00:00 2001 From: Arya K <73596856+gi-yt@users.noreply.github.com> Date: Wed, 8 Jun 2022 21:51:40 +0530 Subject: [PATCH 04/21] Add lt.vern.cc instance to instance list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c3eb804..8e211a9 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,7 @@ URL |API Key Required|Payment Link|Cost [libretranslate.pussthecat.org](https://libretranslate.pussthecat.org/)|-|- [translate.fortytwo-it.com](https://translate.fortytwo-it.com/)|-|- [translate.terraprint.co](https://translate.terraprint.co/)|-|- +[lt.vern.cc](https://lt.vern.cc)|-|- ## Adding New Languages From 89dde2d46835711578fac5ea874590c6d2dd5dc6 Mon Sep 17 00:00:00 2001 From: Jon Wiggins Date: Mon, 20 Jun 2022 16:20:07 -0600 Subject: [PATCH 05/21] Add CUDA docker version --- README.md | 10 ++++++++++ docker-compose.cuda.yml | 12 ++++++++++++ docker/Dockerfile.cuda | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 docker-compose.cuda.yml create mode 100644 docker/Dockerfile.cuda diff --git a/README.md b/README.md index 8e211a9..706cb24 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,16 @@ docker-compose up -d --build > Feel free to change the [`docker-compose.yml`](https://github.com/LibreTranslate/LibreTranslate/blob/main/docker-compose.yml) file to adapt it to your deployment needs, or use an extra `docker-compose.prod.yml` file for your deployment configuration. +### CUDA + +You can use hardware acceleration to speed up translations on a GPU machine with CUDA 11.2 and [nvidia-docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) installed. + +Run this version with: + +```bash +docker-compose -f docker-compose.cuda.yml up -d --build +``` + ## Arguments | Argument | Description | Default | Env. name | diff --git a/docker-compose.cuda.yml b/docker-compose.cuda.yml new file mode 100644 index 0000000..5f8beaa --- /dev/null +++ b/docker-compose.cuda.yml @@ -0,0 +1,12 @@ +version: "3" + +services: + libretranslate-cuda: + container_name: libretranslate-cuda + build: + context: . + dockerfile: docker/Dockerfile.cuda + restart: unless-stopped + ports: + - 5000:5000 + command: --gpus all diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda new file mode 100644 index 0000000..a07662e --- /dev/null +++ b/docker/Dockerfile.cuda @@ -0,0 +1,37 @@ +FROM nvidia/cuda:11.2.2-devel-ubuntu20.04 + +ENV ARGOS_DEVICE_TYPE cuda +ARG with_models=true +ARG models= + +WORKDIR /app + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update -qq \ + && apt-get -qqq install --no-install-recommends -y libicu-dev libaspell-dev libcairo2 libcairo2-dev pkg-config gcc g++ python3.8-dev python3-pip libpython3.8-dev\ + && apt-get clean \ + && rm -rf /var/lib/apt + +RUN pip3 install --upgrade pip + +COPY . . + +RUN ln -s /usr/bin/python3 /usr/bin/python + +RUN if [ "$with_models" = "true" ]; then \ + # install only the dependencies first + pip3 install -e .; \ + # initialize the language models + if [ ! -z "$models" ]; then \ + ./install_models.py --load_only_lang_codes "$models"; \ + else \ + ./install_models.py; \ + fi \ + fi + +# Install package from source code +RUN pip3 install . \ + && pip3 cache purge +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib:/usr/local/cuda/lib64 +EXPOSE 5000 +ENTRYPOINT [ "libretranslate", "--host", "0.0.0.0" ] From 647379aea5e54de6f8c372a71609926a0acdcfb3 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jun 2022 14:22:12 -0400 Subject: [PATCH 06/21] Fix check_and_install_transliteration --- app/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/init.py b/app/init.py index b1c6997..476676c 100644 --- a/app/init.py +++ b/app/init.py @@ -65,7 +65,7 @@ def check_and_install_models(force=False, load_only_lang_codes=None): def check_and_install_transliteration(force=False): # 'en' is not a supported transliteration language transliteration_languages = [ - l.code for l in app.language.languages if l.code != "en" + l.code for l in app.language.load_languages() if l.code != "en" ] # check installed From 8f8087f8ae3b073ce0fc19c01315472000176b2b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jun 2022 14:57:32 -0400 Subject: [PATCH 07/21] Add --api-keys-remote --- README.md | 1 + app/api_keys.py | 27 ++++++++++++++++++++++++++- app/app.py | 9 +++++++-- app/default_values.py | 5 +++++ app/main.py | 6 ++++++ requirements.txt | 2 ++ 6 files changed, 47 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 706cb24..bdb4eb3 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,7 @@ docker-compose -f docker-compose.cuda.yml up -d --build | --frontend-language-target | Set frontend default language - target | `es` | LT_FRONTEND_LANGUAGE_TARGET | | --frontend-timeout | Set frontend translation timeout | `500` | LT_FRONTEND_TIMEOUT | | --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | LT_API_KEYS | +| --api-keys-remote | Use this remote endpoint to query for valid API keys instead of using the local database | `Use local API key database` | LT_API_KEYS_REMOTE | | --require-api-key-origin | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` | LT_REQUIRE_API_KEY_ORIGIN | | --load-only | Set available languages | `all from argostranslate` | LT_LOAD_ONLY | | --suggestions | Allow user suggestions | `false` | LT_SUGGESTIONS | diff --git a/app/api_keys.py b/app/api_keys.py index 905047b..eb2d496 100644 --- a/app/api_keys.py +++ b/app/api_keys.py @@ -1,6 +1,6 @@ import sqlite3 import uuid - +import requests from expiringdict import ExpiringDict DEFAULT_DB_PATH = "api_keys.db" @@ -61,3 +61,28 @@ class Database: def all(self): row = self.c.execute("SELECT api_key, req_limit FROM api_keys") return row.fetchall() + + +class RemoteDatabase: + def __init__(self, url, max_cache_len=1000, max_cache_age=600): + self.url = url + self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) + + def lookup(self, api_key): + req_limit = self.cache.get(api_key) + if req_limit is None: + try: + r = requests.post(self.url, data={'api_key': api_key}) + res = r.json() + except Exception as e: + print("Cannot authenticate API key: " + str(e)) + return False + + if res.get('error', None) is None: + req_limit = res.get('req_limit', None) + self.cache[api_key] = req_limit + else: + req_limit = False + self.cache[api_key] = False + + return req_limit diff --git a/app/app.py b/app/app.py index 935d7a0..cba22a2 100644 --- a/app/app.py +++ b/app/app.py @@ -17,7 +17,7 @@ from werkzeug.utils import secure_filename from app import flood, remove_translated_files, security from app.language import detect_languages, transliterate -from .api_keys import Database +from .api_keys import Database, RemoteDatabase from .suggestions import Database as SuggestionsDatabase @@ -146,7 +146,12 @@ def create_app(args): api_keys_db = None if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: - api_keys_db = Database() if args.api_keys else None + api_keys_db = None + if args.api_keys: + if args.api_keys_remote: + api_keys_db = RemoteDatabase(args.api_keys_remote) + else: + api_keys_db = Database() from flask_limiter import Limiter diff --git a/app/default_values.py b/app/default_values.py index 9161aa6..f8043b5 100644 --- a/app/default_values.py +++ b/app/default_values.py @@ -105,6 +105,11 @@ _default_options_objects = [ 'name': 'API_KEYS', 'default_value': False, 'value_type': 'bool' + }, + { + 'name': 'API_KEYS_REMOTE', + 'default_value': '', + 'value_type': 'str' }, { 'name': 'REQUIRE_API_KEY_ORIGIN', diff --git a/app/main.py b/app/main.py index 1545a77..9e44050 100644 --- a/app/main.py +++ b/app/main.py @@ -89,6 +89,12 @@ def get_args(): action="store_true", help="Enable API keys database for per-user rate limits lookup", ) + parser.add_argument( + "--api-keys-remote", + default=DEFARGS['API_KEYS_REMOTE'], + type=str, + help="Use this remote endpoint to query for valid API keys instead of using the local database", + ) parser.add_argument( "--require-api-key-origin", type=str, diff --git a/requirements.txt b/requirements.txt index 4f605d5..9ea3949 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,5 @@ translatehtml==1.5.2 argos-translate-files==1.1.0 itsdangerous==2.1.2 Werkzeug==2.1.2 +requests==2.28.0 + From ed68f1bcf929ee1d597edd332afcf4af7c81b869 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jun 2022 15:17:42 -0400 Subject: [PATCH 08/21] Add --get-api-key-link --- README.md | 1 + app/app.py | 1 + app/default_values.py | 7 ++++++- app/main.py | 6 ++++++ app/static/js/app.js | 4 +++- app/templates/index.html | 9 ++++++++- 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bdb4eb3..0f32081 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,7 @@ docker-compose -f docker-compose.cuda.yml up -d --build | --frontend-timeout | Set frontend translation timeout | `500` | LT_FRONTEND_TIMEOUT | | --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | LT_API_KEYS | | --api-keys-remote | Use this remote endpoint to query for valid API keys instead of using the local database | `Use local API key database` | LT_API_KEYS_REMOTE | +| --get-api-key-link | Show a link in the UI where to direct users to get an API key | `Don't show a link` | LT_GET_API_KEY_LINK | | --require-api-key-origin | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` | LT_REQUIRE_API_KEY_ORIGIN | | --load-only | Set available languages | `all from argostranslate` | LT_LOAD_ONLY | | --suggestions | Allow user suggestions | `false` | LT_SUGGESTIONS | diff --git a/app/app.py b/app/app.py index cba22a2..17c595a 100644 --- a/app/app.py +++ b/app/app.py @@ -232,6 +232,7 @@ def create_app(args): gaId=args.ga_id, frontendTimeout=args.frontend_timeout, api_keys=args.api_keys, + get_api_key_link=args.get_api_key_link, web_version=os.environ.get("LT_WEB") is not None, version=get_version() ) diff --git a/app/default_values.py b/app/default_values.py index f8043b5..70d3911 100644 --- a/app/default_values.py +++ b/app/default_values.py @@ -105,12 +105,17 @@ _default_options_objects = [ 'name': 'API_KEYS', 'default_value': False, 'value_type': 'bool' - }, + }, { 'name': 'API_KEYS_REMOTE', 'default_value': '', 'value_type': 'str' }, + { + 'name': 'GET_API_KEY_LINK', + 'default_value': '', + 'value_type': 'str' + }, { 'name': 'REQUIRE_API_KEY_ORIGIN', 'default_value': '', diff --git a/app/main.py b/app/main.py index 9e44050..e179661 100644 --- a/app/main.py +++ b/app/main.py @@ -95,6 +95,12 @@ def get_args(): type=str, help="Use this remote endpoint to query for valid API keys instead of using the local database", ) + parser.add_argument( + "--get-api-key-link", + default=DEFARGS['GET_API_KEY_LINK'], + type=str, + help="Show a link in the UI where to direct users to get an API key", + ) parser.add_argument( "--require-api-key-origin", type=str, diff --git a/app/static/js/app.js b/app/static/js/app.js index f81cdb2..de29041 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -436,7 +436,9 @@ function getTextWidth(text) { function setApiKey(){ var prevKey = localStorage.getItem("api_key") || ""; var newKey = ""; - newKey = window.prompt("Type in your API Key. If you need an API key, contact the server operator.", prevKey); + var instructions = "contact the server operator."; + if (window.getApiKeyLink) instructions = "press the \"Get API Key\" link." + newKey = window.prompt("Type in your API Key. If you need an API key, " + instructions, prevKey); if (newKey === null) newKey = ""; localStorage.setItem("api_key", newKey); diff --git a/app/templates/index.html b/app/templates/index.html index 8862ec5..b7d6cc7 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -41,10 +41,14 @@
  • API Docs
    • + {% if get_api_key_link %} +
  • Get API Key
    • + + {% endif %}
  • GitHub
    • {% if api_keys %}
  • vpn_key
    • @@ -53,6 +57,9 @@