install_models.py note

Run helper script, misc improvements

.github/workflows/publish-docker.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Build and push Image
         uses: docker/build-push-action@v2
         with:
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           tags: |
             ${{ steps.get-variables.outputs.gh-username-lower }}/libretranslate:${{ env.TAG }},
             ghcr.io/${{ steps.get-variables.outputs.gh-username-lower }}/libretranslate:${{ env.TAG }}

Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update -qq \
-  && apt-get -qqq install --no-install-recommends -y libicu-dev pkg-config gcc g++ \
+  && apt-get -qqq install --no-install-recommends -y pkg-config gcc g++ \
   && apt-get clean \
   && rm -rf /var/lib/apt
 
@@ -24,8 +24,7 @@ FROM python:3.8.14-slim-bullseye
 ARG with_models=false
 ARG models=
 
-RUN addgroup --system --gid 1032 libretranslate && adduser --system --uid 1032 libretranslate
-RUN apt-get update -qq && apt-get -qqq install --no-install-recommends -y libicu67 && apt-get clean && rm -rf /var/lib/apt
+RUN addgroup --system --gid 1032 libretranslate && adduser --system --uid 1032 libretranslate && mkdir -p /home/libretranslate/.local && chown -R libretranslate:libretranslate /home/libretranslate/.local
 USER libretranslate
 
 COPY --from=builder --chown=1032:1032 /app /app

README.md

@@ -110,12 +110,8 @@ libretranslate [args]
 
 Then open a web browser to http://localhost:5000
 
-If you're on Windows, we recommend you [Run with Docker](#run-with-docker) instead.
-
 On Ubuntu 20.04 you can also use the install script available at https://github.com/argosopentech/LibreTranslate-init
 
-If you would rather run it natively, you can follow the guide [here](https://github.com/nuttolum/LibreOnWindows).
-
 ## Build and Run
 
 If you want to make changes to the code, you can build from source, and run the API:
@@ -134,14 +130,12 @@ Then open a web browser to http://localhost:5000
 
 ### Run with Docker
 
-Simply run:
-
-```bash
-docker run -ti --rm -p 5000:5000 libretranslate/libretranslate
-```
+Linux/MacOS: `./run.sh [args]`
+Windows: `run.bat [args]`
 
 Then open a web browser to http://localhost:5000
 
+
 ### Build with Docker
 
 ```bash
@@ -164,7 +158,7 @@ docker-compose up -d --build
 
 > Feel free to change the [`docker-compose.yml`](https://github.com/LibreTranslate/LibreTranslate/blob/main/docker-compose.yml) file to adapt it to your deployment needs, or use an extra `docker-compose.prod.yml` file for your deployment configuration.
 
-> The models are stored inside the container under `/root/.local/share` and `/root/.local/cache`. Feel free to use volumes if you do not want to redownload the models when the container is destroyed. Be aware that this will prevent the models from being updated!
+> The models are stored inside the container under `/home/libretranslate/.local/share` and `/home/libretranslate/.local/cache`. Feel free to use volumes if you do not want to redownload the models when the container is destroyed. To update the models, use the `--update-models` argument.
 
 ### CUDA
 
@@ -193,7 +187,7 @@ docker-compose -f docker-compose.cuda.yml up -d --build
 | --frontend-language-target  | Set frontend default language - target                                                                      | `es`          | LT_FRONTEND_LANGUAGE_TARGET  |
 | --frontend-timeout          | Set frontend translation timeout                                                                            | `500`         | LT_FRONTEND_TIMEOUT          |
 | --api-keys                  | Enable API keys database for per-user rate limits lookup                                                    | `Don't use API keys` | LT_API_KEYS                  |
-| --api-keys-db-path          | Use a specific path inside the container for the local database. Can be absolute or relative                | `api_keys.db`                      | LT_API_KEYS_DB_PATH          |
+| --api-keys-db-path          | Use a specific path inside the container for the local database. Can be absolute or relative                | `db/api_keys.db`                      | LT_API_KEYS_DB_PATH          |
 | --api-keys-remote           | Use this remote endpoint to query for valid API keys instead of using the local database                    | `Use local API key database` | LT_API_KEYS_REMOTE                  |
 | --get-api-key-link          | Show a link in the UI where to direct users to get an API key                                               | `Don't show a link` | LT_GET_API_KEY_LINK                  |
 | --require-api-key-origin    | Require use of an API key for programmatic access to the API, unless the request origin matches this domain | `No restrictions on domain origin` | LT_REQUIRE_API_KEY_ORIGIN    |
@@ -202,9 +196,28 @@ docker-compose -f docker-compose.cuda.yml up -d --build
 | --suggestions               | Allow user suggestions                                                                                      | `false`    | LT_SUGGESTIONS               |
 | --disable-files-translation | Disable files translation                                                                                   | `false`    | LT_DISABLE_FILES_TRANSLATION |
 | --disable-web-ui            | Disable web ui                                                                                              | `false`    | LT_DISABLE_WEB_UI            |
+| --update-models             | Update language models at startup                                                                                    | `false`    | LT_UPDATE_MODELS            |
 
 Note that each argument has an equivalent environment variable that can be used instead. The env. variables overwrite the default values but have lower priority than the command arguments and are particularly useful if used with Docker. The environment variable names are the upper-snake-case of the equivalent command argument's name with a `LT` prefix.
 
+## Update
+
+### Software
+
+If you installed with pip:
+
+`pip install -U libretranslate`
+
+If you're using docker:
+
+`docker pull libretranslate/libretranslate`
+
+### Language Models
+
+Start the program with the `--update-models` argument. For example: `libretranslate --update-models` or `./run.sh --update-models`.
+
+Alternatively you can also run the `install_models.py` script.
+
 ## Run with WSGI and Gunicorn
 
 ```
@@ -268,7 +281,8 @@ You can use the LibreTranslate API using the following bindings:
  - Swift: https://github.com/wacumov/libretranslate
  - Unix: https://github.com/argosopentech/LibreTranslate-sh
  - Shell: https://github.com/Hayao0819/Hayao-Tools/tree/master/libretranslate-sh
-
+ - Java: https://github.com/suuft/libretranslate-java
+ - 
 ## Discourse Plugin
 
 You can use this [discourse translator plugin](https://github.com/LibreTranslate/discourse-translator) to translate [Discourse](https://discourse.org) topics. To install it simply modify `/var/discourse/containers/app.yml`:
@@ -301,15 +315,15 @@ Then issue `./launcher rebuild app`. From the Discourse's admin panel then selec
 
 This is a list of public LibreTranslate instances, some require an API key. If you want to add a new URL, please open a pull request.
 
-URL |API Key Required|Payment Link|Cost
---- | --- | --- | ---
-[libretranslate.com](https://libretranslate.com)|:heavy_check_mark:|[Buy](https://buy.stripe.com/3cs4j3a4u4c8d3i289)| [$19 / month](https://buy.stripe.com/3cs4j3a4u4c8d3i289), 80 requests / minute limit
-[libretranslate.de](https://libretranslate.de)|-|-
-[translate.argosopentech.com](https://translate.argosopentech.com/)|-|-
-[translate.api.skitzen.com](https://translate.api.skitzen.com/)|-|-
-[translate.fortytwo-it.com](https://translate.fortytwo-it.com/)|-|-
-[translate.terraprint.co](https://translate.terraprint.co/)|-|-
-[lt.vern.cc](https://lt.vern.cc)|-|-
+URL |API Key Required | Links
+--- | --- | ---
+[libretranslate.com](https://libretranslate.com)|:heavy_check_mark:|[Get API Key](https://portal.libretranslate.com)
+[libretranslate.de](https://libretranslate.de)|-
+[translate.argosopentech.com](https://translate.argosopentech.com/)|-
+[translate.api.skitzen.com](https://translate.api.skitzen.com/)|-
+[translate.fortytwo-it.com](https://translate.fortytwo-it.com/)|-
+[translate.terraprint.co](https://translate.terraprint.co/)|-
+[lt.vern.cc](https://lt.vern.cc)|-
 
 ## TOR/i2p Mirrors
 
@@ -339,7 +353,7 @@ Help us by opening a pull request!
 
 ### Can I use your API server at libretranslate.com for my application in production?
 
-In short, no. [You need to buy an API key](https://buy.stripe.com/3cs4j3a4u4c8d3i289). You can always run LibreTranslate for free on your own server of course.
+In short, no. [You need to buy an API key](https://portal.libretranslate.com). You can always run LibreTranslate for free on your own server of course.
 
 ### Can I use LibreTranslate behind a reverse proxy, like Apache2 or Caddy?
 
@@ -410,6 +424,95 @@ Add this to an existing Caddyfile or save it as `Caddyfile` in any directory and
 
 </details>
 
+<details>
+<summary>NGINX config</summary>
+<br>
+
+Replace [YOUR_DOMAIN] with your full domain; for example, `translate.domain.tld` or `libretranslate.domain.tld`.
+  
+Remove `#` on the `access_log` and `error_log` lines to disable logging.
+
+```NginxConf
+server {
+  listen 80;
+  server_name [YOUR_DOMAIN];
+  return 301 https://$server_name$request_uri;
+}
+
+server {
+  listen 443 http2 ssl;
+  server_name [YOUR_DOMAIN];
+
+  #access_log off;
+  #error_log off;
+  
+  # SSL Section
+  ssl_certificate /etc/letsencrypt/live/[YOUR_DOMAIN]/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/[YOUR_DOMAIN]/privkey.pem;
+  
+  ssl_protocols TLSv1.2 TLSv1.3;
+
+  # Using the recommended cipher suite from: https://wiki.mozilla.org/Security/Server_Side_TLS
+  ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
+
+  ssl_session_timeout 10m;
+  ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
+  ssl_session_tickets off;
+
+  # Specifies a curve for ECDHE ciphers.
+  ssl_ecdh_curve prime256v1;
+  # Server should determine the ciphers, not the client
+  ssl_prefer_server_ciphers on;
+
+  
+  # Header section
+  add_header Strict-Transport-Security  "max-age=31536000; includeSubDomains; preload" always;
+  add_header Referrer-Policy            "strict-origin" always;
+
+  add_header X-Frame-Options            "SAMEORIGIN"    always;
+  add_header X-XSS-Protection           "1; mode=block" always;
+  add_header X-Content-Type-Options     "nosniff"       always;
+  add_header X-Download-Options         "noopen"        always;
+  add_header X-Robots-Tag               "none"          always;
+
+  add_header Feature-Policy             "microphone 'none'; camera 'none'; geolocation 'none';"  always;
+  # Newer header but not everywhere supported
+  add_header Permissions-Policy         "microphone=(), camera=(), geolocation=()" always;
+
+  # Remove X-Powered-By, which is an information leak
+  fastcgi_hide_header X-Powered-By;
+
+  # Do not send nginx server header
+  server_tokens off;
+  
+  # GZIP Section
+  gzip on;
+  gzip_disable "msie6";
+
+  gzip_vary on;
+  gzip_proxied any;
+  gzip_comp_level 6;
+  gzip_buffers 16 8k;
+  gzip_http_version 1.1;
+  gzip_min_length 256;
+  gzip_types text/xml text/javascript font/ttf font/eot font/otf application/x-javascript application/atom+xml application/javascript application/json application/manifest+json application/rss+xml application/x-web-app-manifest+json application/xhtml+xml application/xml image/svg+xml image/x-icon text/css text/plain;
+
+  location / {
+      proxy_pass http://127.0.0.1:5000/;
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      client_max_body_size 0;
+  }
+}
+
+```
+
+Add this to an existing NGINX config or save it as `libretranslate` in the `/etc/nginx/site-enabled` directory and run `sudo nginx -s reload`.
+
+</details>
+
 ## Credits
 
 This work is largely possible thanks to [Argos Translate](https://github.com/argosopentech/argos-translate), which powers the translation engine.

TRADEMARK.md

@@ -16,7 +16,8 @@ If you want to report misuse of an LibreTranslate trademark, please contact us v
 
 ## When do I need specific permission to use an LibreTranslate trademark?
 
-You may do the following without receiving specific permission from LibreTranslate (or its affiliates):
+Noting that LibreTranslate software combined with, or integrated into, any other software program, including but not limited to automation software for offering LibreTranslate as a cloud service or orchestration software for offering LibreTranslate in containers is considered "modified" LibreTranslate software, you may do the following without receiving specific permission from LibreTranslate (or its affiliates):
+
  * Use LibreTranslate wordmarks and/or logos in unmodified versions of LibreTranslate programs, products, services and technologies.
  * Use LibreTranslate wordmarks in text to truthfully refer to and/or link to unmodified LibreTranslate programs, products, services and technologies.
  * Use LibreTranslate logos in visuals to truthfully refer to and/or to link to the applicable programs, products, services and technologies hosted on LibreTranslate servers.

VERSION

@@ -1 +1 @@
-1.3.1
+1.3.4

app/__init__.py

@@ -1,13 +1,3 @@
 import os
-
-# override polyglot path
-import polyglot
-from appdirs import user_data_dir
-
-polyglot.polyglot_path = os.path.join(
-    user_data_dir(appname="LibreTranslate", appauthor="uav4geo"), "polyglot_data"
-)
-
-
 from .main import main
 from .manage import manage

app/api_keys.py

@@ -10,6 +10,14 @@ DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH']
 
 class Database:
     def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
+        # Legacy check - this can be removed at some point in the near future
+        if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"):
+            print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db"))
+            try:
+                os.rename("api_keys.db", "db/api_keys.db")
+            except Exception as e:
+                print(str(e))
+
         db_dir = os.path.dirname(db_path)
         if not db_dir == "" and not os.path.exists(db_dir):
             os.makedirs(db_dir)

app/app.py

@@ -15,7 +15,7 @@ from translatehtml import translate_html
 from werkzeug.utils import secure_filename
 
 from app import flood, remove_translated_files, security
-from app.language import detect_languages, transliterate, improve_translation_formatting
+from app.language import detect_languages, improve_translation_formatting
 
 from .api_keys import Database, RemoteDatabase
 from .suggestions import Database as SuggestionsDatabase
@@ -100,7 +100,7 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db):
 def create_app(args):
     from app.init import boot
 
-    boot(args.load_only)
+    boot(args.load_only, args.update_models)
 
     from app.language import load_languages
 
@@ -112,6 +112,9 @@ def create_app(args):
     if not args.disable_files_translation:
         remove_translated_files.setup(get_upload_dir())
     languages = load_languages()
+    language_pairs = {}
+    for lang in languages:
+        language_pairs[lang.code] = sorted([l.to_lang.code for l in lang.translations_from])
 
     # Map userdefined frontend languages to argos language object.
     if args.frontend_language_source == "auto":
@@ -269,17 +272,13 @@ def create_app(args):
                   name:
                     type: string
                     description: Human-readable language name (in English)
-          429:
-            description: Slow down
-            schema:
-              id: error-slow-down
-              type: object
-              properties:
-                error:
-                  type: string
-                  description: Reason for slow down
+                  targets:
+                    type: array
+                    items:
+                      type: string
+                    description: Supported target language codes
         """
-        return jsonify([{"code": l.code, "name": l.name} for l in languages])
+        return jsonify([{"code": l.code, "name": l.name, "targets": language_pairs.get(l.code, [])} for l in languages])
 
     # Add cors
     @app.after_request
@@ -486,11 +485,13 @@ def create_app(args):
                 results = []
                 for idx, text in enumerate(q):
                     translator = src_langs[idx].get_translation(tgt_lang)
+                    if translator is None:
+                        abort(400, description="%s (%s) is not available as a target language from %s (%s)" % (tgt_lang.name, tgt_lang.code, src_langs[idx].name, src_langs[idx].code))
+
                     if text_format == "html":
                         translated_text = str(translate_html(translator, text))
                     else:
-                        translated_text = improve_translation_formatting(text, translator.translate(
-                            transliterate(text, target_lang=source_langs[idx]["language"])))
+                        translated_text = improve_translation_formatting(text, translator.translate(text))
 
                     results.append(unescape(translated_text))
                 if source_lang == "auto":
@@ -502,24 +503,25 @@ def create_app(args):
                     )
                 else:
                     return jsonify(
-                         {
+                          {
                             "translatedText": results
-                         }
+                          }
                     )
             else:
                 translator = src_langs[0].get_translation(tgt_lang)
+                if translator is None:
+                    abort(400, description="%s (%s) is not available as a target language from %s (%s)" % (tgt_lang.name, tgt_lang.code, src_langs[0].name, src_langs[0].code))
 
                 if text_format == "html":
                     translated_text = str(translate_html(translator, q))
                 else:
-                    translated_text = improve_translation_formatting(q, translator.translate(
-                        transliterate(q, target_lang=source_langs[0]["language"])))
+                    translated_text = improve_translation_formatting(q, translator.translate(q))
 
                 if source_lang == "auto":
                     return jsonify(
                         {
                             "translatedText": unescape(translated_text),
-						    "detectedLanguage": source_langs[0]
+                            "detectedLanguage": source_langs[0]
                         }
                     )
                 else:
@@ -942,7 +944,7 @@ def create_app(args):
         return jsonify({"success": True})
 
     swag = swagger(app)
-    swag["info"]["version"] = "1.3.0"
+    swag["info"]["version"] = "1.3.1"
     swag["info"]["title"] = "LibreTranslate"
 
     @app.route("/spec")

app/default_values.py

@@ -113,7 +113,7 @@ _default_options_objects = [
     },
     {
         'name': 'API_KEYS_DB_PATH',
-        'default_value': 'api_keys.db',
+        'default_value': 'db/api_keys.db',
         'value_type': 'str'
     },
     {
@@ -156,6 +156,11 @@ _default_options_objects = [
         'default_value': False,
         'value_type': 'bool'
     },
+    {
+        'name': 'UPDATE_MODELS',
+        'default_value': False,
+        'value_type': 'bool'
+    },
 ]
 
 

app/detect.py

@@ -0,0 +1,72 @@
+# Originally adapted from https://github.com/aboSamoor/polyglot/blob/master/polyglot/base.py
+
+import pycld2 as cld2
+
+class UnknownLanguage(Exception):
+  pass
+
+class Language(object):
+  def __init__(self, choice):
+    name, code, confidence, bytesize = choice
+    self.code = code
+    self.name = name
+    self.confidence = float(confidence)
+    self.read_bytes = int(bytesize)
+
+  def __str__(self):
+    return ("name: {:<12}code: {:<9}confidence: {:>5.1f} "
+            "read bytes:{:>6}".format(self.name, self.code,
+                                    self.confidence, self.read_bytes))
+
+  @staticmethod
+  def from_code(code):
+    return Language(("", code, 100, 0))
+
+
+class Detector(object):
+  """ Detect the language used in a snippet of text."""
+
+  def __init__(self, text, quiet=False):
+    """ Detector of the language used in `text`.
+    Args:
+      text (string): unicode string.
+    """
+    self.__text = text
+    self.reliable = True
+    """False if the detector used Best Effort strategy in detection."""
+    self.quiet = quiet
+    """If true, exceptions will be silenced."""
+    self.detect(text)
+
+  @staticmethod
+  def supported_languages():
+    """Returns a list of the languages that can be detected by pycld2."""
+    return [name.capitalize() for name,code in cld2.LANGUAGES if not name.startswith("X_")]
+
+  def detect(self, text):
+    """Decide which language is used to write the text.
+    The method tries first to detect the language with high reliability. If
+    that is not possible, the method switches to best effort strategy.
+    Args:
+      text (string): A snippet of text, the longer it is the more reliable we
+                     can detect the language used to write the text.
+    """
+    reliable, index, top_3_choices = cld2.detect(text, bestEffort=False)
+
+    if not reliable:
+      self.reliable = False
+      reliable, index, top_3_choices = cld2.detect(text, bestEffort=True)
+      
+      if not self.quiet:
+        if not reliable:
+          raise UnknownLanguage("Try passing a longer snippet of text")
+
+    self.languages = [Language(x) for x in top_3_choices]
+    self.language = self.languages[0]
+    return self.language
+
+  def __str__(self):
+    text = "Prediction is reliable: {}\n".format(self.reliable)
+    text += u"\n".join(["Language {}: {}".format(i+1, str(l))
+                        for i,l in enumerate(self.languages)])
+    return text

app/init.py

@@ -1,15 +1,13 @@
 from pathlib import Path
 
-import polyglot
 from argostranslate import package, translate
 
 import app.language
 
 
-def boot(load_only=None):
+def boot(load_only=None, update_models=False):
     try:
-        check_and_install_models(load_only_lang_codes=load_only)
-        check_and_install_transliteration()
+        check_and_install_models(force=update_models, load_only_lang_codes=load_only)
     except Exception as e:
         print("Cannot update models (normal if you're offline): %s" % str(e))
 
@@ -59,36 +57,4 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
         print(
             "Loaded support for %s languages (%s models total)!"
             % (len(translate.get_installed_languages()), len(available_packages))
-        )
-
-
-def check_and_install_transliteration(force=False):
-    # 'en' is not a supported transliteration language
-    transliteration_languages = [
-        l.code for l in app.language.load_languages() if l.code != "en"
-    ]
-
-    # check installed
-    install_needed = []
-    if not force:
-        t_packages_path = Path(polyglot.polyglot_path) / "transliteration2"
-        for lang in transliteration_languages:
-            if not (
-                t_packages_path / lang / f"transliteration.{lang}.tar.bz2"
-            ).exists():
-                install_needed.append(lang)
-    else:
-        install_needed = transliteration_languages
-
-    # install the needed transliteration packages
-    if install_needed:
-        print(
-            f"Installing transliteration models for the following languages: {', '.join(install_needed)}"
-        )
-
-        from polyglot.downloader import Downloader
-
-        downloader = Downloader()
-
-        for lang in install_needed:
-            downloader.download(f"transliteration2.{lang}")
+        )

app/language.py

@@ -1,8 +1,7 @@
 import string
 
 from argostranslate import translate
-from polyglot.detect.base import Detector, UnknownLanguage
-from polyglot.transliteration.base import Transliterator
+from app.detect import Detector, UnknownLanguage
 
 __languages = None
 
@@ -84,7 +83,10 @@ def improve_translation_formatting(source, translation, improve_punctuation=True
 
     if not len(source):
         return ""
-
+    
+    if not len(translation):
+        return source
+    
     if improve_punctuation:
         source_last_char = source[len(source) - 1]
         translation_last_char = translation[len(translation) - 1]
@@ -113,52 +115,3 @@ def improve_translation_formatting(source, translation, improve_punctuation=True
 
     return translation
 
-
-def __transliterate_line(transliterator, line_text):
-    new_text = []
-
-    # transliteration is done word by word
-    for orig_word in line_text.split(" "):
-        # remove any punctuation on the right side
-        r_word = orig_word.rstrip(string.punctuation)
-        r_diff = set(char for char in orig_word) - set(char for char in r_word)
-        # and on the left side
-        l_word = orig_word.lstrip(string.punctuation)
-        l_diff = set(char for char in orig_word) - set(char for char in l_word)
-
-        # the actual transliteration of the word
-        t_word = transliterator.transliterate(orig_word.strip(string.punctuation))
-
-        # if transliteration fails, default back to the original word
-        if not t_word:
-            t_word = orig_word
-        else:
-            t_word = improve_translation_formatting(orig_word.strip(string.punctuation), t_word, improve_punctuation=False)
-
-            # add back any stripped punctuation
-            if r_diff:
-                t_word = t_word + "".join(r_diff)
-            if l_diff:
-                t_word = "".join(l_diff) + t_word
-
-        new_text.append(t_word)
-
-    # rebuild the text
-    return " ".join(new_text)
-
-
-def transliterate(text, target_lang="en"):
-    # initialize the transliterator from polyglot
-    transliterator = Transliterator(target_lang=target_lang)
-
-    # check for multiline string
-    if "\n" in text:
-        lines = []
-        # process each line separate
-        for line in text.split("\n"):
-            lines.append(__transliterate_line(transliterator, line))
-
-        # rejoin multiline string
-        return "\n".join(lines)
-    else:
-        return __transliterate_line(transliterator, text)

app/main.py

@@ -144,7 +144,9 @@ def get_args():
     parser.add_argument(
         "--disable-web-ui", default=DEFARGS['DISABLE_WEB_UI'], action="store_true", help="Disable web ui"
     )
-
+    parser.add_argument(
+        "--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup"
+    )
     return parser.parse_args()
 
 

app/static/css/main.css

@@ -61,14 +61,6 @@ h3.header {
   position: relative;
 }
 
-@-moz-document url-prefix() {
-  .language-select select {
-    -moz-appearance: none;
-    text-indent: -2px;
-    margin-right: -8px;
-  }
-}
-
 .language-select:after {
   content: "";
   width: 0.5em;

app/static/js/app.js

@@ -88,6 +88,8 @@ document.addEventListener('DOMContentLoaded', function(){
             langsRequest.send();
         },
         updated: function(){
+            if (this.isSuggesting) return;
+
             M.FormSelect.init(this.$refs.sourceLangDropdown);
             M.FormSelect.init(this.$refs.targetLangDropdown);
 
@@ -141,8 +143,16 @@ document.addEventListener('DOMContentLoaded', function(){
             isHtml: function(){
                 return htmlRegex.test(this.inputText);
             },
-            canSendSuggestion() {
+            canSendSuggestion: function(){
                 return this.translatedText.trim() !== "" && this.translatedText !== this.savedTanslatedText;
+            },
+            targetLangs: function(){
+                if (!this.sourceLang) return this.langs;
+                else{
+                    var lang = this.langs.find(l => l.code === this.sourceLang);
+                    if (!lang) return this.langs;
+                    return lang.targets.map(t => this.langs.find(l => l.code === t));
+                }
             }
         },
         filters: {
@@ -161,7 +171,13 @@ document.addEventListener('DOMContentLoaded', function(){
                 }
             },
             swapLangs: function(e){
-                this.closeSuggestTranslation(e)
+                this.closeSuggestTranslation(e);
+
+                // Make sure that we can swap
+                // by checking that the current target language
+                // has source language as target
+                var tgtLang = this.langs.find(l => l.code === this.targetLang);
+                if (tgtLang.targets.indexOf(this.sourceLang) === -1) return; // Not supported
 
                 var t = this.sourceLang;
                 this.sourceLang = this.targetLang;
@@ -269,11 +285,14 @@ document.addEventListener('DOMContentLoaded', function(){
                 this.savedTanslatedText = this.translatedText
 
                 this.isSuggesting = true;
+                this.$nextTick(() => {
+                    this.$refs.translatedTextarea.focus();
+                });
             },
             closeSuggestTranslation: function(e) {
                 if(this.isSuggesting) {
                     e.preventDefault();
-                    this.translatedText = this.savedTanslatedText
+                    // this.translatedText = this.savedTanslatedText
                 }
 
                 this.isSuggesting = false;
@@ -298,7 +317,7 @@ document.addEventListener('DOMContentLoaded', function(){
                     try{
                         var res = JSON.parse(this.response);
                         if (res.success){
-                            M.toast({html: 'Thanks for your correction.'})
+                            M.toast({html: 'Thanks for your correction. Note the suggestion will not take effect right away.'})
                             self.closeSuggestTranslation(e)
                         }else{
                             throw new Error(res.error || "Unknown error");
@@ -405,7 +424,7 @@ function handleLangsResponse(self, response) {
             return;
         }
 
-        self.langs.push({ name: "Auto Detect (Experimental)", code: "auto" })
+        self.langs.push({ name: "Auto Detect", code: "auto", targets: self.langs.map(l => l.code)})
 
         const sourceLanguage = self.langs.find(l => l.code === self.getQueryParam("source"))
         const targetLanguage = self.langs.find(l => l.code === self.getQueryParam("target"))

app/suggestions.py

@@ -1,12 +1,21 @@
 import sqlite3
+import os
 
 from expiringdict import ExpiringDict
 
-DEFAULT_DB_PATH = "suggestions.db"
+DEFAULT_DB_PATH = "db/suggestions.db"
 
 
 class Database:
     def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
+        # Legacy check - this can be removed at some point in the near future
+        if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"):
+            print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db"))
+            try:
+                os.rename("suggestions.db", "db/suggestions.db")
+            except Exception as e:
+                print(str(e))
+
         self.db_path = db_path
         self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
 

app/templates/index.html

@@ -156,7 +156,7 @@
 								</a>
 								<span>Translate into</span>
 								<select class="browser-default" v-model="targetLang" ref="targetLangDropdown" @change="handleInput">
-									<template v-for="option in langs">
+									<template v-for="option in targetLangs">
 										<option v-if="option.code !== 'auto'" :value="option.code">[[ option.name ]]</option>
 									</template>
 								</select>

install_models.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 import argparse
-from app.init import check_and_install_models, check_and_install_transliteration
+from app.init import check_and_install_models
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -10,4 +10,3 @@ if __name__ == "__main__":
     if len(lang_codes) == 0 or lang_codes[0] == '':
         lang_codes = None
     check_and_install_models(force=True, load_only_lang_codes=lang_codes)
-    check_and_install_transliteration(force=True)

requirements.txt

@@ -5,10 +5,8 @@ flask-swagger-ui==4.11.1
 Flask-Limiter==2.6.3
 waitress==2.1.2
 expiringdict==1.2.2
-pyicu>=2.8
-pycld2==0.41
+LTpycld2==0.42
 morfessor==2.0.6
-polyglot==16.7.4
 appdirs==1.4.4
 APScheduler==3.9.1
 translatehtml==1.5.2

run.bat

@@ -0,0 +1,41 @@
+@ECHO OFF
+
+SETLOCAL
+
+SET LT_PORT=5000
+
+:loop
+IF NOT "%1"=="" (
+    IF "%1"=="--port" (
+        SET LT_PORT=%2
+        SHIFT
+    )
+    IF "%1"=="--help" (
+        echo Usage: run.bat [--port N]
+        echo:
+        echo Run LibreTranslate using docker.
+        echo:
+        GOTO :done
+    )
+    IF "%1"=="--api-keys" (
+        SET DB_VOLUME=-v lt-db:/app/db
+        SHIFT
+    )
+    SHIFT
+    GOTO :loop
+)
+
+WHERE /Q docker
+IF %ERRORLEVEL% NEQ 0 GOTO :install_docker
+
+docker run -ti --rm -p %LT_PORT%:%LT_PORT% %DB_VOLUME% -v lt-local:/home/libretranslate/.local libretranslate/libretranslate %*
+
+GOTO :done
+
+:install_docker
+ECHO Cannot find docker! Go to https://docs.docker.com/desktop/install/windows-install/ and install docker before running this script (pressing Enter will open the page) 
+pause
+start "" https://docs.docker.com/desktop/install/windows-install/
+GOTO :done
+
+:done

run.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+set -eo pipefail
+__dirname=$(cd "$(dirname "$0")"; pwd -P)
+cd "${__dirname}"
+
+platform="Linux" # Assumed
+uname=$(uname)
+case $uname in
+    "Darwin")
+    platform="MacOS / OSX"
+    ;;
+    MINGW*)
+    platform="Windows"
+    ;;
+esac
+
+usage(){
+  echo "Usage: $0 [--port N]"
+  echo
+  echo "Run LibreTranslate using docker."
+  echo
+  exit
+}
+
+export LT_PORT=5000
+
+# Parse args for overrides
+ARGS=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+case $key in
+    --port)
+    export LT_PORT="$2"
+    ARGS+=("$1")
+    ARGS+=("$2") # save it in an array for later
+    shift # past argument
+    shift # past value
+    ;;
+    --debug)
+    export LT_DEBUG=YES
+    ARGS+=("$1")
+    shift # past argument
+    ;;
+    --api-keys)
+    export DB_VOLUME="-v lt-db:/app/db"
+    ARGS+=("$1")
+    shift # past argument
+    ;;
+    --help)
+    usage
+    ;;
+    *)    # unknown option
+    ARGS+=("$1")
+    shift # past argument
+    ;;
+esac
+done
+
+# $1 = command | $2 = help_text | $3 = install_command (optional)
+check_command(){
+    hash "$1" 2>/dev/null || not_found=true
+    if [[ $not_found ]]; then
+        check_msg_prefix="Checking for $1... "
+
+        # Can we attempt to install it?
+        if [[ -n "$3" ]]; then
+            echo -e "$check_msg_prefix \033[93mnot found, we'll attempt to install\033[39m"
+            $3 || sudo $3
+
+            # Recurse, but don't pass the install command
+            check_command "$1" "$2"
+        else
+            check_msg_result="\033[91m can't find $1! Check that the program is installed and that you have added the proper path to the program to your PATH environment variable before launching WebODM. If you change your PATH environment variable, remember to close and reopen your terminal. $2\033[39m"
+        fi
+    fi
+
+    echo -e "$check_msg_prefix $check_msg_result"
+    if [[ $not_found ]]; then
+        return 1
+    fi
+}
+
+environment_check(){
+    check_command "docker" "https://www.docker.com/"
+}
+
+environment_check
+docker run -ti --rm -p $LT_PORT:$LT_PORT $DB_VOLUME -v lt-local:/home/libretranslate/.local libretranslate/libretranslate ${ARGS[@]}