API keys support, bug fixes, improvements

.gitignore

@@ -129,3 +129,5 @@ dmypy.json
 .pyre/
 installed_models/
 
+# Misc
+api_keys.db

README.md

@@ -112,7 +112,34 @@ docker-compose up -d --build
 | --frontend-language-source | Set frontend default language - source | `en`          |
 | --frontend-language-target | Set frontend default language - target | `es`          |
 | --frontend-timeout | Set frontend translation timeout | `500`         |
+| --offline | Run user-interface entirely offline (don't use internet CDNs) | `false` |
+| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` |
 
+## Manage API Keys
+
+LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits.
+
+To use API keys simply start LibreTranslate with the `--api-keys` option.
+
+### Add New Keys
+
+To issue a new API key with 120 requests per minute limits:
+
+```bash
+ltmanage keys add 120
+```
+
+### Remove Keys
+
+```bash
+ltmanage keys remove <api-key>
+```
+
+### View Keys
+
+```bash
+ltmanage keys
+```
 
 ## Roadmap
 
@@ -120,14 +147,14 @@ Help us by opening a pull request!
 
 - [x] A docker image (thanks [@vemonet](https://github.com/vemonet) !)
 - [x] Auto-detect input language (thanks [@vemonet](https://github.com/vemonet) !)
-- [ ] User authentication / tokens
+- [X] User authentication / tokens
 - [ ] Language bindings for every computer language
 
 ## FAQ
 
 ### Can I use your API server at libretranslate.com for my application in production?
 
-The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to discuss options.
+The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to get an API key or discuss other options.
 
 ## Credits
 

app/__init__.py

@@ -1 +1,2 @@
 from .main import main
+from .manage import manage

app/api_keys.py

@@ -0,0 +1,54 @@
+import sqlite3
+import uuid
+from expiringdict import ExpiringDict
+
+DEFAULT_DB_PATH = "api_keys.db"
+
+class Database:
+    def __init__(self, db_path = DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
+        self.db_path = db_path
+        self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
+
+        # Make sure to do data synchronization on writes!
+        self.c = sqlite3.connect(db_path, check_same_thread=False)
+        self.c.execute('''CREATE TABLE IF NOT EXISTS api_keys (
+            "api_key"	TEXT NOT NULL,
+            "req_limit"	INTEGER NOT NULL,
+            PRIMARY KEY("api_key")
+        );''')
+
+    def lookup(self, api_key):
+        req_limit = self.cache.get(api_key)
+        if req_limit is None:
+            # DB Lookup
+            stmt = self.c.execute('SELECT req_limit FROM api_keys WHERE api_key = ?', (api_key, ))
+            row = stmt.fetchone()
+            if row is not None:
+                self.cache[api_key] = row[0]
+                req_limit = row[0]
+            else:
+                self.cache[api_key] = False
+                req_limit = False
+        
+        if isinstance(req_limit, bool):
+            req_limit = None
+        
+        return req_limit
+    
+    def add(self, req_limit, api_key = "auto"):
+        if api_key == "auto":
+            api_key = str(uuid.uuid4())
+
+        self.remove(api_key)
+        self.c.execute("INSERT INTO api_keys (api_key, req_limit) VALUES (?, ?)", (api_key, req_limit))
+        self.c.commit()
+        return (api_key, req_limit)
+    
+    def remove(self, api_key):
+        self.c.execute('DELETE FROM api_keys WHERE api_key = ?', (api_key, ))
+        self.c.commit()
+        return api_key
+
+    def all(self):
+        row = self.c.execute("SELECT api_key, req_limit FROM api_keys")
+        return row.fetchall()

app/app.py

@@ -1,11 +1,16 @@
+import os
 from flask import Flask, render_template, jsonify, request, abort, send_from_directory
 from flask_swagger import swagger
 from flask_swagger_ui import get_swaggerui_blueprint
 from langdetect import detect_langs
 from langdetect import DetectorFactory
 from pkg_resources import resource_filename
+from .api_keys import Database
+
 DetectorFactory.seed = 0 # deterministic
 
+api_keys_db = None
+
 def get_remote_address():
     if request.headers.getlist("X-Forwarded-For"):
         ip = request.headers.getlist("X-Forwarded-For")[0]
@@ -14,8 +19,32 @@ def get_remote_address():
 
     return ip
 
-def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=False, frontend_language_source="en", frontend_language_target="en", frontend_timeout=500, offline=False):
-    if not offline:
+def get_routes_limits(default_req_limit, api_keys_db):
+    if default_req_limit == -1:
+        # TODO: better way?
+        default_req_limit = 9999999999999
+
+    def limits():
+        req_limit = default_req_limit
+
+        if api_keys_db:
+            if request.is_json:
+                json = request.get_json()
+                api_key = json.get('api_key')
+            else:
+                api_key = request.values.get("api_key")
+
+            if api_key:
+                db_req_limit = api_keys_db.lookup(api_key)
+                if db_req_limit is not None:
+                    req_limit = db_req_limit
+
+        return "%s per minute" % req_limit
+    
+    return [limits]
+
+def create_app(args):
+    if not args.offline:
         from app.init import boot
         boot()
 
@@ -27,32 +56,32 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
     for l in languages:
         language_map[l.code] = l.name
 
-    if debug:
+    if args.debug:
         app.config['TEMPLATES_AUTO_RELOAD'] = True
 
     # Map userdefined frontend languages to argos language object.
-    if frontend_language_source == "auto":
+    if args.frontend_language_source == "auto":
         frontend_argos_language_source = type('obj', (object,), {
             'code': 'auto',
             'name': 'Auto Detect'
         })
     else:
-        frontend_argos_language_source = next(iter([l for l in languages if l.code == frontend_language_source]), None)
+        frontend_argos_language_source = next(iter([l for l in languages if l.code == args.frontend_language_source]), None)
 
-    frontend_argos_language_target = next(iter([l for l in languages if l.code == frontend_language_target]), None)
+    frontend_argos_language_target = next(iter([l for l in languages if l.code == args.frontend_language_target]), None)
 
     # Raise AttributeError to prevent app startup if user input is not valid.
     if frontend_argos_language_source is None:
-        raise AttributeError(f"{frontend_language_source} as frontend source language is not supported.")
+        raise AttributeError(f"{args.frontend_language_source} as frontend source language is not supported.")
     if frontend_argos_language_target is None:
-        raise AttributeError(f"{frontend_language_target} as frontend target language is not supported.")
+        raise AttributeError(f"{args.frontend_language_target} as frontend target language is not supported.")
 
-    if req_limit > 0:
+    if args.req_limit > 0 or args.api_keys:
         from flask_limiter import Limiter
         limiter = Limiter(
             app,
             key_func=get_remote_address,
-            default_limits=["%s per minute" % req_limit]
+            default_limits=get_routes_limits(args.req_limit, Database() if args.api_keys else None)
         )
 
     @app.errorhandler(400)
@@ -68,10 +97,12 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
         return jsonify({"error": "Slowdown: " + str(e.description)}), 429
 
     @app.route("/")
+    @limiter.exempt
     def index():
-        return render_template('index.html', gaId=ga_id, frontendTimeout=frontend_timeout, offline=offline)
+        return render_template('index.html', gaId=args.ga_id, frontendTimeout=args.frontend_timeout, offline=args.offline, api_keys=args.api_keys, web_version=os.environ.get('LT_WEB') is not None)
 
     @app.route("/languages", methods=['GET', 'POST'])
+    @limiter.exempt
     def langs():
         """
         Retrieve list of supported languages
@@ -149,6 +180,13 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
               example: es
             required: true
             description: Target language code
+          - in: formData
+            name: api_key
+            schema:
+              type: string
+              example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+            required: false
+            description: API key
         responses:
           200:
             description: Translated text
@@ -209,19 +247,19 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
 
         batch = isinstance(q, list)
 
-        if batch and batch_limit != -1:
+        if batch and args.batch_limit != -1:
           batch_size = len(q)
-          if batch_limit < batch_size:
-            abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, batch_limit))
+          if args.batch_limit < batch_size:
+            abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, args.batch_limit))
 
-        if char_limit != -1:
+        if args.char_limit != -1:
             if batch:
               chars = sum([len(text) for text in q])
             else:
               chars = len(q)
 
-            if char_limit < chars:
-              abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, char_limit))
+            if args.char_limit < chars:
+              abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, args.char_limit))
 
         if source_lang == 'auto':
             candidate_langs = list(filter(lambda l: l.lang in language_map, detect_langs(q)))
@@ -229,7 +267,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
             if len(candidate_langs) > 0:
                 candidate_langs.sort(key=lambda l: l.prob, reverse=True)
 
-                if debug:
+                if args.debug:
                     print(candidate_langs)
 
                 source_lang = next(iter([l.code for l in languages if l.code == candidate_langs[0].lang]), None)
@@ -238,7 +276,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
             else:
                 source_lang = 'en'
 
-            if debug:
+            if args.debug:
                 print("Auto detected: %s" % source_lang)
 
         src_lang = next(iter([l for l in languages if l.code == source_lang]), None)
@@ -274,6 +312,13 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
               example: Hello world!
             required: true
             description: Text to detect
+          - in: formData
+            name: api_key
+            schema:
+              type: string
+              example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+            required: false
+            description: API key
         responses:
           200:
             description: Detections
@@ -340,6 +385,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
 
 
     @app.route("/frontend/settings")
+    @limiter.exempt
     def frontend_settings():
         """
         Retrieve frontend specific settings
@@ -381,18 +427,19 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
                           type: string
                           description: Human-readable language name (in English)
         """
-        return jsonify({'charLimit': char_limit,
-                        'frontendTimeout': frontend_timeout,
+        return jsonify({'charLimit': args.char_limit,
+                        'frontendTimeout': args.frontend_timeout,
                         'language': {
                             'source': {'code': frontend_argos_language_source.code, 'name': frontend_argos_language_source.name},
                             'target': {'code': frontend_argos_language_target.code, 'name': frontend_argos_language_target.name}}
                        })
 
     swag = swagger(app)
-    swag['info']['version'] = "1.0"
+    swag['info']['version'] = "1.2"
     swag['info']['title'] = "LibreTranslate"
 
     @app.route("/spec")
+    @limiter.exempt
     def spec():
         return jsonify(swag)
 

app/main.py

@@ -10,7 +10,7 @@ def main():
     parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
                         help='Set character limit (%(default)s)')
     parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
-                        help='Set maximum number of requests per minute per client (%(default)s)')
+                        help='Set the default maximum number of requests per minute per client (%(default)s)')
     parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
                         help='Set maximum number of texts to translate in a batch request (%(default)s)')
     parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
@@ -27,18 +27,13 @@ def main():
                         help='Set frontend translation timeout (%(default)s)')
     parser.add_argument('--offline', default=False, action="store_true",
                         help="Use offline")
+    parser.add_argument('--api-keys', default=False, action="store_true",
+                        help="Enable API keys database for per-user rate limits lookup")
+    
 
     args = parser.parse_args()
+    app = create_app(args)
 
-    app = create_app(char_limit=args.char_limit,
-                     req_limit=args.req_limit,
-                     batch_limit=args.batch_limit,
-                     ga_id=args.ga_id,
-                     debug=args.debug,
-                     frontend_language_source=args.frontend_language_source,
-                     frontend_language_target=args.frontend_language_target,
-                     frontend_timeout=args.frontend_timeout,
-                     offline=args.offline)
     if args.debug:
         app.run(host=args.host, port=args.port)
     else:

app/manage.py

@@ -0,0 +1,45 @@
+import argparse
+from app.api_keys import Database
+
+def manage():
+    parser = argparse.ArgumentParser(description='LibreTranslate Manage Tools')
+    subparsers = parser.add_subparsers(help='', dest='command', required=True, title="Command List")
+
+    keys_parser = subparsers.add_parser('keys', help='Manage API keys database')
+    keys_subparser = keys_parser.add_subparsers(help='', dest='sub_command', title="Command List")
+
+    keys_add_parser = keys_subparser.add_parser('add', help='Add API keys to database')
+    keys_add_parser.add_argument('req_limit',
+                    type=int,
+                    help='Request Limits (per second)')
+    keys_add_parser.add_argument('--key',
+                    type=str,
+                    default="auto",
+                    required=False,
+                    help='API Key')
+
+    keys_remove_parser = keys_subparser.add_parser('remove', help='Remove API keys to database')
+    keys_remove_parser.add_argument('key',
+                    type=str,
+                    help='API Key')
+
+    args = parser.parse_args()
+    
+    if args.command == 'keys':
+        db = Database()
+        if args.sub_command is None:
+            # Print keys
+            keys = db.all()
+            if not keys:
+                print("There are no API keys")
+            else:
+                for item in keys:
+                    print("%s: %s" % item)
+
+        elif args.sub_command == 'add':
+            print(db.add(args.req_limit, args.key)[0])
+        elif args.sub_command == 'remove':
+            print(db.remove(args.key))
+    else:
+        parser.print_help()
+        exit(1)

app/templates/index.html

@@ -61,11 +61,17 @@
 	  <ul class="right hide-on-med-and-down">
 		<li><a href="/docs">API Docs</a></li>
         <li><a href="https://github.com/uav4geo/LibreTranslate">GitHub</a></li>
+        {% if api_keys %}
+        <li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
+        {% endif %}
 	  </ul>
 
 	  <ul id="nav-mobile" class="sidenav">
 		<li><a href="/docs">API Docs</a></li>
         <li><a href="https://github.com/uav4geo/LibreTranslate">GitHub</a></li>
+        {% if api_keys %}
+        <li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
+        {% endif %}
       </ul>
 	  <a href="#" data-target="nav-mobile" class="sidenav-trigger"><i class="material-icons">menu</i></a>
 	</div>
@@ -131,7 +137,7 @@
 						 <div class="input-field col s5">
 							<select  class="browser-default" v-model="targetLang" ref="targetLangDropdown" @change="handleInput">
 								<template v-for="option in langs">
-									 <option :value="option.code">[[ option.name ]]</option>
+									 <option v-if="option.code !== 'auto'" :value="option.code">[[ option.name ]]</option>
 								</template>
 							</select>
 						 </div>
@@ -197,7 +203,7 @@
 	  </div>
 	</div>
   </div>
-
+ {% if web_version %}
   <div class="section no-pad-bot" id="index-banner">
 	<div class="container">
 	   <div class="row center">
@@ -210,20 +216,25 @@
   		</div>
 	  </div>
 	</div>
-
+{% endif %}
 </div>
 </div>
 
 <footer class="page-footer blue darken-3">
   <div class="container">
 	<div class="row">
-	  <div class="col l6 s12">
+	  <div class="col l12 s12">
 		<h5 class="white-text">LibreTranslate</h5>
 		<p class="grey-text text-lighten-4">Free and Open Source Machine Translation API</p>
 		<p class="grey-text text-lighten-4">
 			Made with ❤ by <a class="grey-text text-lighten-3" href="https://uav4geo.com">UAV4GEO</a> and powered by <a class="grey-text text-lighten-3" href="https://github.com/argosopentech/argos-translate/">Argos Translate</a>
 		</p>
         <p><a class="grey-text text-lighten-4"  href="https://www.gnu.org/licenses/agpl-3.0.en.html">License: AGPLv3</a></p>
+{% if web_version %}
+        <p>
+			The public API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please <a href="https://github.com/uav4geo/LibreTranslate" class="grey-text text-lighten-4" style="text-decoration: underline;">host your own server</a> or <a class="grey-text text-lighten-4" href="https://uav4geo.com/contact" style="text-decoration: underline;">get in touch</a> to obtain an API key.
+		</p>
+{% endif %}
 	  </div>
 	  <div class="col l4 offset-l2 s12">
 		<!-- <h5 class="white-text">Links</h5>
@@ -415,6 +426,7 @@ document.addEventListener('DOMContentLoaded', function(){
 	  			data.append("q", self.inputText);
 	  			data.append("source", self.sourceLang);
 	  			data.append("target", self.targetLang);
+				data.append("api_key", localStorage.getItem("api_key") || "");
 
 				request.open('POST', BaseUrl + '/translate', true);
 
@@ -446,7 +458,6 @@ document.addEventListener('DOMContentLoaded', function(){
 
 		copyText: function(e){
 			e.preventDefault();
-			console.log(this.$refs);
 			this.$refs.translatedTextarea.select();
 			this.$refs.translatedTextarea.setSelectionRange(0, 9999999); /* For mobile devices */
 			document.execCommand("copy");
@@ -468,6 +479,16 @@ document.addEventListener('DOMContentLoaded', function(){
 	});
 
 });
+
+function setApiKey(){
+    var prevKey = localStorage.getItem("api_key") || "";
+    var newKey = "";
+    newKey = window.prompt("Type in your API Key. If you need an API key, contact the server operator.", prevKey);
+    if (newKey === null) newKey = "";
+
+	localStorage.setItem("api_key", newKey);
+}
+
 </script>
 </body>
 </html>

manage.py

@@ -0,0 +1,4 @@
+from app import manage
+
+if __name__ == "__main__":
+    manage()

requirements.txt

@@ -5,3 +5,4 @@ flask-swagger-ui==3.36.0
 Flask-Limiter==1.4
 waitress==1.4.4
 langdetect==1.0.8
+expiringdict==1.2.1

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(
-    version='1.1.0',
+    version='1.2.0',
     name='libretranslate',
     license='GNU Affero General Public License v3.0',
     description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.',
@@ -18,6 +18,7 @@ setup(
     entry_points={
         'console_scripts': [
             'libretranslate=app.main:main',
+            'ltmanage=app.manage:manage'
         ],
     },