API keys support, bug fixes, improvements

This commit is contained in:
Piero Toffanin 2021-02-15 13:30:28 -05:00
parent 092990cfb3
commit 90de8e22a0
11 changed files with 242 additions and 44 deletions

2
.gitignore vendored
View file

@ -129,3 +129,5 @@ dmypy.json
.pyre/
installed_models/
# Misc
api_keys.db

View file

@ -112,7 +112,34 @@ docker-compose up -d --build
| --frontend-language-source | Set frontend default language - source | `en` |
| --frontend-language-target | Set frontend default language - target | `es` |
| --frontend-timeout | Set frontend translation timeout | `500` |
| --offline | Run user-interface entirely offline (don't use internet CDNs) | `false` |
| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` |
## Manage API Keys
LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits.
To use API keys simply start LibreTranslate with the `--api-keys` option.
### Add New Keys
To issue a new API key with 120 requests per minute limits:
```bash
ltmanage keys add 120
```
### Remove Keys
```bash
ltmanage keys remove <api-key>
```
### View Keys
```bash
ltmanage keys
```
## Roadmap
@ -120,14 +147,14 @@ Help us by opening a pull request!
- [x] A docker image (thanks [@vemonet](https://github.com/vemonet) !)
- [x] Auto-detect input language (thanks [@vemonet](https://github.com/vemonet) !)
- [ ] User authentication / tokens
- [X] User authentication / tokens
- [ ] Language bindings for every computer language
## FAQ
### Can I use your API server at libretranslate.com for my application in production?
The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to discuss options.
The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to get an API key or discuss other options.
## Credits

View file

@ -1 +1,2 @@
from .main import main
from .manage import manage

54
app/api_keys.py Normal file
View file

@ -0,0 +1,54 @@
import sqlite3
import uuid
from expiringdict import ExpiringDict
DEFAULT_DB_PATH = "api_keys.db"
class Database:
def __init__(self, db_path = DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
self.db_path = db_path
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
# Make sure to do data synchronization on writes!
self.c = sqlite3.connect(db_path, check_same_thread=False)
self.c.execute('''CREATE TABLE IF NOT EXISTS api_keys (
"api_key" TEXT NOT NULL,
"req_limit" INTEGER NOT NULL,
PRIMARY KEY("api_key")
);''')
def lookup(self, api_key):
req_limit = self.cache.get(api_key)
if req_limit is None:
# DB Lookup
stmt = self.c.execute('SELECT req_limit FROM api_keys WHERE api_key = ?', (api_key, ))
row = stmt.fetchone()
if row is not None:
self.cache[api_key] = row[0]
req_limit = row[0]
else:
self.cache[api_key] = False
req_limit = False
if isinstance(req_limit, bool):
req_limit = None
return req_limit
def add(self, req_limit, api_key = "auto"):
if api_key == "auto":
api_key = str(uuid.uuid4())
self.remove(api_key)
self.c.execute("INSERT INTO api_keys (api_key, req_limit) VALUES (?, ?)", (api_key, req_limit))
self.c.commit()
return (api_key, req_limit)
def remove(self, api_key):
self.c.execute('DELETE FROM api_keys WHERE api_key = ?', (api_key, ))
self.c.commit()
return api_key
def all(self):
row = self.c.execute("SELECT api_key, req_limit FROM api_keys")
return row.fetchall()

View file

@ -1,11 +1,16 @@
import os
from flask import Flask, render_template, jsonify, request, abort, send_from_directory
from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint
from langdetect import detect_langs
from langdetect import DetectorFactory
from pkg_resources import resource_filename
from .api_keys import Database
DetectorFactory.seed = 0 # deterministic
api_keys_db = None
def get_remote_address():
if request.headers.getlist("X-Forwarded-For"):
ip = request.headers.getlist("X-Forwarded-For")[0]
@ -14,8 +19,32 @@ def get_remote_address():
return ip
def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=False, frontend_language_source="en", frontend_language_target="en", frontend_timeout=500, offline=False):
if not offline:
def get_routes_limits(default_req_limit, api_keys_db):
if default_req_limit == -1:
# TODO: better way?
default_req_limit = 9999999999999
def limits():
req_limit = default_req_limit
if api_keys_db:
if request.is_json:
json = request.get_json()
api_key = json.get('api_key')
else:
api_key = request.values.get("api_key")
if api_key:
db_req_limit = api_keys_db.lookup(api_key)
if db_req_limit is not None:
req_limit = db_req_limit
return "%s per minute" % req_limit
return [limits]
def create_app(args):
if not args.offline:
from app.init import boot
boot()
@ -27,32 +56,32 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
for l in languages:
language_map[l.code] = l.name
if debug:
if args.debug:
app.config['TEMPLATES_AUTO_RELOAD'] = True
# Map userdefined frontend languages to argos language object.
if frontend_language_source == "auto":
if args.frontend_language_source == "auto":
frontend_argos_language_source = type('obj', (object,), {
'code': 'auto',
'name': 'Auto Detect'
})
else:
frontend_argos_language_source = next(iter([l for l in languages if l.code == frontend_language_source]), None)
frontend_argos_language_source = next(iter([l for l in languages if l.code == args.frontend_language_source]), None)
frontend_argos_language_target = next(iter([l for l in languages if l.code == frontend_language_target]), None)
frontend_argos_language_target = next(iter([l for l in languages if l.code == args.frontend_language_target]), None)
# Raise AttributeError to prevent app startup if user input is not valid.
if frontend_argos_language_source is None:
raise AttributeError(f"{frontend_language_source} as frontend source language is not supported.")
raise AttributeError(f"{args.frontend_language_source} as frontend source language is not supported.")
if frontend_argos_language_target is None:
raise AttributeError(f"{frontend_language_target} as frontend target language is not supported.")
raise AttributeError(f"{args.frontend_language_target} as frontend target language is not supported.")
if req_limit > 0:
if args.req_limit > 0 or args.api_keys:
from flask_limiter import Limiter
limiter = Limiter(
app,
key_func=get_remote_address,
default_limits=["%s per minute" % req_limit]
default_limits=get_routes_limits(args.req_limit, Database() if args.api_keys else None)
)
@app.errorhandler(400)
@ -68,10 +97,12 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
return jsonify({"error": "Slowdown: " + str(e.description)}), 429
@app.route("/")
@limiter.exempt
def index():
return render_template('index.html', gaId=ga_id, frontendTimeout=frontend_timeout, offline=offline)
return render_template('index.html', gaId=args.ga_id, frontendTimeout=args.frontend_timeout, offline=args.offline, api_keys=args.api_keys, web_version=os.environ.get('LT_WEB') is not None)
@app.route("/languages", methods=['GET', 'POST'])
@limiter.exempt
def langs():
"""
Retrieve list of supported languages
@ -149,6 +180,13 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
example: es
required: true
description: Target language code
- in: formData
name: api_key
schema:
type: string
example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
required: false
description: API key
responses:
200:
description: Translated text
@ -209,19 +247,19 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
batch = isinstance(q, list)
if batch and batch_limit != -1:
if batch and args.batch_limit != -1:
batch_size = len(q)
if batch_limit < batch_size:
abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, batch_limit))
if args.batch_limit < batch_size:
abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, args.batch_limit))
if char_limit != -1:
if args.char_limit != -1:
if batch:
chars = sum([len(text) for text in q])
else:
chars = len(q)
if char_limit < chars:
abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, char_limit))
if args.char_limit < chars:
abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, args.char_limit))
if source_lang == 'auto':
candidate_langs = list(filter(lambda l: l.lang in language_map, detect_langs(q)))
@ -229,7 +267,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
if len(candidate_langs) > 0:
candidate_langs.sort(key=lambda l: l.prob, reverse=True)
if debug:
if args.debug:
print(candidate_langs)
source_lang = next(iter([l.code for l in languages if l.code == candidate_langs[0].lang]), None)
@ -238,7 +276,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
else:
source_lang = 'en'
if debug:
if args.debug:
print("Auto detected: %s" % source_lang)
src_lang = next(iter([l for l in languages if l.code == source_lang]), None)
@ -274,6 +312,13 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
example: Hello world!
required: true
description: Text to detect
- in: formData
name: api_key
schema:
type: string
example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
required: false
description: API key
responses:
200:
description: Detections
@ -340,6 +385,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
@app.route("/frontend/settings")
@limiter.exempt
def frontend_settings():
"""
Retrieve frontend specific settings
@ -381,18 +427,19 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
type: string
description: Human-readable language name (in English)
"""
return jsonify({'charLimit': char_limit,
'frontendTimeout': frontend_timeout,
return jsonify({'charLimit': args.char_limit,
'frontendTimeout': args.frontend_timeout,
'language': {
'source': {'code': frontend_argos_language_source.code, 'name': frontend_argos_language_source.name},
'target': {'code': frontend_argos_language_target.code, 'name': frontend_argos_language_target.name}}
})
swag = swagger(app)
swag['info']['version'] = "1.0"
swag['info']['version'] = "1.2"
swag['info']['title'] = "LibreTranslate"
@app.route("/spec")
@limiter.exempt
def spec():
return jsonify(swag)

View file

@ -10,7 +10,7 @@ def main():
parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
help='Set character limit (%(default)s)')
parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
help='Set maximum number of requests per minute per client (%(default)s)')
help='Set the default maximum number of requests per minute per client (%(default)s)')
parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
help='Set maximum number of texts to translate in a batch request (%(default)s)')
parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
@ -27,18 +27,13 @@ def main():
help='Set frontend translation timeout (%(default)s)')
parser.add_argument('--offline', default=False, action="store_true",
help="Use offline")
parser.add_argument('--api-keys', default=False, action="store_true",
help="Enable API keys database for per-user rate limits lookup")
args = parser.parse_args()
app = create_app(args)
app = create_app(char_limit=args.char_limit,
req_limit=args.req_limit,
batch_limit=args.batch_limit,
ga_id=args.ga_id,
debug=args.debug,
frontend_language_source=args.frontend_language_source,
frontend_language_target=args.frontend_language_target,
frontend_timeout=args.frontend_timeout,
offline=args.offline)
if args.debug:
app.run(host=args.host, port=args.port)
else:

45
app/manage.py Normal file
View file

@ -0,0 +1,45 @@
import argparse
from app.api_keys import Database
def manage():
parser = argparse.ArgumentParser(description='LibreTranslate Manage Tools')
subparsers = parser.add_subparsers(help='', dest='command', required=True, title="Command List")
keys_parser = subparsers.add_parser('keys', help='Manage API keys database')
keys_subparser = keys_parser.add_subparsers(help='', dest='sub_command', title="Command List")
keys_add_parser = keys_subparser.add_parser('add', help='Add API keys to database')
keys_add_parser.add_argument('req_limit',
type=int,
help='Request Limits (per second)')
keys_add_parser.add_argument('--key',
type=str,
default="auto",
required=False,
help='API Key')
keys_remove_parser = keys_subparser.add_parser('remove', help='Remove API keys to database')
keys_remove_parser.add_argument('key',
type=str,
help='API Key')
args = parser.parse_args()
if args.command == 'keys':
db = Database()
if args.sub_command is None:
# Print keys
keys = db.all()
if not keys:
print("There are no API keys")
else:
for item in keys:
print("%s: %s" % item)
elif args.sub_command == 'add':
print(db.add(args.req_limit, args.key)[0])
elif args.sub_command == 'remove':
print(db.remove(args.key))
else:
parser.print_help()
exit(1)

View file

@ -60,13 +60,19 @@
<div class="nav-wrapper container"><a id="logo-container" href="/" class="brand-logo"><i class="material-icons">translate</i> LibreTranslate</a>
<ul class="right hide-on-med-and-down">
<li><a href="/docs">API Docs</a></li>
<li><a href="https://github.com/uav4geo/LibreTranslate">GitHub</a></li>
<li><a href="https://github.com/uav4geo/LibreTranslate">GitHub</a></li>
{% if api_keys %}
<li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
{% endif %}
</ul>
<ul id="nav-mobile" class="sidenav">
<li><a href="/docs">API Docs</a></li>
<li><a href="https://github.com/uav4geo/LibreTranslate">GitHub</a></li>
</ul>
<li><a href="https://github.com/uav4geo/LibreTranslate">GitHub</a></li>
{% if api_keys %}
<li><a href="javascript:setApiKey()" title="Set API Key"><i class="material-icons">vpn_key</i></a></li>
{% endif %}
</ul>
<a href="#" data-target="nav-mobile" class="sidenav-trigger"><i class="material-icons">menu</i></a>
</div>
</nav>
@ -131,7 +137,7 @@
<div class="input-field col s5">
<select class="browser-default" v-model="targetLang" ref="targetLangDropdown" @change="handleInput">
<template v-for="option in langs">
<option :value="option.code">[[ option.name ]]</option>
<option v-if="option.code !== 'auto'" :value="option.code">[[ option.name ]]</option>
</template>
</select>
</div>
@ -197,7 +203,7 @@
</div>
</div>
</div>
{% if web_version %}
<div class="section no-pad-bot" id="index-banner">
<div class="container">
<div class="row center">
@ -210,20 +216,25 @@
</div>
</div>
</div>
{% endif %}
</div>
</div>
<footer class="page-footer blue darken-3">
<div class="container">
<div class="row">
<div class="col l6 s12">
<div class="col l12 s12">
<h5 class="white-text">LibreTranslate</h5>
<p class="grey-text text-lighten-4">Free and Open Source Machine Translation API</p>
<p class="grey-text text-lighten-4">
Made with ❤ by <a class="grey-text text-lighten-3" href="https://uav4geo.com">UAV4GEO</a> and powered by <a class="grey-text text-lighten-3" href="https://github.com/argosopentech/argos-translate/">Argos Translate</a>
</p>
<p><a class="grey-text text-lighten-4" href="https://www.gnu.org/licenses/agpl-3.0.en.html">License: AGPLv3</a></p>
<p><a class="grey-text text-lighten-4" href="https://www.gnu.org/licenses/agpl-3.0.en.html">License: AGPLv3</a></p>
{% if web_version %}
<p>
The public API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please <a href="https://github.com/uav4geo/LibreTranslate" class="grey-text text-lighten-4" style="text-decoration: underline;">host your own server</a> or <a class="grey-text text-lighten-4" href="https://uav4geo.com/contact" style="text-decoration: underline;">get in touch</a> to obtain an API key.
</p>
{% endif %}
</div>
<div class="col l4 offset-l2 s12">
<!-- <h5 class="white-text">Links</h5>
@ -415,6 +426,7 @@ document.addEventListener('DOMContentLoaded', function(){
data.append("q", self.inputText);
data.append("source", self.sourceLang);
data.append("target", self.targetLang);
data.append("api_key", localStorage.getItem("api_key") || "");
request.open('POST', BaseUrl + '/translate', true);
@ -446,7 +458,6 @@ document.addEventListener('DOMContentLoaded', function(){
copyText: function(e){
e.preventDefault();
console.log(this.$refs);
this.$refs.translatedTextarea.select();
this.$refs.translatedTextarea.setSelectionRange(0, 9999999); /* For mobile devices */
document.execCommand("copy");
@ -468,6 +479,16 @@ document.addEventListener('DOMContentLoaded', function(){
});
});
function setApiKey(){
var prevKey = localStorage.getItem("api_key") || "";
var newKey = "";
newKey = window.prompt("Type in your API Key. If you need an API key, contact the server operator.", prevKey);
if (newKey === null) newKey = "";
localStorage.setItem("api_key", newKey);
}
</script>
</body>
</html>

4
manage.py Normal file
View file

@ -0,0 +1,4 @@
from app import manage
if __name__ == "__main__":
manage()

View file

@ -5,3 +5,4 @@ flask-swagger-ui==3.36.0
Flask-Limiter==1.4
waitress==1.4.4
langdetect==1.0.8
expiringdict==1.2.1

View file

@ -3,7 +3,7 @@
from setuptools import setup, find_packages
setup(
version='1.1.0',
version='1.2.0',
name='libretranslate',
license='GNU Affero General Public License v3.0',
description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.',
@ -18,6 +18,7 @@ setup(
entry_points={
'console_scripts': [
'libretranslate=app.main:main',
'ltmanage=app.manage:manage'
],
},