run ruff formatting

This commit is contained in:
Vincent Emonet 2023-07-09 12:29:11 +02:00
parent 62712dcad3
commit bf18dcbcf9
22 changed files with 82 additions and 75 deletions

View file

@ -1,8 +1,10 @@
import os import os
import sqlite3 import sqlite3
import uuid import uuid
import requests import requests
from expiringdict import ExpiringDict from expiringdict import ExpiringDict
from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS
DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH'] DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH']
@ -12,14 +14,14 @@ class Database:
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
# Legacy check - this can be removed at some point in the near future # Legacy check - this can be removed at some point in the near future
if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"): if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"):
print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db")) print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db"))
try: try:
os.rename("api_keys.db", "db/api_keys.db") os.rename("api_keys.db", "db/api_keys.db")
except Exception as e: except Exception as e:
print(str(e)) print(str(e))
db_dir = os.path.dirname(db_path) db_dir = os.path.dirname(db_path)
if not db_dir == "" and not os.path.exists(db_dir): if db_dir != '' and not os.path.exists(db_dir):
os.makedirs(db_dir) os.makedirs(db_dir)
self.db_path = db_path self.db_path = db_path
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
@ -91,10 +93,7 @@ class RemoteDatabase:
print("Cannot authenticate API key: " + str(e)) print("Cannot authenticate API key: " + str(e))
return None return None
if res.get('error', None) is None: req_limit = res.get('req_limit', None) if res.get('error', None) is None else None
req_limit = res.get('req_limit', None)
else:
req_limit = None
self.cache[api_key] = req_limit self.cache[api_key] = req_limit
return req_limit return req_limit

View file

@ -1,30 +1,37 @@
import io import io
import os import os
import tempfile
import re import re
import tempfile
import uuid import uuid
from datetime import datetime
from functools import wraps from functools import wraps
from html import unescape from html import unescape
from timeit import default_timer from timeit import default_timer
from datetime import datetime
import argostranslatefiles import argostranslatefiles
from argostranslatefiles import get_supported_formats from argostranslatefiles import get_supported_formats
from flask import (abort, Blueprint, Flask, jsonify, render_template, request, from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for
Response, send_file, url_for, session) from flask_babel import Babel
from flask_session import Session
from flask_swagger import swagger from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint from flask_swagger_ui import get_swaggerui_blueprint
from flask_session import Session
from translatehtml import translate_html from translatehtml import translate_html
from werkzeug.utils import secure_filename
from werkzeug.exceptions import HTTPException from werkzeug.exceptions import HTTPException
from werkzeug.http import http_date from werkzeug.http import http_date
from flask_babel import Babel from werkzeug.utils import secure_filename
from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage
from libretranslate.language import detect_languages, improve_translation_formatting from libretranslate.language import detect_languages, improve_translation_formatting
from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped, from libretranslate.locales import (
gettext_html, lazy_swag, get_alternate_locale_links) _,
_lazy,
get_alternate_locale_links,
get_available_locale_codes,
get_available_locales,
gettext_escaped,
gettext_html,
lazy_swag,
)
from .api_keys import Database, RemoteDatabase from .api_keys import Database, RemoteDatabase
from .suggestions import Database as SuggestionsDatabase from .suggestions import Database as SuggestionsDatabase
@ -150,10 +157,7 @@ def create_app(args):
frontend_argos_language_source = languages[0] frontend_argos_language_source = languages[0]
if len(languages) >= 2: language_target_fallback = languages[1] if len(languages) >= 2 else languages[0]
language_target_fallback = languages[1]
else:
language_target_fallback = languages[0]
if args.frontend_language_target == "locale": if args.frontend_language_target == "locale":
def resolve_language_locale(): def resolve_language_locale():
@ -185,10 +189,7 @@ def create_app(args):
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
api_keys_db = None api_keys_db = None
if args.api_keys: if args.api_keys:
if args.api_keys_remote: api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path)
api_keys_db = RemoteDatabase(args.api_keys_remote)
else:
api_keys_db = Database(args.api_keys_db_path)
from flask_limiter import Limiter from flask_limiter import Limiter
@ -220,7 +221,7 @@ def create_app(args):
os.mkdir(default_mp_dir) os.mkdir(default_mp_dir)
os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir
from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess
@bp.route("/metrics") @bp.route("/metrics")
@limiter.exempt @limiter.exempt
@ -544,10 +545,7 @@ def create_app(args):
) )
if args.char_limit != -1: if args.char_limit != -1:
if batch: chars = sum([len(text) for text in q]) if batch else len(q)
chars = sum([len(text) for text in q])
else:
chars = len(q)
if args.char_limit < chars: if args.char_limit < chars:
abort( abort(
@ -557,10 +555,7 @@ def create_app(args):
if source_lang == "auto": if source_lang == "auto":
source_langs = [] source_langs = []
if batch: auto_detect_texts = q if batch else [q]
auto_detect_texts = q
else:
auto_detect_texts = [q]
overall_candidates = detect_languages(q) overall_candidates = detect_languages(q)
@ -1093,7 +1088,7 @@ def create_app(args):
return override_lang return override_lang
return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes())) return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes()))
babel = Babel(app, locale_selector=get_locale) Babel(app, locale_selector=get_locale)
app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html) app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html)

View file

@ -2,10 +2,11 @@
import pycld2 as cld2 import pycld2 as cld2
class UnknownLanguage(Exception): class UnknownLanguage(Exception):
pass pass
class Language(object): class Language:
def __init__(self, choice): def __init__(self, choice):
name, code, confidence, bytesize = choice name, code, confidence, bytesize = choice
self.code = code self.code = code
@ -23,7 +24,7 @@ class Language(object):
return Language(("", code, 100, 0)) return Language(("", code, 100, 0))
class Detector(object): class Detector:
""" Detect the language used in a snippet of text.""" """ Detect the language used in a snippet of text."""
def __init__(self, text, quiet=False): def __init__(self, text, quiet=False):
@ -57,8 +58,7 @@ class Detector(object):
self.reliable = False self.reliable = False
reliable, index, top_3_choices = cld2.detect(text, bestEffort=True) reliable, index, top_3_choices = cld2.detect(text, bestEffort=True)
if not self.quiet: if not self.quiet and not reliable:
if not reliable:
raise UnknownLanguage("Try passing a longer snippet of text") raise UnknownLanguage("Try passing a longer snippet of text")
self.languages = [Language(x) for x in top_3_choices] self.languages = [Language(x) for x in top_3_choices]
@ -66,7 +66,7 @@ class Detector(object):
return self.language return self.language
def __str__(self): def __str__(self):
text = "Prediction is reliable: {}\n".format(self.reliable) text = f"Prediction is reliable: {self.reliable}\n"
text += u"\n".join(["Language {}: {}".format(i+1, str(l)) text += "\n".join([f"Language {i+1}: {str(l)}"
for i,l in enumerate(self.languages)]) for i,l in enumerate(self.languages)])
return text return text

View file

@ -1,4 +1,3 @@
from pathlib import Path
from argostranslate import package, translate from argostranslate import package, translate

View file

@ -1,6 +1,6 @@
import string
from argostranslate import translate from argostranslate import translate
from libretranslate.detect import Detector, UnknownLanguage from libretranslate.detect import Detector, UnknownLanguage
__languages = None __languages = None

View file

@ -1,10 +1,11 @@
import os
import json import json
import os
from functools import lru_cache from functools import lru_cache
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_babel import lazy_gettext as _lazy from flask_babel import lazy_gettext as _lazy
from markupsafe import Markup, escape
from markupsafe import escape, Markup
@lru_cache(maxsize=None) @lru_cache(maxsize=None)
def get_available_locales(only_reviewed=True, sort_by_name=False): def get_available_locales(only_reviewed=True, sort_by_name=False):

View file

@ -197,7 +197,7 @@ def main():
from waitress import serve from waitress import serve
url_scheme = "https" if args.ssl else "http" url_scheme = "https" if args.ssl else "http"
print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix)) print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}")
serve( serve(
app, app,

View file

@ -49,7 +49,7 @@ def manage():
print("There are no API keys") print("There are no API keys")
else: else:
for item in keys: for item in keys:
print("%s: %s" % item) print("{}: {}".format(*item))
elif args.sub_command == "add": elif args.sub_command == "add":
print(db.add(args.req_limit, args.key)[0]) print(db.add(args.req_limit, args.key)[0])

View file

@ -1,5 +1,7 @@
import atexit import atexit
from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.schedulers.background import BackgroundScheduler
scheduler = None scheduler = None
def setup(args): def setup(args):

View file

@ -1,9 +1,9 @@
import atexit
import random import random
import string import string
from libretranslate.storage import get_storage from libretranslate.storage import get_storage
def generate_secret(): def generate_secret():
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7)) return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7))

View file

@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path):
unsafe_path = os.path.abspath(unsafe_path) unsafe_path = os.path.abspath(unsafe_path)
if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path): if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path):
raise SuspiciousFileOperation("{} is not safe".format(unsafe_path)) raise SuspiciousFileOperation(f"{unsafe_path} is not safe")
# Passes the check # Passes the check
return unsafe_path return unsafe_path

View file

@ -1,5 +1,5 @@
import sqlite3
import os import os
import sqlite3
from expiringdict import ExpiringDict from expiringdict import ExpiringDict
@ -10,7 +10,7 @@ class Database:
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
# Legacy check - this can be removed at some point in the near future # Legacy check - this can be removed at some point in the near future
if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"): if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"):
print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db")) print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db"))
try: try:
os.rename("suggestions.db", "db/suggestions.db") os.rename("suggestions.db", "db/suggestions.db")
except Exception as e: except Exception as e:

View file

@ -1,4 +1,5 @@
import sys import sys
import pytest import pytest
from libretranslate.app import create_app from libretranslate.app import create_app

View file

@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client):
response_json = json.loads(response.data) response_json = json.loads(response.data)
assert "error" in response_json assert "error" in response_json
assert "zz is not supported" == response_json["error"] assert response_json["error"] == "zz is not supported"
assert response.status_code == 400 assert response.status_code == 400
@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client):
response_json = json.loads(response.data) response_json = json.loads(response.data)
assert "error" in response_json assert "error" in response_json
assert "Invalid request: missing q parameter" == response_json["error"] assert response_json["error"] == "Invalid request: missing q parameter"
assert response.status_code == 400 assert response.status_code == 400

View file

@ -1,6 +1,7 @@
from libretranslate.init import boot
from argostranslate import package from argostranslate import package
from libretranslate.init import boot
def test_boot_argos(): def test_boot_argos():
"""Test Argos translate models initialization""" """Test Argos translate models initialization"""

View file

@ -138,18 +138,18 @@ select = [
"B", # flake8-bugbear "B", # flake8-bugbear
# "C", # flake8-comprehensions # "C", # flake8-comprehensions
"ICN", # flake8-import-conventions "ICN", # flake8-import-conventions
"SIM", # flake8-simplify # "SIM", # flake8-simplify
"TID", # flake8-tidy-imports "TID", # flake8-tidy-imports
# "Q", # flake8-quotes # "Q", # flake8-quotes
"FBT", # flake8-boolean-trap # "FBT", # flake8-boolean-trap
"F", # pyflakes "F", # pyflakes
"UP", # pyupgrade "UP", # pyupgrade
# "E", # pycodestyle errors # "E", # pycodestyle errors
# "W", # pycodestyle warnings # "W", # pycodestyle warnings
"PLC", # pylint convention # "PLC", # pylint convention
"PLE", # pylint error "PLE", # pylint error
# "PLR", # pylint refactor # "PLR", # pylint refactor
"PLW", # pylint warning # "PLW", # pylint warning
"RUF", # ruff specific "RUF", # ruff specific
"T", "T",
] ]
@ -158,8 +158,9 @@ ignore = [
# "E741", # From original flake8 ignore # "E741", # From original flake8 ignore
# "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik)
"E501", # line too long "E501", # line too long
# "C901", # too complex "A003", # Class attribute is shadowing a python builtin
"S101", # Use of `assert` detected "S101", # Use of `assert` detected
"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
"T201", "T203", # remove print and pprint "T201", "T203", # remove print and pprint
"E402", # Module level import not at top of file "E402", # Module level import not at top of file
] ]

View file

@ -1,6 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys
import os import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from babel.messages.frontend import main as pybabel from babel.messages.frontend import main as pybabel
@ -15,7 +16,7 @@ if __name__ == "__main__":
link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code'] link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code']
if l['code'] == 'en': if l['code'] == 'en':
link = "https://hosted.weblate.org/projects/libretranslate/app/" link = "https://hosted.weblate.org/projects/libretranslate/app/"
print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
else: else:
locales_dir = os.path.join("libretranslate", "locales") locales_dir = os.path.join("libretranslate", "locales")
if not os.path.isdir(locales_dir): if not os.path.isdir(locales_dir):

View file

@ -1,7 +1,9 @@
from prometheus_client import multiprocess
import re import re
import sys import sys
from prometheus_client import multiprocess
def child_exit(server, worker): def child_exit(server, worker):
multiprocess.mark_process_dead(worker.pid) multiprocess.mark_process_dead(worker.pid)
@ -35,7 +37,7 @@ def on_starting(server):
args = get_args() args = get_args()
from libretranslate import storage, scheduler, flood, secret from libretranslate import flood, scheduler, secret, storage
storage.setup(args.shared_storage) storage.setup(args.shared_storage)
scheduler.setup(args) scheduler.setup(args)
flood.setup(args) flood.setup(args)

View file

@ -1,4 +1,5 @@
import requests import requests
response = requests.post( response = requests.post(
url='http://0.0.0.0:5000/translate', url='http://0.0.0.0:5000/translate',
headers={'Content-Type': 'application/json'}, headers={'Content-Type': 'application/json'},

View file

@ -1,8 +1,10 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys
import os import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import argparse import argparse
from libretranslate.init import check_and_install_models from libretranslate.init import check_and_install_models
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -1,8 +1,8 @@
#!/usr/bin/env python #!/usr/bin/env python
import argparse import argparse
import time
import sqlite3
import json import json
import sqlite3
import time
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db") parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")

View file

@ -1,17 +1,19 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys
import os import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import re
import polib
import json import json
import re
import polib
from babel.messages.frontend import main as pybabel from babel.messages.frontend import main as pybabel
from libretranslate.language import load_languages, improve_translation_formatting
from libretranslate.locales import get_available_locale_codes, swag_eval
from translatehtml import translate_html
from libretranslate.app import get_version, create_app
from libretranslate.main import get_args
from flask_swagger import swagger from flask_swagger import swagger
from libretranslate.app import create_app, get_version
from libretranslate.language import improve_translation_formatting, load_languages
from libretranslate.locales import get_available_locale_codes, swag_eval
from libretranslate.main import get_args
from translatehtml import translate_html
# Update strings # Update strings
if __name__ == "__main__": if __name__ == "__main__":
@ -74,7 +76,7 @@ if __name__ == "__main__":
if not os.path.isfile(meta_file): if not os.path.isfile(meta_file):
with open(meta_file, 'w') as f: with open(meta_file, 'w') as f:
f.write(json.dumps({ f.write(json.dumps({
'name': next((lang.name for lang in languages if lang.code == l)), 'name': next(lang.name for lang in languages if lang.code == l),
'reviewed': False 'reviewed': False
}, indent=4)) }, indent=4))
print("Wrote %s" % meta_file) print("Wrote %s" % meta_file)