diff --git a/libretranslate/api_keys.py b/libretranslate/api_keys.py index dc1da3e..0583531 100644 --- a/libretranslate/api_keys.py +++ b/libretranslate/api_keys.py @@ -1,8 +1,10 @@ import os import sqlite3 import uuid + import requests from expiringdict import ExpiringDict + from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH'] @@ -12,14 +14,14 @@ class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): # Legacy check - this can be removed at some point in the near future if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"): - print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db")) + print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db")) try: os.rename("api_keys.db", "db/api_keys.db") except Exception as e: print(str(e)) db_dir = os.path.dirname(db_path) - if not db_dir == "" and not os.path.exists(db_dir): + if db_dir != '' and not os.path.exists(db_dir): os.makedirs(db_dir) self.db_path = db_path self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) @@ -91,10 +93,7 @@ class RemoteDatabase: print("Cannot authenticate API key: " + str(e)) return None - if res.get('error', None) is None: - req_limit = res.get('req_limit', None) - else: - req_limit = None + req_limit = res.get('req_limit', None) if res.get('error', None) is None else None self.cache[api_key] = req_limit return req_limit diff --git a/libretranslate/app.py b/libretranslate/app.py index c5fe485..77197f2 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -1,30 +1,37 @@ import io import os -import tempfile import re +import tempfile import uuid +from datetime import datetime from functools import wraps from html import unescape from timeit import default_timer -from datetime import datetime import argostranslatefiles from argostranslatefiles import get_supported_formats -from flask import (abort, Blueprint, Flask, jsonify, render_template, request, - Response, send_file, url_for, session) +from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for +from flask_babel import Babel +from flask_session import Session from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint -from flask_session import Session from translatehtml import translate_html -from werkzeug.utils import secure_filename from werkzeug.exceptions import HTTPException from werkzeug.http import http_date -from flask_babel import Babel +from werkzeug.utils import secure_filename -from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage +from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage from libretranslate.language import detect_languages, improve_translation_formatting -from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped, - gettext_html, lazy_swag, get_alternate_locale_links) +from libretranslate.locales import ( + _, + _lazy, + get_alternate_locale_links, + get_available_locale_codes, + get_available_locales, + gettext_escaped, + gettext_html, + lazy_swag, +) from .api_keys import Database, RemoteDatabase from .suggestions import Database as SuggestionsDatabase @@ -150,10 +157,7 @@ def create_app(args): frontend_argos_language_source = languages[0] - if len(languages) >= 2: - language_target_fallback = languages[1] - else: - language_target_fallback = languages[0] + language_target_fallback = languages[1] if len(languages) >= 2 else languages[0] if args.frontend_language_target == "locale": def resolve_language_locale(): @@ -185,10 +189,7 @@ def create_app(args): if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: api_keys_db = None if args.api_keys: - if args.api_keys_remote: - api_keys_db = RemoteDatabase(args.api_keys_remote) - else: - api_keys_db = Database(args.api_keys_db_path) + api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path) from flask_limiter import Limiter @@ -220,7 +221,7 @@ def create_app(args): os.mkdir(default_mp_dir) os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir - from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest + from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess @bp.route("/metrics") @limiter.exempt @@ -544,10 +545,7 @@ def create_app(args): ) if args.char_limit != -1: - if batch: - chars = sum([len(text) for text in q]) - else: - chars = len(q) + chars = sum([len(text) for text in q]) if batch else len(q) if args.char_limit < chars: abort( @@ -557,10 +555,7 @@ def create_app(args): if source_lang == "auto": source_langs = [] - if batch: - auto_detect_texts = q - else: - auto_detect_texts = [q] + auto_detect_texts = q if batch else [q] overall_candidates = detect_languages(q) @@ -1093,7 +1088,7 @@ def create_app(args): return override_lang return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes())) - babel = Babel(app, locale_selector=get_locale) + Babel(app, locale_selector=get_locale) app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html) diff --git a/libretranslate/detect.py b/libretranslate/detect.py index 1b43087..8717658 100644 --- a/libretranslate/detect.py +++ b/libretranslate/detect.py @@ -2,10 +2,11 @@ import pycld2 as cld2 + class UnknownLanguage(Exception): pass -class Language(object): +class Language: def __init__(self, choice): name, code, confidence, bytesize = choice self.code = code @@ -23,7 +24,7 @@ class Language(object): return Language(("", code, 100, 0)) -class Detector(object): +class Detector: """ Detect the language used in a snippet of text.""" def __init__(self, text, quiet=False): @@ -57,16 +58,15 @@ class Detector(object): self.reliable = False reliable, index, top_3_choices = cld2.detect(text, bestEffort=True) - if not self.quiet: - if not reliable: - raise UnknownLanguage("Try passing a longer snippet of text") + if not self.quiet and not reliable: + raise UnknownLanguage("Try passing a longer snippet of text") self.languages = [Language(x) for x in top_3_choices] self.language = self.languages[0] return self.language def __str__(self): - text = "Prediction is reliable: {}\n".format(self.reliable) - text += u"\n".join(["Language {}: {}".format(i+1, str(l)) + text = f"Prediction is reliable: {self.reliable}\n" + text += "\n".join([f"Language {i+1}: {str(l)}" for i,l in enumerate(self.languages)]) return text \ No newline at end of file diff --git a/libretranslate/init.py b/libretranslate/init.py index b6238fa..04aa866 100644 --- a/libretranslate/init.py +++ b/libretranslate/init.py @@ -1,4 +1,3 @@ -from pathlib import Path from argostranslate import package, translate diff --git a/libretranslate/language.py b/libretranslate/language.py index 1386bad..68e3e48 100644 --- a/libretranslate/language.py +++ b/libretranslate/language.py @@ -1,6 +1,6 @@ -import string from argostranslate import translate + from libretranslate.detect import Detector, UnknownLanguage __languages = None diff --git a/libretranslate/locales.py b/libretranslate/locales.py index 896e6fa..9c08d4c 100644 --- a/libretranslate/locales.py +++ b/libretranslate/locales.py @@ -1,10 +1,11 @@ -import os import json +import os from functools import lru_cache + from flask_babel import gettext as _ from flask_babel import lazy_gettext as _lazy +from markupsafe import Markup, escape -from markupsafe import escape, Markup @lru_cache(maxsize=None) def get_available_locales(only_reviewed=True, sort_by_name=False): diff --git a/libretranslate/main.py b/libretranslate/main.py index 0e02145..57dd723 100644 --- a/libretranslate/main.py +++ b/libretranslate/main.py @@ -197,7 +197,7 @@ def main(): from waitress import serve url_scheme = "https" if args.ssl else "http" - print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix)) + print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}") serve( app, diff --git a/libretranslate/manage.py b/libretranslate/manage.py index 68beb61..d296388 100644 --- a/libretranslate/manage.py +++ b/libretranslate/manage.py @@ -49,7 +49,7 @@ def manage(): print("There are no API keys") else: for item in keys: - print("%s: %s" % item) + print("{}: {}".format(*item)) elif args.sub_command == "add": print(db.add(args.req_limit, args.key)[0]) diff --git a/libretranslate/scheduler.py b/libretranslate/scheduler.py index 3300095..de54884 100644 --- a/libretranslate/scheduler.py +++ b/libretranslate/scheduler.py @@ -1,5 +1,7 @@ import atexit + from apscheduler.schedulers.background import BackgroundScheduler + scheduler = None def setup(args): diff --git a/libretranslate/secret.py b/libretranslate/secret.py index a50fcc7..76cf606 100644 --- a/libretranslate/secret.py +++ b/libretranslate/secret.py @@ -1,9 +1,9 @@ -import atexit import random import string from libretranslate.storage import get_storage + def generate_secret(): return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7)) diff --git a/libretranslate/security.py b/libretranslate/security.py index 99415da..3c4023a 100644 --- a/libretranslate/security.py +++ b/libretranslate/security.py @@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path): unsafe_path = os.path.abspath(unsafe_path) if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path): - raise SuspiciousFileOperation("{} is not safe".format(unsafe_path)) + raise SuspiciousFileOperation(f"{unsafe_path} is not safe") # Passes the check return unsafe_path \ No newline at end of file diff --git a/libretranslate/suggestions.py b/libretranslate/suggestions.py index 911b811..73944dd 100644 --- a/libretranslate/suggestions.py +++ b/libretranslate/suggestions.py @@ -1,5 +1,5 @@ -import sqlite3 import os +import sqlite3 from expiringdict import ExpiringDict @@ -10,7 +10,7 @@ class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): # Legacy check - this can be removed at some point in the near future if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"): - print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db")) + print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db")) try: os.rename("suggestions.db", "db/suggestions.db") except Exception as e: diff --git a/libretranslate/tests/test_api/conftest.py b/libretranslate/tests/test_api/conftest.py index 3eb4f79..a1db034 100644 --- a/libretranslate/tests/test_api/conftest.py +++ b/libretranslate/tests/test_api/conftest.py @@ -1,4 +1,5 @@ import sys + import pytest from libretranslate.app import create_app diff --git a/libretranslate/tests/test_api/test_api_translate.py b/libretranslate/tests/test_api/test_api_translate.py index 96f8632..ae08052 100644 --- a/libretranslate/tests/test_api/test_api_translate.py +++ b/libretranslate/tests/test_api/test_api_translate.py @@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client): response_json = json.loads(response.data) assert "error" in response_json - assert "zz is not supported" == response_json["error"] + assert response_json["error"] == "zz is not supported" assert response.status_code == 400 @@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client): response_json = json.loads(response.data) assert "error" in response_json - assert "Invalid request: missing q parameter" == response_json["error"] + assert response_json["error"] == "Invalid request: missing q parameter" assert response.status_code == 400 diff --git a/libretranslate/tests/test_init.py b/libretranslate/tests/test_init.py index 198f223..cd796df 100644 --- a/libretranslate/tests/test_init.py +++ b/libretranslate/tests/test_init.py @@ -1,6 +1,7 @@ -from libretranslate.init import boot from argostranslate import package +from libretranslate.init import boot + def test_boot_argos(): """Test Argos translate models initialization""" diff --git a/pyproject.toml b/pyproject.toml index 7eeab4b..e9eaed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,18 +138,18 @@ select = [ "B", # flake8-bugbear # "C", # flake8-comprehensions "ICN", # flake8-import-conventions - "SIM", # flake8-simplify + # "SIM", # flake8-simplify "TID", # flake8-tidy-imports # "Q", # flake8-quotes - "FBT", # flake8-boolean-trap + # "FBT", # flake8-boolean-trap "F", # pyflakes "UP", # pyupgrade # "E", # pycodestyle errors # "W", # pycodestyle warnings - "PLC", # pylint convention + # "PLC", # pylint convention "PLE", # pylint error # "PLR", # pylint refactor - "PLW", # pylint warning + # "PLW", # pylint warning "RUF", # ruff specific "T", ] @@ -158,8 +158,9 @@ ignore = [ # "E741", # From original flake8 ignore # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) "E501", # line too long - # "C901", # too complex + "A003", # Class attribute is shadowing a python builtin "S101", # Use of `assert` detected + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes "T201", "T203", # remove print and pprint "E402", # Module level import not at top of file ] diff --git a/scripts/compile_locales.py b/scripts/compile_locales.py index 5aff9af..2920354 100755 --- a/scripts/compile_locales.py +++ b/scripts/compile_locales.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from babel.messages.frontend import main as pybabel @@ -15,7 +16,7 @@ if __name__ == "__main__": link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code'] if l['code'] == 'en': link = "https://hosted.weblate.org/projects/libretranslate/app/" - print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) + print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) else: locales_dir = os.path.join("libretranslate", "locales") if not os.path.isdir(locales_dir): diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index 845da5b..8c4f709 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,7 +1,9 @@ -from prometheus_client import multiprocess import re import sys +from prometheus_client import multiprocess + + def child_exit(server, worker): multiprocess.mark_process_dead(worker.pid) @@ -35,7 +37,7 @@ def on_starting(server): args = get_args() - from libretranslate import storage, scheduler, flood, secret + from libretranslate import flood, scheduler, secret, storage storage.setup(args.shared_storage) scheduler.setup(args) flood.setup(args) diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py index cd62b4f..20eba19 100644 --- a/scripts/healthcheck.py +++ b/scripts/healthcheck.py @@ -1,4 +1,5 @@ import requests + response = requests.post( url='http://0.0.0.0:5000/translate', headers={'Content-Type': 'application/json'}, diff --git a/scripts/install_models.py b/scripts/install_models.py index d844afb..2d3d313 100755 --- a/scripts/install_models.py +++ b/scripts/install_models.py @@ -1,8 +1,10 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import argparse + from libretranslate.init import check_and_install_models if __name__ == "__main__": diff --git a/scripts/suggestions-to-jsonl.py b/scripts/suggestions-to-jsonl.py index 6cd8e62..7492ac0 100755 --- a/scripts/suggestions-to-jsonl.py +++ b/scripts/suggestions-to-jsonl.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import argparse -import time -import sqlite3 import json +import sqlite3 +import time if __name__ == "__main__": parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db") diff --git a/scripts/update_locales.py b/scripts/update_locales.py index 1fd2853..93c35fa 100755 --- a/scripts/update_locales.py +++ b/scripts/update_locales.py @@ -1,17 +1,19 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import re -import polib import json +import re + +import polib from babel.messages.frontend import main as pybabel -from libretranslate.language import load_languages, improve_translation_formatting -from libretranslate.locales import get_available_locale_codes, swag_eval -from translatehtml import translate_html -from libretranslate.app import get_version, create_app -from libretranslate.main import get_args from flask_swagger import swagger +from libretranslate.app import create_app, get_version +from libretranslate.language import improve_translation_formatting, load_languages +from libretranslate.locales import get_available_locale_codes, swag_eval +from libretranslate.main import get_args +from translatehtml import translate_html # Update strings if __name__ == "__main__": @@ -74,7 +76,7 @@ if __name__ == "__main__": if not os.path.isfile(meta_file): with open(meta_file, 'w') as f: f.write(json.dumps({ - 'name': next((lang.name for lang in languages if lang.code == l)), + 'name': next(lang.name for lang in languages if lang.code == l), 'reviewed': False }, indent=4)) print("Wrote %s" % meta_file)