run ruff formatting

This commit is contained in:
Vincent Emonet 2023-07-09 12:29:11 +02:00
parent 62712dcad3
commit bf18dcbcf9
22 changed files with 82 additions and 75 deletions

View file

@ -1,8 +1,10 @@
import os
import sqlite3
import uuid
import requests
from expiringdict import ExpiringDict
from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS
DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH']
@ -12,14 +14,14 @@ class Database:
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
# Legacy check - this can be removed at some point in the near future
if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"):
print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db"))
print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db"))
try:
os.rename("api_keys.db", "db/api_keys.db")
except Exception as e:
print(str(e))
db_dir = os.path.dirname(db_path)
if not db_dir == "" and not os.path.exists(db_dir):
if db_dir != '' and not os.path.exists(db_dir):
os.makedirs(db_dir)
self.db_path = db_path
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
@ -91,10 +93,7 @@ class RemoteDatabase:
print("Cannot authenticate API key: " + str(e))
return None
if res.get('error', None) is None:
req_limit = res.get('req_limit', None)
else:
req_limit = None
req_limit = res.get('req_limit', None) if res.get('error', None) is None else None
self.cache[api_key] = req_limit
return req_limit

View file

@ -1,30 +1,37 @@
import io
import os
import tempfile
import re
import tempfile
import uuid
from datetime import datetime
from functools import wraps
from html import unescape
from timeit import default_timer
from datetime import datetime
import argostranslatefiles
from argostranslatefiles import get_supported_formats
from flask import (abort, Blueprint, Flask, jsonify, render_template, request,
Response, send_file, url_for, session)
from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for
from flask_babel import Babel
from flask_session import Session
from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint
from flask_session import Session
from translatehtml import translate_html
from werkzeug.utils import secure_filename
from werkzeug.exceptions import HTTPException
from werkzeug.http import http_date
from flask_babel import Babel
from werkzeug.utils import secure_filename
from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage
from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage
from libretranslate.language import detect_languages, improve_translation_formatting
from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped,
gettext_html, lazy_swag, get_alternate_locale_links)
from libretranslate.locales import (
_,
_lazy,
get_alternate_locale_links,
get_available_locale_codes,
get_available_locales,
gettext_escaped,
gettext_html,
lazy_swag,
)
from .api_keys import Database, RemoteDatabase
from .suggestions import Database as SuggestionsDatabase
@ -150,10 +157,7 @@ def create_app(args):
frontend_argos_language_source = languages[0]
if len(languages) >= 2:
language_target_fallback = languages[1]
else:
language_target_fallback = languages[0]
language_target_fallback = languages[1] if len(languages) >= 2 else languages[0]
if args.frontend_language_target == "locale":
def resolve_language_locale():
@ -185,10 +189,7 @@ def create_app(args):
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
api_keys_db = None
if args.api_keys:
if args.api_keys_remote:
api_keys_db = RemoteDatabase(args.api_keys_remote)
else:
api_keys_db = Database(args.api_keys_db_path)
api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path)
from flask_limiter import Limiter
@ -220,7 +221,7 @@ def create_app(args):
os.mkdir(default_mp_dir)
os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir
from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest
from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess
@bp.route("/metrics")
@limiter.exempt
@ -544,10 +545,7 @@ def create_app(args):
)
if args.char_limit != -1:
if batch:
chars = sum([len(text) for text in q])
else:
chars = len(q)
chars = sum([len(text) for text in q]) if batch else len(q)
if args.char_limit < chars:
abort(
@ -557,10 +555,7 @@ def create_app(args):
if source_lang == "auto":
source_langs = []
if batch:
auto_detect_texts = q
else:
auto_detect_texts = [q]
auto_detect_texts = q if batch else [q]
overall_candidates = detect_languages(q)
@ -1093,7 +1088,7 @@ def create_app(args):
return override_lang
return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes()))
babel = Babel(app, locale_selector=get_locale)
Babel(app, locale_selector=get_locale)
app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html)

View file

@ -2,10 +2,11 @@
import pycld2 as cld2
class UnknownLanguage(Exception):
pass
class Language(object):
class Language:
def __init__(self, choice):
name, code, confidence, bytesize = choice
self.code = code
@ -23,7 +24,7 @@ class Language(object):
return Language(("", code, 100, 0))
class Detector(object):
class Detector:
""" Detect the language used in a snippet of text."""
def __init__(self, text, quiet=False):
@ -57,8 +58,7 @@ class Detector(object):
self.reliable = False
reliable, index, top_3_choices = cld2.detect(text, bestEffort=True)
if not self.quiet:
if not reliable:
if not self.quiet and not reliable:
raise UnknownLanguage("Try passing a longer snippet of text")
self.languages = [Language(x) for x in top_3_choices]
@ -66,7 +66,7 @@ class Detector(object):
return self.language
def __str__(self):
text = "Prediction is reliable: {}\n".format(self.reliable)
text += u"\n".join(["Language {}: {}".format(i+1, str(l))
text = f"Prediction is reliable: {self.reliable}\n"
text += "\n".join([f"Language {i+1}: {str(l)}"
for i,l in enumerate(self.languages)])
return text

View file

@ -1,4 +1,3 @@
from pathlib import Path
from argostranslate import package, translate

View file

@ -1,6 +1,6 @@
import string
from argostranslate import translate
from libretranslate.detect import Detector, UnknownLanguage
__languages = None

View file

@ -1,10 +1,11 @@
import os
import json
import os
from functools import lru_cache
from flask_babel import gettext as _
from flask_babel import lazy_gettext as _lazy
from markupsafe import Markup, escape
from markupsafe import escape, Markup
@lru_cache(maxsize=None)
def get_available_locales(only_reviewed=True, sort_by_name=False):

View file

@ -197,7 +197,7 @@ def main():
from waitress import serve
url_scheme = "https" if args.ssl else "http"
print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix))
print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}")
serve(
app,

View file

@ -49,7 +49,7 @@ def manage():
print("There are no API keys")
else:
for item in keys:
print("%s: %s" % item)
print("{}: {}".format(*item))
elif args.sub_command == "add":
print(db.add(args.req_limit, args.key)[0])

View file

@ -1,5 +1,7 @@
import atexit
from apscheduler.schedulers.background import BackgroundScheduler
scheduler = None
def setup(args):

View file

@ -1,9 +1,9 @@
import atexit
import random
import string
from libretranslate.storage import get_storage
def generate_secret():
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7))

View file

@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path):
unsafe_path = os.path.abspath(unsafe_path)
if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path):
raise SuspiciousFileOperation("{} is not safe".format(unsafe_path))
raise SuspiciousFileOperation(f"{unsafe_path} is not safe")
# Passes the check
return unsafe_path

View file

@ -1,5 +1,5 @@
import sqlite3
import os
import sqlite3
from expiringdict import ExpiringDict
@ -10,7 +10,7 @@ class Database:
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
# Legacy check - this can be removed at some point in the near future
if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"):
print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db"))
print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db"))
try:
os.rename("suggestions.db", "db/suggestions.db")
except Exception as e:

View file

@ -1,4 +1,5 @@
import sys
import pytest
from libretranslate.app import create_app

View file

@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client):
response_json = json.loads(response.data)
assert "error" in response_json
assert "zz is not supported" == response_json["error"]
assert response_json["error"] == "zz is not supported"
assert response.status_code == 400
@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client):
response_json = json.loads(response.data)
assert "error" in response_json
assert "Invalid request: missing q parameter" == response_json["error"]
assert response_json["error"] == "Invalid request: missing q parameter"
assert response.status_code == 400

View file

@ -1,6 +1,7 @@
from libretranslate.init import boot
from argostranslate import package
from libretranslate.init import boot
def test_boot_argos():
"""Test Argos translate models initialization"""

View file

@ -138,18 +138,18 @@ select = [
"B", # flake8-bugbear
# "C", # flake8-comprehensions
"ICN", # flake8-import-conventions
"SIM", # flake8-simplify
# "SIM", # flake8-simplify
"TID", # flake8-tidy-imports
# "Q", # flake8-quotes
"FBT", # flake8-boolean-trap
# "FBT", # flake8-boolean-trap
"F", # pyflakes
"UP", # pyupgrade
# "E", # pycodestyle errors
# "W", # pycodestyle warnings
"PLC", # pylint convention
# "PLC", # pylint convention
"PLE", # pylint error
# "PLR", # pylint refactor
"PLW", # pylint warning
# "PLW", # pylint warning
"RUF", # ruff specific
"T",
]
@ -158,8 +158,9 @@ ignore = [
# "E741", # From original flake8 ignore
# "B008", # do not perform function calls in argument defaults (required for FastAPI afaik)
"E501", # line too long
# "C901", # too complex
"A003", # Class attribute is shadowing a python builtin
"S101", # Use of `assert` detected
"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
"T201", "T203", # remove print and pprint
"E402", # Module level import not at top of file
]

View file

@ -1,6 +1,7 @@
#!/usr/bin/env python
import sys
import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from babel.messages.frontend import main as pybabel
@ -15,7 +16,7 @@ if __name__ == "__main__":
link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code']
if l['code'] == 'en':
link = "https://hosted.weblate.org/projects/libretranslate/app/"
print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
else:
locales_dir = os.path.join("libretranslate", "locales")
if not os.path.isdir(locales_dir):

View file

@ -1,7 +1,9 @@
from prometheus_client import multiprocess
import re
import sys
from prometheus_client import multiprocess
def child_exit(server, worker):
multiprocess.mark_process_dead(worker.pid)
@ -35,7 +37,7 @@ def on_starting(server):
args = get_args()
from libretranslate import storage, scheduler, flood, secret
from libretranslate import flood, scheduler, secret, storage
storage.setup(args.shared_storage)
scheduler.setup(args)
flood.setup(args)

View file

@ -1,4 +1,5 @@
import requests
response = requests.post(
url='http://0.0.0.0:5000/translate',
headers={'Content-Type': 'application/json'},

View file

@ -1,8 +1,10 @@
#!/usr/bin/env python
import sys
import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import argparse
from libretranslate.init import check_and_install_models
if __name__ == "__main__":

View file

@ -1,8 +1,8 @@
#!/usr/bin/env python
import argparse
import time
import sqlite3
import json
import sqlite3
import time
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")

View file

@ -1,17 +1,19 @@
#!/usr/bin/env python
import sys
import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import re
import polib
import json
import re
import polib
from babel.messages.frontend import main as pybabel
from libretranslate.language import load_languages, improve_translation_formatting
from libretranslate.locales import get_available_locale_codes, swag_eval
from translatehtml import translate_html
from libretranslate.app import get_version, create_app
from libretranslate.main import get_args
from flask_swagger import swagger
from libretranslate.app import create_app, get_version
from libretranslate.language import improve_translation_formatting, load_languages
from libretranslate.locales import get_available_locale_codes, swag_eval
from libretranslate.main import get_args
from translatehtml import translate_html
# Update strings
if __name__ == "__main__":
@ -74,7 +76,7 @@ if __name__ == "__main__":
if not os.path.isfile(meta_file):
with open(meta_file, 'w') as f:
f.write(json.dumps({
'name': next((lang.name for lang in languages if lang.code == l)),
'name': next(lang.name for lang in languages if lang.code == l),
'reviewed': False
}, indent=4))
print("Wrote %s" % meta_file)