mirror of
https://github.com/LibreTranslate/LibreTranslate.git
synced 2024-11-25 01:11:00 +00:00
Merge pull request #462 from vemonet/use-standard-build-system
Update the project configuration to use modern PEP standards, and replace `flake8` by `ruff` for linting
This commit is contained in:
commit
e93f22276b
29 changed files with 328 additions and 222 deletions
2
.github/workflows/publish-docker.yml
vendored
2
.github/workflows/publish-docker.yml
vendored
|
@ -12,7 +12,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
|
|
53
.github/workflows/publish-package.yml
vendored
53
.github/workflows/publish-package.yml
vendored
|
@ -9,35 +9,8 @@ on:
|
|||
jobs:
|
||||
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ['3.8', '3.9', '3.10']
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pytest flake8
|
||||
pip install .
|
||||
python scripts/compile_locales.py
|
||||
|
||||
- name: Check code style with flake8 (lint)
|
||||
run: |
|
||||
# warnings if there are Python syntax errors or undefined names
|
||||
# (remove --exit-zero to fail when syntax error)
|
||||
flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
|
||||
- name: Test with pytest
|
||||
run: pytest
|
||||
uses: LibreTranslate/LibreTranslate/.github/workflows/run-tests.yml@main
|
||||
secrets: inherit
|
||||
|
||||
|
||||
publish:
|
||||
|
@ -45,23 +18,23 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install setuptools wheel twine
|
||||
python setup.py sdist bdist_wheel
|
||||
pip install build
|
||||
|
||||
- name: Build and publish to PyPI
|
||||
env:
|
||||
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
|
||||
- name: Build
|
||||
run: |
|
||||
pip install Babel==2.11.0
|
||||
python scripts/compile_locales.py
|
||||
python setup.py sdist bdist_wheel
|
||||
twine upload dist/*
|
||||
python -m build
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
user: ${{ secrets.PYPI_USERNAME }}
|
||||
password: ${{ secrets.PYPI_PASSWORD }}
|
||||
|
|
22
.github/workflows/run-tests.yml
vendored
22
.github/workflows/run-tests.yml
vendored
|
@ -2,6 +2,7 @@ name: Run tests
|
|||
# Run test at each push to main, if changes to package or tests files
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
push:
|
||||
|
@ -21,35 +22,26 @@ jobs:
|
|||
python-version: ['3.8', '3.9', '3.10']
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pytest flake8
|
||||
pip install .
|
||||
|
||||
- name: Check code style with flake8 (lint)
|
||||
run: |
|
||||
# warnings if there are Python syntax errors or undefined names
|
||||
# (remove --exit-zero to fail when syntax error)
|
||||
flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
pipx install hatch
|
||||
hatch run locales
|
||||
|
||||
- name: Test with pytest
|
||||
run: pytest -v
|
||||
run: hatch run test
|
||||
|
||||
|
||||
test_docker_build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
- name: Docker build
|
||||
run: docker build -f docker/Dockerfile -t libretranslate .
|
||||
|
||||
|
|
|
@ -20,17 +20,57 @@ sudo dnf install cmake
|
|||
|
||||
## Getting Started
|
||||
|
||||
Install [`hatch`](https://hatch.pypa.io) to manage the projects dependencies and run dev scripts:
|
||||
|
||||
```bash
|
||||
pipx install hatch
|
||||
```
|
||||
|
||||
Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/LibreTranslate/LibreTranslate.git
|
||||
cd LibreTranslate
|
||||
pip install -e .
|
||||
libretranslate [args]
|
||||
```
|
||||
|
||||
Run in development:
|
||||
|
||||
```bash
|
||||
hatch run dev
|
||||
```
|
||||
|
||||
Then open a web browser to <http://localhost:5000>
|
||||
|
||||
You can also start a new shell in a virtual environment with libretranslate installed:
|
||||
|
||||
```bash
|
||||
hatch shell
|
||||
libretranslate [args]
|
||||
# Or
|
||||
python main.py [args]
|
||||
```
|
||||
|
||||
Then open a web browser to <http://localhost:5000>
|
||||
> You can still use `pip install -e ".[test]"` directly if you don't want to use hatch.
|
||||
|
||||
## Run the tests
|
||||
|
||||
Run the test suite and linting checks:
|
||||
|
||||
```bash
|
||||
hatch run test
|
||||
```
|
||||
|
||||
To display all `print()` when debugging:
|
||||
|
||||
```bash
|
||||
hatch run test -s
|
||||
```
|
||||
|
||||
You can also run the tests on multiple python versions:
|
||||
|
||||
```bash
|
||||
hatch run all:test
|
||||
```
|
||||
|
||||
## Run with Docker
|
||||
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
import os
|
||||
import sqlite3
|
||||
import uuid
|
||||
|
||||
import requests
|
||||
from expiringdict import ExpiringDict
|
||||
|
||||
from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS
|
||||
|
||||
DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH']
|
||||
|
@ -12,14 +14,14 @@ class Database:
|
|||
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
|
||||
# Legacy check - this can be removed at some point in the near future
|
||||
if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"):
|
||||
print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db"))
|
||||
print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db"))
|
||||
try:
|
||||
os.rename("api_keys.db", "db/api_keys.db")
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
|
||||
db_dir = os.path.dirname(db_path)
|
||||
if not db_dir == "" and not os.path.exists(db_dir):
|
||||
if db_dir != '' and not os.path.exists(db_dir):
|
||||
os.makedirs(db_dir)
|
||||
self.db_path = db_path
|
||||
self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
|
||||
|
@ -85,16 +87,13 @@ class RemoteDatabase:
|
|||
req_limit = self.cache.get(api_key)
|
||||
if req_limit is None:
|
||||
try:
|
||||
r = requests.post(self.url, data={'api_key': api_key})
|
||||
r = requests.post(self.url, data={'api_key': api_key}, timeout=60)
|
||||
res = r.json()
|
||||
except Exception as e:
|
||||
print("Cannot authenticate API key: " + str(e))
|
||||
return None
|
||||
|
||||
if res.get('error', None) is None:
|
||||
req_limit = res.get('req_limit', None)
|
||||
else:
|
||||
req_limit = None
|
||||
req_limit = res.get('req_limit', None) if res.get('error', None) is None else None
|
||||
self.cache[api_key] = req_limit
|
||||
|
||||
return req_limit
|
||||
|
|
|
@ -1,30 +1,37 @@
|
|||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import re
|
||||
import tempfile
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from html import unescape
|
||||
from timeit import default_timer
|
||||
from datetime import datetime
|
||||
|
||||
import argostranslatefiles
|
||||
from argostranslatefiles import get_supported_formats
|
||||
from flask import (abort, Blueprint, Flask, jsonify, render_template, request,
|
||||
Response, send_file, url_for, session)
|
||||
from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for
|
||||
from flask_babel import Babel
|
||||
from flask_session import Session
|
||||
from flask_swagger import swagger
|
||||
from flask_swagger_ui import get_swaggerui_blueprint
|
||||
from flask_session import Session
|
||||
from translatehtml import translate_html
|
||||
from werkzeug.utils import secure_filename
|
||||
from werkzeug.exceptions import HTTPException
|
||||
from werkzeug.http import http_date
|
||||
from flask_babel import Babel
|
||||
from werkzeug.utils import secure_filename
|
||||
|
||||
from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage
|
||||
from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage
|
||||
from libretranslate.language import detect_languages, improve_translation_formatting
|
||||
from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped,
|
||||
gettext_html, lazy_swag, get_alternate_locale_links)
|
||||
from libretranslate.locales import (
|
||||
_,
|
||||
_lazy,
|
||||
get_alternate_locale_links,
|
||||
get_available_locale_codes,
|
||||
get_available_locales,
|
||||
gettext_escaped,
|
||||
gettext_html,
|
||||
lazy_swag,
|
||||
)
|
||||
|
||||
from .api_keys import Database, RemoteDatabase
|
||||
from .suggestions import Database as SuggestionsDatabase
|
||||
|
@ -122,8 +129,8 @@ def create_app(args):
|
|||
|
||||
from libretranslate.language import load_languages
|
||||
|
||||
SWAGGER_URL = args.url_prefix + "/docs" # Swagger UI (w/o trailing '/')
|
||||
API_URL = args.url_prefix + "/spec"
|
||||
swagger_url = args.url_prefix + "/docs" # Swagger UI (w/o trailing '/')
|
||||
api_url = args.url_prefix + "/spec"
|
||||
|
||||
bp = Blueprint('Main app', __name__)
|
||||
|
||||
|
@ -150,10 +157,7 @@ def create_app(args):
|
|||
frontend_argos_language_source = languages[0]
|
||||
|
||||
|
||||
if len(languages) >= 2:
|
||||
language_target_fallback = languages[1]
|
||||
else:
|
||||
language_target_fallback = languages[0]
|
||||
language_target_fallback = languages[1] if len(languages) >= 2 else languages[0]
|
||||
|
||||
if args.frontend_language_target == "locale":
|
||||
def resolve_language_locale():
|
||||
|
@ -185,10 +189,7 @@ def create_app(args):
|
|||
if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0:
|
||||
api_keys_db = None
|
||||
if args.api_keys:
|
||||
if args.api_keys_remote:
|
||||
api_keys_db = RemoteDatabase(args.api_keys_remote)
|
||||
else:
|
||||
api_keys_db = Database(args.api_keys_db_path)
|
||||
api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path)
|
||||
|
||||
from flask_limiter import Limiter
|
||||
|
||||
|
@ -220,7 +221,7 @@ def create_app(args):
|
|||
os.mkdir(default_mp_dir)
|
||||
os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir
|
||||
|
||||
from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest
|
||||
from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess
|
||||
|
||||
@bp.route("/metrics")
|
||||
@limiter.exempt
|
||||
|
@ -338,7 +339,7 @@ def create_app(args):
|
|||
get_api_key_link=args.get_api_key_link,
|
||||
web_version=os.environ.get("LT_WEB") is not None,
|
||||
version=get_version(),
|
||||
swagger_url=SWAGGER_URL,
|
||||
swagger_url=swagger_url,
|
||||
available_locales=[{'code': l['code'], 'name': _lazy(l['name'])} for l in get_available_locales(not args.debug)],
|
||||
current_locale=get_locale(),
|
||||
alternate_locales=get_alternate_locale_links()
|
||||
|
@ -544,10 +545,7 @@ def create_app(args):
|
|||
)
|
||||
|
||||
if args.char_limit != -1:
|
||||
if batch:
|
||||
chars = sum([len(text) for text in q])
|
||||
else:
|
||||
chars = len(q)
|
||||
chars = sum([len(text) for text in q]) if batch else len(q)
|
||||
|
||||
if args.char_limit < chars:
|
||||
abort(
|
||||
|
@ -557,10 +555,7 @@ def create_app(args):
|
|||
|
||||
if source_lang == "auto":
|
||||
source_langs = []
|
||||
if batch:
|
||||
auto_detect_texts = q
|
||||
else:
|
||||
auto_detect_texts = [q]
|
||||
auto_detect_texts = q if batch else [q]
|
||||
|
||||
overall_candidates = detect_languages(q)
|
||||
|
||||
|
@ -797,7 +792,7 @@ def create_app(args):
|
|||
checked_filepath = security.path_traversal_check(filepath, get_upload_dir())
|
||||
if os.path.isfile(checked_filepath):
|
||||
filepath = checked_filepath
|
||||
except security.SuspiciousFileOperation:
|
||||
except security.SuspiciousFileOperationError:
|
||||
abort(400, description=_("Invalid filename"))
|
||||
|
||||
return_data = io.BytesIO()
|
||||
|
@ -1080,7 +1075,7 @@ def create_app(args):
|
|||
swag["info"]["version"] = get_version()
|
||||
swag["info"]["title"] = "LibreTranslate"
|
||||
|
||||
@app.route(API_URL)
|
||||
@app.route(api_url)
|
||||
@limiter.exempt
|
||||
def spec():
|
||||
return jsonify(lazy_swag(swag))
|
||||
|
@ -1093,14 +1088,14 @@ def create_app(args):
|
|||
return override_lang
|
||||
return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes()))
|
||||
|
||||
babel = Babel(app, locale_selector=get_locale)
|
||||
Babel(app, locale_selector=get_locale)
|
||||
|
||||
app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html)
|
||||
|
||||
# Call factory function to create our blueprint
|
||||
swaggerui_blueprint = get_swaggerui_blueprint(SWAGGER_URL, API_URL)
|
||||
swaggerui_blueprint = get_swaggerui_blueprint(swagger_url, api_url)
|
||||
if args.url_prefix:
|
||||
app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)
|
||||
app.register_blueprint(swaggerui_blueprint, url_prefix=swagger_url)
|
||||
else:
|
||||
app.register_blueprint(swaggerui_blueprint)
|
||||
|
||||
|
|
|
@ -2,10 +2,11 @@
|
|||
|
||||
import pycld2 as cld2
|
||||
|
||||
class UnknownLanguage(Exception):
|
||||
|
||||
class UnknownLanguageError(Exception):
|
||||
pass
|
||||
|
||||
class Language(object):
|
||||
class Language:
|
||||
def __init__(self, choice):
|
||||
name, code, confidence, bytesize = choice
|
||||
self.code = code
|
||||
|
@ -23,7 +24,7 @@ class Language(object):
|
|||
return Language(("", code, 100, 0))
|
||||
|
||||
|
||||
class Detector(object):
|
||||
class Detector:
|
||||
""" Detect the language used in a snippet of text."""
|
||||
|
||||
def __init__(self, text, quiet=False):
|
||||
|
@ -57,16 +58,15 @@ class Detector(object):
|
|||
self.reliable = False
|
||||
reliable, index, top_3_choices = cld2.detect(text, bestEffort=True)
|
||||
|
||||
if not self.quiet:
|
||||
if not reliable:
|
||||
raise UnknownLanguage("Try passing a longer snippet of text")
|
||||
if not self.quiet and not reliable:
|
||||
raise UnknownLanguageError("Try passing a longer snippet of text")
|
||||
|
||||
self.languages = [Language(x) for x in top_3_choices]
|
||||
self.language = self.languages[0]
|
||||
return self.language
|
||||
|
||||
def __str__(self):
|
||||
text = "Prediction is reliable: {}\n".format(self.reliable)
|
||||
text += u"\n".join(["Language {}: {}".format(i+1, str(l))
|
||||
text = f"Prediction is reliable: {self.reliable}\n"
|
||||
text += "\n".join([f"Language {i+1}: {str(l)}"
|
||||
for i,l in enumerate(self.languages)])
|
||||
return text
|
|
@ -1,4 +1,3 @@
|
|||
from pathlib import Path
|
||||
|
||||
from argostranslate import package, translate
|
||||
|
||||
|
@ -46,14 +45,12 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
|
|||
# Download and install all available packages
|
||||
for available_package in available_packages:
|
||||
print(
|
||||
"Downloading %s (%s) ..."
|
||||
% (available_package, available_package.package_version)
|
||||
f"Downloading {available_package} ({available_package.package_version}) ..."
|
||||
)
|
||||
available_package.install()
|
||||
|
||||
# reload installed languages
|
||||
libretranslate.language.languages = translate.get_installed_languages()
|
||||
print(
|
||||
"Loaded support for %s languages (%s models total)!"
|
||||
% (len(translate.get_installed_languages()), len(available_packages))
|
||||
f"Loaded support for {len(translate.get_installed_languages())} languages ({len(available_packages)} models total)!"
|
||||
)
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import string
|
||||
|
||||
from argostranslate import translate
|
||||
from libretranslate.detect import Detector, UnknownLanguage
|
||||
|
||||
from libretranslate.detect import Detector, UnknownLanguageError
|
||||
|
||||
__languages = None
|
||||
|
||||
|
@ -29,7 +29,7 @@ def detect_languages(text):
|
|||
for i in range(len(d)):
|
||||
d[i].text_length = len(t)
|
||||
candidates.extend(d)
|
||||
except UnknownLanguage:
|
||||
except UnknownLanguageError:
|
||||
pass
|
||||
|
||||
# total read bytes of the provided text
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
import os
|
||||
import json
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
from flask_babel import gettext as _
|
||||
from flask_babel import lazy_gettext as _lazy
|
||||
from markupsafe import Markup, escape
|
||||
|
||||
from markupsafe import escape, Markup
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def get_available_locales(only_reviewed=True, sort_by_name=False):
|
||||
|
|
|
@ -197,7 +197,7 @@ def main():
|
|||
from waitress import serve
|
||||
|
||||
url_scheme = "https" if args.ssl else "http"
|
||||
print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix))
|
||||
print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}")
|
||||
|
||||
serve(
|
||||
app,
|
||||
|
|
|
@ -49,7 +49,7 @@ def manage():
|
|||
print("There are no API keys")
|
||||
else:
|
||||
for item in keys:
|
||||
print("%s: %s" % item)
|
||||
print("{}: {}".format(*item))
|
||||
|
||||
elif args.sub_command == "add":
|
||||
print(db.add(args.req_limit, args.key)[0])
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
import atexit
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
scheduler = None
|
||||
|
||||
def setup(args):
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
import atexit
|
||||
import random
|
||||
import string
|
||||
|
||||
from libretranslate.storage import get_storage
|
||||
|
||||
|
||||
def generate_secret():
|
||||
return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7))
|
||||
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import os
|
||||
|
||||
|
||||
class SuspiciousFileOperation(Exception):
|
||||
class SuspiciousFileOperationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
|
@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path):
|
|||
unsafe_path = os.path.abspath(unsafe_path)
|
||||
|
||||
if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path):
|
||||
raise SuspiciousFileOperation("{} is not safe".format(unsafe_path))
|
||||
raise SuspiciousFileOperationError(f"{unsafe_path} is not safe")
|
||||
|
||||
# Passes the check
|
||||
return unsafe_path
|
|
@ -1,5 +1,5 @@
|
|||
import sqlite3
|
||||
import os
|
||||
import sqlite3
|
||||
|
||||
from expiringdict import ExpiringDict
|
||||
|
||||
|
@ -10,7 +10,7 @@ class Database:
|
|||
def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
|
||||
# Legacy check - this can be removed at some point in the near future
|
||||
if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"):
|
||||
print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db"))
|
||||
print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db"))
|
||||
try:
|
||||
os.rename("suggestions.db", "db/suggestions.db")
|
||||
except Exception as e:
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from libretranslate.app import create_app
|
||||
|
|
|
@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client):
|
|||
response_json = json.loads(response.data)
|
||||
|
||||
assert "error" in response_json
|
||||
assert "zz is not supported" == response_json["error"]
|
||||
assert response_json["error"] == "zz is not supported"
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
|
@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client):
|
|||
response_json = json.loads(response.data)
|
||||
|
||||
assert "error" in response_json
|
||||
assert "Invalid request: missing q parameter" == response_json["error"]
|
||||
assert response_json["error"] == "Invalid request: missing q parameter"
|
||||
assert response.status_code == 400
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
from libretranslate.init import boot
|
||||
from argostranslate import package
|
||||
|
||||
from libretranslate.init import boot
|
||||
|
||||
|
||||
def test_boot_argos():
|
||||
"""Test Argos translate models initialization"""
|
||||
|
|
167
pyproject.toml
Normal file
167
pyproject.toml
Normal file
|
@ -0,0 +1,167 @@
|
|||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
requires-python = ">=3.8"
|
||||
name = "libretranslate"
|
||||
description = "Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services."
|
||||
readme = "README.md"
|
||||
license = { file = "LICENSE" }
|
||||
authors = [
|
||||
{ name = "Piero Toffanin", email = "pt@uav4geo.com" },
|
||||
{ name = "LibreTranslate Authors" },
|
||||
]
|
||||
maintainers = [
|
||||
{ name = "Piero Toffanin", email = "pt@uav4geo.com" },
|
||||
{ name = "LibreTranslate Authors" },
|
||||
]
|
||||
keywords = [
|
||||
"Python",
|
||||
"Translate",
|
||||
"Translation",
|
||||
"API",
|
||||
]
|
||||
classifiers = [
|
||||
"Operating System :: OS Independent",
|
||||
"License :: OSI Approved :: GNU Affero General Public License v3",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10"
|
||||
]
|
||||
dynamic = ["version"]
|
||||
|
||||
dependencies = [
|
||||
"argostranslate ==1.8.0",
|
||||
"Flask ==2.2.2",
|
||||
"flask-swagger ==0.2.14",
|
||||
"flask-swagger-ui ==4.11.1",
|
||||
"Flask-Limiter ==2.6.3",
|
||||
"Flask-Babel ==3.1.0",
|
||||
"Flask-Session ==0.4.0",
|
||||
"waitress ==2.1.2",
|
||||
"expiringdict ==1.2.2",
|
||||
" LTpycld2==0.42",
|
||||
"morfessor ==2.0.6",
|
||||
"appdirs ==1.4.4",
|
||||
"APScheduler ==3.9.1",
|
||||
"translatehtml ==1.5.2",
|
||||
"argos-translate-files ==1.1.1",
|
||||
"itsdangerous ==2.1.2",
|
||||
"Werkzeug ==2.2.2",
|
||||
"requests ==2.28.1",
|
||||
"redis ==4.3.4",
|
||||
"prometheus-client ==0.15.0",
|
||||
"polib ==1.1.1",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
libretranslate = "libretranslate.main:main"
|
||||
ltmanage = "libretranslate.manage:manage"
|
||||
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = [
|
||||
"pytest >=7.2.0",
|
||||
"pytest-cov",
|
||||
"ruff ==0.0.277",
|
||||
"types-requests",
|
||||
]
|
||||
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://libretranslate.com"
|
||||
Source = "https://github.com/LibreTranslate/LibreTranslate"
|
||||
Documentation = "https://github.com/LibreTranslate/LibreTranslate"
|
||||
Tracker = "https://github.com/LibreTranslate/LibreTranslate/issues"
|
||||
History = "https://github.com/LibreTranslate/LibreTranslate/releases"
|
||||
|
||||
|
||||
# ENVIRONMENTS AND SCRIPTS
|
||||
[tool.hatch.envs.default]
|
||||
features = [
|
||||
"test",
|
||||
]
|
||||
|
||||
|
||||
[tool.hatch.envs.default.scripts]
|
||||
dev = "python main.py {args}"
|
||||
locales = "python scripts/compile_locales.py"
|
||||
fmt = [
|
||||
"ruff libretranslate scripts --fix",
|
||||
]
|
||||
test = [
|
||||
"fmt",
|
||||
"pytest {args}",
|
||||
]
|
||||
cov = [
|
||||
"pytest --cov-report html {args}",
|
||||
"python -c 'import webbrowser; webbrowser.open(\"http://0.0.0.0:3000\")'",
|
||||
"python -m http.server 3000 --directory ./htmlcov",
|
||||
]
|
||||
|
||||
|
||||
[[tool.hatch.envs.all.matrix]]
|
||||
python = ["3.8", "3.9", "3.10", "3.11"]
|
||||
|
||||
|
||||
# TOOLS
|
||||
[tool.hatch.version]
|
||||
path = "VERSION"
|
||||
pattern = "^(?P<version>[0-9]*.[0-9]*.[0-9]*)$"
|
||||
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = [
|
||||
"-v",
|
||||
"--cov=libretranslate",
|
||||
"--color=yes",
|
||||
"--cov-report=term-missing",
|
||||
]
|
||||
|
||||
|
||||
# https://beta.ruff.rs/docs/rules
|
||||
[tool.ruff]
|
||||
src = ["libretranslate", "scripts"]
|
||||
target-version = "py38"
|
||||
line-length = 136
|
||||
select = [
|
||||
"I", # isort
|
||||
"N", # pep8-naming
|
||||
"S", # bandit
|
||||
"A", # flake8-builtins
|
||||
"YTT", # flake8-2020
|
||||
"B", # flake8-bugbear
|
||||
# "C", # flake8-comprehensions
|
||||
"ICN", # flake8-import-conventions
|
||||
# "SIM", # flake8-simplify
|
||||
"TID", # flake8-tidy-imports
|
||||
# "Q", # flake8-quotes
|
||||
# "FBT", # flake8-boolean-trap
|
||||
"F", # pyflakes
|
||||
"UP", # pyupgrade
|
||||
# "E", # pycodestyle errors
|
||||
# "W", # pycodestyle warnings
|
||||
# "PLC", # pylint convention
|
||||
"PLE", # pylint error
|
||||
# "PLR", # pylint refactor
|
||||
# "PLW", # pylint warning
|
||||
# "RUF", # ruff specific
|
||||
]
|
||||
|
||||
ignore = [
|
||||
"E501", # line too long
|
||||
"A003", # Class attribute is shadowing a python builtin
|
||||
"S101", # Use of `assert` detected
|
||||
"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
|
||||
"T201", "T203", # remove print and pprint
|
||||
"E402", # Module level import not at top of file
|
||||
]
|
||||
|
||||
[tool.ruff.per-file-ignores]
|
||||
"__init__.py" = ["I", "F401"] # module imported but unused
|
||||
|
||||
|
||||
[tool.ruff.mccabe]
|
||||
max-complexity = 12
|
|
@ -1,21 +0,0 @@
|
|||
argostranslate==1.8.0
|
||||
Flask==2.2.2
|
||||
flask-swagger==0.2.14
|
||||
flask-swagger-ui==4.11.1
|
||||
Flask-Limiter==2.6.3
|
||||
Flask-Babel==3.1.0
|
||||
Flask-Session==0.4.0
|
||||
waitress==2.1.2
|
||||
expiringdict==1.2.2
|
||||
LTpycld2==0.42
|
||||
morfessor==2.0.6
|
||||
appdirs==1.4.4
|
||||
APScheduler==3.9.1
|
||||
translatehtml==1.5.2
|
||||
argos-translate-files==1.1.1
|
||||
itsdangerous==2.1.2
|
||||
Werkzeug==2.2.2
|
||||
requests==2.28.1
|
||||
redis==4.3.4
|
||||
prometheus-client==0.15.0
|
||||
polib==1.1.1
|
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env python
|
||||
import sys
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
from babel.messages.frontend import main as pybabel
|
||||
|
||||
|
@ -15,7 +16,7 @@ if __name__ == "__main__":
|
|||
link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code']
|
||||
if l['code'] == 'en':
|
||||
link = "https://hosted.weblate.org/projects/libretranslate/app/"
|
||||
print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
|
||||
print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link))
|
||||
else:
|
||||
locales_dir = os.path.join("libretranslate", "locales")
|
||||
if not os.path.isdir(locales_dir):
|
||||
|
|
|
@ -1,7 +1,9 @@
|
|||
from prometheus_client import multiprocess
|
||||
import re
|
||||
import sys
|
||||
|
||||
from prometheus_client import multiprocess
|
||||
|
||||
|
||||
def child_exit(server, worker):
|
||||
multiprocess.mark_process_dead(worker.pid)
|
||||
|
||||
|
@ -35,7 +37,7 @@ def on_starting(server):
|
|||
|
||||
args = get_args()
|
||||
|
||||
from libretranslate import storage, scheduler, flood, secret
|
||||
from libretranslate import flood, scheduler, secret, storage
|
||||
storage.setup(args.shared_storage)
|
||||
scheduler.setup(args)
|
||||
flood.setup(args)
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import requests
|
||||
|
||||
response = requests.post(
|
||||
url='http://0.0.0.0:5000/translate',
|
||||
headers={'Content-Type': 'application/json'},
|
||||
|
@ -6,6 +7,7 @@ response = requests.post(
|
|||
'q': 'Hello World!',
|
||||
'source': 'en',
|
||||
'target': 'en'
|
||||
}
|
||||
},
|
||||
timeout=60
|
||||
)
|
||||
# if server unavailable then requests with raise exception and healthcheck will fail
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
#!/usr/bin/env python
|
||||
import sys
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
import argparse
|
||||
|
||||
from libretranslate.init import check_and_install_models
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
#!/usr/bin/env python
|
||||
import argparse
|
||||
import time
|
||||
import sqlite3
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db")
|
||||
|
|
|
@ -1,17 +1,19 @@
|
|||
#!/usr/bin/env python
|
||||
import sys
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
import re
|
||||
import polib
|
||||
import json
|
||||
import re
|
||||
|
||||
import polib
|
||||
from babel.messages.frontend import main as pybabel
|
||||
from libretranslate.language import load_languages, improve_translation_formatting
|
||||
from libretranslate.locales import get_available_locale_codes, swag_eval
|
||||
from translatehtml import translate_html
|
||||
from libretranslate.app import get_version, create_app
|
||||
from libretranslate.main import get_args
|
||||
from flask_swagger import swagger
|
||||
from libretranslate.app import create_app, get_version
|
||||
from libretranslate.language import improve_translation_formatting, load_languages
|
||||
from libretranslate.locales import get_available_locale_codes, swag_eval
|
||||
from libretranslate.main import get_args
|
||||
from translatehtml import translate_html
|
||||
|
||||
# Update strings
|
||||
if __name__ == "__main__":
|
||||
|
@ -74,7 +76,7 @@ if __name__ == "__main__":
|
|||
if not os.path.isfile(meta_file):
|
||||
with open(meta_file, 'w') as f:
|
||||
f.write(json.dumps({
|
||||
'name': next((lang.name for lang in languages if lang.code == l)),
|
||||
'name': next(lang.name for lang in languages if lang.code == l),
|
||||
'reviewed': False
|
||||
}, indent=4))
|
||||
print("Wrote %s" % meta_file)
|
||||
|
|
12
setup.cfg
12
setup.cfg
|
@ -1,12 +0,0 @@
|
|||
[flake8]
|
||||
exclude = .git,
|
||||
.vscode,
|
||||
.gitignore,
|
||||
README.md,
|
||||
venv,
|
||||
test,
|
||||
setup.py,
|
||||
libretranslate/__init__.py
|
||||
|
||||
max-line-length = 136
|
||||
ignore = E741
|
38
setup.py
38
setup.py
|
@ -1,38 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
version=open('VERSION').read().strip(),
|
||||
name='libretranslate',
|
||||
license='GNU Affero General Public License v3.0',
|
||||
description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.',
|
||||
author='LibreTranslate Authors',
|
||||
author_email='pt@uav4geo.com',
|
||||
url='https://libretranslate.com',
|
||||
packages=find_packages(),
|
||||
# packages=find_packages(include=['openpredict']),
|
||||
# package_dir={'openpredict': 'openpredict'},
|
||||
package_data={'': ['static/*', 'static/**/*', 'templates/*', 'locales/**/meta.json', 'locales/**/**/*.mo']},
|
||||
include_package_data=True,
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'libretranslate=libretranslate.main:main',
|
||||
'ltmanage=libretranslate.manage:manage'
|
||||
],
|
||||
},
|
||||
|
||||
python_requires='>=3.8.0',
|
||||
long_description=open('README.md').read(),
|
||||
long_description_content_type="text/markdown",
|
||||
install_requires=open("requirements.txt", "r").readlines(),
|
||||
tests_require=['pytest==7.2.0'],
|
||||
setup_requires=['pytest-runner'],
|
||||
classifiers=[
|
||||
"License :: OSI Approved :: GNU Affero General Public License v3 ",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10"
|
||||
]
|
||||
)
|
Loading…
Reference in a new issue