From 90d743796b04554104fb24738a83664fb6720ef6 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 01:24:34 +0100 Subject: [PATCH 1/9] Start adding pip setup package requirements (setup.py, __init__.py, use pkg_resource) --- .github/workflows/publish-package.yml | 60 +++++++++++++++++++++++++++ README.md | 2 +- app/__init.py__ | 0 app/app.py | 3 +- app/main.py | 46 ++++++++++++++++++++ main.py | 44 -------------------- setup.py | 37 +++++++++++++++++ 7 files changed, 146 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/publish-package.yml create mode 100644 app/__init.py__ create mode 100644 app/main.py delete mode 100644 main.py create mode 100644 setup.py diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml new file mode 100644 index 0000000..5be5c45 --- /dev/null +++ b/.github/workflows/publish-package.yml @@ -0,0 +1,60 @@ +name: Publish package +# Publish to PyPI when new release on GitHub, if tests pass +on: + release: + types: [created] + workflow_dispatch: + # Manual trigger in case the release needs to be rerun + +jobs: + + # tests: + # runs-on: ubuntu-latest + # strategy: + # matrix: + # python-version: [3.6, 3.7, 3.8] + # steps: + # - uses: actions/checkout@v2 + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v2 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install flake8 pytest + # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # - name: Lint with flake8 + # run: | + # # stop the build if there are Python syntax errors or undefined names + # flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # - name: Test with pytest + # run: | + # python setup.py pytest + + + publish: + # needs: [ tests ] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + python setup.py sdist bdist_wheel + + - name: Build and publish to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/README.md b/README.md index 170b1c4..388d06b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ You can run your own API server in just a few lines of setup! Make sure you have installed Python (3.8 or higher), then simply issue: ```bash -git clone https://github.com/uav4geo/LibreTranslate --recurse-submodules +git clone https://github.com/uav4geo/LibreTranslate cd LibreTranslate pip install -r requirements.txt python main.py [args] diff --git a/app/__init.py__ b/app/__init.py__ new file mode 100644 index 0000000..e69de29 diff --git a/app/app.py b/app/app.py index 9b56f0d..1bf4ae2 100644 --- a/app/app.py +++ b/app/app.py @@ -3,6 +3,7 @@ from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint from langdetect import detect_langs from langdetect import DetectorFactory +from pkg_resources import resource_filename DetectorFactory.seed = 0 # deterministic def get_remote_address(): @@ -67,7 +68,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa @app.route("/") def index(): - return render_template('index.html', gaId=ga_id, frontendTimeout=frontend_timeout) + return render_template(resource_filename('app', 'templates/index.html'), gaId=ga_id, frontendTimeout=frontend_timeout) @app.route("/languages") def langs(): diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..243ba07 --- /dev/null +++ b/app/main.py @@ -0,0 +1,46 @@ +import argparse +from app.app import create_app + +def main(): + parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API') + parser.add_argument('--host', type=str, + help='Hostname (%(default)s)', default="127.0.0.1") + parser.add_argument('--port', type=int, + help='Port (%(default)s)', default=5000) + parser.add_argument('--char-limit', default=-1, type=int, metavar="", + help='Set character limit (%(default)s)') + parser.add_argument('--req-limit', default=-1, type=int, metavar="", + help='Set maximum number of requests per minute per client (%(default)s)') + parser.add_argument('--batch-limit', default=-1, type=int, metavar="", + help='Set maximum number of texts to translate in a batch request (%(default)s)') + parser.add_argument('--ga-id', type=str, default=None, metavar="", + help='Enable Google Analytics on the API client page by providing an ID (%(default)s)') + parser.add_argument('--debug', default=False, action="store_true", + help="Enable debug environment") + parser.add_argument('--ssl', default=None, action="store_true", + help="Whether to enable SSL") + parser.add_argument('--frontend-language-source', type=str, default="en", metavar="", + help='Set frontend default language - source (%(default)s)') + parser.add_argument('--frontend-language-target', type=str, default="es", metavar="", + help='Set frontend default language - target (%(default)s)') + parser.add_argument('--frontend-timeout', type=int, default=500, metavar="", + help='Set frontend translation timeout (%(default)s)') + + args = parser.parse_args() + + app = create_app(char_limit=args.char_limit, + req_limit=args.req_limit, + batch_limit=args.batch_limit, + ga_id=args.ga_id, + debug=args.debug, + frontend_language_source=args.frontend_language_source, + frontend_language_target=args.frontend_language_target, + frontend_timeout=args.frontend_timeout) + if args.debug: + app.run(host=args.host, port=args.port) + else: + from waitress import serve + serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http') + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py deleted file mode 100644 index 348900d..0000000 --- a/main.py +++ /dev/null @@ -1,44 +0,0 @@ -import argparse -from app.app import create_app - -parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API') -parser.add_argument('--host', type=str, - help='Hostname (%(default)s)', default="127.0.0.1") -parser.add_argument('--port', type=int, - help='Port (%(default)s)', default=5000) -parser.add_argument('--char-limit', default=-1, type=int, metavar="", - help='Set character limit (%(default)s)') -parser.add_argument('--req-limit', default=-1, type=int, metavar="", - help='Set maximum number of requests per minute per client (%(default)s)') -parser.add_argument('--batch-limit', default=-1, type=int, metavar="", - help='Set maximum number of texts to translate in a batch request (%(default)s)') -parser.add_argument('--ga-id', type=str, default=None, metavar="", - help='Enable Google Analytics on the API client page by providing an ID (%(default)s)') -parser.add_argument('--debug', default=False, action="store_true", - help="Enable debug environment") -parser.add_argument('--ssl', default=None, action="store_true", - help="Whether to enable SSL") -parser.add_argument('--frontend-language-source', type=str, default="en", metavar="", - help='Set frontend default language - source (%(default)s)') -parser.add_argument('--frontend-language-target', type=str, default="es", metavar="", - help='Set frontend default language - target (%(default)s)') -parser.add_argument('--frontend-timeout', type=int, default=500, metavar="", - help='Set frontend translation timeout (%(default)s)') - -args = parser.parse_args() - - -if __name__ == "__main__": - app = create_app(char_limit=args.char_limit, - req_limit=args.req_limit, - batch_limit=args.batch_limit, - ga_id=args.ga_id, - debug=args.debug, - frontend_language_source=args.frontend_language_source, - frontend_language_target=args.frontend_language_target, - frontend_timeout=args.frontend_timeout) - if args.debug: - app.run(host=args.host, port=args.port) - else: - from waitress import serve - serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http') diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..022ecef --- /dev/null +++ b/setup.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup, find_packages + +setup( + version='1.1.0', + name='libretranslate', + license='GNU Affero General Public License v3.0', + description='Free and Open Source Machine Translation API. 100% self-hosted, no limits, no ties to proprietary services. Built on top of Argos Translate.', + author='Piero Toffanin', + author_email='pierotofy@email.com', + url='https://libretranslate.com', + packages=find_packages(), + # packages=find_packages(include=['openpredict']), + # package_dir={'openpredict': 'openpredict'}, + package_data={'': ['static/*', 'templates/*']}, + include_package_data=True, + entry_points={ + 'console_scripts': [ + 'libretranslate=app.main:main', + ], + }, + + python_requires='>=3.6.0', + long_description=open('README.md').read(), + long_description_content_type="text/markdown", + install_requires=open("requirements.txt", "r").readlines(), + tests_require=['pytest==5.2.0'], + setup_requires=['pytest-runner'], + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8" + ] +) From a0760eb23f935c2032ff5a1fa0c65d7dd30c7d25 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 14:12:32 +0100 Subject: [PATCH 2/9] updated argos to 1.1.2, fix argos translate init and templates.html render --- .github/workflows/publish-package.yml | 2 +- app/app.py | 2 +- app/init.py | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 5be5c45..4045163 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -44,7 +44,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.8' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/app/app.py b/app/app.py index 1bf4ae2..c7b7b5c 100644 --- a/app/app.py +++ b/app/app.py @@ -68,7 +68,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa @app.route("/") def index(): - return render_template(resource_filename('app', 'templates/index.html'), gaId=ga_id, frontendTimeout=frontend_timeout) + return render_template('index.html', gaId=ga_id, frontendTimeout=frontend_timeout) @app.route("/languages") def langs(): diff --git a/app/init.py b/app/init.py index 91e36b9..c540c1c 100644 --- a/app/init.py +++ b/app/init.py @@ -7,7 +7,7 @@ def boot(): check_and_install_models() def check_and_install_models(force=False): - if len(package.get_installed_packages()) == 0 or force: + if len(package.get_installed_packages()) < 2 or force: # Update package definitions from remote print("Updating language models") package.update_package_index() diff --git a/requirements.txt b/requirements.txt index 8cc97da..a82a96c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -argostranslate==1.1.0 +argostranslate==1.1.2 Flask==1.1.2 flask-swagger==0.2.14 flask-swagger-ui==3.36.0 From 7139b2f41be1737ae8bd532c7e15f91d255270a3 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 14:17:06 +0100 Subject: [PATCH 3/9] update Dockerfile and README --- Dockerfile | 11 ++++------- README.md | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index f460d7e..2452638 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,10 @@ WORKDIR /app RUN pip install --upgrade pip -# Avoid rebuilding this step if no changes to requirements.txt -COPY requirements.txt . -RUN pip install -r requirements.txt +COPY . . -# Copy everything else -COPY app app -COPY *.py LICENSE README.md ./ +# Install package from source code +RUN pip install . EXPOSE 5000 -ENTRYPOINT [ "python", "main.py", "--host", "0.0.0.0" ] +ENTRYPOINT [ "libretranslate", "--host", "0.0.0.0" ] diff --git a/README.md b/README.md index 388d06b..87ca66e 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,28 @@ Response: } ``` - -## Build and Run +## Install and Run You can run your own API server in just a few lines of setup! Make sure you have installed Python (3.8 or higher), then simply issue: +```bash +pip install libretranslate +libretranslate [args] +``` + +Then open a web browser to http://localhost:5000 + +## Build and Run + +If you want to make some changes to the code, you can build from source, and run the API: + ```bash git clone https://github.com/uav4geo/LibreTranslate cd LibreTranslate -pip install -r requirements.txt -python main.py [args] +pip install -e . +libretranslate [args] ``` Then open a web browser to http://localhost:5000 From 9fe75094e0603213d67cd8e0c9b79348946b39b6 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 14:46:11 +0100 Subject: [PATCH 4/9] fix license in setup.py --- setup.py | 2 +- tests/__init.py__ | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/__init.py__ diff --git a/setup.py b/setup.py index 022ecef..86538c1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( tests_require=['pytest==5.2.0'], setup_requires=['pytest-runner'], classifiers=[ - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: GNU Affero General Public License v3 ", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tests/__init.py__ b/tests/__init.py__ new file mode 100644 index 0000000..e69de29 From d53e08b57e99afbc637214878e2bd6753ce04fb1 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 15:21:53 +0100 Subject: [PATCH 5/9] Add tests for argos boot --- .github/workflows/publish-package.yml | 56 ++++++++++++++------------- .github/workflows/run-tests.yml | 44 +++++++++++++++++++++ tests/test_init.py | 13 +++++++ 3 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/run-tests.yml create mode 100644 tests/test_init.py diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 4045163..ab83a6a 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -8,35 +8,39 @@ on: jobs: - # tests: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # python-version: [3.6, 3.7, 3.8] - # steps: - # - uses: actions/checkout@v2 - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v2 - # with: - # python-version: ${{ matrix.python-version }} - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install flake8 pytest - # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - # - name: Lint with flake8 - # run: | - # # stop the build if there are Python syntax errors or undefined names - # flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics - # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - # - name: Test with pytest - # run: | - # python setup.py pytest + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Check code style with flake8 (lint) + run: | + # warnings if there are Python syntax errors or undefined names + # (remove --exit-zero to fail when syntax error) + flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install . + + - name: Test with pytest + run: pytest publish: - # needs: [ tests ] + needs: [ tests ] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..de28ec5 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,44 @@ +name: Run tests +# Run test at each push to main, if changes to package or tests files +on: + workflow_dispatch: + push: + branches: [ main ] + paths: + - 'requirements.txt' + - 'app/**' + - 'tests/**' + - '.github/workflows/run-tests.yml' + +jobs: + + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Check code style with flake8 (lint) + run: | + # warnings if there are Python syntax errors or undefined names + # (remove --exit-zero to fail when syntax error) + flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install . + + - name: Test with pytest + run: pytest + diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..bb509f9 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,13 @@ +import pytest +from app.init import boot +from argostranslate import package + +def test_boot_argos(): + """Test Argos translate models initialization""" + boot() + + print(package.get_installed_packages()) + assert len(package.get_installed_packages()) > 2 + # Check length models? + # assert 0.80 < scores['precision'] < 0.95 + From 17c1554c51c17b9638751ad5329c9a8a3566a656 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 15:25:05 +0100 Subject: [PATCH 6/9] Fix test workflow --- .github/workflows/publish-package.yml | 12 ++++++------ .github/workflows/run-tests.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index ab83a6a..0d5b93a 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -21,6 +21,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest flake8 + pip install . + - name: Check code style with flake8 (lint) run: | # warnings if there are Python syntax errors or undefined names @@ -29,12 +35,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest - pip install . - - name: Test with pytest run: pytest diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index de28ec5..3a524d3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -25,6 +25,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest flake8 + pip install . + - name: Check code style with flake8 (lint) run: | # warnings if there are Python syntax errors or undefined names @@ -33,12 +39,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest - pip install . - - name: Test with pytest run: pytest From 5eea0e75e84afac947071e7c6e79e669b5caab39 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 15:30:14 +0100 Subject: [PATCH 7/9] Fix __init__.py files --- app/{__init.py__ => __init__.py} | 0 tests/{__init.py__ => __init__.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename app/{__init.py__ => __init__.py} (100%) rename tests/{__init.py__ => __init__.py} (100%) diff --git a/app/__init.py__ b/app/__init__.py similarity index 100% rename from app/__init.py__ rename to app/__init__.py diff --git a/tests/__init.py__ b/tests/__init__.py similarity index 100% rename from tests/__init.py__ rename to tests/__init__.py From c75c664058a7b9d8d0db3e96523fcdbf01b348ea Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Tue, 9 Feb 2021 18:00:11 +0100 Subject: [PATCH 8/9] Clean up tests --- tests/test_init.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index bb509f9..2dec0f5 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -6,8 +6,4 @@ def test_boot_argos(): """Test Argos translate models initialization""" boot() - print(package.get_installed_packages()) - assert len(package.get_installed_packages()) > 2 - # Check length models? - # assert 0.80 < scores['precision'] < 0.95 - + assert len(package.get_installed_packages()) > 2 \ No newline at end of file From 1b907c17035c27a3410b970b794fc7315ee9c717 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 9 Feb 2021 15:20:46 -0500 Subject: [PATCH 9/9] Add info to setup.py, edit README --- README.md | 2 ++ setup.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 87ca66e..20a9668 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ libretranslate [args] Then open a web browser to http://localhost:5000 +If you're on Windows, we recommend you [Run with Docker](#run-with-docker) instead. + ## Build and Run If you want to make some changes to the code, you can build from source, and run the API: diff --git a/setup.py b/setup.py index 86538c1..3accce4 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,9 @@ setup( version='1.1.0', name='libretranslate', license='GNU Affero General Public License v3.0', - description='Free and Open Source Machine Translation API. 100% self-hosted, no limits, no ties to proprietary services. Built on top of Argos Translate.', - author='Piero Toffanin', - author_email='pierotofy@email.com', + description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.', + author='LibreTranslate Authors', + author_email='pt@uav4geo.com', url='https://libretranslate.com', packages=find_packages(), # packages=find_packages(include=['openpredict']),