Merge pull request #37 from vemonet/add-pip-packaging

Add pip packaging
This commit is contained in:
Piero Toffanin 2021-02-09 15:21:39 -05:00 committed by GitHub
commit 485c63fd4f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 224 additions and 58 deletions

64
.github/workflows/publish-package.yml vendored Normal file
View file

@ -0,0 +1,64 @@
name: Publish package
# Publish to PyPI when new release on GitHub, if tests pass
on:
release:
types: [created]
workflow_dispatch:
# Manual trigger in case the release needs to be rerun
jobs:
tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.6, 3.7, 3.8]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pytest flake8
pip install .
- name: Check code style with flake8 (lint)
run: |
# warnings if there are Python syntax errors or undefined names
# (remove --exit-zero to fail when syntax error)
flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: pytest
publish:
needs: [ tests ]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
python setup.py sdist bdist_wheel
- name: Build and publish to PyPI
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
twine upload dist/*

44
.github/workflows/run-tests.yml vendored Normal file
View file

@ -0,0 +1,44 @@
name: Run tests
# Run test at each push to main, if changes to package or tests files
on:
workflow_dispatch:
push:
branches: [ main ]
paths:
- 'requirements.txt'
- 'app/**'
- 'tests/**'
- '.github/workflows/run-tests.yml'
jobs:
tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.6, 3.7, 3.8]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pytest flake8
pip install .
- name: Check code style with flake8 (lint)
run: |
# warnings if there are Python syntax errors or undefined names
# (remove --exit-zero to fail when syntax error)
flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: pytest

View file

@ -4,13 +4,10 @@ WORKDIR /app
RUN pip install --upgrade pip RUN pip install --upgrade pip
# Avoid rebuilding this step if no changes to requirements.txt COPY . .
COPY requirements.txt .
RUN pip install -r requirements.txt
# Copy everything else # Install package from source code
COPY app app RUN pip install .
COPY *.py LICENSE README.md ./
EXPOSE 5000 EXPOSE 5000
ENTRYPOINT [ "python", "main.py", "--host", "0.0.0.0" ] ENTRYPOINT [ "libretranslate", "--host", "0.0.0.0" ]

View file

@ -36,18 +36,30 @@ Response:
} }
``` ```
## Install and Run
## Build and Run
You can run your own API server in just a few lines of setup! You can run your own API server in just a few lines of setup!
Make sure you have installed Python (3.8 or higher), then simply issue: Make sure you have installed Python (3.8 or higher), then simply issue:
```bash ```bash
git clone https://github.com/uav4geo/LibreTranslate --recurse-submodules pip install libretranslate
libretranslate [args]
```
Then open a web browser to http://localhost:5000
If you're on Windows, we recommend you [Run with Docker](#run-with-docker) instead.
## Build and Run
If you want to make some changes to the code, you can build from source, and run the API:
```bash
git clone https://github.com/uav4geo/LibreTranslate
cd LibreTranslate cd LibreTranslate
pip install -r requirements.txt pip install -e .
python main.py [args] libretranslate [args]
``` ```
Then open a web browser to http://localhost:5000 Then open a web browser to http://localhost:5000

0
app/__init__.py Normal file
View file

View file

@ -3,6 +3,7 @@ from flask_swagger import swagger
from flask_swagger_ui import get_swaggerui_blueprint from flask_swagger_ui import get_swaggerui_blueprint
from langdetect import detect_langs from langdetect import detect_langs
from langdetect import DetectorFactory from langdetect import DetectorFactory
from pkg_resources import resource_filename
DetectorFactory.seed = 0 # deterministic DetectorFactory.seed = 0 # deterministic
def get_remote_address(): def get_remote_address():

View file

@ -7,7 +7,7 @@ def boot():
check_and_install_models() check_and_install_models()
def check_and_install_models(force=False): def check_and_install_models(force=False):
if len(package.get_installed_packages()) == 0 or force: if len(package.get_installed_packages()) < 2 or force:
# Update package definitions from remote # Update package definitions from remote
print("Updating language models") print("Updating language models")
package.update_package_index() package.update_package_index()

46
app/main.py Normal file
View file

@ -0,0 +1,46 @@
import argparse
from app.app import create_app
def main():
parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API')
parser.add_argument('--host', type=str,
help='Hostname (%(default)s)', default="127.0.0.1")
parser.add_argument('--port', type=int,
help='Port (%(default)s)', default=5000)
parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
help='Set character limit (%(default)s)')
parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
help='Set maximum number of requests per minute per client (%(default)s)')
parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
help='Set maximum number of texts to translate in a batch request (%(default)s)')
parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
help='Enable Google Analytics on the API client page by providing an ID (%(default)s)')
parser.add_argument('--debug', default=False, action="store_true",
help="Enable debug environment")
parser.add_argument('--ssl', default=None, action="store_true",
help="Whether to enable SSL")
parser.add_argument('--frontend-language-source', type=str, default="en", metavar="<language code>",
help='Set frontend default language - source (%(default)s)')
parser.add_argument('--frontend-language-target', type=str, default="es", metavar="<language code>",
help='Set frontend default language - target (%(default)s)')
parser.add_argument('--frontend-timeout', type=int, default=500, metavar="<milliseconds>",
help='Set frontend translation timeout (%(default)s)')
args = parser.parse_args()
app = create_app(char_limit=args.char_limit,
req_limit=args.req_limit,
batch_limit=args.batch_limit,
ga_id=args.ga_id,
debug=args.debug,
frontend_language_source=args.frontend_language_source,
frontend_language_target=args.frontend_language_target,
frontend_timeout=args.frontend_timeout)
if args.debug:
app.run(host=args.host, port=args.port)
else:
from waitress import serve
serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http')
if __name__ == "__main__":
main()

44
main.py
View file

@ -1,44 +0,0 @@
import argparse
from app.app import create_app
parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API')
parser.add_argument('--host', type=str,
help='Hostname (%(default)s)', default="127.0.0.1")
parser.add_argument('--port', type=int,
help='Port (%(default)s)', default=5000)
parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
help='Set character limit (%(default)s)')
parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
help='Set maximum number of requests per minute per client (%(default)s)')
parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
help='Set maximum number of texts to translate in a batch request (%(default)s)')
parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
help='Enable Google Analytics on the API client page by providing an ID (%(default)s)')
parser.add_argument('--debug', default=False, action="store_true",
help="Enable debug environment")
parser.add_argument('--ssl', default=None, action="store_true",
help="Whether to enable SSL")
parser.add_argument('--frontend-language-source', type=str, default="en", metavar="<language code>",
help='Set frontend default language - source (%(default)s)')
parser.add_argument('--frontend-language-target', type=str, default="es", metavar="<language code>",
help='Set frontend default language - target (%(default)s)')
parser.add_argument('--frontend-timeout', type=int, default=500, metavar="<milliseconds>",
help='Set frontend translation timeout (%(default)s)')
args = parser.parse_args()
if __name__ == "__main__":
app = create_app(char_limit=args.char_limit,
req_limit=args.req_limit,
batch_limit=args.batch_limit,
ga_id=args.ga_id,
debug=args.debug,
frontend_language_source=args.frontend_language_source,
frontend_language_target=args.frontend_language_target,
frontend_timeout=args.frontend_timeout)
if args.debug:
app.run(host=args.host, port=args.port)
else:
from waitress import serve
serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http')

View file

@ -1,4 +1,4 @@
argostranslate==1.1.0 argostranslate==1.1.2
Flask==1.1.2 Flask==1.1.2
flask-swagger==0.2.14 flask-swagger==0.2.14
flask-swagger-ui==3.36.0 flask-swagger-ui==3.36.0

37
setup.py Normal file
View file

@ -0,0 +1,37 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from setuptools import setup, find_packages
setup(
version='1.1.0',
name='libretranslate',
license='GNU Affero General Public License v3.0',
description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.',
author='LibreTranslate Authors',
author_email='pt@uav4geo.com',
url='https://libretranslate.com',
packages=find_packages(),
# packages=find_packages(include=['openpredict']),
# package_dir={'openpredict': 'openpredict'},
package_data={'': ['static/*', 'templates/*']},
include_package_data=True,
entry_points={
'console_scripts': [
'libretranslate=app.main:main',
],
},
python_requires='>=3.6.0',
long_description=open('README.md').read(),
long_description_content_type="text/markdown",
install_requires=open("requirements.txt", "r").readlines(),
tests_require=['pytest==5.2.0'],
setup_requires=['pytest-runner'],
classifiers=[
"License :: OSI Approved :: GNU Affero General Public License v3 ",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8"
]
)

0
tests/__init__.py Normal file
View file

9
tests/test_init.py Normal file
View file

@ -0,0 +1,9 @@
import pytest
from app.init import boot
from argostranslate import package
def test_boot_argos():
"""Test Argos translate models initialization"""
boot()
assert len(package.get_installed_packages()) > 2