Merge pull request #37 from vemonet/add-pip-packaging

Add pip packaging

.github/workflows/publish-package.yml

@@ -0,0 +1,64 @@
+name: Publish package
+# Publish to PyPI when new release on GitHub, if tests pass
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+  # Manual trigger in case the release needs to be rerun
+
+jobs:
+
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest flake8
+        pip install .
+
+    - name: Check code style with flake8 (lint)
+      run: |
+        # warnings if there are Python syntax errors or undefined names 
+        # (remove --exit-zero to fail when syntax error)
+        flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+    - name: Test with pytest
+      run: pytest
+
+  
+  publish:
+    needs: [ tests ]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+        python setup.py sdist bdist_wheel
+
+    - name: Build and publish to PyPI
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.github/workflows/run-tests.yml

@@ -0,0 +1,44 @@
+name: Run tests
+# Run test at each push to main, if changes to package or tests files
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+    paths:
+    - 'requirements.txt'
+    - 'app/**'
+    - 'tests/**'
+    - '.github/workflows/run-tests.yml'
+
+jobs:
+
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest flake8
+        pip install .
+
+    - name: Check code style with flake8 (lint)
+      run: |
+        # warnings if there are Python syntax errors or undefined names 
+        # (remove --exit-zero to fail when syntax error)
+        flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+    - name: Test with pytest
+      run: pytest
+

Dockerfile

@@ -4,13 +4,10 @@ WORKDIR /app
 
 RUN pip install --upgrade pip
 
-# Avoid rebuilding this step if no changes to requirements.txt
+COPY . .
-COPY requirements.txt .
-RUN pip install -r requirements.txt
 
-# Copy everything else
+# Install package from source code
-COPY app app
+RUN pip install .
-COPY *.py LICENSE README.md ./
 
 EXPOSE 5000
-ENTRYPOINT [ "python", "main.py", "--host", "0.0.0.0" ]
+ENTRYPOINT [ "libretranslate", "--host", "0.0.0.0" ]

README.md

@@ -36,18 +36,30 @@ Response:
 }
 ```
 
-
+## Install and Run
-## Build and Run
 
 You can run your own API server in just a few lines of setup!
 
 Make sure you have installed Python (3.8 or higher), then simply issue:
 
 ```bash
-git clone https://github.com/uav4geo/LibreTranslate --recurse-submodules
+pip install libretranslate
+libretranslate [args]
+```
+
+Then open a web browser to http://localhost:5000
+
+If you're on Windows, we recommend you [Run with Docker](#run-with-docker) instead.
+
+## Build and Run
+
+If you want to make some changes to the code, you can build from source, and run the API:
+
+```bash
+git clone https://github.com/uav4geo/LibreTranslate
 cd LibreTranslate
-pip install -r requirements.txt
+pip install -e .
-python main.py [args]
+libretranslate [args]
 ```
 
 Then open a web browser to http://localhost:5000

app/app.py

@@ -3,6 +3,7 @@ from flask_swagger import swagger
 from flask_swagger_ui import get_swaggerui_blueprint
 from langdetect import detect_langs
 from langdetect import DetectorFactory
+from pkg_resources import resource_filename
 DetectorFactory.seed = 0 # deterministic
 
 def get_remote_address():

app/init.py

@@ -7,7 +7,7 @@ def boot():
 	check_and_install_models()
 
 def check_and_install_models(force=False):
-    if len(package.get_installed_packages()) == 0 or force:
+    if len(package.get_installed_packages()) < 2 or force:
         # Update package definitions from remote
         print("Updating language models")
         package.update_package_index()

app/main.py

@@ -0,0 +1,46 @@
+import argparse
+from app.app import create_app
+
+def main():
+    parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API')
+    parser.add_argument('--host', type=str,
+                        help='Hostname (%(default)s)', default="127.0.0.1")
+    parser.add_argument('--port', type=int,
+                        help='Port (%(default)s)', default=5000)
+    parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
+                        help='Set character limit (%(default)s)')
+    parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
+                        help='Set maximum number of requests per minute per client (%(default)s)')
+    parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
+                        help='Set maximum number of texts to translate in a batch request (%(default)s)')
+    parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
+                        help='Enable Google Analytics on the API client page by providing an ID (%(default)s)')
+    parser.add_argument('--debug', default=False, action="store_true",
+                        help="Enable debug environment")
+    parser.add_argument('--ssl', default=None, action="store_true",
+                        help="Whether to enable SSL")
+    parser.add_argument('--frontend-language-source', type=str, default="en", metavar="<language code>",
+                        help='Set frontend default language - source (%(default)s)')
+    parser.add_argument('--frontend-language-target', type=str, default="es", metavar="<language code>",
+                        help='Set frontend default language - target (%(default)s)')
+    parser.add_argument('--frontend-timeout', type=int, default=500, metavar="<milliseconds>",
+                        help='Set frontend translation timeout (%(default)s)')
+
+    args = parser.parse_args()
+
+    app = create_app(char_limit=args.char_limit,
+                     req_limit=args.req_limit,
+                     batch_limit=args.batch_limit,
+                     ga_id=args.ga_id,
+                     debug=args.debug,
+                     frontend_language_source=args.frontend_language_source,
+                     frontend_language_target=args.frontend_language_target,
+                     frontend_timeout=args.frontend_timeout)
+    if args.debug:
+        app.run(host=args.host, port=args.port)
+    else:
+        from waitress import serve
+        serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http')
+
+if __name__ == "__main__":
+    main()

main.py

@@ -1,44 +0,0 @@
-import argparse
-from app.app import create_app
-
-parser = argparse.ArgumentParser(description='LibreTranslate - Free and Open Source Translation API')
-parser.add_argument('--host', type=str,
-                    help='Hostname (%(default)s)', default="127.0.0.1")
-parser.add_argument('--port', type=int,
-                    help='Port (%(default)s)', default=5000)
-parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
-                    help='Set character limit (%(default)s)')
-parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
-                    help='Set maximum number of requests per minute per client (%(default)s)')
-parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
-                    help='Set maximum number of texts to translate in a batch request (%(default)s)')
-parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
-                    help='Enable Google Analytics on the API client page by providing an ID (%(default)s)')
-parser.add_argument('--debug', default=False, action="store_true",
-                    help="Enable debug environment")
-parser.add_argument('--ssl', default=None, action="store_true",
-                    help="Whether to enable SSL")
-parser.add_argument('--frontend-language-source', type=str, default="en", metavar="<language code>",
-                    help='Set frontend default language - source (%(default)s)')
-parser.add_argument('--frontend-language-target', type=str, default="es", metavar="<language code>",
-                    help='Set frontend default language - target (%(default)s)')
-parser.add_argument('--frontend-timeout', type=int, default=500, metavar="<milliseconds>",
-                    help='Set frontend translation timeout (%(default)s)')
-
-args = parser.parse_args()
-
-
-if __name__ == "__main__":
-    app = create_app(char_limit=args.char_limit,
-                     req_limit=args.req_limit,
-                     batch_limit=args.batch_limit,
-                     ga_id=args.ga_id,
-                     debug=args.debug,
-                     frontend_language_source=args.frontend_language_source,
-                     frontend_language_target=args.frontend_language_target,
-                     frontend_timeout=args.frontend_timeout)
-    if args.debug:
-        app.run(host=args.host, port=args.port)
-    else:
-        from waitress import serve
-        serve(app, host=args.host, port=args.port, url_scheme='https' if args.ssl else 'http')

requirements.txt

@@ -1,4 +1,4 @@
-argostranslate==1.1.0
+argostranslate==1.1.2
 Flask==1.1.2
 flask-swagger==0.2.14
 flask-swagger-ui==3.36.0

setup.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from setuptools import setup, find_packages
+
+setup(
+    version='1.1.0',
+    name='libretranslate',
+    license='GNU Affero General Public License v3.0',
+    description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.',
+    author='LibreTranslate Authors',
+    author_email='pt@uav4geo.com',
+    url='https://libretranslate.com',
+    packages=find_packages(),
+    # packages=find_packages(include=['openpredict']),
+    # package_dir={'openpredict': 'openpredict'},
+    package_data={'': ['static/*', 'templates/*']},
+    include_package_data=True,
+    entry_points={
+        'console_scripts': [
+            'libretranslate=app.main:main',
+        ],
+    },
+
+    python_requires='>=3.6.0',
+    long_description=open('README.md').read(),
+    long_description_content_type="text/markdown",
+    install_requires=open("requirements.txt", "r").readlines(),
+    tests_require=['pytest==5.2.0'],
+    setup_requires=['pytest-runner'],
+    classifiers=[
+        "License :: OSI Approved :: GNU Affero General Public License v3 ",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8"
+    ]
+)

tests/test_init.py

@@ -0,0 +1,9 @@
+import pytest
+from app.init import boot
+from argostranslate import package
+
+def test_boot_argos():
+    """Test Argos translate models initialization"""
+    boot()
+
+    assert len(package.get_installed_packages()) > 2