Merge pull request #503 from rrgeorge/add_update_option

Add option to update models rather than reinstall

libretranslate/app.py

@@ -125,7 +125,7 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db):
 def create_app(args):
     from libretranslate.init import boot
 
-    boot(args.load_only, args.update_models)
+    boot(args.load_only, args.update_models, args.force_update_models)
 
     from libretranslate.language import load_languages
 

libretranslate/default_values.py

@@ -171,6 +171,11 @@ _default_options_objects = [
         'default_value': False,
         'value_type': 'bool'
     },
+    {
+        'name': 'FORCE_UPDATE_MODELS',
+        'default_value': False,
+        'value_type': 'bool'
+    },
     {
         'name': 'METRICS',
         'default_value': False,

libretranslate/init.py

@@ -1,26 +1,30 @@
 
 from argostranslate import package, translate
+from packaging import version
 
 import libretranslate.language
 
 
-def boot(load_only=None, update_models=False):
+def boot(load_only=None, update_models=False, install_models=False):
     try:
-        check_and_install_models(force=update_models, load_only_lang_codes=load_only)
+        if update_models:
+            check_and_install_models(load_only_lang_codes=load_only, update=update_models)
+        else:
+            check_and_install_models(force=install_models, load_only_lang_codes=load_only)
     except Exception as e:
         print("Cannot update models (normal if you're offline): %s" % str(e))
 
 
-def check_and_install_models(force=False, load_only_lang_codes=None):
+def check_and_install_models(force=False, load_only_lang_codes=None,update=False):
-    if len(package.get_installed_packages()) < 2 or force:
+    if len(package.get_installed_packages()) < 2 or force or update:
         # Update package definitions from remote
         print("Updating language models")
         package.update_package_index()
 
         # Load available packages from local package index
         available_packages = package.get_available_packages()
+        installed_packages = package.get_installed_packages()
         print("Found %s models" % len(available_packages))
-
         if load_only_lang_codes is not None:
             # load_only_lang_codes: List[str] (codes)
             # Ensure the user does not use any unavailable language code.
@@ -44,10 +48,24 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
 
         # Download and install all available packages
         for available_package in available_packages:
-            print(
+            update = False
-                f"Downloading {available_package} ({available_package.package_version}) ..."
+            if not force:
-            )
+                for pack in installed_packages:
-            available_package.install()
+                    if (
+                            pack.from_code == available_package.from_code
+                            and pack.to_code == available_package.to_code
+                        ):
+                        update = True
+                        if version.parse(pack.package_version) < version.parse(available_package.package_version):
+                            print(
+                                f"Updating {available_package} ({pack.package_version}->{available_package.package_version}) ..."
+                            )
+                            pack.update()
+            if not update:
+                print(
+                    f"Downloading {available_package} ({available_package.package_version}) ..."
+                )
+                available_package.install()
 
         # reload installed languages
         libretranslate.language.languages = translate.get_installed_languages()

libretranslate/main.py

@@ -160,6 +160,9 @@ def get_args():
     parser.add_argument(
         "--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup"
     )
+    parser.add_argument(
+        "--force-update-models", default=DEFARGS['FORCE_UPDATE_MODELS'], action="store_true", help="Install/Reinstall language models at startup"
+    )
     parser.add_argument(
         "--metrics",
         default=DEFARGS['METRICS'],

pyproject.toml

@@ -54,6 +54,7 @@ dependencies = [
     "redis ==4.3.4",
     "prometheus-client ==0.15.0",
     "polib ==1.1.1",
+    "packaging ==23.1"
 ]
 
 [project.scripts]

scripts/install_models.py

@@ -10,8 +10,12 @@ from libretranslate.init import check_and_install_models
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--load_only_lang_codes", type=str, default="")
+    parser.add_argument("--update", action='store_true')
     args = parser.parse_args()
     lang_codes = args.load_only_lang_codes.split(",")
     if len(lang_codes) == 0 or lang_codes[0] == '':
         lang_codes = None
-    check_and_install_models(force=True, load_only_lang_codes=lang_codes)
+    if args.update:
+        check_and_install_models(update=True, load_only_lang_codes=lang_codes)
+    else:
+        check_and_install_models(force=True, load_only_lang_codes=lang_codes)