Added option to update models rather than reinstall

This commit is contained in:
Robert George 2023-09-29 13:43:59 -07:00
parent 81ba2b81e9
commit 78713d8017
No known key found for this signature in database
GPG key ID: 9B0D835A293663C4
5 changed files with 40 additions and 12 deletions

View file

@ -125,7 +125,7 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db):
def create_app(args): def create_app(args):
from libretranslate.init import boot from libretranslate.init import boot
boot(args.load_only, args.update_models) boot(args.load_only, args.update_models, args.install_models)
from libretranslate.language import load_languages from libretranslate.language import load_languages
@ -1042,7 +1042,6 @@ def create_app(args):
s = request.values.get("s") s = request.values.get("s")
source_lang = request.values.get("source") source_lang = request.values.get("source")
target_lang = request.values.get("target") target_lang = request.values.get("target")
if not q: if not q:
abort(400, description=_("Invalid request: missing %(name)s parameter", name='q')) abort(400, description=_("Invalid request: missing %(name)s parameter", name='q'))
if not s: if not s:

View file

@ -171,6 +171,11 @@ _default_options_objects = [
'default_value': False, 'default_value': False,
'value_type': 'bool' 'value_type': 'bool'
}, },
{
'name': 'INSTALL_MODELS',
'default_value': False,
'value_type': 'bool'
},
{ {
'name': 'METRICS', 'name': 'METRICS',
'default_value': False, 'default_value': False,

View file

@ -4,23 +4,26 @@ from argostranslate import package, translate
import libretranslate.language import libretranslate.language
def boot(load_only=None, update_models=False): def boot(load_only=None, update_models=False, install_models=False):
try: try:
check_and_install_models(force=update_models, load_only_lang_codes=load_only) if update_models:
check_and_install_models(load_only_lang_codes=load_only, update=update_models)
else:
check_and_install_models(force=install_models, load_only_lang_codes=load_only)
except Exception as e: except Exception as e:
print("Cannot update models (normal if you're offline): %s" % str(e)) print("Cannot update models (normal if you're offline): %s" % str(e))
def check_and_install_models(force=False, load_only_lang_codes=None): def check_and_install_models(force=False, load_only_lang_codes=None,update=False):
if len(package.get_installed_packages()) < 2 or force: if len(package.get_installed_packages()) < 2 or force or update:
# Update package definitions from remote # Update package definitions from remote
print("Updating language models") print("Updating language models")
package.update_package_index() package.update_package_index()
# Load available packages from local package index # Load available packages from local package index
available_packages = package.get_available_packages() available_packages = package.get_available_packages()
installed_packages = package.get_installed_packages()
print("Found %s models" % len(available_packages)) print("Found %s models" % len(available_packages))
if load_only_lang_codes is not None: if load_only_lang_codes is not None:
# load_only_lang_codes: List[str] (codes) # load_only_lang_codes: List[str] (codes)
# Ensure the user does not use any unavailable language code. # Ensure the user does not use any unavailable language code.
@ -44,10 +47,24 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
# Download and install all available packages # Download and install all available packages
for available_package in available_packages: for available_package in available_packages:
print( update = False
f"Downloading {available_package} ({available_package.package_version}) ..." if not force:
) for pack in installed_packages:
available_package.install() if (
pack.from_code == available_package.from_code
and pack.to_code == available_package.to_code
):
update = True
if pack.package_version < available_package.package_version:
print(
f"Updating {available_package} ({pack.package_version}->{available_package.package_version}) ..."
)
pack.update()
if not update:
print(
f"Downloading {available_package} ({available_package.package_version}) ..."
)
available_package.install()
# reload installed languages # reload installed languages
libretranslate.language.languages = translate.get_installed_languages() libretranslate.language.languages = translate.get_installed_languages()

View file

@ -160,6 +160,9 @@ def get_args():
parser.add_argument( parser.add_argument(
"--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup" "--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup"
) )
parser.add_argument(
"--install-models", default=DEFARGS['INSTALL_MODELS'], action="store_true", help="Install language models at startup"
)
parser.add_argument( parser.add_argument(
"--metrics", "--metrics",
default=DEFARGS['METRICS'], default=DEFARGS['METRICS'],

View file

@ -10,8 +10,12 @@ from libretranslate.init import check_and_install_models
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--load_only_lang_codes", type=str, default="") parser.add_argument("--load_only_lang_codes", type=str, default="")
parser.add_argument("--update", action='store_true')
args = parser.parse_args() args = parser.parse_args()
lang_codes = args.load_only_lang_codes.split(",") lang_codes = args.load_only_lang_codes.split(",")
if len(lang_codes) == 0 or lang_codes[0] == '': if len(lang_codes) == 0 or lang_codes[0] == '':
lang_codes = None lang_codes = None
check_and_install_models(force=True, load_only_lang_codes=lang_codes) if args.update:
check_and_install_models(update=True, load_only_lang_codes=lang_codes)
else:
check_and_install_models(force=True, load_only_lang_codes=lang_codes)