Add lexilang for language detection on short texts

libretranslate/detect.py

@@ -1,6 +1,7 @@
 from functools import lru_cache
 
 import linguars
+from lexilang.detector import detect as lldetect
 
 
 class Language:
@@ -26,9 +27,15 @@ def load_detector(langcodes = ()):
 
 class Detector:
   def __init__(self, langcodes = ()):
+    self.langcodes = langcodes
     self.detector = load_detector(langcodes)
 
   def detect(self, text):
+    if len(text) < 18:
+      code, conf = lldetect(text, self.langcodes)
+      if conf > 0:
+        return [Language(code, round(conf * 100))]
+
     top_3_choices = self.detector.confidence(text)[:3]
     if top_3_choices[0][1] == 0:
       return [Language("en", 0)]

libretranslate/language.py

@@ -18,7 +18,7 @@ def load_languages():
 @lru_cache(maxsize=None)
 def load_lang_codes():
     languages = load_languages()
-    return (l.code for l in languages)
+    return tuple(l.code for l in languages)
 
 def detect_languages(text):
     # detect batch processing

pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
     "waitress ==2.1.2",
     "expiringdict ==1.2.2",
     "linguars==0.4.0",
+    "lexilang==1.0.1",
     "morfessor ==2.0.6",
     "appdirs ==1.4.4",
     "APScheduler ==3.9.1",