Merge f02d9b523d into 11fdc2f56a

Unfortunately, azlyrics has a bot blocker that makes it impossible to implement
an XPath engine for it [1][2].

[1] https://github.com/searxng/searxng/pull/3302#issuecomment-2013529271
[2] https://github.com/searxng/searxng/issues/3280

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/duckduckgo_definitions.py

@@ -238,7 +238,10 @@ def unit_to_str(unit):
     for prefix in WIKIDATA_PREFIX:
         if unit.startswith(prefix):
             wikidata_entity = unit[len(prefix) :]
-            return WIKIDATA_UNITS.get(wikidata_entity, unit)
+            real_unit = WIKIDATA_UNITS.get(wikidata_entity)
+            if real_unit is None:
+                return unit
+            return real_unit['symbol']
     return unit
 
 

searx/engines/wikicommons.py

@@ -3,6 +3,8 @@
 
 """
 
+import datetime
+
 from urllib.parse import urlencode
 
 # about
@@ -14,6 +16,8 @@ about = {
     "require_api_key": False,
     "results": 'JSON',
 }
+categories = ['images']
+search_type = 'images'
 
 base_url = "https://commons.wikimedia.org"
 search_prefix = (
@@ -29,17 +33,29 @@ search_prefix = (
 paging = True
 number_of_results = 10
 
+search_types = {
+    'images': 'bitmap|drawing',
+    'videos': 'video',
+    'audio': 'audio',
+    'files': 'multimedia|office|archive|3d',
+}
+
 
 def request(query, params):
     language = 'en'
     if params['language'] != 'all':
         language = params['language'].split('-')[0]
 
+    if search_type not in search_types:
+        raise ValueError(f"Unsupported search type: {search_type}")
+
+    filetype = search_types[search_type]
+
     args = {
         'uselang': language,
         'gsrlimit': number_of_results,
         'gsroffset': number_of_results * (params["pageno"] - 1),
-        'gsrsearch': "filetype:bitmap|drawing " + query,
+        'gsrsearch': f"filetype:{filetype} {query}",
     }
 
     params["url"] = f"{base_url}/w/api.php{search_prefix}&{urlencode(args, safe=':|')}"
@@ -52,7 +68,6 @@ def response(resp):
 
     if not json.get("query", {}).get("pages"):
         return results
-
     for item in json["query"]["pages"].values():
         imageinfo = item["imageinfo"][0]
         title = item["title"].replace("File:", "").rsplit('.', 1)[0]
@@ -60,11 +75,28 @@ def response(resp):
             'url': imageinfo["descriptionurl"],
             'title': title,
             'content': item["snippet"],
-            'img_src': imageinfo["url"],
-            'resolution': f'{imageinfo["width"]} x {imageinfo["height"]}',
-            'thumbnail_src': imageinfo["thumburl"],
-            'template': 'images.html',
         }
+
+        if search_type == "images":
+            result['template'] = 'images.html'
+            result['img_src'] = imageinfo["url"]
+            result['thumbnail_src'] = imageinfo["thumburl"]
+            result['resolution'] = f'{imageinfo["width"]} x {imageinfo["height"]}'
+        else:
+            result['thumbnail'] = imageinfo["thumburl"]
+
+        if search_type == "videos":
+            result['template'] = 'videos.html'
+            if imageinfo.get('duration'):
+                result['length'] = datetime.timedelta(seconds=int(imageinfo['duration']))
+            result['iframe_src'] = imageinfo['url']
+        elif search_type == "files":
+            result['template'] = 'files.html'
+            result['metadata'] = imageinfo['mime']
+            result['size'] = imageinfo['size']
+        elif search_type == "audio":
+            result['iframe_src'] = imageinfo['url']
+
         results.append(result)
 
     return results

searx/plugins/unit_converter.py

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Calculate mathematical expressions using ack#eval
+"""
+
+from flask_babel import gettext
+
+from searx.data import WIKIDATA_UNITS
+
+name = "Unit converter plugin"
+description = gettext("Convert between units")
+default_on = True
+
+CONVERT_KEYWORDS = ["in", "to", "as"]
+
+
+def _convert(from_value, source_si_factor, target_si_factor):
+    return from_value * source_si_factor / target_si_factor
+
+
+def _parse_text_and_convert(search, splitted_query):
+    if len(splitted_query) != 2 or splitted_query[0].strip() == "" or splitted_query[1].strip() == "":
+        return
+
+    from_value = ""
+    from_unit_key = ""
+
+    # only parse digits as value that belong together
+    read_alpha = False
+    for c in splitted_query[0]:
+        if not read_alpha and (c in ("-", ".") or str.isdigit(c)):
+            from_value += c
+            read_alpha = True
+        elif c != " ":
+            from_unit_key += c
+
+    to_unit_key = splitted_query[1].strip()
+
+    from_unit = None
+    to_unit = None
+
+    for unit in WIKIDATA_UNITS.values():
+        if unit['symbol'] == from_unit_key:
+            from_unit = unit
+
+        if unit['symbol'] == to_unit_key:
+            to_unit = unit
+
+        if from_unit and to_unit:
+            break
+
+    if from_unit is None or to_unit is None or to_unit.get('si_name') != from_unit.get('si_name'):
+        return
+
+    result = _convert(float(from_value), from_unit['to_si_factor'], to_unit['to_si_factor'])
+    search.result_container.answers['conversion'] = {'answer': f"{result:g} {to_unit['symbol']}"}
+
+
+def post_search(_request, search):
+    # only convert between units on the first page
+    if search.search_query.pageno > 1:
+        return True
+
+    query = search.search_query.query
+    query_parts = query.split(" ")
+
+    if len(query_parts) < 3:
+        return True
+
+    for query_part in query_parts:
+        for keyword in CONVERT_KEYWORDS:
+            if query_part == keyword:
+                keyword_split = query.split(keyword, 1)
+                _parse_text_and_convert(search, keyword_split)
+                return True
+
+    return True

searx/settings.yml

@@ -1082,25 +1082,6 @@ engines:
       require_api_key: false
       results: HTML
 
-  - name: azlyrics
-    shortcut: lyrics
-    engine: xpath
-    timeout: 4.0
-    disabled: true
-    categories: [music, lyrics]
-    paging: true
-    search_url: https://search.azlyrics.com/search.php?q={query}&w=lyrics&p={pageno}
-    url_xpath: //td[@class="text-left visitedlyr"]/a/@href
-    title_xpath: //span/b/text()
-    content_xpath: //td[@class="text-left visitedlyr"]/a/small
-    about:
-      website: https://azlyrics.com
-      wikidata_id: Q66372542
-      official_api_documentation:
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-
   - name: mastodon users
     engine: mastodon
     mastodon_type: accounts
@@ -1907,6 +1888,28 @@ engines:
     engine: wikicommons
     shortcut: wc
     categories: images
+    search_type: images
+    number_of_results: 10
+
+  - name: wikicommons.videos
+    engine: wikicommons
+    shortcut: wcv
+    categories: videos
+    search_type: videos
+    number_of_results: 10
+
+  - name: wikicommons.audio
+    engine: wikicommons
+    shortcut: wca
+    categories: music
+    search_type: audio
+    number_of_results: 10
+
+  - name: wikicommons.files
+    engine: wikicommons
+    shortcut: wcf
+    categories: files
+    search_type: files
     number_of_results: 10
 
   - name: wolframalpha

searx/static/themes/simple/src/js/main/preferences.js

@@ -27,6 +27,22 @@
       searxng.on(el, 'mouseenter', load_engine_descriptions);
     }
 
+    const enableAllEngines = d.querySelectorAll(".enable-all-engines");
+    const disableAllEngines = d.querySelectorAll(".disable-all-engines");
+    const engineToggles = d.querySelectorAll('tbody input[type=checkbox][class~=checkbox-onoff]');
+    const toggleEngines = (enable) => {
+      for (const el of engineToggles) {
+        // check if element visible, so that only engines of the current category are modified
+        if (el.offsetParent !== null) el.checked = !enable;
+      }
+    };
+    for (const el of enableAllEngines) {
+      searxng.on(el, 'click', () => toggleEngines(true));
+    }
+    for (const el of disableAllEngines) {
+      searxng.on(el, 'click', () => toggleEngines(false));
+    }
+
     const copyHashButton = d.querySelector("#copy-hash");
     searxng.on(copyHashButton, 'click', (e) => {
       e.preventDefault();

searx/static/themes/simple/src/less/preferences.less

@@ -185,6 +185,11 @@ table {
     }
   }
 
+  #toggle-all-engines-container {
+    width: max-content;
+    margin-left: auto;
+  }
+
   div.selectable_url {
     pre {
       width: 100%;

searx/templates/simple/preferences/engines.html

@@ -10,6 +10,12 @@
       {{- ' ' -}}<a href="{{ url_for('info', pagename='search-syntax') }}">&#9432;</a>
     </p>
   {%- endif -%}
+
+  <div class="hide_if_nojs" id="toggle-all-engines-container">
+    <button type="button" class="button enable-all-engines">{{ _("Enable all") }}</button>
+    <button type="button" class="button disable-all-engines">{{ _("Disable all") }}</button>
+  </div>
+
   <div class="scrollx">{{- '' -}}
     <table class="striped table_engines">{{- '' -}}
 

searxng_extra/update/update_wikidata_units.py

@@ -29,31 +29,46 @@ set_loggers(wikidata, 'wikidata')
 # * https://www.wikidata.org/wiki/Help:Ranking
 # * https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format ("Statement representation" section)
 # * https://w.wiki/32BT
+# * https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates#Quantities
 #   see the result for https://www.wikidata.org/wiki/Q11582
 #   there are multiple symbols the same rank
 SARQL_REQUEST = """
-SELECT DISTINCT ?item ?symbol
+SELECT DISTINCT ?item ?symbol ?tosi ?tosiUnit
 WHERE
 {
   ?item wdt:P31/wdt:P279 wd:Q47574 .
   ?item p:P5061 ?symbolP .
   ?symbolP ps:P5061 ?symbol ;
            wikibase:rank ?rank .
+  OPTIONAL {
+    ?item p:P2370 ?tosistmt .
+    ?tosistmt psv:P2370 ?tosinode .
+    ?tosinode wikibase:quantityAmount ?tosi .
+    ?tosinode wikibase:quantityUnit ?tosiUnit .
+  }
   FILTER(LANG(?symbol) = "en").
 }
 ORDER BY ?item DESC(?rank) ?symbol
 """
 
+_wikidata_url = "https://www.wikidata.org/entity/"
+
 
 def get_data():
     results = collections.OrderedDict()
     response = wikidata.send_wikidata_query(SARQL_REQUEST)
     for unit in response['results']['bindings']:
-        name = unit['item']['value'].replace('http://www.wikidata.org/entity/', '')
-        unit = unit['symbol']['value']
+        name = unit['item']['value'].replace(_wikidata_url, '')
+        symbol = unit['symbol']['value']
+        si_name = unit.get('tosiUnit', {}).get('value', '').replace(_wikidata_url, '')
+        to_si_factor = unit.get('tosi', {}).get('value', '')
         if name not in results:
             # ignore duplicate: always use the first one
-            results[name] = unit
+            results[name] = {
+                'symbol': symbol,
+                'si_name': si_name if si_name else None,
+                'to_si_factor': float(to_si_factor) if to_si_factor else None,
+            }
     return results