[feat] add favicons to result urls

docs/admin/settings/settings_search.rst

@@ -9,6 +9,7 @@
    search:
      safe_search: 0
      autocomplete: ""
+     favicon_resolver: ""
      default_lang: ""
      ban_time_on_fail: 5
      max_ban_time_on_fail: 120
@@ -41,6 +42,14 @@
   - ``qwant``
   - ``wikipedia``
 
+``favicon_resolver``:
+  Favicon resolver, leave blank to turn off the feature by default.
+
+  - ``allesedv``
+  - ``duckduckgo``
+  - ``google``
+  - ``yandex``
+
 ``default_lang``:
   Default search language - leave blank to detect from browser information or
   use codes from :origin:`searx/languages.py`.

searx/favicon_resolver.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""This module implements functions needed for the favicon resolver.
+
+"""
+# pylint: disable=use-dict-literal
+
+from httpx import HTTPError
+
+from searx import settings
+
+from searx.network import get as http_get, post as http_post
+from searx.exceptions import SearxEngineResponseException
+
+
+def update_kwargs(**kwargs):
+    if 'timeout' not in kwargs:
+        kwargs['timeout'] = settings['outgoing']['request_timeout']
+    kwargs['raise_for_httperror'] = False
+
+
+def get(*args, **kwargs):
+    update_kwargs(**kwargs)
+    return http_get(*args, **kwargs)
+
+
+def post(*args, **kwargs):
+    update_kwargs(**kwargs)
+    return http_post(*args, **kwargs)
+
+
+def allesedv(domain):
+    """Favicon Resolver from allesedv.com"""
+
+    url = 'https://f1.allesedv.com/32/{domain}'
+
+    # will just return a 200 regardless of the favicon existing or not
+    # sometimes will be correct size, sometimes not
+    response = get(url.format(domain=domain))
+
+    # returns image/gif if the favicon does not exist
+    if response.headers['Content-Type'] == 'image/gif':
+        return []
+
+    return response.content
+
+
+def duckduckgo(domain):
+    """Favicon Resolver from duckduckgo.com"""
+
+    url = 'https://icons.duckduckgo.com/ip2/{domain}.ico'
+
+    # will return a 404 if the favicon does not exist and a 200 if it does,
+    response = get(url.format(domain=domain))
+
+    # api will respond with a 32x32 png image
+    if response.status_code == 200:
+        return response.content
+    return []
+
+
+def google(domain):
+    """Favicon Resolver from google.com"""
+
+    url = 'https://www.google.com/s2/favicons?sz=32&domain={domain}'
+
+    # will return a 404 if the favicon does not exist and a 200 if it does,
+    response = get(url.format(domain=domain))
+
+    # api will respond with a 32x32 png image
+    if response.status_code == 200:
+        return response.content
+    return []
+
+
+def yandex(domain):
+    """Favicon Resolver from yandex.com"""
+
+    url = 'https://favicon.yandex.net/favicon/{domain}'
+
+    # will always return 200
+    response = get(url.format(domain=domain))
+
+    # api will respond with a 16x16 png image, if it doesn't exist, it will be a 1x1 png image (70 bytes)
+    if response.status_code == 200:
+        if len(response.content) > 70:
+            return response.content
+    return []
+
+
+backends = {
+    'allesedv': allesedv,
+    'duckduckgo': duckduckgo,
+    'google': google,
+    'yandex': yandex,
+}
+
+
+def search_favicon(backend_name, domain):
+    backend = backends.get(backend_name)
+    if backend is None:
+        return []
+    try:
+        return backend(domain)
+    except (HTTPError, SearxEngineResponseException):
+        return []

searx/preferences.py

@@ -13,7 +13,7 @@ from collections import OrderedDict
 import flask
 import babel
 
-from searx import settings, autocomplete
+from searx import settings, autocomplete, favicon_resolver
 from searx.enginelib import Engine
 from searx.plugins import Plugin
 from searx.locales import LOCALE_NAMES
@@ -406,6 +406,11 @@ class Preferences:
                 locked=is_locked('autocomplete'),
                 choices=list(autocomplete.backends.keys()) + ['']
             ),
+            'favicon_resolver': EnumStringSetting(
+                settings['search']['favicon_resolver'],
+                locked=is_locked('favicon_resolver'),
+                choices=list(favicon_resolver.backends.keys()) + ['']
+            ),
             'image_proxy': BooleanSetting(
                 settings['server']['image_proxy'],
                 locked=is_locked('image_proxy')

searx/settings.yml

@@ -35,6 +35,9 @@ search:
   autocomplete: ""
   # minimun characters to type before autocompleter starts
   autocomplete_min: 4
+  # backend for the favicon near URL in search results.
+  # Available resolvers: "allesedv", "duckduckgo", "google", "yandex" - leave blank to turn it off by default.
+  favicon_resolver: ""
   # Default search language - leave blank to detect from browser information or
   # use codes from 'languages.py'
   default_lang: "auto"

searx/settings_defaults.py

@@ -156,6 +156,7 @@ SCHEMA = {
         'safe_search': SettingsValue((0, 1, 2), 0),
         'autocomplete': SettingsValue(str, ''),
         'autocomplete_min': SettingsValue(int, 4),
+        'favicon_resolver': SettingsValue(str, ''),
         'default_lang': SettingsValue(tuple(SXNG_LOCALE_TAGS + ['']), ''),
         'languages': SettingSublistValue(SXNG_LOCALE_TAGS, SXNG_LOCALE_TAGS),
         'ban_time_on_fail': SettingsValue(numbers.Real, 5),

searx/static/themes/simple/img/empty_favicon.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
+  <path fill="#fff" d="M0 0h24v24H0z"/>
+  <path fill="#58f" d="M11 20.85a.92.92 0 0 1-1.1.93A10 10 0 0 1 2.06 13c-.06-.55.4-1 .95-1h3a1 1 0 0 1 1 1 3 3 0 0 0 3 3 1 1 0 0 1 1 1v3.85Zm6-1.92c0 .77.83 1.23 1.42.74a10 10 0 0 0 2.03-2.32c.39-.61-.09-1.35-.81-1.35H18a1 1 0 0 0-1 1v1.93ZM12 2a10 10 0 0 1 6.65 2.53c.61.55.17 1.47-.65 1.47h-.15A2.85 2.85 0 0 0 15 8.85c0 .33-.18.62-.47.77l-.08.04a1 1 0 0 1-.9 0l-.08-.04a.85.85 0 0 1-.47-.77A2.85 2.85 0 0 0 10.15 6H10a1 1 0 0 1-1-1V3.2c0-.44.28-.84.7-.94C10.45 2.1 11.22 2 12 2Z"/>
+  <path fill="#58f" d="M3.42 10c-.63 0-1.1-.58-.9-1.18.6-1.8 1.7-3.36 3.12-4.53C6.2 3.82 7 4.26 7 5a3 3 0 0 0 3 3h.15c.47 0 .85.38.85.85 0 1.09.61 2.07 1.58 2.56l.08.04a3 3 0 0 0 2.68 0l.08-.04A2.85 2.85 0 0 0 17 8.85c0-.47.38-.85.85-.85h2.66c.4 0 .77.23.9.6a9.98 9.98 0 0 1 .52 4.6.94.94 0 0 1-.95.8H18a3 3 0 0 0-3 3v3.8c0 .44-.28.84-.7.94l-.2.04a.92.92 0 0 1-1.1-.93V17a3 3 0 0 0-3-3 1 1 0 0 1-1-1 3 3 0 0 0-3-3H3.42Z"/>
+</svg>

searx/static/themes/simple/src/less/search.less

@@ -378,3 +378,12 @@ html.no-js #clear_search.hide_if_nojs {
 #categories_container {
   position: relative;
 }
+
+.favicon img {
+  height: 1.8rem;
+  width: 1.8rem;
+  border-radius: 20%;
+  background-color: #ddd;
+  border: 1px solid #ccc;
+  display: flex;
+}

searx/static/themes/simple/src/less/style-ltr.less

@@ -82,4 +82,8 @@
   transform: scale(1, 1);
 }
 
+.favicon {
+  margin: 0 8px 0 0;
+}
+
 @import "style.less";

searx/static/themes/simple/src/less/style-rtl.less

@@ -96,6 +96,10 @@
 
   .result .url_wrapper {
     justify-content: end;
+
+    .favicon {
+      margin: 0 0 0 8px;
+    }
   }
 }
 

searx/static/themes/simple/src/less/style.less

@@ -234,6 +234,7 @@ article[data-vim-selected].category-social {
 
   .url_wrapper {
     display: flex;
+    align-items: center;
     font-size: 1rem;
     color: var(--color-result-url-font);
     flex-wrap: nowrap;

searx/templates/simple/macros.html

@@ -21,9 +21,29 @@
 {% macro result_header(result, favicons, image_proxify) -%}
 <article class="result {% if result['template'] %}result-{{ result.template|replace('.html', '') }}{% else %}result-default{% endif %} {% if result['category'] %}category-{{ result['category'] }}{% endif %}{% for e in result.engines %} {{ e }}{% endfor %}">
   {{- result_open_link(result.url, "url_wrapper") -}}
+  {% if not rtl %}
+    {%- if favicon_resolver != "" %}
+    <div class="favicon">
+      <img
+        alt="{{ result.parsed_url.netloc }}"
+        src="{{ favicon_proxify(result.parsed_url.netloc) }}"
+      >
+    </div>
+    {%- endif -%}
+  {%- endif -%}
   {%- for part in get_pretty_url(result.parsed_url) -%}
   <span class="url_o{{loop.index}}"><span class="url_i{{loop.index}}">{{- part -}}</span></span>
   {%- endfor %}
+  {% if rtl %}
+    {%- if favicon_resolver != "" %}
+    <div class="favicon">
+      <img
+        alt="{{ result.parsed_url.netloc }}"
+        src="{{ favicon_proxify(result.parsed_url.netloc) }}"
+      >
+    </div>
+    {%- endif -%}
+  {%- endif -%}
   {{- result_close_link() -}}
   {%- if result.thumbnail %}{{ result_open_link(result.url) }}<img class="thumbnail" src="{{ image_proxify(result.thumbnail) }}" title="{{ result.title|striptags }}" loading="lazy">{{ result_close_link() }}{% endif -%}
   <h3>{{ result_link(result.url, result.title|safe) }}</h3>

searx/templates/simple/preferences.html

@@ -173,6 +173,9 @@
     {%- if 'autocomplete' not in locked_preferences -%}
       {%- include 'simple/preferences/autocomplete.html' -%}
     {%- endif -%}
+    {%- if 'favicon' not in locked_preferences -%}
+      {%- include 'simple/preferences/favicon.html' -%}
+    {%- endif -%}
     {% if 'safesearch' not in locked_preferences %}
       {%- include 'simple/preferences/safesearch.html' -%}
     {%- endif -%}

searx/templates/simple/preferences/favicon.html

@@ -0,0 +1,17 @@
+<fieldset>{{- '' -}}
+  <legend id="pref_favicon_resolver">{{- _('Favicon Resolver') -}}</legend>{{- '' -}}
+  <div class="value">{{- '' -}}
+    <select name="favicon_resolver" aria-labelledby="pref_favicon_resolver">{{- '' -}}
+      <option value=""> - </option>
+      {%- for backend in favicon_backends -%}
+        <option value="{{ backend }}"
+          {%- if backend == favicon_resolver %} selected="selected" {%- endif -%}>
+          {{- backend -}}
+        </option>
+      {%- endfor -%}
+    </select>{{- '' -}}
+  </div>{{- '' -}}
+  <div class="description">
+    {{- _('Display favicons near search results') -}}
+  </div>{{- '' -}}
+</fieldset>{{- '' -}}

searx/webapp.py

@@ -123,6 +123,7 @@ from searx.locales import (
 
 # renaming names from searx imports ...
 from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
+from searx.favicon_resolver import search_favicon, backends as favicon_backends
 from searx.redisdb import initialize as redis_initialize
 from searx.sxng_locales import sxng_locales
 from searx.search import SearchWithPlugins, initialize as search_initialize
@@ -297,6 +298,24 @@ def morty_proxify(url: str):
     return '{0}?{1}'.format(settings['result_proxy']['url'], urlencode(url_params))
 
 
+def favicon_proxify(url: str):
+    # url is a FQDN (e.g. example.com, en.wikipedia.org)
+
+    resolver = request.preferences.get_value('favicon_resolver')
+
+    # if resolver is empty, just return nothing
+    if not resolver:
+        return ""
+
+    # check resolver is valid
+    if resolver not in favicon_backends:
+        return ""
+
+    h = new_hmac(settings['server']['secret_key'], url.encode())
+
+    return '{0}?{1}'.format(url_for('favicon_proxy'), urlencode(dict(q=url.encode(), h=h)))
+
+
 def image_proxify(url: str):
 
     if url.startswith('//'):
@@ -358,6 +377,7 @@ def get_client_settings():
     return {
         'autocomplete_provider': req_pref.get_value('autocomplete'),
         'autocomplete_min': get_setting('search.autocomplete_min'),
+        'favicon_resolver': req_pref.get_value('favicon_resolver'),
         'http_method': req_pref.get_value('method'),
         'infinite_scroll': req_pref.get_value('infinite_scroll'),
         'translations': get_translations(),
@@ -388,6 +408,7 @@ def render(template_name: str, **kwargs):
     # values from the preferences
     kwargs['preferences'] = request.preferences
     kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
+    kwargs['favicon_resolver'] = request.preferences.get_value('favicon_resolver')
     kwargs['infinite_scroll'] = request.preferences.get_value('infinite_scroll')
     kwargs['search_on_category_select'] = request.preferences.get_value('search_on_category_select')
     kwargs['hotkeys'] = request.preferences.get_value('hotkeys')
@@ -431,6 +452,7 @@ def render(template_name: str, **kwargs):
     # helpers to create links to other pages
     kwargs['url_for'] = custom_url_for  # override url_for function in templates
     kwargs['image_proxify'] = image_proxify
+    kwargs['favicon_proxify'] = favicon_proxify
     kwargs['proxify'] = morty_proxify if settings['result_proxy']['url'] is not None else None
     kwargs['proxify_results'] = settings['result_proxy']['proxify_results']
     kwargs['cache_url'] = settings['ui']['cache_url']
@@ -873,6 +895,42 @@ def autocompleter():
     return Response(suggestions, mimetype=mimetype)
 
 
+@app.route('/favicon', methods=['GET'])
+def favicon_proxy():
+    """Return proxied favicon results"""
+    url = request.args.get('q')
+
+    # malformed request
+    if not url:
+        return '', 400
+
+    # malformed request / does not have authorisation
+    if not is_hmac_of(settings['server']['secret_key'], url.encode(), request.args.get('h', '')):
+        return '', 400
+
+    resolver = request.preferences.get_value('favicon_resolver')
+
+    # check if the favicon resolver is valid
+    if not resolver or resolver not in favicon_backends:
+        return '', 400
+
+    # parse query
+    raw_text_query = RawTextQuery(url, [])
+
+    resp = search_favicon(resolver, raw_text_query)
+
+    # return 404 if the favicon is not found
+    if not resp:
+        theme = request.preferences.get_value("theme")
+        # return favicon from /static/themes/simple/img/empty_favicon.svg
+        # we can't rely on an onerror event in the img tag to display a default favicon as this violates the CSP.
+        # using redirect to save network bandwidth (user will have this location cached).
+        return redirect(url_for('static', filename='themes/' + theme + '/img/empty_favicon.svg'))
+
+    # will always return a PNG image
+    return Response(resp, mimetype='image/png')
+
+
 @app.route('/preferences', methods=['GET', 'POST'])
 def preferences():
     """Render preferences page && save user preferences"""
@@ -1020,6 +1078,7 @@ def preferences():
         ],
         disabled_engines = disabled_engines,
         autocomplete_backends = autocomplete_backends,
+        favicon_backends = favicon_backends,
         shortcuts = {y: x for x, y in engine_shortcuts.items()},
         themes = themes,
         plugins = plugins,

tests/unit/settings/user_settings.yml

@@ -5,6 +5,7 @@ general:
 search:
   safe_search: 0
   autocomplete: ""
+  favicon_resolver: ""
   default_lang: ""
   ban_time_on_fail: 5
   max_ban_time_on_fail: 120