improve "search existing issues from github" link

Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.2.6 to 7.3.7.
- [Release notes](https://github.com/sphinx-doc/sphinx/releases)
- [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst)
- [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.6...v7.3.7)

---
updated-dependencies:
- dependency-name: sphinx
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

.editorconfig

@@ -16,6 +16,9 @@ max_line_length = 119
 [*.html]
 indent_size = 4
 
+[*.js]
+indent_size = 2
+
 [*.json]
 indent_size = 4
 insert_final_newline = ignore

.github/dependabot.yml

@@ -1,4 +1,4 @@
-# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 version: 2
 updates:
   - package-ecosystem: "pip"
@@ -8,6 +8,9 @@ updates:
       day: "friday"
     open-pull-requests-limit: 5
     target-branch: "master"
+    commit-message:
+      prefix: "[upd] pypi:"
+
   - package-ecosystem: "npm"
     directory: "/searx/static/themes/simple"
     schedule:
@@ -15,3 +18,5 @@ updates:
       day: "friday"
     open-pull-requests-limit: 5
     target-branch: "master"
+    commit-message:
+      prefix: "[upd] npm:"

.github/workflows/checker.yml

@@ -0,0 +1,31 @@
+name: "Checker"
+on:
+  schedule:
+    - cron: "0 4 * * 5"
+  workflow_dispatch:
+
+jobs:
+  checker:
+    name: Checker
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Ubuntu packages
+        run: |
+          sudo ./utils/searxng.sh install packages
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          architecture: 'x64'
+
+      - name: Install Python dependencies
+        run: |
+          make V=1 install
+
+      - name: Checker
+        run: |
+          make search.checker

.github/workflows/data-update.yml

@@ -48,16 +48,16 @@ jobs:
         id: cpr
         uses: peter-evans/create-pull-request@v3
         with:
-          commit-message: Update searx.data - ${{ matrix.fetch }}
+          commit-message: '[data] update searx.data - ${{ matrix.fetch }}'
           committer: searxng-bot <noreply@github.com>
           author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
           signoff: false
           branch: update_data_${{ matrix.fetch }}
           delete-branch: true
           draft: false
-          title: 'Update searx.data - ${{ matrix.fetch }}'
+          title: '[data] update searx.data - ${{ matrix.fetch }}'
           body: |
-            Update searx.data - ${{ matrix.fetch }}
+            update searx.data - ${{ matrix.fetch }}
           labels: |
             data
 

.github/workflows/integration.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python-version:  ["3.8", "3.9", "3.10", "3.11"]
+        python-version:  ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -120,7 +120,7 @@ jobs:
         FOLDER: dist/docs
         CLEAN: true # Automatically remove deleted files from the deploy branch
         SINGLE_COMMIT: True
-        COMMIT_MESSAGE: build from commit ${{ github.sha }}
+        COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
 
   babel:
     name: Update translations branch
@@ -163,7 +163,6 @@ jobs:
     - name: Update transations
       id: update
       run: |
-        git restore utils/brand.env
         make V=1 weblate.push.translations
 
   dockers:

.github/workflows/translations-update.yml

@@ -45,15 +45,15 @@ jobs:
       uses: peter-evans/create-pull-request@v3
       with:
         token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
-        commit-message: Update translations
+        commit-message: '[l10n] update translations from Weblate'
         committer: searxng-bot <searxng-bot@users.noreply.github.com>
         author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
         signoff: false
         branch: translations_update
         delete-branch: true
         draft: false
-        title: 'Update translations'
+        title: '[l10n] update translations from Weblate'
         body: |
-          Update translations
+          update translations from Weblate
         labels: |
           translation

.gitignore

@@ -23,3 +23,6 @@ gh-pages/
 .idea/
 
 searx/version_frozen.py
+.dir-locals.el
+
+.python-version

.nvmrc

@@ -1 +1 @@
-v16.20.2
+v20.10

.pylintrc

@@ -1,4 +1,4 @@
-# -*- coding: utf-8; mode: conf -*-
+# -*- coding: utf-8; mode: conf-unix -*-
 # lint Python modules using external checkers.
 #
 # This is the main checker controlling the other ones and the reports
@@ -27,7 +27,7 @@ ignore-patterns=
 #init-hook=
 
 # Use multiple processes to speed up Pylint.
-jobs=1
+jobs=0
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.

.tool-versions

@@ -0,0 +1,2 @@
+python 3.12.0
+shellcheck 0.9.0

AUTHORS.rst

@@ -169,4 +169,6 @@ features or generally made searx better:
 - @llmII
 - @blob42 `<https://blob42.xyz>`_
 - Paolo Basso `<https://github.com/paolobasso99>`
-- Bernie Huang `<https://github.com/BernieHuang2008>`
+- Bernie Huang `<https://github.com/BernieHuang2008>`
+- Austin Olacsi `<https://github.com/Austin-Olacsi>`
+- @micsthepick

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 ## Resources in the documentation
 
-* [Development quickstart](https://docs.searxng.org/dev/contribution_guide.html)
+* [Development quickstart](https://docs.searxng.org/dev/quickstart.html)
 * [Contribution guide](https://docs.searxng.org/dev/contribution_guide.html)
 
 ## Submitting PRs

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.19
 ENTRYPOINT ["/sbin/tini","--","/usr/local/searxng/dockerfiles/docker-entrypoint.sh"]
 EXPOSE 8080
 VOLUME /etc/searxng
@@ -15,7 +15,9 @@ ENV INSTANCE_NAME=searxng \
     MORTY_KEY= \
     MORTY_URL= \
     SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml \
-    UWSGI_SETTINGS_PATH=/etc/searxng/uwsgi.ini
+    UWSGI_SETTINGS_PATH=/etc/searxng/uwsgi.ini \
+    UWSGI_WORKERS=%k \
+    UWSGI_THREADS=4
 
 WORKDIR /usr/local/searxng
 
@@ -43,7 +45,7 @@ RUN apk add --no-cache -t build-dependencies \
     uwsgi \
     uwsgi-python3 \
     brotli \
- && pip3 install --no-cache -r requirements.txt \
+ && pip3 install --break-system-packages --no-cache -r requirements.txt \
  && apk del build-dependencies \
  && rm -rf /root/.cache
 

Makefile

@@ -44,10 +44,10 @@ lxc.clean:
 
 PHONY += search.checker search.checker.%
 search.checker: install
-	$(Q)./manage pyenv.cmd searx-checker -v
+	$(Q)./manage pyenv.cmd searxng-checker -v
 
 search.checker.%: install
-	$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
+	$(Q)./manage pyenv.cmd searxng-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
 
 PHONY += test ci.test test.shell
 ci.test: test.yamllint test.black test.pyright test.pylint test.unit test.robot test.rst test.pybabel
@@ -56,7 +56,7 @@ test.shell:
 	$(Q)shellcheck -x -s dash \
 		dockerfiles/docker-entrypoint.sh
 	$(Q)shellcheck -x -s bash \
-		utils/brand.env \
+		utils/brand.sh \
 		$(MTOOLS) \
 		utils/lib.sh \
 		utils/lib_sxng*.sh \
@@ -74,16 +74,14 @@ test.shell:
 
 # wrap ./manage script
 
-MANAGE += buildenv
 MANAGE += weblate.translations.commit weblate.push.translations
-MANAGE += data.all data.traits data.useragents
+MANAGE += data.all data.traits data.useragents data.locales
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += docker.build docker.push docker.buildx
 MANAGE += gecko.driver
 MANAGE += node.env node.env.dev node.clean
 MANAGE += py.build py.clean
 MANAGE += pyenv pyenv.install pyenv.uninstall
-MANAGE += pypi.upload pypi.upload.test
 MANAGE += format.python
 MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean
 MANAGE += themes.all themes.simple themes.simple.test pygments.less

README.rst

@@ -121,9 +121,9 @@ You can contribute from your browser using `GitHub Codespaces`_:
 - Click on the ``Codespaces`` tab instead of ``Local``
 - Click on ``Create codespace on master``
 - VSCode is going to start in the browser
-- Wait for ``git pull && make install`` to appears and then to disapear
+- Wait for ``git pull && make install`` to appear and then disappear
 - You have `120 hours per month`_ (see also your `list of existing Codespaces`_)
-- You can start SearXNG using ``make run`` in the terminal or by pressing ``Ctrl+Shift+B``.
+- You can start SearXNG using ``make run`` in the terminal or by pressing ``Ctrl+Shift+B``
 
 .. _GitHub Codespaces: https://docs.github.com/en/codespaces/overview
 .. _120 hours per month: https://github.com/settings/billing

dockerfiles/uwsgi.ini

@@ -4,8 +4,12 @@ uid = searxng
 gid = searxng
 
 # Number of workers (usually CPU count)
-workers = %k
-threads = 4
+# default value: %k (= number of CPU core, see Dockerfile)
+workers = $(UWSGI_WORKERS)
+
+# Number of threads per worker
+# default value: 4 (see Dockerfile)
+threads = $(UWSGI_THREADS)
 
 # The right granted on the created socket
 chmod-socket = 666
@@ -38,6 +42,10 @@ buffer-size = 8192
 # See https://github.com/searx/searx-docker/issues/24
 add-header = Connection: close
 
+# Follow SIGTERM convention
+# See https://github.com/searxng/searxng/issues/3427
+die-on-term
+
 # uwsgi serves the static files
 static-map = /static=/usr/local/searxng/searx/static
 # expires set to one day

docs/admin/index.rst

@@ -15,7 +15,7 @@ Administrator documentation
    installation-apache
    update-searxng
    answer-captcha
-   searx.botdetection
+   searx.limiter
    api
    architecture
    plugins

docs/admin/installation-docker.rst

@@ -92,6 +92,9 @@ instance using `docker run <https://docs.docker.com/engine/reference/run/>`_:
                 searxng/searxng
    2f998.... # container's ID
 
+The environment variables UWSGI_WORKERS and UWSGI_THREADS overwrite the default
+number of UWSGI processes and UWSGI threads specified in `/etc/searxng/uwsgi.ini`.
+
 Open your WEB browser and visit the URL:
 
 .. code:: sh

docs/admin/searx.limiter.rst

@@ -0,0 +1,17 @@
+.. _limiter:
+
+=======
+Limiter
+=======
+
+.. sidebar:: info
+
+   The limiter requires a :ref:`Redis <settings redis>` database.
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.limiter
+   :members:

docs/admin/settings/settings_server.rst

@@ -12,35 +12,38 @@
        bind_address: "127.0.0.1"
        secret_key: "ultrasecretkey"           # change this!
        limiter: false
+       public_instance: false
        image_proxy: false
        default_http_headers:
          X-Content-Type-Options : nosniff
-         X-XSS-Protection : 1; mode=block
          X-Download-Options : noopen
          X-Robots-Tag : noindex, nofollow
          Referrer-Policy : no-referrer
 
-
-``base_url`` : ``$SEARXNG_URL`` :ref:`buildenv <make buildenv>`
+``base_url`` : ``$SEARXNG_URL``
   The base URL where SearXNG is deployed.  Used to create correct inbound links.
-  If you change the value, don't forget to rebuild instance's environment
-  (:ref:`utils/brand.env <make buildenv>`)
 
-``port`` & ``bind_address``: ``$SEARXNG_PORT`` & ``$SEARXNG_BIND_ADDRESS`` :ref:`buildenv <make buildenv>`
+``port`` & ``bind_address``: ``$SEARXNG_PORT`` & ``$SEARXNG_BIND_ADDRESS``
   Port number and *bind address* of the SearXNG web application if you run it
   directly using ``python searx/webapp.py``.  Doesn't apply to a SearXNG
-  services running behind a proxy and using socket communications.  If you
-  change the value, don't forget to rebuild instance's environment
-  (:ref:`utils/brand.env <make buildenv>`)
+  services running behind a proxy and using socket communications.
 
 ``secret_key`` : ``$SEARXNG_SECRET``
   Used for cryptography purpose.
 
-.. _limiter:
-
 ``limiter`` :
   Rate limit the number of request on the instance, block some bots.  The
-  :ref:`limiter src` requires a :ref:`settings redis` database.
+  :ref:`limiter` requires a :ref:`settings redis` database.
+
+.. _public_instance:
+
+``public_instance`` :
+
+  Setting that allows to enable features specifically for public instances (not
+  needed for local usage).  By set to ``true`` the following features are
+  activated:
+
+  - :py:obj:`searx.botdetection.link_token` in the :ref:`limiter`
 
 .. _image_proxy:
 

docs/admin/settings/settings_ui.rst

@@ -20,6 +20,7 @@
      theme_args:
        simple_style: auto
      search_on_category_select: true
+     hotkeys: default
 
 .. _static_use_hash:
 
@@ -64,3 +65,6 @@
 
 ``search_on_category_select``:
   Perform search immediately if a category selected. Disable to select multiple categories.
+
+``hotkeys``:
+  Hotkeys to use in the search interface: ``default``, ``vim`` (Vim-like).

docs/conf.py

@@ -1,7 +1,7 @@
-# -*- coding: utf-8 -*-
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import  sys, os
+from pathlib import Path
 from pallets_sphinx_themes import ProjectLink
 
 from searx import get_setting
@@ -13,7 +13,6 @@ project = 'SearXNG'
 copyright = 'SearXNG team'
 author = 'SearXNG team'
 release, version = VERSION_STRING, VERSION_STRING
-
 SEARXNG_URL = get_setting('server.base_url') or 'https://example.org/searxng'
 ISSUE_URL = get_setting('brand.issue_url')
 DOCS_URL = get_setting('brand.docs_url')
@@ -22,6 +21,9 @@ PRIVACYPOLICY_URL = get_setting('general.privacypolicy_url')
 CONTACT_URL = get_setting('general.contact_url')
 WIKI_URL = get_setting('brand.wiki_url')
 
+SOURCEDIR = Path(__file__).parent.parent / "searx"
+os.environ['SOURCEDIR'] = str(SOURCEDIR)
+
 # hint: sphinx.ext.viewcode won't highlight when 'highlight_language' [1] is set
 #       to string 'none' [2]
 #

docs/dev/engines/engine_overview.rst

@@ -271,22 +271,54 @@ type.
 ``images``
 ----------
 
-.. table::  Parameter of the **images** media type:
+.. list-table:: Parameter of the **images** media type
+   :header-rows: 2
    :width: 100%
 
-   ========================= =====================================================
-   result-parameter          information
-   ------------------------- -----------------------------------------------------
-   template                  is set to ``images.html``
-   ========================= =====================================================
-   url                       string, url to the result site
-   title                     string, title of the result *(partly implemented)*
-   content                   *(partly implemented)*
-   publishedDate             :py:class:`datetime.datetime`,
-                             time of publish *(partly implemented)*
-   img\_src                  string, url to the result image
-   thumbnail\_src            string, url to a small-preview image
-   ========================= =====================================================
+   * - result-parameter
+     - Python type
+     - information
+
+   * - template
+     - :py:class:`str`
+     - is set to ``images.html``
+
+   * - url
+     - :py:class:`str`
+     - url to the result site
+
+   * - title
+     - :py:class:`str`
+     - title of the result
+
+   * - content
+     - :py:class:`str`
+     - description of the image
+
+   * - publishedDate
+     - :py:class:`datetime <datetime.datetime>`
+     - time of publish
+
+   * - img_src
+     - :py:class:`str`
+     - url to the result image
+
+   * - thumbnail_src
+     - :py:class:`str`
+     - url to a small-preview image
+
+   * - resolution
+     - :py:class:`str`
+     - the resolution of the image (e.g. ``1920 x 1080`` pixel)
+
+   * - img_format
+     - :py:class:`str`
+     - the format of the image (e.g. ``png``)
+
+   * - filesize
+     - :py:class:`str`
+     - size of bytes in :py:obj:`human readable <searx.humanize_bytes>` notation
+       (e.g. ``MB`` for 1024 \* 1024 Bytes filesize).
 
 
 .. _template videos:
@@ -466,3 +498,68 @@ type.
      - :py:class:`str`
      - URL to full article, HTML version
 
+
+.. _template packages:
+
+``packages``
+------------
+
+.. list-table:: Parameter of the **packages** media type
+   :header-rows: 2
+   :width: 100%
+
+   * - result-parameter
+     - Python type
+     - information
+
+   * - template
+     - :py:class:`str`
+     - is set to ``packages.html``
+
+   * - title
+     - :py:class:`str`
+     - title of the result
+
+   * - content
+     - :py:class:`str`
+     - abstract
+
+   * - package_name
+     - :py:class:`str`
+     - the name of the package
+
+   * - version
+     - :py:class:`str`
+     - the current version of the package
+
+   * - maintainer
+     - :py:class:`str`
+     - the maintainer or author of the project
+
+   * - publishedDate
+     - :py:class:`datetime <datetime.datetime>`
+     - date of latest update or release
+
+   * - tags
+     - :py:class:`List <list>`\ [\ :py:class:`str`\ ]
+     - free tag list
+
+   * - popularity
+     - :py:class:`str`
+     - the popularity of the package, e.g. rating or download count
+
+   * - license_name
+     - :py:class:`str`
+     - the name of the license
+
+   * - license_url
+     - :py:class:`str`
+     - the web location of a license copy
+
+   * - homepage
+     - :py:class:`str`
+     - the url of the project's homepage
+
+   * - source_code_url
+     - :py:class:`str`
+     - the location of the project's source code

docs/dev/engines/online/bpb.rst

@@ -0,0 +1,13 @@
+.. _bpb engine:
+
+===
+Bpb
+===
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.bpb
+  :members:

docs/dev/engines/online/duckduckgo.rst

@@ -12,7 +12,7 @@ DuckDuckGo Engines
 .. automodule:: searx.engines.duckduckgo
    :members:
 
-.. automodule:: searx.engines.duckduckgo_images
+.. automodule:: searx.engines.duckduckgo_extra
    :members:
 
 .. automodule:: searx.engines.duckduckgo_definitions

docs/dev/engines/online/mastodon.rst

@@ -0,0 +1,13 @@
+.. _mastodon engine:
+
+========
+Mastodon
+========
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.mastodon
+  :members:

docs/dev/engines/online/mrs.rst

@@ -0,0 +1,13 @@
+.. _mrs engine:
+
+=========================
+Matrix Rooms Search (MRS)
+=========================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.mrs
+   :members:

docs/dev/engines/online/mullvad_leta.rst

@@ -0,0 +1,13 @@
+.. _voidlinux mullvad_leta:
+
+============
+Mullvad-Leta
+============
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.mullvad_leta
+   :members:

docs/dev/engines/online/presearch.rst

@@ -0,0 +1,13 @@
+.. _engine presearch:
+
+================
+Presearch Engine
+================
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.presearch
+   :members:

docs/dev/engines/online/void.rst

@@ -0,0 +1,13 @@
+.. _voidlinux engine:
+
+==========================
+Void Linux binary packages
+==========================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.voidlinux
+  :members:

docs/dev/index.rst

@@ -6,6 +6,7 @@ Developer documentation
    :maxdepth: 2
 
    quickstart
+   rtm_asdf
    contribution_guide
    engines/index
    search_api

docs/dev/makefile.rst

@@ -61,13 +61,9 @@ working tree and release a ``make install`` to get a virtualenv with a
    $ make install
    PYENV     [virtualenv] installing ./requirements*.txt into local/py3
    ...
-   PYENV     OK
    PYENV     [install] pip install -e 'searx[test]'
    ...
-   Successfully installed argparse-1.4.0 searx
-   BUILDENV  INFO:searx:load the default settings from ./searx/settings.yml
-   BUILDENV  INFO:searx:Initialisation done
-   BUILDENV  build utils/brand.env
+   Successfully installed searxng-2023.7.19+a446dea1b
 
 If you release ``make install`` multiple times the installation will only
 rebuild if the sha256 sum of the *requirement files* fails.  With other words:
@@ -82,13 +78,9 @@ the check fails if you edit the requirements listed in
    ...
    PYENV     [virtualenv] installing ./requirements*.txt into local/py3
    ...
-   PYENV     OK
    PYENV     [install] pip install -e 'searx[test]'
    ...
-   Successfully installed argparse-1.4.0 searx
-   BUILDENV  INFO:searx:load the default settings from ./searx/settings.yml
-   BUILDENV  INFO:searx:Initialisation done
-   BUILDENV  build utils/brand.env
+   Successfully installed searxng-2023.7.19+a446dea1b
 
 .. sidebar:: drop environment
 
@@ -98,67 +90,6 @@ the check fails if you edit the requirements listed in
 If you think, something goes wrong with your ./local environment or you change
 the :origin:`setup.py` file, you have to call :ref:`make clean`.
 
-.. _make buildenv:
-
-``make buildenv``
-=================
-
-Rebuild instance's environment with the modified settings from the
-:ref:`settings brand` and :ref:`settings server` section of your
-:ref:`settings.yml <settings location>`.
-
-  What is the :origin:`utils/brand.env` needed for and why do you need to rebuild
-  it if necessary?
-
-  Short answer: :ref:`installation and maintenance <searxng maintenance>`
-  scripts are running outside of instance's runtime environment and need some
-  values defined in the runtime environment.
-
-All the SearXNG setups are centralized in the :ref:`settings.yml` file.  This
-setup is available as long we are in a *installed instance*.  E.g. the
-*installed instance* on the server or the *installed developer instance* at
-``./local`` (the later one is created by a :ref:`make install <make install>` or
-:ref:`make run <make run>`).
-
-Tasks running outside of an *installed instance*, especially :ref:`installation
-and maintenance <searxng maintenance>` tasks running at (pre-) installation time
-do not have access to the SearXNG setup (from a *installed instance*).  Those
-tasks need a *build environment*.
-
-The ``make buildenv`` target will update the *build environment* in:
-
-- :origin:`utils/brand.env`
-
-Tasks running outside of an *installed instance*, need the following settings
-from the YAML configuration:
-
-- ``SEARXNG_URL`` from :ref:`server.base_url <settings  server>` (aka
-  ``PUBLIC_URL``)
-- ``SEARXNG_BIND_ADDRESS`` from :ref:`server.bind_address <settings server>`
-- ``SEARXNG_PORT`` from :ref:`server.port <settings server>`
-
-The ``GIT_URL`` and ``GIT_BRANCH`` in the origin:`utils/brand.env` file, are
-read from the git VCS and the branch that is checked out when ``make
-buildenv`` command runs.
-
-.. _brand:
-
-**I would like to create my own brand, how should I proceed?**
-
-Create a remote branch (``example.org``), checkout the remote branch (on your
-local developer desktop) and in the :origin:`searx/settings.yml` file in the
-:ref:`settings server` section set ``base_url``.  Run ``make buildenv`` and
-create a commit for your brand.
-
-On your server you clone the branch (``example.org``) into your HOME folder
-``~`` from where you run the :ref:`installation <installation>` and
-:ref:`maintenance <searxng maintenance>` task.
-
-To upgrade you brand, rebase on SearXNG's master branch (on your local
-developer desktop), force push it to your remote branch.  Go to your server, do
-a force pull and run :ref:`sudo -H ./utils/searxng.sh instance update <update
-searxng>`.
-
 .. _make node.env:
 
 Node.js environment (``make node.env``)

docs/dev/reST.rst

@@ -292,7 +292,7 @@ content becomes smart.
    files & folders origin     :origin:`docs/dev/reST.rst`        ``:origin:`docs/dev/reST.rst```
    pull request               :pull:`4`                          ``:pull:`4```
    patch                      :patch:`af2cae6`                   ``:patch:`af2cae6```
-   PyPi package               :pypi:`searx`                      ``:pypi:`searx```
+   PyPi package               :pypi:`httpx`                      ``:pypi:`httpx```
    manual page man            :man:`bash`                        ``:man:`bash```
    intersphinx_
    --------------------------------------------------------------------------------------------------

docs/dev/rtm_asdf.rst

@@ -0,0 +1,121 @@
+==================
+Runtime Management
+==================
+
+The runtimes are managed with asdf and are activated in this project via the
+`.tool-versions <.tool-versions>`_. If you have not yet installed asdf_, then
+chapter :ref:`introduce asdf` may be of help to you.
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+Get started
+===========
+
+If you have asdf installed you can install the runtimes of this project by:
+
+.. code:: bash
+
+   $ cd /path/to/searxng
+   $ asdf install          # will install runtimes listed in .tool-versions
+   ...
+
+Manage Versions
+===============
+
+If you want to perform a ``test`` with special runtime versions of nodejs,
+python or shellcheck, you can patch the ``.tool-versions``:
+
+.. code:: diff
+
+   --- a/.tool-versions
+   +++ b/.tool-versions
+   @@ -1,2 +1,2 @@
+   -python 3.12.0
+   -shellcheck 0.9.0
+   +python 3.11.6
+   +shellcheck 0.8.0
+
+To install use ``asdf install`` again.  If the runtime tools have changed, any
+existing (nodejs and python) environments should be cleaned up with a ``make
+clean``.
+
+.. code:: bash
+
+   $ asdf install
+   ...
+   $ make clean test
+
+
+.. _introduce asdf:
+
+Introduce asdf
+==============
+
+To `download asdf`_ and `install asdf`_:
+
+.. code:: bash
+
+   $ git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch <version>
+   $ echo '. "$HOME/.asdf/asdf.sh"' >> ~/.bashrc
+   $ echo '. "$HOME/.asdf/completions/asdf.bash"' >> ~/.bashrc
+
+Start a new shell and try to `install plugins`_:
+
+.. code:: bash
+
+   $ asdf plugin-list-all | grep -E '(golang|python|nodejs|shellcheck).git'
+   golang                        https://github.com/asdf-community/asdf-golang.git
+   nodejs                        https://github.com/asdf-vm/asdf-nodejs.git
+   python                        https://github.com/danhper/asdf-python.git
+   shellcheck                    https://github.com/luizm/asdf-shellcheck.git
+
+   $ asdf plugin add golang https://github.com/asdf-community/asdf-golang.git
+   $ asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git
+   $ asdf plugin add python https://github.com/danhper/asdf-python.git
+   $ asdf plugin add shellcheck https://github.com/luizm/asdf-shellcheck.git
+
+Each plugin has dependencies, to compile runtimes visit the URLs from above and
+look out for the dependencies you need to install on your OS, on Debian for the
+runtimes listed above you will need:
+
+.. code:: bash
+
+  $ sudo apt update
+  $ sudo apt install \
+         dirmngr gpg curl gawk coreutils build-essential libssl-dev zlib1g-dev \
+         libbz2-dev libreadline-dev libsqlite3-dev \
+         libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+With dependencies installed you can install/compile runtimes:
+
+.. code:: bash
+
+  $ asdf install golang latest
+  $ asdf install nodejs latest
+  $ asdf install python latest
+  $ asdf install shellcheck latest
+
+Python will be compiled and will take a while.
+
+In the repository the version is defined in `.tool-versions`_. Outside the
+repository, its recommended that the runtime should use the versions of the OS
+(`Fallback to System Version`_) / if not already done register the system
+versions global:
+
+.. code:: bash
+
+   $ cd /
+   $ asdf global golang system
+   $ asdf global nodejs system
+   $ asdf global python system
+   $ asdf global shellcheck system
+
+.. _asdf: https://asdf-vm.com/
+.. _download asdf: https://asdf-vm.com/guide/getting-started.html#_2-download-asdf
+.. _install asdf: https://asdf-vm.com/guide/getting-started.html#_3-install-asdf
+.. _install plugins: https://asdf-vm.com/guide/getting-started.html#install-the-plugin
+.. _Fallback to System Version: https://asdf-vm.com/manage/versions.html#fallback-to-system-version

docs/dev/search_api.rst

@@ -69,7 +69,7 @@ Parameters
 
 ``autocomplete`` : default from :ref:`settings search`
   [ ``google``, ``dbpedia``, ``duckduckgo``, ``mwmbl``, ``startpage``,
-  ``wikipedia``, ``swisscows``, ``qwant`` ]
+  ``wikipedia``, ``stract``, ``swisscows``, ``qwant`` ]
 
   Service which completes words as you type.
 

docs/dev/searxng_extra/update.rst

@@ -78,6 +78,16 @@ Scripts to update static data in :origin:`searx/data/`
 .. automodule:: searxng_extra.update.update_pygments
   :members:
 
+.. _update_locales.py:
+
+``update_locales.py``
+=====================
+
+:origin:`[source] <searxng_extra/update/update_locales.py>`
+
+.. automodule:: searxng_extra.update.update_locales
+  :members:
+
 
 ``update_wikidata_units.py``
 ============================

docs/src/searx.botdetection.rst

@@ -12,8 +12,10 @@ Bot Detection
 .. automodule:: searx.botdetection
   :members:
 
-.. automodule:: searx.botdetection.limiter
-  :members:
+.. _botdetection ip_lists:
+
+IP lists
+========
 
 .. automodule:: searx.botdetection.ip_lists
   :members:
@@ -50,3 +52,11 @@ Probe HTTP headers
 
 .. automodule:: searx.botdetection.http_user_agent
   :members:
+
+.. _botdetection config:
+
+Config
+======
+
+.. automodule:: searx.botdetection.config
+  :members:

docs/src/searx.locales.rst

@@ -10,11 +10,6 @@ Locales
    :backlinks: entry
 
 .. automodule:: searx.locales
-  :members:
+   :members:
 
 
-SearXNG's locale codes
-======================
-
-.. automodule:: searx.sxng_locales
-  :members:

examples/basic_engine.py

@@ -1,25 +0,0 @@
-
-categories = ['general']  # optional
-
-
-def request(query, params):
-    '''pre-request callback
-    params<dict>:
-      method  : POST/GET
-      headers : {}
-      data    : {} # if method == POST
-      url     : ''
-      category: 'search category'
-      pageno  : 1 # number of the requested page
-    '''
-
-    params['url'] = 'https://host/%s' % query
-
-    return params
-
-
-def response(resp):
-    '''post-response callback
-    resp: requests response object
-    '''
-    return [{'url': '', 'title': '', 'content': ''}]

manage

@@ -41,10 +41,10 @@ PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
 
 PYOBJECTS="searx"
 PY_SETUP_EXTRAS='[test]'
-GECKODRIVER_VERSION="v0.33.0"
+GECKODRIVER_VERSION="v0.34.0"
 # SPHINXOPTS=
 BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
-BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
+BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
 
 _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
 # set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
@@ -52,18 +52,6 @@ if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then
     export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0"
 fi
 
-pylint.FILES() {
-
-    # List files tagged by comment:
-    #
-    #   # lint: pylint
-    #
-    # These py files are linted by test.pylint()
-
-    grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
-    find . -name searxng.msg
-}
-
 YAMLLINT_FILES=()
 while IFS= read -r line; do
    YAMLLINT_FILES+=("$line")
@@ -73,18 +61,9 @@ RST_FILES=(
     'README.rst'
 )
 
-PYLINT_SEARXNG_DISABLE_OPTION="\
-I,C,R,\
-W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
-E1136"
-PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="traits,supported_languages,language_aliases,logger,categories"
-PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
-
 help() {
     nvm.help
     cat <<EOF
-buildenv:
-  rebuild ./utils/brand.env
 webapp.:
   run       : run developer instance
 docs.:
@@ -111,8 +90,6 @@ pyenv.:
   uninstall : uninstall developer installation
   cmd ...   : run command ... in virtualenv
   OK        : test if virtualenv is OK
-pypi.upload:
-  Upload python packages to PyPi (to test use pypi.upload.test)
 format.:
   python    : format Python code source using black
 pygments.:
@@ -155,29 +132,6 @@ webapp.run() {
     SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
 }
 
-buildenv() {
-
-    # settings file from repository's working tree are used by default
-    SEARXNG_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml"
-
-    if [ -f /etc/searx/settings.yml ]; then
-        err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/"
-    fi
-
-    if [ -r '/etc/searxng/settings.yml' ]; then
-        if ask_yn "should settings read from: /etc/searxng/settings.yml"; then
-            SEARXNG_SETTINGS_PATH='/etc/searxng/settings.yml'
-        fi
-    fi
-    export SEARXNG_SETTINGS_PATH
-    (
-        set -e
-        SEARXNG_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
-            | prefix_stdout "${_Blue}BUILDENV${_creset}  "
-    )
-    return "${PIPESTATUS[0]}"
-}
-
 docker.push() {
     docker.build push
 }
@@ -345,7 +299,6 @@ pyenv.install() {
         pyenv
         build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
         "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
-        buildenv
     )
     local exit_val=$?
     if [ ! $exit_val -eq 0 ]; then
@@ -360,31 +313,12 @@ pyenv.uninstall() {
 
 }
 
-pypi.upload() {
-    py.clean
-    py.build
-    # https://github.com/pypa/twine
-    pyenv.cmd twine upload "${PYDIST}"/*
-}
-
-pypi.upload.test() {
-    py.clean
-    py.build
-    pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
-}
-
 format.python() {
     build_msg TEST "[format.python] black \$BLACK_TARGETS"
     pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
     dump_return $?
 }
 
-
-PYLINT_FILES=()
-while IFS= read -r line; do
-   PYLINT_FILES+=("$line")
-done <<< "$(pylint.FILES)"
-
 # shellcheck disable=SC2119
 main() {
 

package.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "eslint": "^8.50.0",
+    "eslint": "^9.0.0",
     "pyright": "^1.1.329"
   },
   "scripts": {

requirements-dev.txt

@@ -1,23 +1,22 @@
 mock==5.1.0
-nose2[coverage_plugin]==0.13.0
+nose2[coverage_plugin]==0.14.1
 cov-core==1.15.0
-black==22.12.0
-pylint==2.17.6
-splinter==0.19.0
-selenium==4.13.0
-twine==4.0.2
-Pallets-Sphinx-Themes==2.1.1
+black==24.3.0
+pylint==3.1.0
+splinter==0.21.0
+selenium==4.20.0
+Pallets-Sphinx-Themes==2.1.2
 Sphinx<=7.1.2; python_version == '3.8'
-Sphinx==7.2.6; python_version > '3.8'
-sphinx-issues==3.0.1
+Sphinx==7.3.7; python_version > '3.8'
+sphinx-issues==4.1.0
 sphinx-jinja==2.0.2
-sphinx-tabs==3.4.1
+sphinx-tabs==3.4.5
 sphinxcontrib-programoutput==0.17
 sphinx-autobuild==2021.3.14
 sphinx-notfound-page==1.0.0
 myst-parser==2.0.0
-linuxdoc==20230827
+linuxdoc==20231020
 aiounittest==1.4.2
-yamllint==1.32.0
-wlc==1.13
+yamllint==1.35.1
+wlc==1.14
 coloredlogs==15.0.1

requirements.txt

@@ -1,19 +1,18 @@
-certifi==2023.7.22
-babel==2.12.1
-flask-babel==3.1.0
-flask==2.3.3
-jinja2==3.1.2
-lxml==4.9.3
-pygments==2.16.1
-python-dateutil==2.8.2
+certifi==2024.2.2
+babel==2.14.0
+flask-babel==4.0.0
+flask==3.0.3
+jinja2==3.1.3
+lxml==5.2.1
+pygments==2.17.2
+python-dateutil==2.9.0.post0
 pyyaml==6.0.1
 httpx[http2]==0.24.1
 Brotli==1.1.0
-uvloop==0.17.0
+uvloop==0.19.0
 httpx-socks[asyncio]==0.7.7
-setproctitle==1.3.2
-redis==4.6.0
+setproctitle==1.3.3
+redis==5.0.4
 markdown-it-py==3.0.0
-typing_extensions==4.8.0
-fasttext-predict==0.9.2.1
+fasttext-predict==0.9.2.2
 pytomlpp==1.0.13

searx/__init__.py

@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
-# pylint: disable=missing-module-docstring
+# pylint: disable=missing-module-docstring, cyclic-import
 
 import sys
 import os
@@ -108,5 +107,6 @@ else:
 if settings['server']['public_instance']:
     logger.warning(
         "Be aware you have activated features intended only for public instances. "
-        + "This force the usage of the bot limiter and link_token plugins."
+        "This force the usage of the limiter and link_token / "
+        "see https://docs.searxng.org/admin/searx.limiter.html"
     )

searx/answerers/__init__.py

@@ -1,25 +1,30 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=missing-module-docstring
+
+import sys
 from os import listdir
 from os.path import realpath, dirname, join, isdir
-from searx.utils import load_module
 from collections import defaultdict
 
+from searx.utils import load_module
 
 answerers_dir = dirname(realpath(__file__))
 
 
 def load_answerers():
-    answerers = []
+    answerers = []  # pylint: disable=redefined-outer-name
+
     for filename in listdir(answerers_dir):
         if not isdir(join(answerers_dir, filename)) or filename.startswith('_'):
             continue
         module = load_module('answerer.py', join(answerers_dir, filename))
-        if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
-            exit(2)
+        if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not module.keywords:
+            sys.exit(2)
         answerers.append(module)
     return answerers
 
 
-def get_answerers_by_keywords(answerers):
+def get_answerers_by_keywords(answerers):  # pylint:disable=redefined-outer-name
     by_keyword = defaultdict(list)
     for answerer in answerers:
         for keyword in answerer.keywords:

searx/answerers/random/answerer.py

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 import hashlib
 import random
 import string

searx/answerers/statistics/answerer.py

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
 from functools import reduce
 from operator import mul
 

searx/autocomplete.py

@@ -1,12 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """This module implements functions needed for the autocompleter.
 
 """
 # pylint: disable=use-dict-literal
 
 import json
-from urllib.parse import urlencode
+from urllib.parse import urlencode, quote_plus
 
 import lxml
 from httpx import HTTPError
@@ -16,17 +15,26 @@ from searx.engines import (
     engines,
     google,
 )
-from searx.network import get as http_get
+from searx.network import get as http_get, post as http_post
 from searx.exceptions import SearxEngineResponseException
 
 
-def get(*args, **kwargs):
+def update_kwargs(**kwargs):
     if 'timeout' not in kwargs:
         kwargs['timeout'] = settings['outgoing']['request_timeout']
     kwargs['raise_for_httperror'] = True
+
+
+def get(*args, **kwargs):
+    update_kwargs(**kwargs)
     return http_get(*args, **kwargs)
 
 
+def post(*args, **kwargs):
+    update_kwargs(**kwargs)
+    return http_post(*args, **kwargs)
+
+
 def brave(query, _lang):
     # brave search autocompleter
     url = 'https://search.brave.com/api/suggest?'
@@ -145,6 +153,18 @@ def seznam(query, _lang):
     ]
 
 
+def stract(query, _lang):
+    # stract autocompleter (beta)
+    url = f"https://stract.com/beta/api/autosuggest?q={quote_plus(query)}"
+
+    resp = post(url)
+
+    if not resp.ok:
+        return []
+
+    return [suggestion['raw'] for suggestion in resp.json()]
+
+
 def startpage(query, sxng_locale):
     """Autocomplete from Startpage. Supports Startpage's languages"""
     lui = engines['startpage'].traits.get_language(sxng_locale, 'english')
@@ -223,6 +243,7 @@ backends = {
     'mwmbl': mwmbl,
     'seznam': seznam,
     'startpage': startpage,
+    'stract': stract,
     'swisscows': swisscows,
     'qwant': qwant,
     'wikipedia': wikipedia,

searx/babel_extract.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """This module implements the :origin:`searxng_msg <babel.cfg>` extractor to
 extract messages from:
 

searx/botdetection/__init__.py

@@ -1,44 +1,22 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """.. _botdetection src:
 
-The :ref:`limiter <limiter src>` implements several methods to block bots:
-
-a. Analysis of the HTTP header in the request / can be easily bypassed.
-
-b. Block and pass lists in which IPs are listed / difficult to maintain, since
-   the IPs of bots are not all known and change over the time.
-
-c. Detection of bots based on the behavior of the requests and blocking and, if
-   necessary, unblocking of the IPs via a dynamically changeable IP block list.
-
-For dynamically changeable IP lists a Redis database is needed and for any kind
-of IP list the determination of the IP of the client is essential.  The IP of
-the client is determined via the X-Forwarded-For_ HTTP header
-
-.. _X-Forwarded-For:
-   https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
-
-X-Forwarded-For
-===============
-
-.. attention::
-
-   A correct setup of the HTTP request headers ``X-Forwarded-For`` and
-   ``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
-
-   - `NGINX RequestHeader`_
-   - `Apache RequestHeader`_
-
-.. _NGINX RequestHeader:
-    https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
-.. _Apache RequestHeader:
-    https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
-
-.. autofunction:: searx.botdetection.get_real_ip
+Implementations used for bot detection.
 
 """
 
 from ._helpers import dump_request
 from ._helpers import get_real_ip
+from ._helpers import get_network
 from ._helpers import too_many_requests
+
+__all__ = ['dump_request', 'get_network', 'get_real_ip', 'too_many_requests']
+
+redis_client = None
+cfg = None
+
+
+def init(_cfg, _redis_client):
+    global redis_client, cfg  # pylint: disable=global-statement
+    redis_client = _redis_client
+    cfg = _cfg

searx/botdetection/_helpers.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 # pylint: disable=missing-module-docstring, invalid-name
 from __future__ import annotations
 
@@ -13,8 +12,8 @@ from ipaddress import (
 import flask
 import werkzeug
 
-from searx.tools import config
 from searx import logger
+from . import config
 
 logger = logger.getChild('botdetection')
 
@@ -104,10 +103,10 @@ def get_real_ip(request: flask.Request) -> str:
     if not forwarded_for:
         _log_error_only_once("X-Forwarded-For header is not set!")
     else:
-        from .limiter import get_cfg  # pylint: disable=import-outside-toplevel, cyclic-import
+        from . import cfg  # pylint: disable=import-outside-toplevel, cyclic-import
 
         forwarded_for = [x.strip() for x in forwarded_for.split(',')]
-        x_for: int = get_cfg()['real_ip.x_for']  # type: ignore
+        x_for: int = cfg['real_ip.x_for']  # type: ignore
         forwarded_for = forwarded_for[-min(len(forwarded_for), x_for)]
 
     if not real_ip:

searx/botdetection/config.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Configuration class :py:class:`Config` with deep-update, schema validation
 and deprecated names.
 
@@ -211,7 +210,6 @@ def value(name: str, data_dict: dict):
 def validate(
     schema_dict: typing.Dict, data_dict: typing.Dict, deprecated: typing.Dict[str, str]
 ) -> typing.Tuple[bool, list]:
-
     """Deep validation of dictionary in ``data_dict`` against dictionary in
     ``schema_dict``.  Argument deprecated is a dictionary that maps deprecated
     configuration names to a messages::

searx/botdetection/http_accept.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Method ``http_accept``
 ----------------------
@@ -24,7 +23,7 @@ from ipaddress import (
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

searx/botdetection/http_accept_encoding.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Method ``http_accept_encoding``
 -------------------------------
@@ -25,7 +24,7 @@ from ipaddress import (
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

searx/botdetection/http_accept_language.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Method ``http_accept_language``
 -------------------------------
@@ -21,7 +20,7 @@ from ipaddress import (
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

searx/botdetection/http_connection.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Method ``http_connection``
 --------------------------
@@ -22,7 +21,7 @@ from ipaddress import (
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

searx/botdetection/http_user_agent.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Method ``http_user_agent``
 --------------------------
@@ -24,7 +23,7 @@ from ipaddress import (
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 
@@ -35,7 +34,7 @@ USER_AGENT = (
     + r'|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser|Googlebot|GoogleImageProxy'
     + r'|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot'
     + r'|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot'
-    + r'|ZmEu|BLEXBot|bitlybot'
+    + r'|ZmEu|BLEXBot|bitlybot|HeadlessChrome'
     # unmaintained Farside instances
     + r'|'
     + re.escape(r'Mozilla/5.0 (compatible; Farside/0.1.0; +https://farside.link)')

searx/botdetection/ip_limit.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """.. _botdetection.ip_limit:
 
 Method ``ip_limit``
@@ -13,8 +12,7 @@ and at least for a maximum of 10 minutes.
 
 The :py:obj:`.link_token` method can be used to investigate whether a request is
 *suspicious*.  To activate the :py:obj:`.link_token` method in the
-:py:obj:`.ip_limit` method add the following to your
-``/etc/searxng/limiter.toml``:
+:py:obj:`.ip_limit` method add the following configuration:
 
 .. code:: toml
 
@@ -46,13 +44,12 @@ from ipaddress import (
 
 import flask
 import werkzeug
-from searx.tools import config
-from searx import settings
 
 from searx import redisdb
 from searx.redislib import incr_sliding_window, drop_counter
 
 from . import link_token
+from . import config
 from ._helpers import (
     too_many_requests,
     logger,
@@ -110,7 +107,7 @@ def filter_request(
         if c > API_MAX:
             return too_many_requests(network, "too many request in API_WINDOW")
 
-    if settings['server']['public_instance'] or cfg['botdetection.ip_limit.link_token']:
+    if cfg['botdetection.ip_limit.link_token']:
 
         suspicious = link_token.is_suspicious(network, request, True)
 

searx/botdetection/ip_lists.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """.. _botdetection.ip_lists:
 
 Method ``ip_lists``
@@ -13,7 +12,7 @@ The ``ip_lists`` method implements IP :py:obj:`block- <block_ip>` and
    [botdetection.ip_lists]
 
    pass_ip = [
-    '140.238.172.132', # IPv4 of check.searx.space
+    '167.235.158.251', # IPv4 of check.searx.space
     '192.168.0.0/16',  # IPv4 private network
     'fe80::/10'        # IPv6 linklocal
    ]
@@ -33,15 +32,15 @@ from ipaddress import (
     IPv6Address,
 )
 
-from searx.tools import config
+from . import config
 from ._helpers import logger
 
 logger = logger.getChild('ip_limit')
 
 SEARXNG_ORG = [
     # https://github.com/searxng/searxng/pull/2484#issuecomment-1576639195
-    '140.238.172.132',  # IPv4 check.searx.space
-    '2603:c022:0:4900::/56',  # IPv6 check.searx.space
+    '167.235.158.251',  # IPv4 check.searx.space
+    '2a01:04f8:1c1c:8fc2::/64',  # IPv6 check.searx.space
 ]
 """Passlist of IPs from the SearXNG organization, e.g. `check.searx.space`."""
 

searx/botdetection/limiter.py

@@ -1,145 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
-""".. _limiter src:
-
-Limiter
-=======
-
-.. sidebar:: info
-
-   The limiter requires a :ref:`Redis <settings redis>` database.
-
-Bot protection / IP rate limitation.  The intention of rate limitation is to
-limit suspicious requests from an IP.  The motivation behind this is the fact
-that SearXNG passes through requests from bots and is thus classified as a bot
-itself.  As a result, the SearXNG engine then receives a CAPTCHA or is blocked
-by the search engine (the origin) in some other way.
-
-To avoid blocking, the requests from bots to SearXNG must also be blocked, this
-is the task of the limiter.  To perform this task, the limiter uses the methods
-from the :py:obj:`searx.botdetection`.
-
-To enable the limiter activate:
-
-.. code:: yaml
-
-   server:
-     ...
-     limiter: true  # rate limit the number of request on the instance, block some bots
-
-and set the redis-url connection. Check the value, it depends on your redis DB
-(see :ref:`settings redis`), by example:
-
-.. code:: yaml
-
-   redis:
-     url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
-
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from ipaddress import ip_address
-import flask
-import werkzeug
-
-from searx.tools import config
-from searx import logger
-
-from . import (
-    http_accept,
-    http_accept_encoding,
-    http_accept_language,
-    http_user_agent,
-    ip_limit,
-    ip_lists,
-)
-
-from ._helpers import (
-    get_network,
-    get_real_ip,
-    dump_request,
-)
-
-logger = logger.getChild('botdetection.limiter')
-
-CFG: config.Config = None  # type: ignore
-
-LIMITER_CFG_SCHEMA = Path(__file__).parent / "limiter.toml"
-"""Base configuration (schema) of the botdetection."""
-
-LIMITER_CFG = Path('/etc/searxng/limiter.toml')
-"""Local Limiter configuration."""
-
-CFG_DEPRECATED = {
-    # "dummy.old.foo": "config 'dummy.old.foo' exists only for tests.  Don't use it in your real project config."
-}
-
-
-def get_cfg() -> config.Config:
-    global CFG  # pylint: disable=global-statement
-    if CFG is None:
-        CFG = config.Config.from_toml(LIMITER_CFG_SCHEMA, LIMITER_CFG, CFG_DEPRECATED)
-    return CFG
-
-
-def filter_request(request: flask.Request) -> werkzeug.Response | None:
-    # pylint: disable=too-many-return-statements
-
-    cfg = get_cfg()
-    real_ip = ip_address(get_real_ip(request))
-    network = get_network(real_ip, cfg)
-
-    if request.path == '/healthz':
-        return None
-
-    # link-local
-
-    if network.is_link_local:
-        return None
-
-    # block- & pass- lists
-    #
-    # 1. The IP of the request is first checked against the pass-list; if the IP
-    #    matches an entry in the list, the request is not blocked.
-    # 2. If no matching entry is found in the pass-list, then a check is made against
-    #    the block list; if the IP matches an entry in the list, the request is
-    #    blocked.
-    # 3. If the IP is not in either list, the request is not blocked.
-
-    match, msg = ip_lists.pass_ip(real_ip, cfg)
-    if match:
-        logger.warning("PASS %s: matched PASSLIST - %s", network.compressed, msg)
-        return None
-
-    match, msg = ip_lists.block_ip(real_ip, cfg)
-    if match:
-        logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
-        return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
-
-    # methods applied on /
-
-    for func in [
-        http_user_agent,
-    ]:
-        val = func.filter_request(network, request, cfg)
-        if val is not None:
-            return val
-
-    # methods applied on /search
-
-    if request.path == '/search':
-
-        for func in [
-            http_accept,
-            http_accept_encoding,
-            http_accept_language,
-            http_user_agent,
-            ip_limit,
-        ]:
-            val = func.filter_request(network, request, cfg)
-            if val is not None:
-                return val
-    logger.debug(f"OK {network}: %s", dump_request(flask.request))
-    return None

searx/botdetection/link_token.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Method ``link_token``
 ---------------------
@@ -99,15 +98,13 @@ def ping(request: flask.Request, token: str):
     The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
 
     """
-    from . import limiter  # pylint: disable=import-outside-toplevel, cyclic-import
+    from . import redis_client, cfg  # pylint: disable=import-outside-toplevel, cyclic-import
 
-    redis_client = redisdb.client()
     if not redis_client:
         return
     if not token_is_valid(token):
         return
 
-    cfg = limiter.get_cfg()
     real_ip = ip_address(get_real_ip(request))
     network = get_network(real_ip, cfg)
 

searx/compat.py

@@ -1,73 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
-# pyright: basic
-"""Module for backward compatibility.
-
-"""
-# pylint: disable=C,R
-
-
-__all__ = ('cached_property',)
-
-
-try:
-    from functools import cached_property  # type: ignore
-
-except ImportError:
-
-    # cache_property has been added in py3.8 [1]
-    #
-    # To support cache_property in py3.7 the implementation from 3.8 has been
-    # copied here.  This code can be cleanup with EOL of py3.7.
-    #
-    # [1] https://docs.python.org/3/library/functools.html#functools.cached_property
-
-    from threading import RLock
-
-    _NOT_FOUND = object()
-
-    class cached_property:
-        def __init__(self, func):
-            self.func = func
-            self.attrname = None
-            self.__doc__ = func.__doc__
-            self.lock = RLock()
-
-        def __set_name__(self, owner, name):
-            if self.attrname is None:
-                self.attrname = name
-            elif name != self.attrname:
-                raise TypeError(
-                    "Cannot assign the same cached_property to two different names "
-                    f"({self.attrname!r} and {name!r})."
-                )
-
-        def __get__(self, instance, owner=None):
-            if instance is None:
-                return self
-            if self.attrname is None:
-                raise TypeError("Cannot use cached_property instance without calling __set_name__ on it.")
-            try:
-                cache = instance.__dict__
-            except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
-                msg = (
-                    f"No '__dict__' attribute on {type(instance).__name__!r} "
-                    f"instance to cache {self.attrname!r} property."
-                )
-                raise TypeError(msg) from None
-            val = cache.get(self.attrname, _NOT_FOUND)
-            if val is _NOT_FOUND:
-                with self.lock:
-                    # check if another thread filled cache while we awaited lock
-                    val = cache.get(self.attrname, _NOT_FOUND)
-                    if val is _NOT_FOUND:
-                        val = self.func(instance)
-                        try:
-                            cache[self.attrname] = val
-                        except TypeError:
-                            msg = (
-                                f"The '__dict__' attribute on {type(instance).__name__!r} instance "
-                                f"does not support item assignment for caching {self.attrname!r} property."
-                            )
-                            raise TypeError(msg) from None
-            return val

searx/data/__init__.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """This module holds the *data* created by::
 
   make data.all
@@ -15,6 +14,7 @@ __all__ = [
     'EXTERNAL_BANGS',
     'OSM_KEYS_TAGS',
     'ENGINE_DESCRIPTIONS',
+    'LOCALES',
     'ahmia_blacklist_loader',
 ]
 
@@ -50,3 +50,4 @@ EXTERNAL_BANGS = _load('external_bangs.json')
 OSM_KEYS_TAGS = _load('osm_keys_tags.json')
 ENGINE_DESCRIPTIONS = _load('engine_descriptions.json')
 ENGINE_TRAITS = _load('engine_traits.json')
+LOCALES = _load('locales.json')

searx/data/locales.json

@@ -0,0 +1,69 @@
+{
+  "LOCALE_NAMES": {
+    "af": "Afrikaans",
+    "ar": "العربية (Arabic)",
+    "bg": "Български (Bulgarian)",
+    "bn": "বাংলা (Bangla)",
+    "bo": "བོད་སྐད་ (Tibetan)",
+    "ca": "Català (Catalan)",
+    "cs": "Čeština (Czech)",
+    "cy": "Cymraeg (Welsh)",
+    "da": "Dansk (Danish)",
+    "de": "Deutsch (German)",
+    "dv": "ދިވެހި (Dhivehi)",
+    "el-GR": "Ελληνικά, Ελλάδα (Greek, Greece)",
+    "en": "English",
+    "eo": "Esperanto",
+    "es": "Español (Spanish)",
+    "et": "Eesti (Estonian)",
+    "eu": "Euskara (Basque)",
+    "fa-IR": "فارسی, ایران (Persian, Iran)",
+    "fi": "Suomi (Finnish)",
+    "fil": "Filipino",
+    "fr": "Français (French)",
+    "gl": "Galego (Galician)",
+    "he": "עברית (Hebrew)",
+    "hr": "Hrvatski (Croatian)",
+    "hu": "Magyar (Hungarian)",
+    "ia": "Interlingua",
+    "id": "Indonesia (Indonesian)",
+    "it": "Italiano (Italian)",
+    "ja": "日本語 (Japanese)",
+    "ko": "한국어 (Korean)",
+    "lt": "Lietuvių (Lithuanian)",
+    "lv": "Latviešu (Latvian)",
+    "ml": "മലയാളം (Malayalam)",
+    "ms": "Melayu (Malay)",
+    "nb-NO": "Norsk bokmål, Norge (Norwegian bokmål, Norway)",
+    "nl": "Nederlands (Dutch)",
+    "nl-BE": "Nederlands, België (Dutch, Belgium)",
+    "oc": "Occitan",
+    "pa": "ਪੰਜਾਬੀ (Punjabi)",
+    "pap": "Papiamento",
+    "pl": "Polski (Polish)",
+    "pt": "Português (Portuguese)",
+    "pt-BR": "Português, Brasil (Portuguese, Brazil)",
+    "ro": "Română (Romanian)",
+    "ru": "Русский (Russian)",
+    "si": "සිංහල (Sinhala)",
+    "sk": "Slovenčina (Slovak)",
+    "sl": "Slovenščina (Slovenian)",
+    "sr": "Српски (Serbian)",
+    "sv": "Svenska (Swedish)",
+    "szl": "Ślōnski (Silesian)",
+    "ta": "தமிழ் (Tamil)",
+    "te": "తెలుగు (Telugu)",
+    "th": "ไทย (Thai)",
+    "tr": "Türkçe (Turkish)",
+    "uk": "Українська (Ukrainian)",
+    "vi": "Tiếng việt (Vietnamese)",
+    "zh-HK": "中文, 中國香港特別行政區 (Chinese, Hong Kong SAR China)",
+    "zh-Hans-CN": "中文, 中国 (Chinese, China)",
+    "zh-Hant-TW": "中文, 台灣 (Chinese, Taiwan)"
+  },
+  "RTL_LOCALES": [
+    "ar",
+    "fa-IR",
+    "he"
+  ]
+}

searx/data/useragents.json

@@ -1,11 +1,11 @@
 {
-    "versions": [
-        "118.0",
-        "117.0"
-    ],
     "os": [
         "Windows NT 10.0; Win64; x64",
         "X11; Linux x86_64"
     ],
-    "ua": "Mozilla/5.0 ({os}; rv:109.0) Gecko/20100101 Firefox/{version}"
+    "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
+    "versions": [
+        "124.0",
+        "123.0"
+    ]
 }

searx/enginelib/__init__.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Implementations of the framework for the SearXNG engines.
 
 .. hint::

searx/enginelib/traits.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Engine's traits are fetched from the origin engines and stored in a JSON file
 in the *data folder*.  Most often traits are languages and region codes and
 their mapping from SearXNG's representation to the representation in the origin
@@ -14,8 +13,7 @@ from __future__ import annotations
 import json
 import dataclasses
 import types
-from typing import Dict, Iterable, Union, Callable, Optional, TYPE_CHECKING
-from typing_extensions import Literal, Self
+from typing import Dict, Literal, Iterable, Union, Callable, Optional, TYPE_CHECKING
 
 from searx import locales
 from searx.data import data_dir, ENGINE_TRAITS
@@ -136,7 +134,7 @@ class EngineTraits:
         return EngineTraits(**dataclasses.asdict(self))
 
     @classmethod
-    def fetch_traits(cls, engine: Engine) -> Union[Self, None]:
+    def fetch_traits(cls, engine: Engine) -> Union['EngineTraits', None]:
         """Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch
         and set properties from the origin engine in the object ``engine_traits``.  If
         function does not exists, ``None`` is returned.
@@ -168,7 +166,7 @@ class EngineTraits:
         #   - name: google italian
         #     engine: google
         #     language: it
-        #     region: it-IT
+        #     region: it-IT                                      # type: ignore
 
         traits = self.copy()
 
@@ -204,7 +202,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
             json.dump(self, f, indent=2, sort_keys=True, cls=EngineTraitsEncoder)
 
     @classmethod
-    def from_data(cls) -> Self:
+    def from_data(cls) -> 'EngineTraitsMap':
         """Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`"""
         obj = cls()
         for k, v in ENGINE_TRAITS.items():
@@ -212,7 +210,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
         return obj
 
     @classmethod
-    def fetch_traits(cls, log: Callable) -> Self:
+    def fetch_traits(cls, log: Callable) -> 'EngineTraitsMap':
         from searx import engines  # pylint: disable=cyclic-import, import-outside-toplevel
 
         names = list(engines.engines)

searx/engines/1337x.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- 1337x
+# pylint: disable=invalid-name
+"""1337x
+
 """
 
 from urllib.parse import quote, urljoin

searx/engines/9gag.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 # pylint: disable=invalid-name
 """9GAG (social media)"""
 

searx/engines/__init__.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Load and initialize the ``engines``, see :py:func:`load_engines` and register
 :py:obj:`engine_shortcuts`.
 
@@ -45,6 +44,7 @@ ENGINE_DEFAULT_ARGS = {
     "using_tor_proxy": False,
     "send_accept_language_header": False,
     "tokens": [],
+    "max_page": 0,
 }
 # set automatically when an engine does not have any tab category
 DEFAULT_CATEGORY = 'other'

searx/engines/ahmia.py

@@ -74,7 +74,7 @@ def response(resp):
     if number_of_results:
         try:
             results.append({'number_of_results': int(extract_text(number_of_results))})
-        except:
+        except:  # pylint: disable=bare-except
             pass
 
     return results

searx/engines/annas_archive.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """`Anna's Archive`_ is a free non-profit online shadow library metasearch
 engine providing access to a variety of book resources (also via IPFS), created
 by a team of anonymous archivists (AnnaArchivist_).
@@ -25,7 +24,7 @@ for *newest* articles and journals (PDF) / by shortcut ``!aaa <search-term>``.
   - name: annas articles
     engine: annas_archive
     shortcut: aaa
-    aa_content: 'journal_article'
+    aa_content: 'magazine'
     aa_ext: 'pdf'
     aa_sort: 'newest'
 
@@ -61,7 +60,7 @@ base_url: str = "https://annas-archive.org"
 aa_content: str = ""
 """Anan's search form field **Content** / possible values::
 
-    journal_article, book_any, book_fiction, book_unknown, book_nonfiction,
+    book_fiction, book_unknown, book_nonfiction,
     book_comic, magazine, standards_document
 
 To not filter use an empty string (default).

searx/engines/apkmirror.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """APKMirror
 """
 

searx/engines/apple_app_store.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
-"""
-    Apple App Store
+"""Apple App Store
+
 """
 
 from json import loads

searx/engines/apple_maps.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Apple Maps"""
 
 from json import loads

searx/engines/archlinux.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """
 Arch Linux Wiki
 ~~~~~~~~~~~~~~~

searx/engines/artic.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """The Art Institute of Chicago
 
 Explore thousands of artworks from The Art Institute of Chicago.
@@ -57,11 +56,10 @@ def response(resp):
         results.append(
             {
                 'url': 'https://artic.edu/artworks/%(id)s' % result,
-                'title': result['title'] + " (%(date_display)s) //  %(artist_display)s" % result,
-                'content': result['medium_display'],
+                'title': result['title'] + " (%(date_display)s) // %(artist_display)s" % result,
+                'content': "%(medium_display)s // %(dimensions)s" % result,
                 'author': ', '.join(result['artist_titles']),
                 'img_src': image_api + '/%(image_id)s/full/843,/0/default.jpg' % result,
-                'img_format': result['dimensions'],
                 'template': 'images.html',
             }
         )

searx/engines/arxiv.py

@@ -1,11 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+"""ArXiV (Scientific preprints)
+
 """
- ArXiV (Scientific preprints)
-"""
+
+from datetime import datetime
 
 from lxml import etree
 from lxml.etree import XPath
-from datetime import datetime
 from searx.utils import eval_xpath, eval_xpath_list, eval_xpath_getindex
 
 # about
@@ -50,7 +51,7 @@ def request(query, params):
     # basic search
     offset = (params['pageno'] - 1) * number_of_results
 
-    string_args = dict(query=query, offset=offset, number_of_results=number_of_results)
+    string_args = {'query': query, 'offset': offset, 'number_of_results': number_of_results}
 
     params['url'] = base_url.format(**string_args)
 

searx/engines/ask.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Ask.com"""
+
+from urllib.parse import urlencode
+import dateutil
+from lxml import html
+from searx import utils
+
+# Metadata
+about = {
+    "website": "https://www.ask.com/",
+    "wikidata_id": 'Q847564',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+# Engine Configuration
+categories = ['general']
+paging = True
+max_page = 5
+
+# Base URL
+base_url = "https://www.ask.com/web"
+
+
+def request(query, params):
+
+    query_params = {
+        "q": query,
+        "page": params["pageno"],
+    }
+
+    params["url"] = f"{base_url}?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+
+    start_tag = 'window.MESON.initialState = {'
+    end_tag = '}};'
+
+    dom = html.fromstring(resp.text)
+    script = utils.eval_xpath_getindex(dom, '//script', 0, default=None).text
+
+    pos = script.index(start_tag) + len(start_tag) - 1
+    script = script[pos:]
+    pos = script.index(end_tag) + len(end_tag) - 1
+    script = script[:pos]
+
+    json_resp = utils.js_variable_to_python(script)
+
+    results = []
+
+    for item in json_resp['search']['webResults']['results']:
+
+        pubdate_original = item.get('pubdate_original')
+        if pubdate_original:
+            pubdate_original = dateutil.parser.parse(pubdate_original)
+        metadata = [item.get(field) for field in ['category_l1', 'catsy'] if item.get(field)]
+
+        results.append(
+            {
+                "url": item['url'].split('&ueid')[0],
+                "title": item['title'],
+                "content": item['abstract'],
+                "publishedDate": pubdate_original,
+                # "img_src": item.get('image_url') or None, # these are not thumbs / to large
+                "metadata": ' | '.join(metadata),
+            }
+        )
+
+    return results

searx/engines/bandcamp.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Bandcamp (Music)
 
 @website     https://bandcamp.com/
@@ -38,16 +37,6 @@ iframe_src = "https://bandcamp.com/EmbeddedPlayer/{type}={result_id}/size=large/
 
 
 def request(query, params):
-    '''pre-request callback
-
-    params<dict>:
-      method  : POST/GET
-      headers : {}
-      data    : {} # if method == POST
-      url     : ''
-      category: 'search category'
-      pageno  : 1 # number of the requested page
-    '''
 
     search_path = search_string.format(query=urlencode({'q': query}), page=params['pageno'])
     params['url'] = base_url + search_path
@@ -55,10 +44,7 @@ def request(query, params):
 
 
 def response(resp):
-    '''post-response callback
 
-    resp: requests response object
-    '''
     results = []
     dom = html.fromstring(resp.text)
 

searx/engines/base.py

@@ -1,12 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+"""BASE (Scholar publications)
+
 """
- BASE (Scholar publications)
-"""
+from datetime import datetime
+import re
 
 from urllib.parse import urlencode
 from lxml import etree
-from datetime import datetime
-import re
 from searx.utils import searx_useragent
 
 # about
@@ -55,13 +55,17 @@ shorcut_dict = {
 
 def request(query, params):
     # replace shortcuts with API advanced search keywords
-    for key in shorcut_dict.keys():
-        query = re.sub(key, shorcut_dict[key], query)
+    for key, val in shorcut_dict.items():
+        query = re.sub(key, val, query)
 
     # basic search
     offset = (params['pageno'] - 1) * number_of_results
 
-    string_args = dict(query=urlencode({'query': query}), offset=offset, hits=number_of_results)
+    string_args = {
+        'query': urlencode({'query': query}),
+        'offset': offset,
+        'hits': number_of_results,
+    }
 
     params['url'] = base_url.format(**string_args)
 
@@ -99,7 +103,7 @@ def response(resp):
             try:
                 publishedDate = datetime.strptime(date, date_format)
                 break
-            except:
+            except:  # pylint: disable=bare-except
                 pass
 
         if publishedDate is not None:

searx/engines/bilibili.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Bilibili is a Chinese video sharing website.
 
 .. _Bilibili: https://www.bilibili.com

searx/engines/bing.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """This is the implementation of the Bing-WEB engine. Some of this
 implementations are shared by other engines:
 
@@ -59,6 +58,9 @@ about = {
 # engine dependent config
 categories = ['general', 'web']
 paging = True
+max_page = 200
+"""200 pages maximum (``&first=1991``)"""
+
 time_range_support = True
 safesearch = True
 """Bing results are always SFW.  To get NSFW links from bing some age

searx/engines/bing_images.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Bing-Images: description see :py:obj:`searx.engines.bing`.
 """
 # pylint: disable=invalid-name
@@ -92,7 +91,7 @@ def response(resp):
 
         metadata = json.loads(result.xpath('.//a[@class="iusc"]/@m')[0])
         title = ' '.join(result.xpath('.//div[@class="infnmpt"]//a/text()')).strip()
-        img_format = ' '.join(result.xpath('.//div[@class="imgpt"]/div/span/text()')).strip()
+        img_format = ' '.join(result.xpath('.//div[@class="imgpt"]/div/span/text()')).strip().split(" · ")
         source = ' '.join(result.xpath('.//div[@class="imgpt"]//div[@class="lnkw"]//a/text()')).strip()
         results.append(
             {
@@ -103,7 +102,8 @@ def response(resp):
                 'content': metadata['desc'],
                 'title': title,
                 'source': source,
-                'img_format': img_format,
+                'resolution': img_format[0],
+                'img_format': img_format[1] if len(img_format) >= 2 else None,
             }
         )
     return results

searx/engines/bing_news.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Bing-News: description see :py:obj:`searx.engines.bing`.
 
 .. hint::

searx/engines/bing_videos.py

@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
+# pylint: disable=invalid-name
 """Bing-Videos: description see :py:obj:`searx.engines.bing`.
 """
-# pylint: disable=invalid-name
 
 from typing import TYPE_CHECKING
 import json

searx/engines/bpb.py

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""BPB refers to ``Bundeszentrale für poltische Bildung``, which is a German
+governmental institution aiming to reduce misinformation by providing resources
+about politics and history.
+"""
+
+from datetime import datetime
+from urllib.parse import urlencode
+
+about = {
+    'website': "https://www.bpb.de",
+    'official_api_documentation': None,
+    'use_official_api': False,
+    'require_api_key': False,
+    'results': 'JSON',
+    'language': 'de',
+}
+
+paging = True
+categories = ['general']
+
+
+base_url = "https://www.bpb.de"
+
+
+def request(query, params):
+    args = {
+        'query[term]': query,
+        'page': params['pageno'] - 1,
+        'sort[direction]': 'descending',
+        'payload[nid]': 65350,
+    }
+    params['url'] = f"{base_url}/bpbapi/filter/search?{urlencode(args)}"
+    return params
+
+
+def response(resp):
+    results = []
+
+    json_resp = resp.json()
+
+    for result in json_resp['teaser']:
+        img_src = None
+        if result['teaser']['image']:
+            img_src = base_url + result['teaser']['image']['sources'][-1]['url']
+
+        metadata = result['extension']['overline']
+        authors = ', '.join(author['name'] for author in result['extension'].get('authors', []))
+        if authors:
+            metadata += f" | {authors}"
+
+        publishedDate = None
+        if result['extension'].get('publishingDate'):
+            publishedDate = datetime.utcfromtimestamp(result['extension']['publishingDate'])
+
+        results.append(
+            {
+                'url': base_url + result['teaser']['link']['url'],
+                'title': result['teaser']['title'],
+                'content': result['teaser']['text'],
+                'img_src': img_src,
+                'publishedDate': publishedDate,
+                'metadata': metadata,
+            }
+        )
+
+    return results

searx/engines/brave.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Brave supports the categories listed in :py:obj:`brave_category` (General,
 news, videos, images).  The support of :py:obj:`paging` and :py:obj:`time range
 <time_range_support>` is limited (see remarks).
@@ -30,6 +29,13 @@ Configured ``brave`` engines:
     ...
     brave_category: news
 
+  - name: brave.goggles
+    brave_category: goggles
+    time_range_support: true
+    paging: true
+    ...
+    brave_category: goggles
+
 
 .. _brave regions:
 
@@ -58,6 +64,23 @@ region are mapped to regions in SearXNG (see :py:obj:`babel
    low quality.
 
 
+.. _brave googles:
+
+Brave Goggles
+=============
+
+.. _list of Goggles: https://search.brave.com/goggles/discover
+.. _Goggles Whitepaper: https://brave.com/static-assets/files/goggles.pdf
+.. _Goggles Quickstart: https://github.com/brave/goggles-quickstart
+
+Goggles allow you to choose, alter, or extend the ranking of Brave Search
+results (`Goggles Whitepaper`_).  Goggles are openly developed by the community
+of Brave Search users.
+
+Select from the `list of Goggles`_ people have published, or create your own
+(`Goggles Quickstart`_).
+
+
 .. _brave languages:
 
 Brave languages
@@ -95,7 +118,7 @@ Implementations
 
 """
 
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
 
 from urllib.parse import (
     urlencode,
@@ -103,11 +126,13 @@ from urllib.parse import (
     parse_qs,
 )
 
+from dateutil import parser
 from lxml import html
 
 from searx import locales
 from searx.utils import (
     extract_text,
+    eval_xpath,
     eval_xpath_list,
     eval_xpath_getindex,
     js_variable_to_python,
@@ -133,12 +158,14 @@ about = {
 base_url = "https://search.brave.com/"
 categories = []
 brave_category = 'search'
-"""Brave supports common web-search, video search, image and video search.
+Goggles = Any
+"""Brave supports common web-search, videos, images, news, and goggles search.
 
 - ``search``: Common WEB search
 - ``videos``: search for videos
 - ``images``: search for images
 - ``news``: search for news
+- ``goggles``: Common WEB search with custom rules
 """
 
 brave_spellcheck = False
@@ -151,14 +178,18 @@ in SearXNG, the spellchecking is disabled by default.
 send_accept_language_header = True
 paging = False
 """Brave only supports paging in :py:obj:`brave_category` ``search`` (UI
-category All)."""
+category All) and in the goggles category."""
+max_page = 10
+"""Tested 9 pages maximum (``&offset=8``), to be save max is set to 10.  Trying
+to do more won't return any result and you will most likely be flagged as a bot.
+"""
 
 safesearch = True
 safesearch_map = {2: 'strict', 1: 'moderate', 0: 'off'}  # cookie: safesearch=off
 
 time_range_support = False
 """Brave only supports time-range in :py:obj:`brave_category` ``search`` (UI
-category All)."""
+category All) and in the goggles category."""
 
 time_range_map = {
     'day': 'pd',
@@ -179,12 +210,15 @@ def request(query, params):
     if brave_spellcheck:
         args['spellcheck'] = '1'
 
-    if brave_category == 'search':
+    if brave_category in ('search', 'goggles'):
         if params.get('pageno', 1) - 1:
             args['offset'] = params.get('pageno', 1) - 1
         if time_range_map.get(params['time_range']):
             args['tf'] = time_range_map.get(params['time_range'])
 
+    if brave_category == 'goggles':
+        args['goggles_id'] = Goggles
+
     params["url"] = f"{base_url}{brave_category}?{urlencode(args)}"
 
     # set properties in the cookies
@@ -203,9 +237,19 @@ def request(query, params):
     logger.debug("cookies %s", params['cookies'])
 
 
+def _extract_published_date(published_date_raw):
+    if published_date_raw is None:
+        return None
+
+    try:
+        return parser.parse(published_date_raw)
+    except parser.ParserError:
+        return None
+
+
 def response(resp):
 
-    if brave_category == 'search':
+    if brave_category in ('search', 'goggles'):
         return _parse_search(resp)
 
     datastr = ""
@@ -244,17 +288,21 @@ def _parse_search(resp):
     for result in eval_xpath_list(dom, xpath_results):
 
         url = eval_xpath_getindex(result, './/a[contains(@class, "h")]/@href', 0, default=None)
-        title_tag = eval_xpath_getindex(result, './/div[contains(@class, "title")]', 0, default=None)
+        title_tag = eval_xpath_getindex(
+            result, './/a[contains(@class, "h")]//div[contains(@class, "title")]', 0, default=None
+        )
         if url is None or title_tag is None or not urlparse(url).netloc:  # partial url likely means it's an ad
             continue
 
-        content_tag = eval_xpath_getindex(result, './/div[@class="snippet-description"]', 0, default='')
+        content_tag = eval_xpath_getindex(result, './/div[contains(@class, "snippet-description")]', 0, default='')
+        pub_date_raw = eval_xpath(result, 'substring-before(.//div[contains(@class, "snippet-description")], "-")')
         img_src = eval_xpath_getindex(result, './/img[contains(@class, "thumb")]/@src', 0, default='')
 
         item = {
             'url': url,
             'title': extract_text(title_tag),
             'content': extract_text(content_tag),
+            'publishedDate': _extract_published_date(pub_date_raw),
             'img_src': img_src,
         }
 
@@ -271,6 +319,10 @@ def _parse_search(resp):
                 item['iframe_src'] = iframe_src
                 item['template'] = 'videos.html'
                 item['thumbnail'] = eval_xpath_getindex(video_tag, './/img/@src', 0, default='')
+                pub_date_raw = extract_text(
+                    eval_xpath(video_tag, './/div[contains(@class, "snippet-attributes")]/div/text()')
+                )
+                item['publishedDate'] = _extract_published_date(pub_date_raw)
             else:
                 item['img_src'] = eval_xpath_getindex(video_tag, './/img/@src', 0, default='')
 
@@ -296,6 +348,7 @@ def _parse_news(json_resp):
             'url': result['url'],
             'title': result['title'],
             'content': result['description'],
+            'publishedDate': _extract_published_date(result['age']),
         }
         if result['thumbnail'] is not None:
             item['img_src'] = result['thumbnail']['src']
@@ -313,9 +366,10 @@ def _parse_images(json_resp):
             'title': result['title'],
             'content': result['description'],
             'template': 'images.html',
-            'img_format': result['properties']['format'],
+            'resolution': result['properties']['format'],
             'source': result['source'],
             'img_src': result['properties']['url'],
+            'thumbnail_src': result['thumbnail']['src'],
         }
         result_list.append(item)
 
@@ -335,6 +389,7 @@ def _parse_videos(json_resp):
             'template': 'videos.html',
             'length': result['video']['duration'],
             'duration': result['video']['duration'],
+            'publishedDate': _extract_published_date(result['age']),
         }
 
         if result['thumbnail'] is not None:

searx/engines/bt4g.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """BT4G_ (bt4g.com) is not a tracker and doesn't store any content and only
 collects torrent metadata (such as file names and file sizes) and a magnet link
 (torrent identifier).

searx/engines/btdigg.py

@@ -3,8 +3,9 @@
  BTDigg (Videos, Music, Files)
 """
 
-from lxml import html
 from urllib.parse import quote, urljoin
+
+from lxml import html
 from searx.utils import extract_text, get_torrent_size
 
 # about
@@ -67,7 +68,7 @@ def response(resp):
         # convert files to int if possible
         try:
             files = int(files)
-        except:
+        except:  # pylint: disable=bare-except
             files = None
 
         magnetlink = result.xpath('.//div[@class="torrent_magnet"]//a')[0].attrib['href']

searx/engines/ccc_media.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""media.ccc.de"""
+
+import datetime
+from urllib.parse import urlencode
+
+from dateutil import parser
+
+about = {
+    'website': 'https://media.ccc.de',
+    'official_api_documentation': 'https://github.com/voc/voctoweb',
+    'use_official_api': True,
+    'require_api_key': False,
+    'results': 'JSON',
+}
+categories = ['videos']
+paging = True
+
+api_url = "https://api.media.ccc.de"
+
+
+def request(query, params):
+    args = {'q': query, 'page': params['pageno']}
+    params['url'] = f"{api_url}/public/events/search?{urlencode(args)}"
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    for item in resp.json()['events']:
+        publishedDate = None
+        if item.get('date'):
+            publishedDate = parser.parse(item['date'])
+
+        iframe_src = None
+        if len(item['recordings']) > 0:
+            iframe_src = item['recordings'][0]['recording_url']
+
+        results.append(
+            {
+                'template': 'videos.html',
+                'url': item['frontend_link'],
+                'title': item['title'],
+                'content': item['description'],
+                'thumbnail': item['thumb_url'],
+                'publishedDate': publishedDate,
+                'length': datetime.timedelta(seconds=item['length']),
+                'iframe_src': iframe_src,
+            }
+        )
+
+    return results

searx/engines/chefkoch.py

@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# lint: pylint
 """Chefkoch is a German database of recipes.
 """