diff --git a/Makefile b/Makefile index 005a4a057..fbed8a512 100644 --- a/Makefile +++ b/Makefile @@ -59,16 +59,16 @@ test.shell: utils/brand.env \ $(MTOOLS) \ utils/lib.sh \ - utils/lib_nvm.sh \ - utils/lib_static.sh \ + utils/lib_sxng*.sh \ utils/lib_go.sh \ + utils/lib_nvm.sh \ utils/lib_redis.sh \ - utils/filtron.sh \ - utils/searx.sh \ utils/searxng.sh \ - utils/morty.sh \ utils/lxc.sh \ - utils/lxc-searxng.env + utils/lxc-searxng.env \ + utils/searx.sh \ + utils/filtron.sh \ + utils/morty.sh $(Q)$(MTOOLS) build_msg TEST "$@ OK" diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index d31e4b79b..b7f317f74 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -4,22 +4,30 @@ Buildhosts ========== -.. sidebar:: This article needs some work - - If you have any contribution send us your :pull:`PR <../pulls>`, see - :ref:`how to contribute`. - .. contents:: :depth: 2 :local: :backlinks: entry -To get best results from build, it's recommend to install additional packages -on build hosts (see :ref:`searxng.sh`).:: +To get best results from build, it's recommend to install additional packages on +build hosts (see :ref:`searxng.sh`). - sudo -H ./utils/searxng.sh install buildhost +.. _searxng.sh install buildhost: -This will install packages needed by searx: +Build and Development tools +=========================== + +To Install tools used by build and development tasks in once: + +.. tabs:: + + .. group-tab:: SearXNG's development tools + + .. code:: sh + + $ sudo -H ./utils/searxng.sh install buildhost + +This will install packages needed by SearXNG: .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages @@ -73,7 +81,7 @@ If your docs build (``make docs.html``) shows warnings like this:: display), check the imgmath_latex setting you need to install additional packages on your build host, to get better HTML -output. +output (:ref:`install buildhost `). .. tabs:: @@ -93,7 +101,7 @@ output. .. code-block:: sh - $ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools + $ sudo dnf install graphviz graphviz-gd ImageMagick texlive-xetex-bin librsvg2-tools For PDF output you also need: @@ -117,9 +125,8 @@ For PDF output you also need: .. code:: sh $ sudo dnf install \ - texlive-collection-fontsrecommended texlive-collection-latex \ - dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \ - ImageMagick + texlive-collection-fontsrecommended texlive-collection-latex \ + dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts .. _sh lint: @@ -128,7 +135,8 @@ Lint shell scripts .. _ShellCheck: https://github.com/koalaman/shellcheck -To lint shell scripts, we use ShellCheck_ - a shell script static analysis tool. +To lint shell scripts we use ShellCheck_ - a shell script static analysis tool +(:ref:`install buildhost `). .. SNIP sh lint requirements diff --git a/docs/admin/settings/settings_redis.rst b/docs/admin/settings/settings_redis.rst index 49640f314..df5dc743f 100644 --- a/docs/admin/settings/settings_redis.rst +++ b/docs/admin/settings/settings_redis.rst @@ -24,20 +24,26 @@ developer) account needs to be added to the *searxng-redis* group. rediss://[[username]:[password]]@localhost:6379/0 unix://[[username]:[password]]@/path/to/socket.sock?db=0 -.. admonition:: Tip for developers +.. _Redis Developer Notes: - To set up a local redis instance, first set the socket path of the Redis DB - in your YAML setting: +Redis Developer Notes +===================== - .. code:: yaml +To set up a local redis instance, first set the socket path of the Redis DB +in your YAML setting: - redis: - url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 +.. code:: yaml - Then use the following commands to install the redis instance :: + redis: + url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 - $ ./manage redis.build - $ sudo -H ./manage redis.install - $ sudo -H ./manage redis.addgrp "${USER}" - # don't forget to logout & login to get member of group +Then use the following commands to install the redis instance (:ref:`manage +redis.help`): + +.. code:: sh + + $ ./manage redis.build + $ sudo -H ./manage redis.install + $ sudo -H ./manage redis.addgrp "${USER}" + # don't forget to logout & login to get member of group diff --git a/docs/build-templates/filtron.rst b/docs/build-templates/filtron.rst deleted file mode 100644 index a858f7ddd..000000000 --- a/docs/build-templates/filtron.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. START create user - -.. tabs:: - - .. group-tab:: bash - - .. code-block:: sh - - $ sudo -H useradd --shell /bin/bash --system \\ - --home-dir \"$SERVICE_HOME\" \\ - --comment \"Privacy-respecting metasearch engine\" $SERVICE_USER - - $ sudo -H mkdir \"$SERVICE_HOME\" - $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\" - -.. END create user - -.. START install go - -.. tabs:: - - .. group-tab:: os: linux / arch: amd64 - - - .. code-block:: bash - - $ cat > \"$GO_ENV\" <> ~/.profile - (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local - (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\ - \"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\" - (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\" - (${SERVICE_USER}) $ which go - ${SERVICE_HOME}/local/go/bin/go - -.. END install go - -.. START install filtron - -.. tabs:: - - .. group-tab:: bash - - .. code-block:: bash - - $ sudo -i -u \"${SERVICE_USER}\" - (${SERVICE_USER}) $ go get -v -u github.com/searxng/filtron - -.. END install filtron diff --git a/docs/build-templates/morty.rst b/docs/build-templates/morty.rst deleted file mode 100644 index b03a54f96..000000000 --- a/docs/build-templates/morty.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. START create user - -.. tabs:: - - .. group-tab:: bash - - .. code-block:: sh - - $ sudo -H useradd --shell /bin/bash --system \\ - --home-dir \"$SERVICE_HOME\" \\ - --comment \"Privacy-respecting metasearch engine\" $SERVICE_USER - - $ sudo -H mkdir \"$SERVICE_HOME\" - $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\" - -.. END create user - -.. START install go - -.. tabs:: - - .. group-tab:: os: linux / arch: amd64 - - - .. code-block:: bash - - $ cat > \"$GO_ENV\" <> ~/.profile - (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local - (${SERVICE_USER}) $ wget --progress=bar -O \"${GO_VERSION}.linux-amd64.tar.gz\" \\ - \"${GO_DL_URL}/${GO_VERSION}.linux-amd64.tar.gz\" - (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local -xzf \"${GO_VERSION}.linux-amd64.tar.gz\" - (${SERVICE_USER}) $ which go - ${SERVICE_HOME}/local/go/bin/go - -.. END install go - -.. START install morty - -.. tabs:: - - .. group-tab:: bash - - .. code-block:: bash - - $ sudo -i -u \"${SERVICE_USER}\" - (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty - -.. END install morty diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 67133b6b3..bc4f6514d 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -148,7 +148,7 @@ live build Live build is like WYSIWYG. If you want to edit the documentation, its recommended to use. The Makefile target ``docs.live`` builds the docs, opens URL in your favorite browser and rebuilds every time a reST file has been -changed. +changed (:ref:`make docs.clean`). .. code:: sh @@ -183,3 +183,8 @@ commit and push: .. code:: sh $ make docs.clean docs.gh-pages + +.. attention:: + + If you are working in your own brand, don't forgett to adjust your + :ref:`settings brand`. diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index ad5665de7..62ac2e905 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -1,32 +1,49 @@ .. _makefile: -======== -Makefile -======== +======================= +Makefile & ``./manage`` +======================= .. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction +All relevant build and development tasks are implemented in the +:origin:`./manage ` script and for CI or IDE integration a small +:origin:`Makefile` wrapper is available. If you are not familiar with +Makefiles, we recommend to read gnu-make_ introduction. + .. sidebar:: build environment Before looking deeper at the targets, first read about :ref:`make install`. - To install system requirements follow :ref:`buildhosts`. + To install developer requirements follow :ref:`buildhosts`. -All relevant build tasks are implemented in :origin:`manage` and for CI or -IDE integration a small ``Makefile`` wrapper is available. If you are not -familiar with Makefiles, we recommend to read gnu-make_ introduction. - -The usage is simple, just type ``make {target-name}`` to *build* a target. -Calling the ``help`` target gives a first overview (``make help``): - -.. program-output:: bash -c "cd ..; make --no-print-directory help" .. contents:: :depth: 2 :local: :backlinks: entry +The usage is simple, just type ``make {target-name}`` to *build* a target. +Calling the ``help`` target gives a first overview (``make help``): + +.. tabs:: + + .. group-tab:: ``make`` + + .. program-output:: bash -c "cd ..; make --no-print-directory help" + + + .. group-tab:: ``./manage`` + + The Makefile targets are implemented for comfort, if you can do without + tab-completion and need to have a more granular control, use + :origin:`manage` without the Makefile wrappers. + + .. code:: sh + + $ ./manage help + .. _make install: Python environment (``make install``) @@ -158,29 +175,49 @@ Node.js environment (``make node.env``) Manager) to install latest LTS of Node.js_ locally: there is no need to install nvm_ or npm_ on your system. -Use ``make nvm.status`` to get the current status of you Node.js_ and nvm_ setup. +To install NVM_ and Node.js_ in once you can use :ref:`make nvm.nodejs`. -Here is the output you will typically get on a Ubuntu 20.04 system which serves -only a `no longer active `_ Release -`Node.js v10.19.0 `_. +.. _make nvm: -:: +NVM ``make nvm.install nvm.status`` +----------------------------------- - $ make nvm.status - INFO: Node.js is installed at /usr/bin/node - INFO: Node.js is version v10.19.0 - WARN: minimal Node.js version is 16.13.0 - INFO: npm is installed at /usr/bin/npm - INFO: npm is version 6.14.4 - WARN: NVM is not installed - INFO: to install NVM and Node.js (LTS) use: manage nvm install --lts +Use ``make nvm.status`` to get the current status of your Node.js_ and nvm_ +setup. -To install you can also use :ref:`make nvm.nodejs` +.. tabs:: + + .. group-tab:: nvm.install + + .. code:: sh + + $ LANG=C make nvm.install + INFO: install (update) NVM at ./searxng/.nvm + INFO: clone: https://github.com/nvm-sh/nvm.git + || Cloning into './searxng/.nvm'... + INFO: checkout v0.39.4 + || HEAD is now at 8fbf8ab v0.39.4 + + .. group-tab:: nvm.status (ubu2004) + + Here is the output you will typically get on a Ubuntu 20.04 system which + serves only a `no longer active `_ + Release `Node.js v10.19.0 `_. + + .. code:: sh + + $ make nvm.status + INFO: Node.js is installed at /usr/bin/node + INFO: Node.js is version v10.19.0 + WARN: minimal Node.js version is 16.13.0 + INFO: npm is installed at /usr/bin/npm + INFO: npm is version 6.14.4 + WARN: NVM is not installed .. _make nvm.nodejs: ``make nvm.nodejs`` -=================== +------------------- Install latest Node.js_ LTS locally (uses nvm_):: @@ -213,10 +250,29 @@ sources of the theme need to be rebuild. You can do that by running:: $ make themes.all Alternatively to ``themes.all`` you can run *live builds* of the theme you are -modify:: +modify (:ref:`make themes`):: $ LIVE_THEME=simple make run +.. _make format.python: + +``make format.python`` +====================== + +Format Python sourcee code using `Black code style`_. See ``$BLACK_OPTIONS`` +and ``$BLACK_TARGETS`` in :origin:`Makefile`. + +.. attention:: + + We stuck at Black 22.12.0, please read comment in PR `Bump black from 22.12.0 + to 23.1.0`_ + +.. _Bump black from 22.12.0 to 23.1.0: + https://github.com/searxng/searxng/pull/2159#pullrequestreview-1284094735 + +.. _Black code style: + https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html + .. _make clean: ``make clean`` @@ -237,18 +293,34 @@ calling ``make clean`` stop all processes using the :ref:`make install` or .. _make docs: -``make docs docs.autobuild docs.clean`` -======================================= +``make docs`` +============= + +Target ``docs`` builds the documentation: + +.. code:: bash + + $ make docs + HTML ./docs --> file:// + DOCS build build/docs/includes + ... + The HTML pages are in dist/docs. + +.. _make docs.clean: + +``make docs.clean docs.live`` +---------------------------------- We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute / Documentation ` section. If you want to edit the documentation read our :ref:`make docs.live` section. If you are working in your own brand, adjust your :ref:`settings brand`. + .. _make docs.gh-pages: ``make docs.gh-pages`` -====================== +---------------------- To deploy on github.io first adjust your :ref:`settings brand`. For any further read :ref:`deploy on github.io`. @@ -261,17 +333,17 @@ further read :ref:`deploy on github.io`. Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit`` and ``test.robot``. You can run tests selective, e.g.:: - $ make test.pep8 test.unit test.sh + $ make test.pep8 test.unit test.shell TEST test.pep8 OK ... TEST test.unit OK ... - TEST test.sh OK + TEST test.shell OK .. _make test.shell: ``make test.shell`` -=================== +------------------- :ref:`sh lint` / if you have changed some bash scripting run this test before commit. @@ -279,7 +351,7 @@ commit. .. _make test.pylint: ``make test.pylint`` -==================== +-------------------- .. _Pylint: https://www.pylint.org/ @@ -289,8 +361,8 @@ found in project's root folder :origin:`.pylintrc`. .. _make search.checker: -``search.checker.{engine name}`` -================================ +``make search.checker.{engine name}`` +===================================== To check all engines:: @@ -318,3 +390,63 @@ To filter out HTTP redirects (3xx_):: https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0 https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None -- + +.. _make themes: + +``make themes.*`` +================= + +.. sidebar:: further read + + - :ref:`devquickstart` + +The :origin:`Makefile` targets ``make theme.*`` cover common tasks to build the +theme(s). The ``./manage themes.*`` command line can be used to convenient run +common theme build tasks. + +.. program-output:: bash -c "cd ..; ./manage themes.help" + +To get live builds while modifying CSS & JS use (:ref:`make run`): + +.. code:: sh + + $ LIVE_THEME=simple make run + +.. _make static.build: + +``make static.build.*`` +======================= + +.. sidebar:: further read + + - :ref:`devquickstart` + +The :origin:`Makefile` targets ``static.build.*`` cover common tasks to build (a +commit of) the static files. The ``./manage static.build..*`` command line +can be used to convenient run common build tasks of the satic files. + +.. program-output:: bash -c "cd ..; ./manage static.help" + + +.. _manage redis.help: + +``./manage redis.help`` +======================= + +The ``./manage redis.*`` command line can be used to convenient run common Redis +tasks (:ref:`Redis developer notes`). + +.. program-output:: bash -c "cd ..; ./manage redis.help" + + +.. _manage go.help: + +``./manage go.help`` +==================== + +The ``./manage go.*`` command line can be used to convenient run common `go +(wiki)`_ tasks. + +.. _go (wiki): https://en.wikipedia.org/wiki/Go_(programming_language) + +.. program-output:: bash -c "cd ..; ./manage go.help" diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index 921384aab..9bee8f60c 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -7,8 +7,16 @@ Development Quickstart .. _npm: https://www.npmjs.com/ .. _Node.js: https://nodejs.org/ -SearXNG loves developers, just clone and start hacking. All the rest is done for -you simply by using :ref:`make `. + +.. sidebar:: further read + + - :ref:`makefile` + - :ref:`buildhosts` + +SearXNG loves developers; Developers do not need to worry about tool chains, the +usual developer tasks can be comfortably executed via :ref:`make `. + +Don't hesitate, just clone SearXNG's sources and start hacking right now .. .. code:: bash @@ -18,25 +26,23 @@ Here is how a minimal workflow looks like: 1. *start* hacking 2. *run* your code: :ref:`make run` -3. *test* your code: :ref:`make test` +3. *format & test* your code: :ref:`make format.python` and :ref:`make test` If you think at some point something fails, go back to *start*. Otherwise, choose a meaningful commit message and we are happy to receive your pull request. To not end in *wild west* we have some directives, please pay attention to our ":ref:`how to contribute`" guideline. -If you implement themes, you will need to setup a :ref:`make node.env` once: +.. sidebar:: further read -.. code:: bash + - :ref:`make nvm` + - :ref:`make themes` - make node.env +If you implement themes, you will need to setup a :ref:`Node.js environment +`: ``make node.env`` Before you call *make run* (2.), you need to compile the modified styles and -JavaScript: - -.. code:: bash - - make themes.all +JavaScript: ``make themes.all`` Alternatively you can also compile selective the theme you have modified, e.g. the *simple* theme. @@ -49,6 +55,10 @@ e.g. the *simple* theme. To get live builds while modifying CSS & JS use: ``LIVE_THEME=simple make run`` +.. sidebar:: further read + + - :ref:`make static.build` + If you finished your *tests* you can start to commit your changes. To separate the modified source code from the build products first run: diff --git a/manage b/manage index 52c038a0c..380560362 100755 --- a/manage +++ b/manage @@ -11,8 +11,23 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" -# shellcheck source=utils/lib_static.sh -source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh" +# shellcheck source=utils/lib_sxng_data.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh" + +# shellcheck source=utils/lib_sxng_weblate.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh" + +# shellcheck source=utils/lib_sxng_static.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh" + +# shellcheck source=utils/lib_sxng_node.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh" + +# shellcheck source=utils/lib_sxng_themes.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh" + +# shellcheck source=utils/lib_sxng_test.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh" # shellcheck source=utils/lib_go.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh" @@ -27,9 +42,8 @@ PATH="${REPO_ROOT}/node_modules/.bin:${PATH}" PYOBJECTS="searx" PY_SETUP_EXTRAS='[test]' GECKODRIVER_VERSION="v0.30.0" -export NODE_MINIMUM_VERSION="16.13.0" # SPHINXOPTS= -BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization") +BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization") BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests") _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock" @@ -73,13 +87,6 @@ buildenv: rebuild ./utils/brand.env webapp.: run : run developer instance -weblate.: - push.translations: push translation changes from SearXNG to Weblate's counterpart - to.translations: Update 'translations' branch with last additions from Weblate. -data.: - all : update searx/sxng_locales.py and searx/data/* - traits : update searx/data/engine_traits.json & searx/sxng_locales.py - useragents: update searx/data/useragents.json with the most recent versions of Firefox docs.: html : build HTML documentation live : autobuild HTML documentation while editing @@ -96,10 +103,6 @@ redis: build : build redis binaries at $(redis._get_dist) install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME}) help : show more redis commands -node.: - env : download & install SearXNG's npm dependencies locally - env.dev : download & install developer and CI tools - clean : drop locally npm installations py.: build : Build python packages at ./${PYDIST} clean : delete virtualenv and intermediate py files @@ -112,24 +115,16 @@ pypi.upload: Upload python packages to PyPi (to test use pypi.upload.test) format.: python : format Python code source using black -test.: - yamllint : lint YAML files (YAMLLINT_FILES) - pylint : lint PYLINT_FILES, searx/engines, searx & tests - pyright : static type check of python sources - black : check black code format - unit : run unit tests - coverage : run unit tests with coverage - robot : run robot test - rst : test .rst files incl. README.rst - clean : clean intermediate test stuff -themes.: - all : build all themes - simple : build simple theme pygments.: less : build LESS files for pygments EOF go.help - static_help + node.help + weblate.help + data.help + test.help + themes.help + static.help cat < /dev/null; then - git remote add weblate https://translate.codeberg.org/git/searxng/searxng/ - fi - if [ -d "${TRANSLATIONS_WORKTREE}" ]; then - pushd . - cd "${TRANSLATIONS_WORKTREE}" - git reset --hard HEAD - git pull origin translations - popd - else - mkdir -p "${TRANSLATIONS_WORKTREE}" - git worktree add "${TRANSLATIONS_WORKTREE}" translations - fi - ) -} - -weblate.to.translations() { - - # Update 'translations' branch of SearXNG (origin) with last additions from - # Weblate. - - # 1. Check if Weblate is locked, if not die with error message - # 2. On Weblate's counterpart (weblate), pull master and translations branch - # from SearXNG (origin). - # 3. Commit changes made in a Weblate object on Weblate's counterpart - # (weblate). - # 4. In translations worktree, merge changes of branch 'translations' from - # remote 'weblate' and push it on branch 'translations' of 'origin' - - ( set -e - pyenv.activate - if [ "$(wlc lock-status)" != "locked: True" ]; then - die 1 "weblate must be locked, currently: $(wlc lock-status)" - fi - # weblate: commit pending changes - wlc pull - wlc commit - - # get the translations in a worktree - weblate.translations.worktree - - pushd "${TRANSLATIONS_WORKTREE}" - git remote update weblate - git merge weblate/translations - git push - popd - ) - dump_return $? -} - -weblate.translations.commit() { - - # Update 'translations' branch of SearXNG (origin) with last additions from - # Weblate. Copy the changes to the master branch, compile translations and - # create a commit in the local branch (master) - - local existing_commit_hash commit_body commit_message exitcode - ( set -e - pyenv.activate - # lock change on weblate - wlc lock - - # get translations branch in git worktree (TRANSLATIONS_WORKTREE) - weblate.translations.worktree - existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h') - - # pull weblate commits - weblate.to.translations - - # copy the changes to the master branch - cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx" - - # compile translations - build_msg BABEL 'compile translation catalogs into binary MO files' - pybabel compile --statistics \ - -d "searx/translations" - # git add/commit (no push) - commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD") - commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}") - git add searx/translations - git commit -m "${commit_message}" - ) - exitcode=$? - ( # make sure to always unlock weblate - set -e - pyenv.cmd wlc unlock - ) - dump_return $exitcode -} - -weblate.push.translations() { - - # Push *translation changes* from SearXNG (origin) to Weblate's counterpart - # (weblate). - - # In branch master of SearXNG (origin) check for meaningful changes in - # folder 'searx/translations', commit changes on branch 'translations' and - # at least, pull updated branches on Weblate's counterpart (weblate). - - # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch - # 'translations' from remote 'weblate'. - # 2. Stop if there is no meaningful change in the 'master' branch (origin), - # compared to the 'translations' branch (weblate), otherwise ... - # 3. Update 'translations' branch of SearXNG (origin) with last additions - # from Weblate. - # 5. Notify Weblate to pull updated 'master' & 'translations' branch. - - local messages_pot diff_messages_pot last_commit_hash last_commit_detail \ - exitcode - messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot" - ( set -e - pyenv.activate - # get translations branch in git worktree (TRANSLATIONS_WORKTREE) - weblate.translations.worktree - - # update messages.pot in the master branch - build_msg BABEL 'extract messages from source files and generate POT file' - pybabel extract -F babel.cfg \ - -o "${messages_pot}" \ - "searx/" - - # stop if there is no meaningful change in the master branch - diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\ - git diff -- "searx/translations/messages.pot") - if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then - build_msg BABEL 'no changes detected, exiting' - return 42 - fi - return 0 - ) - exitcode=$? - if [ "$exitcode" -eq 42 ]; then - return 0 - fi - if [ "$exitcode" -gt 0 ]; then - return $exitcode - fi - ( - set -e - pyenv.activate - - # lock change on weblate - # weblate may add commit(s) since the call to "weblate.translations.worktree". - # this is not a problem because after this line, "weblate.to.translations" - # calls again "weblate.translations.worktree" which calls "git pull" - wlc lock - - # save messages.pot in the translations branch for later - pushd "${TRANSLATIONS_WORKTREE}" - git stash push - popd - - # merge weblate commits into the translations branch - weblate.to.translations - - # restore messages.pot in the translations branch - pushd "${TRANSLATIONS_WORKTREE}" - git stash pop - popd - - # update messages.po files in the master branch - build_msg BABEL 'update existing message catalogs from POT file' - pybabel update -N \ - -i "${messages_pot}" \ - -d "${TRANSLATIONS_WORKTREE}/searx/translations" - - # git add/commit/push - last_commit_hash=$(git log -n1 --pretty=format:'%h') - last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}") - - pushd "${TRANSLATIONS_WORKTREE}" - git add searx/translations - git commit \ - -m "[translations] update messages.pot and messages.po files" \ - -m "From ${last_commit_detail}" - git push - popd - - # notify weblate to pull updated master & translations branch - wlc pull - ) - exitcode=$? - ( # make sure to always unlock weblate - set -e - pyenv.activate - wlc unlock - ) - dump_return $exitcode -} - -data.all() { - ( set -e - - pyenv.activate - data.traits - data.useragents - - build_msg DATA "update searx/data/osm_keys_tags.json" - pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py - build_msg DATA "update searx/data/ahmia_blacklist.txt" - python searxng_extra/update/update_ahmia_blacklist.py - build_msg DATA "update searx/data/wikidata_units.json" - python searxng_extra/update/update_wikidata_units.py - build_msg DATA "update searx/data/currencies.json" - python searxng_extra/update/update_currencies.py - build_msg DATA "update searx/data/external_bangs.json" - python searxng_extra/update/update_external_bangs.py - build_msg DATA "update searx/data/engine_descriptions.json" - python searxng_extra/update/update_engine_descriptions.py - ) -} - - -data.traits() { - ( set -e - pyenv.activate - build_msg DATA "update searx/data/engine_traits.json" - python searxng_extra/update/update_engine_traits.py - build_msg ENGINES "update searx/sxng_locales.py" - ) - dump_return $? -} - -data.useragents() { - build_msg DATA "update searx/data/useragents.json" - pyenv.cmd python searxng_extra/update/update_firefox_version.py - dump_return $? -} - -docs.prebuild() { - build_msg DOCS "build ${DOCS_BUILD}/includes" - ( - set -e - [ "$VERBOSE" = "1" ] && set -x - mkdir -p "${DOCS_BUILD}/includes" - ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" - pyenv.cmd searxng_extra/docs_prebuild - ) - dump_return $? -} - docker.push() { docker.build push } @@ -554,44 +297,6 @@ gecko.driver() { dump_return $? } -nodejs.ensure() { - if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then - info_msg "install Node.js by NVM" - nvm.nodejs - fi -} - -node.env() { - nodejs.ensure - ( set -e - build_msg INSTALL "./searx/static/themes/simple/package.json" - npm --prefix searx/static/themes/simple install - ) - dump_return $? -} - -node.env.dev() { - nodejs.ensure - build_msg INSTALL "./package.json: developer and CI tools" - npm install -} - -node.clean() { - if ! required_commands npm 2>/dev/null; then - build_msg CLEAN "npm is not installed / ignore npm dependencies" - return 0 - fi - build_msg CLEAN "themes -- locally installed npm dependencies" - ( set -e - npm --prefix searx/static/themes/simple run clean - ) - build_msg CLEAN "locally installed developer and CI tools" - ( set -e - npm --prefix . run clean - ) - dump_return $? -} - pygments.less() { build_msg PYGMENTS "searxng_extra/update/update_pygments.py" if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then @@ -674,150 +379,6 @@ format.python() { dump_return $? } -test.yamllint() { - build_msg TEST "[yamllint] \$YAMLLINT_FILES" - pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}" - dump_return $? -} - -test.pylint() { - # shellcheck disable=SC2086 - ( set -e - build_msg TEST "[pylint] \$PYLINT_FILES" - pyenv.activate - python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ - --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \ - "${PYLINT_FILES[@]}" - - build_msg TEST "[pylint] searx/engines" - python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ - --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \ - --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \ - searx/engines - - build_msg TEST "[pylint] searx tests" - python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ - --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \ - --ignore=searx/engines \ - searx tests - ) - dump_return $? -} - -test.pyright() { - build_msg TEST "[pyright] static type check of python sources" - node.env.dev - # We run Pyright in the virtual environment because Pyright - # executes "python" to determine the Python version. - build_msg TEST "[pyright] suppress warnings related to intentional monkey patching" - pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \ - | grep -v ".py$" \ - | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\ - | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\ - | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \ - | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \ - | grep -v '/engines/.*.py.* - warning: "categories" is not defined' - dump_return $? -} - -test.black() { - build_msg TEST "[black] \$BLACK_TARGETS" - pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}" - dump_return $? -} - -test.unit() { - build_msg TEST 'tests/unit' - pyenv.cmd python -m nose2 -s tests/unit - dump_return $? -} - -test.coverage() { - build_msg TEST 'unit test coverage' - ( set -e - pyenv.activate - python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit - coverage report - coverage html - ) - dump_return $? -} - -test.robot() { - build_msg TEST 'robot' - gecko.driver - PYTHONPATH=. pyenv.cmd python -m tests.robot - dump_return $? -} - -test.rst() { - build_msg TEST "[reST markup] ${RST_FILES[*]}" - for rst in "${RST_FILES[@]}"; do - pyenv.cmd rst2html.py --halt error "$rst" > /dev/null || die 42 "fix issue in $rst" - done -} - -test.pybabel() { - TEST_BABEL_FOLDER="build/test/pybabel" - build_msg TEST "[extract messages] pybabel" - mkdir -p "${TEST_BABEL_FOLDER}" - pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx -} - -test.clean() { - build_msg CLEAN "test stuff" - rm -rf geckodriver.log .coverage coverage/ - dump_return $? -} - -themes.all() { - ( set -e - pygments.less - node.env - themes.simple - ) - dump_return $? -} - -themes.live() { - local LIVE_THEME="${LIVE_THEME:-${1}}" - case "${LIVE_THEME}" in - simple) - theme="searx/static/themes/${LIVE_THEME}" - ;; - '') - die_caller 42 "missing theme argument" - ;; - *) - die_caller 42 "unknown theme '${LIVE_THEME}' // [simple]'" - ;; - esac - build_msg GRUNT "theme: $1 (live build)" - nodejs.ensure - cd "${theme}" - { - npm install - npm run watch - } 2>&1 \ - | prefix_stdout "${_Blue}THEME ${1} ${_creset} " \ - | grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |' -} - -themes.simple() { - ( set -e - build_msg GRUNT "theme: simple" - npm --prefix searx/static/themes/simple run build - ) - dump_return $? -} - -themes.simple.test() { - build_msg TEST "theme: simple" - nodejs.ensure - npm --prefix searx/static/themes/simple install - npm --prefix searx/static/themes/simple run test - dump_return $? -} PYLINT_FILES=() while IFS= read -r line; do diff --git a/utils/lib_go.sh b/utils/lib_go.sh index 314204e1a..462bbbba0 100755 --- a/utils/lib_go.sh +++ b/utils/lib_go.sh @@ -20,9 +20,6 @@ # shellcheck source=utils/lib.sh . /dev/null -# shellcheck disable=SC2034 -declare main_cmd - # configure golang environment # ---------------------------- diff --git a/utils/lib_sxng_data.sh b/utils/lib_sxng_data.sh new file mode 100755 index 000000000..549e6dbec --- /dev/null +++ b/utils/lib_sxng_data.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +data.help(){ + cat < "${DOCS_BUILD}/includes/searxng.rst" + pyenv.cmd searxng_extra/docs_prebuild + ) + dump_return $? +} diff --git a/utils/lib_sxng_node.sh b/utils/lib_sxng_node.sh new file mode 100755 index 000000000..eac5bb16b --- /dev/null +++ b/utils/lib_sxng_node.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +export NODE_MINIMUM_VERSION="16.13.0" + +node.help(){ + cat </dev/null; then + build_msg CLEAN "npm is not installed / ignore npm dependencies" + return 0 + fi + build_msg CLEAN "themes -- locally installed npm dependencies" + ( set -e + npm --prefix searx/static/themes/simple run clean + ) + build_msg CLEAN "locally installed developer and CI tools" + ( set -e + npm --prefix . run clean + ) + dump_return $? +} diff --git a/utils/lib_static.sh b/utils/lib_sxng_static.sh similarity index 99% rename from utils/lib_static.sh rename to utils/lib_sxng_static.sh index 754531700..1d76bfe70 100755 --- a/utils/lib_static.sh +++ b/utils/lib_sxng_static.sh @@ -12,7 +12,7 @@ STATIC_BUILT_PATHS=( 'searx/templates/simple/icons.html' ) -static_help(){ +static.help(){ cat < /dev/null || die 42 "fix issue in $rst" + done +} + +test.pybabel() { + TEST_BABEL_FOLDER="build/test/pybabel" + build_msg TEST "[extract messages] pybabel" + mkdir -p "${TEST_BABEL_FOLDER}" + pyenv.cmd pybabel extract -F babel.cfg -o "${TEST_BABEL_FOLDER}/messages.pot" searx +} + +test.clean() { + build_msg CLEAN "test stuff" + rm -rf geckodriver.log .coverage coverage/ + dump_return $? +} + diff --git a/utils/lib_sxng_themes.sh b/utils/lib_sxng_themes.sh new file mode 100755 index 000000000..8d41b6480 --- /dev/null +++ b/utils/lib_sxng_themes.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +declare _Blue +declare _creset + +themes.help(){ + cat <&1 \ + | prefix_stdout "${_Blue}THEME ${1} ${_creset} " \ + | grep -E --ignore-case --color 'error[s]?[:]? |warning[s]?[:]? |' +} + +themes.simple() { + ( set -e + build_msg GRUNT "theme: simple" + npm --prefix searx/static/themes/simple run build + ) + dump_return $? +} + +themes.simple.test() { + build_msg TEST "theme: simple" + nodejs.ensure + npm --prefix searx/static/themes/simple install + npm --prefix searx/static/themes/simple run test + dump_return $? +} diff --git a/utils/lib_sxng_weblate.sh b/utils/lib_sxng_weblate.sh new file mode 100755 index 000000000..f52b75d7c --- /dev/null +++ b/utils/lib_sxng_weblate.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +weblate.help(){ + cat < /dev/null; then + git remote add weblate https://translate.codeberg.org/git/searxng/searxng/ + fi + if [ -d "${TRANSLATIONS_WORKTREE}" ]; then + pushd . + cd "${TRANSLATIONS_WORKTREE}" + git reset --hard HEAD + git pull origin translations + popd + else + mkdir -p "${TRANSLATIONS_WORKTREE}" + git worktree add "${TRANSLATIONS_WORKTREE}" translations + fi + ) +} + +weblate.to.translations() { + + # Update 'translations' branch of SearXNG (origin) with last additions from + # Weblate. + + # 1. Check if Weblate is locked, if not die with error message + # 2. On Weblate's counterpart (weblate), pull master and translations branch + # from SearXNG (origin). + # 3. Commit changes made in a Weblate object on Weblate's counterpart + # (weblate). + # 4. In translations worktree, merge changes of branch 'translations' from + # remote 'weblate' and push it on branch 'translations' of 'origin' + + ( set -e + pyenv.activate + if [ "$(wlc lock-status)" != "locked: True" ]; then + die 1 "weblate must be locked, currently: $(wlc lock-status)" + fi + # weblate: commit pending changes + wlc pull + wlc commit + + # get the translations in a worktree + weblate.translations.worktree + + pushd "${TRANSLATIONS_WORKTREE}" + git remote update weblate + git merge weblate/translations + git push + popd + ) + dump_return $? +} + +weblate.translations.commit() { + + # Update 'translations' branch of SearXNG (origin) with last additions from + # Weblate. Copy the changes to the master branch, compile translations and + # create a commit in the local branch (master) + + local existing_commit_hash commit_body commit_message exitcode + ( set -e + pyenv.activate + # lock change on weblate + wlc lock + + # get translations branch in git worktree (TRANSLATIONS_WORKTREE) + weblate.translations.worktree + existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h') + + # pull weblate commits + weblate.to.translations + + # copy the changes to the master branch + cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx" + + # compile translations + build_msg BABEL 'compile translation catalogs into binary MO files' + pybabel compile --statistics \ + -d "searx/translations" + # git add/commit (no push) + commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD") + commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}") + git add searx/translations + git commit -m "${commit_message}" + ) + exitcode=$? + ( # make sure to always unlock weblate + set -e + pyenv.cmd wlc unlock + ) + dump_return $exitcode +} + +weblate.push.translations() { + + # Push *translation changes* from SearXNG (origin) to Weblate's counterpart + # (weblate). + + # In branch master of SearXNG (origin) check for meaningful changes in + # folder 'searx/translations', commit changes on branch 'translations' and + # at least, pull updated branches on Weblate's counterpart (weblate). + + # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch + # 'translations' from remote 'weblate'. + # 2. Stop if there is no meaningful change in the 'master' branch (origin), + # compared to the 'translations' branch (weblate), otherwise ... + # 3. Update 'translations' branch of SearXNG (origin) with last additions + # from Weblate. + # 5. Notify Weblate to pull updated 'master' & 'translations' branch. + + local messages_pot diff_messages_pot last_commit_hash last_commit_detail \ + exitcode + messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot" + ( set -e + pyenv.activate + # get translations branch in git worktree (TRANSLATIONS_WORKTREE) + weblate.translations.worktree + + # update messages.pot in the master branch + build_msg BABEL 'extract messages from source files and generate POT file' + pybabel extract -F babel.cfg \ + -o "${messages_pot}" \ + "searx/" + + # stop if there is no meaningful change in the master branch + diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\ + git diff -- "searx/translations/messages.pot") + if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then + build_msg BABEL 'no changes detected, exiting' + return 42 + fi + return 0 + ) + exitcode=$? + if [ "$exitcode" -eq 42 ]; then + return 0 + fi + if [ "$exitcode" -gt 0 ]; then + return $exitcode + fi + ( + set -e + pyenv.activate + + # lock change on weblate + # weblate may add commit(s) since the call to "weblate.translations.worktree". + # this is not a problem because after this line, "weblate.to.translations" + # calls again "weblate.translations.worktree" which calls "git pull" + wlc lock + + # save messages.pot in the translations branch for later + pushd "${TRANSLATIONS_WORKTREE}" + git stash push + popd + + # merge weblate commits into the translations branch + weblate.to.translations + + # restore messages.pot in the translations branch + pushd "${TRANSLATIONS_WORKTREE}" + git stash pop + popd + + # update messages.po files in the master branch + build_msg BABEL 'update existing message catalogs from POT file' + pybabel update -N \ + -i "${messages_pot}" \ + -d "${TRANSLATIONS_WORKTREE}/searx/translations" + + # git add/commit/push + last_commit_hash=$(git log -n1 --pretty=format:'%h') + last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}") + + pushd "${TRANSLATIONS_WORKTREE}" + git add searx/translations + git commit \ + -m "[translations] update messages.pot and messages.po files" \ + -m "From ${last_commit_detail}" + git push + popd + + # notify weblate to pull updated master & translations branch + wlc pull + ) + exitcode=$? + ( # make sure to always unlock weblate + set -e + pyenv.activate + wlc unlock + ) + dump_return $exitcode +}