#!/usr/bin/env bash # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck disable=SC2031 # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/brand.env source "${REPO_ROOT}/utils/brand.env" source_dot_config # config PYOBJECTS="searx" PY_SETUP_EXTRAS='[test]' NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli" GECKODRIVER_VERSION="v0.28.0" # SPHINXOPTS= pylint.FILES() { # List files tagged by comment: # # # lint: pylint # # These py files are linted by test.pylint(), all other files are linted by # test.pep8() grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searx_extra tests } PYLINT_SEARX_DISABLE_OPTION="\ I,C,R,\ W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\ E1136" PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases" PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc" help() { cat <<EOF buildenv rebuild ./utils/brand.env babel.compile pybabel compile ./searx/translations data.* all : update searx/languages.py and ./data/* languages : update searx/data/engines_languages.json & searx/languages.py useragents: update searx/data/useragents.json with the most recent versions of Firefox. docs.* html : build HTML documentation live : autobuild HTML documentation while editing gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build docker build : build docker image push : build and push docker image gecko.driver download & install geckodriver if not already installed (required for robot_tests) node.* env : download & install npm dependencies locally clean : drop npm installations py.* build : Build python packages at ./${PYDIST} clean : delete virtualenv and intermediate py files pyenv.* : install : developer install of searx into virtualenv uninstall : uninstall developer installation cmd ... : run command ... in virtualenv OK : test if virtualenv is OK pypi.upload: Upload python packages to PyPi (to test use pypi.upload.test) test.* : pylint : lint PYLINT_FILES, searx/engines, searx & tests pep8 : pycodestyle (pep8) for all files except PYLINT_FILES unit : run unit tests coverage : run unit tests with coverage robot : run robot test clean : clean intermediate test stuff themes.* : all : build all themes oscar : build oscar theme simple : build simple theme EOF } if [ "$VERBOSE" = "1" ]; then SPHINX_VERBOSE="-v" PYLINT_VERBOSE="-v" fi # needed by sphinx-docs export DOCS_BUILD buildenv() { SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \ | prefix_stdout "${_Blue}BUILDENV${_creset} " return "${PIPESTATUS[0]}" } babel.compile() { build_msg BABEL compile pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations" dump_return $? } data.all() { data.languages data.useragents build_msg DATA "update searx/data/ahmia_blacklist.txt" pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py build_msg DATA "update searx/data/wikidata_units.json" pyenv.cmd python searx_extra/update/update_wikidata_units.py build_msg DATA "update searx/data/currencies.json" pyenv.cmd python searx_extra/update/update_currencies.py } data.languages() { ( set -e build_msg ENGINES "fetch languages .." pyenv.cmd python searx_extra/update/update_languages.py build_msg ENGINES "update update searx/languages.py" build_msg DATA "update searx/data/engines_languages.json" ) dump_return $? } data.useragents() { build_msg DATA "update searx/data/useragents.json" pyenv.cmd python searx_extra/update/update_firefox_version.py dump_return $? } docs.prebuild() { build_msg DOCS "build ${DOCS_BUILD}/includes" ( set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst" ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst" ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst" ) dump_return $? } docker.push() { docker.build push } docker.buildx() { docker.build buildx } # shellcheck disable=SC2119 docker.build() { pyenv.install build_msg DOCKER build # run installation in a subprocess and activate pyenv # See https://www.shellcheck.net/wiki/SC1001 and others .. # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 ( set -e # shellcheck source=/dev/null source "${PY_ENV_BIN}/activate" # Check if it is a git repository if [ ! -d .git ]; then die 1 "This is not Git repository" fi if [ ! -x "$(which git)" ]; then die 1 "git is not installed" fi if ! git remote get-url origin 2> /dev/null; then die 1 "there is no remote origin" fi # This is a git repository # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e) # awk to remove the "v" and the "g" SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}') # add the suffix "-dirty" if the repository has uncommited change # /!\ HACK for searx/searx: ignore utils/brand.env git update-index -q --refresh if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty" fi # Get the last git commit id, will be added to the Searx version (see Dockerfile) VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4) build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" # Check consistency between the git tag and the searx/version.py file # /! HACK : parse Python file with bash /! # otherwise it is not possible build the docker image without all Python # dependencies ( version.py loads __init__.py ) # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)") SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -) if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then err_msg "git tag: $SEARX_GIT_VERSION" err_msg "searx/version.py: $SEARX_PYTHON_VERSION" die 1 "Inconsistency between the last git tag and the searx/version.py file" fi # define the docker image name GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" BUILD="build" if [ "$1" = "buildx" ]; then # buildx includes the push option CACHE_TAG="${SEARX_IMAGE_NAME}:latest-build-cache" BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" shift fi build_msg DOCKER "Build command: ${BUILD}" # build Docker image build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" # shellcheck disable=SC2086 docker $BUILD \ --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ --build-arg GIT_URL="${GIT_URL}" \ --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ --build-arg LABEL_VCS_URL="${GIT_URL}" \ --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" . if [ "$1" = "push" ]; then docker push "${SEARX_IMAGE_NAME}:latest" docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" fi ) dump_return $? } # shellcheck disable=SC2119 gecko.driver() { pyenv.install build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv ( set -e # shellcheck source=/dev/null source "${PY_ENV_BIN}/activate" # TODO : check the current geckodriver version geckodriver -V > /dev/null 2>&1 || NOTFOUND=1 set +e if [ -z "$NOTFOUND" ]; then build_msg INSTALL "geckodriver already installed" return fi PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in "linux 32bit" | "linux2 32bit") ARCH="linux32";; "linux 64bit" | "linux2 64bit") ARCH="linux64";; "windows 32 bit") ARCH="win32";; "windows 64 bit") ARCH="win64";; "mac 64bit") ARCH="macos";; esac GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" FILE="$(mktemp)" wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver rm -- "$FILE" chmod 755 -- "${PY_ENV_BIN}/geckodriver" ) dump_return $? } node.env() { local err=0 pushd "${REPO_ROOT}" &> /dev/null # shellcheck disable=SC2230 which npm &> /dev/null || die 1 'node.env - npm is not found!' ( set -e # shellcheck disable=SC2030 PATH="$(npm bin):$PATH" export PATH build_msg INSTALL "npm install $NPM_PACKAGES" # shellcheck disable=SC2086 npm install $NPM_PACKAGES cd "${REPO_ROOT}/searx/static/themes/oscar" build_msg INSTALL "($(pwd)) npm install" npm install build_msg INSTALL "($(pwd)) npm install" cd "${REPO_ROOT}/searx/static/themes/simple" npm install ) err=$? popd &> /dev/null dump_return "$err" } node.clean() { build_msg CLEAN "locally installed npm dependencies" rm -rf \ ./node_modules \ ./package-lock.json \ ./searx/static/themes/oscar/package-lock.json \ ./searx/static/themes/oscar/node_modules \ ./searx/static/themes/simple/package-lock.json \ ./searx/static/themes/simple/node_modules dump_return $? } py.build() { build_msg BUILD "[pylint] python package ${PYDIST}" pyenv.cmd python setup.py \ sdist -d "${PYDIST}" \ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv ( set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name __pycache__ -exec rm -rf {} + ) } pyenv.check() { cat <<EOF import yaml print('import yaml --> OK') EOF } pyenv.install() { if ! pyenv.OK; then py.clean > /dev/null fi if pyenv.install.OK > /dev/null; then return 0 fi ( set -e pyenv build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}" buildenv ) local exit_val=$? if [ ! $exit_val -eq 0 ]; then die 42 "error while pip install (${PY_ENV_BIN})" fi } pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" pyenv.cmd python setup.py develop --uninstall 2>&1 \ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } pypi.upload() { py.clean py.build # https://github.com/pypa/twine pyenv.cmd twine upload "${PYDIST}"/* } pypi.upload.test() { py.clean py.build pyenv.cmd twine upload -r testpypi "${PYDIST}"/* } test.pylint() { # shellcheck disable=SC2086 ( set -e build_msg TEST "[pylint] \$PYLINT_FILES" pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ "${PYLINT_FILES[@]}" build_msg TEST "[pylint] searx/engines" pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \ searx/engines build_msg TEST "[pylint] searx tests" pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \ --ignore=searx/engines \ searx tests ) dump_return $? } test.pep8() { build_msg TEST 'pycodestyle (formerly pep8)' local _exclude="" printf -v _exclude '%s, ' "${PYLINT_FILES[@]}" pyenv.cmd pycodestyle \ --exclude="searx/static, searx/languages.py, $_exclude " \ --max-line-length=120 \ --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \ searx tests dump_return $? } test.unit() { build_msg TEST 'tests/unit' pyenv.cmd python -m nose2 -s tests/unit dump_return $? } test.coverage() { build_msg TEST 'unit test coverage' ( set -e pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit pyenv.cmd coverage report pyenv.cmd coverage html ) dump_return $? } test.robot() { build_msg TEST 'robot' gecko.driver PYTHONPATH=. pyenv.cmd python searx/testing.py robot dump_return $? } test.clean() { build_msg CLEAN "test stuff" rm -rf geckodriver.log .coverage coverage/ dump_return $? } themes.all() { ( set -e node.env themes.oscar themes.simple ) dump_return $? } themes.oscar() { local gruntfile=searx/static/themes/oscar/gruntfile.js build_msg GRUNT "${gruntfile}" PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}" dump_return $? } themes.simple() { local gruntfile=searx/static/themes/simple/gruntfile.js build_msg GRUNT "${gruntfile}" PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}" dump_return $? } PYLINT_FILES=() while IFS= read -r line; do PYLINT_FILES+=("$line") done <<< "$(pylint.FILES)" # shellcheck disable=SC2119 main() { local _type local cmd="$1"; shift if [ "$cmd" == "" ]; then help err_msg "missing command" return 42 fi case "$cmd" in --getenv) var="$1"; echo "${!var}";; --help) help;; --*) help err_msg "unknown option $cmd" return 42 ;; *) _type="$(type -t "$cmd")" if [ "$_type" != 'function' ]; then err_msg "unknown command $1 / use --help" return 42 else "$cmd" "$@" fi ;; esac } main "$@"