From bdf392093af92da7d8fc4e32b68ff0716700522d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 8 Jan 2020 18:09:36 +0100 Subject: [PATCH 001/146] build requirements: add a shell script static analysis tool ShellCheck: https://github.com/koalaman/shellcheck Signed-off-by: Markus Heiser --- Makefile | 6 +++-- docs/admin/buildhosts.rst | 50 ++++++++++++++++++++++++++++++++------- docs/dev/makefile.rst | 4 +++- docs/dev/reST.rst | 5 ++-- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index f35b86c41..6adba1037 100644 --- a/Makefile +++ b/Makefile @@ -70,12 +70,14 @@ $(GH_PAGES):: # test # ---- -PHONY += test test.pylint test.pep8 test.unit test.robot +PHONY += test test.sh test.pylint test.pep8 test.unit test.robot # TODO: balance linting with pylint -test: test.pep8 test.unit test.robot +test: test.pep8 test.unit test.sh test.robot - make pylint +test.sh: + test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index 5260da033..c1582cef2 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -35,8 +35,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF creation, its also needed for :ref:`math` when HTML output is build. To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered -as images (``sphinx.ext.imgmath`` extension). If your docs build (``make -docs``) shows warnings like this:: +as images (``sphinx.ext.imgmath`` extension). + +Here is the extract from the :origin:`docs/conf.py` file, setting math renderer +to ``imgmath``: + +.. literalinclude:: ../conf.py + :language: python + :start-after: # sphinx.ext.imgmath setup + :end-before: # sphinx.ext.imgmath setup END + +If your docs build (``make docs``) shows warnings like this:: WARNING: dot(1) not found, for better output quality install \ graphviz from http://www.graphviz.org @@ -47,8 +56,6 @@ docs``) shows warnings like this:: you need to install additional packages on your build host, to get better HTML output. -.. _system requirements: - .. tabs:: .. group-tab:: Ubuntu / debian @@ -94,10 +101,35 @@ For PDF output you also need: texlive-collection-fontsrecommended texlive-collection-latex \ dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts -.. _system requirements END: +.. _sh lint: -.. literalinclude:: ../conf.py - :language: python - :start-after: # sphinx.ext.imgmath setup - :end-before: # sphinx.ext.imgmath setup END +Lint shell scripts +================== +.. _ShellCheck: https://github.com/koalaman/shellcheck + +To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool. + +.. SNIP sh lint requirements + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo apt install shellcheck + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S shellcheck + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo dnf install ShellCheck + +.. SNAP sh lint requirements diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index f5957001c..8e54aef48 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -11,6 +11,8 @@ Makefile Targets Before looking deeper at the targets, first read about :ref:`makefile setup` and :ref:`make pyenv`. + To install system requirements follow :ref:`buildhosts`. + With the aim to simplify development cycles, started with :pull:`1756` a ``Makefile`` based boilerplate was added. If you are not familiar with Makefiles, we recommend to read gnu-make_ introduction. @@ -170,7 +172,7 @@ e.g.: .. code:: sh - $ make test.pep8 test.unit + $ make test.pep8 test.unit test.sh . ./local/py3/bin/activate; ./manage.sh pep8_check [!] Running pep8 check . ./local/py3/bin/activate; ./manage.sh unit_tests diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 4dc1279f0..7c82ebe5a 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -1312,9 +1312,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from .. literalinclude:: ../admin/buildhosts.rst :language: reST - :start-after: .. _system requirements: - :end-before: .. _system requirements END: - + :start-after: .. SNIP sh lint requirements + :end-before: .. SNAP sh lint requirements .. _math: From 28dacee28819ea2d24fd8ff6547fd3a175677084 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 8 Jan 2020 18:13:22 +0100 Subject: [PATCH 002/146] utils: add lib.sh containing common shell script Signed-off-by: Markus Heiser --- .gitignore | 1 + Makefile | 1 + utils/lib.sh | 316 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 318 insertions(+) create mode 100755 utils/lib.sh diff --git a/.gitignore b/.gitignore index 069dfd35b..3c998afae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # to sync with .dockerignore .coverage coverage/ +cache/ .installed.cfg engines.cfg env diff --git a/Makefile b/Makefile index 6adba1037..a5265ff86 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,7 @@ test: test.pep8 test.unit test.sh test.robot - make pylint test.sh: + shellcheck -x utils/lib.sh test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/utils/lib.sh b/utils/lib.sh new file mode 100755 index 000000000..66790bc5f --- /dev/null +++ b/utils/lib.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2059,SC1117,SC2162,SC2004 + +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + while [ -h "${REPO_ROOT}" ] ; do + REPO_ROOT=$(readlink "${REPO_ROOT}") + done + REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P ) +fi + +if [[ -z ${TEMPLATES} ]]; then + TEMPLATES="${REPO_ROOT}/utils/templates" +fi + +if [[ -z "$CACHE" ]]; then + CACHE="${REPO_ROOT}/cache" +fi + +if [[ -z "$SYSTEMD_UNITS" ]]; then + SYSTEMD_UNITS="/lib/systemd/system/" +fi + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +rst_title() { + # usage: rst_title [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${1//?/=}\n$1\n${1//?/=}\n";; + chapter) printf "\n${1}\n${1//?/=}\n";; + section) printf "\n${1}\n${1//?/-}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +err_msg() { echo -e "ERROR: $*" >&2; } +warn_msg() { echo -e "WARN: $*" >&2; } +info_msg() { echo -e "INFO: $*"; } + +wait_key(){ + # usage: waitKEY [] + + local _t=$1 + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + # shellcheck disable=SC2086 + read -n1 $_t -p "** press any [KEY] to continue **" + echo +} + +ask_yn() { + # usage: ask_yn [Ny|Yn] [] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + case "${2}" in + Yn) + local exit_val=${EXIT_YES} + local choice="[YES/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[NO/yes]" + local default="No" + ;; + esac + echo + while true; do + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <>> print("hello") + # hello + + local _t="0"; + if [[ ! -z $1 ]] ; then _t="$1"; fi + + (while read line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: | prefix_stdout [prefix] + + local prefix="-->| " + + if [[ ! -z $1 ]] ; then prefix="$1"; fi + + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download + + local exit_value=0 + + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if $exit_value; then + err_msg "failed to download: $1" + fi + fi +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ ! -z $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "Menu::" + for ((i=1; i<= $(($max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " $i.) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + printf "$1 [$default] " + + if (( 10 > $max )); then + # shellcheck disable=SC2086 + read -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( $REPLY > 0 )) && (( $REPLY < $max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + echo + eval "$env_name"='${list[${REPLY}]}' +} + +install_template() { + + # usage: + # + # install_template [--no-eval] {file} [{owner} [{group} [{chmod}]]] + # + # install_template --no-eval /etc/updatedb.conf root root 644 + + local do_eval=0 + if [[ "$1" == "--no-eval" ]]; then + do_eval=1; shift + fi + local dst="${1}" + local owner=${2-$(id -un)} + local group=${3-$(id -gn)} + local chmod=${4-644} + local _reply="" + + info_msg "install: ${dst}" + + if [[ ! -f "${TEMPLATES}${dst}" ]] ; then + err_msg "${TEMPLATES}${dst} does not exists" + err_msg "... can't install $dst / exit installation with error 42" + wait_key 30 + return 42 + fi + + local template_file="${TEMPLATES}${dst}" + if [[ "$do_eval" == "1" ]]; then + info_msg "BUILD template ${template_file}" + if [[ -f "${TEMPLATES}${dst}" ]] ; then + template_file="${CACHE}${dst}" + mkdir -p "$(dirname "${template_file}")" + # shellcheck disable=SC2086 + eval "echo \"$(cat ${TEMPLATES}${dst})\"" > "${template_file}" + else + err_msg "failed ${template_file}" + return 42 + fi + fi + + if [[ -f "${dst}" ]] ; then + info_msg "file ${dst} allready exists on this host" + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" + ;; + "leave file unchanged") + ;; + "interactiv shell") + echo "// exit with STRG-D" + sudo -H -u "${owner}" -i + ;; + esac + + else + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" + fi + +} + From 4139c63d23a1f4cc427eb428bcff0594c395c1c5 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 8 Jan 2020 19:21:07 +0100 Subject: [PATCH 003/146] utils/filtron.sh: add script to install filtron middleware Signed-off-by: Markus Heiser --- Makefile | 1 + utils/filtron.sh | 214 ++++++++++++++++++ utils/lib.sh | 12 +- utils/templates/etc/filtron/rules.json | 56 +++++ .../lib/systemd/system/filtron.service | 29 +++ 5 files changed, 307 insertions(+), 5 deletions(-) create mode 100755 utils/filtron.sh create mode 100644 utils/templates/etc/filtron/rules.json create mode 100644 utils/templates/lib/systemd/system/filtron.service diff --git a/Makefile b/Makefile index a5265ff86..fec004a5e 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,7 @@ test: test.pep8 test.unit test.sh test.robot test.sh: shellcheck -x utils/lib.sh + shellcheck -x utils/filtron.sh test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 000000000..f310991a0 --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2119 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +FILTRON_ETC="/etc/filtron" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_NAME}" +SERVICE_HOME="/home/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# ---------------------------------------------------------------------------- +usage(){ +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <&1 | prefix_stdout +systemctl enable $SERVICE_NAME.service +systemctl restart $SERVICE_NAME.service +EOF + tee_stderr <&1 | prefix_stdout +systemctl status $SERVICE_NAME.service +EOF + wait_key +} + +deactivate_server () { + rst_title "De-Activate $SERVICE_NAME (service)" section + echo + tee_stderr <&1 | prefix_stdout +systemctl stop $SERVICE_NAME.service +systemctl disable $SERVICE_NAME.service +EOF + wait_key +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 < "$GO_ENV" <> ~/.profile +EOF +} + +remove_user() { + rst_title "Drop $SERVICE_USER HOME" section + if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then + userdel -r -f "$SERVICE_USER" + else + rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." + fi +} + +interactive_shell(){ + echo "// exit with STRG-D" + sudo -H -u ${SERVICE_USER} -i +} + +_service_prefix="$SERVICE_USER@$(hostname) -->| " + +install_go(){ + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${GO_PKG_URL}" "${GO_TAR}" + + tee_stderr 0.1 </dev/null && echo "Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF + wait_key +} + +install_filtron() { + tee_stderr <&1 +EOF + install_template "$FILTRON_ETC/rules.json" root root 644 +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh index 66790bc5f..b5e897549 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -19,7 +19,7 @@ if [[ -z "$CACHE" ]]; then fi if [[ -z "$SYSTEMD_UNITS" ]]; then - SYSTEMD_UNITS="/lib/systemd/system/" + SYSTEMD_UNITS="/lib/systemd/system" fi sudo_or_exit() { @@ -253,9 +253,9 @@ install_template() { # # install_template --no-eval /etc/updatedb.conf root root 644 - local do_eval=0 + local do_eval=1 if [[ "$1" == "--no-eval" ]]; then - do_eval=1; shift + do_eval=0; shift fi local dst="${1}" local owner=${2-$(id -un)} @@ -286,6 +286,8 @@ install_template() { fi fi + mkdir -p "$(dirname "${dst}")" + if [[ -f "${dst}" ]] ; then info_msg "file ${dst} allready exists on this host" choose_one _reply "choose next step with file $dst" \ @@ -296,7 +298,7 @@ install_template() { "replace file") info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ - "${template_file}" "${dst}" + "${template_file}" "${dst}" | prefix_stdout ;; "leave file unchanged") ;; @@ -309,7 +311,7 @@ install_template() { else info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ - "${template_file}" "${dst}" + "${template_file}" "${dst}" | prefix_stdout fi } diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json new file mode 100644 index 000000000..4a232388e --- /dev/null +++ b/utils/templates/etc/filtron/rules.json @@ -0,0 +1,56 @@ +[ + { + "name": "api limit", + "interval": 60, + "limit": 1000, + "filters": ["Path=^/api"], + "aggregations": ["Path"], + "actions": [ + {"name": "block"} + ], + "subrules": [ + { + "name": "drop put", + "interval": 60, + "limit": 100, + "filters": ["Method=PUT"], + "aggregations": ["Header:X-Forwarded-For"], + "actions": [ + {"name": "shell", + "params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}} + ] + } + ] + }, + { + "name": "log'n'block rss", + "interval": 300, + "limit": 2500, + "filters": ["Path=^/$", "GET:format=rss"], + "actions": [ + {"name": "log"}, + {"name": "block"} + ] + }, + { + "name": "log rule", + "filters": ["Path=/"], + "actions": [ {"name": "log"} ], + "subrules": [ + { + "name": "block missing accept-language", + "filters": ["!Header:Accept-Language"], + "actions": [ + {"name": "block"} + ] + }, + { + "name": "block curl", + "filters": ["Header:User-Agent=[Cc]url"], + "actions": [ + {"name": "block"} + ] + } + ] + } +] diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service new file mode 100644 index 000000000..fdb67731a --- /dev/null +++ b/utils/templates/lib/systemd/system/filtron.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES} + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target From 4990b07b4bc42b0caf0d890f8c81c3545bbb807b Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 9 Jan 2020 16:25:05 +0100 Subject: [PATCH 004/146] utils/filtron.sh: various fix from first installation test (WIP) Signed-off-by: Markus Heiser --- utils/filtron.sh | 75 +++++++-- utils/lib.sh | 72 ++++++--- utils/templates/etc/filtron/rules.json | 142 ++++++++++++------ .../lib/systemd/system/filtron.service | 2 +- 4 files changed, 206 insertions(+), 85 deletions(-) diff --git a/utils/filtron.sh b/utils/filtron.sh index f310991a0..593c0fcf7 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -11,6 +11,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" FILTRON_ETC="/etc/filtron" +FILTRON_RULES="$FILTRON_ETC/rules.json" +FILTRON_API="127.0.0.1:4005" +FILTRON_LISTEN="127.0.0.1:4004" +FILTRON_TARGET="127.0.0.1:8888" + SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_NAME}" SERVICE_HOME="/home/${SERVICE_USER}" @@ -23,6 +28,11 @@ GO_ENV="${SERVICE_HOME}/.go_env" GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" GO_TAR=$(basename "$GO_PKG_URL") +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + # ---------------------------------------------------------------------------- usage(){ # ---------------------------------------------------------------------------- @@ -37,10 +47,16 @@ usage: $(basename "$0") remove [all] $(basename "$0") activate [server] $(basename "$0") deactivate [server] + $(basename "$0") show [server] -shell - start interactive shell with user ${SERVICE_USER} -install user - add service user '$SERVICE_USER' at $SERVICE_HOME - +shell + start interactive shell from user ${SERVICE_USER} +show server + show server status and log +install / remove + all - complete setup of filtron server +install user + add service user '$SERVICE_USER' at $SERVICE_HOME EOF [ ! -z ${1+x} ] && echo -e "$1" } @@ -58,6 +74,14 @@ main(){ sudo_or_exit interactive_shell ;; + show) + case $2 in + server) + sudo_or_exit + show_server + ;; + *) usage "$_usage"; exit 42;; + esac ;; install) sudo_or_exit case $2 in @@ -91,21 +115,27 @@ main(){ install_all() { rst_title "Install $SERVICE_NAME (service)" assert_user + wait_key install_go + wait_key install_filtron + wait_key install_server + wait_key } remove_all() { rst_title "De-Install $SERVICE_NAME (service)" remove_server + wait_key remove_user - rm -rf "$FILTRON_ETC" + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout wait_key } install_server() { - rst_title "Install System-D Unit ${SERVICE_NAME}.service ..." section + rst_title "Install System-D Unit ${SERVICE_NAME}.service" section + echo install_template ${SERVICE_SYSTEMD_UNIT} root root 644 wait_key activate_server @@ -116,12 +146,12 @@ remove_server() { return fi deactivate_server - rm "${SERVICE_SYSTEMD_UNIT}" + rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout } - activate_server () { rst_title "Activate $SERVICE_NAME (service)" section + echo tee_stderr <&1 | prefix_stdout systemctl enable $SERVICE_NAME.service systemctl restart $SERVICE_NAME.service @@ -129,7 +159,6 @@ EOF tee_stderr <&1 | prefix_stdout systemctl status $SERVICE_NAME.service EOF - wait_key } deactivate_server () { @@ -139,7 +168,6 @@ deactivate_server () { systemctl stop $SERVICE_NAME.service systemctl disable $SERVICE_NAME.service EOF - wait_key } assert_user() { @@ -168,18 +196,18 @@ EOF remove_user() { rst_title "Drop $SERVICE_USER HOME" section if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then - userdel -r -f "$SERVICE_USER" + userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout else rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." fi } interactive_shell(){ - echo "// exit with STRG-D" + echo "// exit with CTRL-D" sudo -H -u ${SERVICE_USER} -i } -_service_prefix="$SERVICE_USER@$(hostname) -->| " +_service_prefix=" |$SERVICE_USER| " install_go(){ rst_title "Install Go in user's HOME" section @@ -199,14 +227,29 @@ EOF ! which go >/dev/null && echo "Go Installation not found in PATH!?!" which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" EOF - wait_key } install_filtron() { - tee_stderr <&1 + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <&1 | prefix_stdout "$_service_prefix" +go get -v -u github.com/asciimoo/filtron EOF - install_template "$FILTRON_ETC/rules.json" root root 644 + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +show_server () { + rst_title "server status & log" + echo + systemctl status filtron.service + echo + read -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + echo + while true; do + trap break 2 + journalctl -f -u filtron + done + return 0 } # ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh index b5e897549..d06cabf26 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -22,6 +22,13 @@ if [[ -z "$SYSTEMD_UNITS" ]]; then SYSTEMD_UNITS="/lib/systemd/system" fi +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + sudo_or_exit() { # usage: sudo_or_exit @@ -55,10 +62,10 @@ rst_para() { # usage: RST_INDENT=1 rst_para "lorem ipsum ..." local prefix='' if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then - prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" - echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" else - echo -en "\n$*\n" | $FMT + echo -en "\n$*\n" | $FMT fi } @@ -66,15 +73,23 @@ err_msg() { echo -e "ERROR: $*" >&2; } warn_msg() { echo -e "WARN: $*" >&2; } info_msg() { echo -e "INFO: $*"; } +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while $(read -n1 -t 0.1); do : ; done + fi +} + wait_key(){ # usage: waitKEY [] + clean_stdin local _t=$1 [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT [[ ! -z $_t ]] && _t="-t $_t" # shellcheck disable=SC2086 - read -n1 $_t -p "** press any [KEY] to continue **" + read -s -n1 $_t -p "** press any [KEY] to continue **" echo + clean_stdin } ask_yn() { @@ -100,6 +115,7 @@ ask_yn() { esac echo while true; do + clean_stdin printf "$1 ${choice} " # shellcheck disable=SC2086 read -n1 $_t @@ -117,6 +133,7 @@ ask_yn() { _t="" err_msg "invalid choice" done + clean_stdin return $exit_val } @@ -144,7 +161,7 @@ tee_stderr () { prefix_stdout () { # usage: | prefix_stdout [prefix] - local prefix="-->| " + local prefix=" | " if [[ ! -z $1 ]] ; then prefix="$1"; fi @@ -223,6 +240,7 @@ choose_one() { fi done while true; do + clean_stdin printf "$1 [$default] " if (( 10 > $max )); then @@ -242,6 +260,7 @@ choose_one() { err_msg "invalid choice" done echo + clean_stdin eval "$env_name"='${list[${REPLY}]}' } @@ -288,31 +307,48 @@ install_template() { mkdir -p "$(dirname "${dst}")" - if [[ -f "${dst}" ]] ; then - info_msg "file ${dst} allready exists on this host" + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "file ${dst} allready exists on this host" + + while true; do choose_one _reply "choose next step with file $dst" \ "replace file" \ - "leave file unchanged" + "leave file unchanged" \ + "interactiv shell" \ + "diff files" case $_reply in "replace file") info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ "${template_file}" "${dst}" | prefix_stdout + break ;; "leave file unchanged") + break ;; "interactiv shell") - echo "// exit with STRG-D" + echo "// edit ${dst} to your needs" + echo "// exit with CTRL-D" sudo -H -u "${owner}" -i - ;; + $DIFF_CMD "${dst}" "${template_file}" + if ask_yn "did you edit ${template_file} to your needs?"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout esac - - else - info_msg "install: ${template_file}" - sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ - "${template_file}" "${dst}" | prefix_stdout - fi - + done } - diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json index 4a232388e..b54e097a5 100644 --- a/utils/templates/etc/filtron/rules.json +++ b/utils/templates/etc/filtron/rules.json @@ -1,56 +1,98 @@ -[ +[{ + "name":"search request", + "filters":[ + "Param:q", + "Path=^(/|/search)$" + ], + "interval":60, + "limit":15, + "subrules":[ { - "name": "api limit", - "interval": 60, - "limit": 1000, - "filters": ["Path=^/api"], - "aggregations": ["Path"], - "actions": [ - {"name": "block"} - ], - "subrules": [ - { - "name": "drop put", - "interval": 60, - "limit": 100, - "filters": ["Method=PUT"], - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "shell", - "params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}} - ] - } - ] + "name":"roboagent limit", + "interval":60, + "limit":15, + "filters":[ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] }, { - "name": "log'n'block rss", - "interval": 300, - "limit": 2500, - "filters": ["Path=^/$", "GET:format=rss"], - "actions": [ - {"name": "log"}, - {"name": "block"} - ] + "name":"botlimit", + "limit":0, + "stop":true, + "filters":[ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] }, { - "name": "log rule", - "filters": ["Path=/"], - "actions": [ {"name": "log"} ], - "subrules": [ - { - "name": "block missing accept-language", - "filters": ["!Header:Accept-Language"], - "actions": [ - {"name": "block"} - ] - }, - { - "name": "block curl", - "filters": ["Header:User-Agent=[Cc]url"], - "actions": [ - {"name": "block"} - ] - } - ] + "name":"IP limit", + "interval":60, + "limit":15, + "stop":true, + "aggregations":[ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"rss/json limit", + "interval":60, + "limit":15, + "stop":true, + "filters":[ + "Param:format=(csv|json|rss)" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"useragent limit", + "interval":60, + "limit":15, + "aggregations":[ + "Header:User-Agent" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] } -] + ] +}] diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service index fdb67731a..3b0c6edcc 100644 --- a/utils/templates/lib/systemd/system/filtron.service +++ b/utils/templates/lib/systemd/system/filtron.service @@ -10,7 +10,7 @@ Type=simple User=${SERVICE_USER} Group=${SERVICE_GROUP} WorkingDirectory=${SERVICE_HOME} -ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' Restart=always Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} From 876988a34f76988fc7375caace382480f122f940 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 11 Jan 2020 12:49:02 +0100 Subject: [PATCH 005/146] docs: add Sphinx-doc extension sphinxcontrib.programoutput - https://github.com/NextThought/sphinxcontrib-programoutput Signed-off-by: Markus Heiser --- docs/conf.py | 1 + requirements-dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index af255e230..8c5f6b311 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,6 +61,7 @@ extensions = [ "pallets_sphinx_themes", "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja + "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive. 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling. "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e8f617af..fe92bdce5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,3 +14,4 @@ selenium==3.141.0 linuxdoc @ git+http://github.com/return42/linuxdoc.git sphinx-jinja sphinx-tabs +sphinxcontrib-programoutput From 39feb141bc8361915b3d80a976852b38851e0419 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 11 Jan 2020 12:50:40 +0100 Subject: [PATCH 006/146] docs(admin): add description of the utils/filtron.sh script Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 34 +++++++++++++++++++++++++++--- utils/filtron.sh | 48 +++++++++++++++++++++++------------------- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 07dcb9bc5..009ca1c29 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -2,12 +2,40 @@ How to protect an instance ========================== +.. _filtron: https://github.com/asciimoo/filtron + Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. -An application firewall, ``filtron`` solves exactly this problem. Information -on how to install it can be found at the `project page of filtron -`__. +An application firewall, filtron_ solves exactly this problem. Filtron is just +a middleware between your web server (nginx, apache, ...) and searx. + + +filtron & go +============ + +.. _Go: https://golang.org/ +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply +installed by ``go get`` package management (see `filtron README`_). If you use +filtron as middleware, a more isolated setup is recommended. + +#. Create a separated user account (``filtron``). +#. Download and install Go_ binary in users $HOME (``~filtron``). +#. Install filtron with the package management of Go_ (``go get -v -u + github.com/asciimoo/filtron``) +#. Setup a proper rule configuration :origin:`[ref] + ` (``/etc/filtron/rules.json``). +#. Setup a systemd service unit :origin:`[ref] + ` + (``/lib/systemd/system/filtron.service``). + +To simplify such a installation and the maintenance of; use our script +``utils/filtron.sh``: + +.. program-output:: ../utils/filtron.sh --help + :ellipsis: 0,5 Sample configuration of filtron diff --git a/utils/filtron.sh b/utils/filtron.sh index 593c0fcf7..597d0c634 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -45,18 +45,22 @@ usage: $(basename "$0") shell $(basename "$0") install [all|user] $(basename "$0") remove [all] - $(basename "$0") activate [server] - $(basename "$0") deactivate [server] - $(basename "$0") show [server] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") show [service] shell start interactive shell from user ${SERVICE_USER} -show server - show server status and log -install / remove - all - complete setup of filtron server +install / remove all + complete setup of filtron service +activate + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) install user add service user '$SERVICE_USER' at $SERVICE_HOME +show service + show service status and log EOF [ ! -z ${1+x} ] && echo -e "$1" } @@ -68,7 +72,7 @@ main(){ case $1 in --source-only) ;; - -h|--help) usage ;; + -h|--help) usage; exit 0;; shell) sudo_or_exit @@ -76,9 +80,9 @@ main(){ ;; show) case $2 in - server) + service) sudo_or_exit - show_server + show_service ;; *) usage "$_usage"; exit 42;; esac ;; @@ -99,13 +103,13 @@ main(){ activate) sudo_or_exit case $2 in - server) activate_server ;; + service) activate_service ;; *) usage "$_usage"; exit 42;; esac ;; deactivate) sudo_or_exit case $2 in - server) deactivate_server ;; + service) deactivate_service ;; *) usage "$_usage"; exit 42;; esac ;; *) usage "ERROR: unknown or missing command $1"; exit 42;; @@ -120,36 +124,36 @@ install_all() { wait_key install_filtron wait_key - install_server + install_service wait_key } remove_all() { rst_title "De-Install $SERVICE_NAME (service)" - remove_server + remove_service wait_key remove_user rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout wait_key } -install_server() { +install_service() { rst_title "Install System-D Unit ${SERVICE_NAME}.service" section echo install_template ${SERVICE_SYSTEMD_UNIT} root root 644 wait_key - activate_server + activate_service } -remove_server() { +remove_service() { if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then return fi - deactivate_server + deactivate_service rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout } -activate_server () { +activate_service () { rst_title "Activate $SERVICE_NAME (service)" section echo tee_stderr <&1 | prefix_stdout @@ -161,7 +165,7 @@ systemctl status $SERVICE_NAME.service EOF } -deactivate_server () { +deactivate_service () { rst_title "De-Activate $SERVICE_NAME (service)" section echo tee_stderr <&1 | prefix_stdout @@ -238,8 +242,8 @@ EOF install_template --no-eval "$FILTRON_RULES" root root 644 } -show_server () { - rst_title "server status & log" +show_service () { + rst_title "service status & log" echo systemctl status filtron.service echo From 3ed82a97af4f0c65d59bbf748a19ac907db40753 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 12 Jan 2020 15:51:04 +0100 Subject: [PATCH 007/146] utils/filtron.sh: add command 'update filtron' BTW: tidy up some code style and fix some issues reported by the linter. Signed-off-by: Markus Heiser --- utils/filtron.sh | 50 ++++++++++++++++++++++++++++++++++-------------- utils/lib.sh | 2 +- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/utils/filtron.sh b/utils/filtron.sh index 597d0c634..5c8a738b0 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -12,8 +12,12 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" FILTRON_ETC="/etc/filtron" FILTRON_RULES="$FILTRON_ETC/rules.json" + +# shellcheck disable=SC2034 FILTRON_API="127.0.0.1:4005" +# shellcheck disable=SC2034 FILTRON_LISTEN="127.0.0.1:4004" +# shellcheck disable=SC2034 FILTRON_TARGET="127.0.0.1:8888" SERVICE_NAME="filtron" @@ -28,6 +32,7 @@ GO_ENV="${SERVICE_HOME}/.go_env" GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" GO_TAR=$(basename "$GO_PKG_URL") +# shellcheck disable=SC2034 CONFIG_FILES=( "${FILTRON_RULES}" "${SERVICE_SYSTEMD_UNIT}" @@ -44,6 +49,7 @@ usage: $(basename "$0") shell $(basename "$0") install [all|user] + $(basename "$0") update [filtron] $(basename "$0") remove [all] $(basename "$0") activate [service] $(basename "$0") deactivate [service] @@ -51,8 +57,10 @@ usage: shell start interactive shell from user ${SERVICE_USER} -install / remove all +install / remove all complete setup of filtron service +update filtron + Update filtron installation of user ${SERVICE_USER} activate activate and start service daemon (systemd unit) deactivate service @@ -71,33 +79,39 @@ main(){ local _usage="ERROR: unknown or missing $1 command $2" case $1 in - --source-only) ;; + --source-only) ;; -h|--help) usage; exit 0;; - shell) - sudo_or_exit - interactive_shell - ;; + shell) + sudo_or_exit + interactive_shell + ;; show) case $2 in service) - sudo_or_exit - show_service - ;; + sudo_or_exit + show_service + ;; *) usage "$_usage"; exit 42;; esac ;; install) sudo_or_exit case $2 in all) install_all ;; - user) assert_user ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + filtron) update_filtron ;; *) usage "$_usage"; exit 42;; esac ;; remove) sudo_or_exit case $2 in all) remove_all;; - user) remove_user ;; + user) remove_user ;; *) usage "$_usage"; exit 42;; esac ;; activate) @@ -242,16 +256,24 @@ EOF install_template --no-eval "$FILTRON_RULES" root root 644 } +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <&1 | prefix_stdout "$_service_prefix" +go get -v -u github.com/asciimoo/filtron +EOF +} + show_service () { rst_title "service status & log" echo systemctl status filtron.service echo - read -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + read -r -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" echo while true; do - trap break 2 - journalctl -f -u filtron + trap break 2 + journalctl -f -u filtron done return 0 } diff --git a/utils/lib.sh b/utils/lib.sh index d06cabf26..fd6b92129 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -75,7 +75,7 @@ info_msg() { echo -e "INFO: $*"; } clean_stdin() { if [[ $(uname -s) != 'Darwin' ]]; then - while $(read -n1 -t 0.1); do : ; done + while read -n1 -t 0.1; do : ; done fi } From fe64274fb0405c47b3ba9039ab4e5b1440370a4d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 12 Jan 2020 15:53:35 +0100 Subject: [PATCH 008/146] docs: filtron.sh add remark about init systems Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 009ca1c29..e8a2bfb15 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -17,6 +17,13 @@ filtron & go .. _Go: https://golang.org/ .. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +.. sidebar:: init system + + ATM the ``filtron.sh`` supports only systemd init process used by debian, + ubuntu and many other dists. If you have a working init.d file to start/stop + filtron service, please contribute. + Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply installed by ``go get`` package management (see `filtron README`_). If you use filtron as middleware, a more isolated setup is recommended. From b5449ec47cff805a05329a3e5f925cd661457530 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 13 Jan 2020 18:37:05 +0100 Subject: [PATCH 009/146] filtron: log suspiciously frequent queries (WIP) Signed-off-by: Markus Heiser --- utils/templates/etc/filtron/rules.json | 59 +++++++++++++++++--------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json index b54e097a5..634f5f2d6 100644 --- a/utils/templates/etc/filtron/rules.json +++ b/utils/templates/etc/filtron/rules.json @@ -1,42 +1,63 @@ [{ + "name":"suspiciously frequent queries", + "filters":[ + "Param:q", + "Path=^(/|/search)$" + ], + "interval":120, + "limit":9, + "actions":[ + {"name":"log"} + ] + }, + { "name":"search request", "filters":[ "Param:q", "Path=^(/|/search)$" ], - "interval":60, - "limit":15, + "interval":120, + "limit":19, + "actions":[ + { + "name":"block", + "params":{ + "message":"common rate limit exceeded" + } + } + ], "subrules":[ { "name":"roboagent limit", "interval":60, - "limit":15, + "limit":3, "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)" ], "actions":[ - {"name": "log"}, - { + {"name":"log"}, + { "name":"block", "params":{ - "message":"Rate limit exceeded" + "message":"rate limit exceeded" } } ] }, { "name":"botlimit", + "interval":60, "limit":0, "stop":true, "filters":[ "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" ], "actions":[ - {"name": "log"}, + {"name":"log"}, { "name":"block", "params":{ - "message":"Rate limit exceeded" + "message":"rate limit exceeded" } } ] @@ -44,17 +65,17 @@ { "name":"IP limit", "interval":60, - "limit":15, + "limit":13, "stop":true, "aggregations":[ "Header:X-Forwarded-For" ], "actions":[ - {"name": "log"}, + {"name":"log"}, { "name":"block", "params":{ - "message":"Rate limit exceeded" + "message":"rate limit exceeded" } } ] @@ -62,34 +83,34 @@ { "name":"rss/json limit", "interval":60, - "limit":15, + "limit":13, "stop":true, "filters":[ "Param:format=(csv|json|rss)" ], "actions":[ - {"name": "log"}, + {"name":"log"}, { "name":"block", "params":{ - "message":"Rate limit exceeded" + "message":"rate limit exceeded" } } ] - }, + }, { "name":"useragent limit", "interval":60, - "limit":15, + "limit":13, "aggregations":[ "Header:User-Agent" ], "actions":[ - {"name": "log"}, + {"name":"log"}, { "name":"block", "params":{ - "message":"Rate limit exceeded" + "message":"rate limit exceeded" } } ] From 3cf31528f33c2a600cad21952b46a6ebe1e31420 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 14 Jan 2020 19:26:54 +0100 Subject: [PATCH 010/146] utils/searx.sh: add script to install isolated searx service (WIP) WIP: written from scratch / linted but untested Signed-off-by: Markus Heiser --- Makefile | 1 + utils/filtron.sh | 3 +- utils/lib.sh | 186 ++++++++++++++++++++++++- utils/searx.sh | 349 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 532 insertions(+), 7 deletions(-) create mode 100755 utils/searx.sh diff --git a/Makefile b/Makefile index fec004a5e..6577b374e 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ test: test.pep8 test.unit test.sh test.robot test.sh: shellcheck -x utils/lib.sh shellcheck -x utils/filtron.sh + shellcheck -x utils/searx.sh test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/utils/filtron.sh b/utils/filtron.sh index 5c8a738b0..50cce6e65 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -192,7 +192,8 @@ assert_user() { rst_title "user $SERVICE_USER" section echo tee_stderr 1 < $max )); then @@ -333,7 +339,7 @@ install_template() { info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ "${template_file}" "${dst}" | prefix_stdout - break + break ;; "leave file unchanged") break @@ -343,12 +349,180 @@ install_template() { echo "// exit with CTRL-D" sudo -H -u "${owner}" -i $DIFF_CMD "${dst}" "${template_file}" - if ask_yn "did you edit ${template_file} to your needs?"; then - break - fi + if ask_yn "did you edit ${template_file} to your needs?"; then + break + fi ;; "diff files") $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout esac done } + + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" + +uWSGI_restart() { + + # usage: uWSGI_restart() + + info_msg "restart uWSGI service" + sudo -H systemctl restart uwsgi +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ... + + local do_eval="" + local CONF + + if [[ "$1" == "--no-eval" ]]; then + no_eval=$1; shift + fi + + for CONF in "$@"; do + install_template "$no_eval" "${CONF}" root root 644 + uWSGI_enable_app "$(basename "${CONF}")" + info_msg "enabled uWSGI app: $(basename "${CONF}")" + done + uWSGI_restart +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app ... + + local CONF + for CONF in "$@"; do + uWSGI_disable_app "$(basename "${CONF}")" + rm -f "$CONF" + info_msg "removed uWSGI app: $(basename "${CONF}")" + done + uWSGI_restart +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app + + local CONF=$1 + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app missing arguments" + return 42 + fi + pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + # shellcheck disable=SC2226 + ln -s "../apps-available/$(basename "${CONF}")" + info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" + popd >/dev/null +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app + + local CONF=$1 + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app missing arguments" + return 42 + fi + + rm -f "${uWSGI_SETUP}/apps-enabled/$CONF" + info_msg "disabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" +} + +# distro's package manager +# ------------------------ +# +# FIXME: Arch Linux & RHEL should be added +# + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + # shellcheck disable=SC2068 + apt-get install -y $@ + wait_key 30 +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + apt-get purge --autoremove --ignore-missing -y "$@" + wait_key 30 +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + dpkg -l "$1" &> /dev/null + return $? +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone [ []] + # git_clone [ []] + # + # First form uses $CACHE/ as destination folder, second form clones + # into . If repository is allready cloned, merge from origin and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local prefix="" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ -z $user ]] && prefix="sudo -H -u $user" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + pushd "${dest}" > /dev/null + $prefix git checkout -b "$(basename "$branch")" --track "$branch" + $prefix git pull --all + popd > /dev/null + + else + info_msg "clone into: $dest" + $prefix mkdir -p "$(dirname "$dest")" + pushd "${dest}" > /dev/null + git clone "$url" "$(basename "$dest")" + popd > /dev/null + fi +} diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..0d47820b0 --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2119 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_NAME}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +SERVICE_HOME="/home/${SERVICE_USER}" + +SEARX_GIT_URL="https://github.com/asciimoo/searx.git" +SEARX_GIT_BRANCH="origin/master" + +# FIXME: Arch Linux & RHEL should be added + +SEARX_APT_PACKAGES="\ +libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \ + git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \ + libffi-dev libssl-dev" + +SEARX_VENV="${SEARX_HOME}/searx-venv" +SEARX_SRC="${SEARX_HOME}/searx-src" +SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" +SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS}" +) + +# ---------------------------------------------------------------------------- +usage(){ +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat < https://${SEARX_APACHE_DOMAIN}${SEARX_APACHE_URL}" + +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup +git stash push -m "BACKUP -- 'update server' at ($(date))" +git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH" +git pull "$SEARX_GIT_BRANCH" +${SEARX_SRC}/manage.sh update_packages +EOF + configure_searx + + rst_title "${SEARX_SETTINGS}" section + rstBlock 'Diff between new setting file (<) and backup (>):' + echo + diff "$SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup" + + local action + choose_one action "What should happen to the settings file? " \ + "keep new configuration" \ + "revert to the old configuration (backup file)" \ + "start interactiv shell" + case $action in + "keep new configuration") + info_msg "continue using new settings file" + ;; + "revert to the old configuration (backup file)") + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} +EOF + ;; + "start interactiv shell") + interactive_shell + ;; + esac + chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" + + # shellcheck disable=SC2016 + rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):' + echo + git_diff + wait_key + uWSGI_restart +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + remove_service + wait_key + remove_user +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." + fi +} + +# shellcheck disable=SC2164 +clone_searx(){ + rst_title "Clone searx sources" section + echo + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ + "$SEARX_GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git checkout "$SEARX_GIT_BRANCH" +EOF + popd > /dev/null +} + +create_venv(){ + rst_title "Create virtualenv (python)" section + + rst_para "Create venv in ${SEARX_VENV} and install needed python packages." + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_VENV}" +python3 -m venv "${SEARX_VENV}" +. ${SEARX_VENV}/bin/activate +${SEARX_SRC}/manage.sh update_packages +EOF + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile +EOF + +} + +configure_searx(){ + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS" + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS" +EOF +} + +test_local_searx(){ + rstHeading "Testing searx instance localy" section + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +timeout 5 python3 searx/webapp.py & +sleep 1 +curl --location --verbose --head --insecure http://127.0.0.1:8888/ +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF + waitKEY +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service () { + rst_title "Activate $SERVICE_NAME (service)" section + uWSGI_enable_app "$SEARX_UWSGI_APP" +} + +deactivate_service () { + rst_title "De-Activate $SERVICE_NAME (service)" section + uWSGI_disable_app "$SEARX_UWSGI_APP" +} + +interactive_shell(){ + echo "// exit with CTRL-D" + sudo -H -u "${SERVICE_USER}" -i +} + +git_diff(){ + sudo -H -u "${SERVICE_USER}" -i < Date: Thu, 16 Jan 2020 14:01:38 +0100 Subject: [PATCH 011/146] utils/searx.sh: add script to install isolated searx service (WIP) Signed-off-by: Markus Heiser --- utils/filtron.sh | 2 +- utils/lib.sh | 61 +++++----- utils/searx.sh | 112 ++++++++++++------ .../etc/uwsgi/apps-available/searx.ini | 62 ++++++++++ 4 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 utils/templates/etc/uwsgi/apps-available/searx.ini diff --git a/utils/filtron.sh b/utils/filtron.sh index 50cce6e65..bf6016b0f 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh -*- +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # shellcheck disable=SC2119 # shellcheck source=utils/lib.sh diff --git a/utils/lib.sh b/utils/lib.sh index f43a117cb..827862b7c 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh -*- +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # shellcheck disable=SC2059,SC1117,SC2162,SC2004 ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" @@ -265,9 +265,9 @@ choose_one() { _t="" err_msg "invalid choice" done + eval "$env_name"='${list[${REPLY}]}' echo clean_stdin - eval "$env_name"='${list[${REPLY}]}' } install_template() { @@ -375,34 +375,32 @@ uWSGI_restart() { uWSGI_install_app() { - # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ... + # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini - local do_eval="" - local CONF + local no_eval="" + local CONF="" if [[ "$1" == "--no-eval" ]]; then no_eval=$1; shift fi - for CONF in "$@"; do - install_template "$no_eval" "${CONF}" root root 644 - uWSGI_enable_app "$(basename "${CONF}")" - info_msg "enabled uWSGI app: $(basename "${CONF}")" - done + CONF=$1 + # shellcheck disable=SC2086 + install_template $no_eval "${CONF}" root root 644 + uWSGI_enable_app "$(basename "${CONF}")" uWSGI_restart + info_msg "installed uWSGI app: $(basename "${CONF}")" } uWSGI_remove_app() { - # usage: uWSGI_remove_app ... + # usage: uWSGI_remove_app - local CONF - for CONF in "$@"; do - uWSGI_disable_app "$(basename "${CONF}")" - rm -f "$CONF" - info_msg "removed uWSGI app: $(basename "${CONF}")" - done + local CONF=$1 + uWSGI_disable_app "$(basename "${CONF}")" uWSGI_restart + rm -f "$CONF" + info_msg "removed uWSGI app: $(basename "${CONF}")" } # shellcheck disable=SC2164 @@ -416,6 +414,7 @@ uWSGI_enable_app() { return 42 fi pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + rm -f "$(basename "${CONF}")" # shellcheck disable=SC2226 ln -s "../apps-available/$(basename "${CONF}")" info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" @@ -454,7 +453,6 @@ pkg_install() { fi # shellcheck disable=SC2068 apt-get install -y $@ - wait_key 30 } pkg_remove() { @@ -468,7 +466,6 @@ pkg_remove() { return 42 fi apt-get purge --autoremove --ignore-missing -y "$@" - wait_key 30 } pkg_is_installed() { @@ -491,7 +488,7 @@ git_clone() { # git_clone [ []] # # First form uses $CACHE/ as destination folder, second form clones - # into . If repository is allready cloned, merge from origin and + # into . If repository is allready cloned, pull from and # update working tree (if needed, the caller has to stash local changes). # # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin @@ -501,7 +498,8 @@ git_clone() { local dest="$2" local branch="$3" local user="$4" - local prefix="" + local bash_cmd="bash" + local remote="origin" if [[ ! "${dest:0:1}" = "/" ]]; then dest="$CACHE/$dest" @@ -509,20 +507,21 @@ git_clone() { [[ -z $branch ]] && branch=master [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" - [[ -z $user ]] && prefix="sudo -H -u $user" + [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i" if [[ -d "${dest}" ]] ; then info_msg "already cloned: $dest" - pushd "${dest}" > /dev/null - $prefix git checkout -b "$(basename "$branch")" --track "$branch" - $prefix git pull --all - popd > /dev/null - + tee_stderr 0.1 <&1 | prefix_stdout " |$user| " +cd "${dest}" +git checkout -m -B "$branch" --track "$remote/$branch" +git pull --all +EOF else info_msg "clone into: $dest" - $prefix mkdir -p "$(dirname "$dest")" - pushd "${dest}" > /dev/null - git clone "$url" "$(basename "$dest")" - popd > /dev/null + tee_stderr 0.1 <&1 | prefix_stdout " |$user| " +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" +EOF fi } diff --git a/utils/searx.sh b/utils/searx.sh index 0d47820b0..6955f82a2 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# -*- coding: utf-8; mode: sh -*- +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # shellcheck disable=SC2119 # shellcheck source=utils/lib.sh @@ -15,8 +15,11 @@ SERVICE_USER="${SERVICE_NAME}" SERVICE_GROUP="${SERVICE_USER}" SERVICE_HOME="/home/${SERVICE_USER}" +# shellcheck disable=SC2034 +SEARX_URL="127.0.0.1:8888" + SEARX_GIT_URL="https://github.com/asciimoo/searx.git" -SEARX_GIT_BRANCH="origin/master" +SEARX_GIT_BRANCH="master" # FIXME: Arch Linux & RHEL should be added @@ -25,8 +28,8 @@ libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \ git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \ libffi-dev libssl-dev" -SEARX_VENV="${SEARX_HOME}/searx-venv" -SEARX_SRC="${SEARX_HOME}/searx-src" +SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARX_SRC="${SERVICE_HOME}/searx-src" SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" @@ -51,25 +54,26 @@ usage(){ usage: $(basename "$0") shell - $(basename "$0") install [all|user] + $(basename "$0") install [all|user|pyenv|searx-src] $(basename "$0") update [searx] - $(basename "$0") remove [all] + $(basename "$0") remove [all|user|pyenv|searx-src] $(basename "$0") activate [service] $(basename "$0") deactivate [service] $(basename "$0") show [service] shell start interactive shell from user ${SERVICE_USER} -install / remove all - complete setup of searx service +install / remove + all: complete (de-) installation of searx service + user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME + searx-src: clone $SEARX_GIT_URL + pyenv: create/remove virtualenv (python) in $SEARX_PYENV update searx Update searx installation of user ${SERVICE_USER} activate activate and start service daemon (systemd unit) deactivate service stop and deactivate service daemon (systemd unit) -install user - add service user '$SERVICE_USER' at $SERVICE_HOME show service show service status and log EOF @@ -102,6 +106,8 @@ main(){ case $2 in all) install_all ;; user) assert_user ;; + pyenv) create_pyenv ;; + searx-src) clone_searx ;; *) usage "$_usage"; exit 42;; esac ;; update) @@ -115,6 +121,8 @@ main(){ case $2 in all) remove_all;; user) remove_user ;; + pyenv) remove_pyenv ;; + searx-src) remove_searx ;; *) usage "$_usage"; exit 42;; esac ;; activate) @@ -143,7 +151,7 @@ install_all() { wait_key clone_searx wait_key - create_venv + create_pyenv wait_key configure_searx wait_key @@ -167,7 +175,7 @@ update_searx() { cd ${SEARX_SRC} cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup git stash push -m "BACKUP -- 'update server' at ($(date))" -git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH" +git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH" git pull "$SEARX_GIT_BRANCH" ${SEARX_SRC}/manage.sh update_packages EOF @@ -185,16 +193,16 @@ EOF "start interactiv shell" case $action in "keep new configuration") - info_msg "continue using new settings file" - ;; + info_msg "continue using new settings file" + ;; "revert to the old configuration (backup file)") tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} EOF - ;; - "start interactiv shell") - interactive_shell - ;; + ;; + "start interactiv shell") + interactive_shell + ;; esac chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" @@ -208,7 +216,10 @@ EOF remove_all() { rst_title "De-Install $SERVICE_NAME (service)" - remove_service + if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then + return + fi + remove_searx_uwsgi wait_key remove_user } @@ -240,35 +251,71 @@ remove_user() { clone_searx(){ rst_title "Clone searx sources" section echo + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + return 42 + fi + export SERVICE_HOME + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ - "$SEARX_GIT_BRANCH" "$SERVICE_USER" + "$SEARX_GIT_BRANCH" "$SERVICE_USER" pushd "${SEARX_SRC}" > /dev/null tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd "${SEARX_SRC}" git config user.email "$ADMIN_EMAIL" git config user.name "$ADMIN_NAME" -git checkout "$SEARX_GIT_BRANCH" +git config --list EOF popd > /dev/null } -create_venv(){ - rst_title "Create virtualenv (python)" section +remove_searx() { + rst_title "Drop searx sources" section + if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + rm -rf "$SEARX_SRC" + else + rst_para "Leave searx sources unchanged." + fi +} - rst_para "Create venv in ${SEARX_VENV} and install needed python packages." +create_pyenv(){ + rst_title "Create virtualenv (python)" section echo + if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + err_msg "to create pyenv for searx, searx has to be cloned first" + return 42 + fi + info_msg "create pyenv in ${SEARX_PYENV}" + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_PYENV}" +python3 -m venv "${SEARX_PYENV}" +grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" -rm -rf "${SEARX_VENV}" -python3 -m venv "${SEARX_VENV}" -. ${SEARX_VENV}/bin/activate ${SEARX_SRC}/manage.sh update_packages EOF - tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" -grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \ - || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile -EOF +} +remove_pyenv(){ + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARX_PYENV}" } configure_searx(){ @@ -283,7 +330,7 @@ EOF } test_local_searx(){ - rstHeading "Testing searx instance localy" section + rst_title "Testing searx instance localy" section echo tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -293,7 +340,6 @@ sleep 1 curl --location --verbose --head --insecure http://127.0.0.1:8888/ sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" EOF - waitKEY } install_searx_uwsgi() { diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini new file mode 100644 index 000000000..138a57384 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -0,0 +1,62 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3 + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SERVICE_HOME} + + +# plugin http +# ----------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http + +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html +http = ${SEARX_URL} From 9b5a7f7559faf121e44ba3e7260290bd7efe74bf Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 20 Jan 2020 16:55:05 +0100 Subject: [PATCH 012/146] utils/searx.sh: add script to install isolated searx service First version which serves searx over uwsgi at http://127.0.0.1:8888 Signed-off-by: Markus Heiser --- utils/lib.sh | 64 +++++--- utils/searx.sh | 137 +++++++++++++++--- .../etc/uwsgi/apps-available/searx.ini | 6 +- 3 files changed, 160 insertions(+), 47 deletions(-) diff --git a/utils/lib.sh b/utils/lib.sh index 827862b7c..fadbcd648 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -373,66 +373,84 @@ uWSGI_restart() { sudo -H systemctl restart uwsgi } +uWSGI_app_available() { + # usage: uWSGI_app_available + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_app_available: missing arguments" + return 42 + fi + [[ -f "${uWSGI_SETUP}/apps-available/${CONF}" ]] +} + uWSGI_install_app() { - # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini + # usage: uWSGI_install_app [--no-eval] local no_eval="" - local CONF="" + local CONF="$1" if [[ "$1" == "--no-eval" ]]; then no_eval=$1; shift fi - CONF=$1 # shellcheck disable=SC2086 - install_template $no_eval "${CONF}" root root 644 - uWSGI_enable_app "$(basename "${CONF}")" + install_template $no_eval "${uWSGI_SETUP}/apps-available/${CONF}" root root 644 + uWSGI_enable_app "${CONF}" uWSGI_restart - info_msg "installed uWSGI app: $(basename "${CONF}")" + info_msg "installed uWSGI app: ${CONF}" } uWSGI_remove_app() { - # usage: uWSGI_remove_app + # usage: uWSGI_remove_app - local CONF=$1 - uWSGI_disable_app "$(basename "${CONF}")" + local CONF="$1" + uWSGI_disable_app "${CONF}" uWSGI_restart - rm -f "$CONF" - info_msg "removed uWSGI app: $(basename "${CONF}")" + rm -f "${uWSGI_SETUP}/apps-available/${CONF}" + info_msg "removed uWSGI app: ${CONF}" +} + +uWSGI_app_enabled() { + # usage: uWSGI_app_enabled + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_app_enabled: missing arguments" + return 42 + fi + [[ -f "${uWSGI_SETUP}/apps-enabled/${CONF}" ]] } # shellcheck disable=SC2164 uWSGI_enable_app() { - # usage: uWSGI_enable_app + # usage: uWSGI_enable_app - local CONF=$1 + local CONF="$1" if [[ -z $CONF ]]; then - err_msg "uWSGI_enable_app missing arguments" + err_msg "uWSGI_enable_app: missing arguments" return 42 fi pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null - rm -f "$(basename "${CONF}")" + rm -f "$CONF" # shellcheck disable=SC2226 - ln -s "../apps-available/$(basename "${CONF}")" - info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" + ln -s "../apps-available/${CONF}" + info_msg "enabled uWSGI app: ${CONF} (restart uWSGI required)" popd >/dev/null } uWSGI_disable_app() { - # usage: uWSGI_disable_app + # usage: uWSGI_disable_app - local CONF=$1 + local CONF="$1" if [[ -z $CONF ]]; then - err_msg "uWSGI_enable_app missing arguments" + err_msg "uWSGI_enable_app: missing arguments" return 42 fi - - rm -f "${uWSGI_SETUP}/apps-enabled/$CONF" - info_msg "disabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" + rm -f "${uWSGI_SETUP}/apps-enabled/${CONF}" + info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)" } # distro's package manager diff --git a/utils/searx.sh b/utils/searx.sh index 6955f82a2..c9adecc8c 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -32,11 +32,11 @@ SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" SEARX_SRC="${SERVICE_HOME}/searx-src" SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" -SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" +SEARX_UWSGI_APP="searx.ini" # shellcheck disable=SC2034 CONFIG_FILES=( - "${SEARX_UWSGI_APP}" + "${uWSGI_SETUP}/apps-available/${SEARX_UWSGI_APP}" ) # shellcheck disable=SC2034 @@ -45,7 +45,7 @@ CONFIG_BACKUP_ENCRYPTED=( ) # ---------------------------------------------------------------------------- -usage(){ +usage() { # ---------------------------------------------------------------------------- # shellcheck disable=SC1117 @@ -60,6 +60,7 @@ usage: $(basename "$0") activate [service] $(basename "$0") deactivate [service] $(basename "$0") show [service] + $(basename "$0") option [debug-on|debug-off] shell start interactive shell from user ${SERVICE_USER} @@ -75,12 +76,14 @@ activate deactivate service stop and deactivate service daemon (systemd unit) show service - show service status and log + run some small tests and show service's status and log +option + set one of te available options EOF [ ! -z ${1+x} ] && echo -e "$1" } -main(){ +main() { rst_title "$SERVICE_NAME" part local _usage="ERROR: unknown or missing $1 command $2" @@ -128,13 +131,21 @@ main(){ activate) sudo_or_exit case $2 in - service) activate_service ;; + service) + activate_service; uWSGI_restart ;; *) usage "$_usage"; exit 42;; esac ;; deactivate) sudo_or_exit case $2 in - service) deactivate_service ;; + service) deactivate_service; uWSGI_restart ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; *) usage "$_usage"; exit 42;; esac ;; *) usage "ERROR: unknown or missing command $1"; exit 42;; @@ -158,6 +169,11 @@ install_all() { test_local_searx wait_key install_searx_uwsgi + if service_is_available; then + info_msg "URL http://$SEARX_URL is available." + else + err_msg "URL http://$SEARX_URL not available, check searx & uwsgi setup!" + fi wait_key # ToDo ... @@ -224,6 +240,10 @@ remove_all() { remove_user } +user_is_available() { + sudo -i -u "$SERVICE_USER" echo \$HOME &>/dev/null +} + assert_user() { rst_title "user $SERVICE_USER" section echo @@ -247,8 +267,12 @@ remove_user() { fi } +clone_is_available() { + [[ -f "$SEARX_SETTINGS" ]] +} + # shellcheck disable=SC2164 -clone_searx(){ +clone_searx() { rst_title "Clone searx sources" section echo SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" @@ -280,7 +304,11 @@ remove_searx() { fi } -create_pyenv(){ +pyenv_is_available() { + [[ -f "${SEARX_PYENV}/bin/activate" ]] +} + +create_pyenv() { rst_title "Create virtualenv (python)" section echo if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then @@ -305,7 +333,7 @@ ${SEARX_SRC}/manage.sh update_packages EOF } -remove_pyenv(){ +remove_pyenv() { rst_title "Remove virtualenv (python)" section if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then return @@ -318,7 +346,7 @@ EOF rm -rf "${SEARX_PYENV}" } -configure_searx(){ +configure_searx() { rst_title "Configure searx" section rst_para "Setup searx config located at $SEARX_SETTINGS" echo @@ -329,7 +357,7 @@ sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS" EOF } -test_local_searx(){ +test_local_searx() { rst_title "Testing searx instance localy" section echo tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" @@ -337,7 +365,7 @@ cd ${SEARX_SRC} sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" timeout 5 python3 searx/webapp.py & sleep 1 -curl --location --verbose --head --insecure http://127.0.0.1:8888/ +curl --location --verbose --head --insecure $SEARX_URL sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" EOF } @@ -354,39 +382,106 @@ remove_searx_uwsgi() { uWSGI_remove_app "$SEARX_UWSGI_APP" } -activate_service () { +activate_service() { rst_title "Activate $SERVICE_NAME (service)" section + echo uWSGI_enable_app "$SEARX_UWSGI_APP" } -deactivate_service () { +deactivate_service() { rst_title "De-Activate $SERVICE_NAME (service)" section + echo uWSGI_disable_app "$SEARX_UWSGI_APP" } -interactive_shell(){ +interactive_shell() { echo "// exit with CTRL-D" sudo -H -u "${SERVICE_USER}" -i } -git_diff(){ +git_diff() { sudo -H -u "${SERVICE_USER}" -i </dev/null +} + +enable_debug() { + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +EOF + uWSGI_restart +} + +disable_debug() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF + uWSGI_restart +} + + +show_service() { rst_title "service status & log" echo - systemctl status uwsgi.service + + if user_is_available; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if service_is_available; then + info_msg "URL http://$SEARX_URL is available." + else + err_msg "URL http://$SEARX_URL not available!" + fi + if pyenv_is_available; then + info_msg "${SEARX_PYENV}/bin/activate is available." + else + err_msg "${SEARX_PYENV}/bin/activate not available!" + fi + if clone_is_available; then + info_msg "Searx software is installed." + else + err_msg "Missing searx software!" + fi + + uWSGI_app_available "$SEARX_UWSGI_APP" \ + || err_msg "uWSGI app $SEARX_UWSGI_APP not available!" + + if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then + info_msg "uWSGI app $SEARX_UWSGI_APP is enabled." + else + err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!" + fi + + local _debug_on + if ask_yn "Enable searx debug mode?"; then + enable_debug + _debug_on=1 + fi + wait_key echo - read -r -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + systemctl status uwsgi.service + read -r -s -n1 -t 2 -p "// use CTRL-C to stop monitoring the log" echo while true; do trap break 2 - journalctl -f -u uwsgi.service + #journalctl -f -u uwsgi.service + tail -f /var/log/uwsgi/app/searx.log done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi return 0 } diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini index 138a57384..d3893b3ad 100644 --- a/utils/templates/etc/uwsgi/apps-available/searx.ini +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -10,7 +10,7 @@ uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} # chdir to specified directory before apps loading -chdir = ${SEARX_SRC} +chdir = ${SEARX_SRC}/searx # disable logging for privacy disable-logging = true @@ -28,7 +28,7 @@ master = true lazy-apps = true # load uWSGI plugins -plugin = python3 +plugin = python3,http # By default the Python plugin does not initialize the GIL. This means your # app-generated threads will not run. If you need threads, remember to enable @@ -50,7 +50,7 @@ module = searx.webapp virtualenv = ${SEARX_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SERVICE_HOME} +pythonpath = ${SEARX_SRC} # plugin http From af2f58fc5847756d20741bb4c782f07943b0af60 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 20 Jan 2020 19:08:56 +0100 Subject: [PATCH 013/146] utils/filtron.sh: add script to install filtron middleware (apache) Preparation for the installation of an apache site. Signed-off-by: Markus Heiser --- utils/filtron.sh | 98 +++++++++++++++++++++++++++++++++++++++++------- utils/lib.sh | 53 ++++++++++++++++++++++++++ utils/searx.sh | 22 ++++++++--- 3 files changed, 154 insertions(+), 19 deletions(-) diff --git a/utils/filtron.sh b/utils/filtron.sh index bf6016b0f..bb6597fa5 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -13,11 +13,8 @@ FILTRON_ETC="/etc/filtron" FILTRON_RULES="$FILTRON_ETC/rules.json" -# shellcheck disable=SC2034 FILTRON_API="127.0.0.1:4005" -# shellcheck disable=SC2034 FILTRON_LISTEN="127.0.0.1:4004" -# shellcheck disable=SC2034 FILTRON_TARGET="127.0.0.1:8888" SERVICE_NAME="filtron" @@ -32,6 +29,8 @@ GO_ENV="${SERVICE_HOME}/.go_env" GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" GO_TAR=$(basename "$GO_PKG_URL") +APACHE_SITE="searx.conf" + # shellcheck disable=SC2034 CONFIG_FILES=( "${FILTRON_RULES}" @@ -39,7 +38,7 @@ CONFIG_FILES=( ) # ---------------------------------------------------------------------------- -usage(){ +usage() { # ---------------------------------------------------------------------------- # shellcheck disable=SC1117 @@ -73,7 +72,7 @@ EOF [ ! -z ${1+x} ] && echo -e "$1" } -main(){ +main() { rst_title "$SERVICE_NAME" part local _usage="ERROR: unknown or missing $1 command $2" @@ -140,6 +139,10 @@ install_all() { wait_key install_service wait_key + if apache_is_installed; then + install_apache_site + wait_key + fi } remove_all() { @@ -151,10 +154,22 @@ remove_all() { wait_key } +filtron_is_available() { + curl --insecure "http://${FILTRON_LISTEN}" &>/dev/null +} + +api_is_available() { + curl --insecure "http://${FILTRON_API}" &>/dev/null +} + +target_is_available() { + curl --insecure "http://${FILTRON_TARGET}" &>/dev/null +} + install_service() { rst_title "Install System-D Unit ${SERVICE_NAME}.service" section echo - install_template ${SERVICE_SYSTEMD_UNIT} root root 644 + install_template "${SERVICE_SYSTEMD_UNIT}" root root 644 wait_key activate_service } @@ -167,19 +182,19 @@ remove_service() { rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout } -activate_service () { +activate_service() { rst_title "Activate $SERVICE_NAME (service)" section echo - tee_stderr <&1 | prefix_stdout + tee_stderr <&1 systemctl enable $SERVICE_NAME.service systemctl restart $SERVICE_NAME.service EOF - tee_stderr <&1 | prefix_stdout + tee_stderr <&1 systemctl status $SERVICE_NAME.service EOF } -deactivate_service () { +deactivate_service() { rst_title "De-Activate $SERVICE_NAME (service)" section echo tee_stderr <&1 | prefix_stdout @@ -188,6 +203,10 @@ systemctl disable $SERVICE_NAME.service EOF } +user_is_available() { + sudo -i -u "$SERVICE_USER" echo \$HOME &>/dev/null +} + assert_user() { rst_title "user $SERVICE_USER" section echo @@ -228,7 +247,11 @@ interactive_shell(){ _service_prefix=" |$SERVICE_USER| " -install_go(){ +go_is_available() { + sudo -i -u "$SERVICE_USER" which go &>/dev/null +} + +install_go() { rst_title "Install Go in user's HOME" section rst_para "download and install go binary .." @@ -248,6 +271,10 @@ which go >/dev/null && go version && echo "congratulations -- Go installation O EOF } +filtron_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] +} + install_filtron() { rst_title "Install filtron in user's ~/go-apps" section echo @@ -265,12 +292,48 @@ go get -v -u github.com/asciimoo/filtron EOF } -show_service () { +show_service() { rst_title "service status & log" echo - systemctl status filtron.service + + apache_is_installed && info_msg "Apache is installed." + + if user_is_available; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if filtron_is_installed; then + info_msg "~$SERVICE_USER: filtron app is installed" + else + err_msg "~$SERVICE_USER: filtron app is not installed!" + fi + if api_is_available; then + info_msg "API available at: http://${FILTRON_API}" + else + err_msg "API not available at: http://${FILTRON_API}" + fi + if filtron_is_available; then + info_msg "Filtron listening on: http://${FILTRON_LISTEN}" + else + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + if target_is_available; then + info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" + else + err_msg "Filtron's target is not available at: http://${FILTRON_TARGET}" + fi + + wait_key echo - read -r -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + systemctl --no-pager -l status filtron.service + echo + read -r -s -n1 -t 2 -p "// use CTRL-C to stop monitoring the log" echo while true; do trap break 2 @@ -279,6 +342,13 @@ show_service () { return 0 } +install_apache_site() { + rst_title "Install Apache site $APACHE_SITE" section + echo + err_msg "not yet implemented (${APACHE_SITE})"; return 42 + # apache_install_site "${APACHE_SITE}" +} + # ---------------------------------------------------------------------------- main "$@" # ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh index fadbcd648..40ecbb92e 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -359,6 +359,59 @@ install_template() { done } +# Apache +# ------ + +# FIXME: Arch Linux & RHEL should be added + +if [[ -z "${APACHE_SITES_AVAILABE}" ]]; then + APACHE_SITES_AVAILABE="/etc/apache2/sites-available" +fi + +apache_is_installed() { + (command -v apachectl \ + && command -v a2ensite \ + && command -v a2dissite ) &>/dev/null +} + +apache_reload() { + + info_msg "reload apache .." + echo + sudo -H apachectl configtest + sudo -H service apache2 force-reload +} + +apache_install_site() { + + # usage: apache_install_site [--no-eval] + + local no_eval="" + local CONF="$1" + + if [[ "$1" == "--no-eval" ]]; then + no_eval=$1; shift + fi + + # shellcheck disable=SC2086 + install_template $no_eval "${APACHE_SITES_AVAILABE}/${CONF}" root root 644 + + apache_enable_site "${CONF}" + apache_reload + info_msg "installed apache site: ${CONF}" +} + +apache_enable_site() { + info_msg "enable apache site $1 .." + sudo -H a2ensite -q "$1" + apache_reload +} + +apache_dissable_site() { + info_msg "disable apache site $1 .." + sudo -H a2dissite -q "$1" + apache_reload +} # uWSGI # ----- diff --git a/utils/searx.sh b/utils/searx.sh index c9adecc8c..a1b1140ed 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -15,7 +15,6 @@ SERVICE_USER="${SERVICE_NAME}" SERVICE_GROUP="${SERVICE_USER}" SERVICE_HOME="/home/${SERVICE_USER}" -# shellcheck disable=SC2034 SEARX_URL="127.0.0.1:8888" SEARX_GIT_URL="https://github.com/asciimoo/searx.git" @@ -34,6 +33,8 @@ SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" SEARX_UWSGI_APP="searx.ini" +APACHE_SITE="searx.conf" + # shellcheck disable=SC2034 CONFIG_FILES=( "${uWSGI_SETUP}/apps-available/${SEARX_UWSGI_APP}" @@ -175,9 +176,12 @@ install_all() { err_msg "URL http://$SEARX_URL not available, check searx & uwsgi setup!" fi wait_key + if apache_is_installed; then + install_apache_site + wait_key + fi # ToDo ... - # install_apache_site # test_public_searx # info_msg "searX --> https://${SEARX_APACHE_DOMAIN}${SEARX_APACHE_URL}" @@ -428,11 +432,12 @@ EOF uWSGI_restart } - show_service() { rst_title "service status & log" echo + apache_is_installed && info_msg "Apache is installed." + if user_is_available; then info_msg "service account $SERVICE_USER available." else @@ -468,9 +473,9 @@ show_service() { enable_debug _debug_on=1 fi - wait_key echo - systemctl status uwsgi.service + systemctl --no-pager -l status uwsgi.service + echo read -r -s -n1 -t 2 -p "// use CTRL-C to stop monitoring the log" echo while true; do @@ -485,6 +490,13 @@ show_service() { return 0 } +install_apache_site() { + rst_title "Install Apache site $APACHE_SITE" section + echo + err_msg "not yet implemented (${APACHE_SITE})"; return 42 + # apache_install_site "${APACHE_SITE}" +} + # ---------------------------------------------------------------------------- main "$@" # ---------------------------------------------------------------------------- From d171fcd56ea0444598c6ae6d6d089dd2488bd64d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 21 Jan 2020 18:38:57 +0100 Subject: [PATCH 014/146] utils/searx.sh: add apache site searx.conf:uwsgi Signed-off-by: Markus Heiser --- utils/lib.sh | 125 ++++++++++++------ utils/searx.sh | 42 +++--- .../apache2/sites-available/searx.conf:uwsgi | 25 ++++ 3 files changed, 132 insertions(+), 60 deletions(-) create mode 100644 utils/templates/etc/apache2/sites-available/searx.conf:uwsgi diff --git a/utils/lib.sh b/utils/lib.sh index 40ecbb92e..b89d4d2f9 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -274,41 +274,65 @@ install_template() { # usage: # - # install_template [--no-eval] {file} [{owner} [{group} [{chmod}]]] + # install_template [--no-eval] [--variant=] \ + # {file} [{owner} [{group} [{chmod}]]] # - # install_template --no-eval /etc/updatedb.conf root root 644 + # E.g. the origin of variant 'raw' of /etc/updatedb.conf is:: + # + # ${TEMPLATES}/etc/updatedb.conf:raw + # + # To install variant 'raw' of /etc/updatedb.conf without evaluated + # replacements you can use:: + # + # install_template --variant=raw --no-eval \ + # /etc/updatedb.conf root root 644 - local do_eval=1 - if [[ "$1" == "--no-eval" ]]; then - do_eval=0; shift - fi - local dst="${1}" - local owner=${2-$(id -un)} - local group=${3-$(id -gn)} - local chmod=${4-644} local _reply="" + local do_eval=1 + local variant="" + local pos_args=("$0") - info_msg "install: ${dst}" + for i in "$@"; do + case $i in + --no-eval) do_eval=0; shift ;; + --variant=*) variant=":${i#*=}"; shift ;; + *) pos_args+=("$i") ;; + esac + done - if [[ ! -f "${TEMPLATES}${dst}" ]] ; then - err_msg "${TEMPLATES}${dst} does not exists" - err_msg "... can't install $dst / exit installation with error 42" + local dst="${pos_args[1]}" + local template_origin="${TEMPLATES}${dst}${variant}" + local template_file="${TEMPLATES}${dst}" + + local owner="${pos_args[2]-$(id -un)}" + local group="${pos_args[3]-$(id -gn)}" + local chmod="${pos_args[4]-644}" + + info_msg "install (eval=$do_eval): ${dst}" + [[ ! -z $variant ]] && info_msg "variant: ${variant}" + + if [[ ! -f "${template_origin}" ]] ; then + err_msg "${template_origin} does not exists" + err_msg "... can't install $dst" wait_key 30 return 42 fi - local template_file="${TEMPLATES}${dst}" if [[ "$do_eval" == "1" ]]; then + template_file="${CACHE}${dst}${variant}" info_msg "BUILD template ${template_file}" - if [[ -f "${TEMPLATES}${dst}" ]] ; then - template_file="${CACHE}${dst}" - mkdir -p "$(dirname "${template_file}")" - # shellcheck disable=SC2086 - eval "echo \"$(cat ${TEMPLATES}${dst})\"" > "${template_file}" + if [[ ! -z ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")" else - err_msg "failed ${template_file}" - return 42 + mkdir -p "$(dirname "${template_file}")" fi + # shellcheck disable=SC2086 + eval "echo \"$(cat ${template_origin})\"" > "${template_file}" + if [[ ! -z ${SUDO_USER} ]]; then + chown "${SUDO_USER}:${SUDO_USER}" "${template_file}" + fi + else + template_file=$template_origin fi mkdir -p "$(dirname "${dst}")" @@ -325,7 +349,7 @@ install_template() { return 0 fi - info_msg "file ${dst} allready exists on this host" + info_msg "diffrent file ${dst} allready exists on this host" while true; do choose_one _reply "choose next step with file $dst" \ @@ -349,7 +373,10 @@ install_template() { echo "// exit with CTRL-D" sudo -H -u "${owner}" -i $DIFF_CMD "${dst}" "${template_file}" - if ask_yn "did you edit ${template_file} to your needs?"; then + echo + echo "did you edit file ..." + printf " ${template_file}" + if ask_yn "... to your needs?"; then break fi ;; @@ -384,21 +411,27 @@ apache_reload() { apache_install_site() { - # usage: apache_install_site [--no-eval] + # usage: apache_install_site [