utils/searx.sh & filtron.sh: misc changes from first tests (WIP)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-01-29 20:00:50 +01:00
parent 971a8264b2
commit 924bf65517
6 changed files with 286 additions and 102 deletions

1
.gitignore vendored
View file

@ -25,3 +25,4 @@ dist/
local/
gh-pages/
searx.egg-info/
.config

8
utils/dot_config Normal file
View file

@ -0,0 +1,8 @@
# -*- coding: utf-8; mode: sh -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# Set environment used by ./utils scripts like filtron.sh or searx.sh
#
# Public URL of the searx instance
PUBLIC_URL="${PUBLIC_URL:-https://$(uname -n)/searx}"

View file

@ -5,6 +5,7 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
source_dot_config
# ----------------------------------------------------------------------------
# config
@ -30,7 +31,9 @@ GO_ENV="${SERVICE_HOME}/.go_env"
GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
GO_TAR=$(basename "$GO_PKG_URL")
APACHE_SITE="searx.conf"
# Apache Settings
APACHE_FILTRON_SITE="searx.conf"
# shellcheck disable=SC2034
CONFIG_FILES=(
@ -53,22 +56,32 @@ usage:
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") show [service]
$(basename "$0") inspect [service]
$(basename "$0") apache [install|remove]
shell
start interactive shell from user ${SERVICE_USER}
install / remove all
complete setup of filtron service
install / remove
all: complete setup of filtron service
user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME
update filtron
Update filtron installation of user ${SERVICE_USER}
activate
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
install user
add service user '$SERVICE_USER' at $SERVICE_HOME
show service
inspect service
show service status and log
apache
install: apache site with a reverse proxy (ProxyPass)
remove: apache site ${APACHE_FILTRON_SITE}
If needed change the environment variable PUBLIC_URL of your WEB service in the
${DOT_CONFIG#"$REPO_ROOT/"} file:
PUBLIC_URL : ${PUBLIC_URL}
EOF
[ ! -z ${1+x} ] && echo -e "$1"
}
@ -76,6 +89,10 @@ EOF
main() {
rst_title "$SERVICE_NAME" part
required_commands \
dpkg apt-get install git wget curl \
|| exit
local _usage="ERROR: unknown or missing $1 command $2"
case $1 in
@ -86,11 +103,11 @@ main() {
sudo_or_exit
interactive_shell
;;
show)
inspect)
case $2 in
service)
sudo_or_exit
show_service
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
@ -126,6 +143,14 @@ main() {
service) deactivate_service ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;;
esac
}
@ -140,14 +165,29 @@ install_all() {
wait_key
install_service
wait_key
if apache_is_installed; then
install_apache_site
wait_key
echo
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
fi
if ask_yn "Do you want to inspect the installation?" Yn; then
inspect_service
fi
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
rst_para "\
It goes without saying that this script can only be used to remove
installations that were installed with this script."
remove_service
wait_key
remove_user
@ -155,18 +195,6 @@ remove_all() {
wait_key
}
filtron_is_available() {
curl --insecure "http://${FILTRON_LISTEN}" &>/dev/null
}
api_is_available() {
curl --insecure "http://${FILTRON_API}" &>/dev/null
}
target_is_available() {
curl --insecure "http://${FILTRON_TARGET}" &>/dev/null
}
install_service() {
rst_title "Install System-D Unit ${SERVICE_NAME}.service" section
echo
@ -191,7 +219,7 @@ systemctl enable $SERVICE_NAME.service
systemctl restart $SERVICE_NAME.service
EOF
tee_stderr <<EOF | bash 2>&1
systemctl status $SERVICE_NAME.service
systemctl status --no-pager $SERVICE_NAME.service
EOF
}
@ -265,9 +293,8 @@ mkdir -p \$HOME/local
rm -rf \$HOME/local/go
tar -C \$HOME/local -xzf ${CACHE}/${GO_TAR}
EOF
echo
sudo -i -u "$SERVICE_USER" <<EOF | prefix_stdout
! which go >/dev/null && echo "Go Installation not found in PATH!?!"
! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!"
which go >/dev/null && go version && echo "congratulations -- Go installation OK :)"
EOF
}
@ -293,9 +320,20 @@ go get -v -u github.com/asciimoo/filtron
EOF
}
show_service() {
inspect_service() {
rst_title "service status & log"
echo
cat <<EOF
sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
PUBLIC_URL : ${PUBLIC_URL}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_TARGET : ${FILTRON_TARGET}
EOF
apache_is_installed && info_msg "Apache is installed."
@ -314,20 +352,21 @@ show_service() {
else
err_msg "~$SERVICE_USER: filtron app is not installed!"
fi
if api_is_available; then
info_msg "API available at: http://${FILTRON_API}"
else
if ! service_is_available "http://${FILTRON_API}"; then
err_msg "API not available at: http://${FILTRON_API}"
fi
if filtron_is_available; then
info_msg "Filtron listening on: http://${FILTRON_LISTEN}"
else
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"
fi
if target_is_available; then
if ! service_is_available ""http://${FILTRON_TARGET}"" ; then
info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
else
err_msg "Filtron's target is not available at: http://${FILTRON_TARGET}"
fi
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
wait_key
@ -344,10 +383,44 @@ show_service() {
}
install_apache_site() {
rst_title "Install Apache site $APACHE_SITE" section
rst_title "Install Apache site $APACHE_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then
return
fi
a2enmod proxy
a2enmod proxy_http
echo
err_msg "not yet implemented (${APACHE_SITE})"; return 42
# apache_install_site "${APACHE_SITE}"
apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site $APACHE_FILTRON_SITE"
rst_para "\
This removes apache site ${APACHE_FILTRON_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then
return
fi
apache_remove_site "$APACHE_FILTRON_SITE"
}
# ----------------------------------------------------------------------------

View file

@ -1,6 +1,7 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# shellcheck disable=SC2059,SC1117,SC2162,SC2004
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2059,SC1117
ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
ADMIN_NAME="${ADMIN_NAME:-$USER}"
@ -35,6 +36,20 @@ if [[ -z ${DIFF_CMD} ]]; then
fi
fi
DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config}"
source_dot_config() {
if [[ ! -e "$DOT_CONFIG" ]]; then
info_msg "installing $DOT_CONFIG"
cp "$(dirname "${BASH_SOURCE[0]}")/dot_config" "$DOT_CONFIG"
if [[ ! -z ${SUDO_USER} ]]; then
chown "${SUDO_USER}:${SUDO_USER}" "$DOT_CONFIG"
fi
fi
# shellcheck disable=SC1090
source "${REPO_ROOT}/.config"
}
sudo_or_exit() {
# usage: sudo_or_exit
@ -44,6 +59,22 @@ sudo_or_exit() {
fi
}
required_commands() {
# usage: requires_commands [cmd1 ...]
local exit_val=0
while [ ! -z "$1" ]; do
if ! command -v "$1" &>/dev/null; then
err_msg "missing command $1"
exit_val=42
fi
shift
done
return $exit_val
}
rst_title() {
# usage: rst_title <header-text> [part|chapter|section]
@ -81,7 +112,7 @@ info_msg() { echo -e "INFO: $*"; }
clean_stdin() {
if [[ $(uname -s) != 'Darwin' ]]; then
while read -n1 -t 0.1; do : ; done
while read -r -n1 -t 0.1; do : ; done
fi
}
@ -93,7 +124,7 @@ wait_key(){
[[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
[[ ! -z $_t ]] && _t="-t $_t"
# shellcheck disable=SC2086
read -s -n1 $_t -p "** press any [KEY] to continue **"
read -r -s -n1 $_t -p "** press any [KEY] to continue **"
echo
clean_stdin
}
@ -124,7 +155,7 @@ ask_yn() {
clean_stdin
printf "$1 ${choice} "
# shellcheck disable=SC2086
read -n1 $_t
read -r -n1 $_t
if [[ -z $REPLY ]]; then
printf "$default\n"; break
elif [[ $REPLY =~ ^[Yy]$ ]]; then
@ -156,7 +187,7 @@ tee_stderr () {
local _t="0";
if [[ ! -z $1 ]] ; then _t="$1"; fi
(while read line; do
(while read -r line; do
# shellcheck disable=SC2086
sleep $_t
echo -e "$line" >&2
@ -171,6 +202,7 @@ prefix_stdout () {
if [[ ! -z $1 ]] ; then prefix="$1"; fi
# shellcheck disable=SC2162
(while IFS= read line; do
echo -e "${prefix}$line"
done)
@ -214,7 +246,7 @@ cache_download() {
else
wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
fi
if $exit_value; then
if [[ $exit_value = 0 ]]; then
err_msg "failed to download: $1"
fi
fi
@ -238,7 +270,7 @@ choose_one() {
list=("$@")
echo -e "Menu::"
for ((i=1; i<= $(($max -1)); i++)); do
for ((i=1; i<= $((max -1)); i++)); do
if [[ "$i" == "$default" ]]; then
echo -e " $i.) ${list[$i]} [default]"
else
@ -249,15 +281,15 @@ choose_one() {
clean_stdin
printf "$1 [$default] "
if (( 10 > $max )); then
if (( 10 > max )); then
# shellcheck disable=SC2086
read -n1 $_t
read -r -n1 $_t
else
# shellcheck disable=SC2086,SC2229
read $_t
read -r $_t
fi
# selection fits
[[ $REPLY =~ ^-?[0-9]+$ ]] && (( $REPLY > 0 )) && (( $REPLY < $max )) && break
[[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break
# take default
[[ -z $REPLY ]] && REPLY=$default && break
@ -386,6 +418,28 @@ install_template() {
done
}
service_is_available() {
# usage: service_is_available <URL>
local URL="$1"
if [[ -z $URL ]]; then
err_msg "service_is_available: missing arguments"
return 42
fi
http_code=$(curl -H 'Cache-Control: no-cache' \
--silent -o /dev/null --head --write-out '%{http_code}' --insecure \
"${URL}")
exit_val=$?
if [[ $exit_val = 0 ]]; then
info_msg "got $http_code from ${URL}"
fi
return $exit_val
}
# Apache
# ------
@ -430,18 +484,32 @@ apache_install_site() {
root root 644
apache_enable_site "${pos_args[1]}"
apache_reload
info_msg "installed apache site: ${pos_args[1]}"
}
apache_remove_site() {
# usage: apache_remove_site <mysite.conf>
info_msg "remove apache site: $1"
apache_dissable_site "$1"
rm -f "${APACHE_SITES_AVAILABE}/$1"
}
apache_enable_site() {
info_msg "enable apache site $1 .."
# usage: apache_enable_site <mysite.conf>
info_msg "enable apache site: $1"
sudo -H a2ensite -q "$1"
apache_reload
}
apache_dissable_site() {
info_msg "disable apache site $1 .."
# usage: apache_disable_site <mysite.conf>
info_msg "disable apache site: $1"
sudo -H a2dissite -q "$1"
apache_reload
}
@ -456,7 +524,7 @@ uWSGI_restart() {
# usage: uWSGI_restart()
info_msg "restart uWSGI service"
sudo -H systemctl restart uwsgi
systemctl restart uwsgi
}
uWSGI_app_available() {
@ -498,10 +566,10 @@ uWSGI_remove_app() {
# usage: uWSGI_remove_app <myapp.ini>
local CONF="$1"
info_msg "remove uWSGI app: ${CONF}"
uWSGI_disable_app "${CONF}"
uWSGI_restart
rm -f "${uWSGI_SETUP}/apps-available/${CONF}"
info_msg "removed uWSGI app: ${CONF}"
}
uWSGI_app_enabled() {
@ -542,6 +610,9 @@ uWSGI_disable_app() {
return 42
fi
rm -f "${uWSGI_SETUP}/apps-enabled/${CONF}"
# FIXME: restart uwsgi service won't stop wsgi forked processes of user searx.
# I had to kill them manually here ...
pkill -f "${uWSGI_SETUP}/apps-enabled/${CONF}" -9
info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)"
}

View file

@ -1,19 +1,20 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2119
# shellcheck disable=SC2001
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
source_dot_config
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
SEARX_PUBLIC_URL="${SEARX_PUBLIC_URL:-https://$(uname -n)/searx}"
SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "$SEARX_PUBLIC_URL" \
SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \
| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$SEARX_PUBLIC_URL" \
[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/
SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \
| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }"
SERVICE_USER="searx"
@ -42,10 +43,10 @@ SEARX_APT_PACKAGES="\
# Apache Settings
APACHE_APT_PACKAGES="\
apache2 libapache2-mod-uwsgi \
libapache2-mod-uwsgi \
"
SEARX_APACHE_SITE="searx.conf"
APACHE_SEARX_SITE="searx.conf"
# shellcheck disable=SC2034
CONFIG_FILES=(
@ -74,6 +75,7 @@ usage:
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off]
$(basename "$0") apache [install|remove]
shell
start interactive shell from user ${SERVICE_USER}
@ -82,22 +84,24 @@ install / remove
user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME
searx-src: clone $SEARX_GIT_URL
pyenv: create/remove virtualenv (python) in $SEARX_PYENV
apache: install apache site for searx-uwsgi app
update searx
Update searx installation of user ${SERVICE_USER}
activate
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
run some small tests and inspect service's status and log
option
set one of te available options
set one of the available options
apache
install: apache site with the searx uwsgi app
remove: apache site ${APACHE_FILTRON_SITE}
Use environment SEARX_PUBLIC_URL to set public URL of your WEB-Server:
If needed change the environment variable PUBLIC_URL of your WEB service in the
${DOT_CONFIG#"$REPO_ROOT/"} file:
SEARX_PUBLIC_URL : ${SEARX_PUBLIC_URL}
SEARX_URL_PATH : ${SEARX_URL_PATH}
PUBLIC_URL : ${PUBLIC_URL}
SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
EOF
@ -107,6 +111,10 @@ EOF
main() {
rst_title "$SEARX_INSTANCE_NAME" part
required_commands \
dpkg systemctl apt-get install git wget curl \
|| exit
local _usage="ERROR: unknown or missing $1 command $2"
case $1 in
@ -132,7 +140,6 @@ main() {
user) assert_user ;;
pyenv) create_pyenv ;;
searx-src) clone_searx ;;
apache) install_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
@ -154,13 +161,13 @@ main() {
sudo_or_exit
case $2 in
service)
activate_service; uWSGI_restart ;;
activate_service ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) deactivate_service; uWSGI_restart ;;
service) deactivate_service ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
@ -170,6 +177,14 @@ main() {
debug-off) echo; disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;;
esac
}
@ -191,9 +206,7 @@ install_all() {
test_local_searx
wait_key
install_searx_uwsgi
if service_is_available "http://$SEARX_INTERNAL_URL" &>/dev/null; then
info_msg "URL http://$SEARX_INTERNAL_URL is available."
else
if ! service_is_available "http://$SEARX_INTERNAL_URL"; then
err_msg "URL http://$SEARX_INTERNAL_URL not available, check searx & uwsgi setup!"
fi
if ask_yn "Do you want to inspect the installation?" Yn; then
@ -418,12 +431,14 @@ activate_service() {
rst_title "Activate $SEARX_INSTANCE_NAME (service)" section
echo
uWSGI_enable_app "$SEARX_UWSGI_APP"
uWSGI_restart
}
deactivate_service() {
rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section
echo
uWSGI_disable_app "$SEARX_UWSGI_APP"
uWSGI_restart
}
interactive_shell() {
@ -438,12 +453,6 @@ git --no-pager diff
EOF
}
service_is_available() {
curl -H 'Cache-Control: no-cache' -o /dev/null \
--silent --head --write-out '%{http_code}' --insecure \
"${1?missing URL argument}"
}
enable_debug() {
info_msg "try to enable debug mode ..."
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
@ -464,7 +473,16 @@ EOF
inspect_service() {
rst_title "service status & log"
echo
cat <<EOF
sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
PUBLIC_URL : ${PUBLIC_URL}
SEARX_URL_PATH : ${SEARX_URL_PATH}
SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME}
SEARX_INTERNAL_URL : ${SEARX_INTERNAL_URL}
EOF
apache_is_installed && info_msg "Apache is installed."
@ -475,15 +493,15 @@ inspect_service() {
fi
if pyenv_is_available; then
info_msg "${SEARX_PYENV}/bin/activate is available."
info_msg "~$SERVICE_USER: python environment is available."
else
err_msg "${SEARX_PYENV}/bin/activate not available!"
err_msg "~$SERVICE_USER: python environment is not available!"
fi
if clone_is_available; then
info_msg "Searx software is installed."
info_msg "~$SERVICE_USER: Searx software is installed."
else
err_msg "Missing searx software!"
err_msg "~$SERVICE_USER: Missing searx software!"
fi
if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then
@ -495,16 +513,12 @@ inspect_service() {
uWSGI_app_available "$SEARX_UWSGI_APP" \
|| err_msg "uWSGI app $SEARX_UWSGI_APP not available!"
if service_is_available "http://$SEARX_INTERNAL_URL" &>/dev/null; then
info_msg "uWSGI app (service) at http://$SEARX_INTERNAL_URL is available"
else
if ! service_is_available "http://$SEARX_INTERNAL_URL"; then
err_msg "uWSGI app (service) at http://$SEARX_INTERNAL_URL is not available!"
fi
if service_is_available "${SEARX_PUBLIC_URL}" &>/dev/null; then
info_msg "Public service at ${SEARX_PUBLIC_URL} is available"
else
err_msg "Public service at ${SEARX_PUBLIC_URL} is not available!"
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
local _debug_on
@ -530,7 +544,7 @@ inspect_service() {
}
install_apache_site() {
rst_title "Install Apache site $SEARX_APACHE_SITE"
rst_title "Install Apache site $APACHE_SEARX_SITE"
rst_para "\
This installs the searx uwsgi app as apache site. If your server ist public to
@ -547,15 +561,29 @@ excessively bot queries."
a2enmod uwsgi
echo
apache_install_site --variant=uwsgi "${SEARX_APACHE_SITE}"
apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
if service_is_available "${SEARX_PUBLIC_URL}" &>/dev/null; then
info_msg "Public service at ${SEARX_PUBLIC_URL} is available"
else
err_msg "Public service at ${SEARX_PUBLIC_URL} is not available!"
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site ${APACHE_SEARX_SITE}"
rst_para "\
This removes apache site ${APACHE_SEARX_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?"; then
return
fi
apache_remove_site "${APACHE_SEARX_SITE}"
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

View file

@ -0,0 +1,3 @@
# -*- coding: utf-8; mode: apache -*-
ProxyPass "/searx" "http://127.0.0.1:4004/"