utils/searx.sh: add script to install isolated searx service (WIP)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2020-01-16 14:01:38 +01:00
parent 3cf31528f3
commit 89df9d9141
4 changed files with 172 additions and 65 deletions

View file

@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# -*- coding: utf-8; mode: sh -*- # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# shellcheck disable=SC2119 # shellcheck disable=SC2119
# shellcheck source=utils/lib.sh # shellcheck source=utils/lib.sh

View file

@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# -*- coding: utf-8; mode: sh -*- # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# shellcheck disable=SC2059,SC1117,SC2162,SC2004 # shellcheck disable=SC2059,SC1117,SC2162,SC2004
ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
@ -265,9 +265,9 @@ choose_one() {
_t="" _t=""
err_msg "invalid choice" err_msg "invalid choice"
done done
eval "$env_name"='${list[${REPLY}]}'
echo echo
clean_stdin clean_stdin
eval "$env_name"='${list[${REPLY}]}'
} }
install_template() { install_template() {
@ -375,34 +375,32 @@ uWSGI_restart() {
uWSGI_install_app() { uWSGI_install_app() {
# usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ... # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini
local do_eval="" local no_eval=""
local CONF local CONF=""
if [[ "$1" == "--no-eval" ]]; then if [[ "$1" == "--no-eval" ]]; then
no_eval=$1; shift no_eval=$1; shift
fi fi
for CONF in "$@"; do CONF=$1
install_template "$no_eval" "${CONF}" root root 644 # shellcheck disable=SC2086
install_template $no_eval "${CONF}" root root 644
uWSGI_enable_app "$(basename "${CONF}")" uWSGI_enable_app "$(basename "${CONF}")"
info_msg "enabled uWSGI app: $(basename "${CONF}")"
done
uWSGI_restart uWSGI_restart
info_msg "installed uWSGI app: $(basename "${CONF}")"
} }
uWSGI_remove_app() { uWSGI_remove_app() {
# usage: uWSGI_remove_app <path.ini> ... # usage: uWSGI_remove_app <path.ini>
local CONF local CONF=$1
for CONF in "$@"; do
uWSGI_disable_app "$(basename "${CONF}")" uWSGI_disable_app "$(basename "${CONF}")"
uWSGI_restart
rm -f "$CONF" rm -f "$CONF"
info_msg "removed uWSGI app: $(basename "${CONF}")" info_msg "removed uWSGI app: $(basename "${CONF}")"
done
uWSGI_restart
} }
# shellcheck disable=SC2164 # shellcheck disable=SC2164
@ -416,6 +414,7 @@ uWSGI_enable_app() {
return 42 return 42
fi fi
pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null
rm -f "$(basename "${CONF}")"
# shellcheck disable=SC2226 # shellcheck disable=SC2226
ln -s "../apps-available/$(basename "${CONF}")" ln -s "../apps-available/$(basename "${CONF}")"
info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"
@ -454,7 +453,6 @@ pkg_install() {
fi fi
# shellcheck disable=SC2068 # shellcheck disable=SC2068
apt-get install -y $@ apt-get install -y $@
wait_key 30
} }
pkg_remove() { pkg_remove() {
@ -468,7 +466,6 @@ pkg_remove() {
return 42 return 42
fi fi
apt-get purge --autoremove --ignore-missing -y "$@" apt-get purge --autoremove --ignore-missing -y "$@"
wait_key 30
} }
pkg_is_installed() { pkg_is_installed() {
@ -491,7 +488,7 @@ git_clone() {
# git_clone <url> <path> [<branch> [<user>]] # git_clone <url> <path> [<branch> [<user>]]
# #
# First form uses $CACHE/<name> as destination folder, second form clones # First form uses $CACHE/<name> as destination folder, second form clones
# into <path>. If repository is allready cloned, merge from origin and # into <path>. If repository is allready cloned, pull from <branch> and
# update working tree (if needed, the caller has to stash local changes). # update working tree (if needed, the caller has to stash local changes).
# #
# git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin
@ -501,7 +498,8 @@ git_clone() {
local dest="$2" local dest="$2"
local branch="$3" local branch="$3"
local user="$4" local user="$4"
local prefix="" local bash_cmd="bash"
local remote="origin"
if [[ ! "${dest:0:1}" = "/" ]]; then if [[ ! "${dest:0:1}" = "/" ]]; then
dest="$CACHE/$dest" dest="$CACHE/$dest"
@ -509,20 +507,21 @@ git_clone() {
[[ -z $branch ]] && branch=master [[ -z $branch ]] && branch=master
[[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"
[[ -z $user ]] && prefix="sudo -H -u $user" [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i"
if [[ -d "${dest}" ]] ; then if [[ -d "${dest}" ]] ; then
info_msg "already cloned: $dest" info_msg "already cloned: $dest"
pushd "${dest}" > /dev/null tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| "
$prefix git checkout -b "$(basename "$branch")" --track "$branch" cd "${dest}"
$prefix git pull --all git checkout -m -B "$branch" --track "$remote/$branch"
popd > /dev/null git pull --all
EOF
else else
info_msg "clone into: $dest" info_msg "clone into: $dest"
$prefix mkdir -p "$(dirname "$dest")" tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| "
pushd "${dest}" > /dev/null mkdir -p "$(dirname "$dest")"
git clone "$url" "$(basename "$dest")" cd "$(dirname "$dest")"
popd > /dev/null git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"
EOF
fi fi
} }

View file

@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# -*- coding: utf-8; mode: sh -*- # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# shellcheck disable=SC2119 # shellcheck disable=SC2119
# shellcheck source=utils/lib.sh # shellcheck source=utils/lib.sh
@ -15,8 +15,11 @@ SERVICE_USER="${SERVICE_NAME}"
SERVICE_GROUP="${SERVICE_USER}" SERVICE_GROUP="${SERVICE_USER}"
SERVICE_HOME="/home/${SERVICE_USER}" SERVICE_HOME="/home/${SERVICE_USER}"
# shellcheck disable=SC2034
SEARX_URL="127.0.0.1:8888"
SEARX_GIT_URL="https://github.com/asciimoo/searx.git" SEARX_GIT_URL="https://github.com/asciimoo/searx.git"
SEARX_GIT_BRANCH="origin/master" SEARX_GIT_BRANCH="master"
# FIXME: Arch Linux & RHEL should be added # FIXME: Arch Linux & RHEL should be added
@ -25,8 +28,8 @@ libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \
git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \ git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \
libffi-dev libssl-dev" libffi-dev libssl-dev"
SEARX_VENV="${SEARX_HOME}/searx-venv" SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"
SEARX_SRC="${SEARX_HOME}/searx-src" SEARX_SRC="${SERVICE_HOME}/searx-src"
SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"
SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"
SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"
@ -51,25 +54,26 @@ usage(){
usage: usage:
$(basename "$0") shell $(basename "$0") shell
$(basename "$0") install [all|user] $(basename "$0") install [all|user|pyenv|searx-src]
$(basename "$0") update [searx] $(basename "$0") update [searx]
$(basename "$0") remove [all] $(basename "$0") remove [all|user|pyenv|searx-src]
$(basename "$0") activate [service] $(basename "$0") activate [service]
$(basename "$0") deactivate [service] $(basename "$0") deactivate [service]
$(basename "$0") show [service] $(basename "$0") show [service]
shell shell
start interactive shell from user ${SERVICE_USER} start interactive shell from user ${SERVICE_USER}
install / remove all install / remove
complete setup of searx service all: complete (de-) installation of searx service
user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME
searx-src: clone $SEARX_GIT_URL
pyenv: create/remove virtualenv (python) in $SEARX_PYENV
update searx update searx
Update searx installation of user ${SERVICE_USER} Update searx installation of user ${SERVICE_USER}
activate activate
activate and start service daemon (systemd unit) activate and start service daemon (systemd unit)
deactivate service deactivate service
stop and deactivate service daemon (systemd unit) stop and deactivate service daemon (systemd unit)
install user
add service user '$SERVICE_USER' at $SERVICE_HOME
show service show service
show service status and log show service status and log
EOF EOF
@ -102,6 +106,8 @@ main(){
case $2 in case $2 in
all) install_all ;; all) install_all ;;
user) assert_user ;; user) assert_user ;;
pyenv) create_pyenv ;;
searx-src) clone_searx ;;
*) usage "$_usage"; exit 42;; *) usage "$_usage"; exit 42;;
esac ;; esac ;;
update) update)
@ -115,6 +121,8 @@ main(){
case $2 in case $2 in
all) remove_all;; all) remove_all;;
user) remove_user ;; user) remove_user ;;
pyenv) remove_pyenv ;;
searx-src) remove_searx ;;
*) usage "$_usage"; exit 42;; *) usage "$_usage"; exit 42;;
esac ;; esac ;;
activate) activate)
@ -143,7 +151,7 @@ install_all() {
wait_key wait_key
clone_searx clone_searx
wait_key wait_key
create_venv create_pyenv
wait_key wait_key
configure_searx configure_searx
wait_key wait_key
@ -167,7 +175,7 @@ update_searx() {
cd ${SEARX_SRC} cd ${SEARX_SRC}
cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup
git stash push -m "BACKUP -- 'update server' at ($(date))" git stash push -m "BACKUP -- 'update server' at ($(date))"
git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH" git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH"
git pull "$SEARX_GIT_BRANCH" git pull "$SEARX_GIT_BRANCH"
${SEARX_SRC}/manage.sh update_packages ${SEARX_SRC}/manage.sh update_packages
EOF EOF
@ -208,7 +216,10 @@ EOF
remove_all() { remove_all() {
rst_title "De-Install $SERVICE_NAME (service)" rst_title "De-Install $SERVICE_NAME (service)"
remove_service if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then
return
fi
remove_searx_uwsgi
wait_key wait_key
remove_user remove_user
} }
@ -240,6 +251,13 @@ remove_user() {
clone_searx(){ clone_searx(){
rst_title "Clone searx sources" section rst_title "Clone searx sources" section
echo echo
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"
if [[ ! "${SERVICE_HOME}" ]]; then
err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"
return 42
fi
export SERVICE_HOME
git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \
"$SEARX_GIT_BRANCH" "$SERVICE_USER" "$SEARX_GIT_BRANCH" "$SERVICE_USER"
@ -248,27 +266,56 @@ clone_searx(){
cd "${SEARX_SRC}" cd "${SEARX_SRC}"
git config user.email "$ADMIN_EMAIL" git config user.email "$ADMIN_EMAIL"
git config user.name "$ADMIN_NAME" git config user.name "$ADMIN_NAME"
git checkout "$SEARX_GIT_BRANCH" git config --list
EOF EOF
popd > /dev/null popd > /dev/null
} }
create_venv(){ remove_searx() {
rst_title "Create virtualenv (python)" section rst_title "Drop searx sources" section
if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then
rm -rf "$SEARX_SRC"
else
rst_para "Leave searx sources unchanged."
fi
}
rst_para "Create venv in ${SEARX_VENV} and install needed python packages." create_pyenv(){
rst_title "Create virtualenv (python)" section
echo echo
if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then
err_msg "to create pyenv for searx, searx has to be cloned first"
return 42
fi
info_msg "create pyenv in ${SEARX_PYENV}"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
rm -rf "${SEARX_PYENV}"
python3 -m venv "${SEARX_PYENV}"
grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \
|| echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile
EOF
info_msg "inspect python's virtual environment"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
command -v python && python --version
EOF
wait_key
info_msg "install needed python packages"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
rm -rf "${SEARX_VENV}"
python3 -m venv "${SEARX_VENV}"
. ${SEARX_VENV}/bin/activate
${SEARX_SRC}/manage.sh update_packages ${SEARX_SRC}/manage.sh update_packages
EOF EOF
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" }
grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \
|| echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile
EOF
remove_pyenv(){
rst_title "Remove virtualenv (python)" section
if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then
return
fi
info_msg "remove pyenv activation from ~/.profile"
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##
mv ~/.profile.## ~/.profile
EOF
rm -rf "${SEARX_PYENV}"
} }
configure_searx(){ configure_searx(){
@ -283,7 +330,7 @@ EOF
} }
test_local_searx(){ test_local_searx(){
rstHeading "Testing searx instance localy" section rst_title "Testing searx instance localy" section
echo echo
tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
cd ${SEARX_SRC} cd ${SEARX_SRC}
@ -293,7 +340,6 @@ sleep 1
curl --location --verbose --head --insecure http://127.0.0.1:8888/ curl --location --verbose --head --insecure http://127.0.0.1:8888/
sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"
EOF EOF
waitKEY
} }
install_searx_uwsgi() { install_searx_uwsgi() {

View file

@ -0,0 +1,62 @@
[uwsgi]
# uWSGI core
# ----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}
# disable logging for privacy
disable-logging = true
# The right granted on the created socket
chmod-socket = 666
# Plugin to use and interpretor config
single-interpreter = true
# enable master process
master = true
# load apps in each worker instead of the master
lazy-apps = true
# load uWSGI plugins
plugin = python3
# By default the Python plugin does not initialize the GIL. This means your
# app-generated threads will not run. If you need threads, remember to enable
# them with enable-threads. Running uWSGI in multithreading mode (with the
# threads options) will automatically enable threading support. This *strange*
# default behaviour is for performance reasons.
enable-threads = true
# plugin: python
# --------------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
# load a WSGI module
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SERVICE_HOME}
# plugin http
# -----------
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
http = ${SEARX_URL}