Merge pull request #688 from dalf/settings_enable_stats

[enh] settings.yml: implement general.enable_metrics
This commit is contained in:
Markus Heiser 2022-01-05 18:53:49 +00:00 committed by GitHub
commit 0ebad8220f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 35 additions and 11 deletions

View file

@ -81,6 +81,9 @@ Global Settings
``contact_url``:
Contact ``mailto:`` address or WEB form.
``enable_metrics``:
Enabled by default. Record various anonymous metrics availabled at ``/stats``,
``/stats/errors`` and ``/preferences``.
.. _settings global server:

View file

@ -9,7 +9,7 @@ from timeit import default_timer
from operator import itemgetter
from searx.engines import engines
from .models import HistogramStorage, CounterStorage
from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
from .error_recorder import count_error, count_exception, errors_per_engines
__all__ = [
@ -69,14 +69,18 @@ def counter(*args):
return counter_storage.get(*args)
def initialize(engine_names=None):
def initialize(engine_names=None, enabled=True):
"""
Initialize metrics
"""
global counter_storage, histogram_storage # pylint: disable=global-statement
if enabled:
counter_storage = CounterStorage()
histogram_storage = HistogramStorage()
else:
counter_storage = VoidCounterStorage()
histogram_storage = HistogramStorage(histogram_class=VoidHistogram)
# max_timeout = max of all the engine.timeout
max_timeout = 2

View file

@ -9,7 +9,7 @@ from searx.exceptions import (
SearxEngineAPIException,
SearxEngineAccessDeniedException,
)
from searx import searx_parent_dir
from searx import searx_parent_dir, settings
from searx.engines import engines
@ -165,6 +165,8 @@ def get_error_context(framerecords, exception_classname, log_message, log_parame
def count_exception(engine_name: str, exc: Exception, secondary: bool = False) -> None:
if not settings['general']['enable_metrics']:
return
framerecords = inspect.trace()
try:
exception_classname = get_exception_classname(exc)
@ -178,6 +180,8 @@ def count_exception(engine_name: str, exc: Exception, secondary: bool = False) -
def count_error(
engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None, secondary: bool = False
) -> None:
if not settings['general']['enable_metrics']:
return
framerecords = list(reversed(inspect.stack()[1:]))
try:
error_context = get_error_context(framerecords, None, log_message, log_parameters or (), secondary)

View file

@ -102,16 +102,17 @@ class Histogram:
class HistogramStorage:
__slots__ = 'measures'
__slots__ = 'measures', 'histogram_class'
def __init__(self):
def __init__(self, histogram_class=Histogram):
self.clear()
self.histogram_class = histogram_class
def clear(self):
self.measures = {}
def configure(self, width, size, *args):
measure = Histogram(width, size)
measure = self.histogram_class(width, size)
self.measures[args] = measure
return measure
@ -154,3 +155,13 @@ class CounterStorage:
logger.debug("Counters:")
for k in ks:
logger.debug("- %-60s %s", '|'.join(k), self.counters[k])
class VoidHistogram(Histogram):
def observe(self, value):
pass
class VoidCounterStorage(CounterStorage):
def add(self, value, *args):
pass

View file

@ -24,13 +24,13 @@ from searx.search.checker import initialize as initialize_checker
logger = logger.getChild('search')
def initialize(settings_engines=None, enable_checker=False, check_network=False):
def initialize(settings_engines=None, enable_checker=False, check_network=False, enable_metrics=True):
settings_engines = settings_engines or settings['engines']
load_engines(settings_engines)
initialize_network(settings_engines, settings['outgoing'])
if check_network:
check_network_configuration()
initialize_metrics([engine['name'] for engine in settings_engines])
initialize_metrics([engine['name'] for engine in settings_engines], enable_metrics)
initialize_processors(settings_engines)
if enable_checker:
initialize_checker()

View file

@ -2,6 +2,7 @@ general:
debug: false # Debug mode, only for development
instance_name: "SearXNG" # displayed name
contact_url: false # mailto:contact@example.com
enable_metrics: true # record stats
brand:
new_issue_url: https://github.com/searxng/searxng/issues/new

View file

@ -142,6 +142,7 @@ SCHEMA = {
'debug': SettingsValue(bool, False, 'SEARXNG_DEBUG'),
'instance_name': SettingsValue(str, 'SearXNG'),
'contact_url': SettingsValue((None, False, str), None),
'enable_metrics': SettingsValue(bool, True),
},
'brand': {
'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'),

View file

@ -1333,7 +1333,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
# initialize the engines except on the first run of the werkzeug server.
if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
plugin_initialize(app)
search_initialize(enable_checker=True, check_network=True)
search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
def run():