searxng/searx/metrics/__init__.py

# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring

import typing
import math
import contextlib
from timeit import default_timer
from operator import itemgetter

from searx.engines import engines
from searx.openmetrics import OpenMetricsFamily
from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
from .error_recorder import count_error, count_exception, errors_per_engines

__all__ = [
    "initialize",
    "get_engines_stats",
    "get_engine_errors",
    "histogram",
    "histogram_observe",
    "histogram_observe_time",
    "counter",
    "counter_inc",
    "counter_add",
    "count_error",
    "count_exception",
]


ENDPOINTS = {'search'}


histogram_storage: typing.Optional[HistogramStorage] = None
counter_storage: typing.Optional[CounterStorage] = None


@contextlib.contextmanager
def histogram_observe_time(*args):
    h = histogram_storage.get(*args)
    before = default_timer()
    yield before
    duration = default_timer() - before
    if h:
        h.observe(duration)
    else:
        raise ValueError("histogram " + repr((*args,)) + " doesn't not exist")


def histogram_observe(duration, *args):
    histogram_storage.get(*args).observe(duration)


def histogram(*args, raise_on_not_found=True):
    h = histogram_storage.get(*args)
    if raise_on_not_found and h is None:
        raise ValueError("histogram " + repr((*args,)) + " doesn't not exist")
    return h


def counter_inc(*args):
    counter_storage.add(1, *args)


def counter_add(value, *args):
    counter_storage.add(value, *args)


def counter(*args):
    return counter_storage.get(*args)


def initialize(engine_names=None, enabled=True):
    """
    Initialize metrics
    """
    global counter_storage, histogram_storage  # pylint: disable=global-statement

    if enabled:
        counter_storage = CounterStorage()
        histogram_storage = HistogramStorage()
    else:
        counter_storage = VoidCounterStorage()
        histogram_storage = HistogramStorage(histogram_class=VoidHistogram)

    # max_timeout = max of all the engine.timeout
    max_timeout = 2
    for engine_name in engine_names or engines:
        if engine_name in engines:
            max_timeout = max(max_timeout, engines[engine_name].timeout)

    # histogram configuration
    histogram_width = 0.1
    histogram_size = int(1.5 * max_timeout / histogram_width)

    # engines
    for engine_name in engine_names or engines:
        # search count
        counter_storage.configure('engine', engine_name, 'search', 'count', 'sent')
        counter_storage.configure('engine', engine_name, 'search', 'count', 'successful')
        # global counter of errors
        counter_storage.configure('engine', engine_name, 'search', 'count', 'error')
        # score of the engine
        counter_storage.configure('engine', engine_name, 'score')
        # result count per requests
        histogram_storage.configure(1, 100, 'engine', engine_name, 'result', 'count')
        # time doing HTTP requests
        histogram_storage.configure(histogram_width, histogram_size, 'engine', engine_name, 'time', 'http')
        # total time
        # .time.request and ...response times may overlap .time.http time.
        histogram_storage.configure(histogram_width, histogram_size, 'engine', engine_name, 'time', 'total')


def get_engine_errors(engline_name_list):
    result = {}
    engine_names = list(errors_per_engines.keys())
    engine_names.sort()
    for engine_name in engine_names:
        if engine_name not in engline_name_list:
            continue

        error_stats = errors_per_engines[engine_name]
        sent_search_count = max(counter('engine', engine_name, 'search', 'count', 'sent'), 1)
        sorted_context_count_list = sorted(error_stats.items(), key=lambda context_count: context_count[1])
        r = []
        for context, count in sorted_context_count_list:
            percentage = round(20 * count / sent_search_count) * 5
            r.append(
                {
                    'filename': context.filename,
                    'function': context.function,
                    'line_no': context.line_no,
                    'code': context.code,
                    'exception_classname': context.exception_classname,
                    'log_message': context.log_message,
                    'log_parameters': context.log_parameters,
                    'secondary': context.secondary,
                    'percentage': percentage,
                }
            )
        result[engine_name] = sorted(r, reverse=True, key=lambda d: d['percentage'])
    return result


def get_reliabilities(engline_name_list, checker_results):
    reliabilities = {}

    engine_errors = get_engine_errors(engline_name_list)

    for engine_name in engline_name_list:
        checker_result = checker_results.get(engine_name, {})
        checker_success = checker_result.get('success', True)
        errors = engine_errors.get(engine_name) or []
        sent_count = counter('engine', engine_name, 'search', 'count', 'sent')

        if sent_count == 0:
            # no request
            reliability = None
        elif checker_success and not errors:
            reliability = 100
        elif 'simple' in checker_result.get('errors', {}):
            # the basic (simple) test doesn't work: the engine is broken according to the checker
            # even if there is no exception
            reliability = 0
        else:
            # pylint: disable=consider-using-generator
            reliability = 100 - sum([error['percentage'] for error in errors if not error.get('secondary')])

        reliabilities[engine_name] = {
            'reliability': reliability,
            'sent_count': sent_count,
            'errors': errors,
            'checker': checker_result.get('errors', {}),
        }
    return reliabilities


def get_engines_stats(engine_name_list):
    assert counter_storage is not None
    assert histogram_storage is not None

    list_time = []
    max_time_total = max_result_count = None

    for engine_name in engine_name_list:

        sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
        if sent_count == 0:
            continue

        result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
        result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
        successful_count = counter('engine', engine_name, 'search', 'count', 'successful')

        time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
        max_time_total = max(time_total or 0, max_time_total or 0)
        max_result_count = max(result_count or 0, max_result_count or 0)

        stats = {
            'name': engine_name,
            'total': None,
            'total_p80': None,
            'total_p95': None,
            'http': None,
            'http_p80': None,
            'http_p95': None,
            'processing': None,
            'processing_p80': None,
            'processing_p95': None,
            'score': 0,
            'score_per_result': 0,
            'result_count': result_count,
        }

        if successful_count and result_count_sum:
            score = counter('engine', engine_name, 'score')

            stats['score'] = score
            stats['score_per_result'] = score / float(result_count_sum)

        time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
        time_http_p80 = time_http_p95 = 0

        if time_http is not None:

            time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
            time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)

            stats['http'] = round(time_http, 1)
            stats['http_p80'] = round(time_http_p80, 1)
            stats['http_p95'] = round(time_http_p95, 1)

        if time_total is not None:

            time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
            time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)

            stats['total'] = round(time_total, 1)
            stats['total_p80'] = round(time_total_p80, 1)
            stats['total_p95'] = round(time_total_p95, 1)

            stats['processing'] = round(time_total - (time_http or 0), 1)
            stats['processing_p80'] = round(time_total_p80 - time_http_p80, 1)
            stats['processing_p95'] = round(time_total_p95 - time_http_p95, 1)

        list_time.append(stats)

    return {
        'time': list_time,
        'max_time': math.ceil(max_time_total or 0),
        'max_result_count': math.ceil(max_result_count or 0),
    }


def openmetrics(engine_stats, engine_reliabilities):
    metrics = [
        OpenMetricsFamily(
            key="searxng_engines_response_time_total_seconds",
            type_hint="gauge",
            help_hint="The average total response time of the engine",
            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
            data=[engine['total'] or 0 for engine in engine_stats['time']],
        ),
        OpenMetricsFamily(
            key="searxng_engines_response_time_processing_seconds",
            type_hint="gauge",
            help_hint="The average processing response time of the engine",
            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
            data=[engine['processing'] or 0 for engine in engine_stats['time']],
        ),
        OpenMetricsFamily(
            key="searxng_engines_response_time_http_seconds",
            type_hint="gauge",
            help_hint="The average HTTP response time of the engine",
            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
            data=[engine['http'] or 0 for engine in engine_stats['time']],
        ),
        OpenMetricsFamily(
            key="searxng_engines_result_count_total",
            type_hint="counter",
            help_hint="The total amount of results returned by the engine",
            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
            data=[engine['result_count'] or 0 for engine in engine_stats['time']],
        ),
        OpenMetricsFamily(
            key="searxng_engines_request_count_total",
            type_hint="counter",
            help_hint="The total amount of user requests made to this engine",
            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
            data=[
                engine_reliabilities.get(engine['name'], {}).get('sent_count', 0) or 0
                for engine in engine_stats['time']
            ],
        ),
        OpenMetricsFamily(
            key="searxng_engines_reliability_total",
            type_hint="counter",
            help_hint="The overall reliability of the engine",
            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
            data=[
                engine_reliabilities.get(engine['name'], {}).get('reliability', 0) or 0
                for engine in engine_stats['time']
            ],
        ),
    ]
    return "".join([str(metric) for metric in metrics])