searxng/searx/metrology/error_recorder.py

import typing
import inspect
import logging
from json import JSONDecodeError
from urllib.parse import urlparse
from requests.exceptions import RequestException
from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException,
                              SearxEngineAccessDeniedException)
from searx import logger


logging.basicConfig(level=logging.INFO)

errors_per_engines = {}


class ErrorContext:

    __slots__ = 'filename', 'function', 'line_no', 'code', 'exception_classname', 'log_message', 'log_parameters'

    def __init__(self, filename, function, line_no, code, exception_classname, log_message, log_parameters):
        self.filename = filename
        self.function = function
        self.line_no = line_no
        self.code = code
        self.exception_classname = exception_classname
        self.log_message = log_message
        self.log_parameters = log_parameters

    def __eq__(self, o) -> bool:
        if not isinstance(o, ErrorContext):
            return False
        return self.filename == o.filename and self.function == o.function and self.line_no == o.line_no\
            and self.code == o.code and self.exception_classname == o.exception_classname\
            and self.log_message == o.log_message and self.log_parameters == o.log_parameters

    def __hash__(self):
        return hash((self.filename, self.function, self.line_no, self.code, self.exception_classname, self.log_message,
                     self.log_parameters))

    def __repr__(self):
        return "ErrorContext({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
            format(self.filename, self.line_no, self.code, self.exception_classname, self.log_message,
                   self.log_parameters)


def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
    errors_for_engine = errors_per_engines.setdefault(engine_name, {})
    errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
    logger.debug('%s: %s', engine_name, str(error_context))


def get_trace(traces):
    for trace in reversed(traces):
        split_filename = trace.filename.split('/')
        if '/'.join(split_filename[-3:-1]) == 'searx/engines':
            return trace
        if '/'.join(split_filename[-4:-1]) == 'searx/search/processors':
            return trace
    return traces[-1]


def get_hostname(exc: RequestException) -> typing.Optional[None]:
    url = exc.request.url
    if url is None and exc.response is not None:
        url = exc.response.url
    return urlparse(url).netloc


def get_request_exception_messages(exc: RequestException)\
        -> typing.Tuple[typing.Optional[str], typing.Optional[str], typing.Optional[str]]:
    url = None
    status_code = None
    reason = None
    hostname = None
    if exc.request is not None:
        url = exc.request.url
    if url is None and exc.response is not None:
        url = exc.response.url
    if url is not None:
        hostname = str(urlparse(url).netloc)
    if exc.response is not None:
        status_code = str(exc.response.status_code)
        reason = exc.response.reason
    return (status_code, reason, hostname)


def get_messages(exc, filename) -> typing.Tuple:
    if isinstance(exc, JSONDecodeError):
        return (exc.msg, )
    if isinstance(exc, TypeError):
        return (str(exc), )
    if isinstance(exc, ValueError) and 'lxml' in filename:
        return (str(exc), )
    if isinstance(exc, RequestException):
        return get_request_exception_messages(exc)
    if isinstance(exc, SearxXPathSyntaxException):
        return (exc.xpath_str, exc.message)
    if isinstance(exc, SearxEngineXPathException):
        return (exc.xpath_str, exc.message)
    if isinstance(exc, SearxEngineAPIException):
        return (str(exc.args[0]), )
    if isinstance(exc, SearxEngineAccessDeniedException):
        return (exc.message, )
    return ()


def get_exception_classname(exc: Exception) -> str:
    exc_class = exc.__class__
    exc_name = exc_class.__qualname__
    exc_module = exc_class.__module__
    if exc_module is None or exc_module == str.__class__.__module__:
        return exc_name
    return exc_module + '.' + exc_name


def get_error_context(framerecords, exception_classname, log_message, log_parameters) -> ErrorContext:
    searx_frame = get_trace(framerecords)
    filename = searx_frame.filename
    function = searx_frame.function
    line_no = searx_frame.lineno
    code = searx_frame.code_context[0].strip()
    del framerecords
    return ErrorContext(filename, function, line_no, code, exception_classname, log_message, log_parameters)


def record_exception(engine_name: str, exc: Exception) -> None:
    framerecords = inspect.trace()
    try:
        exception_classname = get_exception_classname(exc)
        log_parameters = get_messages(exc, framerecords[-1][1])
        error_context = get_error_context(framerecords, exception_classname, None, log_parameters)
        add_error_context(engine_name, error_context)
    finally:
        del framerecords


def record_error(engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None) -> None:
    framerecords = list(reversed(inspect.stack()[1:]))
    try:
        error_context = get_error_context(framerecords, None, log_message, log_parameters or ())
        add_error_context(engine_name, error_context)
    finally:
        del framerecords