mirror of
https://github.com/searxng/searxng.git
synced 2024-11-15 14:41:57 +00:00
Merge pull request #85 from searxng/fix-metrics-offline-engine
Fix metrics offline engine
This commit is contained in:
Markus Heiser
GitHub
commit
2398e9a1fe
8 changed files with 86 additions and 61 deletions
|
|
@ -156,57 +156,76 @@ def get_reliabilities(engline_name_list, checker_results):
|
|||
return reliabilities
|
||||
|
||||
|
||||
def round_or_none(number, digits):
|
||||
return round(number, digits) if number else number
|
||||
|
||||
|
||||
def get_engines_stats(engine_name_list):
|
||||
assert counter_storage is not None
|
||||
assert histogram_storage is not None
|
||||
|
||||
list_time = []
|
||||
max_time_total = max_result_count = None
|
||||
|
||||
max_time_total = max_result_count = None # noqa
|
||||
for engine_name in engine_name_list:
|
||||
|
||||
sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
|
||||
if sent_count == 0:
|
||||
continue
|
||||
|
||||
result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
|
||||
result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
|
||||
successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
|
||||
|
||||
time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
|
||||
time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
|
||||
time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
|
||||
time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
|
||||
time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
|
||||
time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
|
||||
|
||||
result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
|
||||
result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
|
||||
if successful_count and result_count_sum:
|
||||
score = counter('engine', engine_name, 'score') # noqa
|
||||
score_per_result = score / float(result_count_sum)
|
||||
else:
|
||||
score = score_per_result = 0.0
|
||||
|
||||
max_time_total = max(time_total or 0, max_time_total or 0)
|
||||
max_result_count = max(result_count or 0, max_result_count or 0)
|
||||
|
||||
list_time.append({
|
||||
stats = {
|
||||
'name': engine_name,
|
||||
'total': round_or_none(time_total, 1),
|
||||
'total_p80': round_or_none(time_total_p80, 1),
|
||||
'total_p95': round_or_none(time_total_p95, 1),
|
||||
'http': round_or_none(time_http, 1),
|
||||
'http_p80': round_or_none(time_http_p80, 1),
|
||||
'http_p95': round_or_none(time_http_p95, 1),
|
||||
'processing': round(time_total - time_http, 1) if time_total else None,
|
||||
'processing_p80': round(time_total_p80 - time_http_p80, 1) if time_total else None,
|
||||
'processing_p95': round(time_total_p95 - time_http_p95, 1) if time_total else None,
|
||||
'score': score,
|
||||
'score_per_result': score_per_result,
|
||||
'total': None,
|
||||
'total_p80': None,
|
||||
'total_p95': None,
|
||||
'http': None,
|
||||
'http_p80': None,
|
||||
'http_p95': None,
|
||||
'processing': None,
|
||||
'processing_p80': None,
|
||||
'processing_p95': None,
|
||||
'score': 0,
|
||||
'score_per_result': 0,
|
||||
'result_count': result_count,
|
||||
})
|
||||
}
|
||||
|
||||
if successful_count and result_count_sum:
|
||||
score = counter('engine', engine_name, 'score')
|
||||
|
||||
stats['score'] = score
|
||||
stats['score_per_result'] = score / float(result_count_sum)
|
||||
|
||||
time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
|
||||
time_http_p80 = time_http_p95 = 0
|
||||
|
||||
if time_http is not None:
|
||||
|
||||
time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
|
||||
time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
|
||||
|
||||
stats['http'] = round(time_http, 1)
|
||||
stats['http_p80'] = round(time_http_p80, 1)
|
||||
stats['http_p95'] = round(time_http_p95, 1)
|
||||
|
||||
if time_total is not None:
|
||||
|
||||
time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
|
||||
time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
|
||||
|
||||
stats['total'] = round(time_total, 1)
|
||||
stats['total_p80'] = round(time_total_p80, 1)
|
||||
stats['total_p95'] = round(time_total_p95, 1)
|
||||
|
||||
stats['processing'] = round(time_total - (time_http or 0), 1)
|
||||
stats['processing_p80'] = round(time_total_p80 - time_http_p80, 1)
|
||||
stats['processing_p95'] = round(time_total_p95 - time_http_p95, 1)
|
||||
|
||||
list_time.append(stats)
|
||||
|
||||
return {
|
||||
'time': list_time,
|
||||
'max_time': math.ceil(max_time_total or 0),
|
||||
|
|
|
|||
|
|
@ -44,7 +44,8 @@ def reset_time_for_thread():
|
|||
|
||||
|
||||
def get_time_for_thread():
|
||||
return THREADLOCAL.total_time
|
||||
"""returns thread's total time or None"""
|
||||
return THREADLOCAL.__dict__.get('total_time')
|
||||
|
||||
|
||||
def set_timeout_for_thread(timeout, start_time=None):
|
||||
|
|
@ -57,10 +58,11 @@ def set_context_network_name(network_name):
|
|||
|
||||
|
||||
def get_context_network():
|
||||
try:
|
||||
return THREADLOCAL.network
|
||||
except AttributeError:
|
||||
return get_network()
|
||||
"""If set return thread's network.
|
||||
|
||||
If unset, return value from :py:obj:`get_network`.
|
||||
"""
|
||||
return THREADLOCAL.__dict__.get('network') or get_network()
|
||||
|
||||
|
||||
def request(method, url, **kwargs):
|
||||
|
|
|
|||
|
|
@ -371,11 +371,12 @@ class ResultContainer:
|
|||
self.unresponsive_engines.add((engine_name, error_type, error_message, suspended))
|
||||
|
||||
def add_timing(self, engine_name, engine_time, page_load_time):
|
||||
self.timings.append({
|
||||
timing = {
|
||||
'engine': engines[engine_name].shortcut,
|
||||
'total': engine_time,
|
||||
'load': page_load_time
|
||||
})
|
||||
'load': page_load_time,
|
||||
}
|
||||
self.timings.append(timing)
|
||||
|
||||
def get_timings(self):
|
||||
return self.timings
|
||||
|
|
|
|||
|
|
@ -25,11 +25,13 @@
|
|||
<td class="{{ label }}" style="padding: 2px">{{- "" -}}
|
||||
{%- if stats[engine_name].time != None -%}
|
||||
<span class="stacked-bar-chart-value">{{- stats[engine_name].time -}}</span>{{- "" -}}
|
||||
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">{{- "" -}}
|
||||
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">
|
||||
{%- if max_rate95 is not none and max_rate95 > 0 -%}
|
||||
<span style="width: calc(max(2px, 100%*{{ (stats[engine_name].time / max_rate95)|round(3) }}))" class="stacked-bar-chart-median"></span>{{- "" -}}
|
||||
<span style="width: calc(100%*{{ ((stats[engine_name].rate80 - stats[engine_name].time) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate80"></span>{{- "" -}}
|
||||
<span style="width: calc(100%*{{ ((stats[engine_name].rate95 - stats[engine_name].rate80) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate95"></span>{{- "" -}}
|
||||
<span class="stacked-bar-chart-rate100"></span>{{- "" -}}
|
||||
<span class="stacked-bar-chart-rate100"></span>
|
||||
{%- endif -%}
|
||||
</span>{{- "" -}}
|
||||
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_graph">{{- "" -}}
|
||||
<p>{{ _('Median') }}: {{ stats[engine_name].time }}</p>{{- "" -}}
|
||||
|
|
|
|||
|
|
@ -52,11 +52,11 @@
|
|||
{%- endif -%}
|
||||
</td>
|
||||
<td class="response-time">
|
||||
{%- if engine_stat.total -%}
|
||||
{%- if engine_stat.total is not none -%}
|
||||
<div class="bar-chart-value">{{- engine_stat.total | round(1) -}}</div>{{- "" -}}
|
||||
<div class="bar-chart-graph" aria-labelledby="{{engine_stat.name}}_time" aria-hidden="true">{{- "" -}}
|
||||
<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{{- "" -}}
|
||||
<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{{- "" -}}
|
||||
{% if engine_stat.http is not none and engine_stats.max_time %}<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{%- endif -%}
|
||||
{% if engine_stat.processing is not none and engine_stats.max_time %}<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{%- endif -%}
|
||||
</div>
|
||||
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_stat.name}}_time">{{- "" -}}
|
||||
<table class="table table-striped">
|
||||
|
|
@ -69,19 +69,19 @@
|
|||
<tr>
|
||||
<th scope="col">{{ _('Median') }}</th>
|
||||
<td>{{ engine_stat.total }}</td>
|
||||
<td>{{ engine_stat.http }}</td>
|
||||
<td>{{ engine_stat.http or '' }}</td>
|
||||
<td>{{ engine_stat.processing }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th scope="col">{{ _('P80') }}</th>
|
||||
<td>{{ engine_stat.total_p80 }}</td>
|
||||
<td>{{ engine_stat.http_p80 }}</td>
|
||||
<td>{{ engine_stat.http_p80 or '' }}</td>
|
||||
<td>{{ engine_stat.processing_p80 }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th scope="col">{{ _('P95') }}</th>
|
||||
<td>{{ engine_stat.total_p95 }}</td>
|
||||
<td>{{ engine_stat.http_p95 }}</td>
|
||||
<td>{{ engine_stat.http_p95 or '' }}</td>
|
||||
<td>{{ engine_stat.processing_p95 }}</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -39,11 +39,13 @@
|
|||
<td class="{{ label }}" style="padding: 2px; width: 13rem;">{{- "" -}}
|
||||
{%- if stats[engine_name].time != None -%}
|
||||
<span class="stacked-bar-chart-value">{{- stats[engine_name].time -}}</span>{{- "" -}}
|
||||
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">{{- "" -}}
|
||||
<span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">
|
||||
{%- if max_rate95 is not none and max_rate95 > 0 -%}
|
||||
<span style="width: calc(max(2px, 100%*{{ (stats[engine_name].time / max_rate95)|round(3) }}))" class="stacked-bar-chart-median"></span>{{- "" -}}
|
||||
<span style="width: calc(100%*{{ ((stats[engine_name].rate80 - stats[engine_name].time) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate80"></span>{{- "" -}}
|
||||
<span style="width: calc(100%*{{ ((stats[engine_name].rate95 - stats[engine_name].rate80) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate95"></span>{{- "" -}}
|
||||
<span class="stacked-bar-chart-rate100"></span>{{- "" -}}
|
||||
<span class="stacked-bar-chart-rate100"></span>
|
||||
{%- endif -%}
|
||||
</span>{{- "" -}}
|
||||
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_graph">{{- "" -}}
|
||||
<p>{{ _('Median') }}: {{ stats[engine_name].time }}</p>{{- "" -}}
|
||||
|
|
|
|||
|
|
@ -52,12 +52,11 @@
|
|||
{%- endif -%}
|
||||
</td>
|
||||
<td class="response-time">
|
||||
{%- if engine_stat.total -%}
|
||||
|
||||
{%- if engine_stat.total is not none -%}
|
||||
<div class="bar-chart-value">{{- engine_stat.total | round(1) -}}</div>{{- "" -}}
|
||||
<div class="bar-chart-graph" aria-labelledby="{{engine_stat.name}}_time" aria-hidden="true">
|
||||
<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{{- "" -}}
|
||||
<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{{- "" -}}
|
||||
{% if engine_stat.http is not none and engine_stats.max_time %}<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{%- endif -%}
|
||||
{% if engine_stat.processing is not none and engine_stats.max_time %}<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{%- endif -%}
|
||||
</div>
|
||||
<div class="engine-tooltip" role="tooltip" id="{{engine_stat.name}}_time">{{- "" -}}
|
||||
<table>
|
||||
|
|
@ -70,19 +69,19 @@
|
|||
<tr>
|
||||
<th scope="col">{{ _('Median') }}</th>
|
||||
<td>{{ engine_stat.total }}</td>
|
||||
<td>{{ engine_stat.http }}</td>
|
||||
<td>{{ engine_stat.http or ''}}</td>
|
||||
<td>{{ engine_stat.processing }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th scope="col">{{ _('P80') }}</th>
|
||||
<td>{{ engine_stat.total_p80 }}</td>
|
||||
<td>{{ engine_stat.http_p80 }}</td>
|
||||
<td>{{ engine_stat.http_p80 or '' }}</td>
|
||||
<td>{{ engine_stat.processing_p80 }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th scope="col">{{ _('P95') }}</th>
|
||||
<td>{{ engine_stat.total_p95 }}</td>
|
||||
<td>{{ engine_stat.http_p95 }}</td>
|
||||
<td>{{ engine_stat.http_p95 or '' }}</td>
|
||||
<td>{{ engine_stat.processing_p95 }}</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -568,7 +568,7 @@ def post_request(response):
|
|||
timings_total = ['total_' + str(i) + '_' + v['engine'] +
|
||||
';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)]
|
||||
timings_load = ['load_' + str(i) + '_' + v['engine'] +
|
||||
';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)]
|
||||
';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings) if v.get('load')]
|
||||
timings_all = timings_all + timings_total + timings_load
|
||||
response.headers.add('Server-Timing', ', '.join(timings_all))
|
||||
return response
|
||||
|
|
@ -923,9 +923,9 @@ def preferences():
|
|||
result_count = int(result_count_sum / float(successful_count)) if successful_count else 0
|
||||
|
||||
stats[e.name] = {
|
||||
'time': median if median else None,
|
||||
'rate80': rate80 if rate80 else None,
|
||||
'rate95': rate95 if rate95 else None,
|
||||
'time': median,
|
||||
'rate80': rate80,
|
||||
'rate95': rate95,
|
||||
'warn_timeout': e.timeout > settings['outgoing']['request_timeout'],
|
||||
'supports_selected_language': _is_selected_language_supported(e, request.preferences),
|
||||
'result_count': result_count,
|
||||
|
|
|
|||
Loading…
Reference in a new issue