From 9bf594cbcfbe59736f093f3d40bda5e302d96304 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 7 Dec 2020 10:31:11 +0100 Subject: [PATCH] [mod] duden engine * add params['soft_max_redirects'] = 1 (when there is spelling suggestion) * avoid try..except * use eval_xpath_* functions --- searx/engines/duden.py | 43 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 1484a21e5..1475fb846 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -8,11 +8,10 @@ @parse url, title, content """ -from lxml import html, etree import re from urllib.parse import quote, urljoin -from searx.utils import extract_text, eval_xpath -from searx import logger +from lxml import html +from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex categories = ['general'] paging = True @@ -40,6 +39,9 @@ def request(query, params): params['url'] = search_url_fmt.format(query=quote(query)) else: params['url'] = search_url.format(offset=offset, query=quote(query)) + # after the last page of results, spelling corrections are returned after a HTTP redirect + # whatever the page number is + params['soft_max_redirects'] = 1 return params @@ -51,28 +53,21 @@ def response(resp): dom = html.fromstring(resp.text) - try: - number_of_results_string =\ - re.sub('[^0-9]', '', - eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]) - + number_of_results_element =\ + eval_xpath_getindex(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()', + 0, default=None) + if number_of_results_element is not None: + number_of_results_string = re.sub('[^0-9]', '', number_of_results_element) results.append({'number_of_results': int(number_of_results_string)}) - except: - logger.debug("Couldn't read number of results.") - - for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'): - try: - url = eval_xpath(result, './/h2/a')[0].get('href') - url = urljoin(base_url, url) - title = eval_xpath(result, 'string(.//h2/a)').strip() - content = extract_text(eval_xpath(result, './/p')) - # append result - results.append({'url': url, - 'title': title, - 'content': content}) - except: - logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True)) - continue + for result in eval_xpath_list(dom, '//section[not(contains(@class, "essay"))]'): + url = eval_xpath_getindex(result, './/h2/a', 0).get('href') + url = urljoin(base_url, url) + title = eval_xpath(result, 'string(.//h2/a)').strip() + content = extract_text(eval_xpath(result, './/p')) + # append result + results.append({'url': url, + 'title': title, + 'content': content}) return results