searxng/searx/engines/duden.py

80 lines
2.2 KiB
Python
Raw Normal View History

2018-08-18 17:24:02 +00:00
"""
Duden
@website https://www.duden.de
@provide-api no
@using-api no
@results HTML (using search portal)
@stable no (HTML can change)
@parse url, title, content
"""
from lxml import html, etree
import re
from searx.engines.xpath import extract_text
2019-07-25 06:17:45 +00:00
from searx.url_utils import quote, urljoin
2018-08-18 17:24:02 +00:00
from searx import logger
categories = ['general']
paging = True
language_support = False
# search-url
base_url = 'https://www.duden.de/'
2019-07-25 06:17:45 +00:00
search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}'
2018-08-18 17:24:02 +00:00
def request(query, params):
'''pre-request callback
params<dict>:
method : POST/GET
headers : {}
data : {} # if method == POST
url : ''
category: 'search category'
pageno : 1 # number of the requested page
'''
offset = (params['pageno'] - 1)
2019-07-25 06:17:45 +00:00
if offset == 0:
search_url_fmt = base_url + 'suchen/dudenonline/{query}'
params['url'] = search_url_fmt.format(query=quote(query))
else:
params['url'] = search_url.format(offset=offset, query=quote(query))
2018-08-18 17:24:02 +00:00
return params
def response(resp):
'''post-response callback
resp: requests response object
'''
results = []
dom = html.fromstring(resp.text)
try:
number_of_results_string = re.sub('[^0-9]', '', dom.xpath(
'//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]
)
results.append({'number_of_results': int(number_of_results_string)})
except:
logger.debug("Couldn't read number of results.")
pass
2019-07-25 06:17:45 +00:00
for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
2018-08-18 17:24:02 +00:00
try:
2019-07-25 06:17:45 +00:00
url = result.xpath('.//h2/a')[0].get('href')
url = urljoin(base_url, url)
title = result.xpath('string(.//h2/a)').strip()
2018-08-18 17:24:02 +00:00
content = extract_text(result.xpath('.//p'))
# append result
results.append({'url': url,
'title': title,
'content': content})
except:
logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))
continue
return results