searxng/searx/engines/gigablast.py
Alexandre Flament 1a6207574e [fix] gigablast engine
fetch extra params after 3000 seconds
2021-12-06 22:55:15 +01:00

134 lines
3.6 KiB
Python

# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""
Gigablast (Web)
"""
# pylint: disable=invalid-name
import re
from time import time
from json import loads
from urllib.parse import urlencode
from searx.network import get
# about
about = {
"website": 'https://www.gigablast.com',
"wikidata_id": 'Q3105449',
"official_api_documentation": 'https://gigablast.com/api.html',
"use_official_api": True,
"require_api_key": False,
"results": 'JSON',
}
# engine dependent config
categories = ['general']
# gigablast's pagination is totally damaged, don't use it
paging = False
safesearch = True
# search-url
base_url = 'https://gigablast.com'
search_path = '/search?'
# ugly hack: gigablast requires a random extra parameter which can be extracted
# from the source code of the gigablast HTTP client
extra_param = ''
# timestamp of the last fetch of extra_param
extra_param_ts = 0
# after how many seconds extra_param expire
extra_param_expiration_delay = 3000
def fetch_extra_param(query_args, headers):
# example:
#
# var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n';
# uxrl=uxrl+'sab=730863287';
#
# extra_param --> "rand=1590740241635&nsab=730863287"
global extra_param, extra_param_ts # pylint: disable=global-statement
extra_param_ts = time()
extra_param_path = search_path + urlencode(query_args)
text = get(base_url + extra_param_path, headers=headers).text
re_var= None
for line in text.splitlines():
if re_var is None and extra_param_path in line:
var = line.split("=")[0].split()[1] # e.g. var --> 'uxrl'
re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)")
extra_param = line.split("'")[1][len(extra_param_path):]
continue
if re_var is not None and re_var.search(line):
extra_param += re_var.search(line).group(1)
break
# do search-request
def request(query, params): # pylint: disable=unused-argument
query_args = dict(
c = 'main'
, q = query
, dr = 1
, showgoodimages = 0
)
if params['language'] and params['language'] != 'all':
query_args['qlangcountry'] = params['language']
query_args['qlang'] = params['language'].split('-')[0]
if params['safesearch'] >= 1:
query_args['ff'] = 1
# see API http://www.gigablast.com/api.html#/search
# Take into account, that the API has some quirks ..
if time() > (extra_param_ts + extra_param_expiration_delay):
fetch_extra_param(query_args, params['headers'])
query_args['format'] = 'json'
params['url'] = base_url + search_path + urlencode(query_args) + extra_param
return params
# get response from search-request
def response(resp):
results = []
response_json = loads(resp.text)
# logger.debug('gigablast returns %s results', len(response_json['results']))
for result in response_json['results']:
# see "Example JSON Output (&format=json)"
# at http://www.gigablast.com/api.html#/search
# sort out meaningless result
title = result.get('title')
if len(title) < 2:
continue
url = result.get('url')
if len(url) < 9:
continue
content = result.get('sum')
if len(content) < 5:
continue
# extend fields
subtitle = result.get('title')
if len(subtitle) > 3 and subtitle != title:
title += " - " + subtitle
results.append(dict(
url = url
, title = title
, content = content
))
return results