searxng/searx/engines/faroo.py

97 lines
2.5 KiB
Python
Raw Normal View History

"""
Faroo (Web, News)
@website http://www.faroo.com
@provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
@using-api no
@results JSON
@stable yes
@parse url, title, content, publishedDate, img_src
"""
2014-10-17 10:34:51 +00:00
from json import loads
import datetime
from searx.utils import searx_useragent
2016-11-30 17:43:03 +00:00
from searx.url_utils import urlencode
2014-10-17 10:34:51 +00:00
# engine dependent config
categories = ['general', 'news']
paging = True
language_support = True
number_of_results = 10
# search-url
url = 'http://www.faroo.com/'
search_url = url + 'instant.json?{query}'\
2017-08-30 17:48:57 +00:00
'&start={offset}'\
'&length={number_of_results}'\
'&l={language}'\
'&src={categorie}'\
'&i=false'\
'&c=false'
2014-10-17 10:34:51 +00:00
search_category = {'general': 'web',
2014-12-16 16:26:16 +00:00
'news': 'news'}
2014-10-17 10:34:51 +00:00
2014-10-17 10:34:51 +00:00
# do search-request
def request(query, params):
2015-02-07 14:35:12 +00:00
offset = (params['pageno'] - 1) * number_of_results + 1
2014-10-17 10:34:51 +00:00
categorie = search_category.get(params['category'], 'web')
if params['language'] == 'all':
language = 'en'
else:
language = params['language'].split('-')[0]
2014-10-17 10:34:51 +00:00
2015-02-07 14:35:12 +00:00
# if language is not supported, put it in english
2014-10-17 10:34:51 +00:00
if language != 'en' and\
language != 'de' and\
language != 'zh':
2015-02-07 14:35:12 +00:00
language = 'en'
2014-10-17 10:34:51 +00:00
params['url'] = search_url.format(offset=offset,
number_of_results=number_of_results,
query=urlencode({'q': query}),
language=language,
categorie=categorie)
2014-10-17 10:34:51 +00:00
params['headers']['Referer'] = url
2014-10-17 10:34:51 +00:00
return params
# get response from search-request
def response(resp):
# HTTP-Code 429: rate limit exceeded
if resp.status_code == 429:
raise Exception("rate limit has been exceeded!")
results = []
search_res = loads(resp.text)
# return empty array if there are no results
if not search_res.get('results', {}):
return []
# parse results
for result in search_res['results']:
publishedDate = None
2017-08-30 17:48:57 +00:00
result_json = {'url': result['url'], 'title': result['title'],
'content': result['kwic']}
2014-10-17 10:54:22 +00:00
if result['news']:
result_json['publishedDate'] = \
2017-08-30 17:48:57 +00:00
datetime.datetime.fromtimestamp(result['date'] / 1000.0)
2014-10-17 10:34:51 +00:00
2014-10-17 10:54:22 +00:00
# append image result if image url is set
if result['iurl']:
2017-08-30 17:48:57 +00:00
result_json['template'] = 'videos.html'
result_json['thumbnail'] = result['iurl']
results.append(result_json)
2014-10-17 10:34:51 +00:00
# return results
return results