Merge pull request #13 from pw3t/master

Filecrop & Yacy engine
This commit is contained in:
Adam Tauber 2014-01-04 14:55:22 -08:00
commit 27cae54197
3 changed files with 114 additions and 0 deletions

3
.gitignore vendored
View file

@ -1,2 +1,5 @@
env
engines.cfg
*.pyc
*/*.pyc

73
searx/engines/filecrop.py Normal file
View file

@ -0,0 +1,73 @@
from json import loads
from urllib import urlencode
from searx.utils import html_to_text
from HTMLParser import HTMLParser
url = 'http://www.filecrop.com/'
search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1'
class FilecropResultParser(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self.__start_processing = False
self.results = []
self.result = {}
self.tr_counter = 0
self.data_counter = 0
def handle_starttag(self, tag, attrs):
if tag == 'tr':
if ('bgcolor', '#edeff5') in attrs or ('bgcolor', '#ffffff') in attrs:
self.__start_processing = True
if not self.__start_processing:
return
if tag == 'label':
self.result['title'] = [attr[1] for attr in attrs if attr[0] == 'title'][0]
elif tag == 'a' and ('rel', 'nofollow') in attrs and ('class', 'sourcelink') in attrs:
if 'content' in self.result:
self.result['content'] += [attr[1] for attr in attrs if attr[0] == 'title'][0]
else:
self.result['content'] = [attr[1] for attr in attrs if attr[0] == 'title'][0]
self.result['content'] += ' '
elif tag == 'a':
self.result['url'] = url + [attr[1] for attr in attrs if attr[0] == 'href'][0]
def handle_endtag(self, tag):
if self.__start_processing is False:
return
if tag == 'tr':
self.tr_counter += 1
if self.tr_counter == 2:
self.__start_processing = False
self.tr_counter = 0
self.data_counter = 0
self.results.append(self.result)
self.result = {}
def handle_data(self, data):
if not self.__start_processing:
return
if 'content' in self.result:
self.result['content'] += data + ' '
else:
self.result['content'] = data + ' '
self.data_counter += 1
def request(query, params):
params['url'] = search_url.format(query=urlencode({'w' :query}))
return params
def response(resp):
parser = FilecropResultParser()
parser.feed(resp.text)
return parser.results

38
searx/engines/yacy.py Normal file
View file

@ -0,0 +1,38 @@
from json import loads
from urllib import urlencode, quote
url = 'http://localhost:8090'
search_url = '/yacysearch.json?{query}&maximumRecords=10'
def request(query, params):
params['url'] = url + search_url.format(query=urlencode({'query':query}))
return params
def response(resp):
raw_search_results = loads(resp.text)
if not len(raw_search_results):
return []
search_results = raw_search_results.get('channels', {})[0].get('items', [])
results = []
for result in search_results:
tmp_result = {}
tmp_result['title'] = result['title']
tmp_result['url'] = result['link']
tmp_result['content'] = ''
if len(result['description']):
tmp_result['content'] += result['description'] +"<br/>"
if len(result['pubDate']):
tmp_result['content'] += result['pubDate'] + "<br/>"
if result['size'] != '-1':
tmp_result['content'] += result['sizename']
results.append(tmp_result)
return results