From 2cd980134b9905d527a8eff861f66c925a470276 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Wed, 3 Sep 2014 13:47:12 +0200 Subject: [PATCH] fix yacy-engine and add comments * add language support * add image-support * add basic-implementation for video, music and file handling * refactore engine * add comments --- searx/engines/yacy.py | 85 ++++++++++++++++++++++++++++++++++--------- searx/settings.yml | 7 ++++ 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py index efdf846ac..2345b24f3 100644 --- a/searx/engines/yacy.py +++ b/searx/engines/yacy.py @@ -1,40 +1,89 @@ +## Yacy (Web, Images, Videos, Music, Files) +# +# @website http://yacy.net +# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse (general) url, title, content, publishedDate +# @parse (images) url, title, img_src +# +# @todo parse video, audio and file results + from json import loads from urllib import urlencode +from dateutil import parser -url = 'http://localhost:8090' -search_url = '/yacysearch.json?{query}&maximumRecords=10' +# engine dependent config +categories = ['general', 'images'] #TODO , 'music', 'videos', 'files' +paging = True +language_support = True +number_of_results = 5 + +# search-url +base_url = 'http://localhost:8090' +search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limit}&contentdom={search_type}&resource=global' + +# yacy specific type-definitions +search_types = {'general': 'text', + 'images': 'image', + 'files': 'app', + 'music': 'audio', + 'videos': 'video'} +# do search-request def request(query, params): - params['url'] = url + search_url.format(query=urlencode({'query': query})) + offset = (params['pageno'] - 1) * number_of_results + search_type = search_types.get(params['category'], '0') + + params['url'] = base_url + search_url.format(query=urlencode({'query': query}), + offset=offset, + limit=number_of_results, + search_type=search_type) + + # add language tag if specified + if params['language'] != 'all': + params['url'] += '&lr=lang_' + params['language'].split('_')[0] + return params +# get response from search-request def response(resp): + results = [] + raw_search_results = loads(resp.text) + # return empty array if there are no results if not raw_search_results: return [] search_results = raw_search_results.get('channels', {})[0].get('items', []) - results = [] + if resp.search_params['category'] == 'general': + # parse general results + for result in search_results: + publishedDate = parser.parse(result['pubDate']) - for result in search_results: - tmp_result = {} - tmp_result['title'] = result['title'] - tmp_result['url'] = result['link'] - tmp_result['content'] = '' + # append result + results.append({'url': result['link'], + 'title': result['title'], + 'content': result['description'], + 'publishedDate': publishedDate}) - if result['description']: - tmp_result['content'] += result['description'] + "
" + elif resp.search_params['category'] == 'images': + # parse image results + for result in search_results: + # append result + results.append({'url': result['url'], + 'title': result['title'], + 'content': '', + 'img_src': result['image'], + 'template': 'images.html'}) - if result['pubDate']: - tmp_result['content'] += result['pubDate'] + "
" - - if result['size'] != '-1': - tmp_result['content'] += result['sizename'] - - results.append(tmp_result) + #TODO parse video, audio and file results + # return results return results diff --git a/searx/settings.yml b/searx/settings.yml index 7d2a4387d..c2d6f28a8 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -143,6 +143,13 @@ engines: locale : en-US shortcut : vm +# - name : yacy +# engine : yacy +# shortcut : ya +# base_url : 'http://localhost:8090' +# number_of_results : 5 +# timeout: 3.0 + locales: en : English de : Deutsch