mirrors/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-23 11:21:00 +00:00
Code Issues Projects Releases Wiki Activity

[fix] when two results are merged, really use the content with more text

Browse source
This commit is contained in:
Dalf 2014-09-22 22:55:51 +02:00
parent bd2db71fa6
commit 6b058962e1
1 changed files with 12 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
searx/search.py
View file

@ -16,6 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
'''
import grequests
import re
from itertools import izip_longest, chain
from datetime import datetime
from operator import itemgetter
@ -76,6 +77,13 @@ def make_callback(engine_name, results, suggestions, callback, params):
return process_callback
# return the meaningful length of the content for a result
def content_result_len(result):
if isinstance(result.get('content'), basestring):
content = re.sub('[,;:!?\./\\\\ ()-_]', '', result.get('content'))
return len(content)
else:
return 0
# score results and remove duplications
def score_results(results):
@ -110,6 +118,9 @@ def score_results(results):
duplicated = False
# check for duplicates
if 'content' in res:
res['content'] = re.sub(' +', ' ', res['content'].strip().replace('\n', ''))
for new_res in results:
# remove / from the end of the url if required
p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
@ -126,7 +137,7 @@ def score_results(results):
# merge duplicates together
if duplicated:
# using content with more text
if res.get('content') > duplicated.get('content'):
if content_result_len(res) > content_result_len(duplicated):
duplicated['content'] = res['content']
# increase result-score