Merge branch 'master' of https://github.com/asciimoo/searx into filtron

searx/data/useragents.json

@@ -1,12 +1,12 @@
 {
     "versions": [
-        "74.0",
+        "75.0",
-        "73.0.1",
+        "74.0.1",
-        "73.0"
+        "74.0"
     ],
     "os": [
-        "Windows NT 10; WOW64",
+        "Windows NT 10.0; WOW64",
         "X11; Linux x86_64"
     ],
     "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
-}
+}

searx/engines/faroo.py

@@ -1,96 +0,0 @@
-"""
- Faroo (Web, News)
-
- @website     http://www.faroo.com
- @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
-
- @using-api   no
- @results     JSON
- @stable      yes
- @parse       url, title, content, publishedDate, img_src
-"""
-
-from json import loads
-import datetime
-from searx.utils import searx_useragent
-from searx.url_utils import urlencode
-
-# engine dependent config
-categories = ['general', 'news']
-paging = True
-language_support = True
-number_of_results = 10
-
-# search-url
-url = 'http://www.faroo.com/'
-search_url = url + 'instant.json?{query}'\
-    '&start={offset}'\
-    '&length={number_of_results}'\
-    '&l={language}'\
-    '&src={categorie}'\
-    '&i=false'\
-    '&c=false'
-
-search_category = {'general': 'web',
-                   'news': 'news'}
-
-
-# do search-request
-def request(query, params):
-    offset = (params['pageno'] - 1) * number_of_results + 1
-    categorie = search_category.get(params['category'], 'web')
-
-    if params['language'] == 'all':
-        language = 'en'
-    else:
-        language = params['language'].split('-')[0]
-
-    # if language is not supported, put it in english
-    if language != 'en' and\
-       language != 'de' and\
-       language != 'zh':
-        language = 'en'
-
-    params['url'] = search_url.format(offset=offset,
-                                      number_of_results=number_of_results,
-                                      query=urlencode({'q': query}),
-                                      language=language,
-                                      categorie=categorie)
-
-    params['headers']['Referer'] = url
-
-    return params
-
-
-# get response from search-request
-def response(resp):
-    # HTTP-Code 429: rate limit exceeded
-    if resp.status_code == 429:
-        raise Exception("rate limit has been exceeded!")
-
-    results = []
-
-    search_res = loads(resp.text)
-
-    # return empty array if there are no results
-    if not search_res.get('results', {}):
-        return []
-
-    # parse results
-    for result in search_res['results']:
-        publishedDate = None
-        result_json = {'url': result['url'], 'title': result['title'],
-                       'content': result['kwic']}
-        if result['news']:
-            result_json['publishedDate'] = \
-                datetime.datetime.fromtimestamp(result['date'] / 1000.0)
-
-        # append image result if image url is set
-        if result['iurl']:
-            result_json['template'] = 'videos.html'
-            result_json['thumbnail'] = result['iurl']
-
-        results.append(result_json)
-
-    # return results
-    return results

searx/results.py

@@ -345,8 +345,8 @@ class ResultContainer(object):
             return 0
         return resultnum_sum / len(self._number_of_results)
 
-    def add_unresponsive_engine(self, engine_error):
+    def add_unresponsive_engine(self, engine_name, error_type, error_message=None):
-        self.unresponsive_engines.add(engine_error)
+        self.unresponsive_engines.add((engine_name, error_type, error_message))
 
     def add_timing(self, engine_name, engine_time, page_load_time):
         self.timings.append({

searx/search.py

@@ -127,11 +127,7 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c
         logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
     except Exception as e:
         record_offline_engine_stats_on_error(engine, result_container, start_time)
-
+        result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
-        result_container.add_unresponsive_engine((
-            engine_name,
-            u'{0}: {1}'.format(gettext('unexpected crash'), e),
-        ))
         logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
 
 
@@ -186,24 +182,21 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont
             engine.stats['errors'] += 1
 
         if (issubclass(e.__class__, requests.exceptions.Timeout)):
-            result_container.add_unresponsive_engine((engine_name, gettext('timeout')))
+            result_container.add_unresponsive_engine(engine_name, 'timeout')
             # requests timeout (connect or read)
             logger.error("engine {0} : HTTP requests timeout"
                          "(search duration : {1} s, timeout: {2} s) : {3}"
                          .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))
             requests_exception = True
         elif (issubclass(e.__class__, requests.exceptions.RequestException)):
-            result_container.add_unresponsive_engine((engine_name, gettext('request exception')))
+            result_container.add_unresponsive_engine(engine_name, 'request exception')
             # other requests exception
             logger.exception("engine {0} : requests exception"
                              "(search duration : {1} s, timeout: {2} s) : {3}"
                              .format(engine_name, engine_time, timeout_limit, e))
             requests_exception = True
         else:
-            result_container.add_unresponsive_engine((
+            result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
-                engine_name,
-                u'{0}: {1}'.format(gettext('unexpected crash'), e),
-            ))
             # others errors
             logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
 
@@ -238,7 +231,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim
             remaining_time = max(0.0, timeout_limit - (time() - start_time))
             th.join(remaining_time)
             if th.isAlive():
-                result_container.add_unresponsive_engine((th._engine_name, gettext('timeout')))
+                result_container.add_unresponsive_engine(th._engine_name, 'timeout')
                 logger.warning('engine timeout: {0}'.format(th._engine_name))
 
 

searx/settings.yml

@@ -219,11 +219,6 @@ engines:
     shortcut : et
     disabled : True
 
-  - name : faroo
-    engine : faroo
-    shortcut : fa
-    disabled : True
-
   - name : 1x
     engine : www1x
     shortcut : 1x

searx/webapp.py

@@ -56,6 +56,7 @@ from flask import (
 from babel.support import Translations
 import flask_babel
 from flask_babel import Babel, gettext, format_date, format_decimal
+from flask.ctx import has_request_context
 from flask.json import jsonify
 from searx import brand
 from searx import settings, searx_dir, searx_debug
@@ -165,13 +166,11 @@ _flask_babel_get_translations = flask_babel.get_translations
 
 # monkey patch for flask_babel.get_translations
 def _get_translations():
-    translation_locale = request.form.get('use-translation')
+    if has_request_context() and request.form.get('use-translation') == 'oc':
-    if translation_locale:
         babel_ext = flask_babel.current_app.extensions['babel']
-        translation = Translations.load(next(babel_ext.translation_directories), 'oc')
+        return Translations.load(next(babel_ext.translation_directories), 'oc')
-    else:
+
-        translation = _flask_babel_get_translations()
+    return _flask_babel_get_translations()
-    return translation
 
 
 flask_babel.get_translations = _get_translations
@@ -627,7 +626,7 @@ def index():
                                     'corrections': list(result_container.corrections),
                                     'infoboxes': result_container.infoboxes,
                                     'suggestions': list(result_container.suggestions),
-                                    'unresponsive_engines': list(result_container.unresponsive_engines)},
+                                    'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)},  # noqa
                                    default=lambda item: list(item) if isinstance(item, set) else item),
                         mimetype='application/json')
     elif output_format == 'csv':
@@ -695,7 +694,7 @@ def index():
         corrections=correction_urls,
         infoboxes=result_container.infoboxes,
         paging=result_container.paging,
-        unresponsive_engines=result_container.unresponsive_engines,
+        unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),
         current_language=match_language(search_query.lang,
                                         LANGUAGE_CODES,
                                         fallback=request.preferences.get_value("language")),
@@ -706,6 +705,16 @@ def index():
     )
 
 
+def __get_translated_errors(unresponsive_engines):
+    translated_errors = []
+    for unresponsive_engine in unresponsive_engines:
+        error_msg = gettext(unresponsive_engine[1])
+        if unresponsive_engine[2]:
+            error_msg = "{} {}".format(error_msg, unresponsive_engine[2])
+        translated_errors.append((unresponsive_engine[0], error_msg))
+    return translated_errors
+
+
 @app.route('/about', methods=['GET'])
 def about():
     """Render about page"""