From c1d10bde0227e2ef714aec67a18d1e8180f1ed27 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 2 Oct 2020 18:17:01 +0200 Subject: [PATCH] [mod] searx/utils.py: add docstring --- searx/utils.py | 201 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 160 insertions(+), 41 deletions(-) diff --git a/searx/utils.py b/searx/utils.py index d3969df58..23bcbc124 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -39,12 +39,17 @@ lang_to_lc_cache = dict() def searx_useragent(): + """Return the searx User Agent""" return 'searx/{searx_version} {suffix}'.format( searx_version=VERSION_STRING, suffix=settings['outgoing'].get('useragent_suffix', '')) def gen_useragent(os=None): + """Return a random browser User Agent + + See searx/data/useragents.json + """ return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions']))) @@ -99,24 +104,40 @@ class HTMLTextExtractor(HTMLParser): return ''.join(self.result).strip() -def html_to_text(html): - html = html.replace('\n', ' ') - html = ' '.join(html.split()) +def html_to_text(html_str): + """Extract text from a HTML string + + Args: + * html_str (str): string HTML + + Returns: + * str: extracted text + + Examples: + >>> html_to_text('Example #2') + 'Example #2' + + >>> html_to_text('Example') + 'Example' + """ + html_str = html_str.replace('\n', ' ') + html_str = ' '.join(html_str.split()) s = HTMLTextExtractor() try: - s.feed(html) + s.feed(html_str) except HTMLTextExtractorException: - logger.debug("HTMLTextExtractor: invalid HTML\n%s", html) + logger.debug("HTMLTextExtractor: invalid HTML\n%s", html_str) return s.get_text() def extract_text(xpath_results): - ''' - if xpath_results is list, extract the text from each result and concat the list - if xpath_results is a xml element, extract all the text node from it - ( text_content() method from lxml ) - if xpath_results is a string element, then it's already done - ''' + """Extract text from a lxml result + + * if xpath_results is list, extract the text from each result and concat the list + * if xpath_results is a xml element, extract all the text node from it + ( text_content() method from lxml ) + * if xpath_results is a string element, then it's already done + """ if type(xpath_results) == list: # it's list of result : concat everything using recursive call result = '' @@ -135,29 +156,6 @@ def extract_text(xpath_results): return ' '.join(text.split()) -def extract_url(xpath_results, search_url): - if xpath_results == []: - raise Exception('Empty url resultset') - url = extract_text(xpath_results) - - if url.startswith('//'): - # add http or https to this kind of url //example.com/ - parsed_search_url = urlparse(search_url) - url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url) - elif url.startswith('/'): - # fix relative url to the search engine - url = urljoin(search_url, url) - - # fix relative urls that fall through the crack - if '://' not in url: - url = urljoin(search_url, url) - - # normalize url - url = normalize_url(url) - - return url - - def normalize_url(url): parsed_url = urlparse(url) @@ -178,7 +176,69 @@ def normalize_url(url): return url +def extract_url(xpath_results, search_url): + """Extract and normalize URL from lxml Element + + Args: + * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s) + * search_url (str): Base URL + + Example: + >>> def f(s, search_url): + >>> return searx.utils.extract_url(html.fromstring(s), search_url) + >>> f('https://example.com', 'http://example.com/') + 'https://example.com/' + >>> f('https://example.com', 'http://example.com/') + 'https://example.com/' + >>> f('//example.com', 'http://example.com/') + 'http://example.com/' + >>> f('//example.com', 'https://example.com/') + 'https://example.com/' + >>> f('/path?a=1', 'https://example.com') + 'https://example.com/path?a=1' + >>> f('', 'https://example.com') + raise lxml.etree.ParserError + >>> searx.utils.extract_url([], 'https://example.com') + raise Exception + + Raises: + * Exception + * lxml.etree.ParserError + + Returns: + * str: normalized URL + """ + if xpath_results == []: + raise Exception('Empty url resultset') + url = extract_text(xpath_results) + + if url.startswith('//'): + # add http or https to this kind of url //example.com/ + parsed_search_url = urlparse(search_url) + url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url) + elif url.startswith('/'): + # fix relative url to the search engine + url = urljoin(search_url, url) + + # fix relative urls that fall through the crack + if '://' not in url: + url = urljoin(search_url, url) + + # normalize url + url = normalize_url(url) + + return url + + def dict_subset(d, properties): + """Extract a subset of a dict + + Examples: + >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C']) + {'A': 'a', 'C': 'c'} + >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D']) + {'A': 'a'} + """ result = {} for k in properties: if k in d: @@ -186,8 +246,19 @@ def dict_subset(d, properties): return result -# get element in list or default value def list_get(a_list, index, default=None): + """Get element in list or default value + + Examples: + >>> list_get(['A', 'B', 'C'], 0) + 'A' + >>> list_get(['A', 'B', 'C'], 3) + None + >>> list_get(['A', 'B', 'C'], 3, 'default') + 'default' + >>> list_get(['A', 'B', 'C'], -1) + 'C' + """ if len(a_list) > index: return a_list[index] else: @@ -195,6 +266,21 @@ def list_get(a_list, index, default=None): def get_torrent_size(filesize, filesize_multiplier): + """ + + Args: + * filesize (str): size + * filesize_multiplier (str): TB, GB, .... TiB, GiB... + + Returns: + * int: number of bytes + + Example: + >>> get_torrent_size('5', 'GB') + 5368709120 + >>> get_torrent_size('3.14', 'MiB') + 3140000 + """ try: filesize = float(filesize) @@ -221,14 +307,18 @@ def get_torrent_size(filesize, filesize_multiplier): def convert_str_to_int(number_str): + """Convert number_str to int or 0 if number_str is not a number.""" if number_str.isdigit(): return int(number_str) else: return 0 -# convert a variable to integer or return 0 if it's not a number def int_or_zero(num): + """Convert num to int or 0. num can be either a str or a list. + If num is a list, the first element is converted to int (or return 0 if the list is empty). + If num is a str, see convert_str_to_int + """ if isinstance(num, list): if len(num) < 1: return 0 @@ -237,6 +327,22 @@ def int_or_zero(num): def is_valid_lang(lang): + """Return language code and name if lang describe a language. + + Examples: + >>> is_valid_lang('zz') + False + >>> is_valid_lang('uk') + (True, 'uk', 'ukrainian') + >>> is_valid_lang(b'uk') + (True, 'uk', 'ukrainian') + >>> is_valid_lang('en') + (True, 'en', 'english') + >>> searx.utils.is_valid_lang('Español') + (True, 'es', 'spanish') + >>> searx.utils.is_valid_lang('Spanish') + (True, 'es', 'spanish') + """ if isinstance(lang, bytes): lang = lang.decode() is_abbr = (len(lang) == 2) @@ -264,8 +370,8 @@ def _get_lang_to_lc_dict(lang_list): return value -# auxiliary function to match lang_code in lang_list def _match_language(lang_code, lang_list=[], custom_aliases={}): + """auxiliary function to match lang_code in lang_list""" # replace language code with a custom alias if necessary if lang_code in custom_aliases: lang_code = custom_aliases[lang_code] @@ -287,8 +393,8 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}): return _get_lang_to_lc_dict(lang_list).get(lang_code, None) -# get the language code from lang_list that best matches locale_code def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'): + """get the language code from lang_list that best matches locale_code""" # try to get language from given locale_code language = _match_language(locale_code, lang_list, custom_aliases) if language: @@ -330,6 +436,7 @@ def load_module(filename, module_dir): def to_string(obj): + """Convert obj to its string representation.""" if isinstance(obj, str): return obj if isinstance(obj, Number): @@ -341,13 +448,19 @@ def to_string(obj): def ecma_unescape(s): - """ - python implementation of the unescape javascript function + """Python implementation of the unescape javascript function https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape + + Examples: + >>> ecma_unescape('%u5409') + '吉' + >>> ecma_unescape('%20') + ' ' + >>> ecma_unescape('%F3') + 'ó' """ - # s = unicode(s) # "%u5409" becomes "吉" s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s) # "%20" becomes " ", "%F3" becomes "ó" @@ -371,6 +484,11 @@ def get_engine_from_settings(name): def get_xpath(xpath_str): + """Return cached compiled XPath + + There is no thread lock. + Worst case scenario, xpath_str is compiled more than one time. + """ result = xpath_cache.get(xpath_str, None) if result is None: result = XPath(xpath_str) @@ -379,5 +497,6 @@ def get_xpath(xpath_str): def eval_xpath(element, xpath_str): + """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all.""" xpath = get_xpath(xpath_str) return xpath(element)