From 8256de2fe85a8b11497289d8e6884710838b20cf Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 29 Jan 2023 09:59:44 +0000 Subject: [PATCH 1/2] peertube engine: update _fetch_supported_languages There is now an API to get the list of supported languages https://docs.joinpeertube.org/api-rest-reference.html#tag/Video/operation/getLanguages --- searx/engines/peertube.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/searx/engines/peertube.py b/searx/engines/peertube.py index 089775694..345c2f991 100644 --- a/searx/engines/peertube.py +++ b/searx/engines/peertube.py @@ -22,9 +22,7 @@ about = { categories = ["videos"] paging = True base_url = "https://peer.tube" -supported_languages_url = ( - 'https://framagit.org/framasoft/peertube/search-index/-/raw/master/client/src/views/Search.vue' -) +supported_languages_url = 'https://peer.tube/api/v1/videos/languages' # do search-request @@ -84,9 +82,6 @@ def response(resp): def _fetch_supported_languages(resp): - import re - - # https://docs.python.org/3/howto/regex.html#greedy-versus-non-greedy - videolanguages = re.search(r"videoLanguages \(\)[^\n]+(.*?)\]", resp.text, re.DOTALL) - peertube_languages = [m.group(1) for m in re.finditer(r"\{ id: '([a-z]+)', label:", videolanguages.group(1))] + videolanguages = resp.json() + peertube_languages = list(videolanguages.keys()) return peertube_languages From bfca63c5365aa77cea34b70cbf929d7b6d0a789d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 29 Jan 2023 10:01:02 +0000 Subject: [PATCH 2/2] wikipedia engine: update _fetch_supported_languages the layout https://meta.wikimedia.org/wiki/List_of_Wikipedias has changed --- searx/engines/wikipedia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py index 52b1053ed..ca841e8b3 100644 --- a/searx/engines/wikipedia.py +++ b/searx/engines/wikipedia.py @@ -106,9 +106,9 @@ def _fetch_supported_languages(resp): for tr in trs: td = tr.xpath('./td') code = td[3].xpath('./a')[0].text - name = td[2].xpath('./a')[0].text + name = td[1].xpath('./a')[0].text english_name = td[1].xpath('./a')[0].text - articles = int(td[4].xpath('./a/b')[0].text.replace(',', '')) + articles = int(td[4].xpath('./a')[0].text.replace(',', '')) # exclude languages with too few articles if articles >= 100: supported_languages[code] = {"name": name, "english_name": english_name}