From a1ce141c99bd31017a131eb6bc57107b886f1442 Mon Sep 17 00:00:00 2001 From: Michael Ilsaas Date: Sat, 8 Aug 2020 19:22:53 +0200 Subject: [PATCH] add peertube engine (#2109) --- AUTHORS.rst | 1 + searx/data/engines_languages.json | 196 +++++++++++++++++++++++++++++- searx/engines/peertube.py | 95 +++++++++++++++ searx/settings.yml | 8 ++ 4 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 searx/engines/peertube.py diff --git a/AUTHORS.rst b/AUTHORS.rst index 5c1722499..6538c0901 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -133,3 +133,4 @@ generally made searx better: - @gordon-quad - Sophie Tauchert @999eagle - @bauruine +- Michael Ilsaas ``_ diff --git a/searx/data/engines_languages.json b/searx/data/engines_languages.json index 83701e6d7..0caf5b9b9 100644 --- a/searx/data/engines_languages.json +++ b/searx/data/engines_languages.json @@ -25033,6 +25033,200 @@ "name": "\u4e2d\u6587 (\u7e41\u9ad4)" } }, + "peertube": [ + "aa", + "ab", + "af", + "ak", + "am", + "an", + "ar", + "as", + "ase", + "av", + "avk", + "ay", + "az", + "ba", + "be", + "bfi", + "bg", + "bi", + "bm", + "bn", + "bo", + "br", + "bs", + "bzs", + "ca", + "ce", + "ch", + "co", + "cr", + "cs", + "cse", + "csl", + "cv", + "cy", + "da", + "de", + "dsl", + "dv", + "dz", + "ee", + "el", + "en", + "eo", + "es", + "et", + "eu", + "fa", + "ff", + "fi", + "fj", + "fo", + "fr", + "fsl", + "fy", + "ga", + "gd", + "gl", + "gn", + "gsg", + "gu", + "gv", + "ha", + "he", + "hi", + "ho", + "hr", + "ht", + "hu", + "hy", + "hz", + "id", + "ig", + "ii", + "ik", + "is", + "it", + "iu", + "ja", + "jbo", + "jsl", + "jv", + "ka", + "kg", + "ki", + "kj", + "kk", + "kl", + "km", + "kn", + "ko", + "kr", + "ks", + "ku", + "kv", + "kw", + "ky", + "lb", + "lg", + "li", + "ln", + "lo", + "lt", + "lu", + "lv", + "mg", + "mh", + "mi", + "mk", + "ml", + "mn", + "mr", + "ms", + "mt", + "my", + "na", + "nb", + "nd", + "ne", + "ng", + "nl", + "nn", + "no", + "nr", + "nv", + "ny", + "oc", + "oj", + "om", + "or", + "os", + "pa", + "pks", + "pl", + "ps", + "pt", + "qu", + "rm", + "rn", + "ro", + "rsl", + "ru", + "rw", + "sc", + "sd", + "sdl", + "se", + "sfs", + "sg", + "sh", + "si", + "sk", + "sl", + "sm", + "sn", + "so", + "sq", + "sr", + "ss", + "st", + "su", + "sv", + "sw", + "swl", + "ta", + "te", + "tg", + "th", + "ti", + "tk", + "tl", + "tlh", + "tn", + "to", + "tr", + "ts", + "tt", + "tw", + "ty", + "ug", + "uk", + "ur", + "uz", + "ve", + "vi", + "wa", + "wo", + "xh", + "yi", + "yo", + "za", + "zh", + "zu" + ], "qwant": [ "bg-BG", "br-FR", @@ -28331,4 +28525,4 @@ "zh-CHS", "zh-CHT" ] -} \ No newline at end of file +} diff --git a/searx/engines/peertube.py b/searx/engines/peertube.py new file mode 100644 index 000000000..b3795bf83 --- /dev/null +++ b/searx/engines/peertube.py @@ -0,0 +1,95 @@ +""" + peertube (Videos) + + @website https://www.peertube.live + @provide-api yes (https://docs.joinpeertube.org/api-rest-reference.html) + + @using-api yes + @results JSON + @stable yes + @parse url, title, thumbnail, publishedDate, embedded + + @todo implement time range support +""" + +from json import loads +from datetime import datetime +from searx.url_utils import urlencode +from searx.utils import html_to_text + +# engine dependent config +categories = ["videos"] +paging = True +language_support = True +base_url = "https://peer.tube/" +supported_languages_url = base_url + "api/v1/videos/languages" + + +# do search-request +def request(query, params): + pageno = (params["pageno"] - 1) * 15 + search_url = base_url + "api/v1/search/videos/?pageno={pageno}&{query}" + query_dict = {"search": query} + language = params["language"].split("-")[0] + # pylint: disable=undefined-variable + if "all" != language and language in supported_languages: + query_dict["languageOneOf"] = language + params["url"] = search_url.format( + query=urlencode(query_dict), pageno=pageno + ) + return params + + +def _get_offset_from_pageno(pageno): + return (pageno - 1) * 15 + 1 + + +# get response from search-request +def response(resp): + results = [] + + search_res = loads(resp.text) + + embedded_url = ( + '' + ) + # return empty array if there are no results + if "data" not in search_res: + return [] + + # parse results + for res in search_res["data"]: + title = res["name"] + url = base_url + "/videos/watch/" + res["uuid"] + description = res["description"] + if description: + content = html_to_text(res["description"]) + else: + content = None + thumbnail = base_url + res["thumbnailPath"] + publishedDate = datetime.strptime(res["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + embedded = embedded_url.format(embed_path=res["embedPath"][1:]) + + results.append( + { + "template": "videos.html", + "url": url, + "title": title, + "content": content, + "publishedDate": publishedDate, + "embedded": embedded, + "thumbnail": thumbnail, + } + ) + + # return results + return results + + +def _fetch_supported_languages(resp): + ret_val = {} + peertube_languages = list(loads(resp.text).keys()) + return peertube_languages diff --git a/searx/settings.yml b/searx/settings.yml index 6d80fc534..63685be8b 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -877,6 +877,14 @@ engines: categories: it disabled : True + - name : peertube + engine: peertube + shortcut: ptb + paging : True + base_url : https://peer.tube/ + categories: videos + disabled : True + # - name : yacy # engine : yacy # shortcut : ya