From 44decaa524279e2fc10ce42f044e1ee69bcc2aca Mon Sep 17 00:00:00 2001 From: Bnyro Date: Wed, 6 Mar 2024 11:04:55 +0100 Subject: [PATCH] [feat] pkg.go.dev: use packages.html template --- searx/engines/pkg_go_dev.py | 85 +++++++++++++++++++++++++++++++++++++ searx/settings.yml | 14 +----- 2 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 searx/engines/pkg_go_dev.py diff --git a/searx/engines/pkg_go_dev.py b/searx/engines/pkg_go_dev.py new file mode 100644 index 000000000..dcbd88451 --- /dev/null +++ b/searx/engines/pkg_go_dev.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""pkg.go.dev (packages)""" + +import re +from urllib.parse import urlencode +from dateutil import parser + +import babel +import flask_babel +from lxml import html +from searx.utils import eval_xpath, eval_xpath_list, extract_text + +about = { + 'website': 'https://pkg.go.dev/', + 'use_official_api': False, + 'official_api_documentation': None, + 'require_api_key': False, + 'results': 'HTML', +} + +categories = ["packages", "it"] + +base_url = "https://pkg.go.dev" +max_result_count = 50 + +results_xpath = '/html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"]' +url_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/@href' +title_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/text()' +package_name_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/span/text()' +version_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span/strong[1]/text()' +updated_xpath = ( + './div[contains(@class, "SearchSnippet-infoLabel")]/span/span[@data-test-id="snippet-published"]/strong/text()' +) +content_xpath = './p[@class="SearchSnippet-synopsis"]' +popularity_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/a/strong/text()' +license_name_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span[@data-test-id="snippet-license"]/a/text()' +license_url_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span[@data-test-id="snippet-license"]/a/@href' + + +def request(query, params): + args = { + 'q': query, + 'm': 'package', + 'limit': max_result_count, + } + params['url'] = f"{base_url}/search?{urlencode(args)}" + + return params + + +def response(resp): + results = [] + + doc = html.fromstring(resp.text) + + for result in eval_xpath_list(doc, results_xpath): + publishedDate = extract_text(eval_xpath(result, updated_xpath)) + try: + publishedDate = parser.parse(publishedDate) + except parser.ParserError: + publishedDate = None + + # 110n 15,000.00 (EN) --> 15.000,00 (DE) + popularity = extract_text(eval_xpath(result, popularity_xpath)).strip() + popularity = babel.numbers.parse_decimal(popularity, locale='en_US') + # popularity is of type str .. + popularity = flask_babel.format_decimal(popularity) + + results.append( + { + 'template': 'packages.html', + 'url': base_url + extract_text(eval_xpath(result, url_xpath)), + 'title': extract_text(eval_xpath(result, title_xpath)), + 'content': extract_text(eval_xpath(result, content_xpath)), + 'package_name': re.sub(r"\(|\)", "", extract_text(eval_xpath(result, package_name_xpath))), + 'version': extract_text(eval_xpath(result, version_xpath)), + 'popularity': popularity, + 'license_name': extract_text(eval_xpath(result, license_name_xpath)), + 'license_url': base_url + extract_text(eval_xpath(result, license_url_xpath)), + 'publishedDate': publishedDate, + } + ) + + return results diff --git a/searx/settings.yml b/searx/settings.yml index f292de18a..865cb6b77 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -2314,21 +2314,9 @@ engines: shortcut: bt4g - name: pkg.go.dev - engine: xpath + engine: pkg_go_dev shortcut: pgo - search_url: https://pkg.go.dev/search?limit=100&m=package&q={query} - results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"] - url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href - title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a - content_xpath: ./p[@class="SearchSnippet-synopsis"] - categories: [packages, it] - timeout: 3.0 disabled: true - about: - website: https://pkg.go.dev/ - use_official_api: false - require_api_key: false - results: HTML # Doku engine lets you access to any Doku wiki instance: # A public one or a privete/corporate one.