[feat] pkg.go.dev: use packages.html template

This commit is contained in:
Bnyro 2024-03-06 11:04:55 +01:00 committed by Markus Heiser
parent fbea0dd3ce
commit 44decaa524
2 changed files with 86 additions and 13 deletions

View file

@ -0,0 +1,85 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""pkg.go.dev (packages)"""
import re
from urllib.parse import urlencode
from dateutil import parser
import babel
import flask_babel
from lxml import html
from searx.utils import eval_xpath, eval_xpath_list, extract_text
about = {
'website': 'https://pkg.go.dev/',
'use_official_api': False,
'official_api_documentation': None,
'require_api_key': False,
'results': 'HTML',
}
categories = ["packages", "it"]
base_url = "https://pkg.go.dev"
max_result_count = 50
results_xpath = '/html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"]'
url_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/@href'
title_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/text()'
package_name_xpath = './div[@class="SearchSnippet-headerContainer"]/h2/a/span/text()'
version_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span/strong[1]/text()'
updated_xpath = (
'./div[contains(@class, "SearchSnippet-infoLabel")]/span/span[@data-test-id="snippet-published"]/strong/text()'
)
content_xpath = './p[@class="SearchSnippet-synopsis"]'
popularity_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/a/strong/text()'
license_name_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span[@data-test-id="snippet-license"]/a/text()'
license_url_xpath = './div[contains(@class, "SearchSnippet-infoLabel")]/span[@data-test-id="snippet-license"]/a/@href'
def request(query, params):
args = {
'q': query,
'm': 'package',
'limit': max_result_count,
}
params['url'] = f"{base_url}/search?{urlencode(args)}"
return params
def response(resp):
results = []
doc = html.fromstring(resp.text)
for result in eval_xpath_list(doc, results_xpath):
publishedDate = extract_text(eval_xpath(result, updated_xpath))
try:
publishedDate = parser.parse(publishedDate)
except parser.ParserError:
publishedDate = None
# 110n 15,000.00 (EN) --> 15.000,00 (DE)
popularity = extract_text(eval_xpath(result, popularity_xpath)).strip()
popularity = babel.numbers.parse_decimal(popularity, locale='en_US')
# popularity is of type str ..
popularity = flask_babel.format_decimal(popularity)
results.append(
{
'template': 'packages.html',
'url': base_url + extract_text(eval_xpath(result, url_xpath)),
'title': extract_text(eval_xpath(result, title_xpath)),
'content': extract_text(eval_xpath(result, content_xpath)),
'package_name': re.sub(r"\(|\)", "", extract_text(eval_xpath(result, package_name_xpath))),
'version': extract_text(eval_xpath(result, version_xpath)),
'popularity': popularity,
'license_name': extract_text(eval_xpath(result, license_name_xpath)),
'license_url': base_url + extract_text(eval_xpath(result, license_url_xpath)),
'publishedDate': publishedDate,
}
)
return results

View file

@ -2314,21 +2314,9 @@ engines:
shortcut: bt4g
- name: pkg.go.dev
engine: xpath
engine: pkg_go_dev
shortcut: pgo
search_url: https://pkg.go.dev/search?limit=100&m=package&q={query}
results_xpath: /html/body/main/div[contains(@class,"SearchResults")]/div[not(@class)]/div[@class="SearchSnippet"]
url_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a/@href
title_xpath: ./div[@class="SearchSnippet-headerContainer"]/h2/a
content_xpath: ./p[@class="SearchSnippet-synopsis"]
categories: [packages, it]
timeout: 3.0
disabled: true
about:
website: https://pkg.go.dev/
use_official_api: false
require_api_key: false
results: HTML
# Doku engine lets you access to any Doku wiki instance:
# A public one or a privete/corporate one.