[mod] fetch firefox versions in a standalone script

This commit is contained in:
Alexandre Flament 2017-05-28 15:46:45 +02:00 committed by Alex
parent d51732c0e5
commit 066bd916bf
3 changed files with 92 additions and 20 deletions

View file

@ -0,0 +1,15 @@
{
"os": [
"Windows NT 10; WOW64",
"X11; Linux x86_64"
],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [
"59.0.2",
"59.0.1",
"59.0",
"58.0.2",
"58.0.1",
"58.0"
]
}

View file

@ -10,8 +10,10 @@ from codecs import getincrementalencoder
from imp import load_source from imp import load_source
from numbers import Number from numbers import Number
from os.path import splitext, join from os.path import splitext, join
from io import open
from random import choice from random import choice
import sys import sys
import json
from searx import settings from searx import settings
from searx.version import VERSION_STRING from searx.version import VERSION_STRING
@ -39,29 +41,11 @@ else:
logger = logger.getChild('utils') logger = logger.getChild('utils')
ua_versions = ('52.8.1',
'53.0',
'54.0',
'55.0',
'56.0',
'57.0',
'58.0',
'59.0',
'60.0.2')
ua_os = ('Windows NT 6.3; WOW64',
'X11; Linux x86_64',
'X11; Linux x86')
ua = "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
blocked_tags = ('script', blocked_tags = ('script',
'style') 'style')
useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
def gen_useragent(os=None): + "/data/useragents.json", 'r', encoding='utf-8').read())
# TODO
return ua.format(os=os or choice(ua_os), version=choice(ua_versions))
def searx_useragent(): def searx_useragent():
@ -70,6 +54,10 @@ def searx_useragent():
suffix=settings['outgoing'].get('useragent_suffix', '')) suffix=settings['outgoing'].get('useragent_suffix', ''))
def gen_useragent():
return str(useragents['ua'].format(os=choice(useragents['os']), version=choice(useragents['versions'])))
def highlight_content(content, query): def highlight_content(content, query):
if not content: if not content:

69
utils/fetch_firefox_version.py Executable file
View file

@ -0,0 +1,69 @@
#!/usr/bin/env python
# set path
from sys import path
from os.path import realpath, dirname
path.append(realpath(dirname(realpath(__file__)) + '/../'))
#
import json
import requests
import re
from distutils.version import LooseVersion, StrictVersion
from lxml import html
from searx.url_utils import urlparse, urljoin
URL = 'https://ftp.mozilla.org/pub/firefox/releases/'
RELEASE_PATH = '/pub/firefox/releases/'
NORMAL_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?(esr)?$')
# BETA_REGEX = re.compile('.*[0-9]b([0-9\-a-z]+)$')
# ESR_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?esr$')
#
useragent = {
"versions": (),
"os": ('Windows NT 10; WOW64',
'X11; Linux x86_64'),
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
}
def fetch_firefox_versions():
resp = requests.get(URL, timeout=2.0)
if resp.status_code != 200:
raise Exception("Error fetching firefox versions, HTTP code " + resp.status_code)
else:
dom = html.fromstring(resp.text)
versions = []
for link in dom.xpath('//a/@href'):
url = urlparse(urljoin(URL, link))
path = url.path
if path.startswith(RELEASE_PATH):
version = path[len(RELEASE_PATH):-1]
if NORMAL_REGEX.match(version):
versions.append(LooseVersion(version))
list.sort(versions, reverse=True)
return versions
def fetch_firefox_last_versions():
versions = fetch_firefox_versions()
result = []
major_last = versions[0].version[0]
major_list = (major_last, major_last - 1)
for version in versions:
major_current = version.version[0]
if major_current in major_list and 'esr' not in version.version:
result.append(version.vstring)
return result
useragent["versions"] = fetch_firefox_last_versions()
f = open("../searx/data/useragents.json", "wb")
json.dump(useragent, f, sort_keys=True, indent=4, ensure_ascii=False, encoding="utf-8")
f.close()