Merge pull request #1792 from unixfox/google-images-internal-api

use the internal API for google images
This commit is contained in:
Alexandre Flament 2022-09-21 19:50:38 +02:00 committed by GitHub
commit 1a7b6872b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -1,28 +1,20 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint # lint: pylint
"""This is the implementation of the google images engine. """This is the implementation of the google images engine using the google
internal API used the Google Go Android app.
.. admonition:: Content-Security-Policy (CSP) This internal API offer results in
This engine needs to allow images from the `data URLs`_ (prefixed with the - JSON (_fmt:json)
``data:`` scheme):: - Protobuf (_fmt:pb)
- Protobuf compressed? (_fmt:pc)
- HTML (_fmt:html)
- Protobuf encoded in JSON (_fmt:jspb).
Header set Content-Security-Policy "img-src 'self' data: ;"
.. _data URLs:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
""" """
import re from urllib.parse import urlencode
from urllib.parse import urlencode, unquote from json import loads
from lxml import html
from searx.utils import (
eval_xpath,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.google import ( from searx.engines.google import (
get_lang_info, get_lang_info,
@ -42,12 +34,12 @@ about = {
"official_api_documentation": 'https://developers.google.com/custom-search', "official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False, "use_official_api": False,
"require_api_key": False, "require_api_key": False,
"results": 'HTML', "results": 'JSON',
} }
# engine dependent config # engine dependent config
categories = ['images', 'web'] categories = ['images', 'web']
paging = False paging = True
use_locale_domain = True use_locale_domain = True
time_range_support = True time_range_support = True
safesearch = True safesearch = True
@ -56,74 +48,8 @@ send_accept_language_header = True
filter_mapping = {0: 'images', 1: 'active', 2: 'active'} filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
def scrap_out_thumbs(dom):
"""Scrap out thumbnail data from <script> tags."""
ret_val = {}
for script in eval_xpath(dom, '//script[contains(., "_setImgSrc(")]'):
_script = script.text
# _setImgSrc('0','data:image\/jpeg;base64,\/9j\/4AAQSkZJR ....');
_thumb_no, _img_data = _script[len("_setImgSrc(") : -2].split(",", 1)
_thumb_no = _thumb_no.replace("'", "")
_img_data = _img_data.replace("'", "")
_img_data = _img_data.replace(r"\/", r"/")
ret_val[_thumb_no] = _img_data.replace(r"\x3d", "=")
return ret_val
# [0, "-H96xjSoW5DsgM", ["https://encrypted-tbn0.gstatic.com/images?q...", 155, 324]
# , ["https://assets.cdn.moviepilot.de/files/d3bf..", 576, 1200],
_RE_JS_IMAGE_URL = re.compile(
r'"'
r'([^"]*)' # -H96xjSoW5DsgM
r'",\s*\["'
r'https://[^\.]*\.gstatic.com/images[^"]*' # https://encrypted-tbn0.gstatic.com/images?q...
r'[^\[]*\["'
r'(https?://[^"]*)' # https://assets.cdn.moviepilot.de/files/d3bf...
)
def parse_urls_img_from_js(dom):
# There are two HTML script tags starting with a JS function
# 'AF_initDataCallback(...)'
#
# <script nonce="zscm+Ab/JzBk1Qd4GY6wGQ">
# AF_initDataCallback({key: 'ds:0', hash: '1', data:[], sideChannel: {}});
# </script>
# <script nonce="zscm+Ab/JzBk1Qd4GY6wGQ">
# AF_initDataCallback({key: 'ds:1', hash: '2', data:[null,[[["online_chips",[["the big",
# ["https://encrypted-tbn0.gstatic.com/images?q...",null,null,true,[null,0],f
# ...
# </script>
#
# The second script contains the URLs of the images.
# The AF_initDataCallback(..) is called with very large dictionary, that
# looks like JSON but it is not JSON since it contains JS variables and
# constants like 'null' (we can't use a JSON parser for).
#
# The alternative is to parse the entire <script> and find all image URLs by
# a regular expression.
img_src_script = eval_xpath_getindex(dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text
data_id_to_img_url = {}
for data_id, url in _RE_JS_IMAGE_URL.findall(img_src_script):
data_id_to_img_url[data_id] = url
return data_id_to_img_url
def get_img_url_by_data_id(data_id_to_img_url, img_node):
"""Get full image URL by @data-id from parent element."""
data_id = eval_xpath_getindex(img_node, '../../../@data-id', 0)
img_url = data_id_to_img_url.get(data_id, '')
img_url = unquote(img_url.replace(r'\u00', r'%'))
return img_url
def request(query, params): def request(query, params):
"""Google-Video search request""" """Google-Image search request"""
lang_info = get_lang_info(params, supported_languages, language_aliases, False) lang_info = get_lang_info(params, supported_languages, language_aliases, False)
@ -132,7 +58,17 @@ def request(query, params):
+ lang_info['subdomain'] + lang_info['subdomain']
+ '/search' + '/search'
+ "?" + "?"
+ urlencode({'q': query, 'tbm': "isch", **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'num': 30}) + urlencode(
{
'q': query,
'tbm': "isch",
**lang_info['params'],
'ie': "utf8",
'oe': "utf8",
'asearch': 'isch',
'async': '_fmt:json,p:1,ijn:' + str(params['pageno']),
}
)
) )
if params['time_range'] in time_range_dict: if params['time_range'] in time_range_dict:
@ -141,9 +77,9 @@ def request(query, params):
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]}) query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url params['url'] = query_url
params['cookies']['CONSENT'] = "YES+"
params['headers'].update(lang_info['headers']) params['headers'].update(lang_info['headers'])
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' params['headers']['User-Agent'] = 'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12; US) gzip'
params['headers']['Accept'] = '*/*'
return params return params
@ -153,78 +89,34 @@ def response(resp):
detect_google_sorry(resp) detect_google_sorry(resp)
# convert the text to dom json_start = resp.text.find('{"ischj":')
dom = html.fromstring(resp.text) json_data = loads(resp.text[json_start:])
img_bas64_map = scrap_out_thumbs(dom)
data_id_to_img_url = parse_urls_img_from_js(dom)
# parse results for item in json_data["ischj"]["metadata"]:
#
# root element::
# <div id="islmp" ..>
# result div per image::
# <div jsmodel="tTXmib"> / <div jsaction="..." data-id="..."
# The data-id matches to a item in a json-data structure in::
# <script nonce="I+vqelcy/01CKiBJi5Z1Ow">AF_initDataCallback({key: 'ds:1', ... data:function(){return [ ...
# In this structure the link to the origin PNG, JPG or whatever is given
# first link per image-div contains a <img> with the data-iid for bas64 encoded image data::
# <img class="rg_i Q4LuWd" data-iid="0"
# second link per image-div is the target link::
# <a class="VFACy kGQAp" href="https://en.wikipedia.org/wiki/The_Sacrament_of_the_Last_Supper">
# the second link also contains two div tags with the *description* and *publisher*::
# <div class="WGvvNb">The Sacrament of the Last Supper ...</div>
# <div class="fxgdke">en.wikipedia.org</div>
root = eval_xpath(dom, '//div[@id="islmp"]') result_item = {
if not root: 'url': item["result"]["referrer_url"],
logger.error("did not find root element id='islmp'") 'title': item["result"]["page_title"],
return results 'content': item["text_in_grid"]["snippet"],
'source': item["result"]["site_title"],
'img_format': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
'img_src': item["original_image"]["url"],
'thumbnail_src': item["thumbnail"]["url"],
'template': 'images.html',
}
root = root[0] author = item["result"].get('iptc', {}).get('creator')
for img_node in eval_xpath_list(root, './/img[contains(@class, "rg_i")]'): if author:
result_item['author'] = ', '.join(author)
img_alt = eval_xpath_getindex(img_node, '@alt', 0) copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
if copyright_notice:
result_item['source'] += ' / ' + copyright_notice
img_base64_id = eval_xpath(img_node, '@data-iid') file_size = item.get('gsa', {}).get('file_size')
if img_base64_id: if file_size:
img_base64_id = img_base64_id[0] result_item['source'] += ' (%s)' % file_size
thumbnail_src = img_bas64_map[img_base64_id]
else:
thumbnail_src = eval_xpath(img_node, '@src')
if not thumbnail_src:
thumbnail_src = eval_xpath(img_node, '@data-src')
if thumbnail_src:
thumbnail_src = thumbnail_src[0]
else:
thumbnail_src = ''
link_node = eval_xpath_getindex(img_node, '../../../a[2]', 0) results.append(result_item)
url = eval_xpath_getindex(link_node, '@href', 0, None)
if url is None:
logger.error("missing @href in node: %s", html.tostring(link_node))
continue
pub_nodes = eval_xpath(link_node, './div/div')
pub_descr = img_alt
pub_source = ''
if pub_nodes:
pub_descr = extract_text(pub_nodes[0])
pub_source = extract_text(pub_nodes[1])
src_url = get_img_url_by_data_id(data_id_to_img_url, img_node)
if not src_url:
src_url = thumbnail_src
results.append(
{
'url': url,
'title': img_alt,
'content': pub_descr,
'source': pub_source,
'img_src': src_url,
'thumbnail_src': thumbnail_src,
'template': 'images.html',
}
)
return results return results