searxng/searx/engines/filecrop.py

from html.parser import HTMLParser
from urllib.parse import urlencode


url = 'http://www.filecrop.com/'
search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}'  # noqa

paging = True


class FilecropResultParser(HTMLParser):  # pylint: disable=W0223  # (see https://bugs.python.org/issue31844)

    def __init__(self):
        HTMLParser.__init__(self)
        self.__start_processing = False

        self.results = []
        self.result = {}

        self.tr_counter = 0
        self.data_counter = 0

    def handle_starttag(self, tag, attrs):

        if tag == 'tr':
            if ('bgcolor', '#edeff5') in attrs or\
               ('bgcolor', '#ffffff') in attrs:
                self.__start_processing = True

        if not self.__start_processing:
            return

        if tag == 'label':
            self.result['title'] = [attr[1] for attr in attrs
                                    if attr[0] == 'title'][0]
        elif tag == 'a' and ('rel', 'nofollow') in attrs\
                and ('class', 'sourcelink') in attrs:
            if 'content' in self.result:
                self.result['content'] += [attr[1] for attr in attrs
                                           if attr[0] == 'title'][0]
            else:
                self.result['content'] = [attr[1] for attr in attrs
                                          if attr[0] == 'title'][0]
            self.result['content'] += ' '
        elif tag == 'a':
            self.result['url'] = url + [attr[1] for attr in attrs
                                        if attr[0] == 'href'][0]

    def handle_endtag(self, tag):
        if self.__start_processing is False:
            return

        if tag == 'tr':
            self.tr_counter += 1

            if self.tr_counter == 2:
                self.__start_processing = False
                self.tr_counter = 0
                self.data_counter = 0
                self.results.append(self.result)
                self.result = {}

    def handle_data(self, data):
        if not self.__start_processing:
            return

        if 'content' in self.result:
            self.result['content'] += data + ' '
        else:
            self.result['content'] = data + ' '

        self.data_counter += 1


def request(query, params):
    index = 1 + (params['pageno'] - 1) * 30
    params['url'] = search_url.format(query=urlencode({'w': query}), index=index)
    return params


def response(resp):
    parser = FilecropResultParser()
    parser.feed(resp.text)

    return parser.results