diff --git a/hooks/pre-commit-python.hook b/hooks/pre-commit-python.hook index 1c0efb6f58..5129f007d5 100755 --- a/hooks/pre-commit-python.hook +++ b/hooks/pre-commit-python.hook @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 import os import subprocess import sys @@ -68,13 +68,13 @@ def main(): break if output_message: - print output_message + print(output_message) if non_compliant_files: - print NOT_PEP8_COMPLIANT_MESSAGE_POST + print(NOT_PEP8_COMPLIANT_MESSAGE_POST) for non_compliant_file in non_compliant_files: - print "autopep8 -i ", non_compliant_file, "; git add ", \ - non_compliant_file - print "git commit" + print("autopep8 -i ", non_compliant_file, "; git add ", + non_compliant_file) + print("git commit") sys.exit(1) diff --git a/validate/launcher/RangeHTTPServer.py b/validate/launcher/RangeHTTPServer.py index 9f97c4708d..c29ac96d88 100644 --- a/validate/launcher/RangeHTTPServer.py +++ b/validate/launcher/RangeHTTPServer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Portions Copyright (C) 2009,2010 Xyne # Portions Copyright (C) 2011 Sean Goller @@ -34,24 +34,21 @@ __all__ = ["RangeHTTPRequestHandler"] import os import sys + import posixpath -import BaseHTTPServer -from SocketServer import ThreadingMixIn -import urllib -import cgi +import http.server +import urllib.parse +import html import shutil import mimetypes +import io import time -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO _bandwidth = 0 -class RangeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RangeHTTPRequestHandler(http.server.BaseHTTPRequestHandler): """Simple HTTP request handler with GET and HEAD commands. @@ -69,7 +66,7 @@ class RangeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def do_GET(self): """Serve a GET request.""" f, start_range, end_range = self.send_head() - print "Got values of ", start_range, " and ", end_range, "...\n" + print ("Got values of {} and {}".format(start_range, end_range)) if f: f.seek(start_range, 0) chunk = 0x1000 @@ -110,13 +107,13 @@ class RangeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): path = self.translate_path(self.path) f = None if os.path.isdir(path): - if not self.path.endswith('/'): - # redirect browser - doing basically what apache does + if not self.path.endswith("/"): + # redirect browser self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return (None, 0, 0) - for index in "index.html", "index.htm": + for index in "index.html", "index.html": index = os.path.join(path, index) if os.path.exists(index): path = index @@ -124,83 +121,97 @@ class RangeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): else: return self.list_directory(path) ctype = self.guess_type(path) + try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! - f = open(path, 'rb') + f = open(path, "rb") except IOError: self.send_error(404, "File not found") return (None, 0, 0) + if "Range" in self.headers: - self.send_response(206) - else: + self.send_response(206) #partial content response + else : self.send_response(200) + self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - size = int(fs[6]) + file_size = os.path.getsize(path) + start_range = 0 - end_range = size + end_range = file_size + self.send_header("Accept-Ranges", "bytes") if "Range" in self.headers: - s, e = self.headers['range'][6:].split('-', 1) + s, e = self.headers['range'][6:].split('-', 1) #bytes:%d-%d sl = len(s) el = len(e) - if sl > 0: + + if sl: start_range = int(s) - if el > 0: + if el: end_range = int(e) + 1 - elif el > 0: - ei = int(e) - if ei < size: - start_range = size - ei - self.send_header("Content-Range", 'bytes ' + str( - start_range) + '-' + str(end_range - 1) + '/' + str(size)) + elif el: + start_range = file_size - min(file_size, int(e)) + + self.send_header("Content-Range", "bytes {}-{}/{}".format(start_range, end_range, file_size)) self.send_header("Content-Length", end_range - start_range) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() + + print ("Sending bytes {} to {}...".format(start_range, end_range)) return (f, start_range, end_range) def list_directory(self, path): """Helper to produce a directory listing (absent index.html). - Return value is either a file object, or None (indicating an - error). In either case, the headers are sent, making the - interface the same as for send_head(). + Return value is either a file object, or None (indicating an + error). In either case, the headers are sent, making the + interface the same as for send_head(). - """ + """ try: - list = os.listdir(path) - except os.error: - self.send_error(404, "No permission to list directory") + lst = os.listdir(path) + except OSError: + self.send_error(404, "Access Forbidden") return None - list.sort(key=lambda a: a.lower()) - f = StringIO() - displaypath = cgi.escape(urllib.unquote(self.path)) - f.write('') - f.write("\nDirectory listing for %s\n" % - displaypath) - f.write("\n

Directory listing for %s

\n" % displaypath) - f.write("
\n\n\n\n\n') + + byte_encoded_string = "\n".join(html_text).encode("utf-8", "surrogateescape") + f = io.BytesIO() + f.write(byte_encoded_string) + length = len(byte_encoded_string) + f.seek(0) + self.send_response(200) self.send_header("Content-type", "text/html") - self.send_header("Content-Length", str(length)) + self.send_header("Content-length", str(length)) self.end_headers() + return (f, 0, length) def translate_path(self, path): @@ -211,36 +222,21 @@ class RangeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): probably be diagnosed.) """ - # abandon query parameters - path = path.split('?', 1)[0] - path = path.split('#', 1)[0] - path = posixpath.normpath(urllib.unquote(path)) - words = path.split('/') + #abandon query parameters + path = path.split("?", 1)[0] + path = path.split("#", 1)[0] + path = posixpath.normpath(urllib.parse.unquote(path)) + words = path.split("/") words = filter(None, words) path = os.getcwd() + for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) - if word in (os.curdir, os.pardir): - continue + if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path - def copyfile(self, source, outputfile): - """Copy all data between two file objects. - - The SOURCE argument is a file object open for reading - (or anything with a read() method) and the DESTINATION - argument is a file object open for writing (or - anything with a write() method). - - The only reason for overriding this would be to change - the block size or perhaps to replace newlines by CRLF - -- note however that this the default server uses this - to copy binary data as well. - - """ - shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. @@ -258,37 +254,31 @@ class RangeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """ base, ext = posixpath.splitext(path) - if ext in self.extensions_map: - return self.extensions_map[ext] + if ext in self.extension_map: + return self.extension_map[ext] ext = ext.lower() - if ext in self.extensions_map: - return self.extensions_map[ext] + if ext in self.extension_map: + return self.extension_map[ext] else: - return self.extensions_map[''] + return self.extension_map[''] - if not mimetypes.inited: - mimetypes.init() # try to read system mime.types - extensions_map = mimetypes.types_map.copy() - extensions_map.update({ - '': 'application/octet-stream', # Default - '.py': 'text/plain', - '.c': 'text/plain', - '.h': 'text/plain', - '.mp4': 'video/mp4', - '.ogg': 'video/ogg', - }) + if not mimetypes.inited: + mimetypes.init() + extension_map = mimetypes.types_map.copy() + extension_map.update({ + '': 'application/octet-stream', # Default + '.py': 'text/plain', + '.c': 'text/plain', + '.h': 'text/plain', + '.mp4': 'video/mp4', + '.ogg': 'video/ogg', + '.java' : 'text/plain', + }) -class ThreadedHTTPServer(ThreadingMixIn, BaseHTTPServer.HTTPServer): - """Handle requests in a separate thread.""" +def test(handler_class = RangeHTTPRequestHandler,server_class = http.server.HTTPServer): + http.server.test(handler_class, server_class) - -def test(HandlerClass=RangeHTTPRequestHandler, - ServerClass=ThreadedHTTPServer): - BaseHTTPServer.test(HandlerClass, ServerClass) - - -if __name__ == '__main__': - if len(sys.argv) > 2: - _bandwidth = int(sys.argv[2]) - test() +if __name__ == "__main__": + httpd = http.server.HTTPServer(("0.0.0.0", int(sys.argv[1])), RangeHTTPRequestHandler) + httpd.serve_forever() diff --git a/validate/launcher/__init__.py b/validate/launcher/__init__.py index 4d1ecfcbeb..13694d6507 100644 --- a/validate/launcher/__init__.py +++ b/validate/launcher/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2014,Thibault Saunier # diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py index a70e7bc1c2..711ee7866f 100644 --- a/validate/launcher/apps/gstvalidate.py +++ b/validate/launcher/apps/gstvalidate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2013,Thibault Saunier # @@ -19,9 +19,9 @@ import argparse import os import time -import urlparse +import urllib.parse import subprocess -import ConfigParser +import configparser from launcher.loggable import Loggable from launcher.baseclasses import GstValidateTest, Test, \ @@ -338,7 +338,7 @@ class GstValidateMixerTestsGenerator(GstValidatePipelineTestsGenerator): self.mixed_srcs[name] = tuple(srcs) - for name, srcs in self.mixed_srcs.iteritems(): + for name, srcs in self.mixed_srcs.items(): if isinstance(srcs, dict): pipe_arguments = { "mixer": self.mixer + " %s" % srcs["mixer_props"]} @@ -454,7 +454,7 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa extra_env_variables = extra_env_variables or {} - file_dur = long(media_descriptor.get_duration()) / GST_SECOND + file_dur = int(media_descriptor.get_duration()) / GST_SECOND if not media_descriptor.get_num_tracks("video"): self.debug("%s audio only file applying transcoding ratio." "File 'duration' : %s" % (classname, file_dur)) @@ -490,8 +490,8 @@ class GstValidateTranscodingTest(GstValidateTest, GstValidateEncodingTestInterfa self.dest_file = os.path.join(self.options.dest, self.classname.replace(".transcode.", os.sep). replace(".", os.sep)) - mkdir(os.path.dirname(urlparse.urlsplit(self.dest_file).path)) - if urlparse.urlparse(self.dest_file).scheme == "": + mkdir(os.path.dirname(urllib.parse.urlsplit(self.dest_file).path)) + if urllib.parse.urlparse(self.dest_file).scheme == "": self.dest_file = path2url(self.dest_file) profile = self.get_profile() @@ -647,7 +647,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") uri.startswith("http://127.0.0.1:8079/"): uri = uri.replace("http://127.0.0.1:8079/", "http://127.0.0.1:%r/" % self.options.http_server_port, 1) - media_descriptor.set_protocol(urlparse.urlparse(uri).scheme) + media_descriptor.set_protocol(urllib.parse.urlparse(uri).scheme) for caps2, prot in GST_VALIDATE_CAPS_TO_PROTOCOL: if caps2 == caps: media_descriptor.set_protocol(prot) @@ -660,7 +660,7 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""") NamedDic({"path": media_info, "media_descriptor": media_descriptor}), special_scenarios)) - except ConfigParser.NoOptionError as e: + except configparser.NoOptionError as e: self.debug("Exception: %s for %s", e, media_info) def _discover_file(self, uri, fpath): diff --git a/validate/launcher/baseclasses.py b/validate/launcher/baseclasses.py index 73f0328c17..a8c0a3c8c1 100644 --- a/validate/launcher/baseclasses.py +++ b/validate/launcher/baseclasses.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2013,Thibault Saunier # @@ -24,22 +24,22 @@ import os import sys import re import copy -import SocketServer +import socketserver import struct import time -import utils +from . import utils import signal -import urlparse +import urllib.parse import subprocess import threading -import Queue -import reporters -import ConfigParser -import loggable -from loggable import Loggable +import queue +from . import reporters +import configparser +from . import loggable +from .loggable import Loggable import xml.etree.cElementTree as ET -from utils import mkdir, Result, Colors, printc, DEFAULT_TIMEOUT, GST_SECOND, \ +from .utils import mkdir, Result, Colors, printc, DEFAULT_TIMEOUT, GST_SECOND, \ Protocols, look_for_file_in_source_dir, get_data_file # The factor by which we increase the hard timeout when running inside @@ -208,8 +208,8 @@ class Test(Loggable): self.process.communicate() else: pname = subprocess.check_output(("readlink -e /proc/%s/exe" - % self.process.pid).split(' ')).replace('\n', '') - raw_input("%sTimeout happened you can attach gdb doing: $gdb %s %d%s\n" + % self.process.pid).decode().split(' ')).replace('\n', '') + input("%sTimeout happened you can attach gdb doing: $gdb %s %d%s\n" "Press enter to continue" % (Colors.FAIL, pname, self.process.pid, Colors.ENDC)) @@ -354,7 +354,7 @@ class Test(Loggable): for supp in self.get_valgrind_suppressions(): vg_args.append(('suppressions', supp)) - self.command = "valgrind %s %s" % (' '.join(map(lambda x: '--%s=%s' % (x[0], x[1]), vg_args)), + self.command = "valgrind %s %s" % (' '.join(['--%s=%s' % (x[0], x[1]) for x in vg_args]), self.command) # Tune GLib's memory allocator to be more valgrind friendly @@ -387,7 +387,7 @@ class Test(Loggable): self.build_arguments() self.proc_env = self.get_subproc_env() - for var, value in self.extra_env_variables.items(): + for var, value in list(self.extra_env_variables.items()): value = self.proc_env.get(var, '') + os.pathsep + value self.proc_env[var] = value.strip(os.pathsep) self.add_env_variable(var, self.proc_env[var]) @@ -428,7 +428,7 @@ class Test(Loggable): printc(message, Colors.FAIL) with open(logfile, 'r') as fin: - print fin.read() + print(fin.read()) def _dump_log_files(self): printc("Dumping log files on failure\n", Colors.FAIL) @@ -454,15 +454,15 @@ class Test(Loggable): return self.result -class GstValidateListener(SocketServer.BaseRequestHandler): +class GstValidateListener(socketserver.BaseRequestHandler): def handle(self): """Implements BaseRequestHandler handle method""" while True: raw_len = self.request.recv(4) - if raw_len == '': + if raw_len == b'': return msglen = struct.unpack('>I', raw_len)[0] - msg = self.request.recv(msglen) + msg = self.request.recv(msglen).decode() if msg == '': return @@ -575,7 +575,7 @@ class GstValidateTest(Test): self.actions_infos.append(action_infos) def server_wrapper(self, ready): - self.server = SocketServer.TCPServer(('localhost', 0), GstValidateListener) + self.server = socketserver.TCPServer(('localhost', 0), GstValidateListener) self.server.socket.settimeout(0.0) self.server.test = self self.serverport = self.server.socket.getsockname()[1] @@ -709,7 +709,7 @@ class GstValidateTest(Test): for key in ['bug', 'sometimes']: if key in expected_failure: del expected_failure[key] - for key, value in report.items(): + for key, value in list(report.items()): if key in expected_failure: if not re.findall(expected_failure[key], value): return False @@ -826,7 +826,7 @@ class GstValidateEncodingTestInterface(object): def get_current_size(self): try: - size = os.stat(urlparse.urlparse(self.dest_file).path).st_size + size = os.stat(urllib.parse.urlparse(self.dest_file).path).st_size except OSError: return None @@ -963,7 +963,7 @@ class TestsManager(Loggable): self.wanted_tests_patterns = [] self.blacklisted_tests_patterns = [] self._generators = [] - self.queue = Queue.Queue() + self.queue = queue.Queue() self.jobs = [] self.total_num_tests = 0 self.starting_test_num = 0 @@ -979,7 +979,7 @@ class TestsManager(Loggable): def add_expected_issues(self, expected_failures): expected_failures_re = {} - for test_name_regex, failures in expected_failures.items(): + for test_name_regex, failures in list(expected_failures.items()): regex = re.compile(test_name_regex) expected_failures_re[regex] = failures for test in self.tests: @@ -989,7 +989,7 @@ class TestsManager(Loggable): self.expected_failures.update(expected_failures_re) def add_test(self, test): - for regex, failures in self.expected_failures.items(): + for regex, failures in list(self.expected_failures.items()): if regex.findall(test.classname): test.expected_failures.extend(failures) @@ -1094,7 +1094,7 @@ class TestsManager(Loggable): # Check process every second for timeout try: self.queue.get(timeout=1) - except Queue.Empty: + except queue.Empty: pass for test in self.jobs: @@ -1232,7 +1232,7 @@ class _TestsLauncher(Loggable): files = [] for f in files: if f.endswith(".py"): - execfile(os.path.join(app_dir, f), env) + exec(compile(open(os.path.join(app_dir, f)).read(), os.path.join(app_dir, f), 'exec'), env) def _exec_apps(self, env): app_dirs = self._list_app_dirs() @@ -1315,7 +1315,7 @@ class _TestsLauncher(Loggable): globals()["options"] = options c__file__ = __file__ globals()["__file__"] = self.options.config - execfile(self.options.config, globals()) + exec(compile(open(self.options.config).read(), self.options.config, 'exec'), globals()) globals()["__file__"] = c__file__ def set_settings(self, options, args): @@ -1374,7 +1374,7 @@ class _TestsLauncher(Loggable): and tester.check_testslist: try: testlist_file = open(os.path.splitext(testsuite.__file__)[0] + ".testslist", - 'rw') + 'r+') know_tests = testlist_file.read().split("\n") testlist_file.close() @@ -1410,7 +1410,7 @@ class _TestsLauncher(Loggable): return -1 self.tests.extend(tests) - return sorted(list(self.tests)) + return sorted(list(self.tests), key=lambda t: t.classname) def _run_tests(self): cur_test_num = 0 @@ -1458,7 +1458,7 @@ class NamedDic(object): def __init__(self, props): if props: - for name, value in props.iteritems(): + for name, value in props.items(): setattr(self, name, value) @@ -1562,7 +1562,7 @@ class ScenarioManager(Loggable): except subprocess.CalledProcessError: pass - config = ConfigParser.ConfigParser() + config = configparser.RawConfigParser() f = open(scenario_defs) config.readfp(f) @@ -1582,7 +1582,8 @@ class ScenarioManager(Loggable): name = section path = None - scenarios.append(Scenario(name, config.items(section), path)) + props = config.items(section) + scenarios.append(Scenario(name, props, path)) if not scenario_paths: self.discovered = True @@ -1744,7 +1745,7 @@ class GstValidateMediaDescriptor(MediaDescriptor): self.media_xml.attrib["duration"] self.media_xml.attrib["seekable"] - self.set_protocol(urlparse.urlparse(urlparse.urlparse(self.get_uri()).scheme).scheme) + self.set_protocol(urllib.parse.urlparse(urllib.parse.urlparse(self.get_uri()).scheme).scheme) @staticmethod def new_from_uri(uri, verbose=False, full=False): @@ -1808,7 +1809,7 @@ class GstValidateMediaDescriptor(MediaDescriptor): return self.media_xml.attrib["uri"] def get_duration(self): - return long(self.media_xml.attrib["duration"]) + return int(self.media_xml.attrib["duration"]) def set_protocol(self, protocol): self.media_xml.attrib["protocol"] = protocol diff --git a/validate/launcher/config.py.in b/validate/launcher/config.py.in index 5739c6e387..3c6e0cd7f2 100644 --- a/validate/launcher/config.py.in +++ b/validate/launcher/config.py.in @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2015,Thibault Saunier # diff --git a/validate/launcher/httpserver.py b/validate/launcher/httpserver.py index 42dc003e04..f813bceef3 100644 --- a/validate/launcher/httpserver.py +++ b/validate/launcher/httpserver.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2013,Thibault Saunier # @@ -19,10 +19,10 @@ import os import time -import loggable +from . import loggable import subprocess import sys -import urllib2 +import urllib.request, urllib.error, urllib.parse logcat = "httpserver" @@ -42,10 +42,10 @@ class HTTPServer(loggable.Loggable): start = time.time() while True: try: - response = urllib2.urlopen('http://127.0.0.1:%s' % ( + response = urllib.request.urlopen('http://127.0.0.1:%s' % ( self.options.http_server_port)) return True - except urllib2.URLError as e: + except urllib.error.URLError as e: pass if time.time() - start > timeout: @@ -61,7 +61,7 @@ class HTTPServer(loggable.Loggable): if self._check_is_up(timeout=2): return True - print "Starting Server" + print("Starting Server") try: self.debug("Launching http server") cmd = "%s %s %d %s" % (sys.executable, os.path.join(os.path.dirname(__file__), @@ -85,14 +85,14 @@ class HTTPServer(loggable.Loggable): time.sleep(1) if self._check_is_up(): - print "Started" + print("Started") return True else: - print "Failed starting server" + print("Failed starting server") self._process.terminate() self._process = None except OSError as ex: - print "Failed starting server" + print("Failed starting server") self.warning(logcat, "Could not launch server %s" % ex) return False diff --git a/validate/launcher/loggable.py b/validate/launcher/loggable.py index d033ece3d2..9ed389dada 100644 --- a/validate/launcher/loggable.py +++ b/validate/launcher/loggable.py @@ -1,5 +1,5 @@ -# CC'd from 'pitivi/log/loggable.py' -# +# -*- coding: utf-8 -*- +# Pitivi video editor # Copyright (c) 2009, Alessandro Decina # # This program is free software; you can redistribute it and/or @@ -16,16 +16,16 @@ # License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, # Boston, MA 02110-1301, USA. - +import collections import errno -import sys -import re -import os import fnmatch +import os +import re +import sys +import threading import time -import types import traceback -import thread +import types # environment variables controlling levels for each category @@ -43,11 +43,11 @@ _log_handlers = [] _log_handlers_limited = [] _initialized = False -_enableCrackOutput = False _stdout = None _stderr = None _old_hup_handler = None +_outfile = None # public log levels @@ -56,7 +56,7 @@ _old_hup_handler = None FIXME, INFO, DEBUG, - LOG) = range(1, 7) + LOG) = list(range(1, 7)) COLORS = {ERROR: 'RED', WARN: 'YELLOW', @@ -69,11 +69,8 @@ _FORMATTED_LEVELS = [] _LEVEL_NAMES = ['ERROR', 'WARN', 'FIXME', 'INFO', 'DEBUG', 'LOG'] -class TerminalController(object): - - """ - A class that can be used to portably generate formatted output to - a terminal. +class TerminalController: + """A class for generating formatted output to a terminal. `TerminalController` defines a set of instance variables whose values are initialized to the control sequence necessary to @@ -81,13 +78,13 @@ class TerminalController(object): output to the terminal: >>> term = TerminalController() - >>> print 'This is '+term.GREEN+'green'+term.NORMAL + >>> print('This is '+term.GREEN+'green'+term.NORMAL) Alternatively, the `render()` method can used, which replaces '${action}' with the string required to perform 'action': >>> term = TerminalController() - >>> print term.render('This is ${GREEN}green${NORMAL}') + >>> print(term.render('This is ${GREEN}green${NORMAL}')) If the terminal doesn't support a given action, then the value of the corresponding instance variable will be set to ''. As a @@ -99,10 +96,15 @@ class TerminalController(object): >>> term = TerminalController() >>> if term.CLEAR_SCREEN: - ... print 'This terminal supports clearning the screen.' + ... print('This terminal supports clearning the screen.') Finally, if the width and height of the terminal are known, then they will be stored in the `COLS` and `LINES` attributes. + + Args: + term_stream (Optional): The stream that will be used for terminal + output; if this stream is not a tty, then the terminal is + assumed to be a dumb terminal (i.e., have no capabilities). """ # Cursor movement: BOL = '' # : Move the cursor to the beginning of the line @@ -148,13 +150,6 @@ class TerminalController(object): _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split() def __init__(self, term_stream=sys.stdout): - """ - Create a `TerminalController` and initialize its attributes - with appropriate values for the current terminal. - `term_stream` is the stream that will be used for terminal - output; if this stream is not a tty, then the terminal is - assumed to be a dumb terminal (i.e., have no capabilities). - """ # Curses isn't available on all platforms try: import curses @@ -179,42 +174,42 @@ class TerminalController(object): # Look up string capabilities. for capability in self._STRING_CAPABILITIES: (attrib, cap_name) = capability.split('=') - setattr(self, attrib, self._tigetstr(cap_name) or '') + setattr(self, attrib, self._tigetstr(cap_name) or b'') # Colors set_fg = self._tigetstr('setf') if set_fg: - for i, color in zip(range(len(self._COLORS)), self._COLORS): - setattr(self, color, curses.tparm(set_fg, i) or '') + for i, color in zip(list(range(len(self._COLORS))), self._COLORS): + setattr(self, color, curses.tparm(set_fg, i) or b'') set_fg_ansi = self._tigetstr('setaf') if set_fg_ansi: - for i, color in zip(range(len(self._ANSICOLORS)), + for i, color in zip(list(range(len(self._ANSICOLORS))), self._ANSICOLORS): - setattr(self, color, curses.tparm(set_fg_ansi, i) or '') + setattr(self, color, curses.tparm(set_fg_ansi, i) or b'') set_bg = self._tigetstr('setb') if set_bg: - for i, color in zip(range(len(self._COLORS)), self._COLORS): - setattr(self, 'BG_' + color, curses.tparm(set_bg, i) or '') + for i, color in zip(list(range(len(self._COLORS))), self._COLORS): + setattr(self, 'BG_' + color, curses.tparm(set_bg, i) or b'') set_bg_ansi = self._tigetstr('setab') if set_bg_ansi: - for i, color in zip(range(len(self._ANSICOLORS)), + for i, color in zip(list(range(len(self._ANSICOLORS))), self._ANSICOLORS): setattr( - self, 'BG_' + color, curses.tparm(set_bg_ansi, i) or '') + self, 'BG_' + color, curses.tparm(set_bg_ansi, i) or b'') def _tigetstr(self, cap_name): # String capabilities can include "delays" of the form "$<2>". # For any modern terminal, we should be able to just ignore # these, so strip them out. import curses - cap = curses.tigetstr(cap_name) or '' - return re.sub(r'\$<\d+>[/*]?', '', cap) + cap = curses.tigetstr(cap_name) or b'' + return re.sub(r'\$<\d+>[/*]?', '', cap.decode()).encode() def render(self, template): - """ - Replace each $-substitutions in the given template string with - the corresponding terminal control string (if it's defined) or - '' (if it's not). + """Replaces each $-substitutions in the specified template string. + + The placeholders are replaced with the corresponding terminal control + string (if it's defined) or '' (if it's not). """ return re.sub(r'\$\$|\${\w+}', self._render_sub, template) @@ -231,9 +226,9 @@ class TerminalController(object): class ProgressBar: + """A 3-line progress bar. - """ - A 3-line progress bar, which looks like:: + Looks like this: Header 20% [===========----------------------------------] @@ -242,6 +237,7 @@ class ProgressBar: The progress bar is colored, if the terminal supports color output; and adjusts to the width of the terminal. """ + BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n' HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n' @@ -275,34 +271,36 @@ class ProgressBar: def getLevelName(level): + """Returns the name of the specified log level. + + Args: + level (int): The level we want to know the name. + + Returns: + str: The name of the level. """ - Return the name of a log level. - @param level: The level we want to know the name - @type level: int - @return: The name of the level - @rtype: str - """ - assert isinstance(level, int) and level > 0 and level < 6, \ + assert isinstance(level, int) and level > 0 and level < 7, \ TypeError("Bad debug level") return getLevelNames()[level - 1] def getLevelNames(): - """ - Return a list with the level names - @return: A list with the level names - @rtype: list of str + """Returns a list with the level names. + + Returns: + List[str]: A list with the level names. """ return _LEVEL_NAMES def getLevelInt(levelName): - """ - Return the integer value of the levelName. - @param levelName: The string value of the level name - @type levelName: str - @return: The value of the level name we are interested in. - @rtype: int + """Returns the integer value of the levelName. + + Args: + levelName (str): The string value of the level name. + + Returns: + int: The value of the level name we are interested in. """ assert isinstance(levelName, str) and levelName in getLevelNames(), \ "Bad debug level name" @@ -316,8 +314,8 @@ def getFormattedLevelName(level): def registerCategory(category): - """ - Register a given category in the debug system. + """Registers the specified category in the debug system. + A level will be assigned to it based on previous calls to setDebug. """ # parse what level it is set to based on _DEBUG @@ -352,11 +350,13 @@ def registerCategory(category): def getCategoryLevel(category): - """ - @param category: string + """Gets the debug level at which the specified category is being logged. - Get the debug level at which this category is being logged, adding it - if it wasn't registered yet. + Registers the category and thus assigns a log level if it wasn't registered + yet. + + Args: + category (string): The category we are interested in. """ global _categories if category not in _categories: @@ -365,10 +365,13 @@ def getCategoryLevel(category): def setLogSettings(state): - """Update the current log settings. + """Updates the current log settings. + This can restore an old saved log settings object returned by - getLogSettings - @param state: the settings to set + getLogSettings. + + Args: + state: The settings to set. """ global _DEBUG @@ -386,9 +389,12 @@ def setLogSettings(state): def getLogSettings(): """Fetches the current log settings. + The returned object can be sent to setLogSettings to restore the returned settings - @returns: the current settings + + Returns: + The current settings. """ return (_DEBUG, _categories, @@ -419,29 +425,27 @@ def scrubFilename(filename): def getFileLine(where=-1): - """ - Return the filename and line number for the given location. + """Returns the filename and line number for the specified location. - If where is a negative integer, look for the code entry in the current - stack that is the given number of frames above this module. - If where is a function, look for the code entry of the function. + Args: + where(int or function): If it's a (negative) integer, looks for + the code entry in the current stack that is the given number + of frames above this module. + If it's a function, look for the code entry of the function. - @param where: how many frames to go back up, or function - @type where: int (negative) or function - - @return: tuple of (file, line) - @rtype: tuple of (str, int) + Returns: + str, int, str: file, line, function_name. """ co = None lineno = None name = None if isinstance(where, types.FunctionType): - co = where.func_code + co = where.__code__ lineno = co.co_firstlineno name = co.co_name elif isinstance(where, types.MethodType): - co = where.im_func.func_code + co = where.__func__.__code__ lineno = co.co_firstlineno name = co.co_name else: @@ -449,10 +453,6 @@ def getFileLine(where=-1): while stackFrame: co = stackFrame.f_code if not co.co_filename.endswith('loggable.py'): - # wind up the stack according to frame - while where < -1: - stackFrame = stackFrame.f_back - where += 1 co = stackFrame.f_code lineno = stackFrame.f_lineno name = co.co_name @@ -460,15 +460,13 @@ def getFileLine(where=-1): stackFrame = stackFrame.f_back if not co: - return "", 0 + return "", 0, None return scrubFilename(co.co_filename), lineno, name def ellipsize(o): - """ - Ellipsize the representation of the given object. - """ + """Ellipsizes the representation of the given object.""" r = repr(o) if len(r) < 800: return r @@ -478,15 +476,15 @@ def ellipsize(o): def getFormatArgs(startFormat, startArgs, endFormat, endArgs, args, kwargs): - """ - Helper function to create a format and args to use for logging. + """Creates a format and args to use for logging. + This avoids needlessly interpolating variables. """ debugArgs = startArgs[:] for a in args: debugArgs.append(ellipsize(a)) - for items in kwargs.items(): + for items in list(kwargs.items()): debugArgs.extend(items) debugArgs.extend(endArgs) format = startFormat \ @@ -498,22 +496,22 @@ def getFormatArgs(startFormat, startArgs, endFormat, endArgs, args, kwargs): def doLog(level, object, category, format, args, where=-1, filePath=None, line=None): - """ - @param where: what to log file and line number for; - -1 for one frame above log.py; -2 and down for higher up; - a function for a (future) code object - @type where: int or callable - @param filePath: file to show the message as coming from, if caller - knows best - @type filePath: str - @param line: line to show the message as coming from, if caller - knows best - @type line: int + """Logs something. - @return: dict of calculated variables, if they needed calculating. - currently contains file and line; this prevents us from - doing this work in the caller when it isn't needed because - of the debug level + Args: + where (int or function): What to log file and line number for; + -1 for one frame above log.py; -2 and down for higher up; + a function for a (future) code object. + filePath (Optional[str]): The file to show the message as coming from, + if caller knows best. + line (Optional[int]): The line to show the message as coming from, + if caller knows best. + + Returns: + A dict of calculated variables, if they needed calculating. + currently contains file and line; this prevents us from + doing this work in the caller when it isn't needed because + of the debug level. """ ret = {} @@ -521,105 +519,80 @@ def doLog(level, object, category, format, args, where=-1, filePath=None, line=N message = format % args else: message = format + funcname = None - # first all the unlimited ones - if _log_handlers: + if level > getCategoryLevel(category): + handlers = _log_handlers + else: + handlers = _log_handlers + _log_handlers_limited + + if handlers: if filePath is None and line is None: (filePath, line, funcname) = getFileLine(where=where) ret['filePath'] = filePath ret['line'] = line if funcname: message = "\033[00m\033[32;01m%s:\033[00m %s" % (funcname, message) - for handler in _log_handlers: + for handler in handlers: try: - handler(level, object, category, file, line, message) - except TypeError, e: + handler(level, object, category, filePath, line, message) + except TypeError as e: raise SystemError("handler %r raised a TypeError: %s" % ( handler, getExceptionMessage(e))) - if level > getCategoryLevel(category): - return ret - - if _log_handlers_limited: - if filePath is None and line is None: - (filePath, line, funcname) = getFileLine(where=where) - ret['filePath'] = filePath - ret['line'] = line - if funcname: - message = "\033[00m\033[32;01m%s:\033[00m %s" % (funcname, message) - for handler in _log_handlers_limited: - # set this a second time, just in case there weren't unlimited - # loggers there before - try: - handler(level, object, category, filePath, line, message) - except TypeError: - raise SystemError("handler %r raised a TypeError" % handler) - - return ret + return ret def errorObject(object, cat, format, *args): - """ - Log a fatal error message in the given category. - This will also raise a L{SystemExit}. + """Logs a fatal error message in the specified category. + + This will also raise a `SystemExit`. """ doLog(ERROR, object, cat, format, args) - # we do the import here because having it globally causes weird import - # errors if our gstreactor also imports .log, which brings in errors - # and pb stuff - if args: - raise SystemExit(format % args) - else: - raise SystemExit(format) - def warningObject(object, cat, format, *args): - """ - Log a warning message in the given category. + """Logs a warning message in the specified category. + This is used for non-fatal problems. """ doLog(WARN, object, cat, format, args) def fixmeObject(object, cat, format, *args): - """ - Log a fixme message in the given category. - This is used for not implemented codepaths or known issues in the code + """Logs a fixme message in the specified category. + + This is used for not implemented codepaths or known issues in the code. """ doLog(FIXME, object, cat, format, args) def infoObject(object, cat, format, *args): - """ - Log an informational message in the given category. - """ + """Logs an informational message in the specified category.""" doLog(INFO, object, cat, format, args) def debugObject(object, cat, format, *args): - """ - Log a debug message in the given category. - """ + """Logs a debug message in the specified category.""" doLog(DEBUG, object, cat, format, args) def logObject(object, cat, format, *args): - """ - Log a log message. Used for debugging recurring events. + """Logs a log message. + + Used for debugging recurring events. """ doLog(LOG, object, cat, format, args) def safeprintf(file, format, *args): - """Write to a file object, ignoring errors. - """ + """Writes to a file object, ignoring errors.""" try: if args: file.write(format % args) else: file.write(format) - except IOError, e: + except IOError as e: if e.errno == errno.EPIPE: # if our output is closed, exit; e.g. when logging over an # ssh connection and the ssh connection is closed @@ -627,17 +600,19 @@ def safeprintf(file, format, *args): # otherwise ignore it, there's nothing you can do -def stderrHandler(level, object, category, file, line, message): - """ - A log handler that writes to stderr. +def printHandler(level, object, category, file, line, message): + """Writes to stderr. + The output will be different depending the value of "_enableCrackOutput"; in Pitivi's case, that is True when the GST_DEBUG env var is defined. - @type level: string - @type object: string (or None) - @type category: string - @type message: string + Args: + level (str): + object (str): Can be None. + category (str): + message (str): """ + global _outfile # Make the file path more compact for readability file = os.path.relpath(file) @@ -646,9 +621,9 @@ def stderrHandler(level, object, category, file, line, message): # If GST_DEBUG is not set, we can assume only PITIVI_DEBUG is set, so don't # show a bazillion of debug details that are not relevant to Pitivi. if not _enableCrackOutput: - safeprintf(sys.stderr, '%s %-8s %-17s %-2s %s %s\n', + safeprintf(_outfile, '%s %-8s %-17s %-2s %s %s\n', getFormattedLevelName(level), time.strftime("%H:%M:%S"), - category, "", message, where) + category, object, message, where) else: o = "" if object: @@ -656,49 +631,55 @@ def stderrHandler(level, object, category, file, line, message): # level pid object cat time # 5 + 1 + 7 + 1 + 32 + 1 + 17 + 1 + 15 == 80 safeprintf( - sys.stderr, '%s [%5d] [0x%12x] %-32s %-17s %-15s %-4s %s %s\n', - getFormattedLevelName(level), os.getpid(), thread.get_ident(), + _outfile, '%s [%5d] [0x%12x] %-32s %-17s %-15s %-4s %s %s\n', + getFormattedLevelName(level), os.getpid(), + threading.current_thread().ident, o[:32], category, time.strftime("%b %d %H:%M:%S"), "", message, where) - sys.stderr.flush() + _outfile.flush() -def _colored_formatter(level): - format = '%-5s' - - t = TerminalController() - return ''.join((t.BOLD, getattr(t, COLORS[level]), - format % (_LEVEL_NAMES[level - 1], ), t.NORMAL)) - - -def _formatter(level): +def logLevelName(level): format = '%-5s' return format % (_LEVEL_NAMES[level - 1], ) -def _preformatLevels(noColorEnvVarName): - if (noColorEnvVarName is not None - and (noColorEnvVarName not in os.environ - or not os.environ[noColorEnvVarName])): - formatter = _colored_formatter - else: - formatter = _formatter - +def _preformatLevels(enableColorOutput): + terminal_controller = TerminalController() for level in ERROR, WARN, FIXME, INFO, DEBUG, LOG: - _FORMATTED_LEVELS.append(formatter(level)) + if enableColorOutput: + if type(terminal_controller.BOLD) == bytes: + formatter = ''.join( + (terminal_controller.BOLD.decode(), + getattr(terminal_controller, COLORS[level]).decode(), + logLevelName(level), + terminal_controller.NORMAL.decode())) + else: + formatter = ''.join( + (terminal_controller.BOLD, + getattr(terminal_controller, COLORS[level]), + logLevelName(level), + terminal_controller.NORMAL)) + else: + formatter = logLevelName(level) + _FORMATTED_LEVELS.append(formatter) # "public" useful API # setup functions -def init(envVarName, enableColorOutput=False, enableCrackOutput=True): - """ - Initialize the logging system and parse the environment variable - of the given name. - Needs to be called before starting the actual application. +def init(envVarName, enableColorOutput=True, enableCrackOutput=True): + """Initializes the logging system. + + Needs to be called before using the log methods. + + Args: + envVarName (str): The name of the environment variable with additional + settings. """ global _initialized + global _outfile global _enableCrackOutput _enableCrackOutput = enableCrackOutput @@ -708,21 +689,29 @@ def init(envVarName, enableColorOutput=False, enableCrackOutput=True): global _ENV_VAR_NAME _ENV_VAR_NAME = envVarName - if enableColorOutput: - _preformatLevels(envVarName + "_NO_COLOR") - else: - _preformatLevels(None) + _preformatLevels(enableColorOutput) if envVarName in os.environ: # install a log handler that uses the value of the environment var setDebug(os.environ[envVarName]) - addLimitedLogHandler(stderrHandler) + filenameEnvVarName = envVarName + "_FILE" + + if filenameEnvVarName in os.environ: + # install a log handler that uses the value of the environment var + _outfile = open(os.environ[filenameEnvVarName], "w+") + else: + _outfile = sys.stderr + + addLimitedLogHandler(printHandler) _initialized = True def setDebug(string): - """Set the DEBUG string. This controls the log output.""" + """Sets the DEBUG string. + + This controls the log output. + """ global _DEBUG global _ENV_VAR_NAME global _categories @@ -736,30 +725,26 @@ def setDebug(string): def getDebug(): - """ - Returns the currently active DEBUG string. - @rtype: str - """ + """Returns the currently active DEBUG string.""" global _DEBUG return _DEBUG def setPackageScrubList(*packages): - """ - Set the package names to scrub from filenames. + """Sets the package names to scrub from filenames. + Filenames from these paths in log messages will be scrubbed to their relative file path instead of the full absolute path. - @type packages: list of str + Args: + *packages (List[str]): The packages names to scrub. """ global _PACKAGE_SCRUB_LIST _PACKAGE_SCRUB_LIST = packages def reset(): - """ - Resets the logging system, removing all log handlers. - """ + """Resets the logging system, removing all log handlers.""" global _log_handlers, _log_handlers_limited, _initialized _log_handlers = [] @@ -768,19 +753,22 @@ def reset(): def addLogHandler(func): - """ - Add a custom log handler. + """Adds a custom log handler. - @param func: a function object with prototype (level, object, category, - message) where level is either ERROR, WARN, INFO, DEBUG, or - LOG, and the rest of the arguments are strings or None. Use - getLevelName(level) to get a printable name for the log level. - @type func: a callable function + The log handler receives all the log messages. - @raises TypeError: if func is not a callable + Args: + func (function): A function object with prototype + (level, object, category, message) where level is either + ERROR, WARN, INFO, DEBUG, or LOG, and the rest of the arguments are + strings or None. Use getLevelName(level) to get a printable name + for the log level. + + Raises: + TypeError: When func is not a callable. """ - if not callable(func): + if not isinstance(func, collections.Callable): raise TypeError("func must be callable") if func not in _log_handlers: @@ -788,18 +776,21 @@ def addLogHandler(func): def addLimitedLogHandler(func): - """ - Add a custom log handler. + """Adds a custom limited log handler. - @param func: a function object with prototype (level, object, category, - message) where level is either ERROR, WARN, INFO, DEBUG, or - LOG, and the rest of the arguments are strings or None. Use - getLevelName(level) to get a printable name for the log level. - @type func: a callable function + The log handler receives only the messages passing the filter. - @raises TypeError: TypeError if func is not a callable + Args: + func (function): A function object with prototype + (level, object, category, message) where level is either + ERROR, WARN, INFO, DEBUG, or LOG, and the rest of the arguments are + strings or None. Use getLevelName(level) to get a printable name + for the log level. + + Raises: + TypeError: When func is not a callable. """ - if not callable(func): + if not isinstance(func, collections.Callable): raise TypeError("func must be callable") if func not in _log_handlers_limited: @@ -807,31 +798,19 @@ def addLimitedLogHandler(func): def removeLogHandler(func): - """ - Remove a registered log handler. + """Removes a registered log handler. - @param func: a function object with prototype (level, object, category, - message) where level is either ERROR, WARN, INFO, DEBUG, or - LOG, and the rest of the arguments are strings or None. Use - getLevelName(level) to get a printable name for the log level. - @type func: a callable function - - @raises ValueError: if func is not registered + Raises: + ValueError: When func is not registered. """ _log_handlers.remove(func) def removeLimitedLogHandler(func): - """ - Remove a registered limited log handler. + """Removes a registered limited log handler. - @param func: a function object with prototype (level, object, category, - message) where level is either ERROR, WARN, INFO, DEBUG, or - LOG, and the rest of the arguments are strings or None. Use - getLevelName(level) to get a printable name for the log level. - @type func: a callable function - - @raises ValueError: if func is not registered + Raises: + ValueError: When func is not registered. """ _log_handlers_limited.remove(func) @@ -865,8 +844,9 @@ def log(cat, format, *args): def getExceptionMessage(exception, frame=-1, filename=None): - """ - Return a short message based on an exception, useful for debugging. + """Returns a short message based on an exception. + + Useful for debugging. Tries to find where the exception was triggered. """ stack = traceback.extract_tb(sys.exc_info()[2]) @@ -886,16 +866,13 @@ def getExceptionMessage(exception, frame=-1, filename=None): def reopenOutputFiles(): - """ - Reopens the stdout and stderr output files, as set by - L{outputToFiles}. - """ + """Reopens the stdout and stderr output files, as set by `outputToFiles`.""" if not _stdout and not _stderr: debug('log', 'told to reopen log files, but log files not set') return def reopen(name, fileno, *args): - oldmask = os.umask(0026) + oldmask = os.umask(0o026) try: f = open(name, 'a+', *args) finally: @@ -912,8 +889,7 @@ def reopenOutputFiles(): def outputToFiles(stdout=None, stderr=None): - """ - Redirect stdout and stderr to named files. + """Redirects stdout and stderr to the specified files. Records the file names so that a future call to reopenOutputFiles() can open the same files. Installs a SIGHUP handler that will reopen @@ -935,7 +911,7 @@ def outputToFiles(stdout=None, stderr=None): _old_hup_handler(signum, frame) debug('log', 'installing SIGHUP handler') - import signal + from . import signal handler = signal.signal(signal.SIGHUP, sighup) if handler == signal.SIG_DFL or handler == signal.SIG_IGN: _old_hup_handler = None @@ -947,94 +923,93 @@ def outputToFiles(stdout=None, stderr=None): class BaseLoggable(object): + """Base class for objects that want to be able to log messages. - """ - Base class for objects that want to be able to log messages with - different level of severity. The levels are, in order from least + The levels of severity for the messages are, in order from least to most: log, debug, info, warning, error. - @cvar logCategory: Implementors can provide a category to log their - messages under. + Attributes: + logCategory (str): The category under which the messages will be filed. + Can be used to set a display filter. """ - def writeMarker(self, marker, level): - """ - Sets a marker that written to the logs. Setting this - marker to multiple elements at a time helps debugging. - @param marker: A string write to the log. - @type marker: str - @param level: The log level. It can be log.WARN, log.INFO, - log.DEBUG, log.ERROR or log.LOG. - @type level: int - """ - logHandlers = {WARN: self.warning, - INFO: self.info, - DEBUG: self.debug, - ERROR: self.error, - LOG: self.log} - logHandler = logHandlers.get(level) - if logHandler: - logHandler('%s', marker) - def error(self, *args): - """Log an error. By default this will also raise an exception.""" + """Logs an error. + + By default this will also raise an exception. + """ if _canShortcutLogging(self.logCategory, ERROR): return errorObject(self.logObjectName(), self.logCategory, *self.logFunction(*args)) def warning(self, *args): - """Log a warning. Used for non-fatal problems.""" + """Logs a warning. + + Used for non-fatal problems. + """ if _canShortcutLogging(self.logCategory, WARN): return warningObject( self.logObjectName(), self.logCategory, *self.logFunction(*args)) def fixme(self, *args): - """Log a fixme. Used for FIXMEs .""" + """Logs a fixme. + + Used for FIXMEs. + """ if _canShortcutLogging(self.logCategory, FIXME): return fixmeObject(self.logObjectName(), self.logCategory, *self.logFunction(*args)) def info(self, *args): - """Log an informational message. Used for normal operation.""" + """Logs an informational message. + + Used for normal operation. + """ if _canShortcutLogging(self.logCategory, INFO): return infoObject(self.logObjectName(), self.logCategory, *self.logFunction(*args)) def debug(self, *args): - """Log a debug message. Used for debugging.""" + """Logs a debug message. + + Used for debugging. + """ if _canShortcutLogging(self.logCategory, DEBUG): return debugObject(self.logObjectName(), self.logCategory, *self.logFunction(*args)) def log(self, *args): - """Log a log message. Used for debugging recurring events.""" + """Logs a log message. + + Used for debugging recurring events. + """ if _canShortcutLogging(self.logCategory, LOG): return logObject(self.logObjectName(), self.logCategory, *self.logFunction(*args)) def doLog(self, level, where, format, *args, **kwargs): - """ - Log a message at the given level, with the possibility of going + """Logs a message at the specified level, with the possibility of going higher up in the stack. - @param level: log level - @type level: int - @param where: how many frames to go back from the last log frame; - or a function (to log for a future call) - @type where: int (negative), or function + Args: + level (int): The log level. + where (int or function): How many frames to go back from + the last log frame, must be negative; or a function + (to log for a future call). + format (str): The string template for the message. + *args: The arguments used when converting the `format` + string template to the message. + **kwargs: The pre-calculated values from a previous doLog call. - @param kwargs: a dict of pre-calculated values from a previous - doLog call - - @return: a dict of calculated variables, to be reused in a - call to doLog that should show the same location - @rtype: dict + Returns: + dict: The calculated variables, to be reused in a + call to doLog that should show the same location. """ if _canShortcutLogging(self.logCategory, level): return {} @@ -1042,29 +1017,15 @@ class BaseLoggable(object): return doLog(level, self.logObjectName(), self.logCategory, format, args, where=where, **kwargs) - def warningFailure(self, failure, swallow=True): - """ - Log a warning about a Twisted Failure. Useful as an errback handler: - d.addErrback(self.warningFailure) - - @param swallow: whether to swallow the failure or not - @type swallow: bool - """ - if _canShortcutLogging(self.logCategory, WARN): - if swallow: - return - return failure - warningObject(self.logObjectName(), self.logCategory, - *self.logFunction(getFailureMessage(failure))) - if not swallow: - return failure - def logFunction(self, *args): - """Overridable log function. Default just returns passed message.""" + """Processes the arguments applied to the message template. + + Default just returns the arguments unchanged. + """ return args def logObjectName(self): - """Overridable object name function.""" + """Gets the name of this object.""" # cheat pychecker for name in ['logName', 'name']: if hasattr(self, name): @@ -1075,146 +1036,6 @@ class BaseLoggable(object): def handleException(self, exc): self.warning(getExceptionMessage(exc)) -# Twisted helper stuff - -# private stuff -_initializedTwisted = False - -# make a singleton -__theTwistedLogObserver = None - - -def _getTheTwistedLogObserver(): - # used internally and in test - global __theTwistedLogObserver - - if not __theTwistedLogObserver: - __theTwistedLogObserver = TwistedLogObserver() - - return __theTwistedLogObserver - - -# public helper methods - - -def getFailureMessage(failure): - """ - Return a short message based on L{twisted.python.failure.Failure}. - Tries to find where the exception was triggered. - """ - exc = str(failure.type) - msg = failure.getErrorMessage() - if len(failure.frames) == 0: - return "failure %(exc)s: %(msg)s" % locals() - - (func, filename, line, some, other) = failure.frames[-1] - filename = scrubFilename(filename) - return "failure %(exc)s at %(filename)s:%(line)s: %(func)s(): %(msg)s" % locals() - - -def warningFailure(failure, swallow=True): - """ - Log a warning about a Failure. Useful as an errback handler: - d.addErrback(warningFailure) - - @param swallow: whether to swallow the failure or not - @type swallow: bool - """ - warning('', getFailureMessage(failure)) - if not swallow: - return failure - - -def logTwisted(): - """ - Integrate twisted's logger with our logger. - - This is done in a separate method because calling this imports and sets - up a reactor. Since we want basic logging working before choosing a - reactor, we need to separate these. - """ - global _initializedTwisted - - if _initializedTwisted: - return - - debug('log', 'Integrating twisted logger') - - # integrate twisted's logging with us - from twisted.python import log as tlog - - # this call imports the reactor - # that is why we do this in a separate method - from twisted.spread import pb - - # we don't want logs for pb.Error types since they - # are specifically raised to be handled on the other side - observer = _getTheTwistedLogObserver() - observer.ignoreErrors([pb.Error, ]) - tlog.startLoggingWithObserver(observer.emit, False) - - _initializedTwisted = True - - -# we need an object as the observer because startLoggingWithObserver -# expects a bound method - - -class TwistedLogObserver(BaseLoggable): - - """ - Twisted log observer that integrates with our logging. - """ - logCategory = "logobserver" - - def __init__(self): - self._ignoreErrors = [] # Failure types - - def emit(self, eventDict): - method = log # by default, lowest level - edm = eventDict['message'] - if not edm: - if eventDict['isError'] and 'failure' in eventDict: - f = eventDict['failure'] - for failureType in self._ignoreErrors: - r = f.check(failureType) - if r: - self.debug("Failure of type %r, ignoring", failureType) - return - - self.log("Failure %r" % f) - - method = debug # tracebacks from errors at debug level - msg = "A twisted traceback occurred." - if getCategoryLevel("twisted") < WARN: - msg += " Run with debug level >= 2 to see the traceback." - # and an additional warning - warning('twisted', msg) - text = f.getTraceback() - safeprintf(sys.stderr, "\nTwisted traceback:\n") - safeprintf(sys.stderr, text + '\n') - elif 'format' in eventDict: - text = eventDict['format'] % eventDict - else: - # we don't know how to log this - return - else: - text = ' '.join(map(str, edm)) - - fmtDict = {'system': eventDict['system'], - 'text': text.replace("\n", "\n\t")} - msgStr = " [%(system)s] %(text)s\n" % fmtDict - # because msgstr can contain %, as in a backtrace, make sure we - # don't try to splice it - method('twisted', msgStr) - - def ignoreErrors(self, *types): - for failureType in types: - self._ignoreErrors.append(failureType) - - def clearIgnores(self): - self._ignoreErrors = [] - class Loggable(BaseLoggable): diff --git a/validate/launcher/main.py b/validate/launcher/main.py index cde660450a..0a8d5f0a10 100644 --- a/validate/launcher/main.py +++ b/validate/launcher/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2014,Thibault Saunier # @@ -18,20 +18,20 @@ # Boston, MA 02110-1301, USA. import os import sys -import utils -import urlparse -import loggable +from . import utils +import urllib.parse +from . import loggable import argparse import tempfile -import reporters +from . import reporters import subprocess -from loggable import Loggable -from httpserver import HTTPServer -from vfb_server import get_virual_frame_buffer_server -from baseclasses import _TestsLauncher, ScenarioManager -from utils import printc, path2url, DEFAULT_MAIN_DIR, launch_command, Colors, Protocols, which +from .loggable import Loggable +from .httpserver import HTTPServer +from .vfb_server import get_virual_frame_buffer_server +from .baseclasses import _TestsLauncher, ScenarioManager +from .utils import printc, path2url, DEFAULT_MAIN_DIR, launch_command, Colors, Protocols, which LESS = "less" @@ -264,7 +264,7 @@ class LauncherConfig(Loggable): % self.redirect_logs, Colors.FAIL, True) return False - if urlparse.urlparse(self.dest).scheme == "": + if urllib.parse.urlparse(self.dest).scheme == "": self.dest = path2url(self.dest) if self.no_color: @@ -506,7 +506,7 @@ Note that all testsuite should be inside python modules, so the directory should tests_launcher.add_options(parser) if _help_message == HELP and which(LESS): - tmpf = tempfile.NamedTemporaryFile() + tmpf = tempfile.NamedTemporaryFile(mode='r+') parser.print_help(file=tmpf) exit(os.system("%s %s" % (LESS, tmpf.name))) @@ -560,17 +560,18 @@ Note that all testsuite should be inside python modules, so the directory should # Also happened here: https://cgit.freedesktop.org/gstreamer/gst-plugins-good/commit/tests/check/Makefile.am?id=8e2c1d1de56bddbff22170f8b17473882e0e63f9 os.environ['GSETTINGS_BACKEND'] = "memory" - e = None + exception = None try: tests_launcher.run_tests() except Exception as e: + exception = e pass finally: tests_launcher.final_report() tests_launcher.clean_tests() httpsrv.stop() vfb_server.stop() - if e is not None: - raise + if exception is not None: + raise exception return 0 diff --git a/validate/launcher/reporters.py b/validate/launcher/reporters.py index 9ee58aadf8..486276b88f 100644 --- a/validate/launcher/reporters.py +++ b/validate/launcher/reporters.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2013,Thibault Saunier # @@ -25,11 +25,11 @@ import time import codecs import datetime import tempfile -from loggable import Loggable +from .loggable import Loggable from xml.sax import saxutils -from utils import Result, printc, Colors +from .utils import Result, printc, Colors -UNICODE_STRINGS = (type(unicode()) == type(str())) # noqa +UNICODE_STRINGS = (type(str()) == type(str())) # noqa class UnknownResult(Exception): @@ -91,14 +91,14 @@ class Reporter(Loggable): self.add_results(test) def final_report(self): - print "\n" + print("\n") printc("Final Report:", title=True) for test in sorted(self.results, key=lambda test: test.result): printc(test) if test.result != Result.PASSED: - print "\n" + print("\n") - print "\n" + print("\n") lenstat = (len("Statistics") + 1) printc("Statistics:\n%s" % (lenstat * "-"), Colors.OKBLUE) printc("\n%sTotal time spent: %s seconds\n" % @@ -157,7 +157,7 @@ class XunitReporter(Reporter): def _quoteattr(self, attr): """Escape an XML attribute. Value can be unicode.""" attr = xml_safe(attr) - if isinstance(attr, unicode) and not UNICODE_STRINGS: + if isinstance(attr, str) and not UNICODE_STRINGS: attr = attr.encode(self.encoding) return saxutils.quoteattr(attr) @@ -175,10 +175,10 @@ class XunitReporter(Reporter): self.stats['total'] = (self.stats['timeout'] + self.stats['failures'] + self.stats['passed'] + self.stats['skipped']) - xml_file.write(u'' - u'' % self.stats) + xml_file.write('' + '' % self.stats) tmp_xml_file = codecs.open(self.tmp_xml_file.name, 'r', self.encoding, 'replace') @@ -186,7 +186,7 @@ class XunitReporter(Reporter): for l in tmp_xml_file: xml_file.write(l) - xml_file.write(u'') + xml_file.write('') xml_file.close() tmp_xml_file.close() os.remove(self.tmp_xml_file.name) diff --git a/validate/launcher/utils.py b/validate/launcher/utils.py index 03b3a3f6f8..6d27fc011b 100644 --- a/validate/launcher/utils.py +++ b/validate/launcher/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2013,Thibault Saunier # @@ -22,14 +22,14 @@ import config import os import re import sys -import urllib -import urlparse +import urllib.request, urllib.parse, urllib.error +import urllib.parse import subprocess from operator import itemgetter -GST_SECOND = long(1000000000) +GST_SECOND = int(1000000000) DEFAULT_TIMEOUT = 30 DEFAULT_MAIN_DIR = os.path.join(os.path.expanduser("~"), "gst-validate") DEFAULT_GST_QA_ASSETS = os.path.join(DEFAULT_MAIN_DIR, "gst-integration-testsuites") @@ -86,7 +86,7 @@ def mkdir(directory): def which(name, extra_path=None): - exts = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep)) + exts = [_f for _f in os.environ.get('PATHEXT', '').split(os.pathsep) if _f] path = os.environ.get('PATH', '') if extra_path: path = extra_path + os.pathsep + path @@ -142,20 +142,20 @@ def launch_command(command, color=None, fails=False): def path2url(path): - return urlparse.urljoin('file:', urllib.pathname2url(path)) + return urllib.parse.urljoin('file:', urllib.request.pathname2url(path)) def url2path(url): - path = urlparse.urlparse(url).path + path = urllib.parse.urlparse(url).path if "win32" in sys.platform: if path[0] == '/': return path[1:] # We need to remove the first '/' on windows - path = urllib.unquote(path) + path = urllib.parse.unquote(path) return path def isuri(string): - url = urlparse.urlparse(string) + url = urllib.parse.urlparse(string) if url.scheme != "" and url.scheme != "": return True @@ -169,7 +169,7 @@ def touch(fname, times=None): def get_subclasses(klass, env): subclasses = [] - for symb in env.iteritems(): + for symb in env.items(): try: if issubclass(symb[1], klass) and not symb[1] is klass: subclasses.append(symb[1]) @@ -216,15 +216,15 @@ def get_data_file(subdir, name): def gsttime_from_tuple(stime): - return long((int(stime[0]) * 3600 + int(stime[1]) * 60 + int(stime[2])) * GST_SECOND + int(stime[3])) + return int((int(stime[0]) * 3600 + int(stime[1]) * 60 + int(stime[2])) * GST_SECOND + int(stime[3])) timeregex = re.compile(r'(?P<_0>.+):(?P<_1>.+):(?P<_2>.+)\.(?P<_3>.+)') def parse_gsttimeargs(time): - stime = map(itemgetter(1), sorted( - timeregex.match(time).groupdict().items())) - return long((int(stime[0]) * 3600 + int(stime[1]) * 60 + int(stime[2])) * GST_SECOND + int(stime[3])) + stime = list(map(itemgetter(1), sorted( + timeregex.match(time).groupdict().items()))) + return int((int(stime[0]) * 3600 + int(stime[1]) * 60 + int(stime[2])) * GST_SECOND + int(stime[3])) def get_duration(media_file): @@ -232,7 +232,7 @@ def get_duration(media_file): duration = 0 res = '' try: - res = subprocess.check_output([DISCOVERER_COMMAND, media_file]) + res = subprocess.check_output([DISCOVERER_COMMAND, media_file]).decode() except subprocess.CalledProcessError: # gst-media-check returns !0 if seeking is not possible, we do not care # in that case. diff --git a/validate/launcher/vfb_server.py b/validate/launcher/vfb_server.py index 3b119eae32..fd9433c886 100644 --- a/validate/launcher/vfb_server.py +++ b/validate/launcher/vfb_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2015,Thibault Saunier # @@ -19,7 +19,7 @@ import os import time -import loggable +from . import loggable import subprocess @@ -55,7 +55,7 @@ class Xvfb(VirtualFrameBufferServer): os.environ["DISPLAY"] = self.display_id subprocess.check_output(["xset", "q"], stderr=self._logsfile) - print("DISPLAY set to %s" % self.display_id) + print(("DISPLAY set to %s" % self.display_id)) return True except subprocess.CalledProcessError: pass diff --git a/validate/tools/gst-validate-analyze b/validate/tools/gst-validate-analyze index 3c6d1148c1..8f4bcf8985 100755 --- a/validate/tools/gst-validate-analyze +++ b/validate/tools/gst-validate-analyze @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2015, Edward Hervey # diff --git a/validate/tools/gst-validate-launcher.in b/validate/tools/gst-validate-launcher.in index 1787f678d0..40013bcfda 100755 --- a/validate/tools/gst-validate-launcher.in +++ b/validate/tools/gst-validate-launcher.in @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # # Copyright (c) 2014,Thibault Saunier # @@ -31,7 +31,7 @@ def _get_git_first_hash(path): cdir = os.path.abspath(os.curdir) try: os.chdir(path) - res = subprocess.check_output(['git', 'rev-list', '--max-parents=0', 'HEAD']).rstrip('\n') + res = subprocess.check_output(['git', 'rev-list', '--max-parents=0', 'HEAD']).decode().rstrip('\n') except (subprocess.CalledProcessError, OSError): res = '' finally: @@ -48,7 +48,7 @@ def _in_devel(): def _add_gst_launcher_path(): if _in_devel(): - print "Running with development path" + print("Running with development path") dir_ = os.path.dirname(os.path.abspath(__file__)) root = os.path.split(dir_)[0] elif __file__.startswith(BUILDDIR):