tracer/structure: add more tests and a benchmark

This commit is contained in:
Stefan Sauer 2016-12-16 14:07:45 +01:00
parent 2150443565
commit 91ac5089cf
3 changed files with 154 additions and 75 deletions

View file

@ -1,5 +1,14 @@
import logging
import re import re
logger = logging.getLogger('structure')
UNESCAPE = re.compile(r'(?<!\\)\\(.)')
INT_TYPES = "".join(
("int", "uint", "int8", "uint8", "int16", "uint16", "int32", "uint32", "int64", "uint64")
)
class Structure(object): class Structure(object):
'''Gst Structure parser.''' '''Gst Structure parser.'''
@ -9,84 +18,77 @@ class Structure(object):
self.types = {} self.types = {}
self.values = {} self.values = {}
self.pos = 0 self.pos = 0
self.valid = False
try:
self._parse(self.text) self._parse(self.text)
self.valid = True
except ValueError:
pass
def __repr__(self): def __repr__(self):
return self.text return self.text
def _find_eos(self, s):
# find next '"' without preceeding '\'
l = 0
#logger.debug("find_eos: '%s'", s)
while 1: # faster than regexp for '[^\\]\"'
p = s.index('"')
l += p + 1
if s[p - 1] != '\\':
#logger.debug("... ok : '%s'", s[p:])
return l
s = s[(p + 1):]
#logger.debug("... : '%s'", s)
return -1
def _parse(self, s): def _parse(self, s):
scan = True scan = True
#logger.debug("===: '%s'", s)
# parse id # parse id
p = s.find(',') p = s.find(',')
if p == -1: if p == -1:
p = s.index(';') p = s.index(';')
scan = False scan = False
self.name = s[:p] self.name = s[:p]
s = s[(p + 2):] # skip 'name, '
self.pos += p + 2
# parse fields # parse fields
while scan: while scan:
s = s[(p + 2):] # skip 'name, ' / 'value, '
self.pos += p + 2
#logger.debug("...: '%s'", s)
p = s.index('=') p = s.index('=')
k = s[:p] k = s[:p]
s = s[(p + 1):] # skip 'key=' if not s[p + 1] == '(':
self.pos += p + 1
p = s.index('(')
s = s[(p + 1):] # skip '('
self.pos += p + 1 self.pos += p + 1
raise ValueError
s = s[(p + 2):] # skip 'key=('
self.pos += p + 2
p = s.index(')') p = s.index(')')
t = s[:p] t = s[:p]
s = s[(p + 1):] # skip 'type)' s = s[(p + 1):] # skip 'type)'
self.pos += p + 1 self.pos += p + 1
if t == 'structure':
p = s.index('"') if s[0] == '"':
s = s[(p + 1):] # skip '"' s = s[1:] # skip '"'
self.pos += p + 1 self.pos += 1
# find next '"' without preceeding '\' p = self._find_eos(s)
sub = s if p == -1:
sublen = 0
while True:
p = sub.index('"')
sublen += p + 1
if sub[p - 1] != '\\':
sub = None
break;
sub = sub[(p + 1):]
if not sub:
sub = s[:(sublen - 1)]
# unescape \., but not \\. (using a backref)
# FIXME: try to combine
# also:
# unescape = re.compile('search')
# unescape.sub('replacement', sub)
sub = re.sub(r'\\\\', r'\\', sub)
sub = re.sub(r'(?<!\\)\\(.)', r'\1', sub)
sub = re.sub(r'(?<!\\)\\(.)', r'\1', sub)
# recurse
v = Structure(sub)
if s[sublen] == ';':
scan = False
s = s[(sublen + 2):]
self.pos += sublen + 2
else:
raise ValueError raise ValueError
v = s[:(p - 1)]
if s[p] == ';':
scan = False
# unescape \., but not \\. (using a backref)
# need a reverse for re.escape()
v = v.replace('\\\\', '\\')
v = UNESCAPE.sub(r'\1', v)
else: else:
p = s.find(',') p = s.find(',')
if p == -1: if p == -1:
p = s.index(';') p = s.index(';')
scan = False scan = False
v= s[:p] v= s[:p]
s = s[(p + 2):] # skip "value, "
self.pos += p + 2 if t == 'structure':
if t == 'string' and v[0] == '"': v = Structure(v)
elif t == 'string' and v[0] == '"':
v = v[1:-1] v = v[1:-1]
elif t in ['int', 'uint', 'int8', 'uint8', 'int16', 'uint16', 'int32', 'uint32', 'int64', 'uint64' ]: elif t in INT_TYPES:
v = int(v) v = int(v)
self.types[k] = t self.types[k] = t
self.values[k] = v self.values[k] = v
self.pos += p + 1
self.valid = True

View file

@ -0,0 +1,67 @@
import timeit
from structure import Structure
from gi.repository import Gst
Gst.init(None)
PLAIN_STRUCTURE = r'thread-rusage, thread-id=(guint64)37268592, ts=(guint64)79416000, average-cpuload=(uint)1000, current-cpuload=(uint)1000, time=(guint64)79418045;'
NESTED_STRUCTURE = r'latency.class, src=(structure)"scope\,\ type\=\(type\)gchararray\,\ related-to\=\(GstTracerValueScope\)GST_TRACER_VALUE_SCOPE_PAD\;", sink=(structure)"scope\,\ type\=\(type\)gchararray\,\ related-to\=\(GstTracerValueScope\)GST_TRACER_VALUE_SCOPE_PAD\;", time=(structure)"value\,\ type\=\(type\)guint64\,\ description\=\(string\)\"time\\\ it\\\ took\\\ for\\\ the\\\ buffer\\\ to\\\ go\\\ from\\\ src\\\ to\\\ sink\\\ ns\"\,\ flags\=\(GstTracerValueFlags\)GST_TRACER_VALUE_FLAGS_AGGREGATED\,\ min\=\(guint64\)0\,\ max\=\(guint64\)18446744073709551615\;";'
NAT_STRUCTURE = Structure(PLAIN_STRUCTURE)
GI_STRUCTURE = Gst.Structure.from_string(PLAIN_STRUCTURE)[0]
# native python impl
def nat_parse_plain():
s = Structure(PLAIN_STRUCTURE)
def nat_parse_nested():
s = Structure(NESTED_STRUCTURE)
def nat_get_name():
return NAT_STRUCTURE.name
def nat_get_value():
return NAT_STRUCTURE.values['thread-id']
# gstreamer impl via gi
def gi_parse_plain():
s = Gst.Structure.from_string(PLAIN_STRUCTURE)[0]
def gi_parse_nested():
s = Gst.Structure.from_string(NESTED_STRUCTURE)[0]
def gi_get_name():
return GI_STRUCTURE.get_name()
def gi_get_value():
return GI_STRUCTURE.get_value('thread-id')
def perf(method, n, flavor):
t = timeit.timeit(method + '()', 'from __main__ import ' + method, number=n)
print("%6s: %lf s, (%lf calls/s)" % (flavor, t, (n/t)))
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--iterations', default=10000, type=int,
help='number of iterations (default: 10000)')
args = parser.parse_args()
n = args.iterations
print("parse_plain:")
t = perf('nat_parse_plain', n, 'native')
t = perf('gi_parse_plain', n, 'gi')
print("parse_nested:")
t = perf('nat_parse_nested', n, 'native')
t = perf('gi_parse_nested', n, 'gi')
print("get_name:")
t = perf('nat_get_name', n, 'native')
t = perf('gi_get_name', n, 'gi')
print("get_value:")
t = perf('nat_get_value', n, 'native')
t = perf('gi_get_value', n, 'gi')

View file

@ -1,7 +1,10 @@
import logging
import unittest import unittest
from tracer.structure import Structure from tracer.structure import Structure
logging.basicConfig(level=logging.INFO)
BAD_NAME = r'foo bar' BAD_NAME = r'foo bar'
BAD_KEY = r'foo, bar' BAD_KEY = r'foo, bar'
BAD_TYPE1 = r'foo, bar=[' BAD_TYPE1 = r'foo, bar=['
@ -14,43 +17,48 @@ MISC_TYPES_STRUCTURE = r'foo, key1=(string)"value", key2=(int)5, key3=(boolean)t
NESTED_STRUCTURE = r'foo, nested=(structure)"bar\,\ key1\=\(int\)0\,\ key2\=\(int\)5\;";' NESTED_STRUCTURE = r'foo, nested=(structure)"bar\,\ key1\=\(int\)0\,\ key2\=\(int\)5\;";'
REGRESSIONS = [
r'query, thread-id=(guint64)139839438879824, ts=(guint64)220860464, pad-ix=(uint)8, element-ix=(uint)9, peer-pad-ix=(uint)9, peer-element-ix=(uint)8, name=(string)accept-caps, structure=(structure)"GstQueryAcceptCaps\,\ caps\=\(GstCaps\)\"audio/mpeg\\\,\\\ mpegversion\\\=\\\(int\\\)4\\\,\\\ framed\\\=\\\(boolean\\\)true\\\,\\\ stream-format\\\=\\\(string\\\)raw\\\,\\\ level\\\=\\\(string\\\)2\\\,\\\ base-profile\\\=\\\(string\\\)lc\\\,\\\ profile\\\=\\\(string\\\)lc\\\,\\\ codec_data\\\=\\\(buffer\\\)1210\\\,\\\ rate\\\=\\\(int\\\)44100\\\,\\\ channels\\\=\\\(int\\\)2\"\,\ result\=\(boolean\)false\;", have-res=(boolean)0, res=(boolean)0;',
r'message, thread-id=(guint64)139838900680560, ts=(guint64)1000451258, element-ix=(uint)2, name=(string)tag, structure=(structure)"GstMessageTag\,\ taglist\=\(taglist\)\"taglist\\\,\\\ datetime\\\=\\\(datetime\\\)2009-03-05T12:57:08Z\\\,\\\ private-qt-tag\\\=\\\(sample\\\)\\\{\\\ 00000019677373740000001164617461000000010000000030:None:R3N0U2VnbWVudCwgZmxhZ3M9KEdzdFNlZ21lbnRGbGFncylHU1RfU0VHTUVOVF9GTEFHX05PTkUsIHJhdGU9KGRvdWJsZSkxLCBhcHBsaWVkLXJhdGU9KGRvdWJsZSkxLCBmb3JtYXQ9KEdzdEZvcm1hdClHU1RfRk9STUFUX1RJTUUsIGJhc2U9KGd1aW50NjQpMCwgb2Zmc2V0PShndWludDY0KTAsIHN0YXJ0PShndWludDY0KTAsIHN0b3A9KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTUsIHRpbWU9KGd1aW50NjQpMCwgcG9zaXRpb249KGd1aW50NjQpMCwgZHVyYXRpb249KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTU7AA__:YXBwbGljYXRpb24veC1nc3QtcXQtZ3NzdC10YWcsIHN0eWxlPShzdHJpbmcpaXR1bmVzOwA_\\\,\\\ 0000001e6773746400000016646174610000000100000000313335353130:None:R3N0U2VnbWVudCwgZmxhZ3M9KEdzdFNlZ21lbnRGbGFncylHU1RfU0VHTUVOVF9GTEFHX05PTkUsIHJhdGU9KGRvdWJsZSkxLCBhcHBsaWVkLXJhdGU9KGRvdWJsZSkxLCBmb3JtYXQ9KEdzdEZvcm1hdClHU1RfRk9STUFUX1RJTUUsIGJhc2U9KGd1aW50NjQpMCwgb2Zmc2V0PShndWludDY0KTAsIHN0YXJ0PShndWludDY0KTAsIHN0b3A9KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTUsIHRpbWU9KGd1aW50NjQpMCwgcG9zaXRpb249KGd1aW50NjQpMCwgZHVyYXRpb249KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTU7AA__:YXBwbGljYXRpb24veC1nc3QtcXQtZ3N0ZC10YWcsIHN0eWxlPShzdHJpbmcpaXR1bmVzOwA_\\\,\\\ 0000003867737364000000306461746100000001000000004244354241453530354d4d313239353033343539373733353435370000000000:None:R3N0U2VnbWVudCwgZmxhZ3M9KEdzdFNlZ21lbnRGbGFncylHU1RfU0VHTUVOVF9GTEFHX05PTkUsIHJhdGU9KGRvdWJsZSkxLCBhcHBsaWVkLXJhdGU9KGRvdWJsZSkxLCBmb3JtYXQ9KEdzdEZvcm1hdClHU1RfRk9STUFUX1RJTUUsIGJhc2U9KGd1aW50NjQpMCwgb2Zmc2V0PShndWludDY0KTAsIHN0YXJ0PShndWludDY0KTAsIHN0b3A9KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTUsIHRpbWU9KGd1aW50NjQpMCwgcG9zaXRpb249KGd1aW50NjQpMCwgZHVyYXRpb249KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTU7AA__:YXBwbGljYXRpb24veC1nc3QtcXQtZ3NzZC10YWcsIHN0eWxlPShzdHJpbmcpaXR1bmVzOwA_\\\,\\\ 0000009867737075000000906461746100000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:None:R3N0U2VnbWVudCwgZmxhZ3M9KEdzdFNlZ21lbnRGbGFncylHU1RfU0VHTUVOVF9GTEFHX05PTkUsIHJhdGU9KGRvdWJsZSkxLCBhcHBsaWVkLXJhdGU9KGRvdWJsZSkxLCBmb3JtYXQ9KEdzdEZvcm1hdClHU1RfRk9STUFUX1RJTUUsIGJhc2U9KGd1aW50NjQpMCwgb2Zmc2V0PShndWludDY0KTAsIHN0YXJ0PShndWludDY0KTAsIHN0b3A9KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTUsIHRpbWU9KGd1aW50NjQpMCwgcG9zaXRpb249KGd1aW50NjQpMCwgZHVyYXRpb249KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTU7AA__:YXBwbGljYXRpb24veC1nc3QtcXQtZ3NwdS10YWcsIHN0eWxlPShzdHJpbmcpaXR1bmVzOwA_\\\,\\\ 000000986773706d000000906461746100000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:None:R3N0U2VnbWVudCwgZmxhZ3M9KEdzdFNlZ21lbnRGbGFncylHU1RfU0VHTUVOVF9GTEFHX05PTkUsIHJhdGU9KGRvdWJsZSkxLCBhcHBsaWVkLXJhdGU9KGRvdWJsZSkxLCBmb3JtYXQ9KEdzdEZvcm1hdClHU1RfRk9STUFUX1RJTUUsIGJhc2U9KGd1aW50NjQpMCwgb2Zmc2V0PShndWludDY0KTAsIHN0YXJ0PShndWludDY0KTAsIHN0b3A9KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTUsIHRpbWU9KGd1aW50NjQpMCwgcG9zaXRpb249KGd1aW50NjQpMCwgZHVyYXRpb249KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTU7AA__:YXBwbGljYXRpb24veC1nc3QtcXQtZ3NwbS10YWcsIHN0eWxlPShzdHJpbmcpaXR1bmVzOwA_\\\,\\\ 0000011867736868000001106461746100000001000000007631302e6c736361636865332e632e796f75747562652e636f6d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:None:R3N0U2VnbWVudCwgZmxhZ3M9KEdzdFNlZ21lbnRGbGFncylHU1RfU0VHTUVOVF9GTEFHX05PTkUsIHJhdGU9KGRvdWJsZSkxLCBhcHBsaWVkLXJhdGU9KGRvdWJsZSkxLCBmb3JtYXQ9KEdzdEZvcm1hdClHU1RfRk9STUFUX1RJTUUsIGJhc2U9KGd1aW50NjQpMCwgb2Zmc2V0PShndWludDY0KTAsIHN0YXJ0PShndWludDY0KTAsIHN0b3A9KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTUsIHRpbWU9KGd1aW50NjQpMCwgcG9zaXRpb249KGd1aW50NjQpMCwgZHVyYXRpb249KGd1aW50NjQpMTg0NDY3NDQwNzM3MDk1NTE2MTU7AA__:YXBwbGljYXRpb24veC1nc3QtcXQtZ3NoaC10YWcsIHN0eWxlPShzdHJpbmcpaXR1bmVzOwA_\\\ \\\}\\\,\\\ container-format\\\=\\\(string\\\)\\\"ISO\\\\\\\ MP4/M4A\\\"\\\;\"\;";',
]
class TestStructure(unittest.TestCase): class TestStructure(unittest.TestCase):
def test_handles_bad_name(self): def test_handles_bad_name(self):
structure = None
with self.assertRaises(ValueError):
structure = Structure(BAD_NAME) structure = Structure(BAD_NAME)
self.assertFalse(structure.valid)
self.assertEquals(structure.pos, 0)
def test_handles_bad_key(self): def test_handles_bad_key(self):
structure = None
with self.assertRaises(ValueError):
structure = Structure(BAD_KEY) structure = Structure(BAD_KEY)
self.assertFalse(structure.valid)
self.assertEquals(structure.pos, 5)
def test_handles_bad_type1(self): def test_handles_bad_type1(self):
structure = None
with self.assertRaises(ValueError):
structure = Structure(BAD_TYPE1) structure = Structure(BAD_TYPE1)
self.assertFalse(structure.valid)
self.assertEquals(structure.pos, 9)
def test_handles_bad_type2(self): def test_handles_bad_type2(self):
structure = None
with self.assertRaises(ValueError):
structure = Structure(BAD_TYPE2) structure = Structure(BAD_TYPE2)
self.assertFalse(structure.valid)
self.assertEquals(structure.pos, 10)
def test_parses_empty_structure(self): def test_parses_empty_structure(self):
structure = Structure(EMPTY_STRUCTURE) structure = Structure(EMPTY_STRUCTURE)
self.assertTrue(structure.valid) self.assertEqual(structure.text, EMPTY_STRUCTURE)
def test_parses_name_in_empty_structure(self): def test_parses_name_in_empty_structure(self):
structure = Structure(EMPTY_STRUCTURE) structure = Structure(EMPTY_STRUCTURE)
self.assertEquals(structure.name, 'foo') self.assertEqual(structure.name, 'foo')
def test_parses_single_value_structure(self): def test_parses_single_value_structure(self):
structure = Structure(SINGLE_VALUE_STRUCTURE) structure = Structure(SINGLE_VALUE_STRUCTURE)
self.assertTrue(structure.valid) self.assertEqual(structure.text, SINGLE_VALUE_STRUCTURE)
def test_parses_name(self): def test_parses_name(self):
structure = Structure(SINGLE_VALUE_STRUCTURE) structure = Structure(SINGLE_VALUE_STRUCTURE)
self.assertEquals(structure.name, 'foo') self.assertEqual(structure.name, 'foo')
def test_parses_key(self): def test_parses_key(self):
structure = Structure(SINGLE_VALUE_STRUCTURE) structure = Structure(SINGLE_VALUE_STRUCTURE)
@ -59,23 +67,25 @@ class TestStructure(unittest.TestCase):
def test_parses_type(self): def test_parses_type(self):
structure = Structure(SINGLE_VALUE_STRUCTURE) structure = Structure(SINGLE_VALUE_STRUCTURE)
self.assertEquals(structure.types['key'], 'string') self.assertEqual(structure.types['key'], 'string')
def test_parses_string_value(self): def test_parses_string_value(self):
structure = Structure(MISC_TYPES_STRUCTURE) structure = Structure(MISC_TYPES_STRUCTURE)
self.assertEquals(structure.values['key1'], 'value') self.assertEqual(structure.values['key1'], 'value')
def test_parses_int_value(self): def test_parses_int_value(self):
structure = Structure(MISC_TYPES_STRUCTURE) structure = Structure(MISC_TYPES_STRUCTURE)
self.assertEquals(structure.values['key2'], 5) self.assertEqual(structure.values['key2'], 5)
def test_parses_nested_structure(self): def test_parses_nested_structure(self):
structure = Structure(NESTED_STRUCTURE) structure = Structure(NESTED_STRUCTURE)
self.assertTrue(structure.valid) self.assertEqual(structure.text, NESTED_STRUCTURE)
sub = structure.values['nested']
self.assertTrue(sub.valid)
def test_nested_structure_has_sub_structure(self): def test_nested_structure_has_sub_structure(self):
structure = Structure(NESTED_STRUCTURE) structure = Structure(NESTED_STRUCTURE)
self.assertEquals(structure.types['nested'], 'structure') self.assertEqual(structure.types['nested'], 'structure')
self.assertIsInstance(structure.values['nested'], Structure) self.assertIsInstance(structure.values['nested'], Structure)
def test_regressions(self):
for s in REGRESSIONS:
structure = Structure(s)