From 8f6b940f329d4a49f273619551b5686d8255e1d4 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Wed, 21 Apr 2021 12:06:05 +0100 Subject: [PATCH 01/33] Reminder to select the python2 branch for python 2 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 46fa626..14fce7f 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A Python implementation of ETSI TS 102 818 v3.1 Hybrid Radio SPI, including an X This can be used by broadcasters for producing or parsing Hybrid Radio SPI documents over IP and/or DAB, or for general service and programme information interfacing. +Please make sure you select the Python2 branch if you want to use on Python 2. + # TODO * GI Files From 8248e81412d0f38c64922ad577733a29a5f8ed12 Mon Sep 17 00:00:00 2001 From: Ben Poor Date: Wed, 12 May 2021 12:03:38 +0200 Subject: [PATCH 02/33] interim changes --- src/spi/__init__.py | 63 ++++++++++++++++++++++++++++++++------ src/spi/test/BearerTest.py | 16 ++++++++++ src/spi/xml/__init__.py | 27 ++++++++-------- 3 files changed, 81 insertions(+), 25 deletions(-) create mode 100644 src/spi/test/BearerTest.py diff --git a/src/spi/__init__.py b/src/spi/__init__.py index 2a1db52..652a986 100644 --- a/src/spi/__init__.py +++ b/src/spi/__init__.py @@ -294,7 +294,9 @@ def __repr__(self): class FmBearer(Bearer): - def __init__(self, ecc, pi, frequency, cost=None, offset=None): + BEARER_PATTERN=re.compile("fm:(\w{2}|[0-9a-f]{3})\.([0-9a-f]{4})\.(\*|.{5})", re.IGNORECASE) + + def __init__(self, ecc, pi, frequency=None, cost=None, offset=None): """ FM Service Bearer @@ -307,11 +309,16 @@ def __init__(self, ecc, pi, frequency, cost=None, offset=None): fm:ce1.c479.09580 :: + Wildcards can also be used for frequency, indicating that *any* frequency will match the bearer + + NOTE: the legacy format using the country code instead of GCC is supported for the time being + but may be removed in a future version. + :param ecc: Extended Country Code :type ecc: int :param pi: Programme Info code :type pi: int - :param frequency: Service frequency (Hz) + :param frequency: Service frequency (Hz), None for wildcard :type frequency: int """ @@ -322,7 +329,7 @@ def __init__(self, ecc, pi, frequency, cost=None, offset=None): if not isinstance(ecc, int): raise ValueError("ECC must be an integer") if not isinstance(pi, int): raise ValueError("PI must be an integer") - if not isinstance(frequency, int): raise ValueError("Frequency must be an integer") + if frequency is not None and not isinstance(frequency, int): raise ValueError("Frequency must be an integer") @classmethod def fromstring(cls, string): @@ -330,16 +337,23 @@ def fromstring(cls, string): Parse a FM Bearer URI from its string representation """ - pattern = re.compile("fm:(.{3})\.(.{4})\.(.{5})") - matcher = pattern.search(string) - if not matcher: raise ValueError('bearer %s does not match the pattern: %s' % (string, pattern)) - ecc = int(matcher.group(1)[1:], 16) + matcher = FmBearer.BEARER_PATTERN.search(string) + if not matcher: raise ValueError('bearer %s does not match the pattern: %s' % (string, FmBearer.BEARER_PATTERN)) + if len(matcher.group(1)) == 3: + ecc = int(matcher.group(1)[1:], 16) + else: # we assume the country code for legacy support and map to the ECC + ecc = int(map_countrycode_to_ecc(matcher.group(1)), 16) pi = int(matcher.group(2), 16) - frequency = int(matcher.group(3)) * 10 - return FmBearer(ecc, pi, frequency) + if matcher.group(3) == '*': + return FmBearer(ecc, pi) + else: + return FmBearer(ecc, pi, int(matcher.group(3)) * 10) def __str__(self): - uri = 'fm:{gcc:03x}.{pi:04x}.{frequency:05d}'.format(gcc=(self.pi >> 4 & 0xf00) + self.ecc, pi=self.pi, frequency=int(self.frequency/10)) + if self.frequency is None: + uri = 'fm:{gcc:03x}.{pi:04x}.*'.format(gcc=(self.pi >> 4 & 0xf00) + self.ecc, pi=self.pi) + else: + uri = 'fm:{gcc:03x}.{pi:04x}.{frequency:05d}'.format(gcc=(self.pi >> 4 & 0xf00) + self.ecc, pi=self.pi, frequency=int(self.frequency/10)) return uri def __repr__(self): @@ -348,6 +362,35 @@ def __repr__(self): def __eq__(self, other): return str(self) == str(other) +""" +map the legacy country code to an ECC +""" +COUNTRY_CODE_MAP = { + "al": "e0", + "dz": "e0", + "ad": "e0", + "am": "e4", + "at": "e0", + "az": "e3", + "by": "e3", + "ba": "e4", + "bg": "e1", + "hr": "e3", + "cy": "e1", + "dk": "e1", + "ee": "e4", + "fi": "e1", + "fr": "e1", + "ge": "e4", + "de": "e1", + "gr": "e1", + "hu": "e0", + "ie": "e3", + "il": "e0", # tbc +} +def map_countrycode_to_ecc(countrycode): + return COUNTRY_CODE_MAP.get(countrycode.lower()) + class IpBearer(DigitalBearer): def __init__(self, uri, content, cost=None, offset=None, bitrate=None): diff --git a/src/spi/test/BearerTest.py b/src/spi/test/BearerTest.py new file mode 100644 index 0000000..6b30952 --- /dev/null +++ b/src/spi/test/BearerTest.py @@ -0,0 +1,16 @@ +from spi import * + +import unittest + +class BearerTest(unittest.TestCase): + + def test_fm_parse_valid(self): + print('here') + FmBearer.fromstring('fm:ce1.c985.09580') + FmBearer.fromstring('fm:ce1.c985.*') + + @unittest.expectedFailure + def test_fm_parse_invalid(self): + FmBearer.fromstring('fm:cee1.c85.095802') + FmBearer.fromstring('fm:ce1.*.09580') + diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 34b90cd..c42c25c 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -554,19 +554,16 @@ def parse_time(timeElement): def parse_bearer(bearer_element, listener): listener.on_element(bearer_element) uri = bearer_element.attrib['id'] - try: - if uri.startswith('dab'): - bearer = DabBearer.fromstring(uri) - elif uri.startswith('fm'): - bearer = FmBearer.fromstring(uri) - elif uri.startswith('http'): - bearer = IpBearer(uri) - else: - raise ValueError('bearer %s not recognised' % uri) - except: - bearer = IpBearer("http://null/") - logger.debug('bearer %s is malformed', uri) - + if uri.startswith('dab'): + bearer = DabBearer.fromstring(uri) + elif uri.startswith('fm'): + bearer = FmBearer.fromstring(uri) + elif uri.startswith('http') or uri.startswith('https'): + if "mimeValue" not in bearer_element.attrib: + raise ValueError("missing mimeValue attribute for URL: %s" % uri) + bearer = IpBearer(uri, content=bearer_element.attrib.get('mimeValue')) + else: + raise ValueError('bearer %s not recognised' % uri) if 'cost' in bearer_element.attrib: bearer.cost = int(bearer_element.attrib['cost']) if 'offset' in bearer_element.attrib: @@ -732,7 +729,7 @@ def parse_service(service_element, listener): # bearers for child in service_element.findall("spi:bearer", namespaces): - if child.attrib.has_key("id"): service.bearers.append(parse_bearer(child)) + if "id" in child: service.bearers.append(parse_bearer(child)) service.bearers.append(parse_bearer(child, listener)) # media @@ -750,7 +747,7 @@ def parse_service(service_element, listener): # links for child in service_element.findall("spi:link", namespaces): - if child.attrib.has_key("uri"): service.links.append(parse_link(child)) + if 'uri' in child: service.links.append(parse_link(child)) # keywords for child in service_element.findall("spi:keywords", namespaces): From ee8a2de8063b396030befc355a386721fd9cc160 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 7 Mar 2022 09:45:31 +0000 Subject: [PATCH 03/33] Update __init__.py Added IntendecAudienceCS to the genre_map Made the first 4 bits of the encoded_genre = 0 --- src/spi/binary/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 0987c6c..817e973 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -349,7 +349,8 @@ def __repr__(self): genre_map = dict( IntentionCS=1, FormatCS=2, - ContentCS=3, # what happened to 4?! + ContentCS=3, + IntendedAudienceCS=4, OriginationCS=5, ContentAlertCS=6, MediaTypeCS=7, @@ -365,6 +366,7 @@ def encode_genre(genre): bits.setall(False) # b0-3: RFU(0) + bits += encode_number(0,4) # b4-7: CS cs = segments[4] if cs in list(genre_map.keys()): cs_val = genre_map[cs] @@ -1290,4 +1292,4 @@ def unmarshall(i): epg = parse_epg(e) return epg else: - raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!') \ No newline at end of file + raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!') From e049728fd989cb7129d341e22f3292831b168208 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 7 Mar 2022 14:14:41 +0000 Subject: [PATCH 04/33] Update setup.py Fix bitarray version at 1.2.0 as there are deprecated calls that we use --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 85c034e..e5580ad 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,6 @@ keywords=['dab', 'spi', 'hybrid', 'radio'], packages=['spi', 'spi.xml', 'spi.binary'], package_dir = {'' : 'src'}, - install_requires = ['python-dateutil', 'isodate', 'bitarray', 'asciitree'], + install_requires = ['python-dateutil', 'isodate', 'bitarray==1.2.0', 'asciitree'], scripts=['dump_binary'] ) From 0d90f0d9f6dc34a7163c32a4702d018aa194869c Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 7 Mar 2022 14:15:31 +0000 Subject: [PATCH 05/33] Fix bitarray version at 1.2.0 Fix bitarray version at 1.2.0, as some deprecated methods are used --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 85c034e..e5580ad 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,6 @@ keywords=['dab', 'spi', 'hybrid', 'radio'], packages=['spi', 'spi.xml', 'spi.binary'], package_dir = {'' : 'src'}, - install_requires = ['python-dateutil', 'isodate', 'bitarray', 'asciitree'], + install_requires = ['python-dateutil', 'isodate', 'bitarray==1.2.0', 'asciitree'], scripts=['dump_binary'] ) From 89eb9e43ee80230b971b2949d30c4ac4c5e812e1 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Fri, 11 Mar 2022 13:18:10 +0000 Subject: [PATCH 06/33] Update setup.py Changed path to dump_binary --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e5580ad..0e2e79a 100644 --- a/setup.py +++ b/setup.py @@ -14,5 +14,5 @@ packages=['spi', 'spi.xml', 'spi.binary'], package_dir = {'' : 'src'}, install_requires = ['python-dateutil', 'isodate', 'bitarray==1.2.0', 'asciitree'], - scripts=['dump_binary'] + scripts=['bin/dump_binary'] ) From 54aa6256f34544887e8300fc437b8db6391cb2a1 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Fri, 11 Mar 2022 14:54:01 +0000 Subject: [PATCH 07/33] Update __init__.py Fixed construction of DabBearer GCC for 16 bit Service IDs to use first nibble of Service ID (not Ensemble ID) For 32 bit Service IDs, the GCC is constructed from nibble 3, plus nibbles and 1 2 (e.g. abf00001 produces GCC = fab) Updated the regex for matching DabBeaer to allow for 16 or 32 bit Service ID values --- src/spi/__init__.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/spi/__init__.py b/src/spi/__init__.py index 2a1db52..851df88 100644 --- a/src/spi/__init__.py +++ b/src/spi/__init__.py @@ -200,7 +200,7 @@ def __init__(self, ecc, eid, sid, scids=0, content=DAB_PLUS, xpad=None, cost=Non dab:.... in hex :: - Where ``gcc`` is a combination of the first nibble of the EId and the ECC. + Where ``gcc`` is a combination of the first nibble of the SId and the ECC. For example: :: @@ -211,8 +211,8 @@ def __init__(self, ecc, eid, sid, scids=0, content=DAB_PLUS, xpad=None, cost=Non :type ecc: int :param eid: Ensemble ID :type eid: int - :param ecc: Service ID - :type ecc: int + :param sid: Service ID + :type sid: int :param scids: Service Component ID within the Service :type scids: int :param xpad: X-PAD application @@ -238,7 +238,7 @@ def fromstring(cls, string): Parse a DAB Bearer URI from its string representation """ - pattern = re.compile("^dab:(.{3})\.(.{4})\.(.{4})\.(.{1})[\.(.+?)]{0,1}$") + pattern = re.compile("^dab:([0-9a-f]{3})\.([0-9a-f]{4})\.([0-9a-f]{4,8})\.([0-9a-f]{1})[\.(.+?)]{0,1}$") matcher = pattern.search(string) if not matcher: raise ValueError('bearer %s does not match the pattern: %s' % (string, pattern.pattern)) ecc = int(matcher.group(1)[1:], 16) @@ -251,7 +251,13 @@ def fromstring(cls, string): return DabBearer(ecc, eid, sid, scids, xpad) def __str__(self): - uri = 'dab:{gcc:03x}.{eid:04x}.{sid:04x}.{scids:01x}'.format(gcc=(self.eid >> 4 & 0xf00) + self.ecc, eid=self.eid, sid=self.sid, scids=self.scids) + if self.sid>65535: # this is a long SId which contains both ECC (first two nibbles) and CC (third nibble) + gcc = (self.sid >> 12 & 0xf00) + (self.sid >> 24) + uri = 'dab:{gcc:03x}.{eid:04x}.{sid:08x}.{scids:01x}'.format(gcc=gcc, eid=self.eid, sid=self.sid, scids=self.scids) + else: # this is a short SId which contains only the CC (first nibble) + gcc = (self.sid >> 4 & 0xf00) + self.ecc + uri = 'dab:{gcc:03x}.{eid:04x}.{sid:04x}.{scids:01x}'.format(gcc=gcc, eid=self.eid, sid=self.sid, scids=self.scids) + if self.xpad is not None: uri += '.{xpad:04x}'.format(xpad=self.xpad) return uri From 52ccec3747fbe112e5e76dfb3b35b6f82735c10c Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Fri, 11 Mar 2022 14:57:33 +0000 Subject: [PATCH 08/33] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e5580ad..0e2e79a 100644 --- a/setup.py +++ b/setup.py @@ -14,5 +14,5 @@ packages=['spi', 'spi.xml', 'spi.binary'], package_dir = {'' : 'src'}, install_requires = ['python-dateutil', 'isodate', 'bitarray==1.2.0', 'asciitree'], - scripts=['dump_binary'] + scripts=['bin/dump_binary'] ) From 1aedf3061457e86d636c30da4e86e8405be984ec Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Fri, 11 Mar 2022 15:25:52 +0000 Subject: [PATCH 09/33] Update build_binary_programmeinfo.py Fixed the reference to Scope in Line 6 --- examples/build_binary_programmeinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/build_binary_programmeinfo.py b/examples/build_binary_programmeinfo.py index 99e0922..08753fe 100644 --- a/examples/build_binary_programmeinfo.py +++ b/examples/build_binary_programmeinfo.py @@ -3,7 +3,7 @@ start = datetime.datetime(2014, 4, 25, 6, 0, 0) end = datetime.datetime(2014, 4, 25, 13, 0, 0) -schedule = Schedule(start, end, originator='Global Radio') +schedule = Schedule(Scope(start, end), originator='Global Radio') info = ProgrammeInfo() info.schedules.append(schedule) @@ -47,4 +47,4 @@ schedule.programmes.append(programme) import sys -sys.stdout.buffer.write(marshall(info)) \ No newline at end of file +sys.stdout.buffer.write(marshall(info)) From 6003f3d4cc7e8badc532980361604e9f9368e9f7 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 14 Mar 2022 20:04:15 +0000 Subject: [PATCH 10/33] Lots of python3 tidy up Noteable - re-added the __eq__ method to DabBearer, which had dropped out and was causing many problems --- src/spi/__init__.py | 12 ++++++++++-- src/spi/xml/__init__.py | 10 +++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/spi/__init__.py b/src/spi/__init__.py index 851df88..aabf3f9 100644 --- a/src/spi/__init__.py +++ b/src/spi/__init__.py @@ -264,6 +264,9 @@ def __str__(self): def __repr__(self): return '' % str(self) + + def __eq__(self, other): + return str(self) == str(other) class HdBearer(DigitalBearer): @@ -298,6 +301,9 @@ def __str__(self): def __repr__(self): return '' % str(self) + def __eq__(self, other): + return str(self) == str(other) + class FmBearer(Bearer): def __init__(self, ecc, pi, frequency, cost=None, offset=None): @@ -356,7 +362,7 @@ def __eq__(self, other): class IpBearer(DigitalBearer): - def __init__(self, uri, content, cost=None, offset=None, bitrate=None): + def __init__(self, uri, content=None, cost=None, offset=None, bitrate=None): """ IP Service Bearer @@ -373,7 +379,9 @@ def __str__(self): def __repr__(self): return '' % str(self) - + + def __eq__(self, other): + return str(self) == str(other) class ProgrammeInfo: """The root of a PI document""" diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 34b90cd..49b5499 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -551,8 +551,7 @@ def parse_time(timeElement): else: raise ValueError('unknown time element: %s' % timeElement) -def parse_bearer(bearer_element, listener): - listener.on_element(bearer_element) +def parse_bearer(bearer_element): uri = bearer_element.attrib['id'] try: if uri.startswith('dab'): @@ -732,8 +731,9 @@ def parse_service(service_element, listener): # bearers for child in service_element.findall("spi:bearer", namespaces): - if child.attrib.has_key("id"): service.bearers.append(parse_bearer(child)) - service.bearers.append(parse_bearer(child, listener)) + if "id" in child.attrib: service.bearers.append(parse_bearer(child)) + # service.bearers.append(parse_bearer(child, listener)) + # service.bearers.append(parse_bearer(child) # media for media_element in service_element.findall("spi:mediaDescription", namespaces): @@ -750,7 +750,7 @@ def parse_service(service_element, listener): # links for child in service_element.findall("spi:link", namespaces): - if child.attrib.has_key("uri"): service.links.append(parse_link(child)) + if "uri" in child.attrib: service.links.append(parse_link(child)) # keywords for child in service_element.findall("spi:keywords", namespaces): From dab6c9cd6315e93dfbd824316e643b36c30862e1 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 10:45:29 +0000 Subject: [PATCH 11/33] Fixes to run under python 3 --- src/spi/__init__.py | 5 +++-- src/spi/binary/__init__.py | 6 +++--- src/spi/xml/__init__.py | 12 ++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/spi/__init__.py b/src/spi/__init__.py index aabf3f9..6274fc9 100644 --- a/src/spi/__init__.py +++ b/src/spi/__init__.py @@ -645,7 +645,7 @@ class Scope: Contains the scope of the proposed schedule, in terms of time and bearers """ - def __init__(self, start, end, bearers=[]): + def __init__(self, start=None, end=None, bearers=[]): """ :param start: Scope start time, if not specified this is calculated from the programmes :type start: datetime @@ -663,7 +663,7 @@ class Schedule: Contains programmes within a given time period. """ - def __init__(self, scope : Scope=None, created=datetime.datetime.now(tzlocal()), version=1, originator=None): + def __init__(self, scope=None, created=datetime.datetime.now(tzlocal()), version=1, originator=None): """x :param scope: Defined scope, otherwise proposed from the schedule :type scope: Scope @@ -674,6 +674,7 @@ def __init__(self, scope : Scope=None, created=datetime.datetime.now(tzlocal()), :param originator: Originator of the schedule :type originator: string """ + if scope==None and type(scope) is not Scope: scope=Scope() if type(scope) is not Scope: raise ValueError("scope must be a Scope object") self.scope = scope self.created = created diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 0a0fd1c..c9c4eec 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -634,13 +634,13 @@ def tobytes(self): bits += tmp elif datalength >= 254 and datalength <= 1<<16: tmp = bitarray() - tmp.frombytes('\xfe') + tmp.frombytes(b'\xfe') bits += tmp tmp = encode_number(datalength, 16) bits += tmp elif datalength > 1<<16 and datalength <= 1<<24: tmp = bitarray() - tmp.frombytes('\xff') + tmp.frombytes(b'\xff') bits += tmp tmp = encode_number(datalength, 24) bits += tmp @@ -968,7 +968,7 @@ def build_service(service): # radiodns lookup if service.lookup: from urllib.parse import urlparse - url = urlparse(service.lookup) + url = urlparse(str(service.lookup)) lookup_element = Element(0x31) lookup_element.attributes.append(Attribute(0x80, url.netloc, encode_string)) lookup_element.attributes.append(Attribute(0x81, url.path[1:], encode_string)) diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 49b5499..83b43b1 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -639,21 +639,21 @@ def parse_programme(programmeElement, listener): return programme -def parse_schedule(scheduleElement): +def parse_schedule(scheduleElement, listener): schedule = Schedule() if 'creationTime' in scheduleElement.attrib: schedule.created = isodate.parse_datetime(scheduleElement.attrib['creationTime']) if 'version' in scheduleElement.attrib: schedule.version = int(scheduleElement.attrib['version']) if 'originator' in scheduleElement.attrib: schedule.originator = scheduleElement.attrib['originator'] for programmeElement in scheduleElement.findall('spi:programme', namespaces): - schedule.programmes.append(parse_programme(programmeElement)) + schedule.programmes.append(parse_programme(programmeElement, listener)) return schedule -def parse_programmeinfo(root): +def parse_programmeinfo(root, listener): logger.debug('parsing programme info from root: %s', root) schedules = [] for schedule_element in root.findall('spi:schedule', namespaces): - schedule = parse_schedule(schedule_element) + schedule = parse_schedule(schedule_element, listener) schedules.append(schedule) info = ProgrammeInfo(schedules=schedules) return info @@ -795,9 +795,9 @@ def unmarshall(i, listener=UnmarshallListener()): return parse_serviceinfo(root, listener) elif root.tag == '{%s}epg' % SCHEMA_NS: if len(root.findall("spi:schedule", namespaces)): - return parse_programmeinfo(root) + return parse_programmeinfo(root, listener) if len(root.findall("spi:programmeGroups", namespaces)): - return parse_groupinfo(root) + return parse_groupinfo(root, listener) else: raise Exception('epg element does not contain either schedules or programme groups') else: From a0e043be30ca66c20f8d9af42560f8977bae2849 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 13:33:30 +0000 Subject: [PATCH 12/33] Fix ContentCS for value 4, and calculation of bit in Genre coding --- src/spi/binary/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index c9c4eec..24bc931 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -318,7 +318,8 @@ def __repr__(self): genre_map = dict( IntentionCS=1, FormatCS=2, - ContentCS=3, # what happened to 4?! + ContentCS=3, + IntendedAudienceCS=4, OriginationCS=5, ContentAlertCS=6, MediaTypeCS=7, @@ -334,6 +335,7 @@ def encode_genre(genre): bits.setall(False) # b0-3: RFU(0) + bits += encode_number(0,4) # b4-7: CS cs = segments[4] if cs in list(genre_map.keys()): cs_val = genre_map[cs] @@ -626,9 +628,11 @@ def tobytes(self): bits.frombytes(b'\x01') # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (256-65536 bytes) + # b16-31: extended element length (254-65536 bytes) # b16-39: extended element length (65537-16777216 bytes) + datalength = len(self.value.encode()) # ensure we get the right count for the encoding + if datalength <= 253: tmp = encode_number(datalength, 8) bits += tmp From 0fbdafcfe5b2cdda125107bfb0347d397dffdb3c Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 13:34:51 +0000 Subject: [PATCH 13/33] Update __init__.py Align exactly with the version in the opendigitalradio fork --- src/spi/binary/__init__.py | 48 +++++++++----------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 817e973..b4abe1b 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -222,37 +222,6 @@ def tobytes(self): logger.debug('encoding attribute %s with function %s', self, self.f) data = self.f(self.value, *self.args, **self.kwargs) - #if isinstance(self.value, int) or isinstance(self.value, long): # integer - # if self.bitlength is None: raise ValueError('attribute %s with int value has no bitlength specification' % self) - # logger.debug('encoding attribute %s as int with %d bits', self, self.bitlength) - # data = encode_number(self.value, self.bitlength) - #elif isinstance(self.value, datetime.timedelta): # duration - # data = encode_number(self.value.seconds, 16) - # logger.debug('encoding attribute %s as duration', self) - #elif isinstance(self.value, Crid): # CRID - # data = bitarray() - # data.fromstring(str(self.value)) - # logger.debug('encoding attribute %s as CRID', self) - #elif isinstance(self.value, Genre): # genre - # data = encode_genre(self.value) - # logger.debug('encoding attribute %s as genre', self) - #elif isinstance(self.value, datetime.datetime): # time - # data = encode_timepoint(self.value) - # logger.debug('encoding attribute %s as timepoint', self) - #elif isinstance(self.value, str): # string - # data = bitarray() - # data.fromstring(self.value) - # logger.debug('encoding attribute %s as string', self) - #elif isinstance(self.value, Bearer): - # data = encode_bearer(self.value) - # logger.debug('encoding attribute %s as bearer', self) - #elif isinstance(self.value, Ensemble): - # data = encode_ensembleid(self.value.ecc, self.value.eid) - # logger.debug('encoding attribute %s as ensemble ID', self.value) - #else: - # raise ValueError('dont know how to encode this type: %s = %s' % (self.value.__class__.__name__, str(self.value))) - #data.fill() - # b0-b7: tag bits = encode_number(self.tag, 8) @@ -407,9 +376,14 @@ def encode_string(s): def encode_timepoint(timepoint): + if timepoint.tzinfo is None: + UTC = datetime.timezone.utc + timepoint = timepoint.replace(tzinfo=UTC) bits = bitarray(1) bits.setall(False) - offset = (timepoint.utcoffset().days * 86400 + timepoint.utcoffset().seconds) + (timepoint.dst().days * 86400 + timepoint.dst().days) + tz_offset = timepoint.utcoffset() if timepoint.utcoffset() is not None else datetime.timedelta(0) + tz_dst = timepoint.dst() if timepoint.dst() is not None else datetime.timedelta(0) + offset = (tz_offset.days * 86400 + tz_offset.seconds) + (tz_dst.days * 86400 + tz_dst.days) timepoint = timepoint - timedelta(seconds=offset) # b0: RFA(0) @@ -656,21 +630,21 @@ def tobytes(self): # b8-15: element data length (0-253 bytes) # b16-31: extended element length (254-65536 bytes) # b16-39: extended element length (65537-16777216 bytes) - + datalength = len(self.value.encode()) # ensure we get the right count for the encoding - + if datalength <= 253: tmp = encode_number(datalength, 8) bits += tmp elif datalength >= 254 and datalength <= 1<<16: tmp = bitarray() - tmp.frombytes('\xfe') + tmp.frombytes(b'\xfe') bits += tmp tmp = encode_number(datalength, 16) bits += tmp elif datalength > 1<<16 and datalength <= 1<<24: tmp = bitarray() - tmp.frombytes('\xff') + tmp.frombytes(b'\xff') bits += tmp tmp = encode_number(datalength, 24) bits += tmp @@ -998,7 +972,7 @@ def build_service(service): # radiodns lookup if service.lookup: from urllib.parse import urlparse - url = urlparse(service.lookup) + url = urlparse(str(service.lookup)) lookup_element = Element(0x31) lookup_element.attributes.append(Attribute(0x80, url.netloc, encode_string)) lookup_element.attributes.append(Attribute(0x81, url.path[1:], encode_string)) From c603703129a19c7c6e2e8b9fd34b9a3ebccf99ad Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 13:48:04 +0000 Subject: [PATCH 14/33] Fixed an incorrect byte range --- src/spi/binary/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 24bc931..a3e99d9 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -77,7 +77,7 @@ def tobytes(self): bits = encode_number(self.tag, 8) # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (256-65536 bytes) + # b16-31: extended element length (254-65536 bytes) # b16-39: extended element length (65537-16777216 bytes) datalength = len(data)/8 if datalength == 0: From 1782feba3f344ccf39540684558b598fffea2fae Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 13:53:59 +0000 Subject: [PATCH 15/33] Force characters to by single byte additions --- src/spi/binary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index a3e99d9..5f8638b 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -233,12 +233,12 @@ def tobytes(self): bits += encode_number(datalength, 8) elif datalength >= 254 and datalength <= 1<<16: tmp = bitarray() - tmp.fromstring('\xfe') + tmp.fromstring(b'\xfe') bits += tmp bits += encode_number(datalength, 16) elif datalength > 1<<16 and datalength <= 1<<24: tmp = bitarray() - tmp.fromstring('\xff') + tmp.fromstring(b'\xff') bits += tmp bits += encode_number(datalength, 24) else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) From 1d2e432e0062e384e6ed6285c617158ea779db8b Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 13:55:57 +0000 Subject: [PATCH 16/33] Force character additions to single byte --- src/spi/binary/__init__.py | 2538 ++++++++++++++++++------------------ 1 file changed, 1269 insertions(+), 1269 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index b4abe1b..5f8638b 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -1,1269 +1,1269 @@ -from spi import * - -from bitarray import bitarray, bits2bytes -import math -import datetime, dateutil.tz -import logging -import sys -from datetime import timedelta - -logger = logging.getLogger("spi.binary") - -class Ensemble: - """ - Describes a DAB ensemble - - :param ecc: Extended Country Code (ECC) - :type ecc: integer - :param eid: Ensemble ID (EId) - :type eid: integer - :param version: Description version - :type version: integer - """ - - def __init__(self, ecc, eid, version=1): - self.ecc = ecc - self.eid = eid - self.names = [] - self.descriptions = [] - self.media = [] - self.keywords = [] - self.links = [] - self.services = [] - self.frequencies = [] - self.version = version - - def __str__(self): - return "%02x.%04x" % (self.ecc, self.eid) - - def __repr__(self): - return '' % str(self) - - -class Element: - - def __init__(self, tag, attributes=None, children=None, cdata=None): - self.tag = tag - self.attributes = (attributes if attributes is not None else []) - self.children = (children if children is not None else []) - self.cdata = cdata - logger.debug('created new element: %s', self) - - def tobytes(self): - logger.debug('rendering element %s', repr(self)) - data = bitarray() - - # encode attributes - for attribute in self.attributes: - logger.debug('rendering attribute: %s', attribute) - try: data += attribute.tobytes() - except Exception as e: - raise ValueError('error rendering attribute %s of %s: %s' % (attribute, self, str(e))).with_traceback(sys.exc_info()[2]) - - # encode children - for child in self.children: - logger.debug('rendering child element: %s', child) - try: data += child.tobytes() - except: - logger.exception('error rendering child %s of %s', child, self) - raise - - # encode CData - if self.cdata is not None: - logger.debug('rendering cdata: %s', self.cdata) - data += self.cdata.tobytes() - - # b0-b7: element tag - bits = encode_number(self.tag, 8) - - # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (256-65536 bytes) - # b16-39: extended element length (65537-16777216 bytes) - datalength = len(data)/8 - if datalength == 0: - raise ValueError('element data length is zero') - if datalength <= 253: - bits += encode_number(datalength, 8) - elif datalength >= 254 and datalength <= 1<<16: - bits += encode_number(0xfe, 8) - bits += encode_number(datalength, 16) - elif datalength > 1<<16 and datalength <= 1<<24: - bits += encode_number(0xff, 8) - bits += encode_number(datalength, 24) - else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) - logger.debug('element %s is rendered with data length: %d bytes: %s', repr(self), datalength, bitarray_to_hex(bits)) - bits += data - return bits - - def __iter__(self): - return iter(self.children) - - def has_child(self, tag): - return len(self.get_children(tag)) - - def get_children(self, tag=None): - if tag is not None: - return [x for x in self.children if x.tag == tag] - return self.children - - def has_attribute(self, tag): - return len(self.get_attributes(tag)) - - def get_attributes(self, tag=None): - if tag is not None: - return [x for x in self.attributes if x.tag == tag] - return self.attributes - - @staticmethod - def frombits(bits): - - # b0-b7: element tag - tag = int(bits[0:8].to01(), 2) - if tag < 0x02 or tag > 0x36: raise ValueError('invalid value for tag: 0x%02x' % tag) - - # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (256-65536 bytes) - # b16-39: extended element length (65537-16777216 bytes) - datalength = int(bits[8:16].to01(), 2) - start = 16 - if datalength == 0xfe: - datalength = int(bits[start:start+16].to01(), 2) - start += 16 - elif datalength == 0xff: - datalength = int(bits[start:start+24].to01(), 2) - start += 24 - data = bits[start : start + (datalength * 8)] - - i = 0 - e = Element(tag) - logger.debug('parsing data of length %d bytes for element with tag 0x%02x', datalength, tag) - while i < data.length(): - logger.debug('now parsing at offset %d / %d', i/8, data.length()/8) - child_tag = int(data[i:i+8].to01(), 2) - child_datalength = int(data[i+8:i+16].to01(), 2) - start = 16 - if child_datalength == 0xfe: - child_datalength = int(data[i+16:i+32].to01(), 2) - start = 32 - elif child_datalength == 0xff: - child_datalength = int(data[i+16:i+40].to01(), 2) - start = 40 - logger.debug('child with tag 0x%02x for parent tag 0x%02x at offset %d has data length of %d bytes', child_tag, tag, i/8, child_datalength) - end = start + (child_datalength * 8) - if i + end > data.length(): - raise ValueError('end of data for element with tag 0x%02x at offset %d requested is beyond length: %d > %d: %s' % (child_tag, i/8, (i + end)/8, data.length() / 8, bitarray_to_hex(data[i:i+64]))) - child_data = data[i + start : i + end] - if child_data.length() < 16*8: logger.debug('child element with tag 0x%02x for parent 0x%02x has data: %s', child_tag, tag, bitarray_to_hex(child_data)) - - # attributes - if child_tag >= 0x80 and child_tag <= 0x87: - logger.debug('parsing child as an attribute') - attribute = Attribute.frombits(tag, data[i:i+end]) - logger.debug('parsed child as an attribute: %s', attribute) - e.attributes.append(attribute) - # token table - elif child_tag == 0x04: - logger.debug('parsing child as a token table') - tokens = decode_tokentable(child_data) - e.tokens = tokens - logger.debug('parsed token table: %s', tokens) - # default content ID - elif child_tag == 0x05: - logger.debug('parsing child as a default content ID') - default_contentid = decode_contentid(child_data) - e.default_contentid = default_contentid - # default language - elif child_tag == 0x06: - logger.debug('parsing child as a default language (not yet implemented)') - pass - # children - elif child_tag >= 0x02 and child_tag <= 0x36: - logger.debug('parsing child as an element') - child = Element.frombits(data[i:i+end]) - child.parent = e - logger.debug('parsed child as an element: %s', child) - e.children.append(child) - # cdata - elif child_tag == 0x01: - logger.debug('parsing child as CDATA') - cdata = CData.frombits(data[i:i+end]) - logger.debug('parsed CDATA: %s', cdata) - e.cdata = cdata - else: - raise ValueError('unknown element 0x%02x under parent 0x%02x' % (child_tag, tag)) - - i += end - - return e - - def __str__(self): - return 'tag=0x%02X, attributes=%s, children=%s, cdata=%s' % (self.tag, self.attributes, self.children, self.cdata) - - def __repr__(self): - return '' % self.tag - -class Attribute: - - def __init__(self, tag, value, f=None, *args, **kwargs): - if not isinstance(tag, int): raise ValueError('tag must be an integer') - self.tag = tag - self.value = value - self.f = f - self.args = args - self.kwargs = kwargs - logger.debug('created new attribute: %s', repr(self)) - - def tobytes(self): - - if not self.f: raise ValueError('cant encode this attribute without an encoding function') - - # encode data - data = None - logger.debug('encoding attribute %s with function %s', self, self.f) - data = self.f(self.value, *self.args, **self.kwargs) - - # b0-b7: tag - bits = encode_number(self.tag, 8) - - # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (256-65536 bytes) - # b16-39: extended element length (65537-16777216 bytes) - datalength = bits2bytes(data.length()) - if datalength <= 253: - bits += encode_number(datalength, 8) - elif datalength >= 254 and datalength <= 1<<16: - tmp = bitarray() - tmp.fromstring('\xfe') - bits += tmp - bits += encode_number(datalength, 16) - elif datalength > 1<<16 and datalength <= 1<<24: - tmp = bitarray() - tmp.fromstring('\xff') - bits += tmp - bits += encode_number(datalength, 24) - else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) - - bits += data - return bits - - @staticmethod - def frombits(parent, bits): - - # b0-b7: attribute tag - tag = int(bits[0:8].to01(), 2) - - # b8-15: attribute data length (0-253 bytes) - # b16-31: extended attribute length (256-65536 bytes) - # b16-39: extended attribute length (65537-16777216 bytes) - datalength = int(bits[8:16].to01(), 2) - start = 16 - if datalength >= 254 and datalength <= 1<<16: - datalength = int(bits[16:32].to01(), 2) - start = 32 - elif datalength > 1<<16 and datalength <= 1<<24: - datalength = int(bits[16:40].to01(), 2) - start = 40 - elif datalength > 1<<24: - raise ValueError('attribute data length exceeds the maximum allowed by the extended attribute length (24bits): %s > %s' + datalength + " > " + (1<<24)) - data = bits[start:start+(datalength * 8)] - - # decode data - if isinstance(parent, Element): parent_tag = parent.tag - else: parent_tag = int(parent) - if (parent_tag, tag) in [ # integer - (0x02, 0x80), (0x21, 0x80), (0x23, 0x80), (0x23, 0x81), (0x23, 0x82), (0x23, 0x84), (0x25, 0x80), - (0x1c, 0x81), (0x1c, 0x82), (0x1c, 0x87), (0x17, 0x81), (0x17, 0x82), (0x03, 0x80), (0x26, 0x81), - (0x2b, 0x84), (0x2b, 0x85), (0x2e, 0x81) - ]: - logger.debug('decoding tag/attribute 0x%02x/0x%02x as int', parent_tag, tag) - value = int(data.to01(), 2) - elif (parent_tag, tag) in [ # string - (0x14, 0x80), (0x17, 0x80), (0x18, 0x80), (0x18, 0x83), (0x1c, 0x80), (0x20, 0x80), (0x20, 0x82), - (0x21, 0x82), (0x03, 0x82), (0x03, 0x83), (0x2b, 0x80), (0x2b, 0x82), (0x2e, 0x80), (0x31, 0x80), - (0x31, 0x81), (0x29, 0x82), (0x18, 0x81), (0x18, 0x83), (0x10, 0x80), (0x11, 0x80), (0x12, 0x80), - (0x20, 0x86), (0x2a, 0x80), (0x2b, 0x81), (0x1a, 0x80), (0x1b, 0x80), (0x06, 0x80) - ]: - logger.debug('decoding tag/attribute 0x%02x/0x%02x as string', parent_tag, tag) - value = data.tobytes().decode() - elif (parent_tag, tag) in [(0x2c, 0x81), (0x2c, 0x83), (0x2f, 0x80), (0x2f, 0x81)]: # duration - logger.debug('decoding tag/attribute 0x%02x/0x%02x as duration', parent_tag, tag) - value = datetime.timedelta(seconds=int(data.to01(), 2)) - elif (parent_tag, tag) in []: # genre - logger.debug('decoding tag/attribute 0x%02x/0x%02x as genre', parent_tag, tag) - value = decode_genre(data) - elif (parent_tag, tag) in [(0x20, 0x81), (0x21, 0x81), (0x24, 0x80), (0x24, 0x81), (0x2c, 0x80), (0x2c, 0x82), - (0x03, 0x81)]: # time - logger.debug('decoding tag/attribute 0x%02x/0x%02x as timepoint', parent_tag, tag) - value = decode_timepoint(data) - elif (parent_tag, tag) in [(0x25, 0x80), (0x26, 0x80), (0x29, 0x80), (0x2d, 0x80)]: # content ID - logger.debug('decoding tag/attribute 0x%02x/0x%02x as ContentId', parent_tag, tag) - return Attribute(tag, decode_contentid(data)) - elif (parent_tag, tag) in [(0x1c, 0x83), (0x1c, 0x84), (0x03, 0x84), (0x2b, 0x83), (0x2e, 0x83), (0x2e, 0x84)]: # ENUM - try: - value = decode_enum(parent_tag, tag, data) - except: - logger.warning('error decoding enum for parent 0x%02x from tag: 0x%02x - IGNORING for now' % (parent_tag, tag)) - value = data - else: - raise ValueError('dont know how to decode attribute value for parent 0x%02x from tag: 0x%02x' % (parent_tag, tag)) - - return Attribute(tag, value) - - def __str__(self): - return str('0x%x' % self.tag) - - def __repr__(self): - return '' % (self.tag, self.value) - -genre_map = dict( - IntentionCS=1, - FormatCS=2, - ContentCS=3, - IntendedAudienceCS=4, - OriginationCS=5, - ContentAlertCS=6, - MediaTypeCS=7, - AtmosphereCS=8 -) - -def encode_genre(genre): - - segments = genre.href.split(':') - if len(segments) < 6: raise ValueError('genre is incorrectly formatted: %s' % genre) - - bits = bitarray(4) - bits.setall(False) - - # b0-3: RFU(0) - bits += encode_number(0,4) - # b4-7: CS - cs = segments[4] - if cs in list(genre_map.keys()): cs_val = genre_map[cs] - else: raise ValueError('unknown CS in genre: %s' % cs) - bits += encode_number(cs_val, 4) - - # optional schema levels - if len(segments) >= 6: - levels = segments[6].split('.') - for level in levels: - bits += encode_number(int(level), 8) - - return bits - -def decode_genre(bits): - - # b4-7: CS - cs_val = int(bits[4:8].to01(), 2) - if cs_val in list(genre_map.values()): cs = [x[0] for x in list(genre_map.items()) if x[1] == cs_val] - else: raise ValueError('unknown CS value for genre: %d' % cs_val) - - level = '%d' % cs_val - - # optional schema levels - if bits.length() > 8: - i = 8 - while i < bits.length(): - sublevel = int(bits[i:i+8].to01(), 2) - level += '.%d' % sublevel - i += 8 - - return Genre('urn:tva:metadata:cs:ContentCS:2002:%s' % level) - -def encode_string(s): - b = bitarray() - b.frombytes(s.encode()) - return b - -def encode_timepoint(timepoint): - - if timepoint.tzinfo is None: - UTC = datetime.timezone.utc - timepoint = timepoint.replace(tzinfo=UTC) - bits = bitarray(1) - bits.setall(False) - tz_offset = timepoint.utcoffset() if timepoint.utcoffset() is not None else datetime.timedelta(0) - tz_dst = timepoint.dst() if timepoint.dst() is not None else datetime.timedelta(0) - offset = (tz_offset.days * 86400 + tz_offset.seconds) + (tz_dst.days * 86400 + tz_dst.days) - timepoint = timepoint - timedelta(seconds=offset) - - # b0: RFA(0) - - # b1-17: Date - a = (14 - timepoint.month) / 12 - y = timepoint.year + 4800 - a - m = timepoint.month + (12 * a) - 3 - jdn = timepoint.day + ((153 * m) + 2) / 5 + (365 * y) + (y / 4) - (y / 100) + (y / 400) - 32045 - jd = jdn + timepoint.hour / 24 + timepoint.minute / 1440 + timepoint.second / 86400 - mjd = (int)(jd - 2400000.5) - bits += encode_number(mjd, 17) - - # b18: RFA(0) - bits += bitarray('0') - - # b19: LTO Flag - if timepoint.tzinfo is None or (timepoint.utcoffset().days == 0 and timepoint.utcoffset().seconds == 0): - bits += bitarray('0') - else: - bits += bitarray('1') - - # b20: UTC Flag - # b21: UTC - 11 or 27 bits depending on the form - if timepoint.second > 0: - bits += bitarray('1') - bits += encode_number(timepoint.hour, 5) - bits += encode_number(timepoint.minute, 6) - bits += encode_number(timepoint.second, 6) - bits += bitarray('0' * 10) - else: - bits += bitarray('0') - bits += encode_number(timepoint.hour, 5) - bits += encode_number(timepoint.minute, 6) - - # b32/48: LTO - if bits[19]: - bits += bitarray('00') # b49-50: RFA(0) - bits += bitarray('0' if offset > 0 else '1') # b51: LTO sign - bits += encode_number(offset / (60 * 60) * 2, 5) # b52-56: Half hours - - return bits - -def decode_timepoint(bits): - - if not bits.any(): return None # NOW - - mjd = int(bits[1:18].to01(), 2) - date = datetime.datetime.fromtimestamp((mjd - 40587) * 86400) - timepoint = datetime.datetime.combine(date, datetime.time()) - - # parse timezone - if bits[19]: - sign = bits[-6] - half_hours = int(bits[-5:].to01(), 2) - timezone = dateutil.tz.tzoffset(None, half_hours * 30 * 60 * (-1 if sign else 1)) - else: - timezone = dateutil.tz.tzutc() - - # parse date with UTC short form or long form - if bits[20]: - timepoint = timepoint.replace(hour=int(bits[21:26].to01(), 2), - minute=int(bits[26:32].to01(), 2), - second=int(bits[32:38].to01(), 2), - microsecond=int(bits[38:48].to01(), 2) * 1000, - tzinfo=timezone) - else: - timepoint = timepoint.replace(hour=int(bits[21:26].to01(), 2), - minute=int(bits[26:32].to01(), 2), - tzinfo=timezone) - - return timepoint - -def encode_bearer(bearer): - - if isinstance(bearer, DabBearer): - bits = bitarray(4) - bits.setall(False) - - # b0: RFA(0) - - # b1: Ensemble Flag. Indicates whether ECC and EId are contained with the - # Content ID. - # 0 = ECC and EId are not present. The service that is referenced within the - # contentID is transmitted on the same ensemble as this EPG service - # 1 = ECC and EId are present. - if bearer.ecc is not None and bearer.eid is not None: bits[1] = True - - # b2: X-PAD flag. Indicates whether the addressed component is carried in an - # X-PAD channel. - # 0 = Is not carried in an X-PAD channel. - # 1 = Is carried in an X-PAD channel. - if bearer.xpad is not None: bits[2] = True - - # b3: SId encoding flag - # 0 = Audio service (SId is 16bit) - # 1 = Data service (SId is 32bit) - # no audio support right now - - # b4-7: SCIdS - bits += encode_number(bearer.scids, 4) - - # optional next 8 bits: ECC - if bearer.ecc is not None: - bits += encode_number(bearer.ecc, 8) - - # optional next 16 bits: EId - if bearer.eid is not None: - bits += encode_number(bearer.eid, 16) - - # next 16/32 bits: SId - bits += encode_number(bearer.sid, 16) - - # optional next 8 bits: X-PAD extension - if bearer.xpad is not None: - bits += encode_number(bearer.xpad, 8) - - elif isinstance(bearer, IpBearer): - return encode_string(bearer.uri) - else: - raise ValueError('bearer %s not currently supported', bearer) - - return bits - -def encode_ensembleid(params): - - ecc, eid = params - bits = bitarray() - - # b0: ECC - bits += encode_number(ecc, 8) - - # b8: EId - bits += encode_number(eid, 16) - return bits - -def decode_contentid(bits): - - """decodes a ContentId from a bitarray""" - - # b0: RFA(0) - - # b1: Ensemble Flag. Indicates whether ECC and EId are contained with the - # Content ID. - # 0 = ECC and EId are not present. The service that is referenced within the - # contentID is transmitted on the same ensemble as this EPG service - # 1 = ECC and EId are present. - ecc = None - eid = None - sid = None - scids = None - xpad = None - - try: - if bits.length() == 24: # EnsembleId - # ECC, EId - ecc = int(bits[0:8].to01(), 2) - eid = int(bits[8:24].to01(), 2) - return (ecc, eid) - else: - ensemble_flag = bits[1] - xpad_flag = bits[2] - sid_flag = bits[3] - - # SCIdS - scids = int(bits[4:8].to01(), 2) - - # ECC, EId - i = 8 - if ensemble_flag: - ecc = int(bits[8:16].to01(), 2) - eid = int(bits[16:32].to01(), 2) - i = 32 - - # SId - if not sid_flag: - sid = int(bits[i:i+16].to01(), 2) - i += 16 - elif sid_flag: - sid = int(bits[i:i+32].to01(), 2) - i += 32 - - # XPAD - if xpad_flag: - xpad = int(bits[i+3:i+8].to01(), 2) - - return (ecc, eid, sid, scids, xpad) - except: - raise ValueError('error parsing ContentId from data: %s', bitarray_to_hex(bits)) - -def decode_tokentable(bits): - - tokens = {} - - i = 0 - while i < bits.length(): - tag = int(bits[i:i+8].to01(), 2) - length = int(bits[i+8:i+16].to01(), 2) - data = bits[i+16:i+16+(length*8)].tobytes().decode(DEFAULT_ENCODING) - tokens[tag] = data - i += 16 + (length * 8) - return tokens - -"""Map of possible num values and their binary equivalents. - Note that not all the values are currently implemented, which will - cause the decoder to skip over their details""" -enum_values = { - (0x1c, 0x83, 0x01) : False, - (0x1c, 0x83, 0x02) : True, - (0x1c, 0x84, 0x01) : "on-air", - (0x1c, 0x84, 0x02) : "off-air", - (0x2b, 0x83, 0x02) : Multimedia.LOGO_UNRESTRICTED, - (0x2b, 0x83, 0x04) : Multimedia.LOGO_COLOUR_SQUARE, - (0x2b, 0x83, 0x06) : Multimedia.LOGO_COLOUR_RECTANGLE -} - -def decode_enum(parent_tag, tag, bits): - - if bits.length() != 8: raise ValueError('enum data for parent/attribute 0x%02x/0x%02x is of incorrect length: %d bytes' % (parent_tag, tag, bits.length()/8)) - - value = int(bits.to01(), 2) - key = (parent_tag, tag, value) - if key in list(enum_values.keys()): - return enum_values.get(key) - else: - raise NotImplementedError('enum for parent/attribute 0x%02x/0x%02x not implemented' % (parent_tag, tag)) - -class CData: - - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - def __repr__(self): - return '' % str(self) - - def tobytes(self): - # b0-b7: element tag - bits = bitarray() - bits.frombytes(b'\x01') - - # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (254-65536 bytes) - # b16-39: extended element length (65537-16777216 bytes) - - datalength = len(self.value.encode()) # ensure we get the right count for the encoding - - if datalength <= 253: - tmp = encode_number(datalength, 8) - bits += tmp - elif datalength >= 254 and datalength <= 1<<16: - tmp = bitarray() - tmp.frombytes(b'\xfe') - bits += tmp - tmp = encode_number(datalength, 16) - bits += tmp - elif datalength > 1<<16 and datalength <= 1<<24: - tmp = bitarray() - tmp.frombytes(b'\xff') - bits += tmp - tmp = encode_number(datalength, 24) - bits += tmp - else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) - tmp = bitarray() - tmp.frombytes(self.value.encode()) - bits += tmp - - return bits - - @staticmethod - def frombits(bits): - - # b0-b7: element tag - tag = int(bits[0:8].to01(), 2) - if tag != 0x01: raise ValueError('CData does not have the correct tag: 0x%02x != 0x01', tag) - - # b8-15: element data length (0-253 bytes) - # b16-31: extended element length (256-65536 bytes) - # b16-39: extended element length (65537-16777216 bytes) - datalength = int(bits[8:16].to01(), 2) - start = 16 - if datalength >= 254 and datalength <= 1<<16: - datalength = int(bits[16:32].to01(), 2) - start = 32 - elif datalength > 1<<16 and datalength <= 1<<24: - datalength = int(bits[16:40].to01(), 2) - start = 40 - elif datalength > 1<<24: - raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) - data = bits[start:start+(datalength * 8)] - - return CData(data.tobytes().decode()) - -def marshall(obj, **kwargs): - """Marshalls an :class:Epg or :class:ServiceInfo to its binary document""" - if isinstance(obj, ServiceInfo): return marshall_serviceinfo(obj, kwargs.get('ensemble', None)) - elif isinstance(obj, ProgrammeInfo): return marshall_programmeinfo(obj) - -def marshall_serviceinfo(info, ensemble): - - # serviceInformation - info_element = Element(0x03) - if info.version > 1: info_element.attributes.append(Attribute(0x80, info.version, encode_number, 16)) - if info.created: info_element.attributes.append(Attribute(0x81, info.created, encode_timepoint)) - if info.originator: info_element.attributes.append(Attribute(0x82, info.originator, encode_string)) - if info.provider: info_element.attributes.append(Attribute(0x83, info.provider, encode_string)) - - # default language - default_language_element = Element(0x06) - default_language_element.attributes.append(Attribute(0x80, DEFAULT_LANGUAGE, encode_string)) # TODO make this configurable in a better way - info_element.children.append(default_language_element) - - # ensemble - if ensemble is None: raise ValueError('must specify an ensemble') - ensemble_element = build_ensemble(ensemble, info.services) - - info_element.children.append(ensemble_element) - - return info_element.tobytes() - -def marshall_programmeinfo(info): - - # epg (default type is DAB, so no need to encode) - epg_element = Element(0x02) - - # default language - default_language_element = Element(0x06) - default_language_element.attributes.append(Attribute(0x80, DEFAULT_LANGUAGE, encode_string)) # TODO make this configurable in a better way - epg_element.children.append(default_language_element) - - for schedule in info.schedules: - schedule_element = build_schedule(schedule) - epg_element.children.append(schedule_element) - - return epg_element.tobytes() - -def build_schedule(schedule): - - # schedule - schedule_element = Element(0x21) - if schedule.version is not None and schedule.version > 1: - schedule_element.attributes.append(Attribute(0x80, schedule.version, encode_number, 16)) - schedule_element.attributes.append(Attribute(0x81, schedule.created, encode_timepoint)) - if schedule.originator is not None: - schedule_element.attributes.append(Attribute(0x82, schedule.originator, encode_string)) - - # schedule scope TODO - #scope = schedule.get_scope() - #if scope is not None: - # schedule_element.children.append(build_scope(scope)) - - # programmes - for programme in schedule.programmes: - programme_element = Element(0x1c) - programme_element.attributes.append(Attribute(0x81, programme.shortcrid, encode_number, 24)) - if programme.crid is not None: - programme_element.attributes.append(Attribute(0x80, programme.crid, encode_string)) - if programme.version is not None: - programme_element.attributes.append(Attribute(0x82, programme.version, encode_number, 16)) - if programme.recommendation: - programme_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) # hardcoded to 'yes' - # names - for name in programme.names: - child = build_name(name) - programme_element.children.append(child) - # descriptions - for description in programme.descriptions: - child = build_description(description) - programme_element.children.append(child) - # locations - for location in programme.locations: - child = build_location(location) - programme_element.children.append(child) - # media - if programme.media: programme_element.children.append(build_mediagroup(programme.media)) - # genre - for genre in programme.genres: - child = build_genre(genre) - programme_element.children.append(child) - # membership - for membership in programme.memberships: - child = build_membership(membership) - programme_element.children.append(child) - # link - for link in programme.links: - child = build_link(link) - programme_element.children.append(child) - # events - for event in programme.events: - child = build_programme_event(event) - programme_element.children.append(child) - - schedule_element.children.append(programme_element) - - return schedule_element - - -def build_scope(scope): - scope_element = Element(0x24) - scope_element.attributes.append(Attribute(0x80, scope.start, encode_timepoint)) - scope_element.attributes.append(Attribute(0x81, scope.end, encode_timepoint)) - for bearer in scope.bearers: - service_scope_element = Element(0x25) - service_scope_element.attributes.append(Attribute(0x80, bearer, encode_bearer)) - scope_element.children.append(service_scope_element) - return scope_element - -def build_name(name): - name_element = None - if isinstance(name, ShortName): name_element = Element(0x10) - elif isinstance(name, MediumName): name_element = Element(0x11) - elif isinstance(name, LongName): name_element = Element(0x12) - name_element.cdata = CData(name.text) - if name.language is not None and name.language is not DEFAULT_LANGUAGE: # TODO this should do a comparison with the language of the document - name_element.attributes.append(Attribute(0x80, name.language, encode_string)) - return name_element - -def build_location(location): - location_element = Element(0x19) - for time in location.times: - location_element.children.append(build_time(time)) - for bearer in location.bearers: - bearer_element = Element(0x2d) - bearer_element.attributes.append(Attribute(0x80, bearer, encode_bearer)) - location_element.children.append(bearer_element) - return location_element - -def build_time(time): - time_element = None - if isinstance(time, Time): - time_element = Element(0x2c) - time_element.attributes.append(Attribute(0x80, time.billed_time, encode_timepoint)) - if time.actual_time is not None: - time_element.attributes.append(Attribute(0x82, time.actual_time, encode_timepoint)) - if time.actual_duration is not None: - time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) - time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) - elif isinstance(time, RelativeTime): - time_element = Element(0x2f) - time_element.attributes.append(Attribute(0x80, time.billed_offset.seconds, encode_number, 16)) - time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) - if time.actual_offset is not None: - time_element.attributes.append(Attribute(0x82, time.actual_offset.seconds, encode_number, 16)) - if time.actual_duration is not None: - time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) - return time_element - -def build_description(description): - mediagroup_element = Element(0x13) - if isinstance(description, ShortDescription): - description_element = Element(0x1a) - description_element.cdata = CData(description.text) - elif isinstance(description, LongDescription): - description_element = Element(0x1b) - description_element.cdata = CData(description.text) - if description.language is not None and description.language is not DEFAULT_LANGUAGE: # TODO this should do a comparison with the language of the document - description_element.attributes.append(Attribute(0x80, description.language, encode_string)) - mediagroup_element.children.append(description_element) - return mediagroup_element - -def build_mediagroup(all_media): - mediagroup_element = Element(0x13) - - for media in all_media : - - if not isinstance(media, Multimedia): - raise ValueError('object must be of type %s (is %s)' % (Multimedia.__name__, type(media))) - - media_element = Element(0x2b) - - if media.content is not None: - media_element.attributes.append(Attribute(0x80, media.content, encode_string)) - if media.url is not None: - media_element.attributes.append(Attribute(0x82, media.url, encode_string)) - if media.type == Multimedia.LOGO_UNRESTRICTED: - media_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) - if media.width: media_element.attributes.append(Attribute(0x84, media.width, encode_number, 16)) - if media.height: media_element.attributes.append(Attribute(0x85, media.height, encode_number, 16)) - if media.type == Multimedia.LOGO_COLOUR_SQUARE: - media_element.attributes.append(Attribute(0x83, 0x04, encode_number, 8)) - if media.type == Multimedia.LOGO_COLOUR_RECTANGLE: - media_element.attributes.append(Attribute(0x83, 0x06, encode_number, 8)) - - mediagroup_element.children.append(media_element) - - return mediagroup_element - -def build_genre(genre): - genre_element = Element(0x14) - genre_element.attributes.append(Attribute(0x80, genre.href, encode_string)) - return genre_element - -def build_membership(membership): - membership_element = Element(0x17) - if membership.crid is not None: - membership_element.attributes.append(Attribute(0x80, membership.crid, encode_string)) - membership_element.attributes.append(Attribute(0x81, membership.shortcrid, encode_number, 24)) - if membership.index is not None: - membership_element.attributes.append(0x82, membership.index, encode_number, 16) - return membership_element - -def build_link(link): - link_element = Element(0x18) - link_element.attributes.append(Attribute(0x80, link.uri, encode_string)) - if link.description is not None: - link_element.attributes.append(Attribute(0x83, link.description, encode_string)) - if link.content is not None: - link_element.attributes.append(Attribute(0x81, link.content, encode_string)) - if link.expiry is not None: - link_element.attributes.append(Attribute(0x84, link.expiry, encode_timepoint)) - return link_element - -def build_programme_event(event): - event_element = Element(0x2e) - if event.crid is not None: - event_element.attributes.append(Attribute(0x80, event.crid, encode_string)) - event_element.attributes.append(Attribute(0x81, event.shortcrid, encode_number, 24)) - if event.version is not None and event.version > 1: - event_element.attributes.append(Attribute(0x82, event.version, encode_number, 16)) - if event.recommendation is True: - event_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) - # names - for name in event.names: - event_element.children.append(build_name(name)) - # descriptions - for description in event.descriptions: - event_element.children.append(build_description(description)) - # locations - for location in event.locations: - event_element.children.append(build_location(location)) - # media - if event.media: event_element.children.append(build_mediagroup(event.media)) - # genre - for genre in event.genres: - event_element.children.append(build_genre(genre)) - # membership - for membership in event.memberships: - event_element.children.append(build_membership(membership)) - # link - for link in event.links: - event_element.children.append(build_link(link)) - - return event_element - -def build_service(service): - if not isinstance(service, Service): raise ValueError("must be an object of type %s (is %s)" % (Service.__name__, type(service))) - - logger.debug('building service: %s', service) - - service_element = Element(0x28) - - # version - if service.version > 1: service_element.attributes.append(Attribute(0x80, service.version, encode_number, 16)) - - # service IDs - the first in the list is primary, all others secondary - for bearer in service.bearers: - serviceid_element = Element(0x29) - serviceid_element.attributes.append(Attribute(0x80, bearer, encode_bearer)) - service_element.children.append(serviceid_element) - - # names - for name in service.names: - service_element.children.append(build_name(name)) - - # descriptions - for description in service.descriptions: - service_element.children.append(build_description(description)) - - # media - if service.media: service_element.children.append(build_mediagroup(service.media)) - - # genre - for genre in service.genres: - service_element.children.append(build_genre(genre)) - - # language TODO - - # keywords - if len(service.keywords): - service_element.children.append(build_keywords(service.keywords)) - - # links TODO - - # radiodns lookup - if service.lookup: - from urllib.parse import urlparse - url = urlparse(str(service.lookup)) - lookup_element = Element(0x31) - lookup_element.attributes.append(Attribute(0x80, url.netloc, encode_string)) - lookup_element.attributes.append(Attribute(0x81, url.path[1:], encode_string)) - service_element.children.append(lookup_element) - - return service_element - -def build_keywords(keywords): - keywords_element = Element(0x16) # TODO encode langauges - keywords_element.cdata = CData(",".join(keywords)) - return keywords_element - -def build_ensemble(ensemble, services): - logger.debug('building ensemble: %s', ensemble) - ensemble_element = Element(0x26) - - ensemble_element.attributes.append(Attribute(0x80, (ensemble.ecc, ensemble.eid), encode_ensembleid)) - if ensemble.version > 1: ensemble_element.attributes.append(Attribute(0x81, ensemble.version, encode_number, 16)) - - # names - for name in ensemble.names: - ensemble_element.children.append(build_name(name)) - - # descriptions - for description in ensemble.descriptions: - event_element.children.append(build_description(description)) - - # media - if ensemble.media: ensemble_element.children.append(build_mediagroup(ensemble.media)) - - # keywords - - # links - - # services - for service in services: - service_element = build_service(service) - ensemble_element.children.append(service_element) - - return ensemble_element - -token_table_pattern = re.compile('([\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x0b\\x0c\\x0e\\x0f\\x10\\x11\\x12\\x13])') -def apply_token_table(val, e): - x = e - while x: - if hasattr(x, 'tokens'): - tokens = x.tokens - matcher = re.findall(token_table_pattern, val) - if matcher: - for group in matcher: - logger.debug('replacing 0x%02x with %s', ord(group), tokens[ord(group)]) - val = val.replace(group, tokens[ord(group)]) - matcher = re.search(token_table_pattern, val) - if matcher: - logger.warning('%d tokens (%s) still remain in string "%s" from table: %s', len(matcher.groups()), matcher.groups(), val, tokens) - break - elif hasattr(x, 'parent'): - x = x.parent - else: - break - return val - -def print_info(e): - print(e.attributes) - print(e.children) - print(e.cdata) - -def parse_time(e): - billed_time = e.get_attributes(0x80)[0].value - billed_duration = e.get_attributes(0x81)[0].value - actual_time = None - if e.has_attribute(0x82): actual_time = e.get_attributes(0x82)[0].value - actual_duration = None - if e.has_attribute(0x82): actual_duration = e.get_attributes(0x83)[0].value - time = Time(billed_time, billed_duration, actual_time, actual_duration) - return time - -def parse_bearer(e): - id = e.get_attributes(0x80)[0].value - bearer = Bearer(id) - return bearer - - -def parse_location(e): - - location = Location() - - # times - for c in e.get_children(0x2c): - location.times.append(parse_time(c)) - - # bearer - for c in e.get_children(0x2d): - location.bearers.append(parse_bearer(c)) - - # apply a default content ID - if not len(location.bearers): - x = e - while x: - if hasattr(x, 'default_contentid'): - default_contentid = x.default_contentid - location.bearers.append(default_contentid) - break - elif hasattr(x, 'parent'): - x = x.parent - else: - break - #if not len(location.bearers): - # raise ValueError('location has no bearers and no default content ID is defined') - - return location - -def parse_programme(e): - - crid = e.get_attributes(0x80)[0].value - shortid = e.get_attributes(0x81)[0].value - programme = Programme(crid, shortid) - - # names - for c in e.get_children(0x10): - val = apply_token_table(c.cdata.value, e) - name = ShortName(val) - if(c.has_attribute(0x80)): name.language = c.get_attributes(0x80)[0].value - programme.names.append(name) - for c in e.get_children(0x11): - val = apply_token_table(c.cdata.value, e) - name = MediumName(val) - if(c.has_attribute(0x80)): name.language = c.get_attributes(0x80)[0].value - programme.names.append(name) - for c in e.get_children(0x12): - val = apply_token_table(c.cdata.value, e) - name = LongName(val) - if(c.has_attribute(0x80)): name.language = c.get_attributes(0x80)[0].value - programme.names.append(name) - - # media - for c in e.get_children(0x13): - # short description - for d in c.get_children(0x1a): - val = apply_token_table(d.cdata.value, e) - description = ShortDescription(val) - if(d.has_attribute(0x80)): description.language = d.get_attributes(0x80)[0].value - programme.descriptions.append() - # long description - for d in c.get_children(0x1b): - val = apply_token_table(d.cdata.value, e) - description = LongDescription(val) - if(d.has_attribute(0x80)): description.language = d.get_attributes(0x80)[0].value - programme.descriptions.append(description) - # multimedia - for d in c.get_children(0x2b): - url = d.get_attributes(0x82)[0].value - multimedia = Multimedia(url) - if(d.has_attribute(0x80)): multimedia.content = d.get_getattributes(0x80)[0].value # MIME content type - if(d.has_attribute(0x81)): pass # nowhere to store language yet - if(d.has_attribute(0x83)): multimedia.type = d.get_attributes(0x83)[0].value # logo type - if(d.has_attribute(0x84)): multimedia.width = d.get_attributes(0x84)[0].value # logo width - if(d.has_attribute(0x85)): multimedia.height = d.get_attributes(0x85)[0].value # logo height - programme.media.append(multimedia) - - # location - for c in e.get_children(0x19): - programme.locations.append(parse_location(c)) - - return programme - -def parse_schedule(e): - - # scope start, end - - schedule = Schedule() - - # programmes - programme_elements = e.get_children(0x1c) - for p in programme_elements: - programme = parse_programme(p) - schedule.programmes.append(programme) - - return schedule - -def parse_epg(e): - schedule = parse_schedule(e.get_children(0x21)[0]) - info = ProgrammeInfo() - info.schedules.append(schedule) - return info - -def parse_service(e): - logger.debug('parsing service from element: %s', e) - id = e.get_children(0x29)[0].get_attributes(0x80)[0].value - service = Service(id) - - # names - logger.debug('parsing service names') - for c in e.get_children(0x10): - logger.debug('parsing short name from element: %s', c) - val = apply_token_table(c.cdata.value, e) - service.names.append(ShortName(val)) - for c in e.get_children(0x11): - logger.debug('parsing medium name from element: %s', c) - val = apply_token_table(c.cdata.value, e) - service.names.append(MediumName(val)) - for c in e.get_children(0x12): - logger.debug('parsing long name from element: %s', c) - val = apply_token_table(c.cdata.value, e) - service.names.append(LongName(val)) - return service - - -def parse_ensemble(e): - id = e.get_attributes(0x80)[0].value - ensemble = Ensemble(*id) - - # names - for c in e.get_children(0x10): - val = apply_token_table(c.cdata.value, e) - ensemble.names.append(ShortName(val)) - for c in e.get_children(0x11): - val = apply_token_table(c.cdata.value, e) - ensemble.names.append(MediumName(val)) - for c in e.get_children(0x12): - val = apply_token_table(c.cdata.value, e) - ensemble.names.append(LongName(val)) - - # services - services = [] - for c in e.get_children(0x28): - services.append(parse_service(c)) - - return ensemble, services - -def parse_service_information(e): - service_info = ServiceInfo() - if not e.has_child(0x26): - raise ValueError('no ensemble subelement (0x26): children=%s' % e.get_children()) - ensemble, services = parse_ensemble(e.get_children(0x26)[0]) - service_info.services = services - return service_info, ensemble - -def encode_number(i, n): - if not isinstance(i, (int, float, complex)): raise ValueError('value must be a number (%s is %s)' % (i, type(i))) - if not isinstance(n, (int, float, complex)): raise ValueError('bitlength must be a number') - i = int(i) - n = int(n) - return bitarray(tuple((0,1)[i>>j & 1] for j in range(n-1,-1,-1))) - -def bitarray_to_hex(bits): - rows = [] - for i in range(0, len(bits), 256): - rows.append(' '.join(["%02X" %x for x in bits[i:i+256].tobytes()]).strip()) - return '\r\n'.join(rows) - -def hex_to_bitarray(hex): - b = bitarray() - for byte in hex.split(' '): - b.extend(encode_number(int('0x%s' % byte, 16), 8)) - return b - -def bitarray_to_binary(bits): - rows = [] - for i in range(0, len(bits), 256): - bytes = [] - for j in range(i, i+256, 8): - bytes.append(bits[j:j+8].to01()) - rows.append(' '.join(bytes)) - return '\r\n'.join(rows) - -def unmarshall(i): - """Unmarshalls a PI or SI binary file to its respective :class:Epg or :class:ServiceInfo object - - :param i: String or File object to read binary from - :type i: str, file - """ - - logger.debug('unmarshalling object of type: %s', type(i)) - - import io - b = bitarray() - if isinstance(i, io.IOBase): - logger.debug('object is a file') - b.fromfile(i) - else: - logger.debug('object is a string of %d bytes', len(str(i))) - b.frombytes(i) - - e = Element.frombits(b) - logger.debug('unmarshalled element %s', e) - if e.tag == 0x03: - si = parse_service_information(e) - return si - elif e.tag == 0x02: - epg = parse_epg(e) - return epg - else: - raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!') +from spi import * + +from bitarray import bitarray, bits2bytes +import math +import datetime, dateutil.tz +import logging +import sys +from datetime import timedelta + +logger = logging.getLogger("spi.binary") + +class Ensemble: + """ + Describes a DAB ensemble + + :param ecc: Extended Country Code (ECC) + :type ecc: integer + :param eid: Ensemble ID (EId) + :type eid: integer + :param version: Description version + :type version: integer + """ + + def __init__(self, ecc, eid, version=1): + self.ecc = ecc + self.eid = eid + self.names = [] + self.descriptions = [] + self.media = [] + self.keywords = [] + self.links = [] + self.services = [] + self.frequencies = [] + self.version = version + + def __str__(self): + return "%02x.%04x" % (self.ecc, self.eid) + + def __repr__(self): + return '' % str(self) + + +class Element: + + def __init__(self, tag, attributes=None, children=None, cdata=None): + self.tag = tag + self.attributes = (attributes if attributes is not None else []) + self.children = (children if children is not None else []) + self.cdata = cdata + logger.debug('created new element: %s', self) + + def tobytes(self): + logger.debug('rendering element %s', repr(self)) + data = bitarray() + + # encode attributes + for attribute in self.attributes: + logger.debug('rendering attribute: %s', attribute) + try: data += attribute.tobytes() + except Exception as e: + raise ValueError('error rendering attribute %s of %s: %s' % (attribute, self, str(e))).with_traceback(sys.exc_info()[2]) + + # encode children + for child in self.children: + logger.debug('rendering child element: %s', child) + try: data += child.tobytes() + except: + logger.exception('error rendering child %s of %s', child, self) + raise + + # encode CData + if self.cdata is not None: + logger.debug('rendering cdata: %s', self.cdata) + data += self.cdata.tobytes() + + # b0-b7: element tag + bits = encode_number(self.tag, 8) + + # b8-15: element data length (0-253 bytes) + # b16-31: extended element length (254-65536 bytes) + # b16-39: extended element length (65537-16777216 bytes) + datalength = len(data)/8 + if datalength == 0: + raise ValueError('element data length is zero') + if datalength <= 253: + bits += encode_number(datalength, 8) + elif datalength >= 254 and datalength <= 1<<16: + bits += encode_number(0xfe, 8) + bits += encode_number(datalength, 16) + elif datalength > 1<<16 and datalength <= 1<<24: + bits += encode_number(0xff, 8) + bits += encode_number(datalength, 24) + else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) + logger.debug('element %s is rendered with data length: %d bytes: %s', repr(self), datalength, bitarray_to_hex(bits)) + bits += data + return bits + + def __iter__(self): + return iter(self.children) + + def has_child(self, tag): + return len(self.get_children(tag)) + + def get_children(self, tag=None): + if tag is not None: + return [x for x in self.children if x.tag == tag] + return self.children + + def has_attribute(self, tag): + return len(self.get_attributes(tag)) + + def get_attributes(self, tag=None): + if tag is not None: + return [x for x in self.attributes if x.tag == tag] + return self.attributes + + @staticmethod + def frombits(bits): + + # b0-b7: element tag + tag = int(bits[0:8].to01(), 2) + if tag < 0x02 or tag > 0x36: raise ValueError('invalid value for tag: 0x%02x' % tag) + + # b8-15: element data length (0-253 bytes) + # b16-31: extended element length (256-65536 bytes) + # b16-39: extended element length (65537-16777216 bytes) + datalength = int(bits[8:16].to01(), 2) + start = 16 + if datalength == 0xfe: + datalength = int(bits[start:start+16].to01(), 2) + start += 16 + elif datalength == 0xff: + datalength = int(bits[start:start+24].to01(), 2) + start += 24 + data = bits[start : start + (datalength * 8)] + + i = 0 + e = Element(tag) + logger.debug('parsing data of length %d bytes for element with tag 0x%02x', datalength, tag) + while i < data.length(): + logger.debug('now parsing at offset %d / %d', i/8, data.length()/8) + child_tag = int(data[i:i+8].to01(), 2) + child_datalength = int(data[i+8:i+16].to01(), 2) + start = 16 + if child_datalength == 0xfe: + child_datalength = int(data[i+16:i+32].to01(), 2) + start = 32 + elif child_datalength == 0xff: + child_datalength = int(data[i+16:i+40].to01(), 2) + start = 40 + logger.debug('child with tag 0x%02x for parent tag 0x%02x at offset %d has data length of %d bytes', child_tag, tag, i/8, child_datalength) + end = start + (child_datalength * 8) + if i + end > data.length(): + raise ValueError('end of data for element with tag 0x%02x at offset %d requested is beyond length: %d > %d: %s' % (child_tag, i/8, (i + end)/8, data.length() / 8, bitarray_to_hex(data[i:i+64]))) + child_data = data[i + start : i + end] + if child_data.length() < 16*8: logger.debug('child element with tag 0x%02x for parent 0x%02x has data: %s', child_tag, tag, bitarray_to_hex(child_data)) + + # attributes + if child_tag >= 0x80 and child_tag <= 0x87: + logger.debug('parsing child as an attribute') + attribute = Attribute.frombits(tag, data[i:i+end]) + logger.debug('parsed child as an attribute: %s', attribute) + e.attributes.append(attribute) + # token table + elif child_tag == 0x04: + logger.debug('parsing child as a token table') + tokens = decode_tokentable(child_data) + e.tokens = tokens + logger.debug('parsed token table: %s', tokens) + # default content ID + elif child_tag == 0x05: + logger.debug('parsing child as a default content ID') + default_contentid = decode_contentid(child_data) + e.default_contentid = default_contentid + # default language + elif child_tag == 0x06: + logger.debug('parsing child as a default language (not yet implemented)') + pass + # children + elif child_tag >= 0x02 and child_tag <= 0x36: + logger.debug('parsing child as an element') + child = Element.frombits(data[i:i+end]) + child.parent = e + logger.debug('parsed child as an element: %s', child) + e.children.append(child) + # cdata + elif child_tag == 0x01: + logger.debug('parsing child as CDATA') + cdata = CData.frombits(data[i:i+end]) + logger.debug('parsed CDATA: %s', cdata) + e.cdata = cdata + else: + raise ValueError('unknown element 0x%02x under parent 0x%02x' % (child_tag, tag)) + + i += end + + return e + + def __str__(self): + return 'tag=0x%02X, attributes=%s, children=%s, cdata=%s' % (self.tag, self.attributes, self.children, self.cdata) + + def __repr__(self): + return '' % self.tag + +class Attribute: + + def __init__(self, tag, value, f=None, *args, **kwargs): + if not isinstance(tag, int): raise ValueError('tag must be an integer') + self.tag = tag + self.value = value + self.f = f + self.args = args + self.kwargs = kwargs + logger.debug('created new attribute: %s', repr(self)) + + def tobytes(self): + + if not self.f: raise ValueError('cant encode this attribute without an encoding function') + + # encode data + data = None + logger.debug('encoding attribute %s with function %s', self, self.f) + data = self.f(self.value, *self.args, **self.kwargs) + + # b0-b7: tag + bits = encode_number(self.tag, 8) + + # b8-15: element data length (0-253 bytes) + # b16-31: extended element length (256-65536 bytes) + # b16-39: extended element length (65537-16777216 bytes) + datalength = bits2bytes(data.length()) + if datalength <= 253: + bits += encode_number(datalength, 8) + elif datalength >= 254 and datalength <= 1<<16: + tmp = bitarray() + tmp.fromstring(b'\xfe') + bits += tmp + bits += encode_number(datalength, 16) + elif datalength > 1<<16 and datalength <= 1<<24: + tmp = bitarray() + tmp.fromstring(b'\xff') + bits += tmp + bits += encode_number(datalength, 24) + else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) + + bits += data + return bits + + @staticmethod + def frombits(parent, bits): + + # b0-b7: attribute tag + tag = int(bits[0:8].to01(), 2) + + # b8-15: attribute data length (0-253 bytes) + # b16-31: extended attribute length (256-65536 bytes) + # b16-39: extended attribute length (65537-16777216 bytes) + datalength = int(bits[8:16].to01(), 2) + start = 16 + if datalength >= 254 and datalength <= 1<<16: + datalength = int(bits[16:32].to01(), 2) + start = 32 + elif datalength > 1<<16 and datalength <= 1<<24: + datalength = int(bits[16:40].to01(), 2) + start = 40 + elif datalength > 1<<24: + raise ValueError('attribute data length exceeds the maximum allowed by the extended attribute length (24bits): %s > %s' + datalength + " > " + (1<<24)) + data = bits[start:start+(datalength * 8)] + + # decode data + if isinstance(parent, Element): parent_tag = parent.tag + else: parent_tag = int(parent) + if (parent_tag, tag) in [ # integer + (0x02, 0x80), (0x21, 0x80), (0x23, 0x80), (0x23, 0x81), (0x23, 0x82), (0x23, 0x84), (0x25, 0x80), + (0x1c, 0x81), (0x1c, 0x82), (0x1c, 0x87), (0x17, 0x81), (0x17, 0x82), (0x03, 0x80), (0x26, 0x81), + (0x2b, 0x84), (0x2b, 0x85), (0x2e, 0x81) + ]: + logger.debug('decoding tag/attribute 0x%02x/0x%02x as int', parent_tag, tag) + value = int(data.to01(), 2) + elif (parent_tag, tag) in [ # string + (0x14, 0x80), (0x17, 0x80), (0x18, 0x80), (0x18, 0x83), (0x1c, 0x80), (0x20, 0x80), (0x20, 0x82), + (0x21, 0x82), (0x03, 0x82), (0x03, 0x83), (0x2b, 0x80), (0x2b, 0x82), (0x2e, 0x80), (0x31, 0x80), + (0x31, 0x81), (0x29, 0x82), (0x18, 0x81), (0x18, 0x83), (0x10, 0x80), (0x11, 0x80), (0x12, 0x80), + (0x20, 0x86), (0x2a, 0x80), (0x2b, 0x81), (0x1a, 0x80), (0x1b, 0x80), (0x06, 0x80) + ]: + logger.debug('decoding tag/attribute 0x%02x/0x%02x as string', parent_tag, tag) + value = data.tobytes().decode() + elif (parent_tag, tag) in [(0x2c, 0x81), (0x2c, 0x83), (0x2f, 0x80), (0x2f, 0x81)]: # duration + logger.debug('decoding tag/attribute 0x%02x/0x%02x as duration', parent_tag, tag) + value = datetime.timedelta(seconds=int(data.to01(), 2)) + elif (parent_tag, tag) in []: # genre + logger.debug('decoding tag/attribute 0x%02x/0x%02x as genre', parent_tag, tag) + value = decode_genre(data) + elif (parent_tag, tag) in [(0x20, 0x81), (0x21, 0x81), (0x24, 0x80), (0x24, 0x81), (0x2c, 0x80), (0x2c, 0x82), + (0x03, 0x81)]: # time + logger.debug('decoding tag/attribute 0x%02x/0x%02x as timepoint', parent_tag, tag) + value = decode_timepoint(data) + elif (parent_tag, tag) in [(0x25, 0x80), (0x26, 0x80), (0x29, 0x80), (0x2d, 0x80)]: # content ID + logger.debug('decoding tag/attribute 0x%02x/0x%02x as ContentId', parent_tag, tag) + return Attribute(tag, decode_contentid(data)) + elif (parent_tag, tag) in [(0x1c, 0x83), (0x1c, 0x84), (0x03, 0x84), (0x2b, 0x83), (0x2e, 0x83), (0x2e, 0x84)]: # ENUM + try: + value = decode_enum(parent_tag, tag, data) + except: + logger.warning('error decoding enum for parent 0x%02x from tag: 0x%02x - IGNORING for now' % (parent_tag, tag)) + value = data + else: + raise ValueError('dont know how to decode attribute value for parent 0x%02x from tag: 0x%02x' % (parent_tag, tag)) + + return Attribute(tag, value) + + def __str__(self): + return str('0x%x' % self.tag) + + def __repr__(self): + return '' % (self.tag, self.value) + +genre_map = dict( + IntentionCS=1, + FormatCS=2, + ContentCS=3, + IntendedAudienceCS=4, + OriginationCS=5, + ContentAlertCS=6, + MediaTypeCS=7, + AtmosphereCS=8 +) + +def encode_genre(genre): + + segments = genre.href.split(':') + if len(segments) < 6: raise ValueError('genre is incorrectly formatted: %s' % genre) + + bits = bitarray(4) + bits.setall(False) + + # b0-3: RFU(0) + bits += encode_number(0,4) + # b4-7: CS + cs = segments[4] + if cs in list(genre_map.keys()): cs_val = genre_map[cs] + else: raise ValueError('unknown CS in genre: %s' % cs) + bits += encode_number(cs_val, 4) + + # optional schema levels + if len(segments) >= 6: + levels = segments[6].split('.') + for level in levels: + bits += encode_number(int(level), 8) + + return bits + +def decode_genre(bits): + + # b4-7: CS + cs_val = int(bits[4:8].to01(), 2) + if cs_val in list(genre_map.values()): cs = [x[0] for x in list(genre_map.items()) if x[1] == cs_val] + else: raise ValueError('unknown CS value for genre: %d' % cs_val) + + level = '%d' % cs_val + + # optional schema levels + if bits.length() > 8: + i = 8 + while i < bits.length(): + sublevel = int(bits[i:i+8].to01(), 2) + level += '.%d' % sublevel + i += 8 + + return Genre('urn:tva:metadata:cs:ContentCS:2002:%s' % level) + +def encode_string(s): + b = bitarray() + b.frombytes(s.encode()) + return b + +def encode_timepoint(timepoint): + + if timepoint.tzinfo is None: + UTC = datetime.timezone.utc + timepoint = timepoint.replace(tzinfo=UTC) + bits = bitarray(1) + bits.setall(False) + tz_offset = timepoint.utcoffset() if timepoint.utcoffset() is not None else datetime.timedelta(0) + tz_dst = timepoint.dst() if timepoint.dst() is not None else datetime.timedelta(0) + offset = (tz_offset.days * 86400 + tz_offset.seconds) + (tz_dst.days * 86400 + tz_dst.days) + timepoint = timepoint - timedelta(seconds=offset) + + # b0: RFA(0) + + # b1-17: Date + a = (14 - timepoint.month) / 12 + y = timepoint.year + 4800 - a + m = timepoint.month + (12 * a) - 3 + jdn = timepoint.day + ((153 * m) + 2) / 5 + (365 * y) + (y / 4) - (y / 100) + (y / 400) - 32045 + jd = jdn + timepoint.hour / 24 + timepoint.minute / 1440 + timepoint.second / 86400 + mjd = (int)(jd - 2400000.5) + bits += encode_number(mjd, 17) + + # b18: RFA(0) + bits += bitarray('0') + + # b19: LTO Flag + if timepoint.tzinfo is None or (timepoint.utcoffset().days == 0 and timepoint.utcoffset().seconds == 0): + bits += bitarray('0') + else: + bits += bitarray('1') + + # b20: UTC Flag + # b21: UTC - 11 or 27 bits depending on the form + if timepoint.second > 0: + bits += bitarray('1') + bits += encode_number(timepoint.hour, 5) + bits += encode_number(timepoint.minute, 6) + bits += encode_number(timepoint.second, 6) + bits += bitarray('0' * 10) + else: + bits += bitarray('0') + bits += encode_number(timepoint.hour, 5) + bits += encode_number(timepoint.minute, 6) + + # b32/48: LTO + if bits[19]: + bits += bitarray('00') # b49-50: RFA(0) + bits += bitarray('0' if offset > 0 else '1') # b51: LTO sign + bits += encode_number(offset / (60 * 60) * 2, 5) # b52-56: Half hours + + return bits + +def decode_timepoint(bits): + + if not bits.any(): return None # NOW + + mjd = int(bits[1:18].to01(), 2) + date = datetime.datetime.fromtimestamp((mjd - 40587) * 86400) + timepoint = datetime.datetime.combine(date, datetime.time()) + + # parse timezone + if bits[19]: + sign = bits[-6] + half_hours = int(bits[-5:].to01(), 2) + timezone = dateutil.tz.tzoffset(None, half_hours * 30 * 60 * (-1 if sign else 1)) + else: + timezone = dateutil.tz.tzutc() + + # parse date with UTC short form or long form + if bits[20]: + timepoint = timepoint.replace(hour=int(bits[21:26].to01(), 2), + minute=int(bits[26:32].to01(), 2), + second=int(bits[32:38].to01(), 2), + microsecond=int(bits[38:48].to01(), 2) * 1000, + tzinfo=timezone) + else: + timepoint = timepoint.replace(hour=int(bits[21:26].to01(), 2), + minute=int(bits[26:32].to01(), 2), + tzinfo=timezone) + + return timepoint + +def encode_bearer(bearer): + + if isinstance(bearer, DabBearer): + bits = bitarray(4) + bits.setall(False) + + # b0: RFA(0) + + # b1: Ensemble Flag. Indicates whether ECC and EId are contained with the + # Content ID. + # 0 = ECC and EId are not present. The service that is referenced within the + # contentID is transmitted on the same ensemble as this EPG service + # 1 = ECC and EId are present. + if bearer.ecc is not None and bearer.eid is not None: bits[1] = True + + # b2: X-PAD flag. Indicates whether the addressed component is carried in an + # X-PAD channel. + # 0 = Is not carried in an X-PAD channel. + # 1 = Is carried in an X-PAD channel. + if bearer.xpad is not None: bits[2] = True + + # b3: SId encoding flag + # 0 = Audio service (SId is 16bit) + # 1 = Data service (SId is 32bit) + # no audio support right now + + # b4-7: SCIdS + bits += encode_number(bearer.scids, 4) + + # optional next 8 bits: ECC + if bearer.ecc is not None: + bits += encode_number(bearer.ecc, 8) + + # optional next 16 bits: EId + if bearer.eid is not None: + bits += encode_number(bearer.eid, 16) + + # next 16/32 bits: SId + bits += encode_number(bearer.sid, 16) + + # optional next 8 bits: X-PAD extension + if bearer.xpad is not None: + bits += encode_number(bearer.xpad, 8) + + elif isinstance(bearer, IpBearer): + return encode_string(bearer.uri) + else: + raise ValueError('bearer %s not currently supported', bearer) + + return bits + +def encode_ensembleid(params): + + ecc, eid = params + bits = bitarray() + + # b0: ECC + bits += encode_number(ecc, 8) + + # b8: EId + bits += encode_number(eid, 16) + return bits + +def decode_contentid(bits): + + """decodes a ContentId from a bitarray""" + + # b0: RFA(0) + + # b1: Ensemble Flag. Indicates whether ECC and EId are contained with the + # Content ID. + # 0 = ECC and EId are not present. The service that is referenced within the + # contentID is transmitted on the same ensemble as this EPG service + # 1 = ECC and EId are present. + ecc = None + eid = None + sid = None + scids = None + xpad = None + + try: + if bits.length() == 24: # EnsembleId + # ECC, EId + ecc = int(bits[0:8].to01(), 2) + eid = int(bits[8:24].to01(), 2) + return (ecc, eid) + else: + ensemble_flag = bits[1] + xpad_flag = bits[2] + sid_flag = bits[3] + + # SCIdS + scids = int(bits[4:8].to01(), 2) + + # ECC, EId + i = 8 + if ensemble_flag: + ecc = int(bits[8:16].to01(), 2) + eid = int(bits[16:32].to01(), 2) + i = 32 + + # SId + if not sid_flag: + sid = int(bits[i:i+16].to01(), 2) + i += 16 + elif sid_flag: + sid = int(bits[i:i+32].to01(), 2) + i += 32 + + # XPAD + if xpad_flag: + xpad = int(bits[i+3:i+8].to01(), 2) + + return (ecc, eid, sid, scids, xpad) + except: + raise ValueError('error parsing ContentId from data: %s', bitarray_to_hex(bits)) + +def decode_tokentable(bits): + + tokens = {} + + i = 0 + while i < bits.length(): + tag = int(bits[i:i+8].to01(), 2) + length = int(bits[i+8:i+16].to01(), 2) + data = bits[i+16:i+16+(length*8)].tobytes().decode(DEFAULT_ENCODING) + tokens[tag] = data + i += 16 + (length * 8) + return tokens + +"""Map of possible num values and their binary equivalents. + Note that not all the values are currently implemented, which will + cause the decoder to skip over their details""" +enum_values = { + (0x1c, 0x83, 0x01) : False, + (0x1c, 0x83, 0x02) : True, + (0x1c, 0x84, 0x01) : "on-air", + (0x1c, 0x84, 0x02) : "off-air", + (0x2b, 0x83, 0x02) : Multimedia.LOGO_UNRESTRICTED, + (0x2b, 0x83, 0x04) : Multimedia.LOGO_COLOUR_SQUARE, + (0x2b, 0x83, 0x06) : Multimedia.LOGO_COLOUR_RECTANGLE +} + +def decode_enum(parent_tag, tag, bits): + + if bits.length() != 8: raise ValueError('enum data for parent/attribute 0x%02x/0x%02x is of incorrect length: %d bytes' % (parent_tag, tag, bits.length()/8)) + + value = int(bits.to01(), 2) + key = (parent_tag, tag, value) + if key in list(enum_values.keys()): + return enum_values.get(key) + else: + raise NotImplementedError('enum for parent/attribute 0x%02x/0x%02x not implemented' % (parent_tag, tag)) + +class CData: + + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value + + def __repr__(self): + return '' % str(self) + + def tobytes(self): + # b0-b7: element tag + bits = bitarray() + bits.frombytes(b'\x01') + + # b8-15: element data length (0-253 bytes) + # b16-31: extended element length (254-65536 bytes) + # b16-39: extended element length (65537-16777216 bytes) + + datalength = len(self.value.encode()) # ensure we get the right count for the encoding + + if datalength <= 253: + tmp = encode_number(datalength, 8) + bits += tmp + elif datalength >= 254 and datalength <= 1<<16: + tmp = bitarray() + tmp.frombytes(b'\xfe') + bits += tmp + tmp = encode_number(datalength, 16) + bits += tmp + elif datalength > 1<<16 and datalength <= 1<<24: + tmp = bitarray() + tmp.frombytes(b'\xff') + bits += tmp + tmp = encode_number(datalength, 24) + bits += tmp + else: raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) + tmp = bitarray() + tmp.frombytes(self.value.encode()) + bits += tmp + + return bits + + @staticmethod + def frombits(bits): + + # b0-b7: element tag + tag = int(bits[0:8].to01(), 2) + if tag != 0x01: raise ValueError('CData does not have the correct tag: 0x%02x != 0x01', tag) + + # b8-15: element data length (0-253 bytes) + # b16-31: extended element length (256-65536 bytes) + # b16-39: extended element length (65537-16777216 bytes) + datalength = int(bits[8:16].to01(), 2) + start = 16 + if datalength >= 254 and datalength <= 1<<16: + datalength = int(bits[16:32].to01(), 2) + start = 32 + elif datalength > 1<<16 and datalength <= 1<<24: + datalength = int(bits[16:40].to01(), 2) + start = 40 + elif datalength > 1<<24: + raise ValueError('element data length exceeds the maximum allowed by the extended element length (24bits): %s > %s' + datalength + " > " + (1<<24)) + data = bits[start:start+(datalength * 8)] + + return CData(data.tobytes().decode()) + +def marshall(obj, **kwargs): + """Marshalls an :class:Epg or :class:ServiceInfo to its binary document""" + if isinstance(obj, ServiceInfo): return marshall_serviceinfo(obj, kwargs.get('ensemble', None)) + elif isinstance(obj, ProgrammeInfo): return marshall_programmeinfo(obj) + +def marshall_serviceinfo(info, ensemble): + + # serviceInformation + info_element = Element(0x03) + if info.version > 1: info_element.attributes.append(Attribute(0x80, info.version, encode_number, 16)) + if info.created: info_element.attributes.append(Attribute(0x81, info.created, encode_timepoint)) + if info.originator: info_element.attributes.append(Attribute(0x82, info.originator, encode_string)) + if info.provider: info_element.attributes.append(Attribute(0x83, info.provider, encode_string)) + + # default language + default_language_element = Element(0x06) + default_language_element.attributes.append(Attribute(0x80, DEFAULT_LANGUAGE, encode_string)) # TODO make this configurable in a better way + info_element.children.append(default_language_element) + + # ensemble + if ensemble is None: raise ValueError('must specify an ensemble') + ensemble_element = build_ensemble(ensemble, info.services) + + info_element.children.append(ensemble_element) + + return info_element.tobytes() + +def marshall_programmeinfo(info): + + # epg (default type is DAB, so no need to encode) + epg_element = Element(0x02) + + # default language + default_language_element = Element(0x06) + default_language_element.attributes.append(Attribute(0x80, DEFAULT_LANGUAGE, encode_string)) # TODO make this configurable in a better way + epg_element.children.append(default_language_element) + + for schedule in info.schedules: + schedule_element = build_schedule(schedule) + epg_element.children.append(schedule_element) + + return epg_element.tobytes() + +def build_schedule(schedule): + + # schedule + schedule_element = Element(0x21) + if schedule.version is not None and schedule.version > 1: + schedule_element.attributes.append(Attribute(0x80, schedule.version, encode_number, 16)) + schedule_element.attributes.append(Attribute(0x81, schedule.created, encode_timepoint)) + if schedule.originator is not None: + schedule_element.attributes.append(Attribute(0x82, schedule.originator, encode_string)) + + # schedule scope TODO + #scope = schedule.get_scope() + #if scope is not None: + # schedule_element.children.append(build_scope(scope)) + + # programmes + for programme in schedule.programmes: + programme_element = Element(0x1c) + programme_element.attributes.append(Attribute(0x81, programme.shortcrid, encode_number, 24)) + if programme.crid is not None: + programme_element.attributes.append(Attribute(0x80, programme.crid, encode_string)) + if programme.version is not None: + programme_element.attributes.append(Attribute(0x82, programme.version, encode_number, 16)) + if programme.recommendation: + programme_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) # hardcoded to 'yes' + # names + for name in programme.names: + child = build_name(name) + programme_element.children.append(child) + # descriptions + for description in programme.descriptions: + child = build_description(description) + programme_element.children.append(child) + # locations + for location in programme.locations: + child = build_location(location) + programme_element.children.append(child) + # media + if programme.media: programme_element.children.append(build_mediagroup(programme.media)) + # genre + for genre in programme.genres: + child = build_genre(genre) + programme_element.children.append(child) + # membership + for membership in programme.memberships: + child = build_membership(membership) + programme_element.children.append(child) + # link + for link in programme.links: + child = build_link(link) + programme_element.children.append(child) + # events + for event in programme.events: + child = build_programme_event(event) + programme_element.children.append(child) + + schedule_element.children.append(programme_element) + + return schedule_element + + +def build_scope(scope): + scope_element = Element(0x24) + scope_element.attributes.append(Attribute(0x80, scope.start, encode_timepoint)) + scope_element.attributes.append(Attribute(0x81, scope.end, encode_timepoint)) + for bearer in scope.bearers: + service_scope_element = Element(0x25) + service_scope_element.attributes.append(Attribute(0x80, bearer, encode_bearer)) + scope_element.children.append(service_scope_element) + return scope_element + +def build_name(name): + name_element = None + if isinstance(name, ShortName): name_element = Element(0x10) + elif isinstance(name, MediumName): name_element = Element(0x11) + elif isinstance(name, LongName): name_element = Element(0x12) + name_element.cdata = CData(name.text) + if name.language is not None and name.language is not DEFAULT_LANGUAGE: # TODO this should do a comparison with the language of the document + name_element.attributes.append(Attribute(0x80, name.language, encode_string)) + return name_element + +def build_location(location): + location_element = Element(0x19) + for time in location.times: + location_element.children.append(build_time(time)) + for bearer in location.bearers: + bearer_element = Element(0x2d) + bearer_element.attributes.append(Attribute(0x80, bearer, encode_bearer)) + location_element.children.append(bearer_element) + return location_element + +def build_time(time): + time_element = None + if isinstance(time, Time): + time_element = Element(0x2c) + time_element.attributes.append(Attribute(0x80, time.billed_time, encode_timepoint)) + if time.actual_time is not None: + time_element.attributes.append(Attribute(0x82, time.actual_time, encode_timepoint)) + if time.actual_duration is not None: + time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) + time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) + elif isinstance(time, RelativeTime): + time_element = Element(0x2f) + time_element.attributes.append(Attribute(0x80, time.billed_offset.seconds, encode_number, 16)) + time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) + if time.actual_offset is not None: + time_element.attributes.append(Attribute(0x82, time.actual_offset.seconds, encode_number, 16)) + if time.actual_duration is not None: + time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) + return time_element + +def build_description(description): + mediagroup_element = Element(0x13) + if isinstance(description, ShortDescription): + description_element = Element(0x1a) + description_element.cdata = CData(description.text) + elif isinstance(description, LongDescription): + description_element = Element(0x1b) + description_element.cdata = CData(description.text) + if description.language is not None and description.language is not DEFAULT_LANGUAGE: # TODO this should do a comparison with the language of the document + description_element.attributes.append(Attribute(0x80, description.language, encode_string)) + mediagroup_element.children.append(description_element) + return mediagroup_element + +def build_mediagroup(all_media): + mediagroup_element = Element(0x13) + + for media in all_media : + + if not isinstance(media, Multimedia): + raise ValueError('object must be of type %s (is %s)' % (Multimedia.__name__, type(media))) + + media_element = Element(0x2b) + + if media.content is not None: + media_element.attributes.append(Attribute(0x80, media.content, encode_string)) + if media.url is not None: + media_element.attributes.append(Attribute(0x82, media.url, encode_string)) + if media.type == Multimedia.LOGO_UNRESTRICTED: + media_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) + if media.width: media_element.attributes.append(Attribute(0x84, media.width, encode_number, 16)) + if media.height: media_element.attributes.append(Attribute(0x85, media.height, encode_number, 16)) + if media.type == Multimedia.LOGO_COLOUR_SQUARE: + media_element.attributes.append(Attribute(0x83, 0x04, encode_number, 8)) + if media.type == Multimedia.LOGO_COLOUR_RECTANGLE: + media_element.attributes.append(Attribute(0x83, 0x06, encode_number, 8)) + + mediagroup_element.children.append(media_element) + + return mediagroup_element + +def build_genre(genre): + genre_element = Element(0x14) + genre_element.attributes.append(Attribute(0x80, genre.href, encode_string)) + return genre_element + +def build_membership(membership): + membership_element = Element(0x17) + if membership.crid is not None: + membership_element.attributes.append(Attribute(0x80, membership.crid, encode_string)) + membership_element.attributes.append(Attribute(0x81, membership.shortcrid, encode_number, 24)) + if membership.index is not None: + membership_element.attributes.append(0x82, membership.index, encode_number, 16) + return membership_element + +def build_link(link): + link_element = Element(0x18) + link_element.attributes.append(Attribute(0x80, link.uri, encode_string)) + if link.description is not None: + link_element.attributes.append(Attribute(0x83, link.description, encode_string)) + if link.content is not None: + link_element.attributes.append(Attribute(0x81, link.content, encode_string)) + if link.expiry is not None: + link_element.attributes.append(Attribute(0x84, link.expiry, encode_timepoint)) + return link_element + +def build_programme_event(event): + event_element = Element(0x2e) + if event.crid is not None: + event_element.attributes.append(Attribute(0x80, event.crid, encode_string)) + event_element.attributes.append(Attribute(0x81, event.shortcrid, encode_number, 24)) + if event.version is not None and event.version > 1: + event_element.attributes.append(Attribute(0x82, event.version, encode_number, 16)) + if event.recommendation is True: + event_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) + # names + for name in event.names: + event_element.children.append(build_name(name)) + # descriptions + for description in event.descriptions: + event_element.children.append(build_description(description)) + # locations + for location in event.locations: + event_element.children.append(build_location(location)) + # media + if event.media: event_element.children.append(build_mediagroup(event.media)) + # genre + for genre in event.genres: + event_element.children.append(build_genre(genre)) + # membership + for membership in event.memberships: + event_element.children.append(build_membership(membership)) + # link + for link in event.links: + event_element.children.append(build_link(link)) + + return event_element + +def build_service(service): + if not isinstance(service, Service): raise ValueError("must be an object of type %s (is %s)" % (Service.__name__, type(service))) + + logger.debug('building service: %s', service) + + service_element = Element(0x28) + + # version + if service.version > 1: service_element.attributes.append(Attribute(0x80, service.version, encode_number, 16)) + + # service IDs - the first in the list is primary, all others secondary + for bearer in service.bearers: + serviceid_element = Element(0x29) + serviceid_element.attributes.append(Attribute(0x80, bearer, encode_bearer)) + service_element.children.append(serviceid_element) + + # names + for name in service.names: + service_element.children.append(build_name(name)) + + # descriptions + for description in service.descriptions: + service_element.children.append(build_description(description)) + + # media + if service.media: service_element.children.append(build_mediagroup(service.media)) + + # genre + for genre in service.genres: + service_element.children.append(build_genre(genre)) + + # language TODO + + # keywords + if len(service.keywords): + service_element.children.append(build_keywords(service.keywords)) + + # links TODO + + # radiodns lookup + if service.lookup: + from urllib.parse import urlparse + url = urlparse(str(service.lookup)) + lookup_element = Element(0x31) + lookup_element.attributes.append(Attribute(0x80, url.netloc, encode_string)) + lookup_element.attributes.append(Attribute(0x81, url.path[1:], encode_string)) + service_element.children.append(lookup_element) + + return service_element + +def build_keywords(keywords): + keywords_element = Element(0x16) # TODO encode langauges + keywords_element.cdata = CData(",".join(keywords)) + return keywords_element + +def build_ensemble(ensemble, services): + logger.debug('building ensemble: %s', ensemble) + ensemble_element = Element(0x26) + + ensemble_element.attributes.append(Attribute(0x80, (ensemble.ecc, ensemble.eid), encode_ensembleid)) + if ensemble.version > 1: ensemble_element.attributes.append(Attribute(0x81, ensemble.version, encode_number, 16)) + + # names + for name in ensemble.names: + ensemble_element.children.append(build_name(name)) + + # descriptions + for description in ensemble.descriptions: + event_element.children.append(build_description(description)) + + # media + if ensemble.media: ensemble_element.children.append(build_mediagroup(ensemble.media)) + + # keywords + + # links + + # services + for service in services: + service_element = build_service(service) + ensemble_element.children.append(service_element) + + return ensemble_element + +token_table_pattern = re.compile('([\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x0b\\x0c\\x0e\\x0f\\x10\\x11\\x12\\x13])') +def apply_token_table(val, e): + x = e + while x: + if hasattr(x, 'tokens'): + tokens = x.tokens + matcher = re.findall(token_table_pattern, val) + if matcher: + for group in matcher: + logger.debug('replacing 0x%02x with %s', ord(group), tokens[ord(group)]) + val = val.replace(group, tokens[ord(group)]) + matcher = re.search(token_table_pattern, val) + if matcher: + logger.warning('%d tokens (%s) still remain in string "%s" from table: %s', len(matcher.groups()), matcher.groups(), val, tokens) + break + elif hasattr(x, 'parent'): + x = x.parent + else: + break + return val + +def print_info(e): + print(e.attributes) + print(e.children) + print(e.cdata) + +def parse_time(e): + billed_time = e.get_attributes(0x80)[0].value + billed_duration = e.get_attributes(0x81)[0].value + actual_time = None + if e.has_attribute(0x82): actual_time = e.get_attributes(0x82)[0].value + actual_duration = None + if e.has_attribute(0x82): actual_duration = e.get_attributes(0x83)[0].value + time = Time(billed_time, billed_duration, actual_time, actual_duration) + return time + +def parse_bearer(e): + id = e.get_attributes(0x80)[0].value + bearer = Bearer(id) + return bearer + + +def parse_location(e): + + location = Location() + + # times + for c in e.get_children(0x2c): + location.times.append(parse_time(c)) + + # bearer + for c in e.get_children(0x2d): + location.bearers.append(parse_bearer(c)) + + # apply a default content ID + if not len(location.bearers): + x = e + while x: + if hasattr(x, 'default_contentid'): + default_contentid = x.default_contentid + location.bearers.append(default_contentid) + break + elif hasattr(x, 'parent'): + x = x.parent + else: + break + #if not len(location.bearers): + # raise ValueError('location has no bearers and no default content ID is defined') + + return location + +def parse_programme(e): + + crid = e.get_attributes(0x80)[0].value + shortid = e.get_attributes(0x81)[0].value + programme = Programme(crid, shortid) + + # names + for c in e.get_children(0x10): + val = apply_token_table(c.cdata.value, e) + name = ShortName(val) + if(c.has_attribute(0x80)): name.language = c.get_attributes(0x80)[0].value + programme.names.append(name) + for c in e.get_children(0x11): + val = apply_token_table(c.cdata.value, e) + name = MediumName(val) + if(c.has_attribute(0x80)): name.language = c.get_attributes(0x80)[0].value + programme.names.append(name) + for c in e.get_children(0x12): + val = apply_token_table(c.cdata.value, e) + name = LongName(val) + if(c.has_attribute(0x80)): name.language = c.get_attributes(0x80)[0].value + programme.names.append(name) + + # media + for c in e.get_children(0x13): + # short description + for d in c.get_children(0x1a): + val = apply_token_table(d.cdata.value, e) + description = ShortDescription(val) + if(d.has_attribute(0x80)): description.language = d.get_attributes(0x80)[0].value + programme.descriptions.append() + # long description + for d in c.get_children(0x1b): + val = apply_token_table(d.cdata.value, e) + description = LongDescription(val) + if(d.has_attribute(0x80)): description.language = d.get_attributes(0x80)[0].value + programme.descriptions.append(description) + # multimedia + for d in c.get_children(0x2b): + url = d.get_attributes(0x82)[0].value + multimedia = Multimedia(url) + if(d.has_attribute(0x80)): multimedia.content = d.get_getattributes(0x80)[0].value # MIME content type + if(d.has_attribute(0x81)): pass # nowhere to store language yet + if(d.has_attribute(0x83)): multimedia.type = d.get_attributes(0x83)[0].value # logo type + if(d.has_attribute(0x84)): multimedia.width = d.get_attributes(0x84)[0].value # logo width + if(d.has_attribute(0x85)): multimedia.height = d.get_attributes(0x85)[0].value # logo height + programme.media.append(multimedia) + + # location + for c in e.get_children(0x19): + programme.locations.append(parse_location(c)) + + return programme + +def parse_schedule(e): + + # scope start, end + + schedule = Schedule() + + # programmes + programme_elements = e.get_children(0x1c) + for p in programme_elements: + programme = parse_programme(p) + schedule.programmes.append(programme) + + return schedule + +def parse_epg(e): + schedule = parse_schedule(e.get_children(0x21)[0]) + info = ProgrammeInfo() + info.schedules.append(schedule) + return info + +def parse_service(e): + logger.debug('parsing service from element: %s', e) + id = e.get_children(0x29)[0].get_attributes(0x80)[0].value + service = Service(id) + + # names + logger.debug('parsing service names') + for c in e.get_children(0x10): + logger.debug('parsing short name from element: %s', c) + val = apply_token_table(c.cdata.value, e) + service.names.append(ShortName(val)) + for c in e.get_children(0x11): + logger.debug('parsing medium name from element: %s', c) + val = apply_token_table(c.cdata.value, e) + service.names.append(MediumName(val)) + for c in e.get_children(0x12): + logger.debug('parsing long name from element: %s', c) + val = apply_token_table(c.cdata.value, e) + service.names.append(LongName(val)) + return service + + +def parse_ensemble(e): + id = e.get_attributes(0x80)[0].value + ensemble = Ensemble(*id) + + # names + for c in e.get_children(0x10): + val = apply_token_table(c.cdata.value, e) + ensemble.names.append(ShortName(val)) + for c in e.get_children(0x11): + val = apply_token_table(c.cdata.value, e) + ensemble.names.append(MediumName(val)) + for c in e.get_children(0x12): + val = apply_token_table(c.cdata.value, e) + ensemble.names.append(LongName(val)) + + # services + services = [] + for c in e.get_children(0x28): + services.append(parse_service(c)) + + return ensemble, services + +def parse_service_information(e): + service_info = ServiceInfo() + if not e.has_child(0x26): + raise ValueError('no ensemble subelement (0x26): children=%s' % e.get_children()) + ensemble, services = parse_ensemble(e.get_children(0x26)[0]) + service_info.services = services + return service_info, ensemble + +def encode_number(i, n): + if not isinstance(i, (int, float, complex)): raise ValueError('value must be a number (%s is %s)' % (i, type(i))) + if not isinstance(n, (int, float, complex)): raise ValueError('bitlength must be a number') + i = int(i) + n = int(n) + return bitarray(tuple((0,1)[i>>j & 1] for j in range(n-1,-1,-1))) + +def bitarray_to_hex(bits): + rows = [] + for i in range(0, len(bits), 256): + rows.append(' '.join(["%02X" %x for x in bits[i:i+256].tobytes()]).strip()) + return '\r\n'.join(rows) + +def hex_to_bitarray(hex): + b = bitarray() + for byte in hex.split(' '): + b.extend(encode_number(int('0x%s' % byte, 16), 8)) + return b + +def bitarray_to_binary(bits): + rows = [] + for i in range(0, len(bits), 256): + bytes = [] + for j in range(i, i+256, 8): + bytes.append(bits[j:j+8].to01()) + rows.append(' '.join(bytes)) + return '\r\n'.join(rows) + +def unmarshall(i): + """Unmarshalls a PI or SI binary file to its respective :class:Epg or :class:ServiceInfo object + + :param i: String or File object to read binary from + :type i: str, file + """ + + logger.debug('unmarshalling object of type: %s', type(i)) + + import io + b = bitarray() + if isinstance(i, io.IOBase): + logger.debug('object is a file') + b.fromfile(i) + else: + logger.debug('object is a string of %d bytes', len(str(i))) + b.frombytes(i) + + e = Element.frombits(b) + logger.debug('unmarshalled element %s', e) + if e.tag == 0x03: + si = parse_service_information(e) + return si + elif e.tag == 0x02: + epg = parse_epg(e) + return epg + else: + raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!') From bd8e679c132df7db3c5f6ac29b50622969571024 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 15 Mar 2022 20:18:24 +0000 Subject: [PATCH 17/33] amend setup.py to use setup tools --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0e2e79a..70710ca 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -from distutils.core import setup +import sys +from setuptools import setup setup(name='hybridspi', version='0.2.0', From bf97cb1ed5cad5bd64c12a22ac95fb22a1307729 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Wed, 16 Mar 2022 09:40:40 +0000 Subject: [PATCH 18/33] Change bitarray length calculation from var.length() to len(var) --- src/spi/binary/__init__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 5f8638b..48a425c 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -137,8 +137,8 @@ def frombits(bits): i = 0 e = Element(tag) logger.debug('parsing data of length %d bytes for element with tag 0x%02x', datalength, tag) - while i < data.length(): - logger.debug('now parsing at offset %d / %d', i/8, data.length()/8) + while i < len(data): + logger.debug('now parsing at offset %d / %d', i/8, len(data)/8) child_tag = int(data[i:i+8].to01(), 2) child_datalength = int(data[i+8:i+16].to01(), 2) start = 16 @@ -150,10 +150,10 @@ def frombits(bits): start = 40 logger.debug('child with tag 0x%02x for parent tag 0x%02x at offset %d has data length of %d bytes', child_tag, tag, i/8, child_datalength) end = start + (child_datalength * 8) - if i + end > data.length(): - raise ValueError('end of data for element with tag 0x%02x at offset %d requested is beyond length: %d > %d: %s' % (child_tag, i/8, (i + end)/8, data.length() / 8, bitarray_to_hex(data[i:i+64]))) + if i + end > len(data): + raise ValueError('end of data for element with tag 0x%02x at offset %d requested is beyond length: %d > %d: %s' % (child_tag, i/8, (i + end)/8, len(data) / 8, bitarray_to_hex(data[i:i+64]))) child_data = data[i + start : i + end] - if child_data.length() < 16*8: logger.debug('child element with tag 0x%02x for parent 0x%02x has data: %s', child_tag, tag, bitarray_to_hex(child_data)) + if len(child_data) < 16*8: logger.debug('child element with tag 0x%02x for parent 0x%02x has data: %s', child_tag, tag, bitarray_to_hex(child_data)) # attributes if child_tag >= 0x80 and child_tag <= 0x87: @@ -228,7 +228,7 @@ def tobytes(self): # b8-15: element data length (0-253 bytes) # b16-31: extended element length (256-65536 bytes) # b16-39: extended element length (65537-16777216 bytes) - datalength = bits2bytes(data.length()) + datalength = bits2bytes(len(data)) if datalength <= 253: bits += encode_number(datalength, 8) elif datalength >= 254 and datalength <= 1<<16: @@ -360,9 +360,9 @@ def decode_genre(bits): level = '%d' % cs_val # optional schema levels - if bits.length() > 8: + if len(bits) > 8: i = 8 - while i < bits.length(): + while i < len(bits): sublevel = int(bits[i:i+8].to01(), 2) level += '.%d' % sublevel i += 8 @@ -538,7 +538,7 @@ def decode_contentid(bits): xpad = None try: - if bits.length() == 24: # EnsembleId + if len(bits) == 24: # EnsembleId # ECC, EId ecc = int(bits[0:8].to01(), 2) eid = int(bits[8:24].to01(), 2) @@ -579,7 +579,7 @@ def decode_tokentable(bits): tokens = {} i = 0 - while i < bits.length(): + while i < len(bits): tag = int(bits[i:i+8].to01(), 2) length = int(bits[i+8:i+16].to01(), 2) data = bits[i+16:i+16+(length*8)].tobytes().decode(DEFAULT_ENCODING) @@ -602,7 +602,7 @@ def decode_tokentable(bits): def decode_enum(parent_tag, tag, bits): - if bits.length() != 8: raise ValueError('enum data for parent/attribute 0x%02x/0x%02x is of incorrect length: %d bytes' % (parent_tag, tag, bits.length()/8)) + if len(bits) != 8: raise ValueError('enum data for parent/attribute 0x%02x/0x%02x is of incorrect length: %d bytes' % (parent_tag, tag, len(bits)/8)) value = int(bits.to01(), 2) key = (parent_tag, tag, value) From bd63d344218cfdbe0ad7375dc8991b567ee325c5 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Wed, 16 Mar 2022 09:42:56 +0000 Subject: [PATCH 19/33] Remove requirement for a specific (older) version of bitarray --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 70710ca..45a1a46 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ keywords=['dab', 'spi', 'hybrid', 'radio'], packages=['spi', 'spi.xml', 'spi.binary'], package_dir = {'' : 'src'}, - install_requires = ['python-dateutil', 'isodate', 'bitarray==1.2.0', 'asciitree'], + install_requires = ['python-dateutil', 'isodate', 'bitarray', 'asciitree'], scripts=['bin/dump_binary'] ) From 5e619a3fda566423b62b6a6efb26bbbbb226243b Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Sun, 3 Apr 2022 14:39:16 +0100 Subject: [PATCH 20/33] Added missing DEFAULT_ENCODING value --- src/spi/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spi/__init__.py b/src/spi/__init__.py index 6274fc9..6c08ad8 100644 --- a/src/spi/__init__.py +++ b/src/spi/__init__.py @@ -26,6 +26,7 @@ MAX_SHORTCRID = 16777215 DEFAULT_LANGUAGE = "en" +DEFAULT_ENCODING = "utf-8" class Text: """Abstract class for textual information""" From 9fedf07a646cb7defc63f7629f442d8eb2a4d3da Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Thu, 28 Apr 2022 06:46:58 +0000 Subject: [PATCH 21/33] Fix parsing of lookup into radiodns fqdn and serviceIdentifier attributes --- src/spi/binary/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 48a425c..63e3ca5 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -971,11 +971,10 @@ def build_service(service): # radiodns lookup if service.lookup: - from urllib.parse import urlparse - url = urlparse(str(service.lookup)) + radiodns = str(service.lookup).partition('/') lookup_element = Element(0x31) - lookup_element.attributes.append(Attribute(0x80, url.netloc, encode_string)) - lookup_element.attributes.append(Attribute(0x81, url.path[1:], encode_string)) + lookup_element.attributes.append(Attribute(0x80, radiodns[0], encode_string)) + lookup_element.attributes.append(Attribute(0x81, radiodns[2], encode_string)) service_element.children.append(lookup_element) return service_element From e557608208bf79fce3b661d1feecff7fa0163880 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Thu, 28 Apr 2022 08:44:20 +0000 Subject: [PATCH 22/33] Use jdcal to calculate Modified Julian Dates correctly --- setup.py | 2 +- src/spi/binary/__init__.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 45a1a46..b7eaa85 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ keywords=['dab', 'spi', 'hybrid', 'radio'], packages=['spi', 'spi.xml', 'spi.binary'], package_dir = {'' : 'src'}, - install_requires = ['python-dateutil', 'isodate', 'bitarray', 'asciitree'], + install_requires = ['python-dateutil', 'isodate', 'bitarray', 'asciitree','jdcal'], scripts=['bin/dump_binary'] ) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 63e3ca5..ec757b5 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -5,6 +5,7 @@ import datetime, dateutil.tz import logging import sys +from jdcal import gcal2jd from datetime import timedelta logger = logging.getLogger("spi.binary") @@ -389,12 +390,7 @@ def encode_timepoint(timepoint): # b0: RFA(0) # b1-17: Date - a = (14 - timepoint.month) / 12 - y = timepoint.year + 4800 - a - m = timepoint.month + (12 * a) - 3 - jdn = timepoint.day + ((153 * m) + 2) / 5 + (365 * y) + (y / 4) - (y / 100) + (y / 400) - 32045 - jd = jdn + timepoint.hour / 24 + timepoint.minute / 1440 + timepoint.second / 86400 - mjd = (int)(jd - 2400000.5) + mjd = int((gcal2jd(timepoint.year,timepoint.month,timepoint.day))[1]) bits += encode_number(mjd, 17) # b18: RFA(0) From fab78853ae08063fbe5ccf4d63298c49215a3a9e Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Thu, 28 Apr 2022 17:46:32 +0000 Subject: [PATCH 23/33] Minor change of code order Just shifting two of the billedTime/billedDuration elements together for legibility --- src/spi/binary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index ec757b5..cbf4281 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -814,11 +814,11 @@ def build_time(time): if isinstance(time, Time): time_element = Element(0x2c) time_element.attributes.append(Attribute(0x80, time.billed_time, encode_timepoint)) + time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) if time.actual_time is not None: time_element.attributes.append(Attribute(0x82, time.actual_time, encode_timepoint)) if time.actual_duration is not None: - time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) - time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) + time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) elif isinstance(time, RelativeTime): time_element = Element(0x2f) time_element.attributes.append(Attribute(0x80, time.billed_offset.seconds, encode_number, 16)) From e1761972d5b98514a8099e9b43a6dfbce9935dd4 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 3 May 2022 16:13:25 +0000 Subject: [PATCH 24/33] Added in scope handling Scope specified in PI files was previously not parsed or encoded, but it is now --- src/spi/binary/__init__.py | 8 ++++---- src/spi/xml/__init__.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index ec757b5..ded88ce 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -728,10 +728,10 @@ def build_schedule(schedule): if schedule.originator is not None: schedule_element.attributes.append(Attribute(0x82, schedule.originator, encode_string)) - # schedule scope TODO - #scope = schedule.get_scope() - #if scope is not None: - # schedule_element.children.append(build_scope(scope)) + # schedule scope + scope = schedule.scope + if scope is not None and scope.start is not None and scope.end is not None: + schedule_element.children.append(build_scope(scope)) # programmes for programme in schedule.programmes: diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 83b43b1..2df1a6b 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -645,6 +645,21 @@ def parse_schedule(scheduleElement, listener): if 'version' in scheduleElement.attrib: schedule.version = int(scheduleElement.attrib['version']) if 'originator' in scheduleElement.attrib: schedule.originator = scheduleElement.attrib['originator'] + scope = Scope() + scopeElement = scheduleElement.find('spi:scope', namespaces) + if scopeElement is not None: + if 'startTime' in scopeElement.attrib and 'endTime' in scopeElement.attrib: + scope.start = isodate.parse_datetime(scopeElement.attrib['startTime']) + scope.end = isodate.parse_datetime(scopeElement.attrib['endTime']) + else: + scope.start = None + scope.end = None + + for serviceScope in scopeElement.findall('spi:serviceScope'): + scope.bearers.append(serviceScope) + + schedule.scope = scope + for programmeElement in scheduleElement.findall('spi:programme', namespaces): schedule.programmes.append(parse_programme(programmeElement, listener)) return schedule From 1c584cfe70b784b21eb8496d716abbc88467cb2c Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Fri, 19 May 2023 18:04:08 +0100 Subject: [PATCH 25/33] Updated the encoding of LANGUAGE on the root Service Information element to be a child, not an xml:lang Attribute. This is in-line with TS 102 371 --- src/spi/binary/__init__.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index dc6b512..e4ffe19 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -43,11 +43,12 @@ def __repr__(self): class Element: - def __init__(self, tag, attributes=None, children=None, cdata=None): + def __init__(self, tag, attributes=None, children=None, cdata=None, rawbits=None): self.tag = tag self.attributes = (attributes if attributes is not None else []) self.children = (children if children is not None else []) self.cdata = cdata + self.rawbits = rawbits logger.debug('created new element: %s', self) def tobytes(self): @@ -73,7 +74,12 @@ def tobytes(self): if self.cdata is not None: logger.debug('rendering cdata: %s', self.cdata) data += self.cdata.tobytes() - + + # encode Rawbits + if self.rawbits is not None: + logger.debug('rendering rawbits: %s', self.rawbits) + data += self.rawbits + # b0-b7: element tag bits = encode_number(self.tag, 8) @@ -176,6 +182,7 @@ def frombits(bits): # default language elif child_tag == 0x06: logger.debug('parsing child as a default language (not yet implemented)') + print('trying to decode the language tag') pass # children elif child_tag >= 0x02 and child_tag <= 0x36: @@ -691,7 +698,9 @@ def marshall_serviceinfo(info, ensemble): # default language default_language_element = Element(0x06) - default_language_element.attributes.append(Attribute(0x80, DEFAULT_LANGUAGE, encode_string)) # TODO make this configurable in a better way + language = bitarray() + language.frombytes(DEFAULT_LANGUAGE.encode('utf-8')) + default_language_element.rawbits = language info_element.children.append(default_language_element) # ensemble @@ -814,11 +823,11 @@ def build_time(time): if isinstance(time, Time): time_element = Element(0x2c) time_element.attributes.append(Attribute(0x80, time.billed_time, encode_timepoint)) - time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) if time.actual_time is not None: time_element.attributes.append(Attribute(0x82, time.actual_time, encode_timepoint)) if time.actual_duration is not None: - time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) + time_element.attributes.append(Attribute(0x83, time.actual_duration.seconds, encode_number, 16)) + time_element.attributes.append(Attribute(0x81, time.billed_duration.seconds, encode_number, 16)) elif isinstance(time, RelativeTime): time_element = Element(0x2f) time_element.attributes.append(Attribute(0x80, time.billed_offset.seconds, encode_number, 16)) From 665920ad05124e9e034b177a8b6f6578d50943da Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 22 May 2023 18:08:47 +0000 Subject: [PATCH 26/33] Amended construction of mediaDescription elements so they only have one media element in each one --- src/spi/binary/__init__.py | 53 ++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index e4ffe19..5390fba 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -765,7 +765,8 @@ def build_schedule(schedule): child = build_location(location) programme_element.children.append(child) # media - if programme.media: programme_element.children.append(build_mediagroup(programme.media)) + for media in programme.media: + programme_element.children.append(build_mediagroup(programme.media)) # genre for genre in programme.genres: child = build_genre(genre) @@ -851,30 +852,29 @@ def build_description(description): mediagroup_element.children.append(description_element) return mediagroup_element -def build_mediagroup(all_media): +def build_mediagroup(media): + mediagroup_element = Element(0x13) - - for media in all_media : - - if not isinstance(media, Multimedia): - raise ValueError('object must be of type %s (is %s)' % (Multimedia.__name__, type(media))) - - media_element = Element(0x2b) + + if not isinstance(media, Multimedia): + raise ValueError('object must be of type %s (is %s)' % (Multimedia.__name__, type(media))) + + media_element = Element(0x2b) - if media.content is not None: - media_element.attributes.append(Attribute(0x80, media.content, encode_string)) - if media.url is not None: - media_element.attributes.append(Attribute(0x82, media.url, encode_string)) - if media.type == Multimedia.LOGO_UNRESTRICTED: - media_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) - if media.width: media_element.attributes.append(Attribute(0x84, media.width, encode_number, 16)) - if media.height: media_element.attributes.append(Attribute(0x85, media.height, encode_number, 16)) - if media.type == Multimedia.LOGO_COLOUR_SQUARE: - media_element.attributes.append(Attribute(0x83, 0x04, encode_number, 8)) - if media.type == Multimedia.LOGO_COLOUR_RECTANGLE: - media_element.attributes.append(Attribute(0x83, 0x06, encode_number, 8)) + if media.content is not None: + media_element.attributes.append(Attribute(0x80, media.content, encode_string)) + if media.url is not None: + media_element.attributes.append(Attribute(0x82, media.url, encode_string)) + if media.type == Multimedia.LOGO_UNRESTRICTED: + media_element.attributes.append(Attribute(0x83, 0x02, encode_number, 8)) + if media.width: media_element.attributes.append(Attribute(0x84, media.width, encode_number, 16)) + if media.height: media_element.attributes.append(Attribute(0x85, media.height, encode_number, 16)) + if media.type == Multimedia.LOGO_COLOUR_SQUARE: + media_element.attributes.append(Attribute(0x83, 0x04, encode_number, 8)) + if media.type == Multimedia.LOGO_COLOUR_RECTANGLE: + media_element.attributes.append(Attribute(0x83, 0x06, encode_number, 8)) - mediagroup_element.children.append(media_element) + mediagroup_element.children.append(media_element) return mediagroup_element @@ -922,7 +922,8 @@ def build_programme_event(event): for location in event.locations: event_element.children.append(build_location(location)) # media - if event.media: event_element.children.append(build_mediagroup(event.media)) + for media in event.media: + event_element.children.append(build_mediagroup(event.media)) # genre for genre in event.genres: event_element.children.append(build_genre(genre)) @@ -960,7 +961,8 @@ def build_service(service): service_element.children.append(build_description(description)) # media - if service.media: service_element.children.append(build_mediagroup(service.media)) + for media in service.media: + service_element.childen.append(build_mediagroup(service.media) # genre for genre in service.genres: @@ -1005,7 +1007,8 @@ def build_ensemble(ensemble, services): event_element.children.append(build_description(description)) # media - if ensemble.media: ensemble_element.children.append(build_mediagroup(ensemble.media)) + for media in ensemble.media: + ensemble_element.children.append(build_mediagroup(ensemble.media)) # keywords From 979066ae3053762cf97bfff4697efb87b080ed90 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 22 May 2023 18:18:58 +0000 Subject: [PATCH 27/33] Builds mediaDescription elements with only a single media element --- src/spi/binary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 5390fba..c433b7b 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -962,8 +962,8 @@ def build_service(service): # media for media in service.media: - service_element.childen.append(build_mediagroup(service.media) - + service_element.childen.append(build_mediagroup(service.media)) + # genre for genre in service.genres: service_element.children.append(build_genre(genre)) From df4d2e43c3ec2085d5fbbc06d6f8fa5c16ae5814 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 22 May 2023 22:17:04 +0000 Subject: [PATCH 28/33] Fixed broken bearer parsing from SI XML files --- src/spi/xml/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 7c220bd..7aaff77 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -743,7 +743,7 @@ def parse_service(service_element, listener): # bearers for child in service_element.findall("spi:bearer", namespaces): - if "id" in child: ervice.bearers.append(parse_bearer(child, listener)) + if "id" in child: service.bearers.append(parse_bearer(child, listener)) # media for media_element in service_element.findall("spi:mediaDescription", namespaces): From a74689b4d230f1b16a25239653a4286f58ab903e Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 22 May 2023 22:40:41 +0000 Subject: [PATCH 29/33] Fixes to broken SI XML parsing --- src/spi/binary/__init__.py | 8 ++++---- src/spi/xml/__init__.py | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index c433b7b..78815d1 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -766,7 +766,7 @@ def build_schedule(schedule): programme_element.children.append(child) # media for media in programme.media: - programme_element.children.append(build_mediagroup(programme.media)) + programme_element.children.append(build_mediagroup(media)) # genre for genre in programme.genres: child = build_genre(genre) @@ -923,7 +923,7 @@ def build_programme_event(event): event_element.children.append(build_location(location)) # media for media in event.media: - event_element.children.append(build_mediagroup(event.media)) + event_element.children.append(build_mediagroup(media)) # genre for genre in event.genres: event_element.children.append(build_genre(genre)) @@ -962,7 +962,7 @@ def build_service(service): # media for media in service.media: - service_element.childen.append(build_mediagroup(service.media)) + service_element.children.append(build_mediagroup(media)) # genre for genre in service.genres: @@ -1008,7 +1008,7 @@ def build_ensemble(ensemble, services): # media for media in ensemble.media: - ensemble_element.children.append(build_mediagroup(ensemble.media)) + ensemble_element.children.append(build_mediagroup(media)) # keywords diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 7aaff77..64c3f7d 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -559,8 +559,12 @@ def parse_bearer(bearer_element): bearer = FmBearer.fromstring(uri) elif uri.startswith('http') or uri.startswith('https'): if "mimeValue" not in bearer_element.attrib: - raise ValueError("missing mimeValue attribute for URL: %s" % uri) - bearer = IpBearer(uri, content=bearer_element.attrib.get('mimeValue')) + bearer = IpBearer(uri) + else: + bearer = IpBearer(uri, content=bearer_element.attrib.get('mimeValue')) + elif uri.startswith('hd'): + bearer = IpBearer("http://null/") + logger.debug('bearer %s is useful for DAB SPI', uri) else: raise ValueError('bearer %s not recognised' % uri) if 'cost' in bearer_element.attrib: @@ -743,7 +747,7 @@ def parse_service(service_element, listener): # bearers for child in service_element.findall("spi:bearer", namespaces): - if "id" in child: service.bearers.append(parse_bearer(child, listener)) + if "id" in child.attrib: service.bearers.append(parse_bearer(child)) # media for media_element in service_element.findall("spi:mediaDescription", namespaces): From 10d350d3733f8f515d3fe0a350fbd32709170628 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Mon, 22 May 2023 23:57:58 +0000 Subject: [PATCH 30/33] Add ensemble label and shortlabel to binary encoding --- src/spi/binary/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spi/binary/__init__.py b/src/spi/binary/__init__.py index 78815d1..1eabb66 100644 --- a/src/spi/binary/__init__.py +++ b/src/spi/binary/__init__.py @@ -22,10 +22,10 @@ class Ensemble: :type version: integer """ - def __init__(self, ecc, eid, version=1): + def __init__(self, ecc, eid, version=1, names=[]): self.ecc = ecc self.eid = eid - self.names = [] + self.names = names self.descriptions = [] self.media = [] self.keywords = [] From a041dd06c42ca83f2957211d1e637eba3702474c Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Tue, 23 May 2023 00:00:45 +0000 Subject: [PATCH 31/33] Added the important word "not" for an HD bearer for DAB SPI --- src/spi/xml/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 64c3f7d..1da5d84 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -564,7 +564,7 @@ def parse_bearer(bearer_element): bearer = IpBearer(uri, content=bearer_element.attrib.get('mimeValue')) elif uri.startswith('hd'): bearer = IpBearer("http://null/") - logger.debug('bearer %s is useful for DAB SPI', uri) + logger.debug('bearer %s is not useful for DAB SPI', uri) else: raise ValueError('bearer %s not recognised' % uri) if 'cost' in bearer_element.attrib: From 46f738a531bdeb0de73861d350000ded61b3dcfc Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Thu, 4 Jul 2024 13:35:44 +0100 Subject: [PATCH 32/33] Support 3.1 and 3.3 SPI Allows parsing of /31 and /33 SPI. Will only write out using /31 namespace. --- src/spi/xml/__init__.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 1da5d84..5c74a5b 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -29,8 +29,6 @@ SCHEMA_XSD = 'spi_31.xsd' XSI_NS = 'http://www.w3.org/2001/XMLSchema-instance' -namespaces = { "spi" : SCHEMA_NS } - logger = logging.getLogger('spi.xml') class MarshallListener: @@ -804,7 +802,13 @@ def unmarshall(i, listener=UnmarshallListener()): doc = parse(d) root = doc.getroot() logger.debug('got root element: %s', root) - + + global SCHEMA_NS, namespaces + + #test first for spi 31 + SCHEMA_NS = 'http://www.worlddab.org/schemas/spi/31' + namespaces = { "spi" : SCHEMA_NS } + if root.tag == '{%s}serviceInformation' % SCHEMA_NS: return parse_serviceinfo(root, listener) elif root.tag == '{%s}epg' % SCHEMA_NS: @@ -814,6 +818,22 @@ def unmarshall(i, listener=UnmarshallListener()): return parse_groupinfo(root, listener) else: raise Exception('epg element does not contain either schedules or programme groups') - else: - raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!') - + + #test next for spi 33 + SCHEMA_NS = 'http://www.worlddab.org/schemas/spi/33' + namespaces = { "spi" : SCHEMA_NS } + + if root.tag == '{%s}serviceInformation' % SCHEMA_NS: + return parse_serviceinfo(root, listener) + elif root.tag == '{%s}epg' % SCHEMA_NS: + if len(root.findall("spi:schedule", namespaces)): + return parse_programmeinfo(root, listener) + if len(root.findall("spi:programmeGroups", namespaces)): + return parse_groupinfo(root, listener) + else: + raise Exception('epg element does not contain either schedules or programme groups') + + + + raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!') + From a370b59481a7bbf1e5777a0c465909fed19dd544 Mon Sep 17 00:00:00 2001 From: Nick Piggott Date: Wed, 24 Jul 2024 14:12:35 +0100 Subject: [PATCH 33/33] Add support for 34 namespace The namespace for SPI version 3.4 and higher was changed to a generic value of http://www.worlddab.org/schemas/spi --- src/spi/xml/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/spi/xml/__init__.py b/src/spi/xml/__init__.py index 5c74a5b..aba152b 100644 --- a/src/spi/xml/__init__.py +++ b/src/spi/xml/__init__.py @@ -808,6 +808,7 @@ def unmarshall(i, listener=UnmarshallListener()): #test first for spi 31 SCHEMA_NS = 'http://www.worlddab.org/schemas/spi/31' namespaces = { "spi" : SCHEMA_NS } + logger.debug('Now parsing the namespace %s', SCHEMA_NS) if root.tag == '{%s}serviceInformation' % SCHEMA_NS: return parse_serviceinfo(root, listener) @@ -822,6 +823,7 @@ def unmarshall(i, listener=UnmarshallListener()): #test next for spi 33 SCHEMA_NS = 'http://www.worlddab.org/schemas/spi/33' namespaces = { "spi" : SCHEMA_NS } + logger.debug('Now parsing the namespace %s', SCHEMA_NS) if root.tag == '{%s}serviceInformation' % SCHEMA_NS: return parse_serviceinfo(root, listener) @@ -833,6 +835,23 @@ def unmarshall(i, listener=UnmarshallListener()): else: raise Exception('epg element does not contain either schedules or programme groups') + #test next for spi 34 + SCHEMA_NS = 'http://www.worlddab.org/schemas/spi' + namespaces = { "spi" : SCHEMA_NS } + logger.debug('Now parsing the namespace %s', SCHEMA_NS) + + if root.tag == '{%s}serviceInformation' % SCHEMA_NS: + logger.debug('Matched root %s and going into parse', SCHEMA_NS) + + return parse_serviceinfo(root, listener) + elif root.tag == '{%s}epg' % SCHEMA_NS: + if len(root.findall("spi:schedule", namespaces)): + return parse_programmeinfo(root, listener) + if len(root.findall("spi:programmeGroups", namespaces)): + return parse_groupinfo(root, listener) + else: + raise Exception('epg element does not contain either schedules or programme groups') + raise Exception('Arrgh! this be neither serviceInformation nor epg - to Davy Jones\' locker with ye!')