From e4a2cbefffa10292ee760a1f124b245113b2d0f1 Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Thu, 16 Oct 2025 22:06:08 +1100 Subject: [PATCH 1/8] setparser: Support nested items when generating MetadataList --- klvdata/setparser.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/klvdata/setparser.py b/klvdata/setparser.py index b4701cd..7351b28 100755 --- a/klvdata/setparser.py +++ b/klvdata/setparser.py @@ -104,18 +104,24 @@ def __str__(self): def MetadataList(self): ''' Return metadata dictionary''' - metadata = {} - def repeat(items, indent=1): + def repeat(items): + metadata = OrderedDict() for item in items: - try: - metadata[item.TAG] = (item.LDSName, item.ESDName, item.UDSName, str(item.value.value)) - except: - None + if not hasattr(item, "TAG"): + # Unknown items don't have a tag, so skip + continue + if hasattr(item, 'items'): - repeat(item.items.values(), indent + 1) - repeat(self.items.values()) - return OrderedDict(metadata) + value = repeat(item.items.values()) + else: + value = str(item.value.value) + metadata[item.TAG] = (item.LDSName, item.ESDName, item.UDSName, value) + + return metadata + + return repeat(self.items.values()) + def structure(self): print(str(type(self))) From 89dd70f94754b136ecd8586cf4801cbca21c0f63 Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Thu, 16 Oct 2025 22:06:29 +1100 Subject: [PATCH 2/8] misb0102: Improve and extend support for MISB0102 parsing --- klvdata/misb0102.py | 215 +++++++++++++++++++++++++++++++++++++++++++- klvdata/misb0601.py | 16 ++-- 2 files changed, 222 insertions(+), 9 deletions(-) diff --git a/klvdata/misb0102.py b/klvdata/misb0102.py index 20895e5..ef24d98 100755 --- a/klvdata/misb0102.py +++ b/klvdata/misb0102.py @@ -23,10 +23,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from klvdata.common import hexstr_to_bytes from klvdata.element import UnknownElement from klvdata.elementparser import BytesElementParser +from klvdata.elementparser import StringElementParser from klvdata.misb0601 import UASLocalMetadataSet from klvdata.setparser import SetParser +from klvdata.streamparser import StreamParser _classifying_country_coding = { b'\x01': 'ISO-3166 Two Letter', @@ -83,7 +86,14 @@ class SecurityLocalMetadataSet(SetParser): Must be a subclass of Element or duck type Element. """ key, name = b'\x30', "Security Local Metadata Set" - key_length = 1 + key_length = 1 + + TAG = 48 + UDSKey = hexstr_to_bytes('06 0E 2B 34 - 02 03 01 01 - 0E 01 03 03 - 02 00 00 00') + LDSName = "Security Local Metadata Set" + ESDName = "" + UDSName = "" + parsers = {} _unknown_element = UnknownElement @@ -99,6 +109,12 @@ class SecurityClassification(BytesElementParser): """ key = b'\x01' + TAG = 1 + UDSKey = "-" + LDSName = "Security Classification" + ESDName = "" + UDSName = "" + _classification = { b'\x01': 'UNCLASSIFIED', b'\x02': 'RESTRICTED', @@ -106,3 +122,200 @@ class SecurityClassification(BytesElementParser): b'\x04': 'SECRET', b'\x05': 'TOP SECRET', } + + +@SecurityLocalMetadataSet.add_parser +class ClassifyingCountryAndReleasingInstructionCCM(BytesElementParser): + """ + """ + key = b'\x02' + TAG = 2 + UDSKey = "-" + LDSName = "Classifying Country And Releasing Instruction Country Coding Method" + ESDName = "" + UDSName = "" + + _classification = _classifying_country_coding + + +@SecurityLocalMetadataSet.add_parser +class ClassifyingCountry(StringElementParser): + """ + """ + key = b'\x03' + TAG = 3 + UDSKey = "-" + LDSName = "Classifying Country" + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class SecuritySCISHIInformation(StringElementParser): + """ + """ + key = b'\x04' + TAG = 4 + UDSKey = "-" + LDSName = 'Security-SCI/SHI Information' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class Caveats(StringElementParser): + """ + """ + key = b'\x05' + TAG = 5 + UDSKey = "-" + LDSName = 'Caveats' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ReleasingInstructions(StringElementParser): + """ + """ + key = b'\x06' + TAG = 6 + UDSKey = "-" + LDSName = 'Releasing Instructions' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ClassifiedBy(StringElementParser): + """ + """ + key = b'\x07' + TAG = 7 + UDSKey = "-" + LDSName = 'Classified By' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class DerivedFrom(StringElementParser): + """ + """ + key = b'\x08' + TAG = 8 + UDSKey = "-" + LDSName = 'Derived From' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ClassificationReason(StringElementParser): + """ + """ + key = b'\x09' + TAG = 9 + UDSKey = "-" + LDSName = 'Classification Reason' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class DeclassificationDate(StringElementParser): + """ + """ + key = b'\x0A' + TAG = 10 + UDSKey = "-" + LDSName = 'Declassification Date' + ESDName = "" + UDSName = "" + min_length, max_length = 8, 8 + + +@SecurityLocalMetadataSet.add_parser +class ClassificationAndMarkingSystem(StringElementParser): + """ + """ + key = b'\x0B' + TAG = 11 + UDSKey = "-" + LDSName = 'Classification And Marking System' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ObjectCountryCodingMethod(BytesElementParser): + """ + """ + key = b'\x0C' + TAG = 12 + UDSKey = "-" + LDSName = 'Object Country Coding Method' + ESDName = "" + UDSName = "" + + _classification = _object_country_coding + + +@SecurityLocalMetadataSet.add_parser +class ObjectCountryCodes(StringElementParser): + """ + """ + key = b'\x0D' + TAG = 13 + UDSKey = "-" + LDSName = 'Object Country Codes' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ClassificationComments(StringElementParser): + """ + """ + key = b'\x0E' + TAG = 14 + UDSKey = "-" + LDSName = 'Classification Comments' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class Version(BytesElementParser): + """ + """ + key = b'\x16' + TAG = 22 + UDSKey = "-" + LDSName = 'Version' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ClassifyingCountryAndReleasingInstructionCCMVD(StringElementParser): + """ + """ + key = b'\x17' + TAG = 23 + UDSKey = "-" + LDSName = 'Classifying Country And Releasing Instruction Courntry Coding Method Version Date' + ESDName = "" + UDSName = "" + + +@SecurityLocalMetadataSet.add_parser +class ClassifyingCountryCodeMethodVersionDate(StringElementParser): + """ + """ + key = b'\x18' + TAG = 24 + UDSKey = "-" + LDSName = 'Classifying Country Code Method Version Date' + ESDName = "" + UDSName = "" diff --git a/klvdata/misb0601.py b/klvdata/misb0601.py index 3bd9793..1c24b79 100755 --- a/klvdata/misb0601.py +++ b/klvdata/misb0601.py @@ -701,14 +701,14 @@ class GenericFlagData01(MappedElementParser): _error = None -# @UASLocalMetadataSet.add_parser -# class SecurityLocalMetadataSet(MappedElementParser): -# key = b'\x30' -# TAG = 48 -# UDSKey = "06 0E 2B 34 02 03 01 01 0E 01 03 03 02 00 00 00" -# LDSName = "Security Local Set" -# ESDName = "" -# UDSName = "Security Local Set" +@UASLocalMetadataSet.add_parser +class SecurityLocalMetadataSet(MappedElementParser): + key = b'\x30' + TAG = 48 + UDSKey = "06 0E 2B 34 02 03 01 01 0E 01 03 03 02 00 00 00" + LDSName = "Security Local Set" + ESDName = "" + UDSName = "Security Local Set" @UASLocalMetadataSet.add_parser From cbf10f868c982bd675f48d099d4b8d7cd9db7e4d Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Thu, 16 Oct 2025 22:06:38 +1100 Subject: [PATCH 3/8] misb0601: Add parser for "MIIS Core Identifier (94)" element --- klvdata/misb0601.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/klvdata/misb0601.py b/klvdata/misb0601.py index 1c24b79..64ae658 100755 --- a/klvdata/misb0601.py +++ b/klvdata/misb0601.py @@ -1292,14 +1292,14 @@ class PlatformSideslipAngleFull(MappedElementParser): units = 'degrees' -#@UASLocalMetadataSet.add_parser -# class MIISCoreIdentifier(StringElementParser): -# key = b'\x5E' -# TAG = 94 -# UDSKey = "06 0E 2B 34 01 01 01 01 0E 01 04 05 03 00 00 00" -# LDSName = "MIIS Core Identifier" -# ESDName = "" -# UDSName = "Motion Imagery Identification System Core" +@UASLocalMetadataSet.add_parser +class MIISCoreIdentifier(BytesElementParser): + key = b'\x5E' + TAG = 94 + UDSKey = "06 0E 2B 34 01 01 01 01 0E 01 04 05 03 00 00 00" + LDSName = "MIIS Core Identifier" + ESDName = "" + UDSName = "Motion Imagery Identification System Core" #@UASLocalMetadataSet.add_parser From da359ed4d0c043ca76bdbb470ce81c1877688c4a Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Thu, 16 Oct 2025 22:19:42 +1100 Subject: [PATCH 4/8] setparser: Handle unknown items when generating MetadataList --- klvdata/misb0102.py | 12 +++++++++++- klvdata/misb0601.py | 12 +++++++++++- klvdata/misbEG0104.py | 18 ++++++++++++++---- klvdata/setparser.py | 14 +++++++++----- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/klvdata/misb0102.py b/klvdata/misb0102.py index ef24d98..e75f6da 100755 --- a/klvdata/misb0102.py +++ b/klvdata/misb0102.py @@ -72,7 +72,17 @@ class UnknownElement(UnknownElement): - pass + @property + def LDSName(self): + return "?" + + @property + def ESDName(self): + return "?" + + @property + def UDSName(self): + return "?" @UASLocalMetadataSet.add_parser diff --git a/klvdata/misb0601.py b/klvdata/misb0601.py index 64ae658..e9ce092 100755 --- a/klvdata/misb0601.py +++ b/klvdata/misb0601.py @@ -35,7 +35,17 @@ class UnknownElement(UnknownElement): - pass + @property + def LDSName(self): + return "?" + + @property + def ESDName(self): + return "?" + + @property + def UDSName(self): + return "?" @StreamParser.add_parser diff --git a/klvdata/misbEG0104.py b/klvdata/misbEG0104.py index efdb57d..0446f78 100644 --- a/klvdata/misbEG0104.py +++ b/klvdata/misbEG0104.py @@ -36,7 +36,17 @@ class UnknownElement(UnknownElement): - pass + @property + def LDSName(self): + return "?" + + @property + def ESDName(self): + return "?" + + @property + def UDSName(self): + return "?" @StreamParser.add_parser @@ -266,7 +276,7 @@ class SensorRelativeAzimuthAngle(IEEE754ElementParser): _range = (0, 360) units = 'degrees' -@UAVBasicUniversalMetadataSet.add_parser +@UAVBasicUniversalMetadataSet.add_parser class SensorRelativeElevationAngle(IEEE754ElementParser): key = hexstr_to_bytes("06 0e 2b 34 01 01 01 01 07 01 10 01 03 00 00 00") TAG = 19 @@ -277,8 +287,8 @@ class SensorRelativeElevationAngle(IEEE754ElementParser): _domain = (-(2 ** 31 - 1), 2 ** 31 - 1) _range = (-180, 180) units = 'degrees' - -@UAVBasicUniversalMetadataSet.add_parser + +@UAVBasicUniversalMetadataSet.add_parser class SlantRange(IEEE754ElementParser): key = hexstr_to_bytes("06 0E 2B 34 01 01 01 01 07 01 08 01 01 00 00 00") TAG = 21 diff --git a/klvdata/setparser.py b/klvdata/setparser.py index 7351b28..7c9d320 100755 --- a/klvdata/setparser.py +++ b/klvdata/setparser.py @@ -108,15 +108,19 @@ def MetadataList(self): def repeat(items): metadata = OrderedDict() for item in items: - if not hasattr(item, "TAG"): - # Unknown items don't have a tag, so skip - continue + if hasattr(item, "TAG"): + tag = item.TAG + else: + tag = item.key if hasattr(item, 'items'): value = repeat(item.items.values()) - else: + elif hasattr(item.value, 'value'): value = str(item.value.value) - metadata[item.TAG] = (item.LDSName, item.ESDName, item.UDSName, value) + else: + value = item.value + + metadata[tag] = (item.LDSName, item.ESDName, item.UDSName, value) return metadata From 0b7ab09e818d4f1fad5d9e23cb11db6e9ab56a79 Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Fri, 17 Oct 2025 08:51:42 +1100 Subject: [PATCH 5/8] elementparser: Add EnumElementParser parser --- klvdata/elementparser.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/klvdata/elementparser.py b/klvdata/elementparser.py index 0ff48f2..ee51caf 100755 --- a/klvdata/elementparser.py +++ b/klvdata/elementparser.py @@ -35,7 +35,7 @@ from klvdata.common import float_to_bytes from klvdata.common import str_to_bytes from klvdata.common import ieee754_bytes_to_fp - + class ElementParser(Element, metaclass=ABCMeta): @@ -94,6 +94,29 @@ def __str__(self): return bytes_to_hexstr(self.value, start='0x', sep='') +class EnumElementParser(ElementParser, metaclass=ABCMeta): + def __init__(self, value): + super().__init__(EnumValue(value, self.enumMap)) + + @property + @classmethod + @abstractmethod + def enumMap(cls): + pass + + +class EnumValue(BaseValue): + def __init__(self, value, enumMap): + self.rawValue = value + self.value = enumMap.get(value, f"??? ({self.__bytes__()})") + + def __bytes__(self): + return bytes(self.rawValue) + + def __str__(self): + return self.value + + class DateTimeElementParser(ElementParser, metaclass=ABCMeta): def __init__(self, value): super().__init__(DateTimeValue(value)) From 43d438ef56f52df505b8ed4d51f5f7cbce4ede2e Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Thu, 16 Oct 2025 22:59:41 +1100 Subject: [PATCH 6/8] misb0102: Use EnumElementParser to parse tags 1, 2 and 12 --- klvdata/misb0102.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/klvdata/misb0102.py b/klvdata/misb0102.py index e75f6da..9cdcdef 100755 --- a/klvdata/misb0102.py +++ b/klvdata/misb0102.py @@ -25,7 +25,8 @@ from klvdata.common import hexstr_to_bytes from klvdata.element import UnknownElement -from klvdata.elementparser import BytesElementParser +from klvdata.elementparser import MappedElementParser +from klvdata.elementparser import EnumElementParser from klvdata.elementparser import StringElementParser from klvdata.misb0601 import UASLocalMetadataSet from klvdata.setparser import SetParser @@ -110,7 +111,7 @@ class SecurityLocalMetadataSet(SetParser): @SecurityLocalMetadataSet.add_parser -class SecurityClassification(BytesElementParser): +class SecurityClassification(EnumElementParser): """MISB ST0102 Security Classification value interpretation parser. The Security Classification metadata element contains a value @@ -125,7 +126,7 @@ class SecurityClassification(BytesElementParser): ESDName = "" UDSName = "" - _classification = { + enumMap = { b'\x01': 'UNCLASSIFIED', b'\x02': 'RESTRICTED', b'\x03': 'CONFIDENTIAL', @@ -135,7 +136,7 @@ class SecurityClassification(BytesElementParser): @SecurityLocalMetadataSet.add_parser -class ClassifyingCountryAndReleasingInstructionCCM(BytesElementParser): +class ClassifyingCountryAndReleasingInstructionCCM(EnumElementParser): """ """ key = b'\x02' @@ -145,7 +146,7 @@ class ClassifyingCountryAndReleasingInstructionCCM(BytesElementParser): ESDName = "" UDSName = "" - _classification = _classifying_country_coding + enumMap = _classifying_country_coding @SecurityLocalMetadataSet.add_parser @@ -258,7 +259,7 @@ class ClassificationAndMarkingSystem(StringElementParser): @SecurityLocalMetadataSet.add_parser -class ObjectCountryCodingMethod(BytesElementParser): +class ObjectCountryCodingMethod(EnumElementParser): """ """ key = b'\x0C' @@ -268,7 +269,7 @@ class ObjectCountryCodingMethod(BytesElementParser): ESDName = "" UDSName = "" - _classification = _object_country_coding + enumMap = _object_country_coding @SecurityLocalMetadataSet.add_parser From d182a4c127b40a37e174b594abf2af744f346810 Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Thu, 16 Oct 2025 22:59:50 +1100 Subject: [PATCH 7/8] misb0102: Parse "Version (22)" element as a number --- klvdata/misb0102.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/klvdata/misb0102.py b/klvdata/misb0102.py index 9cdcdef..05860c8 100755 --- a/klvdata/misb0102.py +++ b/klvdata/misb0102.py @@ -297,7 +297,7 @@ class ClassificationComments(StringElementParser): @SecurityLocalMetadataSet.add_parser -class Version(BytesElementParser): +class Version(MappedElementParser): """ """ key = b'\x16' @@ -306,6 +306,11 @@ class Version(BytesElementParser): LDSName = 'Version' ESDName = "" UDSName = "" + _domain = (0, 2**16-1) + _range = (0, 2**16-1) + _error = None + units = 'number' + @SecurityLocalMetadataSet.add_parser From ff0b3aa70c150ee71416d09026a2c45e474edf68 Mon Sep 17 00:00:00 2001 From: Nick Exton Date: Fri, 17 Oct 2025 11:13:03 +1100 Subject: [PATCH 8/8] klvparser: Add support for keys that are BER OIDs --- klvdata/klvparser.py | 21 +++++++++++++++++++-- klvdata/misb0102.py | 2 +- klvdata/misb0601.py | 2 +- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/klvdata/klvparser.py b/klvdata/klvparser.py index 08062d2..26e3809 100755 --- a/klvdata/klvparser.py +++ b/klvdata/klvparser.py @@ -25,7 +25,7 @@ from io import BytesIO from io import IOBase -from klvdata.common import bytes_to_int +from klvdata.common import bytes_to_int, int_to_bytes class KLVParser(object): @@ -42,7 +42,24 @@ def __iter__(self): return self def __next__(self): - key = self.__read(self.key_length) + # A length of None means the key is a variable length BER OID + if self.key_length is None: + oid = 0 + byte = bytes_to_int(self.__read(1)) + # We limit ourselves to four (4) bytes to prevent infinite loops. + for _ in range(4): + isFinal = byte < 128 + oid = (oid << 7) + (byte & 0x7F) + if (isFinal): + break + byte = bytes_to_int(self.__read(1)) + else: + # We've haven't finished reading the key, so any subsequent reads will be invalid. + raise StopIteration + + key = int_to_bytes(oid) + else: + key = self.__read(self.key_length) byte_length = bytes_to_int(self.__read(1)) diff --git a/klvdata/misb0102.py b/klvdata/misb0102.py index 05860c8..542aaa5 100755 --- a/klvdata/misb0102.py +++ b/klvdata/misb0102.py @@ -97,7 +97,7 @@ class SecurityLocalMetadataSet(SetParser): Must be a subclass of Element or duck type Element. """ key, name = b'\x30', "Security Local Metadata Set" - key_length = 1 + key_length = None # A length of None means the key is a variable length BER OID TAG = 48 UDSKey = hexstr_to_bytes('06 0E 2B 34 - 02 03 01 01 - 0E 01 03 03 - 02 00 00 00') diff --git a/klvdata/misb0601.py b/klvdata/misb0601.py index e9ce092..2c2f180 100755 --- a/klvdata/misb0601.py +++ b/klvdata/misb0601.py @@ -54,7 +54,7 @@ class UASLocalMetadataSet(SetParser): """ key = hexstr_to_bytes('06 0E 2B 34 - 02 0B 01 01 – 0E 01 03 01 - 01 00 00 00') name = 'UAS Datalink Local Set' - key_length = 1 + key_length = None # A length of None means the key is a variable length BER OID parsers = {} _unknown_element = UnknownElement