From ba9713332734c22cc8034d94e9ec46a29445338a Mon Sep 17 00:00:00 2001 From: johannes Date: Sun, 3 Apr 2016 23:38:43 +0200 Subject: [PATCH 01/21] adding support for optional keys in templates --- source/lucidity/template.py | 213 +++++++++++++++++++++++------------- 1 file changed, 137 insertions(+), 76 deletions(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index 1393803..2139876 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -20,7 +20,8 @@ class Template(object): _STRIP_EXPRESSION_REGEX = re.compile(r'{(.+?)(:(\\}|.)+?)}') _PLAIN_PLACEHOLDER_REGEX = re.compile(r'{(.+?)}') _TEMPLATE_REFERENCE_REGEX = re.compile(r'{@(?P.+?)}') - + _OPTIONAL_KEY_REGEX = re.compile(r'(\[.*?\])') + ANCHOR_START, ANCHOR_END, ANCHOR_BOTH = (1, 2, 3) RELAXED, STRICT = (1, 2) @@ -118,48 +119,64 @@ def parse(self, path): parsable by this template. ''' - # Construct regular expression for expanded pattern. - regex = self._construct_regular_expression(self.expanded_pattern()) + # Construct a list of regular expression for expanded pattern. + regexes = self._construct_regular_expression(self.expanded_pattern()) # Parse. parsed = {} - - match = regex.search(path) - if match: - data = {} - for key, value in sorted(match.groupdict().items()): - # Strip number that was added to make group name unique. - key = key[:-3] - - # If strict mode enabled for duplicate placeholders, ensure that - # all duplicate placeholders extract the same value. - if self.duplicate_placeholder_mode == self.STRICT: - if key in parsed: - if parsed[key] != value: - raise lucidity.error.ParseError( - 'Different extracted values for placeholder ' - '{0!r} detected. Values were {1!r} and {2!r}.' - .format(key, parsed[key], value) - ) - else: - parsed[key] = value - - # Expand dot notation keys into nested dictionaries. - target = data - - parts = key.split(self._period_code) - for part in parts[:-1]: - target = target.setdefault(part, {}) - - target[parts[-1]] = value - - return data - + for regex in regexes: + match = regex.search(path) + if match: + data = {} + for key, value in sorted(match.groupdict().items()): + # Strip number that was added to make group name unique. + key = key[:-3] + + # If strict mode enabled for duplicate placeholders, ensure that + # all duplicate placeholders extract the same value. + if self.duplicate_placeholder_mode == self.STRICT: + if key in parsed: + if parsed[key] != value: + raise lucidity.error.ParseError( + 'Different extracted values for placeholder ' + '{0!r} detected. Values were {1!r} and {2!r}.' + .format(key, parsed[key], value) + ) + else: + if value: + parsed[key] = value + + # Expand dot notation keys into nested dictionaries. + target = data + + parts = key.split(self._period_code) + for part in parts[:-1]: + target = target.setdefault(part, {}) + + target[parts[-1]] = value + + newData=dict() + for key,value in data.items(): + if value != None: + newData[key]=value + return newData + else: raise lucidity.error.ParseError( 'Path {0!r} did not match template pattern.'.format(path) ) + def missing(self, data, ignoreOptionals=False): + '''Returns a set of missing keys + optional keys are ignored/subtracted + ''' + data_keys = set(data.keys()) + all_key = self.keys().difference(data_keys) + if ignoreOptionals: + return all_key + minus_opt = all_key.difference(self.optional_keys()) + return minus_opt + def format(self, data): '''Return a path formatted by applying *data* to this template. @@ -171,6 +188,13 @@ def format(self, data): format_specification = self._construct_format_specification( self.expanded_pattern() ) + + #remove all missing optional keys from the format spec + format_specification = re.sub( + self._OPTIONAL_KEY_REGEX, + functools.partial(self._remove_optional_keys, data = data), + format_specification + ) return self._PLAIN_PLACEHOLDER_REGEX.sub( functools.partial(self._format, data=data), @@ -189,8 +213,8 @@ def _format(self, match, data): except (TypeError, KeyError): raise lucidity.error.FormatError( - 'Could not format data {0!r} due to missing key {1!r}.' - .format(data, placeholder) + 'Could not format data {0!r} due to missing key(s) {1!r}.' + .format(data, list(self.missing(data))) ) else: @@ -203,6 +227,17 @@ def keys(self): ) return set(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)) + def optional_keys(self): + format_specification = self._construct_format_specification( + self.expanded_pattern() + ) + optional_keys = list() + temp_keys = self._OPTIONAL_KEY_REGEX.findall(format_specification) + for key in temp_keys: + optional_keys.extend(self._PLAIN_PLACEHOLDER_REGEX.findall(key)) + return set(optional_keys) + + def references(self): '''Return unique set of referenced templates in pattern.''' format_specification = self._construct_format_specification( @@ -210,51 +245,77 @@ def references(self): ) return set(self._TEMPLATE_REFERENCE_REGEX.findall(format_specification)) + def _remove_optional_keys(self, match, data): + pattern = match.group(0) + placeholders = list(set(self._PLAIN_PLACEHOLDER_REGEX.findall(pattern))) + for placeholder in placeholders: + if not placeholder in data: + return "" + return pattern[1:-1] + def _construct_format_specification(self, pattern): '''Return format specification from *pattern*.''' return self._STRIP_EXPRESSION_REGEX.sub('{\g<1>}', pattern) + def _construct_expressions(self, pattern): + optionalKeys = re.split(self._OPTIONAL_KEY_REGEX, pattern) + options = [''] + for opt in optionalKeys: + temp_options = [] + if opt == '': + continue + if opt.startswith('['): + temp_options = options[:] + opt = opt.replace("[","").replace("]","") + for option in options: + temp_options.append(option + opt) + options = temp_options + return options + def _construct_regular_expression(self, pattern): '''Return a regular expression to represent *pattern*.''' # Escape non-placeholder components. - expression = re.sub( - r'(?P{(.+?)(:(\\}|.)+?)?})|(?P.+?)', - self._escape, - pattern - ) - - # Replace placeholders with regex pattern. - expression = re.sub( - r'{(?P.+?)(:(?P(\\}|.)+?))?}', - functools.partial( - self._convert, placeholder_count=defaultdict(int) - ), - expression - ) - - if self._anchor is not None: - if bool(self._anchor & self.ANCHOR_START): - expression = '^{0}'.format(expression) - - if bool(self._anchor & self.ANCHOR_END): - expression = '{0}$'.format(expression) - - # Compile expression. - try: - compiled = re.compile(expression) - except re.error as error: - if any([ - 'bad group name' in str(error), - 'bad character in group name' in str(error) - ]): - raise ValueError('Placeholder name contains invalid ' - 'characters.') - else: - _, value, traceback = sys.exc_info() - message = 'Invalid pattern: {0}'.format(value) - raise ValueError, message, traceback #@IgnorePep8 - - return compiled + compiles = list() + + expressions = self._construct_expressions(pattern) + for expression in expressions: + expression = re.sub( + r'(?P{(.+?)(:(\\}|.)+?)?})|(?P.+?)', + self._escape, + expression + ) + + # Replace placeholders with regex pattern. + expression = re.sub( + r'{(?P.+?)(:(?P(\\}|.)+?))?}', + functools.partial( + self._convert, placeholder_count=defaultdict(int) + ), + expression + ) + + if self._anchor is not None: + if bool(self._anchor & self.ANCHOR_START): + expression = '^{0}'.format(expression) + + if bool(self._anchor & self.ANCHOR_END): + expression = '{0}$'.format(expression) + # Compile expression. + try: + compiled = re.compile(expression) + except re.error as error: + if any([ + 'bad group name' in str(error), + 'bad character in group name' in str(error) + ]): + raise ValueError('Placeholder name contains invalid ' + 'characters.') + else: + _, value, traceback = sys.exc_info() + message = 'Invalid pattern: {0}'.format(value) + raise ValueError, message, traceback #@IgnorePep8 + compiles.append(compiled) + return compiles def _convert(self, match, placeholder_count): '''Return a regular expression to represent *match*. @@ -299,7 +360,7 @@ def _escape(self, match): groups = match.groupdict() if groups['other'] is not None: return re.escape(groups['other']) - + return groups['placeholder'] From 6f18214976139a2dadaad86316a56bffe979eb20 Mon Sep 17 00:00:00 2001 From: johannes Date: Mon, 4 Apr 2016 12:15:28 +0200 Subject: [PATCH 02/21] should not react on empty brackets --- source/lucidity/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index 2139876..9bc35d7 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -20,7 +20,7 @@ class Template(object): _STRIP_EXPRESSION_REGEX = re.compile(r'{(.+?)(:(\\}|.)+?)}') _PLAIN_PLACEHOLDER_REGEX = re.compile(r'{(.+?)}') _TEMPLATE_REFERENCE_REGEX = re.compile(r'{@(?P.+?)}') - _OPTIONAL_KEY_REGEX = re.compile(r'(\[.*?\])') + _OPTIONAL_KEY_REGEX = re.compile(r'(\[.+?\])') ANCHOR_START, ANCHOR_END, ANCHOR_BOTH = (1, 2, 3) From adfe8cb22ef30916d31eb542407925b6ba1d93f0 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Fri, 22 Apr 2016 11:41:31 +0200 Subject: [PATCH 03/21] inital commit of key objects --- source/lucidity/__init__.py | 1 + source/lucidity/key.py | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 source/lucidity/key.py diff --git a/source/lucidity/__init__.py b/source/lucidity/__init__.py index 74d1cfe..9df286b 100644 --- a/source/lucidity/__init__.py +++ b/source/lucidity/__init__.py @@ -8,6 +8,7 @@ from ._version import __version__ from .template import Template, Resolver +from .key import Key from .error import ParseError, FormatError, NotFound diff --git a/source/lucidity/key.py b/source/lucidity/key.py new file mode 100644 index 0000000..986415d --- /dev/null +++ b/source/lucidity/key.py @@ -0,0 +1,138 @@ +# :coding: utf-8 +import re +import logging + +class Key(object): + '''Key baseClass + used to store and validate values for templates + a dict needs to be provided + {'name': 'shot', + 'regex': r'^([0-9]){3}$', + 'type': int, + 'padding': '%04d' + } + name and type are the main keys that need to be provided + without them the object cannot initialize + types are "str" and "int" + + ''' + def __init__(self,**kwargs): + super(Key, self).__init__() + if not 'name' in kwargs.keys() or not 'type' in kwargs.keys(): + raise Exception('please provide "name" and "type" to construct a key object') + + self.__value = None + self.__regex = None + self.__padding = None + self.__abstract = '' + + for key, value in kwargs.items(): + if key == 'name': + self.__name = value + if key == 'type': + self.__type = value + if key == 'regex': + self.__regex = re.compile(value) + if key == 'abstract': + self.__abstract = value + if key == 'padding': + if re.match(r'(\%)([0-9]{0,2})(d)', value): + self.__padding = value + else: + raise Exception('provided padding {0} is not a valid padding pattern must be like "%04d"'.format(value)) + + @property + def name(self): + return self.__name + + @property + def type(self): + return self.__type + + @property + def abstract(self): + return self.__abstract + + @property + def padding(self): + return self.__padding + + @property + def regex(self): + return self.__regex + + @property + def value(self): + return self.__value + + def setValue(self, value): + + if self.type == int and type(value) == int and self.padding: + ## we can skip the regex check if the incoming value is an int and we do have a padding + self.__value = self.type(value) + return + + if self.regex: + if re.match(self.regex, value): + self.__value = self.type(value) + return + else: + raise Exception('provided value {0} does not match regex {1} for {2}'.format(value, self.regex.pattern,self.__repr__())) + else: + self.__value = self.type(value) + return + + def __repr__(self): + if self.value: + return ''.format(self.name,str(self)) + else: + return ''.format(self.name) + + def __str__(self): + ''' + used in the format method to fill the keys + ''' + if not self.value and not self.value == 0: + return str(self.name) + if self.type == str: + return str(self.value) + elif self.type == int and self.padding: + return self.padding % self.value + elif self.type == int: + return str(self.value) + + def __cmp__(self,other): + ''' + compare against name + ''' + return cmp(self.name,other) + + +if __name__ == '__main__': + keys = [{'name': 'shot', + 'regex': r'^([0-9]){3}$', + 'type': int, + 'padding': '%04d' + } + , + {'name': 'sequence', + 'regex': r'([0-9]){4}', + 'type': int, + 'padding': '%04d' + } + , + {'name': 'version', + 'regex': r'([0-9]){3}', + 'type': int, + 'padding': '%03d' + } + ] + + foundKeys = list() + for key in keys: + foundKeys.append(Key(**key)) + # + foundKeys[0].setValue('010') + print foundKeys[0].padding + print foundKeys + \ No newline at end of file From d709cd1a30f0a22f234aed00adebccff128374ad Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Fri, 22 Apr 2016 11:42:20 +0200 Subject: [PATCH 04/21] adding support for optional keys and key objects --- source/lucidity/template.py | 283 +++++++++++++++++++++++++----------- 1 file changed, 196 insertions(+), 87 deletions(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index 1393803..e546a80 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -20,15 +20,16 @@ class Template(object): _STRIP_EXPRESSION_REGEX = re.compile(r'{(.+?)(:(\\}|.)+?)}') _PLAIN_PLACEHOLDER_REGEX = re.compile(r'{(.+?)}') _TEMPLATE_REFERENCE_REGEX = re.compile(r'{@(?P.+?)}') - + _OPTIONAL_KEY_REGEX = re.compile(r'(\[.+?\])') + ANCHOR_START, ANCHOR_END, ANCHOR_BOTH = (1, 2, 3) RELAXED, STRICT = (1, 2) - def __init__(self, name, pattern, anchor=ANCHOR_START, - default_placeholder_expression='[\w_.\-]+', - duplicate_placeholder_mode=RELAXED, - template_resolver=None): + def __init__(self, name, pattern, anchor=ANCHOR_BOTH, + default_placeholder_expression='[A-Za-z0-9\-]+', + duplicate_placeholder_mode=STRICT, + template_resolver=None, key_resolver=None): '''Initialise with *name* and *pattern*. *anchor* determines how the pattern is anchored during a parse. A @@ -54,6 +55,7 @@ def __init__(self, name, pattern, anchor=ANCHOR_START, super(Template, self).__init__() self.duplicate_placeholder_mode = duplicate_placeholder_mode self.template_resolver = template_resolver + self.key_resolver = key_resolver self._default_placeholder_expression = default_placeholder_expression self._period_code = '_LPD_' @@ -118,49 +120,82 @@ def parse(self, path): parsable by this template. ''' - # Construct regular expression for expanded pattern. - regex = self._construct_regular_expression(self.expanded_pattern()) - + # Construct a list of regular expression for expanded pattern. + regexes = self._construct_regular_expression(self.expanded_pattern()) # Parse. parsed = {} - - match = regex.search(path) - if match: - data = {} - for key, value in sorted(match.groupdict().items()): - # Strip number that was added to make group name unique. - key = key[:-3] - - # If strict mode enabled for duplicate placeholders, ensure that - # all duplicate placeholders extract the same value. - if self.duplicate_placeholder_mode == self.STRICT: - if key in parsed: - if parsed[key] != value: - raise lucidity.error.ParseError( - 'Different extracted values for placeholder ' - '{0!r} detected. Values were {1!r} and {2!r}.' - .format(key, parsed[key], value) - ) - else: - parsed[key] = value - - # Expand dot notation keys into nested dictionaries. - target = data - - parts = key.split(self._period_code) - for part in parts[:-1]: - target = target.setdefault(part, {}) - - target[parts[-1]] = value - - return data - + for regex in regexes: + match = regex.search(path) + + if match: + data = {} + for key, value in sorted(match.groupdict().items()): + # Strip number that was added to make group name unique. + key = key[:-3] + + # If strict mode enabled for duplicate placeholders, ensure that + # all duplicate placeholders extract the same value. + if self.duplicate_placeholder_mode == self.STRICT: + if key in parsed: + if parsed[key] != value: + raise lucidity.error.ParseError( + 'Different extracted values for placeholder ' + '{0!r} detected. Values were {1!r} and {2!r}.' + .format(key, parsed[key], value) + ) + else: + if value: + parsed[key] = value + + # Expand dot notation keys into nested dictionaries. + target = data + + parts = key.split(self._period_code) + for part in parts[:-1]: + target = target.setdefault(part, {}) + + target[parts[-1]] = value + + newData=dict() + for key,value in data.items(): + if value != None: + newData[key]=value + return newData + else: raise lucidity.error.ParseError( 'Path {0!r} did not match template pattern.'.format(path) ) - def format(self, data): + def missing(self, data, ignoreOptionals=False): + '''Returns a set of missing keys + optional keys are ignored/subtracted + ''' + data_keys = set(data.keys()) + if self.key_resolver: + new_data_keys = list() + for key in data_keys: + if key in self.key_resolver: + new_data_keys.append(self.key_resolver.get(key)) + else: + new_data_keys.append(key) + data_keys = new_data_keys + all_key = self.keys().difference(data_keys) + if ignoreOptionals: + return all_key + minus_opt = all_key.difference(self.optional_keys()) + return minus_opt + + def apply_fields(self,data,abstract=False): + ''' + here for convenience + + :param data: dict of fields + :param abstract: if there are lucidity.key objects with an abstract key the formatting will use the abstract definition + ''' + self.format(data, abstract=abstract) + + def format(self, data, abstract=False): '''Return a path formatted by applying *data* to this template. Raise :py:class:`~lucidity.error.FormatError` if *data* does not @@ -171,26 +206,39 @@ def format(self, data): format_specification = self._construct_format_specification( self.expanded_pattern() ) + + #remove all missing optional keys from the format spec + format_specification = re.sub( + self._OPTIONAL_KEY_REGEX, + functools.partial(self._remove_optional_keys, data = data), + format_specification + ) return self._PLAIN_PLACEHOLDER_REGEX.sub( - functools.partial(self._format, data=data), + functools.partial(self._format, data=data,abstract=abstract), format_specification ) - def _format(self, match, data): + def _format(self, match, data, abstract= False): '''Return value from data for *match*.''' + placeholder = match.group(1) parts = placeholder.split('.') - try: value = data for part in parts: value = value[part] - + if part in self.key_resolver: + key = self.key_resolver.get(part) + key.setValue(value) + value = str(key) + if abstract and key.abstract: + value = str(key.abstract) + except (TypeError, KeyError): raise lucidity.error.FormatError( - 'Could not format data {0!r} due to missing key {1!r}.' - .format(data, placeholder) + 'Could not format data {0!r} due to missing key(s) {1!r}.' + .format(data, list(self.missing(data))) ) else: @@ -201,7 +249,36 @@ def keys(self): format_specification = self._construct_format_specification( self.expanded_pattern() ) - return set(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)) + if not self.key_resolver: + return set(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)) + else: + keys = list() + for key in set(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)): + if key in self.key_resolver: + keys.append(self.key_resolver.get(key)) + else: + keys.append(key) + return set(keys) + + def optional_keys(self): + format_specification = self._construct_format_specification( + self.expanded_pattern() + ) + optional_keys = list() + temp_keys = self._OPTIONAL_KEY_REGEX.findall(format_specification) + for key in temp_keys: + optional_keys.extend(self._PLAIN_PLACEHOLDER_REGEX.findall(key)) + if not self.key_resolver: + return set(optional_keys) + else: + keys = list() + for key in set(optional_keys): + if key in self.key_resolver: + keys.append(self.key_resolver.get(key)) + else: + keys.append(key) + return set(keys) + def references(self): '''Return unique set of referenced templates in pattern.''' @@ -210,51 +287,77 @@ def references(self): ) return set(self._TEMPLATE_REFERENCE_REGEX.findall(format_specification)) + def _remove_optional_keys(self, match, data): + pattern = match.group(0) + placeholders = list(set(self._PLAIN_PLACEHOLDER_REGEX.findall(pattern))) + for placeholder in placeholders: + if not placeholder in data: + return "" + return pattern[1:-1] + def _construct_format_specification(self, pattern): '''Return format specification from *pattern*.''' return self._STRIP_EXPRESSION_REGEX.sub('{\g<1>}', pattern) + def _construct_expressions(self, pattern): + optionalKeys = re.split(self._OPTIONAL_KEY_REGEX, pattern) + options = [''] + for opt in optionalKeys: + temp_options = [] + if opt == '': + continue + if opt.startswith('['): + temp_options = options[:] + opt = opt[1:-1] + for option in options: + temp_options.append(option + opt) + options = temp_options + return options + def _construct_regular_expression(self, pattern): '''Return a regular expression to represent *pattern*.''' # Escape non-placeholder components. - expression = re.sub( - r'(?P{(.+?)(:(\\}|.)+?)?})|(?P.+?)', - self._escape, - pattern - ) - - # Replace placeholders with regex pattern. - expression = re.sub( - r'{(?P.+?)(:(?P(\\}|.)+?))?}', - functools.partial( - self._convert, placeholder_count=defaultdict(int) - ), - expression - ) - - if self._anchor is not None: - if bool(self._anchor & self.ANCHOR_START): - expression = '^{0}'.format(expression) - - if bool(self._anchor & self.ANCHOR_END): - expression = '{0}$'.format(expression) - - # Compile expression. - try: - compiled = re.compile(expression) - except re.error as error: - if any([ - 'bad group name' in str(error), - 'bad character in group name' in str(error) - ]): - raise ValueError('Placeholder name contains invalid ' - 'characters.') - else: - _, value, traceback = sys.exc_info() - message = 'Invalid pattern: {0}'.format(value) - raise ValueError, message, traceback #@IgnorePep8 - - return compiled + compiles = list() + + expressions = self._construct_expressions(pattern) + for expression in expressions: + expression = re.sub( + r'(?P{(.+?)(:(\\}|.)+?)?})|(?P.+?)', + self._escape, + expression + ) + + # Replace placeholders with regex pattern. + expression = re.sub( + r'{(?P.+?)(:(?P(\\}|.)+?))?}', + functools.partial( + self._convert, placeholder_count=defaultdict(int) + ), + expression + ) + + if self._anchor is not None: + if bool(self._anchor & self.ANCHOR_START): + expression = '^{0}'.format(expression) + + if bool(self._anchor & self.ANCHOR_END): + expression = '{0}$'.format(expression) + # Compile expression. + try: + compiled = re.compile(expression) + except re.error as error: + if any([ + 'bad group name' in str(error), + 'bad character in group name' in str(error) + ]): + raise ValueError('Placeholder name contains invalid ' + 'characters.') + else: + _, value, traceback = sys.exc_info() + message = 'Invalid pattern: {0}'.format(value) + raise ValueError, message, traceback #@IgnorePep8 + compiles.append(compiled) + return compiles def _convert(self, match, placeholder_count): '''Return a regular expression to represent *match*. @@ -286,6 +389,12 @@ def _convert(self, match, placeholder_count): ) expression = match.group('expression') + if self.key_resolver: + if placeholder_name[:-3] in self.key_resolver: + #check if there is a regex on the key object + key = self.key_resolver.get(placeholder_name[:-3]) + if key.regex: + expression = key.regex.pattern if expression is None: expression = self._default_placeholder_expression @@ -299,7 +408,7 @@ def _escape(self, match): groups = match.groupdict() if groups['other'] is not None: return re.escape(groups['other']) - + return groups['placeholder'] From 42c35c4afdc2e91d0288c96f45602d244694ad81 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Wed, 18 May 2016 12:14:07 +0200 Subject: [PATCH 05/21] clean up of keys --- source/lucidity/key.py | 68 +++++++++++++----------------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/source/lucidity/key.py b/source/lucidity/key.py index 986415d..ed1e6fe 100644 --- a/source/lucidity/key.py +++ b/source/lucidity/key.py @@ -16,21 +16,17 @@ class Key(object): types are "str" and "int" ''' - def __init__(self,**kwargs): + def __init__(self, name, type,**kwargs): super(Key, self).__init__() - if not 'name' in kwargs.keys() or not 'type' in kwargs.keys(): - raise Exception('please provide "name" and "type" to construct a key object') + self.__name = name + self.__type = type self.__value = None self.__regex = None self.__padding = None self.__abstract = '' for key, value in kwargs.items(): - if key == 'name': - self.__name = value - if key == 'type': - self.__type = value if key == 'regex': self.__regex = re.compile(value) if key == 'abstract': @@ -66,21 +62,24 @@ def value(self): return self.__value def setValue(self, value): - - if self.type == int and type(value) == int and self.padding: - ## we can skip the regex check if the incoming value is an int and we do have a padding - self.__value = self.type(value) - return - - if self.regex: - if re.match(self.regex, value): + if value: + if self.type == int and isinstance(value,int) and self.padding: + ## we can skip the regex check if the incoming value is an int and we do have a padding self.__value = self.type(value) return + + if self.regex: + if re.match(self.regex, value): + if self.abstract: + if value == self.abstract: + return + self.__value = self.type(value) + return + else: + raise Exception('provided value {0} does not match regex {1} for {2}'.format(value, self.regex.pattern,self.__repr__())) else: - raise Exception('provided value {0} does not match regex {1} for {2}'.format(value, self.regex.pattern,self.__repr__())) - else: - self.__value = self.type(value) - return + self.__value = self.type(value) + return def __repr__(self): if self.value: @@ -93,6 +92,8 @@ def __str__(self): used in the format method to fill the keys ''' if not self.value and not self.value == 0: + if self.abstract: + return str(self.abstract) return str(self.name) if self.type == str: return str(self.value) @@ -106,33 +107,4 @@ def __cmp__(self,other): compare against name ''' return cmp(self.name,other) - - -if __name__ == '__main__': - keys = [{'name': 'shot', - 'regex': r'^([0-9]){3}$', - 'type': int, - 'padding': '%04d' - } - , - {'name': 'sequence', - 'regex': r'([0-9]){4}', - 'type': int, - 'padding': '%04d' - } - , - {'name': 'version', - 'regex': r'([0-9]){3}', - 'type': int, - 'padding': '%03d' - } - ] - - foundKeys = list() - for key in keys: - foundKeys.append(Key(**key)) - # - foundKeys[0].setValue('010') - print foundKeys[0].padding - print foundKeys \ No newline at end of file From aa8e650fb611dd9949c813a2f0f67626070ac748 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Wed, 18 May 2016 12:14:35 +0200 Subject: [PATCH 06/21] key resolver defaults to dict so our "is in checks" wont fail --- source/lucidity/template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index 89cd805..c64b04b 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -29,7 +29,7 @@ class Template(object): def __init__(self, name, pattern, anchor=ANCHOR_BOTH, default_placeholder_expression='[A-Za-z0-9\-]+', duplicate_placeholder_mode=STRICT, - template_resolver=None, key_resolver=None): + template_resolver=None, key_resolver={}): '''Initialise with *name* and *pattern*. *anchor* determines how the pattern is anchored during a parse. A From 6a655b5f25e13592c088bc7d499c752ddb224e82 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Wed, 18 May 2016 12:15:13 +0200 Subject: [PATCH 07/21] changed default unit tests to match macke base conf --- test/unit/test_lucidity.py | 2 ++ test/unit/test_template.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/unit/test_lucidity.py b/test/unit/test_lucidity.py index 2bb9ea6..836cffe 100644 --- a/test/unit/test_lucidity.py +++ b/test/unit/test_lucidity.py @@ -4,6 +4,8 @@ import os import operator +import sys +sys.path.append(os.path.join(os.path.dirname(__file__), '..','..', 'source')) import pytest diff --git a/test/unit/test_template.py b/test/unit/test_template.py index d7139e4..e1ca7bd 100644 --- a/test/unit/test_template.py +++ b/test/unit/test_template.py @@ -3,7 +3,9 @@ # :license: See LICENSE.txt. import pytest - +import sys +import os +sys.path.append(os.path.join(os.path.dirname(__file__), '..','..', 'source')) from lucidity import Template, Resolver from lucidity.error import ParseError, FormatError, ResolveError @@ -97,7 +99,7 @@ def test_invalid_pattern(pattern): ]) def test_matching_parse(pattern, path, expected, template_resolver): '''Extract data from matching path.''' - template = Template('test', pattern, template_resolver=template_resolver) + template = Template('test', pattern, template_resolver=template_resolver, duplicate_placeholder_mode=1) data = template.parse(path) assert data == expected From 14dedaebb685e5a17540dbfefb7363b5ccd7044d Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Wed, 18 May 2016 12:15:35 +0200 Subject: [PATCH 08/21] added unit test for key objects --- test/unit/test_key.py | 80 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 test/unit/test_key.py diff --git a/test/unit/test_key.py b/test/unit/test_key.py new file mode 100644 index 0000000..88df7ad --- /dev/null +++ b/test/unit/test_key.py @@ -0,0 +1,80 @@ +# :coding: utf-8 +# :copyright: Copyright (c) 2013 Martin Pengelly-Phillips +# :license: See LICENSE.txt. + +import os +import operator +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '..','..', 'source')) + +import pytest + +import lucidity + +TEST_TEMPLATE_PATH = os.path.join( + os.path.dirname(__file__), '..', 'fixture', 'template' +) +from lucidity import Template +from lucidity import key as TemplateKeys + +@pytest.fixture(scope='session') +def keys(): + '''Register templates.''' + + keys = [ + {'name': 'ver', + 'regex': r'([0-9]){3}', + 'type': int, + 'padding': '%03d' + } + , + {'name': 'asset', + 'regex': r'[a-zA-Z]*', + 'type': str, + } + , + {'name': 'frame', + 'regex': r'([0-9]+|%[0-9]+[di]|[#@?]+)', + 'type': int, + 'abstract': '%04d', + 'padding': '%04d' + } + ] + keyResolver = dict() + for key in keys: + keyResolver[key.get('name')]= TemplateKeys.Key(**key) + return keyResolver + + +@pytest.mark.parametrize(('name','type'), [ + ('version', int), + ('asset', str) +], ids=[ + 'int key', + 'string key' +]) +def test_key(name, type): + '''Construct Key Objects''' + TemplateKeys.Key(**locals()) + + +@pytest.mark.parametrize(('keyName', 'input','expected'), [ + ('ver', None , 'ver'), + ('ver', 3 , '003'), + ('asset', 'test', 'test'), + ('frame', 1, '0001'), + ('frame', 50, '0050'), + ('frame', 15550, '15550') +], ids=[ + 'version key no value', + 'version padding', + 'string key test', + 'frame 0001', + 'frame 0050', + 'frame 15550' +]) +def test_padding(keyName, input, expected, keys): + key = keys[keyName] + key.setValue(input) + assert str(key) == expected From b6975d6c9dd045229deec208f49eb0f0c84cb15c Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Wed, 18 May 2016 12:23:45 +0200 Subject: [PATCH 09/21] removed a whee bit ambigous locals from unit test.... slipped through --- test/unit/test_key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_key.py b/test/unit/test_key.py index 88df7ad..4484952 100644 --- a/test/unit/test_key.py +++ b/test/unit/test_key.py @@ -56,7 +56,7 @@ def keys(): ]) def test_key(name, type): '''Construct Key Objects''' - TemplateKeys.Key(**locals()) + TemplateKeys.Key(name=name,type=type) @pytest.mark.parametrize(('keyName', 'input','expected'), [ From 0734246ab3c58b5183264f2e29fe164897eaae2e Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Mon, 10 Oct 2016 18:01:17 +0200 Subject: [PATCH 10/21] initial commit --- source/lucidity/template.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index c64b04b..f371fa8 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -6,7 +6,7 @@ import sys import re import functools -from collections import defaultdict +from collections import defaultdict,OrderedDict import lucidity.error @@ -14,6 +14,18 @@ _RegexType = type(re.compile('')) +def OrderedSet(alist): + """ Creates an ordered set of type list + from a list of tuples or other hashable items + """ + mmap = {} # implements hashed lookup + oset = [] # storage for set + for item in alist: + if item not in mmap: + mmap[item] = 1 + oset.append(item) + return oset + class Template(object): '''A template.''' @@ -156,10 +168,12 @@ def parse(self, path): target[parts[-1]] = value - newData=dict() - for key,value in data.items(): - if value != None: - newData[key]=value + newData=OrderedDict() + for key in self.keys(): + if key in data: + value = data.get(key,None) + if value: + newData[key]=value return newData else: @@ -168,7 +182,7 @@ def parse(self, path): ) def missing(self, data, ignoreOptionals=False): - '''Returns a set of missing keys + '''Returns an unsorted set of missing keys optional keys are ignored/subtracted ''' data_keys = set(data.keys()) @@ -180,7 +194,7 @@ def missing(self, data, ignoreOptionals=False): else: new_data_keys.append(key) data_keys = new_data_keys - all_key = self.keys().difference(data_keys) + all_key = set(self.keys()).difference(data_keys) if ignoreOptionals: return all_key minus_opt = all_key.difference(self.optional_keys()) @@ -245,20 +259,20 @@ def _format(self, match, data, abstract= False): return value def keys(self): - '''Return unique set of placeholders in pattern.''' + '''Return unique list of placeholders in pattern.''' format_specification = self._construct_format_specification( self.expanded_pattern() ) if not self.key_resolver: - return set(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)) + return OrderedSet(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)) else: keys = list() - for key in set(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)): + for key in OrderedSet(self._PLAIN_PLACEHOLDER_REGEX.findall(format_specification)): if key in self.key_resolver: keys.append(self.key_resolver.get(key)) else: keys.append(key) - return set(keys) + return OrderedSet(keys) def optional_keys(self): format_specification = self._construct_format_specification( From b1e7fe8cc8c651b057fbda7c6e60c98345be1669 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Thu, 13 Oct 2016 17:33:52 +0200 Subject: [PATCH 11/21] added db placeholders to key objcts --- source/lucidity/key.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/lucidity/key.py b/source/lucidity/key.py index ed1e6fe..0e3713c 100644 --- a/source/lucidity/key.py +++ b/source/lucidity/key.py @@ -25,6 +25,8 @@ def __init__(self, name, type,**kwargs): self.__regex = None self.__padding = None self.__abstract = '' + self.__dbEntity = self.__name + self.__dbField = '' for key, value in kwargs.items(): if key == 'regex': @@ -36,6 +38,10 @@ def __init__(self, name, type,**kwargs): self.__padding = value else: raise Exception('provided padding {0} is not a valid padding pattern must be like "%04d"'.format(value)) + if key == 'dbEntity': + self.__dbEntity = value + if key == 'dbField': + self.__dbField = value @property def name(self): @@ -57,6 +63,14 @@ def padding(self): def regex(self): return self.__regex + @property + def dbEntity(self): + return self.__dbEntity + + @property + def dbField(self): + return self.__dbField + @property def value(self): return self.__value From 841eadd445543896622f6c1c9b55ec4f35ab4d87 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Thu, 13 Oct 2016 17:35:05 +0200 Subject: [PATCH 12/21] only sort template.keys() as this is much more efficient to iterate then over this than the hassle of a special return dict --- source/lucidity/template.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index f371fa8..7c6a5d3 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -6,7 +6,7 @@ import sys import re import functools -from collections import defaultdict,OrderedDict +from collections import defaultdict import lucidity.error @@ -18,11 +18,9 @@ def OrderedSet(alist): """ Creates an ordered set of type list from a list of tuples or other hashable items """ - mmap = {} # implements hashed lookup - oset = [] # storage for set + oset = [] for item in alist: - if item not in mmap: - mmap[item] = 1 + if item not in oset: oset.append(item) return oset @@ -168,12 +166,10 @@ def parse(self, path): target[parts[-1]] = value - newData=OrderedDict() - for key in self.keys(): - if key in data: - value = data.get(key,None) - if value: - newData[key]=value + newData=dict() + for key,value in data.items(): + if value != None: + newData[key]=value return newData else: From 970a6051c899146eb88a2b79f7c8d361acfaf849 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Thu, 13 Oct 2016 17:35:33 +0200 Subject: [PATCH 13/21] change unittest to expect a list from keys instead of a set --- test/unit/test_template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/test_template.py b/test/unit/test_template.py index e1ca7bd..0c5318a 100644 --- a/test/unit/test_template.py +++ b/test/unit/test_template.py @@ -303,14 +303,14 @@ def test_keys_mutable_side_effect(): '''Avoid side effects mutating internal keys set.''' template = Template('test', '/single/{variable}') placeholders = template.keys() - assert placeholders == set(['variable']) + assert placeholders == ['variable'] # Mutate returned set. - placeholders.add('other') + placeholders.append('other') # Newly returned set should be unaffected. placeholders_b = template.keys() - assert placeholders_b == set(['variable']) + assert placeholders_b == ['variable'] @pytest.mark.parametrize(('pattern', 'expected'), [ From 1df5658fe82abd2b47a63170f46be765e4fd1375 Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Wed, 16 Nov 2016 18:24:49 +0100 Subject: [PATCH 14/21] updates --- source/lucidity/template.py | 29 +++++++++++++++++++++++++---- test/unit/test_template.py | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index 7c6a5d3..6e398b1 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -38,7 +38,7 @@ class Template(object): def __init__(self, name, pattern, anchor=ANCHOR_BOTH, default_placeholder_expression='[A-Za-z0-9\-]+', - duplicate_placeholder_mode=STRICT, + duplicate_placeholder_mode=STRICT, validateOnInit = False, template_resolver=None, key_resolver={}): '''Initialise with *name* and *pattern*. @@ -75,7 +75,8 @@ def __init__(self, name, pattern, anchor=ANCHOR_BOTH, self._anchor = anchor # Check that supplied pattern is valid and able to be compiled. - self._construct_regular_expression(self.pattern) + if validateOnInit: + self._construct_regular_expression(self.pattern) def __repr__(self): '''Return unambiguous representation of template.''' @@ -279,7 +280,7 @@ def optional_keys(self): for key in temp_keys: optional_keys.extend(self._PLAIN_PLACEHOLDER_REGEX.findall(key)) if not self.key_resolver: - return set(optional_keys) + return OrderedSet(optional_keys) else: keys = list() for key in set(optional_keys): @@ -287,7 +288,7 @@ def optional_keys(self): keys.append(self.key_resolver.get(key)) else: keys.append(key) - return set(keys) + return OrderedSet(keys) def references(self): @@ -322,8 +323,28 @@ def _construct_expressions(self, pattern): for option in options: temp_options.append(option + opt) options = temp_options + if self.duplicate_placeholder_mode == self.STRICT: + temp = list() + for key in self.optional_keys(): + if isinstance(key,lucidity.Key): + key = key.name + occurences = 0 + for optKey in optionalKeys: + if optKey.__contains__(key): + occurences += 1 + if occurences > 1: + # we do have the same key twice or three times as an optional + # we only keep the options where we find the exact number of occurences + # all other variations will be dismissed + for option in options: + if option.__contains__(key): + if len(re.findall(key,option)) != occurences: + temp.append(option) + if temp: + options = list( set(options)-set(temp)) return options + def _construct_regular_expression(self, pattern): '''Return a regular expression to represent *pattern*.''' # Escape non-placeholder components. diff --git a/test/unit/test_template.py b/test/unit/test_template.py index 0c5318a..b856df5 100644 --- a/test/unit/test_template.py +++ b/test/unit/test_template.py @@ -72,7 +72,7 @@ def test_valid_pattern(pattern): def test_invalid_pattern(pattern): '''Construct template with invalid pattern.''' with pytest.raises(ValueError): - Template('test', pattern) + Template('test', pattern, validateOnInit = True) @pytest.mark.parametrize(('pattern', 'path', 'expected'), [ From 4af927b3f6059a18b28b5f7aced5b01ec7c95e88 Mon Sep 17 00:00:00 2001 From: Thorsten Kaufmann Date: Fri, 13 Jan 2017 14:31:55 +0100 Subject: [PATCH 15/21] Add .gitlab-ci.yml --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..184bea5 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,3 @@ +test: + script: + - py.test test \ No newline at end of file From 99397f3e55adf4745ca4c6b6a334fd17fbadd219 Mon Sep 17 00:00:00 2001 From: Thorsten Kaufmann Date: Fri, 13 Jan 2017 14:36:33 +0100 Subject: [PATCH 16/21] Update .gitlab-ci.yml --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 184bea5..188255e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,4 @@ test: script: + - set PYTHONPATH=.\source - py.test test \ No newline at end of file From 4f1a4a8a48a74e185616c8edf75a143351c4514b Mon Sep 17 00:00:00 2001 From: Thorsten Kaufmann Date: Fri, 13 Jan 2017 14:43:21 +0100 Subject: [PATCH 17/21] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 188255e..2c9b81b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ test: script: - set PYTHONPATH=.\source - - py.test test \ No newline at end of file + - py.test test --html=report.html \ No newline at end of file From 47d45816e7b7dbe551286150accfd48e930692e3 Mon Sep 17 00:00:00 2001 From: Thorsten Kaufmann Date: Fri, 13 Jan 2017 14:49:48 +0100 Subject: [PATCH 18/21] Update .gitlab-ci.yml --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c9b81b..5c8ae07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,5 @@ test: script: - set PYTHONPATH=.\source - - py.test test --html=report.html \ No newline at end of file + - py.test test --html=report.html + - coverage html -d coverage_html \ No newline at end of file From d5a5940776daee25ce2326c87610393ae3acc37f Mon Sep 17 00:00:00 2001 From: Thorsten Kaufmann Date: Fri, 13 Jan 2017 14:57:36 +0100 Subject: [PATCH 19/21] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c8ae07..6f13978 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,4 +2,4 @@ test: script: - set PYTHONPATH=.\source - py.test test --html=report.html - - coverage html -d coverage_html \ No newline at end of file +# - coverage html -d coverage_html \ No newline at end of file From a4ac2151e6499fc5ad8c76a6086031c3828f85ca Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Mon, 20 Mar 2017 12:58:28 +0100 Subject: [PATCH 20/21] store regex on the template objects --- source/lucidity/template.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/lucidity/template.py b/source/lucidity/template.py index 6e398b1..66f49f0 100644 --- a/source/lucidity/template.py +++ b/source/lucidity/template.py @@ -73,6 +73,7 @@ def __init__(self, name, pattern, anchor=ANCHOR_BOTH, self._name = name self._pattern = pattern self._anchor = anchor + self.__regexes = None ## once recompiled store the regexes here # Check that supplied pattern is valid and able to be compiled. if validateOnInit: @@ -132,10 +133,11 @@ def parse(self, path): ''' # Construct a list of regular expression for expanded pattern. - regexes = self._construct_regular_expression(self.expanded_pattern()) + if not self.__regexes: + self.__regexes = self._construct_regular_expression(self.expanded_pattern()) # Parse. parsed = {} - for regex in regexes: + for regex in self.__regexes: match = regex.search(path) if match: From d64c63e0b218b69734592bb15509b31975824a6a Mon Sep 17 00:00:00 2001 From: Johannes Hezer Date: Mon, 20 Mar 2017 12:59:53 +0100 Subject: [PATCH 21/21] added support for functional keys --- source/lucidity/key.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/lucidity/key.py b/source/lucidity/key.py index 0e3713c..265cb87 100644 --- a/source/lucidity/key.py +++ b/source/lucidity/key.py @@ -24,6 +24,7 @@ def __init__(self, name, type,**kwargs): self.__value = None self.__regex = None self.__padding = None + self.__function = None self.__abstract = '' self.__dbEntity = self.__name self.__dbField = '' @@ -33,6 +34,9 @@ def __init__(self, name, type,**kwargs): self.__regex = re.compile(value) if key == 'abstract': self.__abstract = value + if key == 'function': + self.__function = value + self.__value = value() #call the function once at init to set a value if key == 'padding': if re.match(r'(\%)([0-9]{0,2})(d)', value): self.__padding = value @@ -55,6 +59,10 @@ def type(self): def abstract(self): return self.__abstract + @property + def function(self): + return self.__function + @property def padding(self): return self.__padding @@ -89,6 +97,9 @@ def setValue(self, value): return self.__value = self.type(value) return + elif str(value) == str(self.name): + self.__value = value + return else: raise Exception('provided value {0} does not match regex {1} for {2}'.format(value, self.regex.pattern,self.__repr__())) else: @@ -115,6 +126,9 @@ def __str__(self): return self.padding % self.value elif self.type == int: return str(self.value) + elif self.function: + self.__value = self.function() + return str(self.value) def __cmp__(self,other): '''