From bfb63a88dd8e5ac371c52cca77dc71f0d171fe29 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 10 Feb 2016 11:03:30 +0200 Subject: [PATCH 01/22] Moves the excepthood closure to the module level --- tools/kcli/kcli/logger.py | 55 ++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/tools/kcli/kcli/logger.py b/tools/kcli/kcli/logger.py index 445d4d404..4fd94b03f 100644 --- a/tools/kcli/kcli/logger.py +++ b/tools/kcli/kcli/logger.py @@ -6,10 +6,7 @@ from kcli import exceptions -LOGGER_NAME = "IRLogger" -DEFAULT_LOGLEVEL = logging.WARNING - -debug_formatter = ColoredFormatter( +logger_formatter = ColoredFormatter( "%(log_color)s%(levelname)-8s%(message)s", log_colors=dict( DEBUG='blue', @@ -20,39 +17,39 @@ ) ) - -def kcli_traceback_handler(log): - """Creates exception hook that sends IRException to log and other - exceptions to stdout (default excepthook) - :param log: logger to log trace - """ - - def my_excepthook(exc_type, exc_value, exc_traceback): - # sends full exception with trace to log - if not isinstance(exc_value, exceptions.IRException): - return sys.__excepthook__(exc_type, exc_value, exc_traceback) - - if log.getEffectiveLevel() <= logging.DEBUG: - formated_exception = "".join( - traceback.format_exception(exc_type, exc_value, exc_traceback)) - log.error(formated_exception + exc_value.message) - else: - log.error(exc_value.message) - - sys.excepthook = my_excepthook - +LOGGER_NAME = "IRLogger" +DEFAULT_LOG_LEVEL = logging.WARNING LOG = logging.getLogger(LOGGER_NAME) -LOG.setLevel(DEFAULT_LOGLEVEL) +LOG.setLevel(DEFAULT_LOG_LEVEL) # Create stream handler with debug level sh = logging.StreamHandler() sh.setLevel(logging.DEBUG) -# Add the debug_formatter to sh -sh.setFormatter(debug_formatter) +# Add the logger_formatter to sh +sh.setFormatter(logger_formatter) # Create logger and add handler to it LOG.addHandler(sh) -kcli_traceback_handler(LOG) + +def ir_excepthook(exc_type, exc_value, exc_traceback): + """ + exception hook that sends IRException to log and other exceptions to + stdout (default excepthook) + """ + + # sends full exception with trace to log + if not isinstance(exc_value, exceptions.IRException): + return sys.__excepthook__(exc_type, exc_value, exc_traceback) + + if LOG.getEffectiveLevel() <= logging.DEBUG: + formated_exception = "".join( + traceback.format_exception(exc_type, exc_value, exc_traceback)) + LOG.error(formated_exception + exc_value.message) + else: + LOG.error(exc_value.message) + + +sys.excepthook = ir_excepthook From ae51f9721e7bd8872282aa0a0559f6322124263e Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 10 Feb 2016 11:36:24 +0200 Subject: [PATCH 02/22] Moves logger import to the haed of local imports --- tools/kcli/kcli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 683212397..7e2e3c2f8 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -8,10 +8,10 @@ import yaml import configure +from kcli import logger from kcli import conf from kcli.exceptions import * from kcli.execute.execute import PLAYBOOKS -from kcli import logger from kcli import parse # Contains meta-classes so we need to import it without using. from kcli import yamls From 5f54f4b108547abe09f37c3576cc71a0f80f4989 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 10 Feb 2016 11:39:07 +0200 Subject: [PATCH 03/22] Changes the exception import way in main.py --- tools/kcli/kcli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 7e2e3c2f8..8834d6ca5 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -10,9 +10,9 @@ from kcli import logger from kcli import conf -from kcli.exceptions import * from kcli.execute.execute import PLAYBOOKS from kcli import parse +from kcli import exceptions # Contains meta-classes so we need to import it without using. from kcli import yamls From 370c59b192b71257eba55b12b3e064742bc094c7 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 10 Feb 2016 13:05:04 +0200 Subject: [PATCH 04/22] Fixes CR comments --- tools/kcli/kcli/logger.py | 6 +++--- tools/kcli/kcli/main.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/kcli/kcli/logger.py b/tools/kcli/kcli/logger.py index 4fd94b03f..602e88c6f 100644 --- a/tools/kcli/kcli/logger.py +++ b/tools/kcli/kcli/logger.py @@ -2,11 +2,11 @@ import sys import traceback -from colorlog import ColoredFormatter +import colorlog from kcli import exceptions -logger_formatter = ColoredFormatter( +logger_formatter = colorlog.ColoredFormatter( "%(log_color)s%(levelname)-8s%(message)s", log_colors=dict( DEBUG='blue', @@ -37,7 +37,7 @@ def ir_excepthook(exc_type, exc_value, exc_traceback): """ exception hook that sends IRException to log and other exceptions to - stdout (default excepthook) + stderr (default excepthook) """ # sends full exception with trace to log diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 8834d6ca5..ac4f9413f 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -8,7 +8,9 @@ import yaml import configure +# logger should be created first from kcli import logger + from kcli import conf from kcli.execute.execute import PLAYBOOKS from kcli import parse From 3864723765fb977b0a1d1421cdb0f6ee5c48f458 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 10 Feb 2016 17:00:42 +0200 Subject: [PATCH 05/22] Splits code to seperate modules & add documention --- tools/kcli/kcli/conf.py | 5 +- tools/kcli/kcli/main.py | 401 ++----------------------------------- tools/kcli/kcli/options.py | 144 +++++++++++++ tools/kcli/kcli/parse.py | 6 +- tools/kcli/kcli/utils.py | 230 +++++++++++++++++++++ tools/kcli/kcli/yamls.py | 81 +++++++- 6 files changed, 474 insertions(+), 393 deletions(-) create mode 100644 tools/kcli/kcli/options.py create mode 100644 tools/kcli/kcli/utils.py diff --git a/tools/kcli/kcli/conf.py b/tools/kcli/kcli/conf.py index faf86bbaa..880cb4aa8 100644 --- a/tools/kcli/kcli/conf.py +++ b/tools/kcli/kcli/conf.py @@ -1,15 +1,16 @@ import ConfigParser import os +import time from kcli import exceptions -from kcli.exceptions import IRFileNotFoundException - ENV_VAR_NAME = "KCLI_CONFIG" KCLI_CONF_FILE = 'kcli.cfg' CWD_PATH = os.path.join(os.getcwd(), KCLI_CONF_FILE) USER_PATH = os.path.expanduser('~/.' + KCLI_CONF_FILE) SYSTEM_PATH = os.path.join('/etc/khaleesi', KCLI_CONF_FILE) +YAML_EXT = ".yml" +TMP_OUTPUT_FILE = 'kcli_settings_' + str(time.time()) + YAML_EXT def load_config_file(): diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index ac4f9413f..6ce618b4c 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -2,389 +2,28 @@ import logging import os -import re -import sys -import yaml -import configure - -# logger should be created first +# logger creation is first thing to be done from kcli import logger from kcli import conf +from kcli import options as kcli_options from kcli.execute.execute import PLAYBOOKS from kcli import parse -from kcli import exceptions -# Contains meta-classes so we need to import it without using. -from kcli import yamls +from kcli import utils -SETTING_FILE_EXT = ".yml" LOG = logger.LOG -kcli_conf = conf.config - -# Representer for Configuration object -yaml.SafeDumper.add_representer( - configure.Configuration, - lambda dumper, value: - yaml.representer.BaseRepresenter.represent_mapping - (dumper, u'tag:yaml.org,2002:map', value)) - - -def dict_lookup(dic, key, *keys): - if LOG.getEffectiveLevel() <= logging.DEBUG: - calling_method_name = sys._getframe().f_back.f_code.co_name - current_method_name = sys._getframe().f_code.co_name - if current_method_name != calling_method_name: - full_key = list(keys) - full_key.insert(0, key) - LOG.debug("looking up the value of \"%s\"" % ".".join(full_key)) - - if key not in dic: - if isinstance(key, str) and key.isdigit(): - key = int(key) - elif isinstance(key, int): - key = str(key) - - if keys: - return dict_lookup(dic.get(key, {}), *keys) - - try: - value = dic[key] - except KeyError: - raise exceptions.IRKeyNotFoundException(key, dic) - - LOG.debug("value has been found: \"%s\"" % value) - return value - - -def dict_insert(dic, val, key, *keys): - if not keys: - dic[key] = val - return - - if key not in dic: - dic[key] = {} - - dict_insert(dic[key], val, *keys) - - -def validate_settings_dir(settings_dir=None): - """ - Checks & returns the full path to the settings dir. - Path is set in the following priority: - 1. Method argument - 2. System environment variable - 3. Settings dir in the current working dir - :param settings_dir: path given as argument by a user - :return: path to settings dir (str) - :raise: IRFileNotFoundException: when the path to the settings dir doesn't - exist - """ - settings_dir = settings_dir or os.environ.get( - 'KHALEESI_SETTINGS') or os.path.join(os.getcwd(), "settings", "") - - if not os.path.exists(settings_dir): - raise exceptions.IRFileNotFoundException( - settings_dir, - "Settings dir doesn't exist: ") - - return settings_dir - - -class Lookup(yaml.YAMLObject): - yaml_tag = u'!lookup' - yaml_dumper = yaml.SafeDumper - - settings = None - - def __init__(self, key, old_style_lookup=False): - self.key = key - if old_style_lookup: - self.convert_old_style_lookup() - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, self.key) - - def convert_old_style_lookup(self): - self.key = '{{!lookup %s}}' % self.key - - parser = re.compile('\[\s*\!lookup\s*[\w.]*\s*\]') - lookups = parser.findall(self.key) - - for lookup in lookups: - self.key = self.key.replace(lookup, '.{{%s}}' % lookup[1:-1]) - - def replace_lookup(self): - """ - Replace any !lookup with the corresponding value from settings table - """ - while True: - parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') - lookups = parser.findall(self.key) - - if not lookups: - break - - for a_lookup in lookups: - lookup_key = re.search('(\w+\.?)+ *?\}\}', a_lookup) - lookup_key = lookup_key.group(0).strip()[:-2].strip() - lookup_value = dict_lookup( - self.settings, *lookup_key.split(".")) - - if isinstance(lookup_value, Lookup): - return - - lookup_value = str(lookup_value) - - self.key = re.sub('\{\{\s*\!lookup\s*[\w.]*\s*\}\}', - lookup_value, self.key, count=1) - - @classmethod - def from_yaml(cls, loader, node): - return Lookup(loader.construct_scalar(node), old_style_lookup=True) - - @classmethod - def to_yaml(cls, dumper, node): - if node.settings: - node.replace_lookup() - - return dumper.represent_data("%s" % node.key) - - -class OptionNode(object): - def __init__(self, path, parent=None): - self.path = path - self.option = self.path.split("/")[-1] - self.parent = parent - self.parent_value = None - if parent: - self.option = "-".join([self.parent.option, self.option]) - self.values = self._get_values() - self.children = {i: dict() for i in self._get_sub_options()} - - if self.parent: - self.parent_value = self.path.split("/")[-2] - self.parent.children[self.parent_value][self.option] = self - - def _get_values(self): - """ - Returns a sorted list of values available for the current option - """ - values = [a_file.split(SETTING_FILE_EXT)[0] - for a_file in os.listdir(self.path) - if os.path.isfile(os.path.join(self.path, a_file)) and - a_file.endswith(SETTING_FILE_EXT)] - - values.sort() - return values - - def _get_sub_options(self): - """ - Returns a sorted list of sup-options available for the current option - """ - options = [options_dir for options_dir in os.listdir(self.path) - if os.path.isdir(os.path.join(self.path, options_dir)) and - options_dir in self.values] - - options.sort() - return options - - -class OptionsTree(object): - def __init__(self, settings_dir, option): - self.root = None - self.name = option - self.action = option[:-2] if option.endswith('er') else option - self.options_dict = {} - self.root_dir = os.path.join(settings_dir, self.name) - - self.build_tree() - self.init_options_dict(self.root) - - def build_tree(self): - """ - Builds the OptionsTree - """ - self.add_node(self.root_dir, None) - - def add_node(self, path, parent): - """ - Adds OptionNode object to the tree - :param path: Path to option dir - :param parent: Parent option (OptionNode) - """ - node = OptionNode(path, parent) - - if not self.root: - self.root = node - - for child in node.children: - sub_options_dir = os.path.join(node.path, child) - sub_options = [a_dir for a_dir in os.listdir(sub_options_dir) if - os.path.isdir(os.path.join(sub_options_dir, a_dir))] - - for sub_option in sub_options: - self.add_node(os.path.join(sub_options_dir, sub_option), node) - - def init_options_dict(self, node): - """ - Initialize "options_dict" dictionary to store all options and their - valid values - :param node: OptionNode object - """ - if node.option not in self.options_dict: - self.options_dict[node.option] = {} - - if node.parent_value: - self.options_dict[node.option][node.parent_value] = node.values - - if 'ALL' not in self.options_dict[node.option]: - self.options_dict[node.option]['ALL'] = set() - - self.options_dict[node.option]['ALL'].update(node.values) - - for pre_value in node.children: - for child in node.children[pre_value].values(): - self.init_options_dict(child) - - def get_options_ymls(self, options): - ymls = [] - if not options: - return ymls - - keys = options.keys() - keys.sort() - - def step_in(key, node): - keys.remove(key) - if node.option != key.replace("_", "-"): - raise exceptions.IRMissingAncestorException(key) - - ymls.append(os.path.join(node.path, options[key] + ".yml")) - child_keys = [child_key for child_key in keys - if child_key.startswith(key) and - len(child_key.split("_")) == len(key.split("_")) + 1 - ] - - for child_key in child_keys: - step_in(child_key, node.children[options[key]][ - child_key.replace("_", "-")]) - - step_in(keys[0], self.root) - LOG.debug("%s tree settings files:\n%s" % (self.name, ymls)) - - return ymls - - def __str__(self): - return yaml.safe_dump(self.options_dict, default_flow_style=False) - - -def merge_settings(settings, file_path): - LOG.debug("Loading setting file: %s" % file_path) - if not os.path.exists(file_path): - raise exceptions.IRFileNotFoundException(file_path) - - loaded_file = configure.Configuration.from_file(file_path).configure() - settings = settings.merge(loaded_file) - - return settings - - -def generate_settings_file(settings_files, extra_vars): - settings = configure.Configuration.from_dict({}) - - for settings_file in settings_files: - settings = merge_settings(settings, settings_file) - - for extra_var in extra_vars: - if extra_var.startswith('@'): - settings_file = normalize_file(extra_var[1:]) - settings = merge_settings(settings, settings_file) - - else: - if '=' not in extra_var: - raise exceptions.IRExtraVarsException(extra_var) - key, value = extra_var.split("=") - dict_insert(settings, value, *key.split(".")) - - # Dump & load again settings, because 'in_string_lookup' can't work with - # 'Configuration' object. - dumped_settings = yaml.safe_dump(settings, default_flow_style=False) - settings = yaml.safe_load(dumped_settings) - - return settings - - -def in_string_lookup(settings): - """ - Convert strings contain the '!lookup' tag in them and don't - already converted into Lookup objects. - """ - if Lookup.settings is None: - Lookup.settings = settings - - my_iter = settings.iteritems() if isinstance(settings, dict) \ - else enumerate(settings) - - for idx_key, value in my_iter: - if isinstance(value, dict): - in_string_lookup(settings[idx_key]) - elif isinstance(value, list): - in_string_lookup(value) - elif isinstance(value, str): - parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') - lookups = parser.findall(value) - - if lookups: - settings[idx_key] = Lookup(value) - - -def normalize_file(file_path): - """ - Return a normalized absolutized version of a file - """ - if not os.path.isabs(file_path): - abspath = os.path.abspath(file_path) - LOG.debug( - "Setting the absolute path of \"%s\" to: \"%s\"" - % (file_path, abspath) - ) - file_path = abspath - - if not os.path.exists(file_path): - raise exceptions.IRFileNotFoundException(file_path) - - return file_path - - -def lookup2lookup(settings): - first_dump = True - while True: - if not first_dump: - Lookup.settings = settings - settings = yaml.load(output) - - in_string_lookup(settings) - output = yaml.safe_dump(settings, default_flow_style=False) - - if first_dump: - first_dump = False - continue - - if not cmp(settings, Lookup.settings): - break - - return output +CONF = conf.config def main(): options_trees = [] settings_files = [] - settings_dir = validate_settings_dir(kcli_conf.get('DEFAULTS', - 'SETTINGS_DIR')) + settings_dir = utils.validate_settings_dir( + CONF.get('DEFAULTS', 'SETTINGS_DIR')) - for option in kcli_conf.options('ROOT_OPTS'): - options_trees.append(OptionsTree(settings_dir, option)) + for option in CONF.options('ROOT_OPTS'): + options_trees.append(kcli_options.OptionsTree(settings_dir, option)) parser = parse.create_parser(options_trees) args = parser.parse_args() @@ -403,7 +42,7 @@ def main(): # settings generation stage if args.which.lower() != 'execute': for input_file in args.input: - settings_files.append(normalize_file(input_file)) + settings_files.append(utils.normalize_file(input_file)) for options_tree in options_trees: options = {key: value for key, value in vars(args).iteritems() @@ -413,9 +52,11 @@ def main(): LOG.debug("All settings files to be loaded:\n%s" % settings_files) - settings = generate_settings_file(settings_files, args.extra_vars) + settings = utils.generate_settings_file(settings_files, + args.extra_vars) - output = lookup2lookup(settings) + # todo: move into Lookup + output = utils.lookup2lookup(settings) if args.output_file: with open(args.output_file, 'w') as output_file: @@ -424,7 +65,7 @@ def main(): print output exec_playbook = (args.which == 'execute') or \ - (not args.dry_run and args.which in kcli_conf.options( + (not args.dry_run and args.which in CONF.options( 'AUTO_EXEC_OPTS')) # playbook execution stage @@ -452,15 +93,11 @@ def main(): % args.output_file) args_list.append('--settings=%s' % args.output_file) else: - from time import time - - tmp_settings_file = 'kcli_settings_' + str(time()) + \ - SETTING_FILE_EXT - with open(tmp_settings_file, 'w') as output_file: + with open(conf.TMP_OUTPUT_FILE, 'w') as output_file: output_file.write(output) LOG.debug('Temporary settings file "%s" has been created for ' - 'execution purpose only.' % tmp_settings_file) - args_list.append('--settings=%s' % tmp_settings_file) + 'execution purpose only.' % conf.TMP_OUTPUT_FILE) + args_list.append('--settings=%s' % conf.TMP_OUTPUT_FILE) execute_args = parser.parse_args(args_list) @@ -469,8 +106,8 @@ def main(): if not args.output_file and args.which != 'execute': LOG.debug('Temporary settings file "%s" has been deleted.' - % tmp_settings_file) - os.remove(tmp_settings_file) + % conf.TMP_OUTPUT_FILE) + os.remove(conf.TMP_OUTPUT_FILE) if __name__ == '__main__': diff --git a/tools/kcli/kcli/options.py b/tools/kcli/kcli/options.py new file mode 100644 index 000000000..e1cf65157 --- /dev/null +++ b/tools/kcli/kcli/options.py @@ -0,0 +1,144 @@ +import os + +import yaml + +from kcli import conf +from kcli import exceptions +from kcli import logger + +LOG = logger.LOG + + +class OptionNode(object): + def __init__(self, path, parent=None): + self.path = path + self.option = self.path.split("/")[-1] + self.parent = parent + self.parent_value = None + if parent: + self.option = "-".join([self.parent.option, self.option]) + self.values = self._get_values() + self.children = {i: dict() for i in self._get_sub_options()} + + if self.parent: + self.parent_value = self.path.split("/")[-2] + self.parent.children[self.parent_value][self.option] = self + + def _get_values(self): + """ + Returns a sorted list of values available for the current option + """ + values = [a_file.split(conf.YAML_EXT)[0] + for a_file in os.listdir(self.path) + if os.path.isfile(os.path.join(self.path, a_file)) and + a_file.endswith(conf.YAML_EXT)] + + values.sort() + return values + + def _get_sub_options(self): + """ + Returns a sorted list of sup-options available for the current option + """ + options = [options_dir for options_dir in os.listdir(self.path) + if os.path.isdir(os.path.join(self.path, options_dir)) and + options_dir in self.values] + + options.sort() + return options + + +class OptionsTree(object): + def __init__(self, settings_dir, option): + self.root = None + self.name = option + self.action = option[:-2] if option.endswith('er') else option + self.options_dict = {} + self.root_dir = os.path.join(settings_dir, self.name) + + self.build_tree() + self.init_options_dict(self.root) + + def build_tree(self): + """ + Builds the OptionsTree + """ + self.add_node(self.root_dir, None) + + def add_node(self, path, parent): + """ + Adds OptionNode object to the tree + :param path: Path to option dir + :param parent: Parent option (OptionNode) + """ + node = OptionNode(path, parent) + + if not self.root: + self.root = node + + for child in node.children: + sub_options_dir = os.path.join(node.path, child) + sub_options = [a_dir for a_dir in os.listdir(sub_options_dir) if + os.path.isdir(os.path.join(sub_options_dir, a_dir))] + + for sub_option in sub_options: + self.add_node(os.path.join(sub_options_dir, sub_option), node) + + def init_options_dict(self, node): + """ + Initialize "options_dict" dictionary to store all options and their + valid values + :param node: OptionNode object + """ + if node.option not in self.options_dict: + self.options_dict[node.option] = {} + + if node.parent_value: + self.options_dict[node.option][node.parent_value] = node.values + + if 'ALL' not in self.options_dict[node.option]: + self.options_dict[node.option]['ALL'] = set() + + self.options_dict[node.option]['ALL'].update(node.values) + + for pre_value in node.children: + for child in node.children[pre_value].values(): + self.init_options_dict(child) + + def get_options_ymls(self, options): + """return list of paths to settings YAML files for a given options + dictionary + + :param options: dictionary of options to get path of their settings + files + :return: list of paths to settings files of 'options' + """ + ymls = [] + if not options: + return ymls + + keys = options.keys() + keys.sort() + + def step_in(key, node): + keys.remove(key) + if node.option != key.replace("_", "-"): + raise exceptions.IRMissingAncestorException(key) + + ymls.append(os.path.join(node.path, options[key] + ".yml")) + child_keys = [child_key for child_key in keys + if child_key.startswith(key) and + len(child_key.split("_")) == len(key.split("_")) + 1 + ] + + for child_key in child_keys: + step_in(child_key, node.children[options[key]][ + child_key.replace("_", "-")]) + + step_in(keys[0], self.root) + LOG.debug("%s tree settings files:\n%s" % (self.name, ymls)) + + return ymls + + def __str__(self): + return yaml.safe_dump(self.options_dict, default_flow_style=False) diff --git a/tools/kcli/kcli/parse.py b/tools/kcli/kcli/parse.py index fb50be260..3f5b46bce 100644 --- a/tools/kcli/kcli/parse.py +++ b/tools/kcli/kcli/parse.py @@ -54,9 +54,9 @@ def create_parser(options_trees): sub_parser.add_argument("-e", "--extra-vars", default=list(), action='append', help="Provide extra vars") sub_parser.add_argument("-n", "--input", action='append', - help="a settings file that will be loaded " - "first, all other settings file will be" - " merged with it", default=list()) + help="Settings files to be loaded first," + " other settings file will be" + " merged with them", default=list()) sub_parser.add_argument("-o", "--output-file", help="file to dump the settings into") sub_parser.add_argument("-v", "--verbose", help="verbosity", diff --git a/tools/kcli/kcli/utils.py b/tools/kcli/kcli/utils.py new file mode 100644 index 000000000..9db908e9c --- /dev/null +++ b/tools/kcli/kcli/utils.py @@ -0,0 +1,230 @@ +import os +import logging +import re +import sys + +import configure +import yaml + +import kcli.yamls +from kcli import exceptions +from kcli import logger + +LOG = logger.LOG + + +# TODO: check if can be moved into Lookup +def dict_lookup(dic, key, *keys): + """lookup and return value of a nested key from a given dictionary + + to get the value of a nested key, all ancestor keys should be given as + method's arguments + + example: + dict_lookup({'key1': {'key2': 'val'}}, 'key1.key2'.split('.')) + + :param dic: dictionary object to get the key's value from + :param key: key / first key in a keys' chain + :param keys: sub keys in a keys' chain + :return: value of a gives keys + """ + if LOG.getEffectiveLevel() <= logging.DEBUG: + calling_method_name = sys._getframe().f_back.f_code.co_name + current_method_name = sys._getframe().f_code.co_name + if current_method_name != calling_method_name: + full_key = list(keys) + full_key.insert(0, key) + LOG.debug("looking up the value of \"%s\"" % ".".join(full_key)) + + if key not in dic: + if isinstance(key, str) and key.isdigit(): + key = int(key) + elif isinstance(key, int): + key = str(key) + + if keys: + return dict_lookup(dic.get(key, {}), *keys) + + try: + value = dic[key] + except KeyError: + raise exceptions.IRKeyNotFoundException(key, dic) + + LOG.debug("value has been found: \"%s\"" % value) + return value + + +def dict_insert(dic, val, key, *keys): + """insert a value of a nested key into a dictionary + + to insert value for a nested key, all ancestor keys should be given as + method's arguments + + example: + dict_lookup({}, 'val', 'key1.key2'.split('.')) + + :param dic: a dictionary object to insert the nested key value into + :param val: a value to insert to the given dictionary + :param key: first key in a chain of key that will store the value + :param keys: sub keys in the keys chain + """ + if not keys: + dic[key] = val + return + + if key not in dic: + dic[key] = {} + + dict_insert(dic[key], val, *keys) + + +# TODO: remove "settings" references in project +def validate_settings_dir(settings_dir=None): + """Checks & returns the full path to the settings dir. + + Path is set in the following priority: + 1. Method argument + 2. System environment variable + 3. Settings dir in the current working dir + + :param settings_dir: path given as argument by a user + :return: path to settings dir (str) + :raise: IRFileNotFoundException: when the path to the settings dir doesn't + exist + """ + settings_dir = settings_dir or os.environ.get( + 'KHALEESI_SETTINGS') or os.path.join(os.getcwd(), "settings", "") + + if not os.path.exists(settings_dir): + raise exceptions.IRFileNotFoundException( + settings_dir, + "Settings dir doesn't exist: ") + + return settings_dir + + +def merge_settings(settings, file_path): + """merge settings in 'file_path' with 'settings' + + :param settings: settings to be merge with + :param file_path: path to file with settings to be merged + :return: merged settings + """ + LOG.debug("Loading setting file: %s" % file_path) + if not os.path.exists(file_path): + raise exceptions.IRFileNotFoundException(file_path) + + loaded_file = configure.Configuration.from_file(file_path).configure() + settings = settings.merge(loaded_file) + + return settings + + +def generate_settings_file(settings_files, extra_vars): + """Generate one settings file from a given list of settings files and + extra-vars + + :param settings_files: list of paths to settings files + :param extra_vars: list of extra-vars + :return: dictionary with merging results of all settings file and + extra-vars + """ + settings = configure.Configuration.from_dict({}) + + for settings_file in settings_files: + settings = merge_settings(settings, settings_file) + + for extra_var in extra_vars: + if extra_var.startswith('@'): + settings_file = normalize_file(extra_var[1:]) + settings = merge_settings(settings, settings_file) + + else: + if '=' not in extra_var: + raise exceptions.IRExtraVarsException(extra_var) + key, value = extra_var.split("=") + dict_insert(settings, value, *key.split(".")) + + # Dump & load again settings, because 'in_string_lookup' can't work with + # 'Configuration' object. + dumped_settings = yaml.safe_dump(settings, default_flow_style=False) + settings = yaml.safe_load(dumped_settings) + + return settings + + +def in_string_lookup(settings): + """convert strings contain the '!lookup' tag in them and don't + already converted into Lookup objects. + + :param settings: a settings dictionary to search and convert lookup from + """ + if kcli.yamls.Lookup.settings is None: + kcli.yamls.Lookup.settings = settings + + my_iter = settings.iteritems() if isinstance(settings, dict) \ + else enumerate(settings) + + for idx_key, value in my_iter: + if isinstance(value, dict): + in_string_lookup(settings[idx_key]) + elif isinstance(value, list): + in_string_lookup(value) + elif isinstance(value, str): + parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') + lookups = parser.findall(value) + + if lookups: + settings[idx_key] = kcli.yamls.Lookup(value) + + +# todo: convert into a file object to be consumed by argparse +def normalize_file(file_path): + """Return a normalized absolutized version of a file + + :param file_path: path to file to be normalized + :return: normalized path of a file + :raise: IRFileNotFoundException if the file doesn't exist + """ + if not os.path.isabs(file_path): + abspath = os.path.abspath(file_path) + LOG.debug( + 'Setting the absolute path of "%s" to: "%s"' + % (file_path, abspath) + ) + file_path = abspath + + if not os.path.exists(file_path): + raise exceptions.IRFileNotFoundException(file_path) + + return file_path + + +# todo: move into lookup +def lookup2lookup(settings): + """handles recursive lookups + + load and dump yaml's dictionary ('settings') until all lookups strings + are been converted into Lookup objects + + :param settings: settings to convert all lookups from + :return: an yml dictionary object without lookup strings + """ + + first_dump = True + while True: + if not first_dump: + kcli.yamls.Lookup.settings = settings + settings = yaml.load(output) + + in_string_lookup(settings) + output = yaml.safe_dump(settings, default_flow_style=False) + + if first_dump: + first_dump = False + continue + + if not cmp(settings, kcli.yamls.Lookup.settings): + break + + return output diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py index 4d5298adc..993065568 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/kcli/kcli/yamls.py @@ -1,13 +1,22 @@ +import re import string -from configure import Configuration +import configure import yaml -from kcli import logger +import kcli.utils from kcli import exceptions +from kcli import logger LOG = logger.LOG +# Representer for Configuration object +yaml.SafeDumper.add_representer( + configure.Configuration, + lambda dumper, value: + yaml.representer.BaseRepresenter.represent_mapping + (dumper, u'tag:yaml.org,2002:map', value)) + def random_generator(size=32, chars=string.ascii_lowercase + string.digits): import random @@ -15,13 +24,13 @@ def random_generator(size=32, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) -@Configuration.add_constructor('join') +@configure.Configuration.add_constructor('join') def _join_constructor(loader, node): seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) -@Configuration.add_constructor('random') +@configure.Configuration.add_constructor('random') def _random_constructor(loader, node): """ usage: @@ -42,7 +51,7 @@ def _limit_chars(_string, length): return _string[:length] -@Configuration.add_constructor('limit_chars') +@configure.Configuration.add_constructor('limit_chars') def _limit_chars_constructor(loader, node): """ Usage: @@ -57,7 +66,7 @@ def _limit_chars_constructor(loader, node): return _limit_chars(params[0], params[1]) -@Configuration.add_constructor('env') +@configure.Configuration.add_constructor('env') def _env_constructor(loader, node): """ usage: @@ -89,3 +98,63 @@ def _env_constructor(loader, node): return ret return os.environ[var] + + +class Lookup(yaml.YAMLObject): + yaml_tag = u'!lookup' + yaml_dumper = yaml.SafeDumper + + settings = None + + def __init__(self, key, old_style_lookup=False): + self.key = key + if old_style_lookup: + self.convert_old_style_lookup() + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, self.key) + + def convert_old_style_lookup(self): + self.key = '{{!lookup %s}}' % self.key + + parser = re.compile('\[\s*\!lookup\s*[\w.]*\s*\]') + lookups = parser.findall(self.key) + + for lookup in lookups: + self.key = self.key.replace(lookup, '.{{%s}}' % lookup[1:-1]) + + def replace_lookup(self): + """ + Replace any !lookup with the corresponding value from settings table + """ + while True: + parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') + lookups = parser.findall(self.key) + + if not lookups: + break + + for a_lookup in lookups: + lookup_key = re.search('(\w+\.?)+ *?\}\}', a_lookup) + lookup_key = lookup_key.group(0).strip()[:-2].strip() + lookup_value = kcli.utils.dict_lookup( + self.settings, *lookup_key.split(".")) + + if isinstance(lookup_value, Lookup): + return + + lookup_value = str(lookup_value) + + self.key = re.sub('\{\{\s*\!lookup\s*[\w.]*\s*\}\}', + lookup_value, self.key, count=1) + + @classmethod + def from_yaml(cls, loader, node): + return Lookup(loader.construct_scalar(node), old_style_lookup=True) + + @classmethod + def to_yaml(cls, dumper, node): + if node.settings: + node.replace_lookup() + + return dumper.represent_data("%s" % node.key) From e7c5516016a6e9946c0b4c33a9eaa2cc3f3c0563 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Thu, 11 Feb 2016 18:25:42 +0200 Subject: [PATCH 06/22] Adds unittest for dict_insert method --- .travis.yml | 2 +- tools/kcli/tests/test_utils.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tools/kcli/tests/test_utils.py diff --git a/.travis.yml b/.travis.yml index 996c2eb20..3e36299a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,4 @@ install: # command to run tests script: - pep8 tools/kcli - - py.test tools/kcli + - py.test tools/kcli -v diff --git a/tools/kcli/tests/test_utils.py b/tools/kcli/tests/test_utils.py new file mode 100644 index 000000000..cfaf4840c --- /dev/null +++ b/tools/kcli/tests/test_utils.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.mark.parametrize('tested, val, key, expected', [ + ({}, 'val', ['key'], {'key': 'val'}), + + ({}, 'val', ['key1', 'key2', 'key3'], {'key1': {'key2': {'key3': 'val'}}}), + + ({'a_key': 'a_val', 'b_key1': {'b_key2': {'b_key3': 'b_val'}}}, 'x_val', + ['b_key1', 'b_key2'], {'a_key': 'a_val', 'b_key1': {'b_key2': 'x_val'}}), +]) +def test_dict_insert(tested, val, key, expected): + from kcli import utils + utils.dict_insert(tested, val, *key) + assert tested == expected From fee3c14ad447c7fb0ac67f47b02a125621f4ba7c Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Sun, 14 Feb 2016 11:39:55 +0200 Subject: [PATCH 07/22] Fixes CR comments (commit # 38647237) --- tools/kcli/README.rst | 21 ++++++++++++++++++++- tools/kcli/kcli/conf.py | 1 + tools/kcli/kcli/main.py | 4 ++-- tools/kcli/kcli/options.py | 28 ++++++++++++++++++++++------ tools/kcli/kcli/parse.py | 7 +++++-- tools/kcli/kcli/utils.py | 35 ++++++++++++++++++++--------------- tools/kcli/kcli/yamls.py | 4 ++++ 7 files changed, 74 insertions(+), 26 deletions(-) diff --git a/tools/kcli/README.rst b/tools/kcli/README.rst index c873bfd9d..7c6a2bf8c 100644 --- a/tools/kcli/README.rst +++ b/tools/kcli/README.rst @@ -54,4 +54,23 @@ This displays options you can pass to ``kcli``. $ export WORKSAPCE=$(dirname `pwd`) - +Extra-Vars +---------- +One can set/overwrite settings in the output file using the '-e/--extra-vars' +option. There are 2 ways of doing so: +1. specific settings: (key=value form) + --extra-vars provisioner.site.user=a_user +2. path to a settings file: (starts with '@') + --extra-vars @path/to/a/settings_file.yml + +The '-e/--extra-vars' can be used more than once. + +Merging order +------------- +Except options based on the settings dir structure, kcli accepts input of +predefined settings files (with -n/--input) and user defined specific options +(-e/--extra-vars). +The merging priority order listed below: +1. Input files +2. Settings dir based options +3. Extra Vars diff --git a/tools/kcli/kcli/conf.py b/tools/kcli/kcli/conf.py index 880cb4aa8..6e8d2bff7 100644 --- a/tools/kcli/kcli/conf.py +++ b/tools/kcli/kcli/conf.py @@ -11,6 +11,7 @@ SYSTEM_PATH = os.path.join('/etc/khaleesi', KCLI_CONF_FILE) YAML_EXT = ".yml" TMP_OUTPUT_FILE = 'kcli_settings_' + str(time.time()) + YAML_EXT +KHALEESI_DIR_ENV_VAR = 'KHALEESI_SETTINGS' def load_config_file(): diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 6ce618b4c..4493ba01e 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -52,8 +52,8 @@ def main(): LOG.debug("All settings files to be loaded:\n%s" % settings_files) - settings = utils.generate_settings_file(settings_files, - args.extra_vars) + settings = utils.generate_settings(settings_files, + args.extra_vars) # todo: move into Lookup output = utils.lookup2lookup(settings) diff --git a/tools/kcli/kcli/options.py b/tools/kcli/kcli/options.py index e1cf65157..a5c693210 100644 --- a/tools/kcli/kcli/options.py +++ b/tools/kcli/kcli/options.py @@ -1,3 +1,9 @@ +""" +This module is the building blocks for the options parsing tree. +It contains the data structures that hold available options & values in a +given directory. +""" + import os import yaml @@ -10,6 +16,13 @@ class OptionNode(object): + """ + represents an option and its properties: + - parent option + - available values + - option's path + - sub options + """ def __init__(self, path, parent=None): self.path = path self.option = self.path.split("/")[-1] @@ -25,9 +38,7 @@ def __init__(self, path, parent=None): self.parent.children[self.parent_value][self.option] = self def _get_values(self): - """ - Returns a sorted list of values available for the current option - """ + """Returns a sorted list of values available for the current option""" values = [a_file.split(conf.YAML_EXT)[0] for a_file in os.listdir(self.path) if os.path.isfile(os.path.join(self.path, a_file)) and @@ -49,6 +60,10 @@ def _get_sub_options(self): class OptionsTree(object): + """ + Tree represents hierarchy of options from rhe same kind (provisioner, + installer etc...) + """ def __init__(self, settings_dir, option): self.root = None self.name = option @@ -60,9 +75,7 @@ def __init__(self, settings_dir, option): self.init_options_dict(self.root) def build_tree(self): - """ - Builds the OptionsTree - """ + """Builds the OptionsTree""" self.add_node(self.root_dir, None) def add_node(self, path, parent): @@ -121,6 +134,9 @@ def get_options_ymls(self, options): keys.sort() def step_in(key, node): + """recursive method that returns the settings files of a given + options + """ keys.remove(key) if node.option != key.replace("_", "-"): raise exceptions.IRMissingAncestorException(key) diff --git a/tools/kcli/kcli/parse.py b/tools/kcli/kcli/parse.py index 3f5b46bce..c416351a3 100644 --- a/tools/kcli/kcli/parse.py +++ b/tools/kcli/kcli/parse.py @@ -52,10 +52,13 @@ def create_parser(options_trees): sub_parser.add_argument("-d", "--dry-run", action='store_true', help="skip playbook execution stage") sub_parser.add_argument("-e", "--extra-vars", default=list(), - action='append', help="Provide extra vars") + action='append', help="Provide extra vars. " + "(key=value, or a path " + "to settings file if " + "starts with '@')") sub_parser.add_argument("-n", "--input", action='append', help="Settings files to be loaded first," - " other settings file will be" + " other settings files will be" " merged with them", default=list()) sub_parser.add_argument("-o", "--output-file", help="file to dump the settings into") diff --git a/tools/kcli/kcli/utils.py b/tools/kcli/kcli/utils.py index 9db908e9c..c925dd619 100644 --- a/tools/kcli/kcli/utils.py +++ b/tools/kcli/kcli/utils.py @@ -1,3 +1,7 @@ +""" +This module provide some general helper methods +""" + import os import logging import re @@ -7,6 +11,7 @@ import yaml import kcli.yamls +import kcli.conf from kcli import exceptions from kcli import logger @@ -72,10 +77,7 @@ def dict_insert(dic, val, key, *keys): dic[key] = val return - if key not in dic: - dic[key] = {} - - dict_insert(dic[key], val, *keys) + dict_insert(dic.setdefault(key, {}), val, *keys) # TODO: remove "settings" references in project @@ -85,7 +87,6 @@ def validate_settings_dir(settings_dir=None): Path is set in the following priority: 1. Method argument 2. System environment variable - 3. Settings dir in the current working dir :param settings_dir: path given as argument by a user :return: path to settings dir (str) @@ -93,7 +94,7 @@ def validate_settings_dir(settings_dir=None): exist """ settings_dir = settings_dir or os.environ.get( - 'KHALEESI_SETTINGS') or os.path.join(os.getcwd(), "settings", "") + kcli.conf.KHALEESI_DIR_ENV_VAR) if not os.path.exists(settings_dir): raise exceptions.IRFileNotFoundException( @@ -103,10 +104,10 @@ def validate_settings_dir(settings_dir=None): return settings_dir -def merge_settings(settings, file_path): +def update_settings(settings, file_path): """merge settings in 'file_path' with 'settings' - :param settings: settings to be merge with + :param settings: settings to be merge with (configure.Configuration) :param file_path: path to file with settings to be merged :return: merged settings """ @@ -120,24 +121,27 @@ def merge_settings(settings, file_path): return settings -def generate_settings_file(settings_files, extra_vars): - """Generate one settings file from a given list of settings files and - extra-vars +def generate_settings(settings_files, extra_vars): + """Generate one settings object (configure.Configuration) by merging all + files in settings file & extra-vars + + files in 'settings_files' are the first to be merged and after them the + 'extra_vars' :param settings_files: list of paths to settings files :param extra_vars: list of extra-vars - :return: dictionary with merging results of all settings file and - extra-vars + :return: Configuration object with merging results of all settings + files and extra-vars """ settings = configure.Configuration.from_dict({}) for settings_file in settings_files: - settings = merge_settings(settings, settings_file) + settings = update_settings(settings, settings_file) for extra_var in extra_vars: if extra_var.startswith('@'): settings_file = normalize_file(extra_var[1:]) - settings = merge_settings(settings, settings_file) + settings = update_settings(settings, settings_file) else: if '=' not in extra_var: @@ -156,6 +160,7 @@ def generate_settings_file(settings_files, extra_vars): def in_string_lookup(settings): """convert strings contain the '!lookup' tag in them and don't already converted into Lookup objects. + (in case of strings that contain and don't start with '!lookup') :param settings: a settings dictionary to search and convert lookup from """ diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py index 993065568..962c155ae 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/kcli/kcli/yamls.py @@ -1,3 +1,7 @@ +""" +This module contains the tools for handling YAML files and tags. +""" + import re import string From 9c7e651b102ab62e3d57dd793f7b60b8b15d4120 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Mon, 15 Feb 2016 15:12:15 +0200 Subject: [PATCH 08/22] Adds missing new line in enumerated lists lines --- tools/kcli/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/kcli/README.rst b/tools/kcli/README.rst index 7c6a2bf8c..513d612a8 100644 --- a/tools/kcli/README.rst +++ b/tools/kcli/README.rst @@ -58,6 +58,7 @@ Extra-Vars ---------- One can set/overwrite settings in the output file using the '-e/--extra-vars' option. There are 2 ways of doing so: + 1. specific settings: (key=value form) --extra-vars provisioner.site.user=a_user 2. path to a settings file: (starts with '@') @@ -71,6 +72,7 @@ Except options based on the settings dir structure, kcli accepts input of predefined settings files (with -n/--input) and user defined specific options (-e/--extra-vars). The merging priority order listed below: + 1. Input files 2. Settings dir based options 3. Extra Vars From 8466816b6fcf6bf539a601074cd52bf1053be188 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Tue, 16 Feb 2016 10:18:34 +0200 Subject: [PATCH 09/22] Moves 'execute' package to a module in kcli dir --- tools/kcli/kcli/{execute => }/execute.py | 31 +++++++++++++++++-- tools/kcli/kcli/execute/__init__.py | 2 -- tools/kcli/kcli/execute/core.py | 38 ------------------------ tools/kcli/kcli/main.py | 2 +- tools/kcli/kcli/parse.py | 15 +++++----- 5 files changed, 37 insertions(+), 51 deletions(-) rename tools/kcli/kcli/{execute => }/execute.py (83%) delete mode 100644 tools/kcli/kcli/execute/__init__.py delete mode 100644 tools/kcli/kcli/execute/core.py diff --git a/tools/kcli/kcli/execute/execute.py b/tools/kcli/kcli/execute.py similarity index 83% rename from tools/kcli/kcli/execute/execute.py rename to tools/kcli/kcli/execute.py index 47efb6f9a..88a94edd1 100644 --- a/tools/kcli/kcli/execute/execute.py +++ b/tools/kcli/kcli/execute.py @@ -1,3 +1,5 @@ +import argparse +import sys from os import path import ansible.color @@ -6,14 +8,17 @@ import ansible.utils from ansible import callbacks -from kcli import conf, exceptions -from kcli.execute import core +from kcli import conf, exceptions, utils +VERBOSITY = 0 HOSTS_FILE = "hosts" LOCAL_HOSTS = "local_hosts" PROVISION = "provision" PLAYBOOKS = [PROVISION, "install", "test", "collect-logs", "cleanup"] +assert "playbooks" == path.basename(conf.PLAYBOOKS_DIR), \ + "Bad path to playbooks" + # ansible-playbook # https://github.com/ansible/ansible/blob/devel/bin/ansible-playbook @@ -139,7 +144,7 @@ def ansible_wrapper(args): playbooks = [p for p in PLAYBOOKS if getattr(args, p, False)] if not playbooks: - core.parser.error("No playbook to execute (%s)" % PLAYBOOKS) + parser.error("No playbook to execute (%s)" % PLAYBOOKS) for playbook in (p for p in PLAYBOOKS if getattr(args, p, False)): print "Executing Playbook: %s" % playbook @@ -147,3 +152,23 @@ def ansible_wrapper(args): execute_ansible(playbook, args) except Exception: raise exceptions.IRPlaybookFailedException(playbook) + + +def main(): + args = parser.parse_args() + args.func(args) + + +parser = argparse.ArgumentParser() +parser.add_argument('-v', '--verbose', default=VERBOSITY, action="count", + help="verbose mode (-vvv for more," + " -vvvv to enable connection debugging)") +parser.add_argument("--settings", + default=conf.KCLI_SETTINGS_YML, + type=lambda file_path: utils.normalize_file(file_path), + help="settings file to use. default: %s" + % conf.KCLI_SETTINGS_YML) +subparsers = parser.add_subparsers(metavar="COMMAND") + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/kcli/kcli/execute/__init__.py b/tools/kcli/kcli/execute/__init__.py deleted file mode 100644 index c8cc42b81..000000000 --- a/tools/kcli/kcli/execute/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# from kcli.execute import core -# from kcli.execute import execute diff --git a/tools/kcli/kcli/execute/core.py b/tools/kcli/kcli/execute/core.py deleted file mode 100644 index 1882534a1..000000000 --- a/tools/kcli/kcli/execute/core.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -import argparse -import os -import sys - -from kcli import conf - -assert "playbooks" == os.path.basename(conf.PLAYBOOKS_DIR), \ - "Bad path to playbooks" - -VERBOSITY = 0 - - -def file_exists(prs, filename): - if not os.path.exists(filename): - prs.error("The file %s does not exist!" % filename) - return filename - - -def main(): - args = parser.parse_args() - args.func(args) - - -parser = argparse.ArgumentParser() -parser.add_argument('-v', '--verbose', default=VERBOSITY, action="count", - help="verbose mode (-vvv for more," - " -vvvv to enable connection debugging)") -parser.add_argument("--settings", - default=conf.KCLI_SETTINGS_YML, - type=lambda x: file_exists(parser, x), - help="settings file to use. default: %s" - % conf.KCLI_SETTINGS_YML) -subparsers = parser.add_subparsers(metavar="COMMAND") - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 4493ba01e..2b9641dde 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -8,7 +8,7 @@ from kcli import conf from kcli import options as kcli_options -from kcli.execute.execute import PLAYBOOKS +from kcli.execute import PLAYBOOKS from kcli import parse from kcli import utils diff --git a/tools/kcli/kcli/parse.py b/tools/kcli/kcli/parse.py index c416351a3..2d0014375 100644 --- a/tools/kcli/kcli/parse.py +++ b/tools/kcli/kcli/parse.py @@ -1,7 +1,6 @@ from argparse import ArgumentParser, RawTextHelpFormatter -from execute.core import * -from execute.execute import * +from kcli import conf, execute, utils def create_parser(options_trees): @@ -17,13 +16,14 @@ def create_parser(options_trees): execute_parser = sub_parsers.add_parser('execute') execute_parser.add_argument('-i', '--inventory', default=None, - type=lambda x: core.file_exists( - execute_parser, x), + type=lambda file_path: utils.normalize_file( + file_path), help="Inventory file to use. " "Default: {lcl}. " "NOTE: to reuse old environment use {" "host}". - format(lcl=LOCAL_HOSTS, host=HOSTS_FILE)) + format(lcl=execute.LOCAL_HOSTS, + host=execute.HOSTS_FILE)) execute_parser.add_argument("-v", "--verbose", help="verbosity", action='count', default=0) execute_parser.add_argument("--provision", action="store_true", @@ -37,10 +37,11 @@ def create_parser(options_trees): execute_parser.add_argument("--cleanup", action="store_true", help="cleanup nodes") execute_parser.add_argument("--settings", - type=lambda x: file_exists(parser, x), + type=lambda file_path: utils.normalize_file( + file_path), help="settings file to use. default: %s" % conf.KCLI_SETTINGS_YML) - execute_parser.set_defaults(func=ansible_wrapper) + execute_parser.set_defaults(func=execute.ansible_wrapper) execute_parser.set_defaults(which='execute') for options_tree in options_trees: From 51cd9b29d8adae70c15fc62e0f27ec82b98a50f9 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Tue, 16 Feb 2016 12:08:51 +0200 Subject: [PATCH 10/22] Moves 'dict_lookup' method int 'Lookup' class --- tools/kcli/kcli/utils.py | 43 +------------------------------ tools/kcli/kcli/yamls.py | 55 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 45 deletions(-) diff --git a/tools/kcli/kcli/utils.py b/tools/kcli/kcli/utils.py index c925dd619..c5b87dbf4 100644 --- a/tools/kcli/kcli/utils.py +++ b/tools/kcli/kcli/utils.py @@ -18,47 +18,6 @@ LOG = logger.LOG -# TODO: check if can be moved into Lookup -def dict_lookup(dic, key, *keys): - """lookup and return value of a nested key from a given dictionary - - to get the value of a nested key, all ancestor keys should be given as - method's arguments - - example: - dict_lookup({'key1': {'key2': 'val'}}, 'key1.key2'.split('.')) - - :param dic: dictionary object to get the key's value from - :param key: key / first key in a keys' chain - :param keys: sub keys in a keys' chain - :return: value of a gives keys - """ - if LOG.getEffectiveLevel() <= logging.DEBUG: - calling_method_name = sys._getframe().f_back.f_code.co_name - current_method_name = sys._getframe().f_code.co_name - if current_method_name != calling_method_name: - full_key = list(keys) - full_key.insert(0, key) - LOG.debug("looking up the value of \"%s\"" % ".".join(full_key)) - - if key not in dic: - if isinstance(key, str) and key.isdigit(): - key = int(key) - elif isinstance(key, int): - key = str(key) - - if keys: - return dict_lookup(dic.get(key, {}), *keys) - - try: - value = dic[key] - except KeyError: - raise exceptions.IRKeyNotFoundException(key, dic) - - LOG.debug("value has been found: \"%s\"" % value) - return value - - def dict_insert(dic, val, key, *keys): """insert a value of a nested key into a dictionary @@ -66,7 +25,7 @@ def dict_insert(dic, val, key, *keys): method's arguments example: - dict_lookup({}, 'val', 'key1.key2'.split('.')) + dict_insert({}, 'val', 'key1.key2'.split('.')) :param dic: a dictionary object to insert the nested key value into :param val: a value to insert to the given dictionary diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py index 962c155ae..0625429a3 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/kcli/kcli/yamls.py @@ -2,13 +2,14 @@ This module contains the tools for handling YAML files and tags. """ +import logging import re +import sys import string import configure import yaml -import kcli.utils from kcli import exceptions from kcli import logger @@ -141,8 +142,7 @@ def replace_lookup(self): for a_lookup in lookups: lookup_key = re.search('(\w+\.?)+ *?\}\}', a_lookup) lookup_key = lookup_key.group(0).strip()[:-2].strip() - lookup_value = kcli.utils.dict_lookup( - self.settings, *lookup_key.split(".")) + lookup_value = self.dict_lookup(lookup_key.split(".")) if isinstance(lookup_value, Lookup): return @@ -152,6 +152,55 @@ def replace_lookup(self): self.key = re.sub('\{\{\s*\!lookup\s*[\w.]*\s*\}\}', lookup_value, self.key, count=1) + def dict_lookup(self, keys, dic=None): + """ Returns the value of a given key from the settings class variable + + to get the value of a nested key, all ancestor keys should be given as + method's arguments + + example: + if one want to get the value of 'key3' in: + {'key1': {'key2': {'key3': 'val1'}}} + + dict_lookup(['key1', 'key2', 'key3']) + return value: + 'val1' + + :param keys: list with keys describing the path to the target key + :param dic: mapping object holds settings. (self.settings by default) + + :return: value of the target key + """ + if LOG.getEffectiveLevel() <= logging.DEBUG: + calling_method_name = sys._getframe().f_back.f_code.co_name + current_method_name = sys._getframe().f_code.co_name + if current_method_name != calling_method_name: + LOG.debug( + 'looking up the value of "{keys}"'.format( + keys=".".join(keys))) + + if dic is None: + dic = self.settings + + key = keys.pop(0) + + if key not in dic: + if isinstance(key, str) and key.isdigit(): + key = int(key) + elif isinstance(key, int): + key = str(key) + + try: + if keys: + return self.dict_lookup(keys, dic[key]) + + value = dic[key] + except KeyError: + raise exceptions.IRKeyNotFoundException(key, dic) + + LOG.debug('value has been found: "{value}"'.format(value=value)) + return value + @classmethod def from_yaml(cls, loader, node): return Lookup(loader.construct_scalar(node), old_style_lookup=True) From 8e92465c6227d8adfea90759608e356fb207a530 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Tue, 16 Feb 2016 16:14:25 +0200 Subject: [PATCH 11/22] Adds a handler for yaml ConstructorError Raise IRYAMLConstructorError instead of yaml.constructor.ConstructorError when loading YAML files, with the excapt path to the file that caused the problem. --- tools/kcli/kcli/exceptions.py | 10 ++++++++++ tools/kcli/kcli/utils.py | 8 +++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tools/kcli/kcli/exceptions.py b/tools/kcli/kcli/exceptions.py index 50b470fc7..e7763594a 100644 --- a/tools/kcli/kcli/exceptions.py +++ b/tools/kcli/kcli/exceptions.py @@ -38,3 +38,13 @@ class IRPlaybookFailedException(IRException): def __init__(self, playbook): super(self.__class__, self).__init__( 'Playbook "%s" failed!' % playbook) + + +class IRYAMLConstructorError(IRException): + def __init__(self, mark_obj, where): + self.message = mark_obj.problem + pm = mark_obj.problem_mark + self.message += ' in:\n "{where}", line {line_no}, column ' \ + '{column_no}'.format(where=where, + line_no=pm.line + 1, + column_no=pm.column + 1) diff --git a/tools/kcli/kcli/utils.py b/tools/kcli/kcli/utils.py index c5b87dbf4..260872da6 100644 --- a/tools/kcli/kcli/utils.py +++ b/tools/kcli/kcli/utils.py @@ -3,9 +3,7 @@ """ import os -import logging import re -import sys import configure import yaml @@ -74,7 +72,11 @@ def update_settings(settings, file_path): if not os.path.exists(file_path): raise exceptions.IRFileNotFoundException(file_path) - loaded_file = configure.Configuration.from_file(file_path).configure() + try: + loaded_file = configure.Configuration.from_file(file_path).configure() + except yaml.constructor.ConstructorError as e: + raise exceptions.IRYAMLConstructorError(e, file_path) + settings = settings.merge(loaded_file) return settings From d7e7f0c360b82c92104216dc5f8c2a0c28a2421c Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 17 Feb 2016 13:40:11 +0200 Subject: [PATCH 12/22] Organizes all lookup method in Lookup class --- tools/kcli/kcli/main.py | 11 ++++--- tools/kcli/kcli/utils.py | 64 +-------------------------------------- tools/kcli/kcli/yamls.py | 65 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 67 deletions(-) diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 2b9641dde..5f44d9b27 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -3,6 +3,8 @@ import logging import os +import yaml + # logger creation is first thing to be done from kcli import logger @@ -11,6 +13,7 @@ from kcli.execute import PLAYBOOKS from kcli import parse from kcli import utils +import kcli.yamls LOG = logger.LOG CONF = conf.config @@ -52,11 +55,11 @@ def main(): LOG.debug("All settings files to be loaded:\n%s" % settings_files) - settings = utils.generate_settings(settings_files, - args.extra_vars) + kcli.yamls.Lookup.settings = utils.generate_settings(settings_files, + args.extra_vars) - # todo: move into Lookup - output = utils.lookup2lookup(settings) + output = yaml.safe_dump(kcli.yamls.Lookup.settings, + default_flow_style=False) if args.output_file: with open(args.output_file, 'w') as output_file: diff --git a/tools/kcli/kcli/utils.py b/tools/kcli/kcli/utils.py index 260872da6..6e1abc075 100644 --- a/tools/kcli/kcli/utils.py +++ b/tools/kcli/kcli/utils.py @@ -3,7 +3,6 @@ """ import os -import re import configure import yaml @@ -83,7 +82,7 @@ def update_settings(settings, file_path): def generate_settings(settings_files, extra_vars): - """Generate one settings object (configure.Configuration) by merging all + """ Generates one settings object (configure.Configuration) by merging all files in settings file & extra-vars files in 'settings_files' are the first to be merged and after them the @@ -110,40 +109,9 @@ def generate_settings(settings_files, extra_vars): key, value = extra_var.split("=") dict_insert(settings, value, *key.split(".")) - # Dump & load again settings, because 'in_string_lookup' can't work with - # 'Configuration' object. - dumped_settings = yaml.safe_dump(settings, default_flow_style=False) - settings = yaml.safe_load(dumped_settings) - return settings -def in_string_lookup(settings): - """convert strings contain the '!lookup' tag in them and don't - already converted into Lookup objects. - (in case of strings that contain and don't start with '!lookup') - - :param settings: a settings dictionary to search and convert lookup from - """ - if kcli.yamls.Lookup.settings is None: - kcli.yamls.Lookup.settings = settings - - my_iter = settings.iteritems() if isinstance(settings, dict) \ - else enumerate(settings) - - for idx_key, value in my_iter: - if isinstance(value, dict): - in_string_lookup(settings[idx_key]) - elif isinstance(value, list): - in_string_lookup(value) - elif isinstance(value, str): - parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') - lookups = parser.findall(value) - - if lookups: - settings[idx_key] = kcli.yamls.Lookup(value) - - # todo: convert into a file object to be consumed by argparse def normalize_file(file_path): """Return a normalized absolutized version of a file @@ -164,33 +132,3 @@ def normalize_file(file_path): raise exceptions.IRFileNotFoundException(file_path) return file_path - - -# todo: move into lookup -def lookup2lookup(settings): - """handles recursive lookups - - load and dump yaml's dictionary ('settings') until all lookups strings - are been converted into Lookup objects - - :param settings: settings to convert all lookups from - :return: an yml dictionary object without lookup strings - """ - - first_dump = True - while True: - if not first_dump: - kcli.yamls.Lookup.settings = settings - settings = yaml.load(output) - - in_string_lookup(settings) - output = yaml.safe_dump(settings, default_flow_style=False) - - if first_dump: - first_dump = False - continue - - if not cmp(settings, kcli.yamls.Lookup.settings): - break - - return output diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py index 0625429a3..1ebe329a3 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/kcli/kcli/yamls.py @@ -110,6 +110,7 @@ class Lookup(yaml.YAMLObject): yaml_dumper = yaml.SafeDumper settings = None + handling_nested_lookups = False def __init__(self, key, old_style_lookup=False): self.key = key @@ -201,12 +202,76 @@ def dict_lookup(self, keys, dic=None): LOG.debug('value has been found: "{value}"'.format(value=value)) return value + @classmethod + def in_string_lookup(cls, settings_dic=None): + """ convert strings containing '!lookup' in them, and didn't already + converted into Lookup objects. + (in case when the strings don't start with '!lookup') + + :param settings_dic: a settings dictionary to search and convert + lookup from + """ + + if settings_dic is None: + settings_dic = cls.settings + + my_iter = settings_dic.iteritems() if isinstance(settings_dic, dict) \ + else enumerate(settings_dic) + + for idx_key, value in my_iter: + if isinstance(value, dict): + cls.in_string_lookup(settings_dic[idx_key]) + elif isinstance(value, list): + cls.in_string_lookup(value) + elif isinstance(value, str): + parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') + lookups = parser.findall(value) + + if lookups: + settings_dic[idx_key] = cls(value) + + @classmethod + def handle_nested_lookup(cls): + """ handles lookup to lookup (nested lookup scenario) + + load and dump 'settings' again and again until all lookups strings + are converted into Lookup objects + """ + + # because there is a call to 'yaml.safe_dump' which call to this + # method, the 'handling_nested_lookups' flag is being set & unset to + # prevent infinite loop between the method + cls.handling_nested_lookups = True + + first_dump = True + settings = cls.settings + + while True: + if not first_dump: + cls.settings = settings + settings = yaml.load(output) + + cls.in_string_lookup() + output = yaml.safe_dump(cls.settings, default_flow_style=False) + + if first_dump: + first_dump = False + continue + + if not cmp(settings, cls.settings): + break + + cls.handling_nested_lookups = False + @classmethod def from_yaml(cls, loader, node): return Lookup(loader.construct_scalar(node), old_style_lookup=True) @classmethod def to_yaml(cls, dumper, node): + if not cls.handling_nested_lookups: + cls.handle_nested_lookup() + if node.settings: node.replace_lookup() From 980ccdb5d3ecfed2ed7f05b6f1627c2319c63a86 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Wed, 17 Feb 2016 16:27:59 +0200 Subject: [PATCH 13/22] Adds unitest for unsupported YAML constructor Moves generic pytest methos into test_cwd/utils.py --- tools/kcli/tests/test_conf.py | 35 ++----------------- .../tests/test_cwd/IRYAMLConstructorError.yml | 3 ++ tools/kcli/tests/test_cwd/consts.py | 5 --- tools/kcli/tests/test_cwd/utils.py | 33 +++++++++++++++++ tools/kcli/tests/test_yaml.py | 17 +++++++++ 5 files changed, 56 insertions(+), 37 deletions(-) create mode 100644 tools/kcli/tests/test_cwd/IRYAMLConstructorError.yml delete mode 100644 tools/kcli/tests/test_cwd/consts.py create mode 100644 tools/kcli/tests/test_cwd/utils.py create mode 100644 tools/kcli/tests/test_yaml.py diff --git a/tools/kcli/tests/test_conf.py b/tools/kcli/tests/test_conf.py index dc70b02dd..528e56e5e 100644 --- a/tools/kcli/tests/test_conf.py +++ b/tools/kcli/tests/test_conf.py @@ -1,41 +1,12 @@ import os -import pytest -from tests.test_cwd import consts as test_const +from tests.test_cwd import utils -MYCWD = test_const.TESTS_CWD - - -@pytest.fixture() -def our_cwd_setup(request): - """Change cwd to test_cwd dir. Revert to original dir on teardown. """ - - bkp = os.getcwd() - - def our_cwd_teardown(): - os.chdir(bkp) - - request.addfinalizer(our_cwd_teardown) - os.chdir(MYCWD) - - -@pytest.yield_fixture -def os_environ(): - """Backups env var from os.environ and restores it at teardown. """ - - from kcli import conf - - backup_flag = False - if conf.ENV_VAR_NAME in os.environ: - backup_flag = True - backup_value = os.environ.get(conf.ENV_VAR_NAME) - yield os.environ - if backup_flag: - os.environ[conf.ENV_VAR_NAME] = backup_value +our_cwd_setup = utils.our_cwd_setup def test_get_config_dir(our_cwd_setup): from kcli import conf conf_file = conf.load_config_file() assert os.path.abspath(conf_file.get("DEFAULTS", "KHALEESI_DIR")) == \ - os.path.abspath(MYCWD) + os.path.abspath(utils.TESTS_CWD) diff --git a/tools/kcli/tests/test_cwd/IRYAMLConstructorError.yml b/tools/kcli/tests/test_cwd/IRYAMLConstructorError.yml new file mode 100644 index 000000000..e567c5144 --- /dev/null +++ b/tools/kcli/tests/test_cwd/IRYAMLConstructorError.yml @@ -0,0 +1,3 @@ +--- +key: + sub_key: !notag diff --git a/tools/kcli/tests/test_cwd/consts.py b/tools/kcli/tests/test_cwd/consts.py deleted file mode 100644 index 534bd4a57..000000000 --- a/tools/kcli/tests/test_cwd/consts.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -TESTS_CWD = os.path.dirname(__file__) -SETTINGS_PATH = os.path.join(TESTS_CWD, "settings") -# CONFIG_FILE = os.path.join(TESTS_CWD, conf.KCLI_CONF_FILE) diff --git a/tools/kcli/tests/test_cwd/utils.py b/tools/kcli/tests/test_cwd/utils.py new file mode 100644 index 000000000..5517c7f12 --- /dev/null +++ b/tools/kcli/tests/test_cwd/utils.py @@ -0,0 +1,33 @@ +import os +import pytest + +TESTS_CWD = os.path.dirname(__file__) +SETTINGS_PATH = os.path.join(TESTS_CWD, "settings") + + +@pytest.yield_fixture +def os_environ(): + """ Backups env var from os.environ and restores it at teardown. """ + + from kcli import conf + + backup_flag = False + if conf.ENV_VAR_NAME in os.environ: + backup_flag = True + backup_value = os.environ.get(conf.ENV_VAR_NAME) + yield os.environ + if backup_flag: + os.environ[conf.ENV_VAR_NAME] = backup_value + + +@pytest.fixture() +def our_cwd_setup(request): + """ Change cwd to test_cwd dir. Revert to original dir on teardown. """ + + bkp = os.getcwd() + + def our_cwd_teardown(): + os.chdir(bkp) + + request.addfinalizer(our_cwd_teardown) + os.chdir(TESTS_CWD) diff --git a/tools/kcli/tests/test_yaml.py b/tools/kcli/tests/test_yaml.py new file mode 100644 index 000000000..d19a3f641 --- /dev/null +++ b/tools/kcli/tests/test_yaml.py @@ -0,0 +1,17 @@ +import os.path + +import pytest +import configure + +from tests.test_cwd import utils + +our_cwd_setup = utils.our_cwd_setup + + +def test_unsupported_yaml_constructor(our_cwd_setup): + from kcli.utils import update_settings + from kcli.exceptions import IRYAMLConstructorError + tester_file = 'IRYAMLConstructorError.yml' + settings = configure.Configuration.from_dict({}) + with pytest.raises(IRYAMLConstructorError): + update_settings(settings, os.path.join(utils.TESTS_CWD, tester_file)) From 36896a64800dcfd48d81596275c8a65b0b6741c6 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Thu, 18 Feb 2016 08:20:14 +0200 Subject: [PATCH 14/22] Fixes comments from pull request #38 - pep8 fix Fixed continuation line over-indented for visual indent in test_conf.py --- tools/kcli/tests/test_conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/kcli/tests/test_conf.py b/tools/kcli/tests/test_conf.py index 528e56e5e..59fed1454 100644 --- a/tools/kcli/tests/test_conf.py +++ b/tools/kcli/tests/test_conf.py @@ -8,5 +8,6 @@ def test_get_config_dir(our_cwd_setup): from kcli import conf conf_file = conf.load_config_file() - assert os.path.abspath(conf_file.get("DEFAULTS", "KHALEESI_DIR")) == \ - os.path.abspath(utils.TESTS_CWD) + assert os.path.abspath( + conf_file.get("DEFAULTS", "KHALEESI_DIR")) == os.path.abspath( + utils.TESTS_CWD) From cad99ce1ca4d71de3a820fcf19cf9b965cbda328 Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Thu, 18 Feb 2016 11:13:50 +0200 Subject: [PATCH 15/22] Removes the un-needed parser in execute.py Removes the extra parser the was in execute.py and not needed anymore. The execute parser is now implemented in parse.py --- tools/kcli/kcli/execute.py | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/tools/kcli/kcli/execute.py b/tools/kcli/kcli/execute.py index 88a94edd1..4e0a01058 100644 --- a/tools/kcli/kcli/execute.py +++ b/tools/kcli/kcli/execute.py @@ -1,5 +1,3 @@ -import argparse -import sys from os import path import ansible.color @@ -8,7 +6,9 @@ import ansible.utils from ansible import callbacks -from kcli import conf, exceptions, utils +from kcli import conf, exceptions, logger + +LOG = logger.LOG VERBOSITY = 0 HOSTS_FILE = "hosts" @@ -140,11 +140,11 @@ def execute_ansible(playbook, args): def ansible_wrapper(args): - """Wraps the 'anisble-playbook' CLI.""" + """ Wraps the 'ansible-playbook' CLI. """ playbooks = [p for p in PLAYBOOKS if getattr(args, p, False)] if not playbooks: - parser.error("No playbook to execute (%s)" % PLAYBOOKS) + LOG.error("No playbook to execute (%s)" % PLAYBOOKS) for playbook in (p for p in PLAYBOOKS if getattr(args, p, False)): print "Executing Playbook: %s" % playbook @@ -152,23 +152,3 @@ def ansible_wrapper(args): execute_ansible(playbook, args) except Exception: raise exceptions.IRPlaybookFailedException(playbook) - - -def main(): - args = parser.parse_args() - args.func(args) - - -parser = argparse.ArgumentParser() -parser.add_argument('-v', '--verbose', default=VERBOSITY, action="count", - help="verbose mode (-vvv for more," - " -vvvv to enable connection debugging)") -parser.add_argument("--settings", - default=conf.KCLI_SETTINGS_YML, - type=lambda file_path: utils.normalize_file(file_path), - help="settings file to use. default: %s" - % conf.KCLI_SETTINGS_YML) -subparsers = parser.add_subparsers(metavar="COMMAND") - -if __name__ == '__main__': - sys.exit(main()) From e0d89903d80c760a6a920e6ae0af984da737ba8e Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Thu, 18 Feb 2016 14:53:05 +0200 Subject: [PATCH 16/22] Adds constructor & unittest for !placeholder tag Implements Placeholder class which creates Placeholder object from '!placeholder' tags and raises IRPlaceholderException if the objects haven't been replaced till the dumping stage. Creates a basic unittest which validates that IRPlaceholderException is being raised for '!placeholder' tag usage: (YAML file content) --- key: value: !placeholder "description of needed value" --- tools/kcli/kcli/exceptions.py | 5 ++++ tools/kcli/kcli/main.py | 1 + tools/kcli/kcli/utils.py | 7 ++++++ tools/kcli/kcli/yamls.py | 24 +++++++++++++++++++ .../tests/test_cwd/placeholder_validator.yml | 5 ++++ tools/kcli/tests/test_yaml.py | 15 +++++++++++- 6 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tools/kcli/tests/test_cwd/placeholder_validator.yml diff --git a/tools/kcli/kcli/exceptions.py b/tools/kcli/kcli/exceptions.py index e7763594a..b79815fd8 100644 --- a/tools/kcli/kcli/exceptions.py +++ b/tools/kcli/kcli/exceptions.py @@ -48,3 +48,8 @@ def __init__(self, mark_obj, where): '{column_no}'.format(where=where, line_no=pm.line + 1, column_no=pm.column + 1) + + +class IRPlaceholderException(IRException): + def __init__(self, trace_message): + self.message = 'Mandatory value is missing.\n' + trace_message diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py index 5f44d9b27..fa9567ad1 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/kcli/kcli/main.py @@ -58,6 +58,7 @@ def main(): kcli.yamls.Lookup.settings = utils.generate_settings(settings_files, args.extra_vars) + LOG.debug("Dumping settings...") output = yaml.safe_dump(kcli.yamls.Lookup.settings, default_flow_style=False) diff --git a/tools/kcli/kcli/utils.py b/tools/kcli/kcli/utils.py index 6e1abc075..23e044539 100644 --- a/tools/kcli/kcli/utils.py +++ b/tools/kcli/kcli/utils.py @@ -3,6 +3,7 @@ """ import os +import re import configure import yaml @@ -73,6 +74,12 @@ def update_settings(settings, file_path): try: loaded_file = configure.Configuration.from_file(file_path).configure() + placeholders_list = kcli.yamls.Placeholder.placeholders_list + for placeholder in placeholders_list[::-1]: + if placeholders_list[-1].file_path is None: + placeholder.file_path = file_path + else: + break except yaml.constructor.ConstructorError as e: raise exceptions.IRYAMLConstructorError(e, file_path) diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py index 1ebe329a3..cb42d42b1 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/kcli/kcli/yamls.py @@ -276,3 +276,27 @@ def to_yaml(cls, dumper, node): node.replace_lookup() return dumper.represent_data("%s" % node.key) + + +class Placeholder(yaml.YAMLObject): + """ Raises 'IRPlaceholderException' when dumping Placeholder objects. """ + yaml_tag = u'!placeholder' + yaml_dumper = yaml.SafeDumper + + # Refs for all Placeholder's objects + placeholders_list = [] + + def __init__(self, message): + self.message = message + self.file_path = None + + @classmethod + def from_yaml(cls, loader, node): + # Create & save references to Placeholder objects + cls.placeholders_list.append(Placeholder(str(node.start_mark))) + return cls.placeholders_list[-1] + + @classmethod + def to_yaml(cls, dumper, node): + message = re.sub("", node.file_path, node.message) + raise exceptions.IRPlaceholderException(message) diff --git a/tools/kcli/tests/test_cwd/placeholder_validator.yml b/tools/kcli/tests/test_cwd/placeholder_validator.yml new file mode 100644 index 000000000..d83de289e --- /dev/null +++ b/tools/kcli/tests/test_cwd/placeholder_validator.yml @@ -0,0 +1,5 @@ +--- +place: + holder: + validtaor: + !placeholder "tester for '!placeholder' tag" diff --git a/tools/kcli/tests/test_yaml.py b/tools/kcli/tests/test_yaml.py index d19a3f641..4152ae419 100644 --- a/tools/kcli/tests/test_yaml.py +++ b/tools/kcli/tests/test_yaml.py @@ -1,7 +1,8 @@ import os.path -import pytest import configure +import pytest +import yaml from tests.test_cwd import utils @@ -15,3 +16,15 @@ def test_unsupported_yaml_constructor(our_cwd_setup): settings = configure.Configuration.from_dict({}) with pytest.raises(IRYAMLConstructorError): update_settings(settings, os.path.join(utils.TESTS_CWD, tester_file)) + + +def test_placeholder_validator(our_cwd_setup): + from kcli.utils import update_settings + from kcli.exceptions import IRPlaceholderException + tester_file = 'placeholder_validator.yml' + settings = configure.Configuration.from_dict({}) + with pytest.raises(IRPlaceholderException) as exc: + settings = update_settings(settings, + os.path.join(utils.TESTS_CWD, tester_file)) + yaml.safe_dump(settings, default_flow_style=False) + assert "Mandatory value is missing." in str(exc.value.message) From 6c8a0fad6bc447d61ab7912b2f4227964b23fd2a Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Thu, 18 Feb 2016 15:41:22 +0200 Subject: [PATCH 17/22] Fixes comments from pull #40 --- tools/kcli/kcli/yamls.py | 12 +++++++++--- ...holder_validator.yml => placeholder_injector.yml} | 0 tools/kcli/tests/test_cwd/placeholder_overwriter.yml | 5 +++++ tools/kcli/tests/test_yaml.py | 10 +++++++++- 4 files changed, 23 insertions(+), 4 deletions(-) rename tools/kcli/tests/test_cwd/{placeholder_validator.yml => placeholder_injector.yml} (100%) create mode 100644 tools/kcli/tests/test_cwd/placeholder_overwriter.yml diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py index cb42d42b1..e20641b37 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/kcli/kcli/yamls.py @@ -279,7 +279,12 @@ def to_yaml(cls, dumper, node): class Placeholder(yaml.YAMLObject): - """ Raises 'IRPlaceholderException' when dumping Placeholder objects. """ + """ Raises 'IRPlaceholderException' when dumping Placeholder objects. + + Objects created by 'from_yaml' method are automatically added to the + 'placeholders_list' class variable so it'll be possible to add for each + object the path to the file where it stored. + """ yaml_tag = u'!placeholder' yaml_dumper = yaml.SafeDumper @@ -293,8 +298,9 @@ def __init__(self, message): @classmethod def from_yaml(cls, loader, node): # Create & save references to Placeholder objects - cls.placeholders_list.append(Placeholder(str(node.start_mark))) - return cls.placeholders_list[-1] + placeholder = Placeholder(str(node.start_mark)) + cls.placeholders_list.append(placeholder) + return placeholder @classmethod def to_yaml(cls, dumper, node): diff --git a/tools/kcli/tests/test_cwd/placeholder_validator.yml b/tools/kcli/tests/test_cwd/placeholder_injector.yml similarity index 100% rename from tools/kcli/tests/test_cwd/placeholder_validator.yml rename to tools/kcli/tests/test_cwd/placeholder_injector.yml diff --git a/tools/kcli/tests/test_cwd/placeholder_overwriter.yml b/tools/kcli/tests/test_cwd/placeholder_overwriter.yml new file mode 100644 index 000000000..a44106d9e --- /dev/null +++ b/tools/kcli/tests/test_cwd/placeholder_overwriter.yml @@ -0,0 +1,5 @@ +--- +place: + holder: + validtaor: + "'!placeholder' has been overwritten" diff --git a/tools/kcli/tests/test_yaml.py b/tools/kcli/tests/test_yaml.py index 4152ae419..b2fc7056f 100644 --- a/tools/kcli/tests/test_yaml.py +++ b/tools/kcli/tests/test_yaml.py @@ -21,10 +21,18 @@ def test_unsupported_yaml_constructor(our_cwd_setup): def test_placeholder_validator(our_cwd_setup): from kcli.utils import update_settings from kcli.exceptions import IRPlaceholderException - tester_file = 'placeholder_validator.yml' + + # Checks that 'IRPlaceholderException' is raised if value isn't been + # overwritten + tester_file = 'placeholder_injector.yml' settings = configure.Configuration.from_dict({}) with pytest.raises(IRPlaceholderException) as exc: settings = update_settings(settings, os.path.join(utils.TESTS_CWD, tester_file)) yaml.safe_dump(settings, default_flow_style=False) assert "Mandatory value is missing." in str(exc.value.message) + + # Checks that exceptions haven't been raised after overwriting the + # placeholder + tester_file = 'placeholder_overwriter.yml' + update_settings(settings, os.path.join(utils.TESTS_CWD, tester_file)) From bdabd7b0da106f88631b7fd50305d1fb6b6ea04d Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Thu, 18 Feb 2016 16:19:48 +0200 Subject: [PATCH 18/22] Fixes typo in placholder's yamles & add data check --- .../tests/test_cwd/placeholder_injector.yml | 2 +- .../tests/test_cwd/placeholder_overwriter.yml | 2 +- tools/kcli/tests/test_yaml.py | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/kcli/tests/test_cwd/placeholder_injector.yml b/tools/kcli/tests/test_cwd/placeholder_injector.yml index d83de289e..150cc1285 100644 --- a/tools/kcli/tests/test_cwd/placeholder_injector.yml +++ b/tools/kcli/tests/test_cwd/placeholder_injector.yml @@ -1,5 +1,5 @@ --- place: holder: - validtaor: + validator: !placeholder "tester for '!placeholder' tag" diff --git a/tools/kcli/tests/test_cwd/placeholder_overwriter.yml b/tools/kcli/tests/test_cwd/placeholder_overwriter.yml index a44106d9e..bedb599e7 100644 --- a/tools/kcli/tests/test_cwd/placeholder_overwriter.yml +++ b/tools/kcli/tests/test_cwd/placeholder_overwriter.yml @@ -1,5 +1,5 @@ --- place: holder: - validtaor: + validator: "'!placeholder' has been overwritten" diff --git a/tools/kcli/tests/test_yaml.py b/tools/kcli/tests/test_yaml.py index b2fc7056f..6473e47fa 100644 --- a/tools/kcli/tests/test_yaml.py +++ b/tools/kcli/tests/test_yaml.py @@ -21,18 +21,27 @@ def test_unsupported_yaml_constructor(our_cwd_setup): def test_placeholder_validator(our_cwd_setup): from kcli.utils import update_settings from kcli.exceptions import IRPlaceholderException + from kcli.yamls import Placeholder + + injector = 'placeholder_injector.yml' + overwriter = 'placeholder_overwriter.yml' # Checks that 'IRPlaceholderException' is raised if value isn't been # overwritten - tester_file = 'placeholder_injector.yml' settings = configure.Configuration.from_dict({}) + settings = update_settings(settings, + os.path.join(utils.TESTS_CWD, injector)) + + assert isinstance(settings['place']['holder']['validator'], Placeholder) with pytest.raises(IRPlaceholderException) as exc: - settings = update_settings(settings, - os.path.join(utils.TESTS_CWD, tester_file)) yaml.safe_dump(settings, default_flow_style=False) assert "Mandatory value is missing." in str(exc.value.message) # Checks that exceptions haven't been raised after overwriting the # placeholder - tester_file = 'placeholder_overwriter.yml' - update_settings(settings, os.path.join(utils.TESTS_CWD, tester_file)) + settings = update_settings(settings, + os.path.join(utils.TESTS_CWD, overwriter)) + + assert settings['place']['holder'][ + 'validator'] == "'!placeholder' has been overwritten" + yaml.safe_dump(settings, default_flow_style=False) From e6552680b0b80e145726f0623b87108bb8d3c4db Mon Sep 17 00:00:00 2001 From: Ariel Opincaru Date: Sun, 21 Feb 2016 13:34:00 +0200 Subject: [PATCH 19/22] WIP - dereference khaleesi & kcli khaleesi -> InfraRed kcli -> cli --- .travis.yml | 6 +++--- tools/{kcli => cli}/README.rst | 0 tools/{kcli/kcli => cli/cli}/__init__.py | 0 tools/{kcli/kcli => cli/cli}/conf.py | 18 ++++++++--------- tools/{kcli/kcli => cli/cli}/exceptions.py | 0 tools/{kcli/kcli => cli/cli}/execute.py | 2 +- tools/{kcli/kcli => cli/cli}/logger.py | 2 +- tools/{kcli/kcli => cli/cli}/main.py | 20 +++++++++---------- tools/{kcli/kcli => cli/cli}/options.py | 6 +++--- tools/{kcli/kcli => cli/cli}/parse.py | 6 +++--- tools/{kcli/kcli => cli/cli}/utils.py | 12 +++++------ tools/{kcli/kcli => cli/cli}/yamls.py | 4 ++-- tools/cli/etc/kcli.cfg.example | 19 ++++++++++++++++++ tools/{kcli => cli}/requirements.txt | 0 tools/{kcli => cli}/setup.py | 8 ++++---- tools/{kcli => cli}/tests/__init__.py | 0 tools/{kcli => cli}/tests/test_conf.py | 4 ++-- .../tests/test_cwd/IRYAMLConstructorError.yml | 0 .../{kcli => cli}/tests/test_cwd/__init__.py | 0 tools/cli/tests/test_cwd/kcli.cfg | 3 +++ .../tests/test_cwd/placeholder_injector.yml | 0 .../tests/test_cwd/placeholder_overwriter.yml | 0 tools/{kcli => cli}/tests/test_cwd/utils.py | 2 +- tools/{kcli => cli}/tests/test_utils.py | 2 +- tools/{kcli => cli}/tests/test_yaml.py | 10 +++++----- tools/kcli/etc/kcli.cfg.example | 19 ------------------ tools/kcli/tests/test_cwd/kcli.cfg | 3 --- 27 files changed, 73 insertions(+), 73 deletions(-) rename tools/{kcli => cli}/README.rst (100%) rename tools/{kcli/kcli => cli/cli}/__init__.py (100%) rename tools/{kcli/kcli => cli/cli}/conf.py (69%) rename tools/{kcli/kcli => cli/cli}/exceptions.py (100%) rename tools/{kcli/kcli => cli/cli}/execute.py (99%) rename tools/{kcli/kcli => cli/cli}/logger.py (97%) rename tools/{kcli/kcli => cli/cli}/main.py (89%) rename tools/{kcli/kcli => cli/cli}/options.py (98%) rename tools/{kcli/kcli => cli/cli}/parse.py (96%) rename tools/{kcli/kcli => cli/cli}/utils.py (95%) rename tools/{kcli/kcli => cli/cli}/yamls.py (99%) create mode 100644 tools/cli/etc/kcli.cfg.example rename tools/{kcli => cli}/requirements.txt (100%) rename tools/{kcli => cli}/setup.py (85%) rename tools/{kcli => cli}/tests/__init__.py (100%) rename tools/{kcli => cli}/tests/test_conf.py (71%) rename tools/{kcli => cli}/tests/test_cwd/IRYAMLConstructorError.yml (100%) rename tools/{kcli => cli}/tests/test_cwd/__init__.py (100%) create mode 100644 tools/cli/tests/test_cwd/kcli.cfg rename tools/{kcli => cli}/tests/test_cwd/placeholder_injector.yml (100%) rename tools/{kcli => cli}/tests/test_cwd/placeholder_overwriter.yml (100%) rename tools/{kcli => cli}/tests/test_cwd/utils.py (96%) rename tools/{kcli => cli}/tests/test_utils.py (94%) rename tools/{kcli => cli}/tests/test_yaml.py (85%) delete mode 100644 tools/kcli/etc/kcli.cfg.example delete mode 100644 tools/kcli/tests/test_cwd/kcli.cfg diff --git a/.travis.yml b/.travis.yml index 3e36299a3..e8e1e8fca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ python: install: - pip install pep8 --use-mirrors - pip install https://github.com/dcramer/pyflakes/tarball/master - - pip install tools/kcli + - pip install tools/cli # command to run tests script: - - pep8 tools/kcli - - py.test tools/kcli -v + - pep8 tools/cli + - py.test tools/cli -v diff --git a/tools/kcli/README.rst b/tools/cli/README.rst similarity index 100% rename from tools/kcli/README.rst rename to tools/cli/README.rst diff --git a/tools/kcli/kcli/__init__.py b/tools/cli/cli/__init__.py similarity index 100% rename from tools/kcli/kcli/__init__.py rename to tools/cli/cli/__init__.py diff --git a/tools/kcli/kcli/conf.py b/tools/cli/cli/conf.py similarity index 69% rename from tools/kcli/kcli/conf.py rename to tools/cli/cli/conf.py index 6e8d2bff7..2e1705c8f 100644 --- a/tools/kcli/kcli/conf.py +++ b/tools/cli/cli/conf.py @@ -2,16 +2,16 @@ import os import time -from kcli import exceptions +from cli import exceptions -ENV_VAR_NAME = "KCLI_CONFIG" -KCLI_CONF_FILE = 'kcli.cfg' -CWD_PATH = os.path.join(os.getcwd(), KCLI_CONF_FILE) -USER_PATH = os.path.expanduser('~/.' + KCLI_CONF_FILE) -SYSTEM_PATH = os.path.join('/etc/khaleesi', KCLI_CONF_FILE) +ENV_VAR_NAME = "CLI_CONFIG" +CLI_CONF_FILE = 'cli.cfg' +CWD_PATH = os.path.join(os.getcwd(), CLI_CONF_FILE) +USER_PATH = os.path.expanduser('~/.' + CLI_CONF_FILE) +SYSTEM_PATH = os.path.join('/etc/infrared', CLI_CONF_FILE) YAML_EXT = ".yml" -TMP_OUTPUT_FILE = 'kcli_settings_' + str(time.time()) + YAML_EXT -KHALEESI_DIR_ENV_VAR = 'KHALEESI_SETTINGS' +TMP_OUTPUT_FILE = 'cli_settings_' + str(time.time()) + YAML_EXT +INFRARED_DIR_ENV_VAR = 'INFRARED_SETTINGS' def load_config_file(): @@ -24,7 +24,7 @@ def load_config_file(): if env_path is not None: env_path = os.path.expanduser(env_path) if os.path.isdir(env_path): - env_path = os.path.join(env_path, KCLI_CONF_FILE) + env_path = os.path.join(env_path, CLI_CONF_FILE) for path in (env_path, CWD_PATH, USER_PATH, SYSTEM_PATH): if path is not None and os.path.exists(path): _config.read(path) diff --git a/tools/kcli/kcli/exceptions.py b/tools/cli/cli/exceptions.py similarity index 100% rename from tools/kcli/kcli/exceptions.py rename to tools/cli/cli/exceptions.py diff --git a/tools/kcli/kcli/execute.py b/tools/cli/cli/execute.py similarity index 99% rename from tools/kcli/kcli/execute.py rename to tools/cli/cli/execute.py index 4e0a01058..66d17826e 100644 --- a/tools/kcli/kcli/execute.py +++ b/tools/cli/cli/execute.py @@ -6,7 +6,7 @@ import ansible.utils from ansible import callbacks -from kcli import conf, exceptions, logger +from cli import conf, exceptions, logger LOG = logger.LOG diff --git a/tools/kcli/kcli/logger.py b/tools/cli/cli/logger.py similarity index 97% rename from tools/kcli/kcli/logger.py rename to tools/cli/cli/logger.py index 602e88c6f..07eac01cd 100644 --- a/tools/kcli/kcli/logger.py +++ b/tools/cli/cli/logger.py @@ -4,7 +4,7 @@ import colorlog -from kcli import exceptions +from cli import exceptions logger_formatter = colorlog.ColoredFormatter( "%(log_color)s%(levelname)-8s%(message)s", diff --git a/tools/kcli/kcli/main.py b/tools/cli/cli/main.py similarity index 89% rename from tools/kcli/kcli/main.py rename to tools/cli/cli/main.py index fa9567ad1..e2437d8d4 100755 --- a/tools/kcli/kcli/main.py +++ b/tools/cli/cli/main.py @@ -6,14 +6,14 @@ import yaml # logger creation is first thing to be done -from kcli import logger +from cli import logger -from kcli import conf -from kcli import options as kcli_options -from kcli.execute import PLAYBOOKS -from kcli import parse -from kcli import utils -import kcli.yamls +from cli import conf +from cli import options as cli_options +from cli.execute import PLAYBOOKS +from cli import parse +from cli import utils +import cli.yamls LOG = logger.LOG CONF = conf.config @@ -26,7 +26,7 @@ def main(): CONF.get('DEFAULTS', 'SETTINGS_DIR')) for option in CONF.options('ROOT_OPTS'): - options_trees.append(kcli_options.OptionsTree(settings_dir, option)) + options_trees.append(cli_options.OptionsTree(settings_dir, option)) parser = parse.create_parser(options_trees) args = parser.parse_args() @@ -55,11 +55,11 @@ def main(): LOG.debug("All settings files to be loaded:\n%s" % settings_files) - kcli.yamls.Lookup.settings = utils.generate_settings(settings_files, + cli.yamls.Lookup.settings = utils.generate_settings(settings_files, args.extra_vars) LOG.debug("Dumping settings...") - output = yaml.safe_dump(kcli.yamls.Lookup.settings, + output = yaml.safe_dump(cli.yamls.Lookup.settings, default_flow_style=False) if args.output_file: diff --git a/tools/kcli/kcli/options.py b/tools/cli/cli/options.py similarity index 98% rename from tools/kcli/kcli/options.py rename to tools/cli/cli/options.py index a5c693210..942fad437 100644 --- a/tools/kcli/kcli/options.py +++ b/tools/cli/cli/options.py @@ -8,9 +8,9 @@ import yaml -from kcli import conf -from kcli import exceptions -from kcli import logger +from cli import conf +from cli import exceptions +from cli import logger LOG = logger.LOG diff --git a/tools/kcli/kcli/parse.py b/tools/cli/cli/parse.py similarity index 96% rename from tools/kcli/kcli/parse.py rename to tools/cli/cli/parse.py index 2d0014375..f2f6dc2b9 100644 --- a/tools/kcli/kcli/parse.py +++ b/tools/cli/cli/parse.py @@ -1,6 +1,6 @@ from argparse import ArgumentParser, RawTextHelpFormatter -from kcli import conf, execute, utils +from cli import conf, execute, utils def create_parser(options_trees): @@ -9,7 +9,7 @@ def create_parser(options_trees): :param options_trees: An iterable with OptionsTree objects :return: Namespace object """ - parser = ArgumentParser(prog="kcli", + parser = ArgumentParser(prog="cli", formatter_class=RawTextHelpFormatter) sub_parsers = parser.add_subparsers() @@ -40,7 +40,7 @@ def create_parser(options_trees): type=lambda file_path: utils.normalize_file( file_path), help="settings file to use. default: %s" - % conf.KCLI_SETTINGS_YML) + % conf.CLI_SETTINGS_YML) execute_parser.set_defaults(func=execute.ansible_wrapper) execute_parser.set_defaults(which='execute') diff --git a/tools/kcli/kcli/utils.py b/tools/cli/cli/utils.py similarity index 95% rename from tools/kcli/kcli/utils.py rename to tools/cli/cli/utils.py index 23e044539..36f3cef9d 100644 --- a/tools/kcli/kcli/utils.py +++ b/tools/cli/cli/utils.py @@ -8,10 +8,10 @@ import configure import yaml -import kcli.yamls -import kcli.conf -from kcli import exceptions -from kcli import logger +import cli.yamls +import cli.conf +from cli import exceptions +from cli import logger LOG = logger.LOG @@ -51,7 +51,7 @@ def validate_settings_dir(settings_dir=None): exist """ settings_dir = settings_dir or os.environ.get( - kcli.conf.KHALEESI_DIR_ENV_VAR) + cli.conf.INFRARED_DIR_ENV_VAR) if not os.path.exists(settings_dir): raise exceptions.IRFileNotFoundException( @@ -74,7 +74,7 @@ def update_settings(settings, file_path): try: loaded_file = configure.Configuration.from_file(file_path).configure() - placeholders_list = kcli.yamls.Placeholder.placeholders_list + placeholders_list = cli.yamls.Placeholder.placeholders_list for placeholder in placeholders_list[::-1]: if placeholders_list[-1].file_path is None: placeholder.file_path = file_path diff --git a/tools/kcli/kcli/yamls.py b/tools/cli/cli/yamls.py similarity index 99% rename from tools/kcli/kcli/yamls.py rename to tools/cli/cli/yamls.py index e20641b37..089c52fc1 100644 --- a/tools/kcli/kcli/yamls.py +++ b/tools/cli/cli/yamls.py @@ -10,8 +10,8 @@ import configure import yaml -from kcli import exceptions -from kcli import logger +from cli import exceptions +from cli import logger LOG = logger.LOG diff --git a/tools/cli/etc/kcli.cfg.example b/tools/cli/etc/kcli.cfg.example new file mode 100644 index 000000000..ac7b7091e --- /dev/null +++ b/tools/cli/etc/kcli.cfg.example @@ -0,0 +1,19 @@ +[DEFAULTS] +INFRARED_DIR = /home/aopincar/Git/Repos/Infrared +SETTINGS_DIR = %(INFRARED_DIR)s/settings +MODULES_DIR = %(INFRARED_DIR)s/library +ROLES_DIR = %(INFRARED_DIR)s/roles +PLAYBOOKS_DIR = %(INFRARED_DIR)s/playbooks +CLI_SETTINGS_YML = cli_settings.yml + +[ROOT_OPTS] +provisioner +installer +tester +distro +product + +[AUTO_EXEC_OPTS] +provision +install +test diff --git a/tools/kcli/requirements.txt b/tools/cli/requirements.txt similarity index 100% rename from tools/kcli/requirements.txt rename to tools/cli/requirements.txt diff --git a/tools/kcli/setup.py b/tools/cli/setup.py similarity index 85% rename from tools/kcli/setup.py rename to tools/cli/setup.py index 83f44a587..b92df7386 100644 --- a/tools/kcli/setup.py +++ b/tools/cli/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages from os.path import join, dirname, abspath -import kcli +import cli # parse_requirements() returns generator of pip.req.InstallRequirement objects install_reqs = req.parse_requirements('requirements.txt', session=False) @@ -13,12 +13,12 @@ prj_dir = dirname(abspath(__file__)) setup( - name='kcli', - version=kcli.__VERSION__, + name='cli', + version=cli.__VERSION__, packages=find_packages(), long_description=open(join(prj_dir, 'README.rst')).read(), entry_points={ - 'console_scripts': ['kcli = kcli.main:main'] + 'console_scripts': ['cli = cli.main:main'] }, install_requires=reqs, author='Yair Fried', diff --git a/tools/kcli/tests/__init__.py b/tools/cli/tests/__init__.py similarity index 100% rename from tools/kcli/tests/__init__.py rename to tools/cli/tests/__init__.py diff --git a/tools/kcli/tests/test_conf.py b/tools/cli/tests/test_conf.py similarity index 71% rename from tools/kcli/tests/test_conf.py rename to tools/cli/tests/test_conf.py index 59fed1454..24b07192b 100644 --- a/tools/kcli/tests/test_conf.py +++ b/tools/cli/tests/test_conf.py @@ -6,8 +6,8 @@ def test_get_config_dir(our_cwd_setup): - from kcli import conf + from cli import conf conf_file = conf.load_config_file() assert os.path.abspath( - conf_file.get("DEFAULTS", "KHALEESI_DIR")) == os.path.abspath( + conf_file.get("DEFAULTS", "INFRARED_DIR")) == os.path.abspath( utils.TESTS_CWD) diff --git a/tools/kcli/tests/test_cwd/IRYAMLConstructorError.yml b/tools/cli/tests/test_cwd/IRYAMLConstructorError.yml similarity index 100% rename from tools/kcli/tests/test_cwd/IRYAMLConstructorError.yml rename to tools/cli/tests/test_cwd/IRYAMLConstructorError.yml diff --git a/tools/kcli/tests/test_cwd/__init__.py b/tools/cli/tests/test_cwd/__init__.py similarity index 100% rename from tools/kcli/tests/test_cwd/__init__.py rename to tools/cli/tests/test_cwd/__init__.py diff --git a/tools/cli/tests/test_cwd/kcli.cfg b/tools/cli/tests/test_cwd/kcli.cfg new file mode 100644 index 000000000..7964cf6c8 --- /dev/null +++ b/tools/cli/tests/test_cwd/kcli.cfg @@ -0,0 +1,3 @@ +[DEFAULTS] +INFRARED_DIR = . +SETTINGS_DIR = %(INFRARED_DIR)s/settings diff --git a/tools/kcli/tests/test_cwd/placeholder_injector.yml b/tools/cli/tests/test_cwd/placeholder_injector.yml similarity index 100% rename from tools/kcli/tests/test_cwd/placeholder_injector.yml rename to tools/cli/tests/test_cwd/placeholder_injector.yml diff --git a/tools/kcli/tests/test_cwd/placeholder_overwriter.yml b/tools/cli/tests/test_cwd/placeholder_overwriter.yml similarity index 100% rename from tools/kcli/tests/test_cwd/placeholder_overwriter.yml rename to tools/cli/tests/test_cwd/placeholder_overwriter.yml diff --git a/tools/kcli/tests/test_cwd/utils.py b/tools/cli/tests/test_cwd/utils.py similarity index 96% rename from tools/kcli/tests/test_cwd/utils.py rename to tools/cli/tests/test_cwd/utils.py index 5517c7f12..db4cb3dfb 100644 --- a/tools/kcli/tests/test_cwd/utils.py +++ b/tools/cli/tests/test_cwd/utils.py @@ -9,7 +9,7 @@ def os_environ(): """ Backups env var from os.environ and restores it at teardown. """ - from kcli import conf + from cli import conf backup_flag = False if conf.ENV_VAR_NAME in os.environ: diff --git a/tools/kcli/tests/test_utils.py b/tools/cli/tests/test_utils.py similarity index 94% rename from tools/kcli/tests/test_utils.py rename to tools/cli/tests/test_utils.py index cfaf4840c..5d35cda04 100644 --- a/tools/kcli/tests/test_utils.py +++ b/tools/cli/tests/test_utils.py @@ -10,6 +10,6 @@ ['b_key1', 'b_key2'], {'a_key': 'a_val', 'b_key1': {'b_key2': 'x_val'}}), ]) def test_dict_insert(tested, val, key, expected): - from kcli import utils + from cli import utils utils.dict_insert(tested, val, *key) assert tested == expected diff --git a/tools/kcli/tests/test_yaml.py b/tools/cli/tests/test_yaml.py similarity index 85% rename from tools/kcli/tests/test_yaml.py rename to tools/cli/tests/test_yaml.py index 6473e47fa..cf0501668 100644 --- a/tools/kcli/tests/test_yaml.py +++ b/tools/cli/tests/test_yaml.py @@ -10,8 +10,8 @@ def test_unsupported_yaml_constructor(our_cwd_setup): - from kcli.utils import update_settings - from kcli.exceptions import IRYAMLConstructorError + from cli.utils import update_settings + from cli.exceptions import IRYAMLConstructorError tester_file = 'IRYAMLConstructorError.yml' settings = configure.Configuration.from_dict({}) with pytest.raises(IRYAMLConstructorError): @@ -19,9 +19,9 @@ def test_unsupported_yaml_constructor(our_cwd_setup): def test_placeholder_validator(our_cwd_setup): - from kcli.utils import update_settings - from kcli.exceptions import IRPlaceholderException - from kcli.yamls import Placeholder + from cli.utils import update_settings + from cli.exceptions import IRPlaceholderException + from cli.yamls import Placeholder injector = 'placeholder_injector.yml' overwriter = 'placeholder_overwriter.yml' diff --git a/tools/kcli/etc/kcli.cfg.example b/tools/kcli/etc/kcli.cfg.example deleted file mode 100644 index eae881d2f..000000000 --- a/tools/kcli/etc/kcli.cfg.example +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULTS] -KHALEESI_DIR = /home/aopincar/Git/Repos/khaleesi -SETTINGS_DIR = %(KHALEESI_DIR)s/settings -MODULES_DIR = %(KHALEESI_DIR)s/library -ROLES_DIR = %(KHALEESI_DIR)s/roles -PLAYBOOKS_DIR = %(KHALEESI_DIR)s/playbooks -KCLI_SETTINGS_YML = kcli_settings.yml - -[ROOT_OPTS] -provisioner -installer -tester -distro -product - -[AUTO_EXEC_OPTS] -provision -install -test diff --git a/tools/kcli/tests/test_cwd/kcli.cfg b/tools/kcli/tests/test_cwd/kcli.cfg deleted file mode 100644 index 27f3d8ac4..000000000 --- a/tools/kcli/tests/test_cwd/kcli.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULTS] -KHALEESI_DIR = . -SETTINGS_DIR = %(KHALEESI_DIR)s/settings From 60b87cc436910d7f4dc152d3ecd5c07dad63a7ac Mon Sep 17 00:00:00 2001 From: Yair Fried Date: Sun, 21 Feb 2016 15:31:46 +0200 Subject: [PATCH 20/22] More deref --- .gitignore | 3 +- tools/__init__.py | 1 - tools/cli/README.rst | 59 ++++++++----------- tools/cli/cli/conf.py | 16 ++--- tools/cli/cli/main.py | 9 +-- tools/cli/cli/parse.py | 5 +- ...{kcli.cfg.example => infrared.cfg.example} | 2 +- tools/cli/setup.py | 4 +- tools/cli/tests/test_conf.py | 2 +- .../tests/test_cwd/{kcli.cfg => infrared.cfg} | 0 tools/cli/tests/test_yaml.py | 2 +- 11 files changed, 47 insertions(+), 56 deletions(-) delete mode 100644 tools/__init__.py rename tools/cli/etc/{kcli.cfg.example => infrared.cfg.example} (89%) rename tools/cli/tests/test_cwd/{kcli.cfg => infrared.cfg} (100%) diff --git a/.gitignore b/.gitignore index 273b8432f..1901e87cf 100644 --- a/.gitignore +++ b/.gitignore @@ -29,9 +29,8 @@ stackrc id_rsa* hosts *-hosts -ksgen_settings.yml instack_hosts doc/_build/ fence_xvm.key vm-host-table -tools/kcli/etc/kcli.cfg +tools/cli/etc/infrared.cfg diff --git a/tools/__init__.py b/tools/__init__.py deleted file mode 100644 index 09668b120..000000000 --- a/tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'yfried' diff --git a/tools/cli/README.rst b/tools/cli/README.rst index 513d612a8..503d9f057 100644 --- a/tools/cli/README.rst +++ b/tools/cli/README.rst @@ -1,8 +1,8 @@ -======================== -kcli - Khaleesi CLI tool -======================== +================= +InfraRed CLI tool +================= -``kcli`` is intended to reduce Khaleesi users' dependency on external CLI tools. +Intended to reduce InfraRed users' dependency on external CLI tools. Setup ===== @@ -14,63 +14,56 @@ Setup Use pip to install from source:: - $ pip install tools/kcli + $ pip install tools/cli .. note:: For development work it's better to install in editable mode:: - $ pip install -e tools/kcli + $ pip install -e tools/cli Conf ==== -.. note:: Assumes that ``kcli`` is installed, else follow Setup_. +.. note:: Assumes that ``infrared`` is installed, else follow Setup_. -``kcli`` will look for ``kcli.cfg`` in the following order: +``infrared`` will look for ``infrared.cfg`` in the following order: -#. In working directory: ``./kcli.cfg`` -#. In user home directory: ``~/.kcli.cfg`` -#. In system settings: ``/etc/khaleesi/kcli.cfg`` +#. In working directory: ``./infrared.cfg`` +#. In user home directory: ``~/.infrared.cfg`` +#. In system settings: ``/etc/infrared /infrared.cfg`` .. note:: To specify a different directory or different filename, override the - lookup order with ``KCLI_CONFIG`` environment variable:: + lookup order with ``IR_CONFIG`` environment variable:: - $ KCLI_CONFIG=/my/config/file.ini kcli --help + $ IR_CONFIG=/my/config/file.ini infrared --help -Running kcli -============ +Running InfraRed +================ -.. note:: Assumes that ``kcli`` is installed, else follow Setup_. +.. note:: Assumes that ``infrared`` is installed, else follow Setup_. You can get general usage information with the ``--help`` option:: - kcli --help + infrared --help -This displays options you can pass to ``kcli``. - -.. note:: Some setting files are hard-coded to look for the ``$WORKSPACE`` - environment variable (see `Khaleesi - Cookbook`) that should point to the - directory where ``khaleesi`` and ``khaleesi-settings`` have been cloned. You - can define it manually to work around that:: - - $ export WORKSAPCE=$(dirname `pwd`) +This displays options you can pass to ``infrared``. Extra-Vars ---------- One can set/overwrite settings in the output file using the '-e/--extra-vars' option. There are 2 ways of doing so: -1. specific settings: (key=value form) - --extra-vars provisioner.site.user=a_user -2. path to a settings file: (starts with '@') - --extra-vars @path/to/a/settings_file.yml +1. specific settings: (``key=value`` form) + ``--extra-vars provisioner.site.user=a_user`` +2. path to a settings file: (starts with ``@``) + ``--extra-vars @path/to/a/settings_file.yml`` -The '-e/--extra-vars' can be used more than once. +The ``-e``/``--extra-vars`` can be used more than once. Merging order ------------- -Except options based on the settings dir structure, kcli accepts input of -predefined settings files (with -n/--input) and user defined specific options -(-e/--extra-vars). +Except options based on the settings dir structure, ``infrared`` accepts input of +predefined settings files (with ``-n``/``--input``) and user defined specific options +(``-e``/``--extra-vars``). The merging priority order listed below: 1. Input files diff --git a/tools/cli/cli/conf.py b/tools/cli/cli/conf.py index 2e1705c8f..7634485cd 100644 --- a/tools/cli/cli/conf.py +++ b/tools/cli/cli/conf.py @@ -4,14 +4,14 @@ from cli import exceptions -ENV_VAR_NAME = "CLI_CONFIG" -CLI_CONF_FILE = 'cli.cfg' -CWD_PATH = os.path.join(os.getcwd(), CLI_CONF_FILE) -USER_PATH = os.path.expanduser('~/.' + CLI_CONF_FILE) -SYSTEM_PATH = os.path.join('/etc/infrared', CLI_CONF_FILE) +ENV_VAR_NAME = "IR_CONFIG" +IR_CONF_FILE = 'infrared.cfg' +CWD_PATH = os.path.join(os.getcwd(), IR_CONF_FILE) +USER_PATH = os.path.expanduser('~/.' + IR_CONF_FILE) +SYSTEM_PATH = os.path.join('/etc/infrared', IR_CONF_FILE) YAML_EXT = ".yml" -TMP_OUTPUT_FILE = 'cli_settings_' + str(time.time()) + YAML_EXT -INFRARED_DIR_ENV_VAR = 'INFRARED_SETTINGS' +TMP_OUTPUT_FILE = 'ir_settings_' + str(time.time()) + YAML_EXT +INFRARED_DIR_ENV_VAR = 'IR_SETTINGS' def load_config_file(): @@ -24,7 +24,7 @@ def load_config_file(): if env_path is not None: env_path = os.path.expanduser(env_path) if os.path.isdir(env_path): - env_path = os.path.join(env_path, CLI_CONF_FILE) + env_path = os.path.join(env_path, IR_CONF_FILE) for path in (env_path, CWD_PATH, USER_PATH, SYSTEM_PATH): if path is not None and os.path.exists(path): _config.read(path) diff --git a/tools/cli/cli/main.py b/tools/cli/cli/main.py index e2437d8d4..224f7d48b 100755 --- a/tools/cli/cli/main.py +++ b/tools/cli/cli/main.py @@ -10,7 +10,7 @@ from cli import conf from cli import options as cli_options -from cli.execute import PLAYBOOKS +from cli import execute from cli import parse from cli import utils import cli.yamls @@ -56,7 +56,7 @@ def main(): LOG.debug("All settings files to be loaded:\n%s" % settings_files) cli.yamls.Lookup.settings = utils.generate_settings(settings_files, - args.extra_vars) + args.extra_vars) LOG.debug("Dumping settings...") output = yaml.safe_dump(cli.yamls.Lookup.settings, @@ -76,9 +76,10 @@ def main(): if exec_playbook: if args.which == 'execute': execute_args = parser.parse_args() - elif args.which not in PLAYBOOKS: + elif args.which not in execute.PLAYBOOKS: LOG.debug("No playbook named \"%s\", nothing to execute.\n" - "Please choose from: %s" % (args.which, PLAYBOOKS)) + "Please choose from: %s" % (args.which, + execute.PLAYBOOKS)) return else: args_list = ["execute"] diff --git a/tools/cli/cli/parse.py b/tools/cli/cli/parse.py index f2f6dc2b9..5325244bc 100644 --- a/tools/cli/cli/parse.py +++ b/tools/cli/cli/parse.py @@ -9,8 +9,7 @@ def create_parser(options_trees): :param options_trees: An iterable with OptionsTree objects :return: Namespace object """ - parser = ArgumentParser(prog="cli", - formatter_class=RawTextHelpFormatter) + parser = ArgumentParser(formatter_class=RawTextHelpFormatter) sub_parsers = parser.add_subparsers() @@ -40,7 +39,7 @@ def create_parser(options_trees): type=lambda file_path: utils.normalize_file( file_path), help="settings file to use. default: %s" - % conf.CLI_SETTINGS_YML) + % conf.IR_SETTINGS_YML) execute_parser.set_defaults(func=execute.ansible_wrapper) execute_parser.set_defaults(which='execute') diff --git a/tools/cli/etc/kcli.cfg.example b/tools/cli/etc/infrared.cfg.example similarity index 89% rename from tools/cli/etc/kcli.cfg.example rename to tools/cli/etc/infrared.cfg.example index ac7b7091e..988365385 100644 --- a/tools/cli/etc/kcli.cfg.example +++ b/tools/cli/etc/infrared.cfg.example @@ -4,7 +4,7 @@ SETTINGS_DIR = %(INFRARED_DIR)s/settings MODULES_DIR = %(INFRARED_DIR)s/library ROLES_DIR = %(INFRARED_DIR)s/roles PLAYBOOKS_DIR = %(INFRARED_DIR)s/playbooks -CLI_SETTINGS_YML = cli_settings.yml +IR_SETTINGS_YML = ir_settings.yml [ROOT_OPTS] provisioner diff --git a/tools/cli/setup.py b/tools/cli/setup.py index b92df7386..ed05ecf84 100644 --- a/tools/cli/setup.py +++ b/tools/cli/setup.py @@ -13,12 +13,12 @@ prj_dir = dirname(abspath(__file__)) setup( - name='cli', + name='infrared', version=cli.__VERSION__, packages=find_packages(), long_description=open(join(prj_dir, 'README.rst')).read(), entry_points={ - 'console_scripts': ['cli = cli.main:main'] + 'console_scripts': ['infrared = cli.main:main'] }, install_requires=reqs, author='Yair Fried', diff --git a/tools/cli/tests/test_conf.py b/tools/cli/tests/test_conf.py index 24b07192b..75535bc56 100644 --- a/tools/cli/tests/test_conf.py +++ b/tools/cli/tests/test_conf.py @@ -1,6 +1,6 @@ import os -from tests.test_cwd import utils +from test_cwd import utils our_cwd_setup = utils.our_cwd_setup diff --git a/tools/cli/tests/test_cwd/kcli.cfg b/tools/cli/tests/test_cwd/infrared.cfg similarity index 100% rename from tools/cli/tests/test_cwd/kcli.cfg rename to tools/cli/tests/test_cwd/infrared.cfg diff --git a/tools/cli/tests/test_yaml.py b/tools/cli/tests/test_yaml.py index cf0501668..7247e32c8 100644 --- a/tools/cli/tests/test_yaml.py +++ b/tools/cli/tests/test_yaml.py @@ -43,5 +43,5 @@ def test_placeholder_validator(our_cwd_setup): os.path.join(utils.TESTS_CWD, overwriter)) assert settings['place']['holder'][ - 'validator'] == "'!placeholder' has been overwritten" + 'validator'] == "'!placeholder' has been overwritten" yaml.safe_dump(settings, default_flow_style=False) From 590c2c4c4921d781b9245a15d292b1358ad32682 Mon Sep 17 00:00:00 2001 From: rhosqeauto Date: Wed, 13 Apr 2016 18:57:47 +1000 Subject: [PATCH 21/22] A testing file only RHOSINFRA-140 #A testing file to check whether or not the sync between jira and github is alright. --- TestingOnly | 1 + 1 file changed, 1 insertion(+) create mode 100644 TestingOnly diff --git a/TestingOnly b/TestingOnly new file mode 100644 index 000000000..d14c6c9eb --- /dev/null +++ b/TestingOnly @@ -0,0 +1 @@ +Just a testing file, can be deleted anytime From 670320960b10a439b48a554cacc3de843e2d68fb Mon Sep 17 00:00:00 2001 From: rhosqeauto Date: Wed, 13 Apr 2016 19:04:58 +1000 Subject: [PATCH 22/22] Delete TestingOnly --- TestingOnly | 1 - 1 file changed, 1 deletion(-) delete mode 100644 TestingOnly diff --git a/TestingOnly b/TestingOnly deleted file mode 100644 index d14c6c9eb..000000000 --- a/TestingOnly +++ /dev/null @@ -1 +0,0 @@ -Just a testing file, can be deleted anytime