From b353ee199a0decbca61a1e6a9aec89075e1ad52e Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Sun, 13 Jan 2019 21:21:08 +0100 Subject: [PATCH 1/9] chg: doc: wrong spelling of "explicitly" --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 31c65e4..5904f7c 100644 --- a/README.rst +++ b/README.rst @@ -535,7 +535,7 @@ an error message will be printed:: $ echo "a: 3" | shyaml get-value b Error: invalid path 'b', missing key 'b' in struct. -You can emulate pre v0.3 behavior by specifying explicitely an empty +You can emulate pre v0.3 behavior by specifying explicitly an empty string as third argument:: $ echo "a: 3" | shyaml get-value b '' @@ -603,7 +603,7 @@ when processing YAML coming out of shyaml, you should probably think about using the ``--yaml`` (or ``-y``) option to output only strict YAML. With the drawback that when you'll want to output string, you'll need to -call a last time ``shyaml get-value`` to explicitely unquote the YAML. +call a last time ``shyaml get-value`` to explicitly unquote the YAML. Object Tag From 69a4bdfbef20a71a32db1ff989f3ecd72b54ef8f Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Fri, 14 Dec 2018 18:34:48 +0100 Subject: [PATCH 2/9] chg: dev: shtest output more columnised output --- shtest.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/shtest.py b/shtest.py index b260e78..343d50c 100755 --- a/shtest.py +++ b/shtest.py @@ -308,6 +308,11 @@ def get_meta_commands(command): def shtest_runner(lines, regex_patterns): + def _lines(start_line_nb, stop_line_nb): + return (("lines %9s" % ("%s-%s" % (start_line_nb, stop_line_nb))) + if start_line_nb != stop_line_nb else + ("line %10s" % start_line_nb)) + for block_nb, block in enumerate(get_docshtest_blocks(lines)): lines = iter(block) command_block = "" @@ -328,28 +333,20 @@ def shtest_runner(lines, regex_patterns): run_and_check(command_block, "".join(line for _, line in lines)) except UnmatchedLine as e: print(format_failed_test( - "shtest %d - failure (line %s):" - % (block_nb + 1, - ("%s-%s" % (start_line_nb, stop_line_nb)) - if start_line_nb != stop_line_nb else - start_line_nb), + "#%04d - failure (%15s):" + % (block_nb + 1, _lines(start_line_nb, stop_line_nb)), command_block, e.args[0], e.args[1])) exit(1) except Ignored as e: - print("shtest %d - ignored (line %s): %s" + print("#%04d - ignored (%15s): %s" % (block_nb + 1, - (("%s-%s" % (start_line_nb, stop_line_nb)) - if start_line_nb != stop_line_nb else - start_line_nb), + _lines(start_line_nb, stop_line_nb), " ".join(e.args))) else: - print("shtest %d - success (line %s)" - % (block_nb + 1, - ("%s-%s" % (start_line_nb, stop_line_nb)) - if start_line_nb != stop_line_nb else - start_line_nb)) + print("#%04d - success (%15s)" + % (block_nb + 1, _lines(start_line_nb, stop_line_nb))) sys.stdout.flush() From 473f609b049298d408a07783d8f642b6e04b0fee Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 12 Feb 2019 10:59:21 +0100 Subject: [PATCH 3/9] new: pkg: update license years. --- LICENSE | 2 +- README.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 06b8177..3c6bb91 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012-2018, Valentin Lab +Copyright (c) 2012-2019, Valentin Lab All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.rst b/README.rst index 5904f7c..275e69c 100644 --- a/README.rst +++ b/README.rst @@ -905,7 +905,7 @@ would show you how to deal with your issue. License ======= -Copyright (c) 2018 Valentin Lab. +Copyright (c) 2019 Valentin Lab. Licensed under the `BSD License`_. From 959bcc5a66e14414218b3c3fd206afa6a81caecb Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 12 Feb 2019 10:58:08 +0100 Subject: [PATCH 4/9] new: pkg: ``shtest`` is now an external test dependency. --- README.rst | 8 +- bin/test | 8 +- setup.py | 4 +- shtest.py | 421 ----------------------------------------------------- 4 files changed, 11 insertions(+), 430 deletions(-) delete mode 100755 shtest.py diff --git a/README.rst b/README.rst index 275e69c..6fc78bb 100644 --- a/README.rst +++ b/README.rst @@ -83,7 +83,7 @@ implementation but some examples will fail depending on the implementation. To make things clear, I'll use some annotation and you can yourself check which version you are using with:: - $ shyaml -V | grep "^libyaml used:" ## shtest: if-success-set LIBYAML + $ shyaml -V | grep "^libyaml used:" ## docshtest: if-success-set LIBYAML libyaml used: True @@ -153,7 +153,7 @@ Get sub YAML from a structure attribute:: $ cat test.yaml | shyaml get-type subvalue struct - $ cat test.yaml | shyaml get-value subvalue ## shtest: ignore-if LIBYAML + $ cat test.yaml | shyaml get-value subvalue ## docshtest: ignore-if LIBYAML how-much: 1.1 how-many: 2 things: @@ -629,7 +629,7 @@ allow parsing their internal structure. ``get-value`` with ``-y`` (see section Strict YAML) will give you the complete yaml tagged value:: - $ shyaml get-value -y 0 < test.yaml ## shtest: ignore-if LIBYAML + $ shyaml get-value -y 0 < test.yaml ## docshtest: ignore-if LIBYAML ! 'bar' @@ -678,7 +678,7 @@ Note that all global tags will be resolved and simplified (as } EOF - $ shyaml get-value < test.yaml ## shtest: ignore-if LIBYAML + $ shyaml get-value < test.yaml ## docshtest: ignore-if LIBYAML sequence: - one - two diff --git a/bin/test b/bin/test index ded8699..eca42be 100755 --- a/bin/test +++ b/bin/test @@ -1,11 +1,11 @@ #!/bin/bash -shtest_opts=() +docshtest_opts=() if [ -z "$DOVIS" -o "$PKG_COVERAGE" ]; then ## with coverage echo "With coverage support enabled" python="coverage run --include ./shyaml.py -a" - shtest_opts+=("-r" '#\bshyaml\b#'"$coverage"' shyaml#') + docshtest_opts+=("-r" '#\bshyaml\b#'"$coverage"' shyaml#') else echo "No coverage support" python=python @@ -17,10 +17,10 @@ $python -m doctest README.rst || exit 1 if python -c 'import yaml; exit(0 if yaml.__with_libyaml__ else 1)' 2>/dev/null; then echo "PyYAML has C libyaml bindings available... Testing with libyaml" export FORCE_PYTHON_YAML_IMPLEMENTATION= - time ./shtest.py README.rst -r '#\bshyaml\b#'"$python"' ./shyaml.py#' || exit 1 + time docshtest README.rst -r '#\bshyaml\b#'"$python"' ./shyaml.py#' || exit 1 else echo "PyYAML has NOT any C libyaml bindings available..." fi echo "Testing with python implementation" export FORCE_PYTHON_YAML_IMPLEMENTATION=1 -time ./shtest.py README.rst -r '#\bshyaml\b#'"$python"' ./shyaml.py#' || exit 1 +time docshtest README.rst -r '#\bshyaml\b#'"$python"' ./shyaml.py#' || exit 1 diff --git a/setup.py b/setup.py index 362dc53..f120147 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,8 @@ setup( setup_requires=['d2to1'], - extras_require={'test': []}, + extras_require={'test': [ + "docshtest==0.0.2", + ]}, d2to1=True ) diff --git a/shtest.py b/shtest.py deleted file mode 100755 index 343d50c..0000000 --- a/shtest.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python -"""Shell Doctest - -First naive implementation, will probably have to move this -into its own project. - - -Major concerns and shortcomings that prevents it to be a serious project: -- end of blocks and final "\n" are not tested correctly -- tests execution in current directory with possible consequences. -- no support of checking errlvl -- no support of proper mixed err and stdout content -- limited to ``bash`` testing - -Minor concerns, but would be better without: -- fail on first error hardwritten. -- hardwritten support of "" - -Possible evolution: -- support of python file (by extracting docs before) -- integration in nosetests ? is it possible ? -- colorize output ? -- move to standalone full fledged program ? -- coverage integration ? - -""" - -from __future__ import print_function - - -import re -import sys -import os.path -import subprocess -import difflib -import threading - - -try: - from Queue import Queue, Empty -except ImportError: - from queue import Queue, Empty # python 3.x - - -PY3 = sys.version_info[0] >= 3 -WIN32 = sys.platform == 'win32' - -EXNAME = os.path.basename(__file__ if WIN32 else sys.argv[0]) - -for ext in (".py", ".pyc", ".exe", "-script.py", "-script.pyc"): - if EXNAME.endswith(ext): - EXNAME = EXNAME[:-len(ext)] - break - - -USAGE = """\ -Usage: - - %(exname)s (-h|--help) - %(exname)s [[-r|--regex REGEX] ...] SHTESTFILE -""" % {"exname": EXNAME} - - -HELP = """\ - -%(exname)s - parse file and run shell doctests - -%(usage)s - -Options: - - -r REGEX, --regex REGEX - Will apply this regex to the lines to be executed. You - can have more than one patterns by re-using this options - as many times as wanted. Regexps will be applied one by one - in the same order than they are provided on the command line. - - -Examples: - - ## run tests but replace executable on-the-fly for coverage support - shtest README.rst -r '/\\bshyaml\\b/coverage run shyaml.py/' - -""" % {"exname": EXNAME, "usage": USAGE} - - -## command line quoting -cmd_line_quote = (lambda e: e.replace('\\', '\\\\')) if WIN32 else (lambda e: e) - - -## -## Helpers coming from othe projects -## - - -## XXXvlab: code comes from kids.txt.diff -def udiff(a, b, fa="", fb=""): - if not a.endswith("\n"): - a += "\n" - if not b.endswith("\n"): - b += "\n" - return "".join( - difflib.unified_diff( - a.splitlines(1), b.splitlines(1), - fa, fb)) - - -## XXXvlab: code comes from ``kids.sh`` -ON_POSIX = 'posix' in sys.builtin_module_names - -__ENV__ = {} - -## XXXvlab: code comes from ``kids.txt`` -## Note that a quite equivalent function was added to textwrap in python 3.3 -def indent(text, prefix=" ", first=None): - if first is not None: - first_line = text.split("\n")[0] - rest = '\n'.join(text.split("\n")[1:]) - return '\n'.join([first + first_line, - indent(rest, prefix=prefix)]) - return '\n'.join([prefix + line - for line in text.split('\n')]) - - -## XXXvlab: consider for inclusion in ``kids.sh`` -def cmd_iter(cmd, encoding="utf-8"): - """Asynchrone subprocess driver - - returns an iterator that yields events of the life of the - process. - - """ - - def thread_enqueue(label, f, q): - t = threading.Thread(target=enqueue_output, args=(label, f, q)) - t.daemon = True ## thread dies with the program - t.start() - return t - - decode = (lambda s: s) if PY3 else (lambda s: s.decode(encoding)) - - def enqueue_output(label, out, queue): - for line in iter(out.readline, '' if PY3 else b''): - # print("%s: %s" % (label, chomp(line))) - queue.put((label, decode(line))) - # print("END of %s" % (label, )) - out.close() - - proc = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=ON_POSIX, - shell=False, - universal_newlines=True, - env=None) - proc.stdin.close() - q = Queue() - t1 = thread_enqueue("out", proc.stdout, q) - t2 = thread_enqueue("err", proc.stderr, q) - running = True - while True: - try: - yield q.get(True, 0.001) - except Empty: - if not running: - break - proc.poll() - running = proc.returncode is None or \ - any(t.is_alive() for t in (t1, t2)) - - yield "errorlevel", proc.returncode - - -## XXXvlab: consider for inclusion in ``kids.txt`` -def chomp(s): - if len(s): - lines = s.splitlines(True) - last = lines.pop() - return ''.join(lines + last.splitlines()) - else: - return '' - - -def get_docshtest_blocks(lines): - """Returns an iterator of shelltest blocks from an iterator of lines""" - - block = [] - consecutive_empty = 0 - for line_nb, line in enumerate(lines): - is_empty_line = not line.strip() - if not is_empty_line: - if not line.startswith(" "): - if block: - yield block[:-consecutive_empty] if consecutive_empty else block - block = [] - continue - else: - line = line[4:] - if line.startswith("$ ") or block: - if line.startswith("$ "): - line = line[2:] - if block: - yield block[:-consecutive_empty] if consecutive_empty else block - block = [] - if is_empty_line: - consecutive_empty += 1 - else: - consecutive_empty = 0 - block.append((line_nb + 1, line)) - - -def valid_syntax(command): - """Check if shell command if complete""" - - for ev, value in cmd_iter(["bash", "-n", "-c", cmd_line_quote(command)]): - if ev == "err": - if value.endswith("syntax error: unexpected end of file"): - return False - if "unexpected EOF while looking for matching" in value: - return False - if "here-document at line" in value: - return False - return value == 0 - - -class UnmatchedLine(Exception): - - def __init__(self, *args): - self.args = args - - -class Ignored(Exception): - - def __init__(self, *args): - self.args = args - - -def run_and_check(command, expected_output): - global __ENV__ - meta_commands = list(get_meta_commands(command)) - for meta_command in meta_commands: - if meta_command[0] == "ignore-if": - if meta_command[1] in __ENV__: - raise Ignored(*meta_command) - if meta_command[0] == "ignore-if-not": - if meta_command[1] not in __ENV__: - raise Ignored(*meta_command) - - - expected_output = expected_output.replace("\n", "\n") - orig_expected_output = expected_output - output = "" - diff = False - for ev, value in cmd_iter(["bash", "-c", cmd_line_quote(command)]): - if ev in ("err", "out"): - output += value - if not diff and expected_output.startswith(value): - expected_output = expected_output[len(value):] - else: - diff = True - if not diff and len(chomp(expected_output)): - diff = True - - for meta_command in meta_commands: - if meta_command[0] == "if-success-set": - if not diff: - __ENV__[meta_command[1]] = 1 - raise Ignored(*meta_command) - else: - raise Ignored(*meta_command) - if diff: - raise UnmatchedLine(output, orig_expected_output) - return value == 0 - - -def format_failed_test(message, command, output, expected): - formatted = [] - if "\n" in command: - formatted.append("command:\n%s" % indent(command, "| ")) - else: - formatted.append("command: %r" % command) - formatted.append("expected:\n%s" % indent(expected, "| ")) - formatted.append("output:\n%s" % indent(output, "| ")) - if len(expected.splitlines() + output.splitlines()) > 10: - formatted.append("diff:\n%s" % udiff(expected, output, "expected", "output")) - - formatted = '\n'.join(formatted) - - return "%s\n%s" % (message, indent(formatted, prefix=" ")) - - -def apply_regex(patterns, s): - for p in patterns: - s = re.sub(p[0], p[1], s) - return s - - -META_COMMAND_REGEX = '##? shtest: (?P.*)$' - - -def get_meta_commands(command): - for m in re.finditer(META_COMMAND_REGEX, command): - raw_cmd = m.groupdict()["cmd"] - cmd = raw_cmd.strip() - cmd = re.sub(' +', ' ', cmd) - yield cmd.split(' ') - - -def shtest_runner(lines, regex_patterns): - def _lines(start_line_nb, stop_line_nb): - return (("lines %9s" % ("%s-%s" % (start_line_nb, stop_line_nb))) - if start_line_nb != stop_line_nb else - ("line %10s" % start_line_nb)) - - for block_nb, block in enumerate(get_docshtest_blocks(lines)): - lines = iter(block) - command_block = "" - start_line_nb = None - stop_line_nb = None - for line_nb, line in lines: - start_line_nb = start_line_nb or line_nb - command_block += line - if valid_syntax(apply_regex(regex_patterns, - command_block)): - stop_line_nb = line_nb - break - else: - raise ValueError("Invalid Block:\n%s" % (indent(command_block, " | "))) - command_block = command_block.rstrip("\n\r") - command_block = apply_regex(regex_patterns, command_block) - try: - run_and_check(command_block, "".join(line for _, line in lines)) - except UnmatchedLine as e: - print(format_failed_test( - "#%04d - failure (%15s):" - % (block_nb + 1, _lines(start_line_nb, stop_line_nb)), - command_block, - e.args[0], - e.args[1])) - exit(1) - except Ignored as e: - print("#%04d - ignored (%15s): %s" - % (block_nb + 1, - _lines(start_line_nb, stop_line_nb), - " ".join(e.args))) - else: - print("#%04d - success (%15s)" - % (block_nb + 1, _lines(start_line_nb, stop_line_nb))) - sys.stdout.flush() - - -def split_quote(s, split_char='/', quote='\\'): - r"""Split args separated by char, possibily quoted with quote char - - - >>> tuple(split_sep_args('/pattern/replace/')) - ('', 'pattern', 'replace', '') - - >>> tuple(split_sep_args('/pat\/tern/replace/')) - ('', 'pat/tern', 'replace', '') - - >>> tuple(split_sep_args('/pat\/ter\n/replace/')) - ('', 'pat/ter\n', 'replace', '') - - """ - - buf = "" - parse_str = iter(s) - for char in parse_str: - if char == split_char: - yield buf - buf = "" - continue - if char == quote: - char = next(parse_str) - if char != split_char: - buf += quote - buf += char - yield buf - - -if __name__ == "__main__": - args = sys.argv[1:] - - pattern = None - if any(arg in args for arg in ["-h", "--help"]) or \ - len(args) == 0: - print(HELP) - exit(0) - - patterns = [] - for arg in ["-r", "--regex"]: - while arg in args: - idx = args.index(arg) - pattern = args[idx + 1] - del args[idx + 1] - del args[idx] - if re.match('^[a-zA-Z1-9]$', pattern[0]): - print("Error: regex %s should start with a delimiter char, " - "not an alphanumerical char." % pattern) - print(USAGE) - exit(1) - parts = tuple(split_quote(pattern, split_char=pattern[0])) - if not (parts[0] == parts[-1] == ''): - print("Error: regex should start and end with a delimiter char.") - exit(1) - parts = parts[1:-1] - if len(parts) > 2: - print("Error: Found too many delimiter char.") - exit(1) - patterns.append(parts) - - if len(args) == 0: - print("Error: please provide a rst filename as argument.") - exit(1) - filename = args[0] - if not os.path.exists(filename): - print("Error: file %r doesn't exists." % filename) - exit(1) - shtest_runner(open(filename), regex_patterns=patterns) From 829e93696e392595645a9d116bce24bb9d9e73ae Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 12 Feb 2019 11:02:26 +0100 Subject: [PATCH 5/9] fix: pkg: python 3.7 is also incompatible with old ``pkg_resources``. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c8db8c3..7b280de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ dist_check: options: exclude: - ["v:3.6", "pkg:old"] ## old version is breaking python 3.6 pkg_resources + - ["v:3.7", "pkg:old"] ## old version is breaking python 3.7 pkg_resources tests: - label: install matrix: From 510323bbc65a1f86f2c491db4ac7640dad627d92 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 12 Feb 2019 11:19:04 +0100 Subject: [PATCH 6/9] fix: pkg: cleaning unused code and variable. !minor --- bin/test | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/test b/bin/test index eca42be..9efef3f 100755 --- a/bin/test +++ b/bin/test @@ -5,22 +5,23 @@ docshtest_opts=() if [ -z "$DOVIS" -o "$PKG_COVERAGE" ]; then ## with coverage echo "With coverage support enabled" python="coverage run --include ./shyaml.py -a" - docshtest_opts+=("-r" '#\bshyaml\b#'"$coverage"' shyaml#') else echo "No coverage support" python=python fi +docshtest_opts+=("-r" '#\bshyaml\b#'"$python"' ./shyaml.py#') + $python -m doctest shyaml.py || exit 1 $python -m doctest README.rst || exit 1 if python -c 'import yaml; exit(0 if yaml.__with_libyaml__ else 1)' 2>/dev/null; then echo "PyYAML has C libyaml bindings available... Testing with libyaml" export FORCE_PYTHON_YAML_IMPLEMENTATION= - time docshtest README.rst -r '#\bshyaml\b#'"$python"' ./shyaml.py#' || exit 1 + time docshtest README.rst "${docshtest_opts[@]}" || exit 1 else echo "PyYAML has NOT any C libyaml bindings available..." fi echo "Testing with python implementation" export FORCE_PYTHON_YAML_IMPLEMENTATION=1 -time docshtest README.rst -r '#\bshyaml\b#'"$python"' ./shyaml.py#' || exit 1 +time docshtest README.rst "${docshtest_opts[@]}" || exit 1 From 8dcf9bb39f995564271600a4dae1fc114077c2e0 Mon Sep 17 00:00:00 2001 From: Valentin Lab Date: Tue, 12 Feb 2019 11:34:26 +0100 Subject: [PATCH 7/9] fix: pkg: remove "unbound variable" error in some matrix paths. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7b280de..acb21f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ dist_check: EOF - | cd /tmp - echo PYTHONPATH: $PYTHONPATH + echo PYTHONPATH: "${PYTHONPATH:-}" python -c 'import shyaml' - | [ -e /tmp/not-installed ] || { From d0024c4ae49ac100c5632b1c58a499c8373e616b Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Wed, 5 Aug 2020 11:09:11 +0200 Subject: [PATCH 8/9] CHANGE: Use argparse module for CLI parsing CLI behaviour has changed slightly: flags cannot be passed after `action`. Updated tests accordingly. --- README.rst | 41 +++++++----- shyaml.py | 193 +++++++++++++++++++++++------------------------------ 2 files changed, 109 insertions(+), 125 deletions(-) diff --git a/README.rst b/README.rst index 6fc78bb..ec6e0e7 100644 --- a/README.rst +++ b/README.rst @@ -363,7 +363,7 @@ So:: break fi echo "---" - done | shyaml get-value -y ingests.0.id + done | shyaml -y get-value ingests.0.id tag-1 ... --- @@ -426,7 +426,7 @@ With the ``-L``, if we kill our shyaml process before the end:: fi echo "---" sleep 10 - done 2>/dev/null | shyaml get-value -L ingests.0.id & pid=$! ; sleep 2; kill $pid + done 2>/dev/null | shyaml -L get-value ingests.0.id & pid=$! ; sleep 2; kill $pid tag-1 @@ -443,7 +443,7 @@ which could help you chain shyaml calls:: fi echo "---" sleep 0.2 - done | shyaml get-value ingests.0 -L -y | shyaml get-value id | tr '\0' '\n' + done | shyaml -L -y get-value ingests.0 | shyaml get-value id | tr '\0' '\n' tag-1 tag-2 tag-3 @@ -629,7 +629,7 @@ allow parsing their internal structure. ``get-value`` with ``-y`` (see section Strict YAML) will give you the complete yaml tagged value:: - $ shyaml get-value -y 0 < test.yaml ## docshtest: ignore-if LIBYAML + $ shyaml -y get-value 0 < test.yaml ## docshtest: ignore-if LIBYAML ! 'bar' @@ -656,7 +656,7 @@ Another example:: And you can still traverse internal value:: - $ shyaml get-value -y 2.start < test.yaml + $ shyaml -y get-value 2.start < test.yaml x: 73 y: 129 @@ -693,7 +693,7 @@ Empty documents When provided with an empty document, ``shyaml`` will consider the document to hold a ``null`` value:: - $ echo | shyaml get-value -y + $ echo | shyaml -y get-value null ... @@ -704,30 +704,26 @@ Usage string A quick reminder of what is available will be printed when calling ``shyaml`` without any argument:: - $ shyaml - Error: Bad number of arguments. - Usage: + $ shyaml + usage: shyaml {-h|--help} shyaml {-V|--version} shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] - The full help is available through the usage of the standard ``-h`` or ``-help``:: $ shyaml --help - - Parses and output chosen subpart or values from YAML input. - It reads YAML in stdin and will output on stdout it's return value. - - Usage: + usage: shyaml {-h|--help} shyaml {-V|--version} shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] + Parses and output chosen subpart or values from YAML input. + It reads YAML in stdin and will output on stdout it's return value. Options: @@ -803,13 +799,24 @@ The full help is available through the usage of the standard ``-h`` or ## get YAML config part of 'myhost' cat hosts_config.yaml | shyaml get-value cfgs.myhost - + positional arguments: + action + key + default + + optional arguments: + -h, --help show this help message and exit + -y, --yaml + -q, --quiet + -L, --line-buffer + -V, --version show program's version number and exit + Using invalid keywords will issue an error and the usage message:: $ shyaml get-foo Error: 'get-foo' is not a valid action. - Usage: + shyaml {-h|--help} shyaml {-V|--version} diff --git a/shyaml.py b/shyaml.py index cf1c252..3e3aa49 100755 --- a/shyaml.py +++ b/shyaml.py @@ -10,6 +10,7 @@ from __future__ import print_function +import argparse import sys import os.path import re @@ -492,44 +493,26 @@ def get_version_info(): def _parse_args(args, USAGE, HELP): - opts = {} - - opts["dump"] = magic_dump - for arg in ["-y", "--yaml"]: - if arg in args: - args.remove(arg) - opts["dump"] = yaml_dump - - opts["quiet"] = False - for arg in ["-q", "--quiet"]: - if arg in args: - args.remove(arg) - opts["quiet"] = True - - for arg in ["-L", "--line-buffer"]: - if arg not in args: - continue - args.remove(arg) - - opts["loader"] = LineLoader - - if len(args) == 0: - stderr("Error: Bad number of arguments.\n") - die(USAGE, errlvl=1, prefix="") - - if len(args) == 1 and args[0] in ("-h", "--help"): - stdout(HELP) - exit(0) - - if len(args) == 1 and args[0] in ("-V", "--version"): - version_info = get_version_info() - print("version: %s\nPyYAML: %s\nlibyaml available: %s\nlibyaml used: %s\nPython: %s" - % version_info) - exit(0) - - opts["action"] = args[0] - opts["key"] = None if len(args) == 1 else args[1] - opts["default"] = args[2] if len(args) > 2 else None + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + usage=USAGE, description=HELP) + parser.add_argument("action") + parser.add_argument("key", nargs=argparse.OPTIONAL) + parser.add_argument("default", nargs=argparse.OPTIONAL) + parser.add_argument("-y", "--yaml", dest="dump", action="store_const", + default=magic_dump, const=yaml_dump) + parser.add_argument("-q", "--quiet", action="store_true") + parser.add_argument("-L", "--line-buffer", dest="loader", + action="store_const", default=ShyamlSafeLoader, + const=LineLoader) + parser.add_argument("-V", "--version", action="version", + version="version: %s\nPyYAML: %s\n" + "libyaml available: %s\nlibyaml used: %s\n" + "Python: %s" % get_version_info()) + + parsed_args = parser.parse_args(args) + + opts = vars(parsed_args) return opts @@ -674,98 +657,92 @@ def main(args): ## pylint: disable=too-many-branches EXNAME = EXNAME[:-len(ext)] break - USAGE = """\ - Usage: + USAGE = """ - %(exname)s {-h|--help} - %(exname)s {-V|--version} - %(exname)s [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] - """ % {"exname": EXNAME} + %(exname)s {-h|--help} + %(exname)s {-V|--version} + %(exname)s [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] +""" % {"exname": EXNAME} HELP = """ - Parses and output chosen subpart or values from YAML input. - It reads YAML in stdin and will output on stdout it's return value. - -%(usage)s - - Options: - - -y, --yaml - Output only YAML safe value, more precisely, even - literal values will be YAML quoted. This behavior - is required if you want to output YAML subparts and - further process it. If you know you have are dealing - with safe literal value, then you don't need this. - (Default: no safe YAML output) +Parses and output chosen subpart or values from YAML input. +It reads YAML in stdin and will output on stdout it's return value. - -q, --quiet - In case KEY value queried is an invalid path, quiet - mode will prevent the writing of an error message on - standard error. - (Default: no quiet mode) +Options: - -L, --line-buffer - Force parsing stdin line by line allowing to process - streamed YAML as it is fed instead of buffering - input and treating several YAML streamed document - at once. This is likely to have some small performance - hit if you have a huge stream of YAML document, but - then you probably don't really care about the - line-buffering. - (Default: no line buffering) + -y, --yaml + Output only YAML safe value, more precisely, even + literal values will be YAML quoted. This behavior + is required if you want to output YAML subparts and + further process it. If you know you have are dealing + with safe literal value, then you don't need this. + (Default: no safe YAML output) - ACTION Depending on the type of data you've targetted - thanks to the KEY, ACTION can be: + -q, --quiet + In case KEY value queried is an invalid path, quiet + mode will prevent the writing of an error message on + standard error. + (Default: no quiet mode) + + -L, --line-buffer + Force parsing stdin line by line allowing to process + streamed YAML as it is fed instead of buffering + input and treating several YAML streamed document + at once. This is likely to have some small performance + hit if you have a huge stream of YAML document, but + then you probably don't really care about the + line-buffering. + (Default: no line buffering) - These ACTIONs applies to any YAML type: + ACTION Depending on the type of data you've targetted + thanks to the KEY, ACTION can be: - get-type ## returns a short string - get-value ## returns YAML + These ACTIONs applies to any YAML type: - These ACTIONs applies to 'sequence' and 'struct' YAML type: + get-type ## returns a short string + get-value ## returns YAML - get-values{,-0} ## returns list of YAML - get-length ## returns an integer + These ACTIONs applies to 'sequence' and 'struct' YAML type: - These ACTION applies to 'struct' YAML type: + get-values{,-0} ## returns list of YAML + get-length ## returns an integer - keys{,-0} ## returns list of YAML - values{,-0} ## returns list of YAML - key-values,{,-0} ## returns list of YAML + These ACTION applies to 'struct' YAML type: - Note that any value returned is returned on stdout, and - when returning ``list of YAML``, it'll be separated by - a newline or ``NUL`` char depending of you've used the - ``-0`` suffixed ACTION. + keys{,-0} ## returns list of YAML + values{,-0} ## returns list of YAML + key-values,{,-0} ## returns list of YAML - KEY Identifier to browse and target subvalues into YAML - structure. Use ``.`` to parse a subvalue. If you need - to use a literal ``.`` or ``\\``, use ``\\`` to quote it. + Note that any value returned is returned on stdout, and + when returning ``list of YAML``, it'll be separated by + a newline or ``NUL`` char depending of you've used the + ``-0`` suffixed ACTION. - Use struct keyword to browse ``struct`` YAML data and use - integers to browse ``sequence`` YAML data. + KEY Identifier to browse and target subvalues into YAML + structure. Use ``.`` to parse a subvalue. If you need + to use a literal ``.`` or ``\\``, use ``\\`` to quote it. - DEFAULT if not provided and given KEY do not match any value in - the provided YAML, then DEFAULT will be returned. If no - default is provided and the KEY do not match any value - in the provided YAML, %(exname)s will fail with an error - message. + Use struct keyword to browse ``struct`` YAML data and use + integers to browse ``sequence`` YAML data. - Examples: + DEFAULT if not provided and given KEY do not match any value in + the provided YAML, then DEFAULT will be returned. If no + default is provided and the KEY do not match any value + in the provided YAML, %(exname)s will fail with an error + message. - ## get last grocery - cat recipe.yaml | %(exname)s get-value groceries.-1 +Examples: - ## get all words of my french dictionary - cat dictionaries.yaml | %(exname)s keys-0 french.dictionary + ## get last grocery + cat recipe.yaml | %(exname)s get-value groceries.-1 - ## get YAML config part of 'myhost' - cat hosts_config.yaml | %(exname)s get-value cfgs.myhost + ## get all words of my french dictionary + cat dictionaries.yaml | %(exname)s keys-0 french.dictionary - """ % {"exname": EXNAME, "usage": USAGE} + ## get YAML config part of 'myhost' + cat hosts_config.yaml | %(exname)s get-value cfgs.myhost - USAGE = textwrap.dedent(USAGE) - HELP = textwrap.dedent(HELP) +""" % {"exname": EXNAME} opts = _parse_args(args, USAGE, HELP) quiet = opts.pop("quiet") From 4637d270611b0238b354bc556d8dc034fd5371ea Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Thu, 6 Aug 2020 11:32:08 +0200 Subject: [PATCH 9/9] ADD file CLI optional argument to read from input file Use custom argparse Action (FileInputAction) to open given file (or stdin if file is "-" or not given). Update usage and help messages and README examples. Issue #11 --- README.rst | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++--- shyaml.py | 31 ++++++++++++++++++++--- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index ec6e0e7..d04c07e 100644 --- a/README.rst +++ b/README.rst @@ -121,11 +121,17 @@ Simple query of simple attribute:: $ cat test.yaml | shyaml get-value name MyName !! + $ shyaml --file test.yaml get-value name + MyName !! + Query nested attributes by using '.' between key labels:: $ cat test.yaml | shyaml get-value subvalue.how-much 1.1 + $ shyaml --file test.yaml get-value subvalue.how-much + 1.1 + Get type of attributes:: $ cat test.yaml | shyaml get-type name @@ -133,6 +139,11 @@ Get type of attributes:: $ cat test.yaml | shyaml get-type subvalue.how-much float + $ shyaml --file test.yaml get-type name + str + $ shyaml --file test.yaml get-type subvalue.how-much + float + Get length of structures or sequences:: $ cat test.yaml | shyaml get-length subvalue @@ -140,6 +151,11 @@ Get length of structures or sequences:: $ cat test.yaml | shyaml get-length subvalue.things 3 + $ shyaml --file test.yaml get-length subvalue + 5 + $ shyaml --file test.yaml get-length subvalue.things + 3 + But this won't work on other types:: $ cat test.yaml | shyaml get-length name @@ -178,6 +194,13 @@ Iteration through keys only:: subvalue.how-much\more subvalue.how-much\.more + $ shyaml --file test.yaml keys + name + subvalue + subvalue.how-much + subvalue.how-much\more + subvalue.how-much\.more + Iteration through keys only (``\0`` terminated strings):: $ cat test.yaml | shyaml keys-0 subvalue | xargs -0 -n 1 echo "VALUE:" @@ -187,6 +210,13 @@ Iteration through keys only (``\0`` terminated strings):: VALUE: maintainer VALUE: description + $ shyaml --file test.yaml keys-0 subvalue | xargs -0 -n 1 echo "VALUE:" + VALUE: how-much + VALUE: how-many + VALUE: things + VALUE: maintainer + VALUE: description + Iteration through values only (``\0`` terminated string highly recommended):: $ cat test.yaml | shyaml values-0 subvalue | @@ -267,6 +297,11 @@ Query a sequence with ``get-value``:: - second - third + $ shyaml --file test.yaml get-value subvalue.things + - first + - second + - third + And access individual elements with python-like indexing:: $ cat test.yaml | shyaml get-value subvalue.things.0 @@ -289,6 +324,11 @@ More usefull, parse a list in one go with ``get-values``:: second third + $ shyaml --file test.yaml get-values subvalue.things + first + second + third + Note that the action is called ``get-values``, and that output is separated by newline char(s) (which is os dependent), this can bring havoc if you are parsing values containing newlines itself. Hopefully, @@ -469,6 +509,13 @@ uses single quotes):: $ cat test.yaml | shyaml get-value 'subvalue\.how-much\\.more' default default + $ shyaml --file test.yaml get-value 'subvalue\.how-much' + 1.2 + $ shyaml --file test.yaml get-value 'subvalue\.how-much\\more' + 1.3 + $ shyaml --file test.yaml get-value 'subvalue\.how-much\\.more' default + default + This last one didn't escape correctly the last ``.``, this is the correct version:: @@ -476,6 +523,10 @@ correct version:: 1.4 + $ shyaml --file test.yaml get-value 'subvalue\.how-much\\\.more' default + 1.4 + + empty string keys ----------------- @@ -582,6 +633,11 @@ For ``shyaml`` version before ``0.4.0``:: b: 3 c: 2 + # shyaml --file test.yaml get-value mapping + a: 1 + b: 3 + c: 2 + For ``shyaml`` version including and after ``0.4.0``:: $ shyaml get-value mapping < test.yaml @@ -590,6 +646,12 @@ For ``shyaml`` version including and after ``0.4.0``:: b: 3 + $ shyaml --file test.yaml get-value mapping + a: 1 + c: 2 + b: 3 + + Strict YAML for further processing ---------------------------------- @@ -709,7 +771,7 @@ A quick reminder of what is available will be printed when calling shyaml {-h|--help} shyaml {-V|--version} - shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] + shyaml [-y|--yaml] [-q|--quiet] [--file FILE] ACTION KEY [DEFAULT] The full help is available through the usage of the standard ``-h`` or ``-help``:: @@ -720,7 +782,7 @@ The full help is available through the usage of the standard ``-h`` or shyaml {-h|--help} shyaml {-V|--version} - shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] + shyaml [-y|--yaml] [-q|--quiet] [--file FILE] ACTION KEY [DEFAULT] Parses and output chosen subpart or values from YAML input. It reads YAML in stdin and will output on stdout it's return value. @@ -741,6 +803,10 @@ The full help is available through the usage of the standard ``-h`` or standard error. (Default: no quiet mode) + --file FILE + Read from FILE + (Default: read from stdin) + -L, --line-buffer Force parsing stdin line by line allowing to process streamed YAML as it is fed instead of buffering @@ -806,6 +872,7 @@ The full help is available through the usage of the standard ``-h`` or optional arguments: -h, --help show this help message and exit + --file FILE -y, --yaml -q, --quiet -L, --line-buffer @@ -820,7 +887,7 @@ Using invalid keywords will issue an error and the usage message:: shyaml {-h|--help} shyaml {-V|--version} - shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] + shyaml [-y|--yaml] [-q|--quiet] [--file FILE] ACTION KEY [DEFAULT] diff --git a/shyaml.py b/shyaml.py index 3e3aa49..0a07c66 100755 --- a/shyaml.py +++ b/shyaml.py @@ -14,7 +14,6 @@ import sys import os.path import re -import textwrap import yaml @@ -493,12 +492,31 @@ def get_version_info(): def _parse_args(args, USAGE, HELP): + class FileInputAction(argparse.Action): + def __init__(self, option_strings, dest, default=sys.stdin, + required=False, help=None, metavar="FILE"): + super(FileInputAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + required=required, + help=help, + metavar=metavar + ) + + def __call__(self, parser, namespace, values, option_string=None): + if values == "-": + setattr(namespace, self.dest, sys.stdin) + else: + setattr(namespace, self.dest, open(values)) + parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, usage=USAGE, description=HELP) parser.add_argument("action") parser.add_argument("key", nargs=argparse.OPTIONAL) parser.add_argument("default", nargs=argparse.OPTIONAL) + parser.add_argument("--file", dest="stream", action=FileInputAction) parser.add_argument("-y", "--yaml", dest="dump", action="store_const", default=magic_dump, const=yaml_dump) parser.add_argument("-q", "--quiet", action="store_true") @@ -661,7 +679,7 @@ def main(args): ## pylint: disable=too-many-branches %(exname)s {-h|--help} %(exname)s {-V|--version} - %(exname)s [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] + %(exname)s [-y|--yaml] [-q|--quiet] [--file FILE] ACTION KEY [DEFAULT] """ % {"exname": EXNAME} HELP = """ @@ -684,6 +702,10 @@ def main(args): ## pylint: disable=too-many-branches standard error. (Default: no quiet mode) + --file FILE + Read from FILE + (Default: read from stdin) + -L, --line-buffer Force parsing stdin line by line allowing to process streamed YAML as it is fed instead of buffering @@ -749,7 +771,7 @@ def main(args): ## pylint: disable=too-many-branches try: first = True - for output in do(stream=sys.stdin, **opts): + for output in do(**opts): if first: first = False else: @@ -770,11 +792,14 @@ def main(args): ## pylint: disable=too-many-branches except (InvalidPath, ActionTypeError) as e: if quiet: exit(1) + opts["stream"].close() else: die(str(e)) + opts["stream"].close() except InvalidAction as e: die("'%s' is not a valid action.\n%s" % (e.args[0], USAGE)) + opts["stream"].close() def entrypoint():