diff --git a/.gitignore b/.gitignore index 054d99c51..3eadcb4c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +*.rej *.py[cod] +/.env +*.orig # C extensions *.so @@ -40,6 +43,9 @@ nosetests.xml # Sublime /*.sublime-* +#PyCharm +/.idea + # virtualenv /.Python /lib diff --git a/CHANGES.rst b/CHANGES.rst index 8e203b798..fe2d3907d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changelog ========= +2.2.0 +----- + +- Implemented cucumber json formatter (bubenkoff, albertjan) + 2.1.2 ----- diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ea526545a --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +# create virtual environment +.env: + virtualenv .env + +# install all needed for development +develop: .env + .env/bin/pip install -e . -r requirements-testing.txt + +# clean the development envrironment +clean: + -rm -rf .env diff --git a/README.rst b/README.rst index 9d12427cd..082aab4ea 100644 --- a/README.rst +++ b/README.rst @@ -594,6 +594,20 @@ Tools recommended to use for browser testing: * `pytest-splinter `_ - pytest `splinter `_ integration for the real browser testing +Reporting +--------- + +It's important to have nice reporting out of your bdd tests. Cucumber introduced some kind of standard for +`json format `_ +which can be used for `this `_ jenkins +plugin + +To have an output in json format: + +:: + + py.test --cucumberjson= + Migration of your tests from versions 0.x.x-1.x.x ------------------------------------------------- diff --git a/pytest_bdd/__init__.py b/pytest_bdd/__init__.py index 9c4c3741b..49aed1c67 100644 --- a/pytest_bdd/__init__.py +++ b/pytest_bdd/__init__.py @@ -1,4 +1,5 @@ -__version__ = '2.1.2' +"""pytest-bdd public api.""" +__version__ = '2.2.0' try: from pytest_bdd.steps import given, when, then # pragma: no cover diff --git a/pytest_bdd/cucumber_json.py b/pytest_bdd/cucumber_json.py new file mode 100644 index 000000000..88a71fad1 --- /dev/null +++ b/pytest_bdd/cucumber_json.py @@ -0,0 +1,120 @@ +"""Cucumber json output formatter.""" +import json +import os +import time + +import py + +from .feature import force_unicode + + +def pytest_addoption(parser): + group = parser.getgroup('pytest-bdd') + group.addoption( + '--cucumberjson', '--cucumber-json', action='store', + dest='cucumber_json_path', metavar='path', default=None, + help='create cucumber json style report file at given path.') + + +def pytest_configure(config): + cucumber_json_path = config.option.cucumber_json_path + # prevent opening json log on slave nodes (xdist) + if cucumber_json_path and not hasattr(config, 'slaveinput'): + config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path) + config.pluginmanager.register(config._bddcucumberjson) + + +def pytest_unconfigure(config): + xml = getattr(config, '_bddcucumberjson', None) + if xml is not None: + del config._bddcucumberjson + config.pluginmanager.unregister(xml) + + +class LogBDDCucumberJSON(object): + + """Logging plugin for cucumber like json output.""" + + def __init__(self, logfile): + logfile = os.path.expanduser(os.path.expandvars(logfile)) + self.logfile = os.path.normpath(os.path.abspath(logfile)) + self.features = {} + + def append(self, obj): + self.features[-1].append(obj) + + def _get_result(self, step, report): + """Get scenario test run result. + + :param step: `Step` step we get result for + :param report: pytest `Report` object + :return: `dict` in form {'status': '', ['error_message': '']} + """ + if report.passed or not step['failed']: # ignore setup/teardown + return {'status': 'passed'} + elif report.failed and step['failed']: + return { + 'status': 'failed', + 'error_message': force_unicode(report.longrepr), + } + elif report.skipped: + return {'status': 'skipped'} + + def pytest_runtest_logreport(self, report): + try: + scenario = report.scenario + except AttributeError: + # skip reporting for non-bdd tests + return + + if not scenario['steps'] or report.when != 'call': + # skip if there isn't a result or scenario has no steps + return + + def stepmap(step): + return { + "keyword": step['keyword'], + "name": step['name'], + "line": step['line_number'], + "match": { + "location": "" + }, + "result": self._get_result(step, report), + } + + if scenario['feature']['filename'] not in self.features: + self.features[scenario['feature']['filename']] = { + "keyword": "Feature", + "uri": scenario['feature']['rel_filename'], + "name": scenario['feature']['name'] or scenario['feature']['rel_filename'], + "id": scenario['feature']['rel_filename'].lower().replace(' ', '-'), + "line": scenario['feature']['line_number'], + "description": scenario['feature']['description'], + "tags": [], + "elements": [], + } + + self.features[scenario['feature']['filename']]['elements'].append({ + "keyword": "Scenario", + "id": report.item['name'], + "name": scenario['name'], + "line": scenario['line_number'], + "description": '', + "tags": [], + "type": "scenario", + "steps": [stepmap(step) for step in scenario['steps']], + }) + + def pytest_sessionstart(self): + self.suite_start_time = time.time() + + def pytest_sessionfinish(self): + if py.std.sys.version_info[0] < 3: + logfile_open = py.std.codecs.open + else: + logfile_open = open + with logfile_open(self.logfile, 'w', encoding='utf-8') as logfile: + logfile.write(json.dumps(list(self.features.values()))) + + def pytest_terminal_summary(self, terminalreporter): + terminalreporter.write_sep('-', 'generated json file: %s' % (self.logfile)) diff --git a/pytest_bdd/feature.py b/pytest_bdd/feature.py index ce0faffe8..393d19bc7 100644 --- a/pytest_bdd/feature.py +++ b/pytest_bdd/feature.py @@ -23,6 +23,8 @@ one line. """ +from os import path as op # pragma: no cover + import re # pragma: no cover import sys # pragma: no cover import textwrap @@ -32,6 +34,7 @@ class FeatureError(Exception): # pragma: no cover + """Feature parse error.""" message = u'{0}.\nLine number: {1}.\nLine: {2}.' @@ -93,18 +96,17 @@ def strip_comments(line): return line.strip() -def remove_prefix(line): - """Remove the step prefix (Scenario, Given, When, Then or And). +def parse_line(line): + """Parse step line to get the step prefix (Scenario, Given, When, Then or And) and the actual step name. :param line: Line of the Feature file. - :return: Line without the prefix. - + :return: `tuple` in form ('', ''). """ for prefix, _ in STEP_PREFIXES: if line.startswith(prefix): - return line[len(prefix):].strip() - return line + return prefix.strip(), line[len(prefix):].strip() + return '', line def _open_file(filename, encoding): @@ -114,11 +116,19 @@ def _open_file(filename, encoding): return open(filename, 'r', encoding=encoding) -def force_unicode(string, encoding='utf-8'): - if sys.version_info < (3, 0) and isinstance(string, str): - return string.decode(encoding) +def force_unicode(obj, encoding='utf-8'): + """Get the unicode string out of given object (python 2 and python 3). + + :param obj: `object`, usually a string + :return: unicode string + """ + if sys.version_info < (3, 0): + if isinstance(obj, str): + return obj.decode(encoding) + else: + return unicode(obj) else: - return string + return str(obj) def force_encode(string, encoding='utf-8'): @@ -129,17 +139,20 @@ def force_encode(string, encoding='utf-8'): class Feature(object): + """Feature.""" - def __init__(self, filename, encoding='utf-8'): + def __init__(self, basedir, filename, encoding='utf-8'): """Parse the feature file. :param filename: Relative path to the feature file. """ self.scenarios = {} - - self.filename = filename + self.rel_filename = op.join(op.basename(basedir), filename) + self.filename = filename = op.abspath(op.join(basedir, filename)) + self.line_number = 1 + self.name = None scenario = None mode = None prev_mode = None @@ -181,16 +194,17 @@ def __init__(self, filename, encoding='utf-8'): if mode == types.FEATURE: if prev_mode != types.FEATURE: - self.name = remove_prefix(clean_line) + _, self.name = parse_line(clean_line) + self.line_number = line_number else: description.append(clean_line) prev_mode = mode # Remove Feature, Given, When, Then, And - clean_line = remove_prefix(clean_line) + keyword, clean_line = parse_line(clean_line) if mode in [types.SCENARIO, types.SCENARIO_OUTLINE]: - self.scenarios[clean_line] = scenario = Scenario(self, clean_line) + self.scenarios[clean_line] = scenario = Scenario(self, clean_line, line_number) elif mode == types.EXAMPLES: mode = types.EXAMPLES_HEADERS elif mode == types.EXAMPLES_VERTICAL: @@ -205,14 +219,16 @@ def __init__(self, filename, encoding='utf-8'): scenario.add_example_row(clean_line[0], clean_line[1:]) elif mode and mode != types.FEATURE: step = scenario.add_step( - step_name=clean_line, step_type=mode, indent=line_indent, line_number=line_number) + step_name=clean_line, step_type=mode, indent=line_indent, line_number=line_number, + keyword=keyword) self.description = u'\n'.join(description) @classmethod - def get_feature(cls, filename, encoding='utf-8'): + def get_feature(cls, base_path, filename, encoding='utf-8'): """Get a feature by the filename. + :param base_path: Base feature directory. :param filename: Filename of the feature file. :return: `Feature` instance from the parsed feature cache. @@ -222,17 +238,19 @@ def get_feature(cls, filename, encoding='utf-8'): when multiple scenarios are referencing the same file. """ - feature = features.get(filename) + full_name = op.abspath(op.join(base_path, filename)) + feature = features.get(full_name) if not feature: - feature = Feature(filename, encoding=encoding) - features[filename] = feature + feature = Feature(base_path, filename, encoding=encoding) + features[full_name] = feature return feature class Scenario(object): + """Scenario.""" - def __init__(self, feature, name, example_converters=None): + def __init__(self, feature, name, line_number, example_converters=None): self.feature = feature self.name = name self.params = set() @@ -240,19 +258,23 @@ def __init__(self, feature, name, example_converters=None): self.example_params = [] self.examples = [] self.vertical_examples = [] + self.line_number = line_number self.example_converters = example_converters - def add_step(self, step_name, step_type, indent, line_number): + def add_step(self, step_name, step_type, indent, line_number, keyword): """Add step to the scenario. :param step_name: Step name. :param step_type: Step type. - + :param indent: `int` step text indent + :param line_number: `int` line number + :param keyword: `str` step keyword """ params = get_step_params(step_name) self.params.update(params) step = Step( - name=step_name, type=step_type, params=params, scenario=self, indent=indent, line_number=line_number) + name=step_name, type=step_type, params=params, scenario=self, indent=indent, line_number=line_number, + keyword=keyword) self.steps.append(step) return step @@ -326,16 +348,19 @@ def validate(self): class Step(object): + """Step.""" - def __init__(self, name, type, params, scenario, indent, line_number): + def __init__(self, name, type, params, scenario, indent, line_number, keyword): self.name = name + self.keyword = keyword self.lines = [] self.indent = indent self.type = type self.params = params self.scenario = scenario self.line_number = line_number + self.failed = False def add_line(self, line): """Add line to the multiple step.""" diff --git a/pytest_bdd/plugin.py b/pytest_bdd/plugin.py index 9ac251e66..87293e976 100644 --- a/pytest_bdd/plugin.py +++ b/pytest_bdd/plugin.py @@ -15,3 +15,42 @@ def pytest_addhooks(pluginmanager): """Register plugin hooks.""" from pytest_bdd import hooks pluginmanager.addhooks(hooks) + + +def pytest_bdd_step_error(request, feature, scenario, step, step_func, step_func_args, exception): + """Mark step as failed for later reporting.""" + step.failed = True + + +@pytest.mark.tryfirst +def pytest_runtest_makereport(item, call, __multicall__): + """Store item in the report object.""" + rep = __multicall__.execute() + try: + scenario = item.obj.__scenario__ + except AttributeError: + pass + else: + rep.scenario = { + 'steps': [{ + 'name': step.name, + 'type': step.type, + 'keyword': step.keyword, + 'line_number': step.line_number, + 'failed': step.failed + } for step in scenario.steps], + 'name': scenario.name, + 'line_number': scenario.line_number, + 'feature': { + 'name': scenario.feature.name, + 'filename': scenario.feature.filename, + 'rel_filename': scenario.feature.rel_filename, + 'line_number': scenario.feature.line_number, + 'description': scenario.feature.description + } + } + rep.item = { + 'name': item.name + } + + return rep diff --git a/pytest_bdd/scenario.py b/pytest_bdd/scenario.py index 7ad9d34dd..0be507d86 100644 --- a/pytest_bdd/scenario.py +++ b/pytest_bdd/scenario.py @@ -17,7 +17,6 @@ import imp import inspect # pragma: no cover -from os import path as op # pragma: no cover import pytest @@ -239,6 +238,7 @@ def decorator(_pytestbdd_function): _scenario.__doc__ = '{feature_name}: {scenario_name}'.format( feature_name=feature_name, scenario_name=scenario_name) + _scenario.__scenario__ = scenario return _scenario return recreate_function(decorator, module=caller_module, firstlineno=caller_function.f_lineno) @@ -253,8 +253,7 @@ def scenario( # Get the feature base_path = get_fixture(caller_module, 'pytestbdd_feature_base_dir') - feature_path = op.abspath(op.join(base_path, feature_name)) - feature = Feature.get_feature(feature_path, encoding=encoding) + feature = Feature.get_feature(base_path, feature_name, encoding=encoding) # Get the scenario try: diff --git a/pytest_bdd/steps.py b/pytest_bdd/steps.py index a31c7c82b..076b8c26b 100644 --- a/pytest_bdd/steps.py +++ b/pytest_bdd/steps.py @@ -39,13 +39,14 @@ def article(author): import pytest # pragma: no cover -from pytest_bdd.feature import remove_prefix # pragma: no cover +from pytest_bdd.feature import parse_line # pragma: no cover from pytest_bdd.types import GIVEN, WHEN, THEN # pragma: no cover PY3 = sys.version_info[0] >= 3 # pragma: no cover class StepError(Exception): # pragma: no cover + """Step declaration error.""" RE_TYPE = type(re.compile('')) # pragma: no cover @@ -63,7 +64,6 @@ def given(name, fixture=None, converters=None): :note: Can't be used as a decorator when the fixture is specified. """ - if fixture is not None: module = get_caller_module() step_func = lambda request: request.getfuncargvalue(fixture) @@ -73,7 +73,8 @@ def given(name, fixture=None, converters=None): step_func.fixture = fixture func = pytest.fixture(lambda: step_func) func.__doc__ = 'Alias for the "{0}" fixture.'.format(fixture) - contribute_to_module(module, remove_prefix(name), func) + _, name = parse_line(name) + contribute_to_module(module, name, func) return _not_a_fixture_decorator return _step_decorator(GIVEN, name, converters=converters) diff --git a/setup.py b/setup.py index 2a13e2aba..a19624de1 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +"""pytest-bdd package config.""" import codecs import os import sys @@ -10,9 +10,12 @@ class Tox(TestCommand): + + """"Custom setup.py test command implementation using tox runner.""" + def finalize_options(self): TestCommand.finalize_options(self) - self.test_args = ['--recreate'] + self.test_args = ['--recreate -vv'] self.test_suite = True def run_tests(self): @@ -59,6 +62,7 @@ def run_tests(self): entry_points={ 'pytest11': [ 'pytest-bdd = pytest_bdd.plugin', + 'pytest-bdd-cucumber-json = pytest_bdd.cucumber_json', ], 'console_scripts': [ 'pytestbdd_migrate_tests = pytest_bdd.scripts:migrate_tests [migrate]' diff --git a/tests/feature/test_cucumber_json.py b/tests/feature/test_cucumber_json.py new file mode 100644 index 000000000..0fc2c41b1 --- /dev/null +++ b/tests/feature/test_cucumber_json.py @@ -0,0 +1,150 @@ +"""Test cucumber json output.""" +import json +import os.path +import textwrap + +import pytest + + +def runandparse(testdir, *args): + """Run tests in testdir and parse json output.""" + resultpath = testdir.tmpdir.join("cucumber.json") + result = testdir.runpytest('--cucumberjson={0}'.format(resultpath), '-s', *args) + jsonobject = json.load(resultpath.open()) + return result, jsonobject + + +@pytest.fixture(scope='session') +def equals_any(): + """Helper object comparison to which is always 'equal'.""" + class equals_any(object): + + def __eq__(self, other): + return True + + def __cmp__(self, other): + return 0 + + return equals_any() + + +def test_step_trace(testdir, equals_any): + """Test step trace.""" + testdir.makefile('.feature', test=textwrap.dedent(""" + Feature: One passing scenario, one failing scenario + + Scenario: Passing + Given a passing step + And some other passing step + + Scenario: Failing + Given a passing step + And a failing step + """)) + testdir.makepyfile(textwrap.dedent(""" + import pytest + from pytest_bdd import given, when, scenario + + @given('a passing step') + def a_passing_step(): + return 'pass' + + @given('some other passing step') + def some_other_passing_step(): + return 'pass' + + @given('a failing step') + def a_failing_step(): + raise Exception('Error') + + @scenario('test.feature', 'Passing') + def test_passing(): + pass + + @scenario('test.feature', 'Failing') + def test_failing(): + pass + """)) + result, jsonobject = runandparse(testdir) + assert result.ret + assert jsonobject == [ + { + "description": "", + "elements": [ + { + "description": "", + "id": "test_passing", + "keyword": "Scenario", + "line": 3, + "name": "Passing", + "steps": [ + { + "keyword": "Given", + "line": 4, + "match": { + "location": "" + }, + "name": "a passing step", + "result": { + "status": "passed" + } + }, + { + "keyword": "And", + "line": 5, + "match": { + "location": "" + }, + "name": "some other passing step", + "result": { + "status": "passed" + } + } + + ], + "tags": [], + "type": "scenario" + }, + { + "description": "", + "id": "test_failing", + "keyword": "Scenario", + "line": 7, + "name": "Failing", + "steps": [ + { + "keyword": "Given", + "line": 8, + "match": { + "location": "" + }, + "name": "a passing step", + "result": { + "status": "passed" + } + }, + { + "keyword": "And", + "line": 9, + "match": { + "location": "" + }, + "name": "a failing step", + "result": { + "error_message": equals_any, + "status": "failed" + } + } + ], + "tags": [], + "type": "scenario" + } + ], + "id": "test_step_trace0/test.feature", + "keyword": "Feature", + "line": 1, + "name": "One passing scenario, one failing scenario", + "tags": [], + "uri": os.path.join(testdir.tmpdir.basename, 'test.feature'), + } + ] diff --git a/tox.ini b/tox.ini index 0415760e9..cf47be8f2 100644 --- a/tox.ini +++ b/tox.ini @@ -30,3 +30,4 @@ deps = [pytest] pep8maxlinelength=120 +addopts=-vvl