diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fac842d5..83991277 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,33 @@ name: build on: [push, pull_request] jobs: + python-tests: + name: "Python Tests" + + runs-on: ubuntu-20.04 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: pre-commit checks - setup cache + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - run: make -j $(nproc) + + - name: pre-commit checks - run checks + uses: pre-commit/action@v3.0.0 + + # Python2 pytest/pylint CI: + - run: sudo apt-get update && sudo apt-get install -y python2 + - run: curl -sSL https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2 - + - run: pip2 install pytest pylint==1.9.4 + - run: python2 -m pytest -v -rA + - run: python2 -m pylint xtf-runner build/*.py tests/python/*.py + build: strategy: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b1143a8f..997ec43a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,6 +9,9 @@ on: jobs: analyse: + # https://github.com/orgs/community/discussions/26409 (private secrets): + # Run this job if the feature branch is in the main repo (not in a fork): + if: github.event.pull_request.head.repo.full_name == github.repository strategy: matrix: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..436ae21e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,54 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +# + +fail_fast: false +default_stages: [commit, push] +repos: + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + # https://pre-commit.com/hooks.html + hooks: + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + args: ['--fix=lf'] + - id: trailing-whitespace + +- repo: local + hooks: + - id: pytest + name: run python3 unit tests for xtf-runner + entry: env PYTHONDEVMODE=yes python3 -m pytest -v -rA + pass_filenames: false + language: python + types: [python] + additional_dependencies: [pytest] + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.8.0 + hooks: + - id: mypy + +- repo: https://github.com/pylint-dev/pylint + rev: v3.0.3 + hooks: + - id: pylint + args: [--jobs=2] + files: '(^xtf-runner|\.py)$' + log_file: ".git/pre-commit-pylint.log" + additional_dependencies: [pytest] + +- repo: local + hooks: + - id: git-diff # For reference: https://github.com/pre-commit/pre-commit/issues/1712 + name: Show not staged changes (fixups may make them too) + entry: git diff --exit-code + language: system + pass_filenames: false + always_run: true diff --git a/.pylintrc b/.pylintrc index 7c7e50c4..f1b15f53 100644 --- a/.pylintrc +++ b/.pylintrc @@ -37,7 +37,7 @@ extension-pkg-whitelist= # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. See also the "--disable" option for examples. -#enable= +enable=spelling # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this @@ -48,7 +48,24 @@ extension-pkg-whitelist= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=bad-whitespace, bad-continuation, global-statement, star-args +disable=bad-whitespace, bad-continuation, global-statement, star-args, + # Allow older pylint-1.9.x for Python2 to tolerate newer pylint options: + bad-option-value, + unrecognized-inline-option, + # Not real problems, returns on the same indentation level can be easier: + consider-using-with, + no-else-raise, + no-else-return, + multiple-imports, + len-as-condition, + # For Python3-only projects: + consider-using-f-string, + deprecated-module, + unrecognized-option, + unspecified-encoding, + use-implicit-booleaness-not-len, + useless-object-inheritance, + useless-option-value, [REPORTS] @@ -103,6 +120,23 @@ ignore-docstrings=yes ignore-imports=no +[SPELLING] + +# Spelling dictionary name. Available dictionaries: en (aspell), en_AU +# (aspell), en_CA (aspell), en_GB (aspell), en_US (aspell). +# To support spelling checks for older python2 pylint versions, +# `sudo apt-get install -y libenchant-2-2` would be needed, +# so we enable it for newer Python3 pylint in .pre-commit-config.yaml: +#spelling-dict=en_US + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file=.pylintrc.project-dict.txt + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=y + + [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A @@ -337,4 +371,4 @@ max-public-methods=20 # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/.pylintrc.project-dict.txt b/.pylintrc.project-dict.txt new file mode 100644 index 00000000..5ca64e92 --- /dev/null +++ b/.pylintrc.project-dict.txt @@ -0,0 +1,50 @@ +arg +basestring +CFG +config +conftest +CWD +dev +dir +dirs +entrypoint +env +ENVS +epilog +hvm +init +invlpg +iopl +json +logfile +logline +logpath +mkcfg +nonexisting +Normalise +O_CREAT +O_RDONLY +os +pre +pseduo +pv +py +pylintrc +pyright +pytest +reportMissingImports +reportUndefinedVariable +src +stdout +subproc +substitue +sys +toolstack +unimported +unioned +Unrecognised +VM +xenconsole +xenconsoled +xl +xtf diff --git a/arch/x86/include/arch/msr-index.h b/arch/x86/include/arch/msr-index.h index be7ba0d5..0495c3f4 100644 --- a/arch/x86/include/arch/msr-index.h +++ b/arch/x86/include/arch/msr-index.h @@ -81,4 +81,3 @@ * indent-tabs-mode: nil * End: */ - diff --git a/build/mkcfg.py b/build/mkcfg.py old mode 100644 new mode 100755 index 7f8e3d52..631992be --- a/build/mkcfg.py +++ b/build/mkcfg.py @@ -9,7 +9,7 @@ import sys, os # Usage: mkcfg.py $OUT $DEFAULT-CFG $EXTRA-CFG $VARY-CFG -_, out, defcfg, vcpus, extracfg, varycfg = sys.argv +_, out, defcfg, vcpus, extracfg, varycfg = sys.argv # pylint: disable=unbalanced-tuple-unpacking # Evaluate environment and name from $OUT _, env, name = out.split('.')[0].split('-', 2) diff --git a/build/mkinfo.py b/build/mkinfo.py old mode 100644 new mode 100755 index 50819e2c..8df3970d --- a/build/mkinfo.py +++ b/build/mkinfo.py @@ -1,10 +1,43 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +mkcfg.py - Generate a configuration JSON file based on provided parameters. + +Usage: + python mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS + +Arguments: + + $OUT: Path to the output file where the generated JSON configuration + will be saved. + $NAME: Name to be assigned in the configuration. + $CATEGORY: Category designation in the configuration. + $ENVS: Optional space-separated list of environments (can be empty). + $VARIATIONS: Optional space-separated list of variations (can be empty). + +Description: + + This script generates a JSON configuration file using provided parameters + and saves it to the specified output file. The generated JSON structure + includes fields for 'name', 'category', 'environments', and 'variations'. + The 'environments' and 'variations' fields can be populated with + space-separated lists if corresponding arguments ($ENVS and $VARIATIONS) + are provided. + +Example: + + python mkcfg.py config.json ExampleConfig Utilities prod dev test + + This example will create a configuration file named 'config.json' with + 'name' as 'ExampleConfig', + 'category' as 'Utilities', and + 'environments' as ['prod', 'dev', 'test']. +""" import sys, json # Usage: mkcfg.py $OUT $NAME $CATEGORY $ENVS $VARIATIONS -_, out, name, cat, envs, variations = sys.argv +_, out, name, cat, envs, variations = sys.argv # pylint: disable=unbalanced-tuple-unpacking template = { "name": name, diff --git a/docs/mainpage.dox b/docs/mainpage.dox index d9558d62..062de1b6 100644 --- a/docs/mainpage.dox +++ b/docs/mainpage.dox @@ -34,7 +34,7 @@ Environment | Guest | Width | Paging Requirements: - GNU Make >= 3.81 -- Python 2.6 or later +- Python 2.7 or later For x86: - GNU compatible 32 and 64-bit toolchain, capable of `-std=gnu99`, `-m64`, diff --git a/include/xen/sysctl.h b/include/xen/sysctl.h old mode 100755 new mode 100644 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8a3ea5c0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,62 @@ +[project] +# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ +name = "xtf" +description = "Xen Test Framework" +requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +license = {file = "COPYING"} +keywords = ["xen", "xen-project"] +authors = [ + {name = "Andrew Cooper"}, + {name = "Bernhard Kaindl"}, + {name = "Jan Beulich"}, + {name = "Michal Orzel"}, + {name = "Haozhong Zhang"}, + {name = "Roger Pau Monne"}, + {name = "Wei Liuq"}, +] +maintainers = [ + {name = "Andrew Cooper"}, +] +readme = "README" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: System :: Virtualization", +] + +[tool.black] +line-length = 80 + +[tool.isort] +profile = "black" + +[tool.mypy] +files = ["xtf-runner", "build/*.py", "tests/python/test_*.py"] +pretty = true +error_summary = true +warn_redundant_casts = true +warn_return_any = true +warn_unreachable = true +warn_unused_configs = true +disallow_any_unimported = true +disallow_any_explicit = false +disallow_any_generics = true +disallow_subclassing_any = true +show_error_context = true +show_error_codes = true +strict_equality = true +# Enables checking the contents of not yet typed functions: +check_untyped_defs = true + +[[tool.mypy.overrides]] +module = ["xtf-runner"] +disable_error_code = ["name-defined", "method-assign"] diff --git a/tests/python/conftest.py b/tests/python/conftest.py new file mode 100644 index 00000000..acf42243 --- /dev/null +++ b/tests/python/conftest.py @@ -0,0 +1,55 @@ +"""pytest fixtures for unit-testing functions in the xtf-runner script""" +import os +import sys + +import pytest + + +def import_script_as_module(relative_script_path): + "Import a Python script without the .py extension as a python module" + + script_path = os.path.join(os.path.dirname(__file__), relative_script_path) + module_name = os.path.basename(script_path) + + if sys.version_info.major == 2: + # Use deprecated imp module because it needs also to run with Python27: + # pylint: disable-next=import-outside-toplevel + import imp # pyright: ignore[reportMissingImports] + + return imp.load_source(module_name, script_path) + else: + # For Python 3.11+: Import Python script without the .py extension: + # https://gist.github.com/bernhardkaindl/1aaa04ea925fdc36c40d031491957fd3: + + # pylint: disable-next=import-outside-toplevel + from importlib import ( # pylint: disable=no-name-in-module + machinery, + util, + ) + + loader = machinery.SourceFileLoader(module_name, script_path) + spec = util.spec_from_loader(module_name, loader) + assert spec + assert spec.loader + module = util.module_from_spec(spec) + # It is probably a good idea to add the imported module to sys.modules: + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="session") +def imported_xtf_runner(): + """Fixture to import a script as a module for unit testing its functions""" + return import_script_as_module("../../xtf-runner") + + +@pytest.fixture(scope="function") +def xtf_runner(imported_xtf_runner): # pylint: disable=redefined-outer-name + """Test fixture for unit tests: initializes module for each test function""" + # Init the imported xtf-runner, so each unit test function gets it pristine: + # May be used to unit-test xtf-runner with other, different test dirs: + imported_xtf_runner._all_test_info = {} # pylint: disable=protected-access + # The GitHub pre-commit action for does not start the checks in the src dir: + # os.chdir(os.path.join(os.path.dirname(__file__), "../..")) + return imported_xtf_runner diff --git a/tests/python/test_xtf_runner.py b/tests/python/test_xtf_runner.py new file mode 100644 index 00000000..fd5b3d6a --- /dev/null +++ b/tests/python/test_xtf_runner.py @@ -0,0 +1,34 @@ +"""Test xtf-runner.parse_test_instance_string() using tests/python/conftest""" +import pytest + + +def test_parse_test_instance_string(xtf_runner): + """ + Test env, name, variation = xtf-runner.parse_test_instance_string(string): + + Valid argument strings conform to: [[test-]$ENV-]$NAME[~$VARIATION] + Returns a tuple with the environment, name, and variation (if present). + + This test uses the pytest fixture xtf_runner from tests/python/conftest.py + to import and initialize the xtf-runner Python script as module under test. + """ + # Test the argument "pv64-example": + for arg in ("pv64-example", "test-pv64-example"): + env, name, variation = xtf_runner.parse_test_instance_string(arg) + assert env == "pv64" + assert name == "example" + assert variation is None + + # Test the argument "xsa-444": + env, name, variation = xtf_runner.parse_test_instance_string("xsa-444") + assert env is None + assert name == "xsa-444" + assert variation is None + + # Test that passing a nonexisting variation argument raises RunnerError: + with pytest.raises(xtf_runner.RunnerError): + xtf_runner.parse_test_instance_string("xsa-444~nonexisting_variation") + + # Test that passing a nonexisting test argument raises RunnerError: + with pytest.raises(xtf_runner.RunnerError): + xtf_runner.parse_test_instance_string("test-nonexisting_test-raises") diff --git a/xtf-runner b/xtf-runner index 94ed1764..92b338e3 100755 --- a/xtf-runner +++ b/xtf-runner @@ -9,19 +9,16 @@ Currently assumes the presence and availability of the `xl` toolstack. from __future__ import print_function, unicode_literals +import json import sys, os, os.path as path from optparse import OptionParser from subprocess import Popen, PIPE, call as subproc_call - -try: - import json -except ImportError: - import simplejson as json +from typing import Dict # pylint: disable=unused-import # Python 2/3 compatibility if sys.version_info >= (3, ): - basestring = str + basestring = str # pylint: disable=invalid-name,redefined-builtin # All results of a test, keep in sync with C code report.h. # Notes: @@ -80,8 +77,7 @@ class TestInstance(object): def __repr__(self): if not self.variation: return "test-{0}-{1}".format(self.env, self.name) - else: - return "test-{0}-{1}~{2}".format(self.env, self.name, self.variation) + return "test-{0}-{1}~{2}".format(self.env, self.name, self.variation) def __hash__(self): return hash(repr(self)) @@ -92,8 +88,9 @@ class TestInstance(object): def __ne__(self, other): return repr(self) != repr(other) - def __cmp__(self, other): - return cmp(repr(self), repr(other)) + # Should be obsolete for Python2.7 as it can use __eq__() already: + def __cmp__(self, other): # pylint: disable-next=undefined-variable + return cmp(repr(self), repr(other)) # pylint: disable=line-too-long #pyright:ignore[reportUndefinedVariable] class TestInfo(object): @@ -145,12 +142,12 @@ class TestInfo(object): if env_filter: envs = set(env_filter).intersection(self.envs) else: - envs = self.envs + envs = set(self.envs) if vary_filter: variations = set(vary_filter).intersection(self.variations) else: - variations = self.variations + variations = set(self.variations) res = [] if variations: @@ -235,41 +232,18 @@ def parse_test_instance_string(arg): # Cached test json from disk -_all_test_info = {} +_all_test_info = {} # type: Dict[str, TestInfo] def get_all_test_info(): """ Open and collate each info.json """ - # Short circuit if already cached - if _all_test_info: - return _all_test_info - - for test in os.listdir("tests"): - - info_file = None - try: - - # Ignore directories which don't have a info.json inside them - try: - info_file = open(path.join("tests", test, "info.json")) - except IOError: - continue - - # Ignore tests which have bad JSON - try: - test_info = TestInfo(json.load(info_file)) - - if test_info.name != test: - continue - - except (ValueError, KeyError, TypeError): - continue - - _all_test_info[test] = test_info - - finally: - if info_file: - info_file.close() + if not _all_test_info: + for test in os.listdir("tests"): + info_file = path.join("tests", test, "info.json") + if not os.path.exists(info_file): + continue # Not a test dir or test was not built successfully + with open(info_file) as info: + _all_test_info[test] = TestInfo(json.loads(info.read())) return _all_test_info @@ -667,7 +641,7 @@ def main(): "\n" " Running all the pv-iopl tests:\n" " ./xtf-runner pv-iopl\n" - " \n" + " \n" " Combined test results:\n" " test-pv64-pv-iopl SUCCESS\n" " test-pv32pae-pv-iopl SUCCESS\n"