From 02337074834b93da9d6b581d0e55d6a44197c89c Mon Sep 17 00:00:00 2001 From: Mikko Rapeli Date: Mon, 2 Jan 2023 10:49:47 +0000 Subject: [PATCH 1/2] test-runner.py: support specifying ssh port with --target-port qemu virtual machines can run on localhost but on non-default ports so support setting the target SSH port with --target-port. Signed-off-by: Mikko Rapeli --- automated/utils/test-runner.py | 63 +++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/automated/utils/test-runner.py b/automated/utils/test-runner.py index be4dd70fe..15a3bfc2f 100755 --- a/automated/utils/test-runner.py +++ b/automated/utils/test-runner.py @@ -61,12 +61,12 @@ def __call__(self, parser, namespace, values, option_string=None): SSH_PARAMS = "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=5" -def run_command(command, target=None): +def run_command(command, target=None, target_port=22): """Run a shell command. If target is specified, ssh to the given target first.""" run = command if target: - run = 'ssh {} {} "{}"'.format(SSH_PARAMS, target, command) + run = 'ssh -p {} {} {} "{}"'.format(target_port, SSH_PARAMS, target, command) logger = logging.getLogger("RUNNER.run_command") logger.debug(run) @@ -446,12 +446,13 @@ def copy_to_target(self): ) self.logger.info("Creating test path") - run_command("mkdir -p %s" % (self.test["target_test_path"]), self.args.target) + run_command("mkdir -p %s" % (self.test["target_test_path"]), self.args.target, self.args.target_port) self.logger.info("Copying test archive to target host") run_command( - "scp %s ./%s %s:%s" + "scp -P %s %s ./%s %s:%s" % ( + self.args.target_port, SSH_PARAMS, tarball_name, self.args.target, @@ -463,16 +464,17 @@ def copy_to_target(self): run_command( "cd %s && tar -xf %s" % (self.test["target_test_path"], tarball_name), self.args.target, + self.args.target_port, ) self.logger.info("Removing test file archive from target") run_command( - "rm %s/%s" % (self.test["target_test_path"], tarball_name), self.args.target + "rm %s/%s" % (self.test["target_test_path"], tarball_name), self.args.target, self.args.target_port ) def cleanup_target(self): self.logger.info("Removing files from target after testing") - run_command("rm -rf %s" % (self.test["target_test_path"]), self.args.target) + run_command("rm -rf %s" % (self.test["target_test_path"]), self.args.target, self.args.target_port) def run(self): self.copy_to_target() @@ -480,7 +482,8 @@ def run(self): "Executing %s/run.sh remotely on %s" % (self.test["target_test_path"], self.args.target) ) - shell_cmd = 'ssh %s %s "%s/run.sh 2>&1"' % ( + shell_cmd = 'ssh -p %s %s %s "%s/run.sh 2>&1"' % ( + self.args.target_port, SSH_PARAMS, self.args.target, self.test["target_test_path"], @@ -620,7 +623,7 @@ def check_result(self): pass -def get_packages(linux_distribution, target=None): +def get_packages(linux_distribution, target=None, target_port=None): """Return a list of installed packages with versions linux_distribution is a string that may be 'debian', @@ -647,13 +650,13 @@ def get_packages(linux_distribution, target=None): if linux_distribution in ["debian", "ubuntu"]: # Debian (apt) based system packages = run_command( - "dpkg-query -W -f '${package}-${version}\n'", target + "dpkg-query -W -f '${package}-${version}\n'", target, target_port ).splitlines() elif linux_distribution in ["centos", "fedora"]: # RedHat (rpm) based system packages = run_command( - "rpm -qa --qf '%{NAME}-%{VERSION}-%{RELEASE}\n'", target + "rpm -qa --qf '%{NAME}-%{VERSION}-%{RELEASE}\n'", target, target_port ).splitlines() else: logger.warning( @@ -666,7 +669,7 @@ def get_packages(linux_distribution, target=None): return packages -def get_environment(target=None, skip_collection=False): +def get_environment(target=None, target_port=None, skip_collection=False): """Return a dictionary with environmental information target: optional ssh host string to gather environment remotely. @@ -696,7 +699,7 @@ def get_environment(target=None, skip_collection=False): return environment try: environment["linux_distribution"] = ( - run_command("grep ^ID= /etc/os-release", target) + run_command("grep ^ID= /etc/os-release", target, target_port) .split("=")[-1] .strip('"') .lower() @@ -706,7 +709,7 @@ def get_environment(target=None, skip_collection=False): try: environment["linux_distribution_version"] = ( - run_command("grep ^VERSION= /etc/os-release", target) + run_command("grep ^VERSION= /etc/os-release", target, target_port) .split("=")[-1] .strip('"') ) @@ -715,7 +718,7 @@ def get_environment(target=None, skip_collection=False): try: environment["linux_distribution_version_id"] = ( - run_command("grep ^VERSION_ID= /etc/os-release", target) + run_command("grep ^VERSION_ID= /etc/os-release", target, target_port) .split("=")[-1] .strip('"') ) @@ -724,7 +727,7 @@ def get_environment(target=None, skip_collection=False): try: environment["linux_distribution_version_codename"] = ( - run_command("grep ^VERSION_CODENAME= /etc/os-release", target) + run_command("grep ^VERSION_CODENAME= /etc/os-release", target, target_port) .split("=")[-1] .strip('"') ) @@ -732,39 +735,39 @@ def get_environment(target=None, skip_collection=False): environment["linux_distribution_version_codename"] = "" try: - environment["kernel"] = run_command("uname -r", target) + environment["kernel"] = run_command("uname -r", target, target_port) except subprocess.CalledProcessError: environment["kernel"] = "" try: - environment["uname"] = run_command("uname -a", target) + environment["uname"] = run_command("uname -a", target, target_port) except subprocess.CalledProcessError: environment["uname"] = "" try: environment["bios_version"] = run_command( - "cat /sys/devices/virtual/dmi/id/bios_version", target + "cat /sys/devices/virtual/dmi/id/bios_version", target, target_port ) except subprocess.CalledProcessError: environment["bios_version"] = "" try: environment["board_vendor"] = run_command( - "cat /sys/devices/virtual/dmi/id/board_vendor", target + "cat /sys/devices/virtual/dmi/id/board_vendor", target, target_port ) except subprocess.CalledProcessError: environment["board_vendor"] = "" try: environment["board_name"] = run_command( - "cat /sys/devices/virtual/dmi/id/board_name", target + "cat /sys/devices/virtual/dmi/id/board_name", target, target_port ) except subprocess.CalledProcessError: environment["board_name"] = "" try: environment["packages"] = get_packages( - environment["linux_distribution"], target + environment["linux_distribution"], target, target_port ) except subprocess.CalledProcessError: environment["packages"] = [] @@ -781,7 +784,7 @@ def __init__(self, test, args): self.results["id"] = test["test_uuid"] self.results["test_plan"] = args.test_plan self.results["environment"] = get_environment( - target=self.args.target, skip_collection=self.args.skip_environment + target=self.args.target, target_port=self.args.target_port, skip_collection=self.args.skip_environment ) self.logger = logging.getLogger("RUNNER.ResultParser") self.results["params"] = {} @@ -1206,6 +1209,18 @@ def get_args(): """ ), ) + parser.add_argument( + "-tp", + "--target-port", + default=22, + dest="target_port", + help=textwrap.dedent( + """\ + Specify SSH target port + Default: 22 + """ + ), + ) parser.add_argument( "-s", "--skip_install", @@ -1356,7 +1371,7 @@ def main(): logger.error("openssh client must be installed on the host.") sys.exit(1) try: - run_command("exit", args.target) + run_command("exit", args.target, args.target_port) except subprocess.CalledProcessError as e: logger.error("ssh login failed.") print(e) @@ -1387,7 +1402,7 @@ def main(): args.kind, tc_dirname.split(args.kind)[1], ) - target_user_home = run_command("echo $HOME", args.target) + target_user_home = run_command("echo $HOME", args.target, args.target_port) test["target_test_path"] = "%s/output/%s" % ( target_user_home, test["test_uuid"], From b1011ae848145f65f8602d7bd34ec6ef37b2d424 Mon Sep 17 00:00:00 2001 From: Mikko Rapeli Date: Mon, 2 Jan 2023 13:01:21 +0000 Subject: [PATCH 2/2] test-runner.py: support opkg in get_packages() Yocto can use rpm, deb or opkg as binary packaging tool. If distribution is not guessed correctly from /etc/os-release, or if it doesn't match debian, fedora or ubuntu, then try the different packaging tools to populate package list. This will work if tools are on the rootfs also on custom distributions like the ones from yocto build. Signed-off-by: Mikko Rapeli --- automated/utils/test-runner.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/automated/utils/test-runner.py b/automated/utils/test-runner.py index 15a3bfc2f..0b46a25fc 100755 --- a/automated/utils/test-runner.py +++ b/automated/utils/test-runner.py @@ -629,6 +629,9 @@ def get_packages(linux_distribution, target=None, target_port=None): linux_distribution is a string that may be 'debian', 'ubuntu', 'centos', or 'fedora'. + If linux_distribution is not provided, then rpm, dpkg and opkg + tools will be tried to get the package list. + For example (ubuntu): 'packages': ['acl-2.2.52-2', 'adduser-3.113+nmu3', @@ -643,6 +646,11 @@ def get_packages(linux_distribution, target=None, target_port=None): "zlib-1.2.7-17.el7", "zlib-devel-1.2.7-17.el7" ] + (yocto/opkg): + "packages": ["alsa-conf - 1.2.6.1-r0.3", + "alsa-state - 0.2.0-r5.3", + ... + ] """ logger = logging.getLogger("RUNNER.get_packages") @@ -660,10 +668,31 @@ def get_packages(linux_distribution, target=None, target_port=None): ).splitlines() else: logger.warning( - "Unknown linux distribution '{}'; package list not populated.".format( + "Unknown linux distribution '{}'; trying to populate package list.".format( linux_distribution ) ) + try: + packages = run_command( + "which rpm > /dev/null && rpm -qa --qf '%{NAME}-%{VERSION}-%{RELEASE}\n'", target, target_port + ).splitlines() + except subprocess.CalledProcessError: + pass + logger.debug("packages = %s" % packages) + try: + packages += run_command( + "which dpkg > /dev/null && dpkg-query -W -f '${package}-${version}\n'", target, target_port + ).splitlines() + except subprocess.CalledProcessError: + pass + logger.debug("packages = %s" % packages) + try: + packages += run_command( + "which opkg > /dev/null && opkg list-installed", target, target_port + ).splitlines() + except subprocess.CalledProcessError: + pass + logger.debug("packages = %s" % packages) packages.sort() return packages