From 63f69791724da173e9770667a7c6850e94c3adcb Mon Sep 17 00:00:00 2001 From: Roman Prilipskii Date: Tue, 11 Feb 2025 02:58:10 +0100 Subject: [PATCH 1/4] CLOS-3205: Rework the clearpackageconflicts actor with set definitoon to work for el8to9 upgrades Seems like the InstalledRPM model was modified at some point, and the previous construction could not be utilized. The logic was changed to process only package names (which is essentially already how the data was used) into the set, which is then used for package lookup. This sidesteps the problem with the InstalledRPM type not being hashable. --- .../actors/clearpackageconflicts/actor.py | 96 +++---------------- .../libraries/clearpackageconflicts.py | 84 ++++++++++++++++ .../tests/unit_test_clearpackageconflicts.py | 17 ++++ 3 files changed, 112 insertions(+), 85 deletions(-) create mode 100644 repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py create mode 100644 repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py index fc642d5029..0bed04ed27 100644 --- a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py @@ -1,102 +1,28 @@ -import os -import errno -import shutil - from leapp.actors import Actor from leapp.models import InstalledRPM from leapp.tags import DownloadPhaseTag, IPUWorkflowTag from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.actor import clearpackageconflicts class ClearPackageConflicts(Actor): """ - Remove several python package files manually to resolve conflicts between versions of packages to be upgraded. + Remove several Python package files manually to resolve conflicts + between versions of packages to be upgraded. + + When the corresponding packages are detected, + the conflicting files are removed to allow for an upgrade to the new package versions. + + While most packages are handled automatically by the package manager, + some specific packages require direct intervention to resolve conflicts + between their own versions on different OS releases. """ name = "clear_package_conflicts" consumes = (InstalledRPM,) produces = () tags = (DownloadPhaseTag.Before, IPUWorkflowTag) - rpm_lookup = None - - def has_package(self, name): - """ - Check whether the package is installed. - Looks only for the package name, nothing else. - """ - if self.rpm_lookup: - return name in self.rpm_lookup - - def problem_packages_installed(self, problem_packages): - """ - Check whether any of the problem packages are present in the system. - """ - for pkg in problem_packages: - if self.has_package(pkg): - self.log.debug("Conflicting package {} detected".format(pkg)) - return True - return False - - def clear_problem_files(self, problem_files, problem_dirs): - """ - Go over the list of problem files and directories and remove them if they exist. - They'll be replaced by the new packages. - """ - for p_dir in problem_dirs: - try: - if os.path.isdir(p_dir): - shutil.rmtree(p_dir) - self.log.debug("Conflicting directory {} removed".format(p_dir)) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - for p_file in problem_files: - try: - if os.path.isfile(p_file): - os.remove(p_file) - self.log.debug("Conflicting file {} removed".format(p_file)) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - def alt_python37_handle(self): - """ - These alt-python37 packages are conflicting with their own builds for EL8. - """ - problem_packages = [ - "alt-python37-six", - "alt-python37-pytz", - ] - problem_files = [] - problem_dirs = [ - "/opt/alt/python37/lib/python3.7/site-packages/six-1.15.0-py3.7.egg-info", - "/opt/alt/python37/lib/python3.7/site-packages/pytz-2017.2-py3.7.egg-info", - ] - - if self.problem_packages_installed(problem_packages): - self.clear_problem_files(problem_files, problem_dirs) - - def lua_cjson_handle(self): - """ - lua-cjson package is conflicting with the incoming lua-cjson package for EL8. - """ - problem_packages = [ - "lua-cjson" - ] - problem_files = [ - "/usr/lib64/lua/5.1/cjson.so", - "/usr/share/lua/5.1/cjson/tests/bench.lua", - "/usr/share/lua/5.1/cjson/tests/genutf8.pl", - "/usr/share/lua/5.1/cjson/tests/test.lua", - ] - problem_dirs = [] - - if self.problem_packages_installed(problem_packages): - self.clear_problem_files(problem_files, problem_dirs) @run_on_cloudlinux def process(self): - # todo: (CLOS-3205) investigate why set is needed here - self.rpm_lookup = [rpm for rpm in self.consume(InstalledRPM)] - self.alt_python37_handle() + clearpackageconflicts.process() diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py new file mode 100644 index 0000000000..420ddce19f --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py @@ -0,0 +1,84 @@ +import os +import errno +from re import L +import shutil + +from leapp.libraries.stdlib import api +from leapp.models import InstalledRPM + + +def problem_packages_installed(problem_packages, lookup): + """ + Check whether any of the problem packages are present in the system. + """ + for pkg in problem_packages: + if pkg in lookup: + api.current_logger().debug("Conflicting package {} detected".format(pkg)) + return True + return False + + +def clear_problem_files(problem_files, problem_dirs): + """ + Go over the list of problem files and directories and remove them if they exist. + They'll be replaced by the new packages. + """ + for p_dir in problem_dirs: + try: + if os.path.isdir(p_dir): + shutil.rmtree(p_dir) + api.current_logger().debug("Conflicting directory {} removed".format(p_dir)) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + for p_file in problem_files: + try: + if os.path.isfile(p_file): + os.remove(p_file) + api.current_logger().debug("Conflicting file {} removed".format(p_file)) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +def alt_python37_handle(package_lookup): + """ + These alt-python37 packages are conflicting with their own builds for EL8. + """ + problem_packages = [ + "alt-python37-six", + "alt-python37-pytz", + ] + problem_files = [] + problem_dirs = [ + "/opt/alt/python37/lib/python3.7/site-packages/six-1.15.0-py3.7.egg-info", + "/opt/alt/python37/lib/python3.7/site-packages/pytz-2017.2-py3.7.egg-info", + ] + + if problem_packages_installed(problem_packages, package_lookup): + clear_problem_files(problem_files, problem_dirs) + + +def lua_cjson_handle(package_lookup): + """ + lua-cjson package is conflicting with the incoming lua-cjson package for EL8. + """ + problem_packages = [ + "lua-cjson" + ] + problem_files = [ + "/usr/lib64/lua/5.1/cjson.so", + "/usr/share/lua/5.1/cjson/tests/bench.lua", + "/usr/share/lua/5.1/cjson/tests/genutf8.pl", + "/usr/share/lua/5.1/cjson/tests/test.lua", + ] + problem_dirs = [] + + if problem_packages_installed(problem_packages, package_lookup): + clear_problem_files(problem_files, problem_dirs) + + +def process(): + rpm_lookup = {rpm.name for rpm in api.consume(InstalledRPM)} + alt_python37_handle(rpm_lookup) diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py new file mode 100644 index 0000000000..efdcbee9ea --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py @@ -0,0 +1,17 @@ +import pytest + +# from leapp import reporting +from leapp.libraries.actor import clearpackageconflicts + + +@pytest.mark.parametrize( + "problem_pkgs,lookup,expected_res", + ( + (["cagefs"], {"cagefs", "dnf"}, True), + (["lve-utils"], {"lve-utils", "dnf"}, True), + (["nonexistent-pkg"], {"cagefs", "dnf"}, False), + (["cagefs"], {"lve-utils", "dnf"}, False), + ), +) +def test_problem_packages_installed(problem_pkgs, lookup, expected_res): + assert expected_res == clearpackageconflicts.problem_packages_installed(problem_pkgs, lookup) From 21de0b764d60b83161c633edaab83415a0ee96cc Mon Sep 17 00:00:00 2001 From: Roman Prilipskii Date: Wed, 12 Feb 2025 13:41:11 +0100 Subject: [PATCH 2/4] CLOS-3230: Refresh EPEL repos after the system upgrade if they remained on the old version --- .../cloudlinux/actors/refreshepel/actor.py | 67 +++++++++++++++++++ .../actors/replacerpmnewconfigs/actor.py | 50 +++++++------- .../cloudlinux/libraries/backup.py | 54 +++++++++++++-- 3 files changed, 140 insertions(+), 31 deletions(-) create mode 100644 repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py diff --git a/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py b/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py new file mode 100644 index 0000000000..f2d280cc36 --- /dev/null +++ b/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py @@ -0,0 +1,67 @@ +from __future__ import print_function +from operator import is_ +import os + +from leapp.actors import Actor +from leapp.tags import ApplicationsPhaseTag, IPUWorkflowTag +from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.common.backup import backup_and_remove +from leapp.libraries.common.config.version import get_target_major_version + +REPO_DIR = '/etc/yum.repos.d' +EPEL_INSTALL_URL = 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm'.format(get_target_major_version()) + + +class RefreshEPEL(Actor): + """ + Check that the EPEL repositories are correctly configured after the upgrade. + + Depending on how the upgrade went, the EPEL repositories might still be targeting the old OS version. + This actor checks that the EPEL repositories are correctly configured and if not, it will install the + correct EPEL release package and refresh the repositories. + """ + + name = 'refresh_epel' + # We can't depend on InstalledRPM message because by this point + # the system is upgraded and the RPMs are not the same as when the data was collected. + consumes = () + produces = () + tags = (ApplicationsPhaseTag.After, IPUWorkflowTag) + + def clear_epel_repo_files(self): + for repofile in os.listdir(REPO_DIR): + if repofile.startswith('epel'): + epel_file = os.path.join(REPO_DIR, repofile) + backup_and_remove(epel_file) + + def install_epel_release_package(self, target_url): + os.system('dnf install {}'.format(target_url)) + self.log.info('EPEL release package installed: {}'.format(target_url)) + + @run_on_cloudlinux + def process(self): + target_version = int(get_target_major_version()) + target_epel_release = EPEL_INSTALL_URL.format(target_version) + + # EPEL release package name is 'epel-release' and the version should match the target OS version + epel_release_package = 'epel-release' + + is_epel_installed = os.system('rpm -q {}'.format(epel_release_package)) == 0 + is_correct_version = os.system('rpm -q {}-{}'.format(epel_release_package, target_version)) == 0 + epel_files_verified = os.system('rpm -V {}'.format(epel_release_package)) == 0 + + # It's possible (although unusual) that the correct EPEL release package is installed during the upgrade, + # but the EPEL repository files still point to the old OS version. + # This was observed on client machines before. + + if (is_epel_installed and not is_correct_version) or not epel_files_verified: + # If the EPEL release package is installed but not the correct version, remove it + # Same if the files from the package were modified + os.system('rpm -e {}'.format(epel_release_package)) + if not is_epel_installed or not is_correct_version or not epel_files_verified: + # Clear the EPEL repository files + self.clear_epel_repo_files() + # Install the correct EPEL release package + self.install_epel_release_package(target_epel_release) + # Logging for clarity + self.log.info('EPEL release package installation invoked for: {}'.format(target_epel_release)) diff --git a/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py b/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py index a454ed381a..e1a4c009fd 100644 --- a/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py +++ b/repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py @@ -7,12 +7,15 @@ from leapp import reporting from leapp.reporting import Report from leapp.libraries.common.cllaunch import run_on_cloudlinux +from leapp.libraries.common.backup import backup_and_remove, LEAPP_BACKUP_SUFFIX REPO_DIR = '/etc/yum.repos.d' -REPO_DELETE_MARKERS = ['cloudlinux', 'imunify', 'epel'] +# These markers are used to identify which repository files should be directly replaced with new versions. +REPO_DELETE_MARKERS = ['cloudlinux', 'imunify'] +# These markers are used to identify which repository files should be replaced with new versions and backed up. REPO_BACKUP_MARKERS = [] +# This suffix is used to identify .rpmnew files that appear after package upgrade. RPMNEW = '.rpmnew' -LEAPP_BACKUP_SUFFIX = '.leapp-backup' class ReplaceRpmnewConfigs(Actor): @@ -30,32 +33,31 @@ def process(self): deleted_repofiles = [] renamed_repofiles = [] - for reponame in os.listdir(REPO_DIR): - if any(mark in reponame for mark in REPO_DELETE_MARKERS) and RPMNEW in reponame: - base_reponame = reponame[:-len(RPMNEW)] - base_path = os.path.join(REPO_DIR, base_reponame) - new_file_path = os.path.join(REPO_DIR, reponame) + for rpmnew_filename in os.listdir(REPO_DIR): + if any(mark in rpmnew_filename for mark in REPO_DELETE_MARKERS) and rpmnew_filename.endswith(RPMNEW): + main_reponame = rpmnew_filename[:-len(RPMNEW)] + main_file_path = os.path.join(REPO_DIR, main_reponame) + rpmnew_file_path = os.path.join(REPO_DIR, rpmnew_filename) - os.unlink(base_path) - os.rename(new_file_path, base_path) - deleted_repofiles.append(base_reponame) - self.log.debug('Yum repofile replaced: {}'.format(base_path)) + os.unlink(main_file_path) + os.rename(rpmnew_file_path, main_file_path) + deleted_repofiles.append(main_reponame) + self.log.debug('Yum repofile replaced: {}'.format(main_file_path)) - if any(mark in reponame for mark in REPO_BACKUP_MARKERS) and RPMNEW in reponame: - base_reponame = reponame[:-len(RPMNEW)] - base_path = os.path.join(REPO_DIR, base_reponame) - new_file_path = os.path.join(REPO_DIR, reponame) - backup_path = os.path.join(REPO_DIR, base_reponame + LEAPP_BACKUP_SUFFIX) + if any(mark in rpmnew_filename for mark in REPO_BACKUP_MARKERS) and rpmnew_filename.endswith(RPMNEW): + main_reponame = rpmnew_filename[:-len(RPMNEW)] + main_file_path = os.path.join(REPO_DIR, main_reponame) + rpmnew_file_path = os.path.join(REPO_DIR, rpmnew_filename) - os.rename(base_path, backup_path) - os.rename(new_file_path, base_path) - renamed_repofiles.append(base_reponame) - self.log.debug('Yum repofile replaced with backup: {}'.format(base_path)) + backup_and_remove(main_file_path) + os.rename(rpmnew_file_path, main_file_path) + renamed_repofiles.append(main_reponame) + self.log.debug('Yum repofile replaced with backup: {}'.format(main_file_path)) # Disable any old repositories. - for reponame in os.listdir(REPO_DIR): - if LEAPP_BACKUP_SUFFIX in reponame: - repofile_path = os.path.join(REPO_DIR, reponame) + for repofile_name in os.listdir(REPO_DIR): + if LEAPP_BACKUP_SUFFIX in repofile_name: + repofile_path = os.path.join(REPO_DIR, repofile_name) for line in fileinput.input(repofile_path, inplace=True): if line.startswith('enabled'): print("enabled = 0") @@ -66,7 +68,7 @@ def process(self): deleted_string = '\n'.join(['{}'.format(repofile_name) for repofile_name in deleted_repofiles]) replaced_string = '\n'.join(['{}'.format(repofile_name) for repofile_name in renamed_repofiles]) reporting.create_report([ - reporting.Title('CloudLinux repository config files replaced by updated versions'), + reporting.Title('Repository config files replaced by updated versions'), reporting.Summary( 'One or more RPM repository configuration files ' 'have been replaced with new versions provided by the upgraded packages. ' diff --git a/repos/system_upgrade/cloudlinux/libraries/backup.py b/repos/system_upgrade/cloudlinux/libraries/backup.py index 9002f569cb..7a40448bfd 100644 --- a/repos/system_upgrade/cloudlinux/libraries/backup.py +++ b/repos/system_upgrade/cloudlinux/libraries/backup.py @@ -1,3 +1,17 @@ +""" +Backup functionality for CloudLinux system upgrade process. + +This module provides utilities for backing up and restoring system configuration files +during the CloudLinux upgrade process. It includes functions for: +- Backing up files to a specified backup directory +- Creating in-place backups with .leapp-backup suffix +- Backing up and removing files +- Restoring files from backups + +Typically used in other CloudLinux upgrade actors to ensure that some specific configuration files +are preserved and can be restored in case of issues during the upgrade process. +""" + import os import shutil from leapp.libraries.stdlib import api @@ -10,6 +24,28 @@ ] BACKUP_DIR = "/var/lib/leapp/cl_backup" +LEAPP_BACKUP_SUFFIX = ".leapp-backup" + + +def backup_and_remove(path): + # type: (str) -> None + """ + Backup the file in-place and remove the original file. + + :param path: Path of the file to backup and remove. + """ + backup_file_in_place(path) + os.unlink(path) + + +def backup_file_in_place(path): + # type: (str) -> None + """ + Backup file in place, creating a copy of it with the same name and .leapp-backup suffix. + + :param path: Path of the file to backup. + """ + backup_file(path, path + LEAPP_BACKUP_SUFFIX) def backup_file(source, destination, backup_directory=""): @@ -19,14 +55,18 @@ def backup_file(source, destination, backup_directory=""): :param source: Path of the file to backup. :param destination: Destination name of a file in the backup directory. - :param dir: Backup directory override, defaults to None + If an absolute path is provided, it will be used as the destination path. + :param backup_directory: Backup directory override, defaults to None """ - if not backup_directory: - backup_directory = BACKUP_DIR - if not os.path.isdir(backup_directory): - os.makedirs(backup_directory) - - dest_path = os.path.join(backup_directory, destination) + # If destination is an absolute path, use it as the destination path + if os.path.isabs(destination): + dest_path = destination + else: + if not backup_directory: + backup_directory = BACKUP_DIR + if not os.path.isdir(backup_directory): + os.makedirs(backup_directory) + dest_path = os.path.join(backup_directory, destination) api.current_logger().debug('Backing up file: {} to {}'.format(source, dest_path)) shutil.copy(source, dest_path) From aac7a9036a8fd4d175b574c7ef111807405a668c Mon Sep 17 00:00:00 2001 From: Roman Prilipskii Date: Thu, 13 Feb 2025 13:48:22 +0100 Subject: [PATCH 3/4] Correct library function usage error in the refreshepel actor --- repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py b/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py index f2d280cc36..a0fd77fc8d 100644 --- a/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py +++ b/repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py @@ -9,7 +9,6 @@ from leapp.libraries.common.config.version import get_target_major_version REPO_DIR = '/etc/yum.repos.d' -EPEL_INSTALL_URL = 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm'.format(get_target_major_version()) class RefreshEPEL(Actor): @@ -40,8 +39,10 @@ def install_epel_release_package(self, target_url): @run_on_cloudlinux def process(self): + epel_install_url = 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm'.format(get_target_major_version()) + target_version = int(get_target_major_version()) - target_epel_release = EPEL_INSTALL_URL.format(target_version) + target_epel_release = epel_install_url.format(target_version) # EPEL release package name is 'epel-release' and the version should match the target OS version epel_release_package = 'epel-release' From 4eb5d1c851b53ea5a9ecbc697c6dc6fc21bfd785 Mon Sep 17 00:00:00 2001 From: Roman Prilipskii Date: Thu, 13 Feb 2025 16:56:34 +0100 Subject: [PATCH 4/4] Fix InstalledRPM usage in the clearpackageconflicts actor --- .../libraries/clearpackageconflicts.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py index 420ddce19f..f872c18947 100644 --- a/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py +++ b/repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py @@ -1,6 +1,5 @@ import os import errno -from re import L import shutil from leapp.libraries.stdlib import api @@ -80,5 +79,12 @@ def lua_cjson_handle(package_lookup): def process(): - rpm_lookup = {rpm.name for rpm in api.consume(InstalledRPM)} + rpm_lookup = set() + # Each InstalledRPM is a list of RPM objects. + # There's a bunch of other fields, but all that we're interested in here is their names. + installed_rpm_messages = api.consume(InstalledRPM) + for rpm_list in installed_rpm_messages: + rpm_names = [item.name for item in rpm_list.items] + rpm_lookup.update(rpm_names) + alt_python37_handle(rpm_lookup)