From cffbe77a64ef159f946ec11a9274782c6131d513 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 19 Dec 2024 16:37:54 -0600 Subject: [PATCH] Install a curated registry of crates with libraries This largely mirrors the approach taken by major Linux distributions like Ubuntu and Fedora. We can curate the crates in a way that should allow us to use them in downstream package builds within this colcon workspace or downstream workspaces. This change doesn't yet add the code necessary to instruct cargo to use our curated registry. --- colcon_cargo/task/cargo/build.py | 86 ++++++++++++++++++++++++++++++++ test/spell_check.words | 2 + test/test_build.py | 14 ++++++ 3 files changed, 102 insertions(+) diff --git a/colcon_cargo/task/cargo/build.py b/colcon_cargo/task/cargo/build.py index 3a7557e..c4f0a2c 100644 --- a/colcon_cargo/task/cargo/build.py +++ b/colcon_cargo/task/cargo/build.py @@ -10,6 +10,8 @@ from colcon_core.logging import colcon_logger from colcon_core.plugin_system import satisfies_version from colcon_core.shell import create_environment_hook, get_command_environment +from colcon_core.task import create_file +from colcon_core.task import install from colcon_core.task import run from colcon_core.task import TaskExtensionPoint @@ -96,6 +98,11 @@ async def build( # noqa: D102 if rc and rc.returncode: return rc.returncode + if self._has_libraries(metadata, pkg.name): + self.progress('package') + await self._install_package( + metadata['packages'][0]['version'], env) + if not skip_hook_creation: create_environment_scripts( pkg, args, additional_hooks=additional_hooks) @@ -197,3 +204,82 @@ def _has_binaries(metadata, package_name): # If no binary target exists in the whole package, then skip running # cargo install because it would produce an error. return False + + # Identify if there are any libraries to install for the current package + @staticmethod + def _has_libraries(metadata, package_name): + for package in metadata.get('packages', {}): + # If the package is part of a cargo workspace, the metadata + # contains all members. We're only interested in our target + # package - ignore the other workspace members here. + if package.get('name') != package_name: + continue + for target in package.get('targets', {}): + if { + 'lib', + 'rlib', + 'proc-macro', + }.intersection(target.get('crate_types', ())): + # If any one binary exists in the package then we + # should go ahead and install the extracted crate + return True + + # If no library target exists in the whole package, then skip extracted + # crate installation because it isn't useful. + return False + + # Determine what files would be part of a packaged crate + async def _get_crate_contents(self, env): + pkg = self.context.pkg + cmd = [ + CARGO_EXECUTABLE, + 'package', + '--list', + '--allow-dirty', + '--quiet', + '--package', pkg.name, + ] + + rc = await run( + self.context, + cmd, + cwd=self.context.pkg.path, + capture_output=True, + env=env + ) + if rc is None or rc.returncode != 0: + raise RuntimeError( + "Could not inspect package using 'cargo package'" + ) + + if rc.stdout is None: + raise RuntimeError( + "Failed to capture stdout from 'cargo package'" + ) + + contents = set(rc.stdout.decode().splitlines()) + contents.difference_update({ + # Ignore stuff that we wouldn't want to copy + '', + None, + 'Cargo.lock', + 'Cargo.toml.orig', + '.cargo_vcs_info.json', + }) + return contents + + async def _install_package(self, version, env): + contents = await self._get_crate_contents(env) + crate_path = Path( + 'share', 'cargo', 'registry', f'{self.context.pkg.name}-{version}') + + for file in contents: + dst = crate_path / file + install(self.context.args, file, dst) + + # Cargo "directory sources" require a checksum file to be included in + # the package metadata (though it need not list all of the files). + create_file( + self.context.args, + crate_path / '.cargo-checksum.json', + content='{"files":{},"package":""}\n') diff --git a/test/spell_check.words b/test/spell_check.words index 88b7069..29ceb3c 100644 --- a/test/spell_check.words +++ b/test/spell_check.words @@ -2,6 +2,7 @@ apache argcomplete asyncio autouse +checksum colcon completers cwpd @@ -26,6 +27,7 @@ pydocstyle pytest returncode rglob +rlib rmtree rtype rustfmt diff --git a/test/test_build.py b/test/test_build.py index 163c525..4320bad 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -27,6 +27,7 @@ TEST_PACKAGE_NAME = 'rust-sample-package' PURE_LIBRARY_PACKAGE_NAME = 'rust-pure-library' WORKSPACE_PACKAGE_NAME = 'rust-workspace' +WORKSPACE_PACKAGE_VERSION = '0.1.0' test_project_path = Path(__file__).parent / TEST_PACKAGE_NAME pure_library_path = Path(__file__).parent / PURE_LIBRARY_PACKAGE_NAME @@ -170,6 +171,7 @@ def test_build_and_test_package(): path=str(test_project_path), build_base=str(tmpdir / 'build'), install_base=str(tmpdir / 'install'), + symlink_install=False, clean_build=None, cargo_args=None, ), @@ -236,6 +238,7 @@ def test_skip_pure_library_package(): path=str(pure_library_path), build_base=str(tmpdir / 'build'), install_base=str(tmpdir / 'install'), + symlink_install=False, clean_build=None, cargo_args=None, ), @@ -290,6 +293,7 @@ def test_workspace_with_package(): path=str(workspace_project_path), build_base=str(tmpdir / 'build'), install_base=str(tmpdir / 'install'), + symlink_install=False, clean_build=None, cargo_args=None, ), @@ -315,5 +319,15 @@ def test_workspace_with_package(): # members didn't get installed as well assert len(tuple((install_base / 'bin').iterdir())) == 1 + # There should also be an unpacked library create + registry_path = install_base / 'share' / 'cargo' / 'registry' + crate_path = registry_path / '-'.join(( + WORKSPACE_PACKAGE_NAME, + WORKSPACE_PACKAGE_VERSION, + )) + assert tuple(registry_path.iterdir()) == (crate_path,) + assert (crate_path / 'Cargo.toml').is_file() + assert (crate_path / 'src' / 'lib.rs').is_file() + finally: event_loop.close()