From a97a7a6211632ebe32fc55fecfa033c9d5cb10cc Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 5 May 2025 10:07:07 +0100 Subject: [PATCH 01/10] test: uncomment commented remove dir at end of tests --- core-dockpack/src/cmd_processes/pull/unpack_files.rs | 2 +- core-dockpack/src/cmd_processes/push/execute_push.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core-dockpack/src/cmd_processes/pull/unpack_files.rs b/core-dockpack/src/cmd_processes/pull/unpack_files.rs index 5b79a2e..d4d323c 100644 --- a/core-dockpack/src/cmd_processes/pull/unpack_files.rs +++ b/core-dockpack/src/cmd_processes/pull/unpack_files.rs @@ -46,6 +46,6 @@ mod tests { assert!(result.is_ok()); let path = result.unwrap(); assert!(Path::new(&path).exists()); - // fs::remove_dir_all(directory).unwrap(); + fs::remove_dir_all(directory).unwrap(); } } diff --git a/core-dockpack/src/cmd_processes/push/execute_push.rs b/core-dockpack/src/cmd_processes/push/execute_push.rs index 4e637b6..c378035 100644 --- a/core-dockpack/src/cmd_processes/push/execute_push.rs +++ b/core-dockpack/src/cmd_processes/push/execute_push.rs @@ -66,7 +66,7 @@ mod tests { assert!(result.is_ok()); - // fs::remove_dir_all(directory).expect("Failed to remove test directory"); + fs::remove_dir_all(directory).expect("Failed to remove test directory"); } #[tokio::test] @@ -82,6 +82,6 @@ mod tests { let result = execute_docker_build(directory, image_name).await; assert!(result.is_ok()); - // fs::remove_dir_all(directory).expect("Failed to remove test directory"); + fs::remove_dir_all(directory).expect("Failed to remove test directory"); } } From 3a4eda1dc1f286293bedd3d5909370e31e9cc168 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 5 May 2025 10:07:17 +0100 Subject: [PATCH 02/10] chore: add anyhow to requirements --- core-dockpack/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/core-dockpack/Cargo.toml b/core-dockpack/Cargo.toml index 1baf63f..326ddfb 100644 --- a/core-dockpack/Cargo.toml +++ b/core-dockpack/Cargo.toml @@ -16,3 +16,4 @@ futures-util = "0.3.31" tokio-util = { version = "0.7.14", features = ["io"] } tokio-tar = "0.3.1" futures-core = "0.3.31" +anyhow = "1.0.98" From d464f24ea533b92e7b3fd0029ad07c970124e551 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 5 May 2025 17:52:58 +0100 Subject: [PATCH 03/10] refactor: use anyhow with dockpack pull --- .../src/cmd_processes/pull/unpack_files.rs | 3 ++- core-dockpack/src/utils/docker_commands.rs | 22 +++++++++------- core-dockpack/src/utils/unpacking.rs | 25 +++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/core-dockpack/src/cmd_processes/pull/unpack_files.rs b/core-dockpack/src/cmd_processes/pull/unpack_files.rs index d4d323c..ab08c70 100644 --- a/core-dockpack/src/cmd_processes/pull/unpack_files.rs +++ b/core-dockpack/src/cmd_processes/pull/unpack_files.rs @@ -1,5 +1,6 @@ //! The API for unpacking Docker images into a directory. use crate::utils::{cache, docker_commands, unpacking}; +use anyhow::Result; use std::path::PathBuf; /// Unpacks the files from a Docker image into a directory. @@ -10,7 +11,7 @@ use std::path::PathBuf; /// /// # Returns /// The path to the directory where the Docker image files are stored. -pub async fn unpack_files_from_image(image: &str, directory: &str) -> Result { +pub async fn unpack_files_from_image(image: &str, directory: &str) -> Result { let main_path = PathBuf::from(directory); cache::wipe_and_create_cache(&main_path); diff --git a/core-dockpack/src/utils/docker_commands.rs b/core-dockpack/src/utils/docker_commands.rs index d23ab48..6a4269e 100644 --- a/core-dockpack/src/utils/docker_commands.rs +++ b/core-dockpack/src/utils/docker_commands.rs @@ -1,5 +1,6 @@ //! Defines the actions around downloading and unpacking docker images to access the files. use super::cache::process_image_name; +use anyhow::{anyhow, Context, Result}; use bollard::image::CreateImageOptions; use bollard::Docker; use futures_util::stream::TryStreamExt; @@ -10,7 +11,7 @@ use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio_tar::Archive; -async fn pull_image(image_name: &str, docker: &Docker) -> Result<(), String> { +async fn pull_image(image_name: &str, docker: &Docker) -> Result<()> { let options = Some(CreateImageOptions { from_image: image_name, ..Default::default() @@ -21,7 +22,7 @@ async fn pull_image(image_name: &str, docker: &Docker) -> Result<(), String> { .create_image(options, None, None) .try_collect::>() .await - .map_err(|err| err.to_string())?; + .with_context(|| "Error creating image")?; Ok(()) } @@ -36,7 +37,7 @@ async fn pull_image(image_name: &str, docker: &Docker) -> Result<(), String> { /// /// # Returns /// The path to where the compressed Docker image files are stored -pub async fn save_docker_image(image_name: &str, tar_path: &str) -> Result { +pub async fn save_docker_image(image_name: &str, tar_path: &str) -> Result { // pull image // // TODO: consider moving to receive in the function let docker = Docker::connect_with_socket_defaults() @@ -56,17 +57,17 @@ pub async fn save_docker_image(image_name: &str, tar_path: &str) -> Result Result v.to_string(), - None => return Err("Failed to convert path to string".to_string()), + None => return Err(anyhow!("Failed to convert path to string")), }) } diff --git a/core-dockpack/src/utils/unpacking.rs b/core-dockpack/src/utils/unpacking.rs index 3db9bf7..2bd25b1 100644 --- a/core-dockpack/src/utils/unpacking.rs +++ b/core-dockpack/src/utils/unpacking.rs @@ -1,4 +1,5 @@ //! Defines the actions around unpacking compressed Docker files from the manifest. +use anyhow::{anyhow, Context, Result}; use flate2::read::GzDecoder; use serde_json::Value; use std::fs::File; @@ -48,16 +49,17 @@ fn read_json_file>(path: P) -> std::io::Result { /// /// # Returns /// The path to where the layers are extracted. -pub fn extract_layers(main_path: &str, unpack_path: &str) -> Result { +pub fn extract_layers(main_path: &str, unpack_path: &str) -> Result { let manifest_path = std::path::Path::new(main_path).join("manifest.json"); let blobs_dir = std::path::Path::new(main_path); let unpack_path = std::path::Path::new(unpack_path); if !unpack_path.exists() { - std::fs::create_dir_all(unpack_path).map_err(|e| e.to_string())?; + std::fs::create_dir_all(unpack_path).with_context(|| "Error creating directory")?; } - let manifest = read_json_file(&manifest_path).map_err(|e| e.to_string())?; + let manifest = read_json_file(&manifest_path) + .with_context(|| format!("Error reading json manifest file"))?; if let Some(layers) = manifest[0]["Layers"].as_array() { println!("Found {} layers in manifest", layers.len()); @@ -70,25 +72,27 @@ pub fn extract_layers(main_path: &str, unpack_path: &str) -> Result layer, None => { return Err( - "Failed to get the layer path when extracting a layer from the Docker image".to_string() + anyhow!("Failed to get the layer path when extracting a layer from the Docker image") ); } }); // Extract the layer's tarball to a directory - let mut tar_file = File::open(&layer_path).map_err(|e| e.to_string())?; - let if_gzipped = check_if_gzipped(&mut tar_file).map_err(|e| e.to_string())?; + let mut tar_file = + File::open(&layer_path).with_context(|| format!("Error reading tar file"))?; + let if_gzipped = check_if_gzipped(&mut tar_file) + .with_context(|| format!("Erorr checcking if tar fle is gzipped"))?; match if_gzipped { true => { println!("Layer is gzipped"); let decompressed = GzDecoder::new(tar_file); let mut archive = Archive::new(decompressed); - archive.unpack(unpack_path).map_err(|e| e.to_string())?; + archive.unpack(unpack_path)?; } false => { println!("Layer is not gzipped"); let mut archive = Archive::new(tar_file); - archive.unpack(unpack_path).map_err(|e| e.to_string())?; + archive.unpack(unpack_path)?; } } } @@ -99,10 +103,9 @@ pub fn extract_layers(main_path: &str, unpack_path: &str) -> Result v.to_string(), None => { - return Err( + return Err(anyhow!( "Failed to convert path to string when extracting layers from the Docker image" - .to_string(), - ); + )); } }) } From dc122c9eca24987e1b50cadf8c1b2111f6663881 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 5 May 2025 18:00:13 +0100 Subject: [PATCH 04/10] refactor: build dockerfile command with anyhow crate --- .../src/cmd_processes/build/build_dockerfile.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core-dockpack/src/cmd_processes/build/build_dockerfile.rs b/core-dockpack/src/cmd_processes/build/build_dockerfile.rs index 7f40982..ebc8e94 100644 --- a/core-dockpack/src/cmd_processes/build/build_dockerfile.rs +++ b/core-dockpack/src/cmd_processes/build/build_dockerfile.rs @@ -1,20 +1,27 @@ //! Builds a Dockerfile from a directory +use anyhow::{Context, Result}; use std::fs::File; use std::io::Write; // directory is the build context -pub fn create_dockerfile(directory: &str) -> Result<(), String> { +pub fn create_dockerfile(directory: &str) -> Result<()> { let docker_file_content = "FROM scratch\nCOPY . .\n".to_string(); let dockerfile_path = format!("{}/Dockerfile", directory); - let mut dockerfile = File::create(&dockerfile_path).map_err(|e| e.to_string())?; + let mut dockerfile = File::create(&dockerfile_path) + .with_context(|| format!("Error creatining file at path {}", dockerfile_path))?; dockerfile .write_all(docker_file_content.as_bytes()) - .map_err(|e| e.to_string())?; + .with_context(|| { + format!( + "Could not write all from scratch content to dockerfile at path {}", + dockerfile_path, + ) + })?; Ok(()) } From 7610e7d795e91af3981b2c0a1757bb5fbd8d7bdf Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 5 May 2025 18:28:33 +0100 Subject: [PATCH 05/10] refactor: make test_execute_push pass again & use anyhow crate for errors --- .../src/cmd_processes/push/execute_push.rs | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/core-dockpack/src/cmd_processes/push/execute_push.rs b/core-dockpack/src/cmd_processes/push/execute_push.rs index c378035..41c28dc 100644 --- a/core-dockpack/src/cmd_processes/push/execute_push.rs +++ b/core-dockpack/src/cmd_processes/push/execute_push.rs @@ -1,36 +1,40 @@ use crate::utils::cache; +use anyhow::{Context, Result}; use bollard::models::CreateImageInfo; use bollard::Docker; use futures_util::stream::TryStreamExt; use tokio::fs::File; use tokio_util::io::ReaderStream; -async fn dir_to_tar(dir: &str, image_name: &str) -> Result<(), String> { - let tar_name = &format!("{}.tar", cache::process_image_name(image_name)); - let tar = File::create(tar_name) +async fn dir_to_tar(dir: &str, image_name: &str) -> Result { + let tar_name = format!("{}.tar", cache::process_image_name(image_name)); + let tar = File::create(&tar_name) .await - .expect("Could not create archive file"); + .with_context(|| "Could not create archive file")?; let mut tar = tokio_tar::Builder::new(tar); tar.append_dir_all("", dir) .await - .expect("Could not add path to target"); + .with_context(|| "Could not add path to target")?; tar.finish() .await - .expect("An error occured in converting dir to tar"); - Ok(()) + .with_context(|| "An error occured in converting dir to tar")?; + Ok(tar_name) } -pub async fn execute_docker_build(directory: &str, image: &str) -> Result<(), String> { +pub async fn execute_docker_build(directory: &str, image: &str) -> Result<()> { // Convert directory to a tar file - dir_to_tar(directory, image).await?; - - let file = File::open(&format!("{}.tar", image)) - .await - .expect("Could not find archive."); + let tar_path = dir_to_tar(directory, image).await?; + + let file = File::open(tar_path).await.with_context(|| { + format!( + "Could not find archive at path {}.", + format!("{}.tar", image) + ) + })?; let stream = ReaderStream::new(file); let docker = Docker::connect_with_socket_defaults() - .expect("Could no connect to docker socket. Is docker running?"); + .with_context(|| "Could no connect to docker socket. Is docker running?")?; let options = bollard::query_parameters::CreateImageOptionsBuilder::default() .from_src("-") // from_src must be "-" when sending the archive in the request body @@ -42,7 +46,7 @@ pub async fn execute_docker_build(directory: &str, image: &str) -> Result<(), St .create_image(Some(options), Some(bollard::body_try_stream(stream)), None) .try_collect() .await - .expect("Could not create image"); + .with_context(|| "Could not create image")?; Ok(()) } From 58a76adb510e2c03c99fdc195b92d32dbaace985 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 5 May 2025 18:41:42 +0100 Subject: [PATCH 06/10] chore: changes for clippy and rust formatting --- core-dockpack/src/cmd_processes/push/execute_push.rs | 9 +++------ core-dockpack/src/utils/unpacking.rs | 9 ++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/core-dockpack/src/cmd_processes/push/execute_push.rs b/core-dockpack/src/cmd_processes/push/execute_push.rs index 41c28dc..ee0506c 100644 --- a/core-dockpack/src/cmd_processes/push/execute_push.rs +++ b/core-dockpack/src/cmd_processes/push/execute_push.rs @@ -25,12 +25,9 @@ pub async fn execute_docker_build(directory: &str, image: &str) -> Result<()> { // Convert directory to a tar file let tar_path = dir_to_tar(directory, image).await?; - let file = File::open(tar_path).await.with_context(|| { - format!( - "Could not find archive at path {}.", - format!("{}.tar", image) - ) - })?; + let file = File::open(tar_path) + .await + .with_context(|| format!("Could not find archive at path {}.tar", image))?; let stream = ReaderStream::new(file); let docker = Docker::connect_with_socket_defaults() diff --git a/core-dockpack/src/utils/unpacking.rs b/core-dockpack/src/utils/unpacking.rs index 2bd25b1..1fcaf3b 100644 --- a/core-dockpack/src/utils/unpacking.rs +++ b/core-dockpack/src/utils/unpacking.rs @@ -58,8 +58,8 @@ pub fn extract_layers(main_path: &str, unpack_path: &str) -> Result { std::fs::create_dir_all(unpack_path).with_context(|| "Error creating directory")?; } - let manifest = read_json_file(&manifest_path) - .with_context(|| format!("Error reading json manifest file"))?; + let manifest = + read_json_file(&manifest_path).with_context(|| "Error reading json manifest file")?; if let Some(layers) = manifest[0]["Layers"].as_array() { println!("Found {} layers in manifest", layers.len()); @@ -78,10 +78,9 @@ pub fn extract_layers(main_path: &str, unpack_path: &str) -> Result { }); // Extract the layer's tarball to a directory - let mut tar_file = - File::open(&layer_path).with_context(|| format!("Error reading tar file"))?; + let mut tar_file = File::open(&layer_path).with_context(|| "Error reading tar file")?; let if_gzipped = check_if_gzipped(&mut tar_file) - .with_context(|| format!("Erorr checcking if tar fle is gzipped"))?; + .with_context(|| "Error checking if tar fle is gzipped")?; match if_gzipped { true => { println!("Layer is gzipped"); From 84ed3da6de8f6b9b6cb45d74025beb5cf5c20e26 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 7 May 2025 18:22:12 +0100 Subject: [PATCH 07/10] refactor: comply with clippy lint * No longer used deptecrated Struct * add clippy allow for unsafe code --- core-dockpack/src/utils/docker_commands.rs | 12 ++++++------ coredockpack/src/lib.rs | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core-dockpack/src/utils/docker_commands.rs b/core-dockpack/src/utils/docker_commands.rs index 6a4269e..1fff47b 100644 --- a/core-dockpack/src/utils/docker_commands.rs +++ b/core-dockpack/src/utils/docker_commands.rs @@ -1,21 +1,21 @@ //! Defines the actions around downloading and unpacking docker images to access the files. use super::cache::process_image_name; use anyhow::{anyhow, Context, Result}; -use bollard::image::CreateImageOptions; +use bollard::query_parameters::CreateImageOptionsBuilder; use bollard::Docker; use futures_util::stream::TryStreamExt; use futures_util::StreamExt; -use std::default::Default; use std::process::Command; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio_tar::Archive; async fn pull_image(image_name: &str, docker: &Docker) -> Result<()> { - let options = Some(CreateImageOptions { - from_image: image_name, - ..Default::default() - }); + let options = Some( + CreateImageOptionsBuilder::new() + .from_image(image_name) + .build(), + ); println!("image_name: {}", image_name); // confirmed: this works docker diff --git a/coredockpack/src/lib.rs b/coredockpack/src/lib.rs index 0e18731..438ec6b 100644 --- a/coredockpack/src/lib.rs +++ b/coredockpack/src/lib.rs @@ -12,6 +12,7 @@ use std::os::raw::c_char; /// A C string with the path to the directory where the Docker image files are stored. /// On error, returns a null pointer. #[allow(clippy::not_unsafe_ptr_arg_deref)] +#[allow(improper_ctypes_definitions)] #[no_mangle] pub async extern "C" fn unpack_files_from_image_c( image: *const c_char, From 596bf8285b2609a7758a32f4e1fffee67e5386fd Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 7 May 2025 18:27:31 +0100 Subject: [PATCH 08/10] chore: correct typo in error message --- core-dockpack/src/cmd_processes/push/execute_push.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-dockpack/src/cmd_processes/push/execute_push.rs b/core-dockpack/src/cmd_processes/push/execute_push.rs index ee0506c..ce685eb 100644 --- a/core-dockpack/src/cmd_processes/push/execute_push.rs +++ b/core-dockpack/src/cmd_processes/push/execute_push.rs @@ -31,7 +31,7 @@ pub async fn execute_docker_build(directory: &str, image: &str) -> Result<()> { let stream = ReaderStream::new(file); let docker = Docker::connect_with_socket_defaults() - .with_context(|| "Could no connect to docker socket. Is docker running?")?; + .with_context(|| "Could not connect to docker socket. Is docker running?")?; let options = bollard::query_parameters::CreateImageOptionsBuilder::default() .from_src("-") // from_src must be "-" when sending the archive in the request body From 68f454a9f6fa94b95f02d89ef30bfbd3a0c10c08 Mon Sep 17 00:00:00 2001 From: Tom Date: Sat, 28 Jun 2025 23:23:10 +0100 Subject: [PATCH 09/10] refactor: make C type function C FFI safe * Function is no longer async. * No longer has opaque C type. * Therefore, can be C FFI safe again. --- coredockpack/Cargo.toml | 1 + coredockpack/src/lib.rs | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/coredockpack/Cargo.toml b/coredockpack/Cargo.toml index ac64187..b98cf8c 100644 --- a/coredockpack/Cargo.toml +++ b/coredockpack/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] core-dockpack = { path = "../core-dockpack" } +tokio = "1.45.1" [lib] diff --git a/coredockpack/src/lib.rs b/coredockpack/src/lib.rs index 438ec6b..c70cb3f 100644 --- a/coredockpack/src/lib.rs +++ b/coredockpack/src/lib.rs @@ -1,6 +1,7 @@ use core_dockpack::cmd_processes::pull::unpack_files::unpack_files_from_image; use std::ffi::{CStr, CString}; use std::os::raw::c_char; +use tokio::runtime::Runtime; /// Unpacks the files from a Docker image into a directory. /// @@ -12,9 +13,8 @@ use std::os::raw::c_char; /// A C string with the path to the directory where the Docker image files are stored. /// On error, returns a null pointer. #[allow(clippy::not_unsafe_ptr_arg_deref)] -#[allow(improper_ctypes_definitions)] #[no_mangle] -pub async extern "C" fn unpack_files_from_image_c( +pub extern "C" fn unpack_files_from_image_c( image: *const c_char, directory: *const c_char, ) -> *const c_char { @@ -22,7 +22,9 @@ pub async extern "C" fn unpack_files_from_image_c( let image = unsafe { CStr::from_ptr(image).to_string_lossy().into_owned() }; let directory = unsafe { CStr::from_ptr(directory).to_string_lossy().into_owned() }; - match unpack_files_from_image(&image, &directory).await { + let rt = Runtime::new().unwrap(); + let result = rt.block_on(unpack_files_from_image(&image, &directory)); + match result { Ok(path) => { let c_string = CString::new(path).unwrap(); c_string.into_raw() // Return the C string From d1843ad7cafed49b62ae27598194ca3cfc22076d Mon Sep 17 00:00:00 2001 From: Tom Date: Sat, 5 Jul 2025 16:39:43 +0100 Subject: [PATCH 10/10] chore: do not give image a tag, it already has one, and do not process image name for push --- core-dockpack/src/cmd_processes/push/execute_push.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core-dockpack/src/cmd_processes/push/execute_push.rs b/core-dockpack/src/cmd_processes/push/execute_push.rs index 40750b0..098d637 100644 --- a/core-dockpack/src/cmd_processes/push/execute_push.rs +++ b/core-dockpack/src/cmd_processes/push/execute_push.rs @@ -37,7 +37,6 @@ pub async fn execute_docker_build(directory: &str, image: &str) -> Result<()> { let options = CreateImageOptionsBuilder::default() .from_src("-") // from_src must be "-" when sending the archive in the request body .repo(image) // The name of the image in the docker daemon. - .tag("1.0.0") // The tag of this particular image. .build(); let _: Vec = docker .create_image(Some(options), Some(bollard::body_try_stream(stream)), None) @@ -47,7 +46,7 @@ pub async fn execute_docker_build(directory: &str, image: &str) -> Result<()> { let options = PushImageOptionsBuilder::new().tag("latest").build(); let _: Vec = docker - .push_image(&cache::process_image_name(image), Some(options), None) + .push_image(image, Some(options), None) .try_collect() .await .with_context(|| "Could not push image")?;