diff --git a/Sources/ContainerClient/Archiver.swift b/Sources/ContainerClient/Archiver.swift index eea80108..5d905026 100644 --- a/Sources/ContainerClient/Archiver.swift +++ b/Sources/ContainerClient/Archiver.swift @@ -78,6 +78,12 @@ public final class Archiver: Sendable { } entryInfo.append(info) } + // Handle empty directories - if no entries were added, add the directory itself + if entryInfo.isEmpty { + if let info = closure(source) { + entryInfo.append(info) + } + } } let archiver = try ArchiveWriter( @@ -98,6 +104,22 @@ public final class Archiver: Sendable { } } + public static func compress( + source: URL, + to handle: FileHandle, + followSymlinks: Bool = false, + writerConfiguration: ArchiveWriterConfiguration = ArchiveWriterConfiguration(format: .paxRestricted, filter: .gzip), + closure: (URL) -> ArchiveEntryInfo? + ) throws { + let destination = URL(fileURLWithPath: "/dev/fd/\(handle.fileDescriptor)") + try compress(source: source, destination: destination, followSymlinks: followSymlinks, writerConfiguration: writerConfiguration, closure: closure) + } + + public static func uncompress(from handle: FileHandle, to destination: URL) throws { + let source = URL(fileURLWithPath: "/dev/fd/\(handle.fileDescriptor)") + try uncompress(source: source, destination: destination) + } + public static func uncompress(source: URL, destination: URL) throws { let source = source.standardizedFileURL let destination = destination.standardizedFileURL diff --git a/Sources/ContainerClient/Core/ClientVolume.swift b/Sources/ContainerClient/Core/ClientVolume.swift index f5f90784..04cdefcd 100644 --- a/Sources/ContainerClient/Core/ClientVolume.swift +++ b/Sources/ContainerClient/Core/ClientVolume.swift @@ -90,4 +90,31 @@ public struct ClientVolume { let size = reply.uint64(key: .volumeSize) return size } + + public static func copyIn(volume: String, path: String, fileHandle: FileHandle) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .volumeCopyIn) + message.set(key: .volumeName, value: volume) + message.set(key: .path, value: path) + + let fd = fileHandle.fileDescriptor + let xpcHandle = FileHandle(fileDescriptor: fd, closeOnDealloc: false) + message.set(key: .fd, value: xpcHandle) + + _ = try await client.send(message) + } + + public static func copyOut(volume: String, path: String, fileHandle: FileHandle) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .volumeCopyOut) + message.set(key: .volumeName, value: volume) + message.set(key: .path, value: path) + + let fd = fileHandle.fileDescriptor + let xpcHandle = FileHandle(fileDescriptor: fd, closeOnDealloc: false) + message.set(key: .fd, value: xpcHandle) + + _ = try await client.send(message) + } + } diff --git a/Sources/ContainerClient/Core/XPC+.swift b/Sources/ContainerClient/Core/XPC+.swift index 41370998..c3e77cde 100644 --- a/Sources/ContainerClient/Core/XPC+.swift +++ b/Sources/ContainerClient/Core/XPC+.swift @@ -117,6 +117,7 @@ public enum XPCKeys: String { case volumeLabels case volumeReadonly case volumeContainerId + case path /// Container statistics case statistics @@ -157,6 +158,8 @@ public enum XPCRoute: String { case volumeDelete case volumeList case volumeInspect + case volumeCopyIn + case volumeCopyOut case volumeDiskUsage case systemDiskUsage diff --git a/Sources/ContainerCommands/Volume/VolumeCommand.swift b/Sources/ContainerCommands/Volume/VolumeCommand.swift index af01a65d..78077aab 100644 --- a/Sources/ContainerCommands/Volume/VolumeCommand.swift +++ b/Sources/ContainerCommands/Volume/VolumeCommand.swift @@ -27,6 +27,7 @@ extension Application { VolumeList.self, VolumeInspect.self, VolumePrune.self, + VolumeCopy.self, ], aliases: ["v"] ) diff --git a/Sources/ContainerCommands/Volume/VolumeCopy.swift b/Sources/ContainerCommands/Volume/VolumeCopy.swift new file mode 100644 index 00000000..b8b8d461 --- /dev/null +++ b/Sources/ContainerCommands/Volume/VolumeCopy.swift @@ -0,0 +1,163 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerClient +import Darwin +import Foundation + +extension Application.VolumeCommand { + struct VolumeCopy: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "cp", + abstract: "Copy files/folders between a container volume and the local filesystem" + ) + + @Argument(help: "Source path (use volume:path for volume paths)") + var source: String + + @Argument(help: "Destination path (use volume:path for volume paths)") + var destination: String + + @Flag(name: .shortAndLong, help: "Copy recursively") + var recursive: Bool = false + + func run() async throws { + let (sourceVol, sourcePath) = parsePath(source) + let (destVol, destPath) = parsePath(destination) + + try validatePaths(sourceVol, destVol) + + if let volName = sourceVol, let volPath = sourcePath { + try await copyFromVolume(volume: volName, volumePath: volPath, localPath: destination) + } else if let volName = destVol, let volPath = destPath { + try await copyToVolume(volume: volName, volumePath: volPath, localPath: source) + } + } + + private func validatePaths(_ sourceVol: String?, _ destVol: String?) throws { + if sourceVol != nil && destVol != nil { + throw ValidationError("copying between volumes is not supported") + } + if sourceVol == nil && destVol == nil { + throw ValidationError("one path must be a volume path (use volume:path format)") + } + } + + private func copyFromVolume(volume: String, volumePath: String, localPath: String) async throws { + let localURL = URL(fileURLWithPath: localPath) + let pipe = Pipe() + + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tempDir) } + + let writeFD = dup(pipe.fileHandleForWriting.fileDescriptor) + let writeHandle = FileHandle(fileDescriptor: writeFD, closeOnDealloc: true) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + defer { try? pipe.fileHandleForWriting.close() } + try await ClientVolume.copyOut(volume: volume, path: volumePath, fileHandle: writeHandle) + } + group.addTask { + try Archiver.uncompress(from: pipe.fileHandleForReading, to: tempDir) + } + try await group.waitForAll() + } + + let contents = try FileManager.default.contentsOfDirectory(at: tempDir, includingPropertiesForKeys: nil) + if contents.count == 1 { + let extracted = contents[0] + var isDir: ObjCBool = false + _ = FileManager.default.fileExists(atPath: extracted.path, isDirectory: &isDir) + if isDir.boolValue && !recursive { + throw ValidationError("source '\(volume):\(volumePath)' is a directory, use -r to copy recursively") + } + if FileManager.default.fileExists(atPath: localPath) { + _ = try FileManager.default.replaceItemAt(localURL, withItemAt: extracted) + } else { + try FileManager.default.moveItem(at: extracted, to: localURL) + } + } else { + if !recursive { + throw ValidationError("source '\(volume):\(volumePath)' is a directory, use -r to copy recursively") + } + try FileManager.default.createDirectory(at: localURL, withIntermediateDirectories: true) + for item in contents { + let dest = localURL.appendingPathComponent(item.lastPathComponent) + try FileManager.default.moveItem(at: item, to: dest) + } + } + } + + private func copyToVolume(volume: String, volumePath: String, localPath: String) async throws { + let localURL = URL(fileURLWithPath: localPath) + let pipe = Pipe() + + var isDir: ObjCBool = false + if !FileManager.default.fileExists(atPath: localPath, isDirectory: &isDir) { + throw ValidationError("source path does not exist: '\(localPath)'") + } + if isDir.boolValue && !recursive { + throw ValidationError("source is a directory, use -r to copy recursively") + } + + let readFD = dup(pipe.fileHandleForReading.fileDescriptor) + let readHandle = FileHandle(fileDescriptor: readFD, closeOnDealloc: true) + + let isDirectory = isDir.boolValue + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + defer { try? pipe.fileHandleForWriting.close() } + try Archiver.compress(source: localURL, to: pipe.fileHandleForWriting) { url in + let relativePath = self.computeRelativePath(source: localURL, current: url, isDirectory: isDirectory) + return Archiver.ArchiveEntryInfo( + pathOnHost: url, + pathInArchive: URL(fileURLWithPath: relativePath) + ) + } + } + group.addTask { + try await ClientVolume.copyIn(volume: volume, path: volumePath, fileHandle: readHandle) + } + try await group.waitForAll() + } + } + + private func computeRelativePath(source: URL, current: URL, isDirectory: Bool) -> String { + guard source.path != current.path else { + return source.lastPathComponent + } + let components = current.pathComponents + let baseComponents = source.pathComponents + guard components.count > baseComponents.count && components.prefix(baseComponents.count) == baseComponents[...] else { + return current.lastPathComponent + } + let relativePart = components[baseComponents.count...].joined(separator: "/") + return "\(source.lastPathComponent)/\(relativePart)" + } + + private func parsePath(_ path: String) -> (String?, String?) { + let parts = path.split(separator: ":", maxSplits: 1) + if parts.count == 2 { + return (String(parts[0]), String(parts[1])) + } + return (nil, nil) + } + } +} diff --git a/Sources/Helpers/APIServer/APIServer+Start.swift b/Sources/Helpers/APIServer/APIServer+Start.swift index 8bf36335..9b347919 100644 --- a/Sources/Helpers/APIServer/APIServer+Start.swift +++ b/Sources/Helpers/APIServer/APIServer+Start.swift @@ -272,6 +272,8 @@ extension APIServer { routes[XPCRoute.volumeList] = harness.list routes[XPCRoute.volumeInspect] = harness.inspect routes[XPCRoute.volumeDiskUsage] = harness.diskUsage + routes[XPCRoute.volumeCopyIn] = harness.copyIn + routes[XPCRoute.volumeCopyOut] = harness.copyOut return service } diff --git a/Sources/Services/ContainerAPIService/Volumes/VolumesHarness.swift b/Sources/Services/ContainerAPIService/Volumes/VolumesHarness.swift index d694abab..9eab54d5 100644 --- a/Sources/Services/ContainerAPIService/Volumes/VolumesHarness.swift +++ b/Sources/Services/ContainerAPIService/Volumes/VolumesHarness.swift @@ -104,4 +104,38 @@ public struct VolumesHarness: Sendable { reply.set(key: .volumeSize, value: size) return reply } + + @Sendable + public func copyIn(_ message: XPCMessage) async throws -> XPCMessage { + guard let name = message.string(key: .volumeName) else { + throw ContainerizationError(.invalidArgument, message: "volume name cannot be empty") + } + guard let path = message.string(key: .path) else { + throw ContainerizationError(.invalidArgument, message: "path cannot be empty") + } + guard let fd = message.fileHandle(key: .fd) else { + throw ContainerizationError(.invalidArgument, message: "file descriptor cannot be empty") + } + defer { try? fd.close() } + + try await service.copyIn(name: name, path: path, fileHandle: fd) + return message.reply() + } + + @Sendable + public func copyOut(_ message: XPCMessage) async throws -> XPCMessage { + guard let name = message.string(key: .volumeName) else { + throw ContainerizationError(.invalidArgument, message: "volume name cannot be empty") + } + guard let path = message.string(key: .path) else { + throw ContainerizationError(.invalidArgument, message: "path cannot be empty") + } + guard let fd = message.fileHandle(key: .fd) else { + throw ContainerizationError(.invalidArgument, message: "file descriptor cannot be empty") + } + defer { try? fd.close() } + + try await service.copyOut(name: name, path: path, fileHandle: fd) + return message.reply() + } } diff --git a/Sources/Services/ContainerAPIService/Volumes/VolumesService.swift b/Sources/Services/ContainerAPIService/Volumes/VolumesService.swift index cddc991b..5e33c769 100644 --- a/Sources/Services/ContainerAPIService/Volumes/VolumesService.swift +++ b/Sources/Services/ContainerAPIService/Volumes/VolumesService.swift @@ -284,4 +284,40 @@ public actor VolumesService { return volume } + public func copyIn(name: String, path: String, fileHandle: FileHandle) async throws { + try await lock.withLock { _ in + let volume = try await self._inspect(name) + let volumePath = self.volumePath(for: volume.name) + let destination = URL(fileURLWithPath: volumePath).appendingPathComponent(path) + try Archiver.uncompress(from: fileHandle, to: destination) + } + } + + public func copyOut(name: String, path: String, fileHandle: FileHandle) async throws { + try await lock.withLock { _ in + let volume = try await self._inspect(name) + let volumePath = self.volumePath(for: volume.name) + let source = URL(fileURLWithPath: volumePath).appendingPathComponent(path) + + try Archiver.compress(source: source, to: fileHandle) { url in + let relativePath: String + let sourcePath = source.path + let urlPath = url.path + + if sourcePath == urlPath { + relativePath = source.lastPathComponent + } else if urlPath.hasPrefix(sourcePath + "/") { + relativePath = String(urlPath.dropFirst(sourcePath.count + 1)) + } else { + relativePath = url.lastPathComponent + } + + return Archiver.ArchiveEntryInfo( + pathOnHost: url, + pathInArchive: URL(fileURLWithPath: relativePath) + ) + } + } + } + } diff --git a/Tests/CLITests/Subcommands/Volumes/TestCLIVolumes.swift b/Tests/CLITests/Subcommands/Volumes/TestCLIVolumes.swift index 75deefce..568089e6 100644 --- a/Tests/CLITests/Subcommands/Volumes/TestCLIVolumes.swift +++ b/Tests/CLITests/Subcommands/Volumes/TestCLIVolumes.swift @@ -452,4 +452,153 @@ class TestCLIVolumes: CLITest { #expect(statusFinal == 0) #expect(!listFinal.contains(volumeName), "volume should be pruned after container is deleted") } + + @Test func testFileCopyOperations() throws { + let testName = getTestName() + let volumeName = "\(testName)_vol" + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + + defer { + doVolumeDeleteIfExists(name: volumeName) + try? FileManager.default.removeItem(at: tempDir) + } + + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + try doVolumeCreate(name: volumeName) + + // Single file + try "Hello World".write(to: tempDir.appendingPathComponent("single.txt"), atomically: true, encoding: .utf8) + let (_, _, _, uploadStatus1) = try run(arguments: ["volume", "cp", tempDir.appendingPathComponent("single.txt").path, "\(volumeName):/single.txt"]) + #expect(uploadStatus1 == 0, "volume cp upload should succeed") + let (_, _, _, downloadStatus1) = try run(arguments: ["volume", "cp", "\(volumeName):/single.txt", tempDir.appendingPathComponent("out.txt").path]) + #expect(downloadStatus1 == 0, "volume cp download should succeed") + + let content1 = try String(contentsOf: tempDir.appendingPathComponent("out.txt"), encoding: .utf8) + #expect(content1 == "Hello World") + + // Subdirectory path + try "subdir-test".write(to: tempDir.appendingPathComponent("subdir.txt"), atomically: true, encoding: .utf8) + let (_, _, _, uploadStatus2) = try run(arguments: ["volume", "cp", tempDir.appendingPathComponent("subdir.txt").path, "\(volumeName):/path/to/subdir.txt"]) + #expect(uploadStatus2 == 0, "volume cp upload should succeed") + let (_, _, _, downloadStatus2) = try run(arguments: ["volume", "cp", "\(volumeName):/path/to/subdir.txt", tempDir.appendingPathComponent("subdir-out.txt").path]) + #expect(downloadStatus2 == 0, "volume cp download should succeed") + + let content2 = try String(contentsOf: tempDir.appendingPathComponent("subdir-out.txt"), encoding: .utf8) + #expect(content2 == "subdir-test") + } + + @Test func testDirectoryCopyOperations() throws { + let testName = getTestName() + let volumeName = "\(testName)_vol" + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + + defer { + doVolumeDeleteIfExists(name: volumeName) + try? FileManager.default.removeItem(at: tempDir) + } + + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + try doVolumeCreate(name: volumeName) + + // Directory with files + let myDir = tempDir.appendingPathComponent("mydir") + try FileManager.default.createDirectory(at: myDir, withIntermediateDirectories: true) + try "file1".write(to: myDir.appendingPathComponent("file1.txt"), atomically: true, encoding: .utf8) + try "file2".write(to: myDir.appendingPathComponent("file2.txt"), atomically: true, encoding: .utf8) + + let (_, _, _, uploadStatus1) = try run(arguments: ["volume", "cp", "-r", myDir.path, "\(volumeName):/mydir"]) + #expect(uploadStatus1 == 0, "volume cp upload should succeed") + let (_, _, _, downloadStatus1) = try run(arguments: ["volume", "cp", "-r", "\(volumeName):/mydir", tempDir.appendingPathComponent("mydir-out").path]) + #expect(downloadStatus1 == 0, "volume cp download should succeed") + + let outputDir = tempDir.appendingPathComponent("mydir-out") + #expect(FileManager.default.fileExists(atPath: outputDir.path)) + #expect(FileManager.default.fileExists(atPath: outputDir.appendingPathComponent("file1.txt").path)) + #expect(FileManager.default.fileExists(atPath: outputDir.appendingPathComponent("file2.txt").path)) + #expect(try String(contentsOf: outputDir.appendingPathComponent("file1.txt"), encoding: .utf8) == "file1") + + // Empty directory + let emptyDir = tempDir.appendingPathComponent("empty") + try FileManager.default.createDirectory(at: emptyDir, withIntermediateDirectories: true) + let (_, _, _, uploadStatus2) = try run(arguments: ["volume", "cp", "-r", emptyDir.path, "\(volumeName):/empty"]) + #expect(uploadStatus2 == 0, "volume cp upload should succeed") + let (_, _, _, downloadStatus2) = try run(arguments: ["volume", "cp", "-r", "\(volumeName):/empty", tempDir.appendingPathComponent("empty-out").path]) + #expect(downloadStatus2 == 0, "volume cp download should succeed") + + var isDirectory: ObjCBool = false + #expect(FileManager.default.fileExists(atPath: tempDir.appendingPathComponent("empty-out").path, isDirectory: &isDirectory) && isDirectory.boolValue) + + // Directory name preservation + let preserveDir = tempDir.appendingPathComponent("preserve-name") + try FileManager.default.createDirectory(at: preserveDir, withIntermediateDirectories: true) + try "test".write(to: preserveDir.appendingPathComponent("file.txt"), atomically: true, encoding: .utf8) + + let (_, _, _, uploadStatus3) = try run(arguments: ["volume", "cp", "-r", preserveDir.path, "\(volumeName):/preserve-name"]) + #expect(uploadStatus3 == 0, "volume cp upload should succeed") + let (_, _, _, downloadStatus3) = try run(arguments: ["volume", "cp", "-r", "\(volumeName):/preserve-name", tempDir.appendingPathComponent("preserve-out").path]) + #expect(downloadStatus3 == 0, "volume cp download should succeed") + + #expect(FileManager.default.fileExists(atPath: tempDir.appendingPathComponent("preserve-out/file.txt").path)) + } + + @Test func testLargeFileHandling() throws { + let testName = getTestName() + let volumeName = "\(testName)_vol" + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + + defer { + doVolumeDeleteIfExists(name: volumeName) + try? FileManager.default.removeItem(at: tempDir) + } + + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + try doVolumeCreate(name: volumeName) + + let largeFile = tempDir.appendingPathComponent("large.bin") + try Data(count: 10 * 1024 * 1024).write(to: largeFile) + + let (_, _, _, uploadStatus) = try run(arguments: ["volume", "cp", largeFile.path, "\(volumeName):/large.bin"]) + #expect(uploadStatus == 0, "volume cp upload should succeed") + let (_, _, _, downloadStatus) = try run(arguments: ["volume", "cp", "\(volumeName):/large.bin", tempDir.appendingPathComponent("large-out.bin").path]) + #expect(downloadStatus == 0, "volume cp download should succeed") + + let size1 = try FileManager.default.attributesOfItem(atPath: largeFile.path)[.size] as? Int + let size2 = try FileManager.default.attributesOfItem(atPath: tempDir.appendingPathComponent("large-out.bin").path)[.size] as? Int + #expect(size1 == size2) + } + + @Test func testErrorHandling() throws { + let testName = getTestName() + let volumeName = "\(testName)_vol" + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + + defer { + doVolumeDeleteIfExists(name: volumeName) + try? FileManager.default.removeItem(at: tempDir) + } + + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + try doVolumeCreate(name: volumeName) + + let noRecursiveDir = tempDir.appendingPathComponent("no-recursive") + try FileManager.default.createDirectory(at: noRecursiveDir, withIntermediateDirectories: true) + try "test".write(to: noRecursiveDir.appendingPathComponent("file.txt"), atomically: true, encoding: .utf8) + + // Test: copy directory TO volume without -r should fail + let (_, _, _, uploadStatus) = try run(arguments: ["volume", "cp", noRecursiveDir.path, "\(volumeName):/no-recursive"]) + #expect(uploadStatus != 0, "copy directory to volume without -r should fail") + + // Test: copy directory TO volume with -r should succeed + let (_, _, _, uploadWithRStatus) = try run(arguments: ["volume", "cp", "-r", noRecursiveDir.path, "\(volumeName):/no-recursive"]) + #expect(uploadWithRStatus == 0, "copy directory to volume with -r should succeed") + + // Test: copy directory FROM volume without -r should fail + let downloadDest = tempDir.appendingPathComponent("downloaded") + let (_, _, _, downloadStatus) = try run(arguments: ["volume", "cp", "\(volumeName):/no-recursive", downloadDest.path]) + #expect(downloadStatus != 0, "copy directory from volume without -r should fail") + + // Test: copy directory FROM volume with -r should succeed + let (_, _, _, downloadWithRStatus) = try run(arguments: ["volume", "cp", "-r", "\(volumeName):/no-recursive", downloadDest.path]) + #expect(downloadWithRStatus == 0, "copy directory from volume with -r should succeed") + } } diff --git a/docs/command-reference.md b/docs/command-reference.md index b01b8923..fd1a050d 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -849,6 +849,41 @@ container volume inspect [--debug] ... No options. +### `container volume cp` + +Copies files and directories between a volume and the local filesystem. Use the `volume:path` format to specify paths within a volume. For directories, the `-r` flag is required. + +**Usage** + +```bash +container volume cp [-r] [--debug] +``` + +**Arguments** + +* ``: Source path (use `volume:path` for volume paths) +* ``: Destination path (use `volume:path` for volume paths) + +**Options** + +* `-r, --recursive`: Copy directories recursively (required for directories) + +**Examples** + +```bash +# copy a file from local to volume +container volume cp ./config.json myvolume:/app/config.json + +# copy a file from volume to local +container volume cp myvolume:/app/data.txt ./data.txt + +# copy a directory to volume (requires -r) +container volume cp -r ./src myvolume:/app/src + +# copy a directory from volume to local (requires -r) +container volume cp -r myvolume:/app/logs ./logs +``` + ## Registry Management The registry commands manage authentication and defaults for container registries.