diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79..b0dd1eec 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/public/manifest" ) var shortCutHelp = "Cut a tree with selected slices" @@ -73,6 +74,24 @@ func (cmd *cmdCut) Execute(args []string) error { } } + mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) + if err != nil { + return err + } + if mfest != nil { + err = mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + sliceKeys = append(sliceKeys, sk) + return nil + }) + if err != nil { + return err + } + } + selection, err := setup.Select(release, sliceKeys, cmd.Arch) if err != nil { return err @@ -125,6 +144,7 @@ func (cmd *cmdCut) Execute(args []string) error { Selection: selection, Archives: archives, TargetDir: cmd.RootDir, + Manifest: mfest, }) return err } diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 0503c96f..60bfc2b0 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -166,11 +166,22 @@ func createDir(o *CreateOptions) error { if err != nil { return err } - err = os.Mkdir(path, o.Mode) - if os.IsExist(err) { - return nil + fileinfo, err := os.Lstat(path) + if err == nil { + if fileinfo.IsDir() { + if fileinfo.Mode() != o.Mode && o.OverrideMode { + return os.Chmod(path, o.Mode) + } + return nil + } + err = os.Remove(path) + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err } - return err + return os.Mkdir(path, o.Mode) } func createFile(o *CreateOptions) error { diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b05402..b2946d57 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -18,22 +18,44 @@ import ( const DefaultFilename = "manifest.wall" +func collectManifests(slice *setup.Slice, collector func(path string, slice *setup.Slice)) { + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + collector(path, slice) + } + } +} + // FindPaths finds the paths marked with "generate:manifest" and // returns a map from the manifest path to all the slices that declare it. func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { manifestSlices := make(map[string][]*setup.Slice) + collector := func(path string, slice *setup.Slice) { + manifestSlices[path] = append(manifestSlices[path], slice) + } for _, slice := range slices { - for path, info := range slice.Contents { - if info.Generate == setup.GenerateManifest { - dir := strings.TrimSuffix(path, "**") - path = filepath.Join(dir, DefaultFilename) - manifestSlices[path] = append(manifestSlices[path], slice) - } - } + collectManifests(slice, collector) } return manifestSlices } +// FindPathsInRelease finds all the paths marked with "generate:manifest" +// for the given release. +func FindPathsInRelease(r *setup.Release) []string { + manifestPaths := make([]string, 0) + collector := func(path string, slice *setup.Slice) { + manifestPaths = append(manifestPaths, path) + } + for _, pkg := range r.Packages { + for _, slice := range pkg.Slices { + collectManifests(slice, collector) + } + } + return manifestPaths +} + type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice @@ -340,3 +362,11 @@ func Validate(mfest *manifest.Manifest) (err error) { } return nil } + +// CompareSchemas compares two manifest schema strings. +func CompareSchemas(va, vb string) int { + if va == manifest.Schema && va == vb { + return 0 + } + return -1 +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb..7923d6cc 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,14 @@ package slicer import ( "archive/tar" "bytes" + "crypto/sha256" + "encoding/hex" "fmt" "io" "io/fs" + "maps" "os" + "path" "path/filepath" "slices" "sort" @@ -21,6 +25,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/manifest" ) const manifestMode fs.FileMode = 0644 @@ -29,6 +34,7 @@ type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive TargetDir string + Manifest *manifest.Manifest } type pathData struct { @@ -90,6 +96,19 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } + var originalTargetDir string + if options.Manifest != nil { + tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*") + if err != nil { + return fmt.Errorf("cannot create temporary working directory: %w", err) + } + originalTargetDir = targetDir + targetDir = tmpWorkDir + defer func() { + os.RemoveAll(tmpWorkDir) + }() + } + pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err @@ -347,7 +366,110 @@ func Run(options *RunOptions) error { return err } - return generateManifests(targetDir, options.Selection, report, pkgInfos) + err = generateManifests(targetDir, options.Selection, report, pkgInfos) + if err != nil { + return err + } + + if options.Manifest != nil { + return upgrade(originalTargetDir, targetDir, report, options.Manifest) + } + return nil +} + +// absPath requires root to be a clean path that ends in "/". +func absPath(root, relPath string) (string, error) { + path := filepath.Clean(filepath.Join(root, relPath)) + if !strings.HasPrefix(path, root) { + return "", fmt.Errorf("cannot create path %s outside of root %s", path, root) + } + return path, nil +} + +// upgrade upgrades content in targetDir using content in tempDir. +func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { + logf("Upgrading existing content...") + missingPaths := make([]string, 0) + err := oldManifest.IteratePaths("", func(path *manifest.Path) error { + _, ok := newReport.Entries[path.Path] + if !ok { + missingPaths = append(missingPaths, path.Path) + } + return nil + }) + if err != nil { + return err + } + + paths := slices.Sorted(maps.Keys(newReport.Entries)) + for _, path := range paths { + srcPath, err := absPath(tempDir, path) + if err != nil { + return err + } + dstPath, err := absPath(targetDir, path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + + entry := newReport.Entries[path] + switch entry.Mode & fs.ModeType { + case 0: + case fs.ModeSymlink: + err = os.Rename(srcPath, dstPath) + case fs.ModeDir: + err = upgradeDir(dstPath, &entry) + default: + err = fmt.Errorf("unsupported file type: %s", path) + } + if err != nil { + return err + } + } + + // Remove missing paths + slices.Sort(missingPaths) + slices.Reverse(missingPaths) + for _, relPath := range missingPaths { + path, err := absPath(targetDir, relPath) + if err != nil { + return err + } + if strings.HasSuffix(path, "/") { + err = syscall.Rmdir(path) + if err != nil && err != syscall.ENOTEMPTY { + return err + } + } else { + err = os.Remove(path) + if err != nil { + return err + } + } + } + return nil +} + +func upgradeDir(path string, entry *manifestutil.ReportEntry) error { + fileinfo, err := os.Lstat(path) + if err == nil { + if fileinfo.IsDir() { + if fileinfo.Mode() != entry.Mode { + return os.Chmod(path, entry.Mode) + } + return nil + } + err = os.Remove(path) + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + return os.Mkdir(path, entry.Mode) } func generateManifests(targetDir string, selection *setup.Selection, @@ -537,3 +659,89 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel } return pkgArchive, nil } + +// SelectValidManifest returns, if found, a valid manifest with the latest +// schema. Consistency with all other manifests with the same schema is verified +// so the selection is deterministic. +func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) { + targetDir = filepath.Clean(targetDir) + if !filepath.IsAbs(targetDir) { + dir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("cannot obtain current directory: %w", err) + } + targetDir = filepath.Join(dir, targetDir) + } + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) == 0 { + return nil, nil + } + + type manifestHash struct { + path string + hash string + } + var selected *manifest.Manifest + schemaManifest := make(map[string]manifestHash) + for _, mfestPath := range manifestPaths { + err := func() error { + mfestFullPath := path.Join(targetDir, mfestPath) + f, err := os.Open(mfestFullPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err := manifest.Read(r) + if err != nil { + return err + } + err = manifestutil.Validate(mfest) + if err != nil { + return err + } + // Verify consistency with other manifests with the same schema. + h, err := contentHash(mfestFullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + refMfest, ok := schemaManifest[mfest.Schema()] + if !ok { + schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} + } else if refMfest.hash != mfestHash { + return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) + } + + if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { + selected = mfest + } + return nil + }() + if err != nil { + return nil, err + } + } + return selected, nil +} + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8..65b362d5 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -68,6 +68,10 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { return manifest, nil } +func (manifest *Manifest) Schema() string { + return manifest.db.Schema() +} + func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) }