From 491cd0c6df5dd77826fc320c3eb855947f1cd056 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 26 Nov 2025 11:01:08 +0100 Subject: [PATCH 1/9] feat: search, load and validates manifests Signed-off-by: Paul Mars --- cmd/chisel/cmd_cut.go | 16 +++ internal/manifestutil/manifestutil.go | 45 ++++++- internal/slicer/check.go | 184 ++++++++++++++++++++++++++ internal/slicer/slicer.go | 92 +++++++++++++ public/manifest/manifest.go | 4 + 5 files changed, 334 insertions(+), 7 deletions(-) create mode 100644 internal/slicer/check.go diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79..cca9055f 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/public/manifest" ) var shortCutHelp = "Cut a tree with selected slices" @@ -73,6 +74,21 @@ func (cmd *cmdCut) Execute(args []string) error { } } + mfest, err := slicer.Inspect(cmd.RootDir, release) + if err != nil { + return err + } + if mfest != nil { + mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + sliceKeys = append(sliceKeys, sk) + return nil + }) + } + selection, err := setup.Select(release, sliceKeys, cmd.Arch) if err != nil { return err diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b05402..f6bb7e60 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -18,22 +18,44 @@ import ( const DefaultFilename = "manifest.wall" +func collectManifests(slice *setup.Slice, collector func(path string, slice *setup.Slice)) { + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + collector(path, slice) + } + } +} + // FindPaths finds the paths marked with "generate:manifest" and // returns a map from the manifest path to all the slices that declare it. func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { manifestSlices := make(map[string][]*setup.Slice) + collector := func(path string, slice *setup.Slice) { + manifestSlices[path] = append(manifestSlices[path], slice) + } for _, slice := range slices { - for path, info := range slice.Contents { - if info.Generate == setup.GenerateManifest { - dir := strings.TrimSuffix(path, "**") - path = filepath.Join(dir, DefaultFilename) - manifestSlices[path] = append(manifestSlices[path], slice) - } - } + collectManifests(slice, collector) } return manifestSlices } +// FindPathsInRelease finds all the paths marked with "generate:manifest" +// for the given release. +func FindPathsInRelease(r *setup.Release) []string { + manifestPaths := make([]string,0) + collector := func(path string, slice *setup.Slice) { + manifestPaths = append(manifestPaths, path) + } + for _, pkg := range r.Packages { + for _, slice := range pkg.Slices { + collectManifests(slice, collector) + } + } + return manifestPaths +} + type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice @@ -340,3 +362,12 @@ func Validate(mfest *manifest.Manifest) (err error) { } return nil } + +// CompareSchemas compares two manifest schema strings. +func CompareSchemas(va, vb string) int { + if va == manifest.Schema && va == vb { + return 0 + } + return -1 +} + diff --git a/internal/slicer/check.go b/internal/slicer/check.go new file mode 100644 index 00000000..ccf97605 --- /dev/null +++ b/internal/slicer/check.go @@ -0,0 +1,184 @@ +package slicer + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" + + "github.com/klauspost/compress/zstd" + + "github.com/canonical/chisel/internal/manifestutil" + "github.com/canonical/chisel/public/manifest" +) + +type pathInfo struct { + mode string + size int64 + link string + hash string +} + +func unixPerm(mode fs.FileMode) (perm uint32) { + perm = uint32(mode.Perm()) + if mode&fs.ModeSticky != 0 { + perm |= 0o1000 + } + return perm +} + +// checkRootDir checks the content of the target directory matches with +// the manifest. Files not managed by chisel are ignored. +// This function works under the assumption the manifest is valid. +func checkRootDir(mfest *manifest.Manifest, rootDir string) error { + singlePathsByFSInode := make(map[uint64]string) + fsInodeByManifestInode := make(map[uint64]uint64) + manifestInfos := make(map[string]*pathInfo) + err := mfest.IteratePaths("", func(path *manifest.Path) error { + pathHash := path.FinalSHA256 + if pathHash == "" { + pathHash = path.SHA256 + } + recordedPathInfo := &pathInfo{ + mode: path.Mode, + size: int64(path.Size), + link: path.Link, + hash: pathHash, + } + + fsInfo := &pathInfo{} + fullPath := filepath.Join(rootDir, path.Path) + info, err := os.Lstat(fullPath) + if err != nil { + return err + } + mode := info.Mode() + fsInfo.mode = fmt.Sprintf("0%o", unixPerm(mode)) + ftype := mode & fs.ModeType + switch ftype { + case fs.ModeDir: + // Nothing to do. + case fs.ModeSymlink: + fsInfo.link, err = os.Readlink(fullPath) + if err != nil { + return fmt.Errorf("cannot read symlink %q: %w", fullPath, err) + } + case 0: // Regular file. + h, err := contentHash(fullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", fullPath, err) + } + fsInfo.hash = hex.EncodeToString(h) + fsInfo.size = info.Size() + default: + return fmt.Errorf("cannot check %q: unrecognized type %s", fullPath, mode.String()) + } + + // Collect manifests for tailored checking later. Adjust observed hash and + // size to still compare in a generic way. + if filepath.Base(path.Path) == manifestutil.DefaultFilename && recordedPathInfo.size == 0 && recordedPathInfo.hash == "" { + mfestInfo := *fsInfo + manifestInfos[path.Path] = &mfestInfo + fsInfo.size = 0 + fsInfo.hash = "" + } + + if recordedPathInfo.mode != fsInfo.mode { + return fmt.Errorf("inconsistent mode at %q: recorded %v, observed %v", path.Path, recordedPathInfo.mode, fsInfo.mode) + } + if recordedPathInfo.size != fsInfo.size { + return fmt.Errorf("inconsistent size at %q: recorded %v, observed %v", path.Path, recordedPathInfo.size, fsInfo.size) + } + if recordedPathInfo.link != fsInfo.link { + return fmt.Errorf("inconsistent link at %q: recorded %v, observed %v", path.Path, recordedPathInfo.link, fsInfo.link) + } + if recordedPathInfo.hash != fsInfo.hash { + return fmt.Errorf("inconsistent hash at %q: recorded %v, observed %v", path.Path, recordedPathInfo.hash, fsInfo.hash) + } + // Check hardlink. + if ftype != fs.ModeDir { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("cannot get syscall stat info for %q", info.Name()) + } + inode := stat.Ino + + if path.Inode == 0 { + // This path must not be linked to any other. + singlePath, ok := singlePathsByFSInode[inode] + if ok { + return fmt.Errorf("inconsistent content at %q: recorded no hardlink, observed hardlinked to %q", path.Path, singlePath) + } + singlePathsByFSInode[inode] = path.Path + } else { + recordedInode, ok := fsInodeByManifestInode[path.Inode] + if !ok { + fsInodeByManifestInode[path.Inode] = inode + } else if recordedInode != inode { + return fmt.Errorf("inconsistent content at %q: file hardlinked to a different inode", path.Path) + } + } + } + return nil + }) + if err != nil { + return err + } + + // Check manifests. + // They must all be valid manifests and be consistent per schema version. + schemaManifestInfos := make(map[string]*pathInfo) + for path, info := range manifestInfos { + fullPath := filepath.Join(rootDir, path) + f, err := os.Open(fullPath) + if err != nil { + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err = manifest.Read(r) + if err != nil { + return err + } + err = manifestutil.Validate(mfest) + if err != nil { + return err + } + schema := mfest.Schema() + refInfo, ok := schemaManifestInfos[schema] + if !ok { + schemaManifestInfos[schema] = info + continue + } + + if refInfo.size != info.size { + return fmt.Errorf("inconsistent manifest size for version %s at %q: recorded %v, observed %v", schema, path, refInfo.size, info.size) + } + if refInfo.hash != info.hash { + return fmt.Errorf("inconsistent manifest hash for version %s at %q: recorded %v, observed %v", schema, path, refInfo.hash, info.hash) + } + } + return nil +} + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb..48eb0e11 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,12 @@ package slicer import ( "archive/tar" "bytes" + "encoding/hex" "fmt" "io" "io/fs" "os" + "path" "path/filepath" "slices" "sort" @@ -21,6 +23,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/manifest" ) const manifestMode fs.FileMode = 0644 @@ -537,3 +540,92 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel } return pkgArchive, nil } + +// Inspect examines and validates the targetDir. It returns, if found and valid +// the manifest representing the content in the targetDir. +func Inspect(targetDir string, release *setup.Release) (*manifest.Manifest, error) { + var mfest *manifest.Manifest + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) > 0 { + logf("Inspecting root directory...") + var err error + mfest, err = selectValidManifest(targetDir, manifestPaths) + if err != nil { + return nil, err + } + if mfest != nil { + err = checkRootDir(mfest, targetDir) + if err != nil { + return nil, err + } + } + } + return mfest, nil +} + +// selectValidManifest returns, if found, a valid manifest with the latest +// schema. Consistency with all other manifests with the same schema is verified +// so the selection is deterministic. +func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Manifest, error) { + targetDir = filepath.Clean(targetDir) + if !filepath.IsAbs(targetDir) { + dir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("cannot obtain current directory: %w", err) + } + targetDir = filepath.Join(dir, targetDir) + } + + type manifestHash struct { + path string + hash string + } + var selected *manifest.Manifest + schemaManifest := make(map[string]manifestHash) + for _, mfestPath := range manifestPaths { + err := func() error { + mfestFullPath := path.Join(targetDir, mfestPath) + f, err := os.Open(mfestFullPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err := manifest.Read(r) + if err != nil { + return nil + } + err = manifestutil.Validate(mfest) + if err != nil { + return nil + } + + if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { + h, err := contentHash(mfestFullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + refMfest, ok := schemaManifest[mfest.Schema()] + if !ok { + schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} + } else if refMfest.hash != mfestHash { + return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) + } + selected = mfest + } + return nil + }() + if err != nil { + return nil, err + } + } + return selected, nil +} diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8..65b362d5 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -68,6 +68,10 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { return manifest, nil } +func (manifest *Manifest) Schema() string { + return manifest.db.Schema() +} + func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) } From a6e57f7290104bec2c17bfefc508d091badc971d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 29 Jan 2026 17:19:12 +0100 Subject: [PATCH 2/9] feat: adapt to revised strategy --- cmd/chisel/cmd_cut.go | 3 +- internal/fsutil/create.go | 16 +- internal/manifestutil/manifestutil.go | 3 +- internal/manifestutil/manifestutil_test.go | 1 - internal/slicer/check.go | 184 --------------------- internal/slicer/slicer.go | 123 +++++++++----- 6 files changed, 100 insertions(+), 230 deletions(-) delete mode 100644 internal/slicer/check.go diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index cca9055f..940ec121 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -74,7 +74,7 @@ func (cmd *cmdCut) Execute(args []string) error { } } - mfest, err := slicer.Inspect(cmd.RootDir, release) + mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) if err != nil { return err } @@ -141,6 +141,7 @@ func (cmd *cmdCut) Execute(args []string) error { Selection: selection, Archives: archives, TargetDir: cmd.RootDir, + Manifest: mfest, }) return err } diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 0503c96f..2f87f670 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -166,11 +166,19 @@ func createDir(o *CreateOptions) error { if err != nil { return err } - err = os.Mkdir(path, o.Mode) - if os.IsExist(err) { - return nil + fileinfo, err := os.Lstat(path) + if err == nil { + if fileinfo.IsDir() { + return nil + } + err = os.Remove(path) + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err } - return err + return os.Mkdir(path, o.Mode) } func createFile(o *CreateOptions) error { diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index f6bb7e60..b2946d57 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -44,7 +44,7 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { // FindPathsInRelease finds all the paths marked with "generate:manifest" // for the given release. func FindPathsInRelease(r *setup.Release) []string { - manifestPaths := make([]string,0) + manifestPaths := make([]string, 0) collector := func(path string, slice *setup.Slice) { manifestPaths = append(manifestPaths, path) } @@ -370,4 +370,3 @@ func CompareSchemas(va, vb string) int { } return -1 } - diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 2bab0a68..0c6c0c06 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -2,7 +2,6 @@ package manifestutil_test import ( "bytes" - "io" "io/fs" "os" "path" diff --git a/internal/slicer/check.go b/internal/slicer/check.go deleted file mode 100644 index ccf97605..00000000 --- a/internal/slicer/check.go +++ /dev/null @@ -1,184 +0,0 @@ -package slicer - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "syscall" - - "github.com/klauspost/compress/zstd" - - "github.com/canonical/chisel/internal/manifestutil" - "github.com/canonical/chisel/public/manifest" -) - -type pathInfo struct { - mode string - size int64 - link string - hash string -} - -func unixPerm(mode fs.FileMode) (perm uint32) { - perm = uint32(mode.Perm()) - if mode&fs.ModeSticky != 0 { - perm |= 0o1000 - } - return perm -} - -// checkRootDir checks the content of the target directory matches with -// the manifest. Files not managed by chisel are ignored. -// This function works under the assumption the manifest is valid. -func checkRootDir(mfest *manifest.Manifest, rootDir string) error { - singlePathsByFSInode := make(map[uint64]string) - fsInodeByManifestInode := make(map[uint64]uint64) - manifestInfos := make(map[string]*pathInfo) - err := mfest.IteratePaths("", func(path *manifest.Path) error { - pathHash := path.FinalSHA256 - if pathHash == "" { - pathHash = path.SHA256 - } - recordedPathInfo := &pathInfo{ - mode: path.Mode, - size: int64(path.Size), - link: path.Link, - hash: pathHash, - } - - fsInfo := &pathInfo{} - fullPath := filepath.Join(rootDir, path.Path) - info, err := os.Lstat(fullPath) - if err != nil { - return err - } - mode := info.Mode() - fsInfo.mode = fmt.Sprintf("0%o", unixPerm(mode)) - ftype := mode & fs.ModeType - switch ftype { - case fs.ModeDir: - // Nothing to do. - case fs.ModeSymlink: - fsInfo.link, err = os.Readlink(fullPath) - if err != nil { - return fmt.Errorf("cannot read symlink %q: %w", fullPath, err) - } - case 0: // Regular file. - h, err := contentHash(fullPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", fullPath, err) - } - fsInfo.hash = hex.EncodeToString(h) - fsInfo.size = info.Size() - default: - return fmt.Errorf("cannot check %q: unrecognized type %s", fullPath, mode.String()) - } - - // Collect manifests for tailored checking later. Adjust observed hash and - // size to still compare in a generic way. - if filepath.Base(path.Path) == manifestutil.DefaultFilename && recordedPathInfo.size == 0 && recordedPathInfo.hash == "" { - mfestInfo := *fsInfo - manifestInfos[path.Path] = &mfestInfo - fsInfo.size = 0 - fsInfo.hash = "" - } - - if recordedPathInfo.mode != fsInfo.mode { - return fmt.Errorf("inconsistent mode at %q: recorded %v, observed %v", path.Path, recordedPathInfo.mode, fsInfo.mode) - } - if recordedPathInfo.size != fsInfo.size { - return fmt.Errorf("inconsistent size at %q: recorded %v, observed %v", path.Path, recordedPathInfo.size, fsInfo.size) - } - if recordedPathInfo.link != fsInfo.link { - return fmt.Errorf("inconsistent link at %q: recorded %v, observed %v", path.Path, recordedPathInfo.link, fsInfo.link) - } - if recordedPathInfo.hash != fsInfo.hash { - return fmt.Errorf("inconsistent hash at %q: recorded %v, observed %v", path.Path, recordedPathInfo.hash, fsInfo.hash) - } - // Check hardlink. - if ftype != fs.ModeDir { - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("cannot get syscall stat info for %q", info.Name()) - } - inode := stat.Ino - - if path.Inode == 0 { - // This path must not be linked to any other. - singlePath, ok := singlePathsByFSInode[inode] - if ok { - return fmt.Errorf("inconsistent content at %q: recorded no hardlink, observed hardlinked to %q", path.Path, singlePath) - } - singlePathsByFSInode[inode] = path.Path - } else { - recordedInode, ok := fsInodeByManifestInode[path.Inode] - if !ok { - fsInodeByManifestInode[path.Inode] = inode - } else if recordedInode != inode { - return fmt.Errorf("inconsistent content at %q: file hardlinked to a different inode", path.Path) - } - } - } - return nil - }) - if err != nil { - return err - } - - // Check manifests. - // They must all be valid manifests and be consistent per schema version. - schemaManifestInfos := make(map[string]*pathInfo) - for path, info := range manifestInfos { - fullPath := filepath.Join(rootDir, path) - f, err := os.Open(fullPath) - if err != nil { - return err - } - defer f.Close() - r, err := zstd.NewReader(f) - if err != nil { - return err - } - defer r.Close() - mfest, err = manifest.Read(r) - if err != nil { - return err - } - err = manifestutil.Validate(mfest) - if err != nil { - return err - } - schema := mfest.Schema() - refInfo, ok := schemaManifestInfos[schema] - if !ok { - schemaManifestInfos[schema] = info - continue - } - - if refInfo.size != info.size { - return fmt.Errorf("inconsistent manifest size for version %s at %q: recorded %v, observed %v", schema, path, refInfo.size, info.size) - } - if refInfo.hash != info.hash { - return fmt.Errorf("inconsistent manifest hash for version %s at %q: recorded %v, observed %v", schema, path, refInfo.hash, info.hash) - } - } - return nil -} - -func contentHash(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return nil, err - } - return h.Sum(nil), nil -} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 48eb0e11..1dcfd019 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,12 @@ package slicer import ( "archive/tar" "bytes" + "crypto/sha256" "encoding/hex" "fmt" "io" "io/fs" + "maps" "os" "path" "path/filepath" @@ -32,6 +34,7 @@ type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive TargetDir string + Manifest *manifest.Manifest } type pathData struct { @@ -93,6 +96,19 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } + var originalTargetDir string + if options.Manifest != nil { + tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*") + if err != nil { + return fmt.Errorf("cannot create temporary working directory: %w", err) + } + originalTargetDir = targetDir + targetDir = tmpWorkDir + defer func() { + os.RemoveAll(tmpWorkDir) + }() + } + pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err @@ -350,7 +366,41 @@ func Run(options *RunOptions) error { return err } - return generateManifests(targetDir, options.Selection, report, pkgInfos) + err = generateManifests(targetDir, options.Selection, report, pkgInfos) + if err != nil { + return err + } + + if options.Manifest != nil { + return upgrade(originalTargetDir, targetDir, report, options.Manifest) + } + return nil +} + +// upgrade upgrades content in targetDir using content in tempDir. +// Work on sorted list of content in tempDir +func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { + paths := slices.Sorted(maps.Keys(newReport.Entries)) + for _, path := range paths { + entry := newReport.Entries[path] + var err error + switch entry.Mode & fs.ModeType { + case 0: + // rename file if hash different than same path in old manifest + case fs.ModeDir: + // create dir with proper mode + // or chmod existing dir + case fs.ModeSymlink: + // move symlink to dest + default: + err = fmt.Errorf("unsupported file type: %s", path) + } + if err != nil { + return err + } + } + + return nil } func generateManifests(targetDir string, selection *setup.Selection, @@ -541,32 +591,10 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return pkgArchive, nil } -// Inspect examines and validates the targetDir. It returns, if found and valid -// the manifest representing the content in the targetDir. -func Inspect(targetDir string, release *setup.Release) (*manifest.Manifest, error) { - var mfest *manifest.Manifest - manifestPaths := manifestutil.FindPathsInRelease(release) - if len(manifestPaths) > 0 { - logf("Inspecting root directory...") - var err error - mfest, err = selectValidManifest(targetDir, manifestPaths) - if err != nil { - return nil, err - } - if mfest != nil { - err = checkRootDir(mfest, targetDir) - if err != nil { - return nil, err - } - } - } - return mfest, nil -} - -// selectValidManifest returns, if found, a valid manifest with the latest +// SelectValidManifest returns, if found, a valid manifest with the latest // schema. Consistency with all other manifests with the same schema is verified // so the selection is deterministic. -func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Manifest, error) { +func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) { targetDir = filepath.Clean(targetDir) if !filepath.IsAbs(targetDir) { dir, err := os.Getwd() @@ -575,6 +603,10 @@ func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Ma } targetDir = filepath.Join(dir, targetDir) } + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) == 0 { + return nil, nil + } type manifestHash struct { path string @@ -600,25 +632,26 @@ func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Ma defer r.Close() mfest, err := manifest.Read(r) if err != nil { - return nil + return err } err = manifestutil.Validate(mfest) if err != nil { - return nil + return err + } + // Verify consistency with other manifests with the same schema. + h, err := contentHash(mfestFullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + refMfest, ok := schemaManifest[mfest.Schema()] + if !ok { + schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} + } else if refMfest.hash != mfestHash { + return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) } if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { - h, err := contentHash(mfestFullPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) - } - mfestHash := hex.EncodeToString(h) - refMfest, ok := schemaManifest[mfest.Schema()] - if !ok { - schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} - } else if refMfest.hash != mfestHash { - return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) - } selected = mfest } return nil @@ -629,3 +662,17 @@ func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Ma } return selected, nil } + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} From 5da04501bd541faf4a898c2ca8a0b8c564210ac8 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 14:26:15 +0100 Subject: [PATCH 3/9] feat: implement upgrade --- internal/fsutil/create.go | 3 ++ internal/slicer/slicer.go | 99 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 2f87f670..60bfc2b0 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -169,6 +169,9 @@ func createDir(o *CreateOptions) error { fileinfo, err := os.Lstat(path) if err == nil { if fileinfo.IsDir() { + if fileinfo.Mode() != o.Mode && o.OverrideMode { + return os.Chmod(path, o.Mode) + } return nil } err = os.Remove(path) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 1dcfd019..67949d10 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -377,21 +377,56 @@ func Run(options *RunOptions) error { return nil } +// absPath requires root to be a clean path that ends in "/". +func absPath(root, relPath string) (string, error) { + path := filepath.Clean(filepath.Join(root, relPath)) + if !strings.HasPrefix(path, root) { + return "", fmt.Errorf("cannot create path %s outside of root %s", path, root) + } + return path, nil +} + // upgrade upgrades content in targetDir using content in tempDir. -// Work on sorted list of content in tempDir func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { + logf("Upgrading existing content...") + filesToDelete := make([]*manifest.Path, 0) + oldPaths := make(map[string]*manifest.Path, 0) + err := oldManifest.IteratePaths("", func(path *manifest.Path) error { + _, ok := newReport.Entries[path.Path] + if !ok && strings.HasSuffix(path.Path, "/") { + // Keep directories. + filesToDelete = append(filesToDelete, path) + return nil + } + oldPaths[path.Path] = path + return nil + }) + if err != nil { + return err + } + paths := slices.Sorted(maps.Keys(newReport.Entries)) for _, path := range paths { + srcPath, err := absPath(tempDir, path) + if err != nil { + return err + } + dstPath, err := absPath(targetDir, path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + entry := newReport.Entries[path] - var err error switch entry.Mode & fs.ModeType { case 0: - // rename file if hash different than same path in old manifest + err = upgradeFile(srcPath, dstPath, &entry) case fs.ModeDir: - // create dir with proper mode - // or chmod existing dir + err = upgradeDir(dstPath, &entry) case fs.ModeSymlink: - // move symlink to dest + err = os.Rename(srcPath, dstPath) default: err = fmt.Errorf("unsupported file type: %s", path) } @@ -400,9 +435,61 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } } + // Delete old files + for _, pathToDelete := range filesToDelete { + path, err := absPath(tempDir, pathToDelete.Path) + if err != nil { + return err + } + err = os.Remove(path) + if err != nil { + return err + } + } return nil } +func upgradeFile(srcPath string, dstPath string, entry *manifestutil.ReportEntry) error { + fileinfo, err := os.Lstat(dstPath) + if err == nil { + h, err := contentHash(dstPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", dstPath, err) + } + oldHash := hex.EncodeToString(h) + newHash := entry.SHA256 + if newHash == "" { + newHash = entry.FinalSHA256 + } + if oldHash == newHash && entry.Mode == fileinfo.Mode() { + // Same file, do nothing. + return nil + } + } else if !os.IsNotExist(err) { + return err + } + return os.Rename(srcPath, dstPath) +} + +func upgradeDir(path string, entry *manifestutil.ReportEntry) error { + fileinfo, err := os.Lstat(path) + if err == nil { + if fileinfo.IsDir() { + if fileinfo.Mode() != entry.Mode { + return os.Chmod(path, entry.Mode) + } + return nil + } + err = os.Remove(path) + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + return os.Mkdir(path, entry.Mode) +} + func generateManifests(targetDir string, selection *setup.Selection, report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { manifestSlices := manifestutil.FindPaths(selection.Slices) From 5703751eff9926771d086ee7d43480e916848522 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 14:43:30 +0100 Subject: [PATCH 4/9] fix: deletion --- cmd/chisel/cmd_cut.go | 2 +- internal/slicer/slicer.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 940ec121..5873c3a3 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -141,7 +141,7 @@ func (cmd *cmdCut) Execute(args []string) error { Selection: selection, Archives: archives, TargetDir: cmd.RootDir, - Manifest: mfest, + Manifest: mfest, }) return err } diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 67949d10..aed6f668 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -393,7 +393,7 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o oldPaths := make(map[string]*manifest.Path, 0) err := oldManifest.IteratePaths("", func(path *manifest.Path) error { _, ok := newReport.Entries[path.Path] - if !ok && strings.HasSuffix(path.Path, "/") { + if !ok && !strings.HasSuffix(path.Path, "/") { // Keep directories. filesToDelete = append(filesToDelete, path) return nil @@ -437,7 +437,7 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o // Delete old files for _, pathToDelete := range filesToDelete { - path, err := absPath(tempDir, pathToDelete.Path) + path, err := absPath(targetDir, pathToDelete.Path) if err != nil { return err } From 26bb2b3361c8fbdb078438eeeb9e144815e6da67 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 14:49:19 +0100 Subject: [PATCH 5/9] fix: revert inadvertent change --- internal/manifestutil/manifestutil_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 0c6c0c06..2bab0a68 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -2,6 +2,7 @@ package manifestutil_test import ( "bytes" + "io" "io/fs" "os" "path" From 4a14230ed08272abf1a7e4badeca243f6bcae7cc Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 15:08:17 +0100 Subject: [PATCH 6/9] fix: check slice collecting error --- cmd/chisel/cmd_cut.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 5873c3a3..b0dd1eec 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -79,7 +79,7 @@ func (cmd *cmdCut) Execute(args []string) error { return err } if mfest != nil { - mfest.IterateSlices("", func(slice *manifest.Slice) error { + err = mfest.IterateSlices("", func(slice *manifest.Slice) error { sk, err := setup.ParseSliceKey(slice.Name) if err != nil { return err @@ -87,6 +87,9 @@ func (cmd *cmdCut) Execute(args []string) error { sliceKeys = append(sliceKeys, sk) return nil }) + if err != nil { + return err + } } selection, err := setup.Select(release, sliceKeys, cmd.Arch) From 865c1b8ae17a03e70df70b2ca7505e5de1053b15 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 17:01:00 +0100 Subject: [PATCH 7/9] fix: simplify upgrade and improve deletion --- internal/slicer/slicer.go | 55 ++++++++++++++------------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index aed6f668..5de5813d 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -389,13 +389,12 @@ func absPath(root, relPath string) (string, error) { // upgrade upgrades content in targetDir using content in tempDir. func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { logf("Upgrading existing content...") - filesToDelete := make([]*manifest.Path, 0) + pathsToDelete := make([]string, 0) oldPaths := make(map[string]*manifest.Path, 0) err := oldManifest.IteratePaths("", func(path *manifest.Path) error { _, ok := newReport.Entries[path.Path] - if !ok && !strings.HasSuffix(path.Path, "/") { - // Keep directories. - filesToDelete = append(filesToDelete, path) + if !ok { + pathsToDelete = append(pathsToDelete, path.Path) return nil } oldPaths[path.Path] = path @@ -422,11 +421,10 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o entry := newReport.Entries[path] switch entry.Mode & fs.ModeType { case 0: - err = upgradeFile(srcPath, dstPath, &entry) - case fs.ModeDir: - err = upgradeDir(dstPath, &entry) case fs.ModeSymlink: err = os.Rename(srcPath, dstPath) + case fs.ModeDir: + err = upgradeDir(dstPath, &entry) default: err = fmt.Errorf("unsupported file type: %s", path) } @@ -435,42 +433,29 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } } - // Delete old files - for _, pathToDelete := range filesToDelete { - path, err := absPath(targetDir, pathToDelete.Path) + // Delete old paths + slices.Sort(pathsToDelete) + slices.Reverse(pathsToDelete) + for _, pathToDelete := range pathsToDelete { + path, err := absPath(targetDir, pathToDelete) if err != nil { return err } - err = os.Remove(path) - if err != nil { - return err + if strings.HasSuffix(path, "/") { + err = syscall.Rmdir(path) + if err != nil && err != syscall.ENOTEMPTY { + return err + } + } else { + err = os.Remove(path) + if err != nil { + return err + } } } return nil } -func upgradeFile(srcPath string, dstPath string, entry *manifestutil.ReportEntry) error { - fileinfo, err := os.Lstat(dstPath) - if err == nil { - h, err := contentHash(dstPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", dstPath, err) - } - oldHash := hex.EncodeToString(h) - newHash := entry.SHA256 - if newHash == "" { - newHash = entry.FinalSHA256 - } - if oldHash == newHash && entry.Mode == fileinfo.Mode() { - // Same file, do nothing. - return nil - } - } else if !os.IsNotExist(err) { - return err - } - return os.Rename(srcPath, dstPath) -} - func upgradeDir(path string, entry *manifestutil.ReportEntry) error { fileinfo, err := os.Lstat(path) if err == nil { From 0343ddcfe0db4d58613e7223ea6adb726ee2e559 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 17:12:33 +0100 Subject: [PATCH 8/9] refactor: cleaning --- internal/slicer/slicer.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 5de5813d..7923d6cc 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -389,15 +389,12 @@ func absPath(root, relPath string) (string, error) { // upgrade upgrades content in targetDir using content in tempDir. func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { logf("Upgrading existing content...") - pathsToDelete := make([]string, 0) - oldPaths := make(map[string]*manifest.Path, 0) + missingPaths := make([]string, 0) err := oldManifest.IteratePaths("", func(path *manifest.Path) error { _, ok := newReport.Entries[path.Path] if !ok { - pathsToDelete = append(pathsToDelete, path.Path) - return nil + missingPaths = append(missingPaths, path.Path) } - oldPaths[path.Path] = path return nil }) if err != nil { @@ -433,11 +430,11 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } } - // Delete old paths - slices.Sort(pathsToDelete) - slices.Reverse(pathsToDelete) - for _, pathToDelete := range pathsToDelete { - path, err := absPath(targetDir, pathToDelete) + // Remove missing paths + slices.Sort(missingPaths) + slices.Reverse(missingPaths) + for _, relPath := range missingPaths { + path, err := absPath(targetDir, relPath) if err != nil { return err } From e11642485b9ffea7ee3bc008bfbaa43e40c4d0d1 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 2 Feb 2026 11:59:10 +0100 Subject: [PATCH 9/9] refactor: refine upgrade --- internal/slicer/slicer.go | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 7923d6cc..2e263fd5 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -386,22 +386,10 @@ func absPath(root, relPath string) (string, error) { return path, nil } -// upgrade upgrades content in targetDir using content in tempDir. -func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { - logf("Upgrading existing content...") - missingPaths := make([]string, 0) - err := oldManifest.IteratePaths("", func(path *manifest.Path) error { - _, ok := newReport.Entries[path.Path] - if !ok { - missingPaths = append(missingPaths, path.Path) - } - return nil - }) - if err != nil { - return err - } - - paths := slices.Sorted(maps.Keys(newReport.Entries)) +// upgrade upgrades content in targetDir with content in tempDir. +func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfest *manifest.Manifest) error { + logf("Upgrading content...") + paths := slices.Sorted(maps.Keys(report.Entries)) for _, path := range paths { srcPath, err := absPath(tempDir, path) if err != nil { @@ -415,7 +403,7 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o return err } - entry := newReport.Entries[path] + entry := report.Entries[path] switch entry.Mode & fs.ModeType { case 0: case fs.ModeSymlink: @@ -431,8 +419,18 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } // Remove missing paths - slices.Sort(missingPaths) - slices.Reverse(missingPaths) + missingPaths := make([]string, 0) + err := mfest.IteratePaths("", func(path *manifest.Path) error { + _, ok := report.Entries[path.Path] + if !ok { + missingPaths = append(missingPaths, path.Path) + } + return nil + }) + if err != nil { + return err + } + sort.Sort(sort.Reverse(sort.StringSlice(missingPaths))) for _, relPath := range missingPaths { path, err := absPath(targetDir, relPath) if err != nil {