Skip to content

feat: update existing filesystem #264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
upils wants to merge 8 commits into
base: main
Choose a base branch
from
+285 −12
Draft

feat: update existing filesystem #264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
491cd0c
feat: search, load and validates manifests
upils Nov 26, 2025
a6e57f7
feat: adapt to revised strategy
upils Jan 29, 2026
5da0450
feat: implement upgrade
upils Jan 30, 2026
5703751
fix: deletion
upils Jan 30, 2026
26bb2b3
fix: revert inadvertent change
upils Jan 30, 2026
4a14230
fix: check slice collecting error
upils Jan 30, 2026
865c1b8
fix: simplify upgrade and improve deletion
upils Jan 30, 2026
0343ddc
refactor: cleaning
upils Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions cmd/chisel/cmd_cut.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/canonical/chisel/internal/cache"
"github.com/canonical/chisel/internal/setup"
"github.com/canonical/chisel/internal/slicer"
"github.com/canonical/chisel/public/manifest"
)

var shortCutHelp = "Cut a tree with selected slices"
Expand Down Expand Up @@ -73,6 +74,24 @@ func (cmd *cmdCut) Execute(args []string) error {
}
}

mfest, err := slicer.SelectValidManifest(cmd.RootDir, release)
if err != nil {
return err
}
if mfest != nil {
err = mfest.IterateSlices("", func(slice *manifest.Slice) error {
sk, err := setup.ParseSliceKey(slice.Name)
if err != nil {
return err
}
sliceKeys = append(sliceKeys, sk)
return nil
})
if err != nil {
return err
}
}

selection, err := setup.Select(release, sliceKeys, cmd.Arch)
if err != nil {
return err
Expand Down Expand Up @@ -125,6 +144,7 @@ func (cmd *cmdCut) Execute(args []string) error {
Selection: selection,
Archives: archives,
TargetDir: cmd.RootDir,
Manifest: mfest,
})
return err
}
19 changes: 15 additions & 4 deletions internal/fsutil/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,11 +166,22 @@ func createDir(o *CreateOptions) error {
if err != nil {
return err
}
err = os.Mkdir(path, o.Mode)
if os.IsExist(err) {
return nil
fileinfo, err := os.Lstat(path)
if err == nil {
if fileinfo.IsDir() {
if fileinfo.Mode() != o.Mode && o.OverrideMode {
return os.Chmod(path, o.Mode)
}
return nil
}
err = os.Remove(path)
if err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
return err
return os.Mkdir(path, o.Mode)
}

func createFile(o *CreateOptions) error {
Expand Down
44 changes: 37 additions & 7 deletions internal/manifestutil/manifestutil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,44 @@ import (

const DefaultFilename = "manifest.wall"

func collectManifests(slice *setup.Slice, collector func(path string, slice *setup.Slice)) {
for path, info := range slice.Contents {
if info.Generate == setup.GenerateManifest {
dir := strings.TrimSuffix(path, "**")
path = filepath.Join(dir, DefaultFilename)
collector(path, slice)
}
}
}

// FindPaths finds the paths marked with "generate:manifest" and
// returns a map from the manifest path to all the slices that declare it.
func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice {
manifestSlices := make(map[string][]*setup.Slice)
collector := func(path string, slice *setup.Slice) {
manifestSlices[path] = append(manifestSlices[path], slice)
}
for _, slice := range slices {
for path, info := range slice.Contents {
if info.Generate == setup.GenerateManifest {
dir := strings.TrimSuffix(path, "**")
path = filepath.Join(dir, DefaultFilename)
manifestSlices[path] = append(manifestSlices[path], slice)
}
}
collectManifests(slice, collector)
}
return manifestSlices
}

// FindPathsInRelease finds all the paths marked with "generate:manifest"
// for the given release.
func FindPathsInRelease(r *setup.Release) []string {
manifestPaths := make([]string, 0)
collector := func(path string, slice *setup.Slice) {
manifestPaths = append(manifestPaths, path)
}
for _, pkg := range r.Packages {
for _, slice := range pkg.Slices {
collectManifests(slice, collector)
}
}
return manifestPaths
}

type WriteOptions struct {
PackageInfo []*archive.PackageInfo
Selection []*setup.Slice
Expand Down Expand Up @@ -340,3 +362,11 @@ func Validate(mfest *manifest.Manifest) (err error) {
}
return nil
}

// CompareSchemas compares two manifest schema strings.
func CompareSchemas(va, vb string) int {
if va == manifest.Schema && va == vb {
return 0
}
return -1
}
210 changes: 209 additions & 1 deletion internal/slicer/slicer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ package slicer
import (
"archive/tar"
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"io/fs"
"maps"
"os"
"path"
"path/filepath"
"slices"
"sort"
Expand All @@ -21,6 +25,7 @@ import (
"github.com/canonical/chisel/internal/manifestutil"
"github.com/canonical/chisel/internal/scripts"
"github.com/canonical/chisel/internal/setup"
"github.com/canonical/chisel/public/manifest"
)

const manifestMode fs.FileMode = 0644
Expand All @@ -29,6 +34,7 @@ type RunOptions struct {
Selection *setup.Selection
Archives map[string]archive.Archive
TargetDir string
Manifest *manifest.Manifest
}

type pathData struct {
Expand Down Expand Up @@ -90,6 +96,19 @@ func Run(options *RunOptions) error {
targetDir = filepath.Join(dir, targetDir)
}

var originalTargetDir string
if options.Manifest != nil {
tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*")
if err != nil {
return fmt.Errorf("cannot create temporary working directory: %w", err)
}
originalTargetDir = targetDir
targetDir = tmpWorkDir
defer func() {
os.RemoveAll(tmpWorkDir)
}()
}

pkgArchive, err := selectPkgArchives(options.Archives, options.Selection)
if err != nil {
return err
Expand Down Expand Up @@ -347,7 +366,110 @@ func Run(options *RunOptions) error {
return err
}

return generateManifests(targetDir, options.Selection, report, pkgInfos)
err = generateManifests(targetDir, options.Selection, report, pkgInfos)
if err != nil {
return err
}

if options.Manifest != nil {
return upgrade(originalTargetDir, targetDir, report, options.Manifest)
}
return nil
}

// absPath requires root to be a clean path that ends in "/".
func absPath(root, relPath string) (string, error) {
path := filepath.Clean(filepath.Join(root, relPath))
if !strings.HasPrefix(path, root) {
return "", fmt.Errorf("cannot create path %s outside of root %s", path, root)
}
return path, nil
}

// upgrade upgrades content in targetDir using content in tempDir.
func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error {
logf("Upgrading existing content...")
missingPaths := make([]string, 0)
err := oldManifest.IteratePaths("", func(path *manifest.Path) error {
_, ok := newReport.Entries[path.Path]
if !ok {
missingPaths = append(missingPaths, path.Path)
}
return nil
})
if err != nil {
return err
}

paths := slices.Sorted(maps.Keys(newReport.Entries))
for _, path := range paths {
srcPath, err := absPath(tempDir, path)
if err != nil {
return err
}
dstPath, err := absPath(targetDir, path)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil {
return err
}

entry := newReport.Entries[path]
switch entry.Mode & fs.ModeType {
case 0:
case fs.ModeSymlink:
err = os.Rename(srcPath, dstPath)
case fs.ModeDir:
err = upgradeDir(dstPath, &entry)
default:
err = fmt.Errorf("unsupported file type: %s", path)
}
if err != nil {
return err
}
}

// Remove missing paths
slices.Sort(missingPaths)
slices.Reverse(missingPaths)
for _, relPath := range missingPaths {
path, err := absPath(targetDir, relPath)
if err != nil {
return err
}
if strings.HasSuffix(path, "/") {
err = syscall.Rmdir(path)
if err != nil && err != syscall.ENOTEMPTY {
return err
}
} else {
err = os.Remove(path)
if err != nil {
return err
}
}
}
return nil
}

func upgradeDir(path string, entry *manifestutil.ReportEntry) error {
fileinfo, err := os.Lstat(path)
if err == nil {
if fileinfo.IsDir() {
if fileinfo.Mode() != entry.Mode {
return os.Chmod(path, entry.Mode)
}
return nil
}
err = os.Remove(path)
if err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
return os.Mkdir(path, entry.Mode)
}

func generateManifests(targetDir string, selection *setup.Selection,
Expand Down Expand Up @@ -537,3 +659,89 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel
}
return pkgArchive, nil
}

// SelectValidManifest returns, if found, a valid manifest with the latest
// schema. Consistency with all other manifests with the same schema is verified
// so the selection is deterministic.
func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) {
targetDir = filepath.Clean(targetDir)
if !filepath.IsAbs(targetDir) {
dir, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("cannot obtain current directory: %w", err)
}
targetDir = filepath.Join(dir, targetDir)
}
manifestPaths := manifestutil.FindPathsInRelease(release)
if len(manifestPaths) == 0 {
return nil, nil
}

type manifestHash struct {
path string
hash string
}
var selected *manifest.Manifest
schemaManifest := make(map[string]manifestHash)
for _, mfestPath := range manifestPaths {
err := func() error {
mfestFullPath := path.Join(targetDir, mfestPath)
f, err := os.Open(mfestFullPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
defer f.Close()
r, err := zstd.NewReader(f)
if err != nil {
return err
}
defer r.Close()
mfest, err := manifest.Read(r)
if err != nil {
return err
}
err = manifestutil.Validate(mfest)
if err != nil {
return err
}
// Verify consistency with other manifests with the same schema.
h, err := contentHash(mfestFullPath)
if err != nil {
return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err)
}
mfestHash := hex.EncodeToString(h)
refMfest, ok := schemaManifest[mfest.Schema()]
if !ok {
schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash}
} else if refMfest.hash != mfestHash {
return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath)
}

if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 {
selected = mfest
}
return nil
}()
if err != nil {
return nil, err
}
}
return selected, nil
}

func contentHash(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()

h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return nil, err
}
return h.Sum(nil), nil
}
4 changes: 4 additions & 0 deletions public/manifest/manifest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ func Read(reader io.Reader) (manifest *Manifest, err error) {
return manifest, nil
}

func (manifest *Manifest) Schema() string {
return manifest.db.Schema()
}

func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) {
return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch)
}
Expand Down
Loading