From 1a05836d25cc322e538d2b73549b9a6dcd6d149c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 3 Feb 2026 11:16:56 +0100 Subject: [PATCH] worker: remove automatic migration of legacy v1 cache This removes the MigrateV2 function, which was added in BuildKit v0.7.0 (Docker v20.10.0) in 31a9aeea88d8b1ccff0c9ec433b3f221f91fcc65. That was in 2019, which is now over 6 Years ago, so it's very unlikely for old files to be still present. Removing this code would impact users migrating from Docker 19.03 or older, which are versions that reached EOL many years ago, so very unlikely. Signed-off-by: Sebastiaan van Stijn --- cache/metadata/metadata.go | 14 ++ cache/migrate_v2.go | 267 -------------------------------- worker/containerd/containerd.go | 16 +- worker/runc/runc.go | 17 +- 4 files changed, 17 insertions(+), 297 deletions(-) delete mode 100644 cache/migrate_v2.go diff --git a/cache/metadata/metadata.go b/cache/metadata/metadata.go index 5ccf182e295d..7eb9c4aad6d2 100644 --- a/cache/metadata/metadata.go +++ b/cache/metadata/metadata.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "encoding/json" + "os" + "path/filepath" "strings" "sync" @@ -27,6 +29,18 @@ type Store struct { } func NewStore(dbPath string) (*Store, error) { + // Check for legacy (v1) cache state. + // + // Automatic migration was removed in https://github.com/moby/buildkit/pull/6509 + if _, err := os.Stat(dbPath); errors.Is(err, os.ErrNotExist) { + legacyMetadata := filepath.Join(filepath.Dir(dbPath), "metadata.db") + if _, err := os.Stat(legacyMetadata); err == nil { + return nil, errors.Errorf( + "legacy (v1) cache metadata found at %q and needs to be removed or migrated; downgrade BuildKit to v0.27.1 to perform automatic migration or remove the existing cache", + legacyMetadata, + ) + } + } db, err := boltutil.Open(dbPath, 0600, nil) if err != nil { return nil, errors.Wrapf(err, "failed to open database file %s", dbPath) diff --git a/cache/migrate_v2.go b/cache/migrate_v2.go deleted file mode 100644 index 67aed2d5eb68..000000000000 --- a/cache/migrate_v2.go +++ /dev/null @@ -1,267 +0,0 @@ -package cache - -import ( - "context" - "io" - "os" - "time" - - "github.com/containerd/containerd/v2/core/content" - "github.com/containerd/containerd/v2/core/images" - "github.com/containerd/containerd/v2/core/leases" - "github.com/containerd/containerd/v2/core/snapshots" - cerrdefs "github.com/containerd/errdefs" - "github.com/moby/buildkit/cache/metadata" - "github.com/moby/buildkit/snapshot" - "github.com/moby/buildkit/util/bklog" - digest "github.com/opencontainers/go-digest" - "github.com/pkg/errors" -) - -func migrateChainID(si *metadata.StorageItem, all map[string]*metadata.StorageItem) (digest.Digest, digest.Digest, error) { - md := &cacheMetadata{si} - diffID := md.getDiffID() - if diffID == "" { - return "", "", nil - } - blobID := md.getBlob() - if blobID == "" { - return "", "", nil - } - chainID := md.getChainID() - blobChainID := md.getBlobChainID() - - if chainID != "" && blobChainID != "" { - return chainID, blobChainID, nil - } - - chainID = diffID - blobChainID = digest.FromBytes([]byte(blobID + " " + diffID)) - - parent := md.getParent() - if parent != "" { - pChainID, pBlobChainID, err := migrateChainID(all[parent], all) - if err != nil { - return "", "", err - } - chainID = digest.FromBytes([]byte(pChainID + " " + chainID)) - blobChainID = digest.FromBytes([]byte(pBlobChainID + " " + blobChainID)) - } - - md.queueChainID(chainID) - md.queueBlobChainID(blobChainID) - - return chainID, blobChainID, md.commitMetadata() -} - -func MigrateV2(ctx context.Context, from, to string, cs content.Store, s snapshot.Snapshotter, lm leases.Manager) error { - _, err := os.Stat(to) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return errors.WithStack(err) - } - } else { - return nil - } - - _, err = os.Stat(from) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return errors.WithStack(err) - } - return nil - } - tmpPath := to + ".tmp" - tmpFile, err := os.Create(tmpPath) - if err != nil { - return errors.WithStack(err) - } - src, err := os.Open(from) - if err != nil { - tmpFile.Close() - return errors.WithStack(err) - } - if _, err = io.Copy(tmpFile, src); err != nil { - tmpFile.Close() - src.Close() - return errors.Wrapf(err, "failed to copy db for migration") - } - src.Close() - tmpFile.Close() - - md, err := metadata.NewStore(tmpPath) - if err != nil { - return err - } - - items, err := md.All() - if err != nil { - return err - } - - byID := map[string]*metadata.StorageItem{} - for _, item := range items { - byID[item.ID()] = item - } - - // add committed, parent, snapshot - for id, item := range byID { - md := &cacheMetadata{item} - em := md.getEqualMutable() - if em == "" { - info, err := s.Stat(ctx, id) - if err != nil { - return err - } - if info.Kind == snapshots.KindCommitted { - md.queueCommitted(true) - } - if info.Parent != "" { - md.queueParent(info.Parent) - } - } else { - md.queueCommitted(true) - } - md.queueSnapshotID(id) - md.commitMetadata() - } - - for _, item := range byID { - md := &cacheMetadata{item} - em := md.getEqualMutable() - if em != "" { - if md.getParent() == "" { - emMd := &cacheMetadata{byID[em]} - md.queueParent(emMd.getParent()) - md.commitMetadata() - } - } - } - - type diffPair struct { - Blobsum string - DiffID string - } - // move diffID, blobsum to new location - for _, item := range byID { - v := item.Get("blobmapping.blob") - if v == nil { - continue - } - var blob diffPair - if err := v.Unmarshal(&blob); err != nil { - return errors.WithStack(err) - } - if _, err := cs.Info(ctx, digest.Digest(blob.Blobsum)); err != nil { - continue - } - md := &cacheMetadata{item} - md.queueDiffID(digest.Digest(blob.DiffID)) - md.queueBlob(digest.Digest(blob.Blobsum)) - md.queueMediaType(images.MediaTypeDockerSchema2LayerGzip) - if err := md.commitMetadata(); err != nil { - return err - } - } - - // calculate new chainid/blobsumid - for _, item := range byID { - if _, _, err := migrateChainID(item, byID); err != nil { - return err - } - } - - ctx = context.TODO() // no cancellation allowed pass this point - - // add new leases - for _, item := range byID { - md := &cacheMetadata{item} - l, err := lm.Create(ctx, func(l *leases.Lease) error { - l.ID = item.ID() - l.Labels = map[string]string{ - "containerd.io/gc.flat": time.Now().UTC().Format(time.RFC3339Nano), - } - return nil - }) - if err != nil { - // if we are running the migration twice - if errors.Is(err, cerrdefs.ErrAlreadyExists) { - continue - } - return errors.Wrap(err, "failed to create lease") - } - - if err := lm.AddResource(ctx, l, leases.Resource{ - ID: md.getSnapshotID(), - Type: "snapshots/" + s.Name(), - }); err != nil { - return errors.Wrapf(err, "failed to add snapshot %s to lease", item.ID()) - } - - if blobID := md.getBlob(); blobID != "" { - if err := lm.AddResource(ctx, l, leases.Resource{ - ID: string(blobID), - Type: "content", - }); err != nil { - return errors.Wrapf(err, "failed to add blob %s to lease", item.ID()) - } - } - } - - // remove old root labels - for _, item := range byID { - md := &cacheMetadata{item} - em := md.getEqualMutable() - if em == "" { - if _, err := s.Update(ctx, snapshots.Info{ - Name: md.getSnapshotID(), - }, "labels.containerd.io/gc.root"); err != nil { - if !errors.Is(err, cerrdefs.ErrNotFound) { - return err - } - } - - if blob := md.getBlob(); blob != "" { - if _, err := cs.Update(ctx, content.Info{ - Digest: blob, - }, "labels.containerd.io/gc.root"); err != nil { - return err - } - } - } - } - - // previous implementation can leak views, just clean up all views - err = s.Walk(ctx, func(ctx context.Context, info snapshots.Info) error { - if info.Kind == snapshots.KindView { - if _, err := s.Update(ctx, snapshots.Info{ - Name: info.Name, - }, "labels.containerd.io/gc.root"); err != nil { - if !errors.Is(err, cerrdefs.ErrNotFound) { - return err - } - } - } - return nil - }) - if err != nil { - return err - } - - // switch to new DB - if err := md.Close(); err != nil { - return err - } - - if err := os.Rename(tmpPath, to); err != nil { - return err - } - - for _, item := range byID { - md := &cacheMetadata{item} - bklog.G(ctx).Infof("migrated %s parent:%q snapshot:%v blob:%v diffid:%v chainID:%v blobChainID:%v", - item.ID(), md.getParent(), md.getSnapshotID(), md.getBlob(), md.getDiffID(), md.getChainID(), md.getBlobChainID()) - } - - return nil -} diff --git a/worker/containerd/containerd.go b/worker/containerd/containerd.go index 0d34fe9b4214..6da1c0310579 100644 --- a/worker/containerd/containerd.go +++ b/worker/containerd/containerd.go @@ -12,7 +12,6 @@ import ( "github.com/containerd/containerd/v2/core/leases" "github.com/containerd/containerd/v2/pkg/gc" "github.com/containerd/platforms" - "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/executor/containerdexecutor" "github.com/moby/buildkit/executor/oci" @@ -135,19 +134,6 @@ func newContainerd(client *ctd.Client, workerOpts WorkerOptions) (base.WorkerOpt } } - snap := containerdsnapshot.NewSnapshotter(workerOpts.SnapshotterName, client.SnapshotService(workerOpts.SnapshotterName), workerOpts.Namespace, nil) - - if err := cache.MigrateV2( - context.TODO(), - filepath.Join(root, "metadata.db"), - filepath.Join(root, "metadata_v2.db"), - cs, - snap, - lm, - ); err != nil { - return base.WorkerOpt{}, err - } - md, err := metadata.NewStore(filepath.Join(root, "metadata_v2.db")) if err != nil { return base.WorkerOpt{}, err @@ -174,7 +160,7 @@ func newContainerd(client *ctd.Client, workerOpts WorkerOptions) (base.WorkerOpt MetadataStore: md, NetworkProviders: np, Executor: containerdexecutor.New(executorOpts), - Snapshotter: snap, + Snapshotter: containerdsnapshot.NewSnapshotter(workerOpts.SnapshotterName, client.SnapshotService(workerOpts.SnapshotterName), workerOpts.Namespace, nil), ContentStore: cs, Applier: winlayers.NewFileSystemApplierWithWindows(cs, df), Differ: winlayers.NewWalkingDiffWithWindows(cs, df), diff --git a/worker/runc/runc.go b/worker/runc/runc.go index ededdc3cc5d3..4b169b10ba52 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -15,7 +15,6 @@ import ( "github.com/containerd/containerd/v2/plugins/content/local" "github.com/containerd/containerd/v2/plugins/diff/walking" "github.com/containerd/platforms" - "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/executor/oci" "github.com/moby/buildkit/executor/resources" @@ -130,18 +129,6 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc } maps.Copy(xlabels, labels) - lm := leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit") - snap := containerdsnapshot.NewSnapshotter(snFactory.Name, mdb.Snapshotter(snFactory.Name), "buildkit", idmap) - if err := cache.MigrateV2( - context.TODO(), - filepath.Join(root, "metadata.db"), - filepath.Join(root, "metadata_v2.db"), - c, - snap, - lm, - ); err != nil { - return opt, err - } md, err := metadata.NewStore(filepath.Join(root, "metadata_v2.db")) if err != nil { @@ -155,14 +142,14 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc MetadataStore: md, NetworkProviders: np, Executor: exe, - Snapshotter: snap, + Snapshotter: containerdsnapshot.NewSnapshotter(snFactory.Name, mdb.Snapshotter(snFactory.Name), "buildkit", idmap), ContentStore: c, Applier: winlayers.NewFileSystemApplierWithWindows(c, apply.NewFileSystemApplier(c)), Differ: winlayers.NewWalkingDiffWithWindows(c, walking.NewWalkingDiff(c)), ImageStore: nil, // explicitly Platforms: []ocispecs.Platform{platforms.Normalize(platforms.DefaultSpec())}, IdentityMapping: idmap, - LeaseManager: lm, + LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"), GarbageCollect: mdb.GarbageCollect, ParallelismSem: parallelismSem, MountPoolRoot: filepath.Join(root, "cachemounts"),