Depot v0.12

api/services/control/control.proto

@@ -35,7 +35,7 @@ message PruneRequest {
 }
 
 message DiskUsageRequest {
-	repeated string filter = 1; 
+	repeated string filter = 1;
 }
 
 message DiskUsageResponse {
@@ -55,6 +55,8 @@ message UsageRecord {
 	string RecordType = 10;
 	bool Shared = 11;
 	repeated string Parents = 12;
+	repeated string stable_digests = 9998;
+	string creator_digest = 9999;
 }
 
 message SolveRequest {
@@ -124,6 +126,11 @@ message Vertex {
 	google.protobuf.Timestamp completed = 6 [(gogoproto.stdtime) = true ];
 	string error = 7; // typed errors?
 	pb.ProgressGroup progressGroup = 8;
+
+	string stableDigest = 9999 [
+		(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest",
+		(gogoproto.nullable) = false
+	];
 }
 
 message VertexStatus {

buf.gen.yaml

@@ -0,0 +1,17 @@
+version: v1
+
+managed:
+  enabled: true
+  go_package_prefix:
+    default: github.com/moby/buildkit/depot/api
+    except:
+      - buf.build/googleapis/googleapis
+      - buf.build/depot/buildkit
+
+plugins:
+  - plugin: buf.build/protocolbuffers/go:v1.31.0
+    out: depot/api
+    opt: paths=source_relative
+  - plugin: buf.build/connectrpc/go:v1.11.1
+    out: depot/api
+    opt: paths=source_relative

buf.work.yaml

@@ -0,0 +1,3 @@
+version: v1
+directories:
+  - depot/api

cache/blobs_linux.go

@@ -11,10 +11,10 @@ import (
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/mount"
+	"github.com/moby/buildkit/depot"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/compression"
 	"github.com/moby/buildkit/util/overlay"
-	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -53,7 +53,7 @@ func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper
 	bufW := bufio.NewWriterSize(cw, 128*1024)
 	var labels map[string]string
 	if compressorFunc != nil {
-		dgstr := digest.SHA256.Digester()
+		dgstr := depot.NewFastDigester()
 		compressed, err := compressorFunc(bufW, mediaType)
 		if err != nil {
 			return emptyDesc, false, errors.Wrap(err, "failed to get compressed stream")

cache/compression_nydus.go

@@ -4,10 +4,12 @@
 package cache
 
 import (
-	"compress/gzip"
 	"context"
 	"io"
 
+	// DEPOT: Using parallel gzip for faster image layer compression
+	gzip "github.com/klauspost/pgzip"
+
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/moby/buildkit/cache/config"

cache/contenthash/checksum.go

@@ -3,7 +3,6 @@ package contenthash
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"io"
 	"os"
 	"path"
@@ -14,6 +13,7 @@ import (
 	iradix "github.com/hashicorp/go-immutable-radix"
 	"github.com/hashicorp/golang-lru/simplelru"
 	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/depot"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/locker"
@@ -434,7 +434,7 @@ func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable,
 		return includedPaths[0].record.Digest, nil
 	}
 
-	digester := digest.Canonical.Digester()
+	digester := depot.NewFastDigester()
 	for i, w := range includedPaths {
 		if i != 0 {
 			digester.Hash().Write([]byte{0})
@@ -906,7 +906,7 @@ func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *ir
 
 	switch cr.Type {
 	case CacheRecordTypeDir:
-		h := sha256.New()
+		digester := depot.NewFastDigester()
 		next := append(k, 0)
 		iter := root.Iterator()
 		iter.SeekLowerBound(append(append([]byte{}, next...), 0))
@@ -916,14 +916,14 @@ func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *ir
 			if !ok || !bytes.HasPrefix(subk, next) {
 				break
 			}
-			h.Write(bytes.TrimPrefix(subk, k))
+			digester.Hash().Write(bytes.TrimPrefix(subk, k))
 
 			subcr, _, err := cc.checksum(ctx, root, txn, m, subk, true)
 			if err != nil {
 				return nil, false, err
 			}
 
-			h.Write([]byte(subcr.Digest))
+			digester.Hash().Write([]byte(subcr.Digest))
 
 			if subcr.Type == CacheRecordTypeDir { // skip subfiles
 				next := append(subk, 0, 0xff)
@@ -932,7 +932,7 @@ func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *ir
 			}
 			subk, _, ok = iter.Next()
 		}
-		dgst = digest.NewDigest(digest.SHA256, h)
+		dgst = digester.Digest()
 
 	default:
 		p := string(convertKeyToPath(bytes.TrimSuffix(k, []byte{0})))

cache/contenthash/filehash.go

@@ -2,12 +2,12 @@ package contenthash
 
 import (
 	"archive/tar"
-	"crypto/sha256"
 	"hash"
 	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/minio/sha256-simd"
 	fstypes "github.com/tonistiigi/fsutil/types"
 )
 

cache/manager.go

@@ -1294,18 +1294,20 @@ func (cm *cacheManager) markShared(m map[string]*cacheUsageInfo) error {
 }
 
 type cacheUsageInfo struct {
-	refs        int
-	parents     []string
-	size        int64
-	mutable     bool
-	createdAt   time.Time
-	usageCount  int
-	lastUsedAt  *time.Time
-	description string
-	doubleRef   bool
-	recordType  client.UsageRecordType
-	shared      bool
-	parentChain []digest.Digest
+	refs          int
+	parents       []string
+	size          int64
+	mutable       bool
+	createdAt     time.Time
+	usageCount    int
+	lastUsedAt    *time.Time
+	description   string
+	doubleRef     bool
+	recordType    client.UsageRecordType
+	shared        bool
+	parentChain   []digest.Digest
+	stableDigests []string
+	creatorDigest string
 }
 
 func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) ([]*client.UsageInfo, error) {
@@ -1329,16 +1331,18 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo)
 
 		usageCount, lastUsedAt := cr.getLastUsed()
 		c := &cacheUsageInfo{
-			refs:        len(cr.refs),
-			mutable:     cr.mutable,
-			size:        cr.getSize(),
-			createdAt:   cr.GetCreatedAt(),
-			usageCount:  usageCount,
-			lastUsedAt:  lastUsedAt,
-			description: cr.GetDescription(),
-			doubleRef:   cr.equalImmutable != nil,
-			recordType:  cr.GetRecordType(),
-			parentChain: cr.layerDigestChain(),
+			refs:          len(cr.refs),
+			mutable:       cr.mutable,
+			size:          cr.getSize(),
+			createdAt:     cr.GetCreatedAt(),
+			usageCount:    usageCount,
+			lastUsedAt:    lastUsedAt,
+			description:   cr.GetDescription(),
+			doubleRef:     cr.equalImmutable != nil,
+			recordType:    cr.GetRecordType(),
+			parentChain:   cr.layerDigestChain(),
+			stableDigests: cr.GetStringSlice("depot.stableDigests"),
+			creatorDigest: cr.GetString("depot.vertexDigest"),
 		}
 		if c.recordType == "" {
 			c.recordType = client.UsageRecordTypeRegular
@@ -1395,17 +1399,19 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo)
 	var du []*client.UsageInfo
 	for id, cr := range m {
 		c := &client.UsageInfo{
-			ID:          id,
-			Mutable:     cr.mutable,
-			InUse:       cr.refs > 0,
-			Size:        cr.size,
-			Parents:     cr.parents,
-			CreatedAt:   cr.createdAt,
-			Description: cr.description,
-			LastUsedAt:  cr.lastUsedAt,
-			UsageCount:  cr.usageCount,
-			RecordType:  cr.recordType,
-			Shared:      cr.shared,
+			ID:            id,
+			Mutable:       cr.mutable,
+			InUse:         cr.refs > 0,
+			Size:          cr.size,
+			Parents:       cr.parents,
+			CreatedAt:     cr.createdAt,
+			Description:   cr.description,
+			LastUsedAt:    cr.lastUsedAt,
+			UsageCount:    cr.usageCount,
+			RecordType:    cr.recordType,
+			Shared:        cr.shared,
+			StableDigests: cr.stableDigests,
+			CreatorDigest: cr.creatorDigest,
 		}
 		if filter.Match(adaptUsageInfo(c)) {
 			du = append(du, c)
@@ -1475,6 +1481,18 @@ func WithDescription(descr string) RefOption {
 	}
 }
 
+func WithStableDigests(digests []string) RefOption {
+	return func(m *cacheMetadata) error {
+		return m.AppendStringSlice("depot.stableDigests", digests...)
+	}
+}
+
+func WithVertexDigest(digest string) RefOption {
+	return func(m *cacheMetadata) error {
+		return m.InsertIfNotExists("depot.vertexDigest", digest)
+	}
+}
+
 func WithRecordType(t client.UsageRecordType) RefOption {
 	return func(m *cacheMetadata) error {
 		return m.queueRecordType(t)

cache/manager_test.go

@@ -3,7 +3,6 @@ package cache
 import (
 	"archive/tar"
 	"bytes"
-	"compress/gzip"
 	"context"
 	"encoding/binary"
 	"fmt"
@@ -18,6 +17,9 @@ import (
 	"testing"
 	"time"
 
+	// DEPOT: Using parallel gzip for faster image layer compression
+	gzip "github.com/klauspost/pgzip"
+
 	ctdcompression "github.com/containerd/containerd/archive/compression"
 	"github.com/containerd/containerd/archive/tarheader"
 	"github.com/containerd/containerd/content"

cache/metadata.go

@@ -72,6 +72,10 @@ type RefMetadata interface {
 	// generic getters/setters for external packages
 	GetString(string) string
 	SetString(key, val, index string) error
+	// AppendStringSlice will append the values to the existing slice; values are deduplicated.
+	AppendStringSlice(key string, values ...string) error
+	// InsertIfNotExists will insert the value if the key does not exist; otherwise, the value is ignored.
+	InsertIfNotExists(key, value string) error
 
 	GetExternal(string) ([]byte, error)
 	SetExternal(string, []byte) error
@@ -292,7 +296,7 @@ func (md *cacheMetadata) appendURLs(urls []string) error {
 	if len(urls) == 0 {
 		return nil
 	}
-	return md.appendStringSlice(keyURLs, urls...)
+	return md.AppendStringSlice(keyURLs, urls...)
 }
 
 func (md *cacheMetadata) getURLs() []string {
@@ -363,7 +367,7 @@ func (md *cacheMetadata) getSize() int64 {
 }
 
 func (md *cacheMetadata) appendImageRef(s string) error {
-	return md.appendStringSlice(keyImageRefs, s)
+	return md.AppendStringSlice(keyImageRefs, s)
 }
 
 func (md *cacheMetadata) getImageRefs() []string {
@@ -542,7 +546,7 @@ func (md *cacheMetadata) getInt64(key string) (int64, bool) {
 	return i, true
 }
 
-func (md *cacheMetadata) appendStringSlice(key string, values ...string) error {
+func (md *cacheMetadata) AppendStringSlice(key string, values ...string) error {
 	return md.si.GetAndSetValue(key, func(v *metadata.Value) (*metadata.Value, error) {
 		var slice []string
 		if v != nil {
@@ -586,3 +590,12 @@ func (md *cacheMetadata) getStringSlice(key string) []string {
 	}
 	return s
 }
+
+func (md *cacheMetadata) InsertIfNotExists(key, value string) error {
+	return md.si.GetAndSetValue(key, func(v *metadata.Value) (*metadata.Value, error) {
+		if v != nil {
+			return nil, metadata.ErrSkipSetValue
+		}
+		return metadata.NewValue(value)
+	})
+}

cache/refs.go

@@ -1098,8 +1098,10 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s
 					if err == nil { // usable as remote snapshot without unlazying.
 						defer func() {
 							// Remove tmp labels appended in this func
-							for k := range tmpLabels {
-								info.Labels[k] = ""
+							if info.Labels != nil {
+								for k := range tmpLabels {
+									info.Labels[k] = ""
+								}
 							}
 							if _, err := r.cm.Snapshotter.Update(ctx, info, tmpFields...); err != nil {
 								bklog.G(ctx).Warn(errors.Wrapf(err,