diff --git a/carstream.go b/carstream.go index 3c98040..b17b289 100644 --- a/carstream.go +++ b/carstream.go @@ -29,23 +29,23 @@ var protoChooser = dagpb.AddSupportToChooser(basicnode.Chooser) // StreamCar streams a DAG in CARv1 format to the given writer, using the given // selector. -func StreamCar(ctx context.Context, requestLsys linking.LinkSystem, rootCid cid.Cid, selNode datamodel.Node, out io.Writer, duplicates bool) error { +func StreamCar(ctx context.Context, requestLsys linking.LinkSystem, rootCid cid.Cid, selNode datamodel.Node, out io.Writer, duplicates bool) (int64, int64, error) { sel, err := selector.CompileSelector(selNode) if err != nil { - return fmt.Errorf("failed to compile selector: %w", err) + return 0, 0, fmt.Errorf("failed to compile selector: %w", err) } carWriter, err := carstorage.NewWritable(out, []cid.Cid{rootCid}, car.WriteAsCarV1(true), car.AllowDuplicatePuts(duplicates)) if err != nil { - return fmt.Errorf("failed to create car writer: %w", err) + return 0, 0, fmt.Errorf("failed to create car writer: %w", err) } - erro := &errorRecordingReadOpener{ctx, requestLsys.StorageReadOpener, carWriter, nil} + erro := newErrorRecordingReadOpener(ctx, requestLsys.StorageReadOpener, carWriter) requestLsys.StorageReadOpener = erro.StorageReadOpener rootNode, err := loadNode(ctx, rootCid, requestLsys) if err != nil { - return fmt.Errorf("failed to load root node: %w", err) + return 0, 0, fmt.Errorf("failed to load root node: %w", err) } progress := traversal.Progress{Cfg: &traversal.Config{ @@ -54,20 +54,26 @@ func StreamCar(ctx context.Context, requestLsys linking.LinkSystem, rootCid cid. LinkTargetNodePrototypeChooser: protoChooser, }} if err := progress.WalkAdv(rootNode, sel, visitNoop); err != nil { - return fmt.Errorf("failed to complete traversal: %w", err) + return 0, 0, fmt.Errorf("failed to complete traversal: %w", err) } if erro.err != nil { - return fmt.Errorf("block load failed during traversal: %w", erro.err) + return 0, 0, fmt.Errorf("block load failed during traversal: %w", erro.err) } - return nil + return erro.byteCount, erro.blockCount, nil } type errorRecordingReadOpener struct { - ctx context.Context - orig linking.BlockReadOpener - car carstorage.WritableCar - err error + ctx context.Context + orig linking.BlockReadOpener + car carstorage.WritableCar + err error + byteCount int64 + blockCount int64 +} + +func newErrorRecordingReadOpener(ctx context.Context, orig linking.BlockReadOpener, car carstorage.WritableCar) *errorRecordingReadOpener { + return &errorRecordingReadOpener{ctx, orig, car, nil, 0, 0} } func (erro *errorRecordingReadOpener) StorageReadOpener(lc linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { @@ -84,6 +90,8 @@ func (erro *errorRecordingReadOpener) StorageReadOpener(lc linking.LinkContext, if err != nil { return nil, err } + erro.byteCount += int64(len(byts)) + erro.blockCount++ return bytes.NewReader(byts), nil } diff --git a/carstream_test.go b/carstream_test.go index 4caa7ce..b6082a0 100644 --- a/carstream_test.go +++ b/carstream_test.go @@ -50,17 +50,21 @@ func TestStreamCar(t *testing.T) { } testCases := []struct { - name string - selector datamodel.Node - root cid.Cid - lsys linking.LinkSystem - validate func(t *testing.T, r io.Reader) + name string + selector datamodel.Node + root cid.Cid + lsys linking.LinkSystem + expectedBytes int64 + expectedBlocks int64 + validate func(t *testing.T, r io.Reader) }{ { - name: "chain: all blocks", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: tbc.TipLink.(cidlink.Link).Cid, - lsys: chainLsys, + name: "chain: all blocks", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: tbc.TipLink.(cidlink.Link).Cid, + lsys: chainLsys, + expectedBytes: sizeOf(allChainBlocks), + expectedBlocks: 100, validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, tbc.TipLink.(cidlink.Link).Cid, root) @@ -68,10 +72,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "chain: just root", - selector: selectorparse.CommonSelector_MatchPoint, - root: tbc.TipLink.(cidlink.Link).Cid, - lsys: chainLsys, + name: "chain: just root", + selector: selectorparse.CommonSelector_MatchPoint, + root: tbc.TipLink.(cidlink.Link).Cid, + lsys: chainLsys, + expectedBytes: sizeOf(allChainBlocks[:1]), + expectedBlocks: 1, validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, tbc.TipLink.(cidlink.Link).Cid, root) @@ -79,10 +85,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "unixfs file", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: fileEnt.Root, - lsys: fileLsys, + name: "unixfs file", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: fileEnt.Root, + lsys: fileLsys, + expectedBytes: sizeOfDirEnt(fileEnt, fileLsys), + expectedBlocks: int64(len(fileEnt.SelfCids)), validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, fileEnt.Root, root) @@ -90,10 +98,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "unixfs directory", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: dirEnt.Root, - lsys: dirLsys, + name: "unixfs directory", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: dirEnt.Root, + lsys: dirLsys, + expectedBytes: sizeOfDirEnt(dirEnt, dirLsys), + expectedBlocks: blocksInDirEnt(dirEnt), validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, dirEnt.Root, root) @@ -101,10 +111,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "unixfs sharded directory", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: shardedDirEnt.Root, - lsys: shardedDirLsys, + name: "unixfs sharded directory", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: shardedDirEnt.Root, + lsys: shardedDirLsys, + expectedBytes: sizeOfDirEnt(shardedDirEnt, shardedDirLsys), + expectedBlocks: blocksInDirEnt(shardedDirEnt), validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, shardedDirEnt.Root, root) @@ -118,8 +130,10 @@ func TestStreamCar(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := require.New(t) var buf bytes.Buffer - err := frisbii.StreamCar(ctx, tc.lsys, tc.root, tc.selector, &buf, false) + byts, blks, err := frisbii.StreamCar(ctx, tc.lsys, tc.root, tc.selector, &buf, false) req.NoError(err) + req.Equal(tc.expectedBytes, byts) + req.Equal(tc.expectedBlocks, blks) tc.validate(t, &buf) }) } @@ -200,3 +214,33 @@ func GenerateNoDupes(gen func() unixfs.DirEntry) unixfs.DirEntry { } } } + +func sizeOf(blks []blocks.Block) int64 { + var size int64 + for _, blk := range blks { + size += int64(len(blk.RawData())) + } + return size +} +func sizeOfDirEnt(dirEnt unixfs.DirEntry, ls linking.LinkSystem) int64 { + var size int64 + for _, c := range dirEnt.SelfCids { + blk, err := ls.LoadRaw(linking.LinkContext{}, cidlink.Link{Cid: c}) + if err != nil { + panic(err) + } + size += int64(len(blk)) + } + for _, c := range dirEnt.Children { + size += sizeOfDirEnt(c, ls) + } + return size +} + +func blocksInDirEnt(dirEnt unixfs.DirEntry) int64 { + size := int64(len(dirEnt.SelfCids)) + for _, c := range dirEnt.Children { + size += blocksInDirEnt(c) + } + return size +} diff --git a/go.mod b/go.mod index ec044ea..ec776a0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/dustin/go-humanize v1.0.1 - github.com/filecoin-project/lassie v0.12.2-0.20230614045620-19b6e938241c + github.com/filecoin-project/lassie v0.14.4-0.20230811073202-26954e00718d github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-graphsync v0.14.7 @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-unixfsnode v1.7.3 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.20.1-0.20230613110822-3142e1304e55 + github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de github.com/ipni/go-libipni v0.3.4 github.com/ipni/index-provider v0.13.5 github.com/libp2p/go-libp2p v0.29.2 @@ -22,6 +22,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.7 golang.org/x/term v0.10.0 + lukechampine.com/blake3 v1.2.1 ) require ( @@ -32,13 +33,11 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect - github.com/felixge/httpsnoop v1.0.0 // indirect github.com/filecoin-project/go-address v1.1.0 // indirect github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect github.com/filecoin-project/go-cbor-util v0.0.1 // indirect @@ -58,7 +57,6 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -70,22 +68,18 @@ require ( github.com/hannahhoward/go-pubsub v1.0.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.10.2-0.20230629143123-2d3edc552442 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blocksutil v0.0.1 // indirect github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect - github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect + github.com/ipfs/go-unixfs v0.4.5 // indirect github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect @@ -101,9 +95,6 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-libp2p-gostream v0.6.0 // indirect github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect - github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.7.0 // indirect - github.com/libp2p/go-mplex v0.7.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect @@ -117,7 +108,6 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/go-server-timing v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -172,5 +162,4 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 99559d3..149a6f9 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= -github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= -github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -78,8 +76,6 @@ github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/ github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.0 h1:gh8fMGz0rlOv/1WmRZm7OgncIOTsAj21iNJot48omJQ= -github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f/go.mod h1:+If3s2VxyjZn+KGGZIoRXBDSFQ9xL404JBJGf4WhEj0= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -121,15 +117,15 @@ github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNdofHZoGPjfNaAo5Q= github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= -github.com/filecoin-project/lassie v0.12.2-0.20230614045620-19b6e938241c h1:2E1hw30L6zKaHsNnqoqXc48A3EDJ/afnGMPFna8c4h0= -github.com/filecoin-project/lassie v0.12.2-0.20230614045620-19b6e938241c/go.mod h1:kbO6Ljk/2ug6CLJuHsgsd4c6EVrAmnVQKFAktmETSI4= +github.com/filecoin-project/lassie v0.14.4-0.20230811073202-26954e00718d h1:QC5m8PZxdJHBAqIp+XIMT8srIZnBpS2y9mFXyWQBIXU= +github.com/filecoin-project/lassie v0.14.4-0.20230811073202-26954e00718d/go.mod h1:4zIWVE9FSeGbx7xQTd9y+vFVt4UIdBxfVxnj4RCI9gM= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gammazero/channelqueue v0.2.1 h1:AcK6wnLrj8koTTn3RxjRCyfmS677TjhIZb1FSMi14qc= github.com/gammazero/channelqueue v0.2.1/go.mod h1:824o5HHE+yO1xokh36BIuSv8YWwXW0364ku91eRMFS4= @@ -158,8 +154,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= -github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -187,7 +181,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= @@ -230,16 +223,12 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= -github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.10.2-0.20230629143123-2d3edc552442 h1:SGbw381zt6c1VFf3QCBaJ+eVJ4AwD9fPaFKFp9U9Apk= -github.com/ipfs/boxo v0.10.2-0.20230629143123-2d3edc552442/go.mod h1:1qgKq45mPRCxf4ZPoJV2lnXxyxucigILMJOrQrVivv8= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -275,8 +264,6 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= -github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= @@ -314,10 +301,10 @@ github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= -github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= +github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= github.com/ipfs/go-unixfsnode v1.7.3 h1:giAxFq7CxAm2Z8h8yFAD7TOQUpf5XG7a2xrR143ci4Y= github.com/ipfs/go-unixfsnode v1.7.3/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= @@ -327,8 +314,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 h1:QAI/Ridj0+foHD6epbxmB4ugxz9B4vmNdYSmQLGa05E= github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:odxGcpiQZLzP5+yGu84Ljo8y3EzCvNAQKEodHNsHLXA= -github.com/ipld/go-ipld-prime v0.20.1-0.20230613110822-3142e1304e55 h1:D1JUX6l0+ugD3PE99l/NmN/97jz9YNP0uZZRLAGZQhs= -github.com/ipld/go-ipld-prime v0.20.1-0.20230613110822-3142e1304e55/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= +github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de h1:N6Wfk6dvcBjF4AJJDSmti6CkgHWZPDZ0fuqSQL+kKnU= +github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipni/go-libipni v0.3.4 h1:ZYgCE2TOZt/QJJcBZb+R63FaBLlA2suZGP2IH1fKv4A= github.com/ipni/go-libipni v0.3.4/go.mod h1:6EIUhN83pd1i6q7SCSCIuuUC3XgR7D/gjKkEnVyIQWE= @@ -390,13 +377,7 @@ github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qk github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA= github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= -github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= -github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.7.0 h1:sirOYVD0wGWjkDwHZvinunIpaqPLBXkcnXApVHwZFGA= -github.com/libp2p/go-libp2p-routing-helpers v0.7.0/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= -github.com/libp2p/go-mplex v0.7.0 h1:BDhFZdlk5tbr0oyFq/xv/NPGfjbnrsDam1EvutpBDbY= -github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= @@ -437,8 +418,6 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE= -github.com/mitchellh/go-server-timing v1.0.1/go.mod h1:Mo6GKi9FSLwWFAMn3bqVPWe20y5ri5QGQuO9D9MCOxk= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= diff --git a/httpipfs.go b/httpipfs.go index a12e0cd..27f2c85 100644 --- a/httpipfs.go +++ b/httpipfs.go @@ -9,12 +9,16 @@ import ( "sync" "time" - lassiehttp "github.com/filecoin-project/lassie/pkg/server/http" + lassiehttp "github.com/filecoin-project/lassie/pkg/httputil" + "github.com/filecoin-project/lassie/pkg/httputil/metadata" + "github.com/filecoin-project/lassie/pkg/types" lassietypes "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" + mh "github.com/multiformats/go-multihash" + "lukechampine.com/blake3" ) var _ http.Handler = (*HttpIpfs)(nil) @@ -86,7 +90,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { return } - includeDupes, err := lassiehttp.CheckFormat(req) + includeDupes, includeMeta, err := lassiehttp.CheckFormat(req) if err != nil { logError(http.StatusBadRequest, err) return @@ -120,30 +124,82 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { selNode := unixfsnode.UnixFSPathSelectorBuilder(path.String(), dagScope.TerminalSelectorSpec(), false) bytesWrittenCh := make(chan struct{}) - writer := newIpfsResponseWriter(res, hi.maxResponseBytes, func() { + var writer io.Writer = newIpfsResponseWriter(res, hi.maxResponseBytes, func() { // called once we start writing blocks into the CAR (on the first Put()) res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName)) res.Header().Set("Accept-Ranges", lassiehttp.ResponseAcceptRangesHeader) res.Header().Set("Cache-Control", lassiehttp.ResponseCacheControlHeader) - res.Header().Set("Content-Type", lassiehttp.ResponseContentTypeHeader) + res.Header().Set("Content-Type", lassiehttp.RequestAcceptHeader) // instead of ResponseContentTypeHeader because we are doing meta=eof as well res.Header().Set("Etag", etag(rootCid, path.String(), dagScope, includeDupes)) res.Header().Set("X-Content-Type-Options", "nosniff") res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String()) close(bytesWrittenCh) }) - if err := StreamCar(ctx, hi.lsys, rootCid, selNode, writer, includeDupes); err != nil { - logError(http.StatusInternalServerError, err) + if includeMeta { + writer = newChecksumWriter(writer) + } + + dataBytes, blockCount, carErr := StreamCar(ctx, hi.lsys, rootCid, selNode, writer, includeDupes) + + if includeMeta { + // write NUL byte to indicate end of CARv1 data + if _, err := res.Write([]byte{0}); err != nil { + if carErr != nil { + carErr = err + } + } + // write the metadata + md := metadata.Metadata{ + Request: metadata.Request{ + Root: rootCid, + Scope: dagScope, + Duplicates: includeDupes, + }, + } + if path.Len() != 0 { + p := "/" + path.String() + md.Request.Path = &p + } + + if carErr != nil { + msg := carErr.Error() + md.Error = &msg + } else { + checksum := writer.(*checksumWriter).Sum() + checksumMh, err := mh.Encode(checksum, mh.BLAKE3) + if err != nil { + msg := fmt.Sprintf("error creating checksum multihash: %s", err.Error()) + md.Error = &msg + carErr = err + } else { + md.Properties = &types.CarProperties{ + CarBytes: writer.(*checksumWriter).Count(), + DataBytes: dataBytes, + BlockCount: blockCount, + ChecksumMultihash: checksumMh, + } + } + } + + err := metadata.CarMetadata{Metadata: &md}.Serialize(res) + if err != nil && carErr != nil { + carErr = err + } + } + + if carErr != nil { + logError(http.StatusInternalServerError, carErr) select { case <-bytesWrittenCh: - logger.Debugw("unclean close", "cid", rootCid, "err", err) + logger.Debugw("unclean close", "cid", rootCid, "err", carErr) if err := closeWithUnterminatedChunk(res); err != nil { logger.Infow("unable to send early termination", "err", err) } return default: } - logger.Debugw("error streaming CAR", "cid", rootCid, "err", err) + logger.Debugw("error streaming CAR", "cid", rootCid, "err", carErr) } } @@ -207,3 +263,32 @@ func closeWithUnterminatedChunk(res http.ResponseWriter) error { } return nil } + +type checksumWriter struct { + w io.Writer + h *blake3.Hasher + c int64 +} + +func newChecksumWriter(w io.Writer) *checksumWriter { + return &checksumWriter{ + w: w, + h: blake3.New(32, nil), + } +} + +func (hw *checksumWriter) Write(p []byte) (n int, err error) { + if _, err := hw.h.Write(p); err != nil { + return 0, err + } + hw.c += int64(len(p)) + return hw.w.Write(p) +} + +func (hw *checksumWriter) Sum() []byte { + return hw.h.Sum(nil) +} + +func (hw *checksumWriter) Count() int64 { + return hw.c +}