From 27bae29f8522c65641342daea58fb43813cc6133 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 10 Aug 2023 22:22:57 +1000 Subject: [PATCH 1/2] feat: add ?meta=eof for trailling metadata dag-json Ref: https://github.com/ipfs/specs/pull/431 --- carstream.go | 32 ++++++---- carstream_test.go | 96 +++++++++++++++++++++--------- go.mod | 4 +- go.sum | 6 +- httpipfs.go | 104 +++++++++++++++++++++++++++++++-- metadata/metadata.go | 76 ++++++++++++++++++++++++ metadata/metadata.ipldsch | 31 ++++++++++ metadata/metadata_test.go | 119 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 420 insertions(+), 48 deletions(-) create mode 100644 metadata/metadata.go create mode 100644 metadata/metadata.ipldsch create mode 100644 metadata/metadata_test.go diff --git a/carstream.go b/carstream.go index 3c98040..b17b289 100644 --- a/carstream.go +++ b/carstream.go @@ -29,23 +29,23 @@ var protoChooser = dagpb.AddSupportToChooser(basicnode.Chooser) // StreamCar streams a DAG in CARv1 format to the given writer, using the given // selector. -func StreamCar(ctx context.Context, requestLsys linking.LinkSystem, rootCid cid.Cid, selNode datamodel.Node, out io.Writer, duplicates bool) error { +func StreamCar(ctx context.Context, requestLsys linking.LinkSystem, rootCid cid.Cid, selNode datamodel.Node, out io.Writer, duplicates bool) (int64, int64, error) { sel, err := selector.CompileSelector(selNode) if err != nil { - return fmt.Errorf("failed to compile selector: %w", err) + return 0, 0, fmt.Errorf("failed to compile selector: %w", err) } carWriter, err := carstorage.NewWritable(out, []cid.Cid{rootCid}, car.WriteAsCarV1(true), car.AllowDuplicatePuts(duplicates)) if err != nil { - return fmt.Errorf("failed to create car writer: %w", err) + return 0, 0, fmt.Errorf("failed to create car writer: %w", err) } - erro := &errorRecordingReadOpener{ctx, requestLsys.StorageReadOpener, carWriter, nil} + erro := newErrorRecordingReadOpener(ctx, requestLsys.StorageReadOpener, carWriter) requestLsys.StorageReadOpener = erro.StorageReadOpener rootNode, err := loadNode(ctx, rootCid, requestLsys) if err != nil { - return fmt.Errorf("failed to load root node: %w", err) + return 0, 0, fmt.Errorf("failed to load root node: %w", err) } progress := traversal.Progress{Cfg: &traversal.Config{ @@ -54,20 +54,26 @@ func StreamCar(ctx context.Context, requestLsys linking.LinkSystem, rootCid cid. LinkTargetNodePrototypeChooser: protoChooser, }} if err := progress.WalkAdv(rootNode, sel, visitNoop); err != nil { - return fmt.Errorf("failed to complete traversal: %w", err) + return 0, 0, fmt.Errorf("failed to complete traversal: %w", err) } if erro.err != nil { - return fmt.Errorf("block load failed during traversal: %w", erro.err) + return 0, 0, fmt.Errorf("block load failed during traversal: %w", erro.err) } - return nil + return erro.byteCount, erro.blockCount, nil } type errorRecordingReadOpener struct { - ctx context.Context - orig linking.BlockReadOpener - car carstorage.WritableCar - err error + ctx context.Context + orig linking.BlockReadOpener + car carstorage.WritableCar + err error + byteCount int64 + blockCount int64 +} + +func newErrorRecordingReadOpener(ctx context.Context, orig linking.BlockReadOpener, car carstorage.WritableCar) *errorRecordingReadOpener { + return &errorRecordingReadOpener{ctx, orig, car, nil, 0, 0} } func (erro *errorRecordingReadOpener) StorageReadOpener(lc linking.LinkContext, lnk datamodel.Link) (io.Reader, error) { @@ -84,6 +90,8 @@ func (erro *errorRecordingReadOpener) StorageReadOpener(lc linking.LinkContext, if err != nil { return nil, err } + erro.byteCount += int64(len(byts)) + erro.blockCount++ return bytes.NewReader(byts), nil } diff --git a/carstream_test.go b/carstream_test.go index 4caa7ce..b6082a0 100644 --- a/carstream_test.go +++ b/carstream_test.go @@ -50,17 +50,21 @@ func TestStreamCar(t *testing.T) { } testCases := []struct { - name string - selector datamodel.Node - root cid.Cid - lsys linking.LinkSystem - validate func(t *testing.T, r io.Reader) + name string + selector datamodel.Node + root cid.Cid + lsys linking.LinkSystem + expectedBytes int64 + expectedBlocks int64 + validate func(t *testing.T, r io.Reader) }{ { - name: "chain: all blocks", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: tbc.TipLink.(cidlink.Link).Cid, - lsys: chainLsys, + name: "chain: all blocks", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: tbc.TipLink.(cidlink.Link).Cid, + lsys: chainLsys, + expectedBytes: sizeOf(allChainBlocks), + expectedBlocks: 100, validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, tbc.TipLink.(cidlink.Link).Cid, root) @@ -68,10 +72,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "chain: just root", - selector: selectorparse.CommonSelector_MatchPoint, - root: tbc.TipLink.(cidlink.Link).Cid, - lsys: chainLsys, + name: "chain: just root", + selector: selectorparse.CommonSelector_MatchPoint, + root: tbc.TipLink.(cidlink.Link).Cid, + lsys: chainLsys, + expectedBytes: sizeOf(allChainBlocks[:1]), + expectedBlocks: 1, validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, tbc.TipLink.(cidlink.Link).Cid, root) @@ -79,10 +85,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "unixfs file", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: fileEnt.Root, - lsys: fileLsys, + name: "unixfs file", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: fileEnt.Root, + lsys: fileLsys, + expectedBytes: sizeOfDirEnt(fileEnt, fileLsys), + expectedBlocks: int64(len(fileEnt.SelfCids)), validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, fileEnt.Root, root) @@ -90,10 +98,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "unixfs directory", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: dirEnt.Root, - lsys: dirLsys, + name: "unixfs directory", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: dirEnt.Root, + lsys: dirLsys, + expectedBytes: sizeOfDirEnt(dirEnt, dirLsys), + expectedBlocks: blocksInDirEnt(dirEnt), validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, dirEnt.Root, root) @@ -101,10 +111,12 @@ func TestStreamCar(t *testing.T) { }, }, { - name: "unixfs sharded directory", - selector: selectorparse.CommonSelector_ExploreAllRecursively, - root: shardedDirEnt.Root, - lsys: shardedDirLsys, + name: "unixfs sharded directory", + selector: selectorparse.CommonSelector_ExploreAllRecursively, + root: shardedDirEnt.Root, + lsys: shardedDirLsys, + expectedBytes: sizeOfDirEnt(shardedDirEnt, shardedDirLsys), + expectedBlocks: blocksInDirEnt(shardedDirEnt), validate: func(t *testing.T, r io.Reader) { root, blks := carToBlocks(t, r) require.Equal(t, shardedDirEnt.Root, root) @@ -118,8 +130,10 @@ func TestStreamCar(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := require.New(t) var buf bytes.Buffer - err := frisbii.StreamCar(ctx, tc.lsys, tc.root, tc.selector, &buf, false) + byts, blks, err := frisbii.StreamCar(ctx, tc.lsys, tc.root, tc.selector, &buf, false) req.NoError(err) + req.Equal(tc.expectedBytes, byts) + req.Equal(tc.expectedBlocks, blks) tc.validate(t, &buf) }) } @@ -200,3 +214,33 @@ func GenerateNoDupes(gen func() unixfs.DirEntry) unixfs.DirEntry { } } } + +func sizeOf(blks []blocks.Block) int64 { + var size int64 + for _, blk := range blks { + size += int64(len(blk.RawData())) + } + return size +} +func sizeOfDirEnt(dirEnt unixfs.DirEntry, ls linking.LinkSystem) int64 { + var size int64 + for _, c := range dirEnt.SelfCids { + blk, err := ls.LoadRaw(linking.LinkContext{}, cidlink.Link{Cid: c}) + if err != nil { + panic(err) + } + size += int64(len(blk)) + } + for _, c := range dirEnt.Children { + size += sizeOfDirEnt(c, ls) + } + return size +} + +func blocksInDirEnt(dirEnt unixfs.DirEntry) int64 { + size := int64(len(dirEnt.SelfCids)) + for _, c := range dirEnt.Children { + size += blocksInDirEnt(c) + } + return size +} diff --git a/go.mod b/go.mod index ec044ea..6ed4009 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-unixfsnode v1.7.3 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.20.1-0.20230613110822-3142e1304e55 + github.com/ipld/go-ipld-prime v0.21.1-0.20230810111002-bdf990edcdeb github.com/ipni/go-libipni v0.3.4 github.com/ipni/index-provider v0.13.5 github.com/libp2p/go-libp2p v0.29.2 @@ -22,6 +22,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.7 golang.org/x/term v0.10.0 + lukechampine.com/blake3 v1.2.1 ) require ( @@ -172,5 +173,4 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 99559d3..c2ba7eb 100644 --- a/go.sum +++ b/go.sum @@ -129,7 +129,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gammazero/channelqueue v0.2.1 h1:AcK6wnLrj8koTTn3RxjRCyfmS677TjhIZb1FSMi14qc= github.com/gammazero/channelqueue v0.2.1/go.mod h1:824o5HHE+yO1xokh36BIuSv8YWwXW0364ku91eRMFS4= @@ -327,8 +327,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 h1:QAI/Ridj0+foHD6epbxmB4ugxz9B4vmNdYSmQLGa05E= github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:odxGcpiQZLzP5+yGu84Ljo8y3EzCvNAQKEodHNsHLXA= -github.com/ipld/go-ipld-prime v0.20.1-0.20230613110822-3142e1304e55 h1:D1JUX6l0+ugD3PE99l/NmN/97jz9YNP0uZZRLAGZQhs= -github.com/ipld/go-ipld-prime v0.20.1-0.20230613110822-3142e1304e55/go.mod h1:PRQpXNcJypaPiiSdarsrJABPkYrBvafwDl0B9HjujZ8= +github.com/ipld/go-ipld-prime v0.21.1-0.20230810111002-bdf990edcdeb h1:frdDF5813yKhOaEHAdEhgArLVk+uBQwygVtfg2jAGR4= +github.com/ipld/go-ipld-prime v0.21.1-0.20230810111002-bdf990edcdeb/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipni/go-libipni v0.3.4 h1:ZYgCE2TOZt/QJJcBZb+R63FaBLlA2suZGP2IH1fKv4A= github.com/ipni/go-libipni v0.3.4/go.mod h1:6EIUhN83pd1i6q7SCSCIuuUC3XgR7D/gjKkEnVyIQWE= diff --git a/httpipfs.go b/httpipfs.go index a12e0cd..13536a1 100644 --- a/httpipfs.go +++ b/httpipfs.go @@ -13,8 +13,11 @@ import ( lassietypes "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" + "github.com/ipld/frisbii/metadata" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" + mh "github.com/multiformats/go-multihash" + "lukechampine.com/blake3" ) var _ http.Handler = (*HttpIpfs)(nil) @@ -113,6 +116,8 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { return } + includeMeta := req.URL.Query().Get("meta") == "eof" + if fileName == "" { fileName = fmt.Sprintf("%s%s", rootCid.String(), lassiehttp.FilenameExtCar) } @@ -120,7 +125,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { selNode := unixfsnode.UnixFSPathSelectorBuilder(path.String(), dagScope.TerminalSelectorSpec(), false) bytesWrittenCh := make(chan struct{}) - writer := newIpfsResponseWriter(res, hi.maxResponseBytes, func() { + var writer io.Writer = newIpfsResponseWriter(res, hi.maxResponseBytes, func() { // called once we start writing blocks into the CAR (on the first Put()) res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName)) res.Header().Set("Accept-Ranges", lassiehttp.ResponseAcceptRangesHeader) @@ -132,18 +137,78 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { close(bytesWrittenCh) }) - if err := StreamCar(ctx, hi.lsys, rootCid, selNode, writer, includeDupes); err != nil { - logError(http.StatusInternalServerError, err) + if includeMeta { + writer = newChecksumWriter(writer) + } + + dataBytes, blockCount, carErr := StreamCar(ctx, hi.lsys, rootCid, selNode, writer, includeDupes) + + if includeMeta { + // write NUL byte to indicate end of CARv1 data + if _, err := res.Write([]byte{0}); err != nil { + if carErr != nil { + carErr = err + } + } + // write the metadata + md := metadata.Metadata{ + Request: metadata.Request{ + Root: rootCid, + Scope: dagScope, + Duplicates: includeDupes, + }, + } + if path.Len() != 0 { + p := "/" + path.String() + md.Request.Path = &p + } + + if carErr != nil { + msg := carErr.Error() + md.Error = &msg + } else { + checksum := writer.(*checksumWriter).Sum() + checksumMh, err := mh.Encode(checksum, mh.BLAKE3) + if err != nil { + msg := fmt.Sprintf("error creating checksum multihash: %s", err.Error()) + md.Error = &msg + carErr = err + } else { + md.Properties = &metadata.CarProperties{ + CarBytes: writer.(*checksumWriter).Count(), + DataBytes: dataBytes, + BlockCount: blockCount, + ChecksumMultihash: checksumMh, + } + } + } + + enc, err := metadata.CarMetadata{Metadata: &md}.Serialize() + if err != nil { + if carErr != nil { + carErr = err + } + } else { + if _, err := res.Write(enc); err != nil { + if carErr != nil { + carErr = err + } + } + } + } + + if carErr != nil { + logError(http.StatusInternalServerError, carErr) select { case <-bytesWrittenCh: - logger.Debugw("unclean close", "cid", rootCid, "err", err) + logger.Debugw("unclean close", "cid", rootCid, "err", carErr) if err := closeWithUnterminatedChunk(res); err != nil { logger.Infow("unable to send early termination", "err", err) } return default: } - logger.Debugw("error streaming CAR", "cid", rootCid, "err", err) + logger.Debugw("error streaming CAR", "cid", rootCid, "err", carErr) } } @@ -207,3 +272,32 @@ func closeWithUnterminatedChunk(res http.ResponseWriter) error { } return nil } + +type checksumWriter struct { + w io.Writer + h *blake3.Hasher + c int64 +} + +func newChecksumWriter(w io.Writer) *checksumWriter { + return &checksumWriter{ + w: w, + h: blake3.New(32, nil), + } +} + +func (hw *checksumWriter) Write(p []byte) (n int, err error) { + if _, err := hw.h.Write(p); err != nil { + return 0, err + } + hw.c += int64(len(p)) + return hw.w.Write(p) +} + +func (hw *checksumWriter) Sum() []byte { + return hw.h.Sum(nil) +} + +func (hw *checksumWriter) Count() int64 { + return hw.c +} diff --git a/metadata/metadata.go b/metadata/metadata.go new file mode 100644 index 0000000..3a3aaf5 --- /dev/null +++ b/metadata/metadata.go @@ -0,0 +1,76 @@ +package metadata + +import ( + "fmt" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime/codec/dagjson" + bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry" + mh "github.com/multiformats/go-multihash" + + _ "embed" +) + +//go:embed metadata.ipldsch +var schema []byte + +var BindnodeRegistry = bindnoderegistry.NewRegistry() + +type CarMetadata struct { + Metadata *Metadata +} + +func (cm CarMetadata) Serialize() ([]byte, error) { + // TODO: do the same checks we do on Deserialize() + return BindnodeRegistry.TypeToBytes(&cm, dagjson.Encode) +} + +func (cm *CarMetadata) Deserialize(byts []byte) error { + cmIface, err := BindnodeRegistry.TypeFromBytes(byts, &CarMetadata{}, dagjson.Decode) + if err != nil { + return fmt.Errorf("invalid CarMetadata: %w", err) + } + cmm := cmIface.(*CarMetadata) // safe to assume type + if cmm.Metadata.Properties == nil && cmm.Metadata.Error == nil { + return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields") + } + if (cmm.Metadata.Properties == nil) == (cmm.Metadata.Error == nil) { + return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields, not both") + } + if cmm.Metadata.Properties != nil { + if _, err := mh.Decode(cmm.Metadata.Properties.ChecksumMultihash); err != nil { + return fmt.Errorf("invalid CarMetadata: checksum multihash: %w", err) + } + } + // TODO: parse and check EntityBytes format + *cm = *cmm + return nil +} + +type Metadata struct { + Request Request + Properties *CarProperties + Error *string +} + +type Request struct { + Root cid.Cid + Path *string + Scope types.DagScope + Duplicates bool + EntityBytes *string +} + +type CarProperties struct { + CarBytes int64 + DataBytes int64 + BlockCount int64 + ChecksumMultihash []byte +} + +func init() { + if err := BindnodeRegistry.RegisterType((*CarMetadata)(nil), string(schema), "CarMetadata"); err != nil { + panic(err.Error()) + } +} diff --git a/metadata/metadata.ipldsch b/metadata/metadata.ipldsch new file mode 100644 index 0000000..ae27840 --- /dev/null +++ b/metadata/metadata.ipldsch @@ -0,0 +1,31 @@ +type CarMetadata union { + | Metadata "car-metadata/v1" +} representation keyed + +type Metadata struct { + request Request + # must contain either a properties or an error + properties optional CarProperties + error optional String +} + +type Request struct { + root &Any + path optional String + scope DagScope + duplicates Bool (rename "dups") + entityBytes optional String (rename "entity-bytes") # Must be a valid entity-bytes param: "from:to" +} + +type DagScope enum { + | all + | entity + | block +} + +type CarProperties struct { + carBytes Int (rename "car_bytes") + dataBytes Int (rename "data_bytes") + blockCount Int (rename "block_count") + checksumMultihash optional Bytes (rename "checksum") # Must be a valid multihash +} diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go new file mode 100644 index 0000000..b7a2415 --- /dev/null +++ b/metadata/metadata_test.go @@ -0,0 +1,119 @@ +package metadata_test + +import ( + "testing" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + "github.com/ipld/frisbii/metadata" + "github.com/stretchr/testify/require" +) + +var testCid = cid.MustParse("bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4") + +func TestCarMetadataRoundtrip(t *testing.T) { + path := "/birb.mp4" + orig := metadata.CarMetadata{ + Metadata: &metadata.Metadata{ + Request: metadata.Request{ + Root: testCid, + Path: &path, + Scope: types.DagScopeAll, + Duplicates: true, + }, + Properties: &metadata.CarProperties{ + CarBytes: 202020, + DataBytes: 101010, + BlockCount: 303, + ChecksumMultihash: testCid.Hash(), + }, + }, + } + byts, err := orig.Serialize() + require.NoError(t, err) + + t.Log("metadata dag-json:", string(byts)) + + var roundtrip metadata.CarMetadata + err = roundtrip.Deserialize(byts) + require.NoError(t, err) + require.Equal(t, orig, roundtrip) + require.NotNil(t, roundtrip.Metadata) + require.Equal(t, testCid, roundtrip.Metadata.Request.Root) + require.NotNil(t, roundtrip.Metadata.Request.Path) + require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path) + require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope) + require.True(t, roundtrip.Metadata.Request.Duplicates) + require.NotNil(t, roundtrip.Metadata.Properties) + require.Nil(t, roundtrip.Metadata.Error) + require.Equal(t, int64(202020), roundtrip.Metadata.Properties.CarBytes) + require.Equal(t, int64(101010), roundtrip.Metadata.Properties.DataBytes) + require.Equal(t, int64(303), roundtrip.Metadata.Properties.BlockCount) + require.Equal(t, []byte(testCid.Hash()), roundtrip.Metadata.Properties.ChecksumMultihash) +} + +func TestCarMetadataErrorRoundtrip(t *testing.T) { + path := "/birb.mp4" + msg := "something bad happened" + orig := metadata.CarMetadata{ + Metadata: &metadata.Metadata{ + Request: metadata.Request{ + Root: testCid, + Path: &path, + Scope: types.DagScopeAll, + Duplicates: true, + }, + Error: &msg, + }, + } + byts, err := orig.Serialize() + require.NoError(t, err) + + t.Log("metadata dag-json:", string(byts)) + + var roundtrip metadata.CarMetadata + err = roundtrip.Deserialize(byts) + require.NoError(t, err) + require.Equal(t, orig, roundtrip) + require.NotNil(t, roundtrip.Metadata) + require.Equal(t, testCid, roundtrip.Metadata.Request.Root) + require.NotNil(t, roundtrip.Metadata.Request.Path) + require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path) + require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope) + require.True(t, roundtrip.Metadata.Request.Duplicates) + require.Nil(t, roundtrip.Metadata.Properties) + require.NotNil(t, roundtrip.Metadata.Error) + require.Equal(t, "something bad happened", *roundtrip.Metadata.Error) +} + +func TestBadMetadata(t *testing.T) { + testCases := []struct { + name string + byts string + err string + }{ + {"empty", `{}`, `union structure constraints for CarMetadata caused rejection: a union must have exactly one entry`}, + {"bad key", `{"not metadata":true}`, `union structure constraints for CarMetadata caused rejection: no member named "not metadata"`}, + { + "bad multihash", + `{"car-metadata/v1":{"properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"bm90IGEgbXVsdGloYXNo"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, + `invalid CarMetadata: checksum multihash:`, + }, + { + "no properties or error", + `{"car-metadata/v1":{"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, + `invalid CarMetadata: must contain either properties or error fields`, + }, + { + "both properties and error", + `{"car-metadata/v1":{"error":"something bad happened","properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"EiBd9neBCasGxUmysJN7nGza4ylHikmbsP2+nXs6BlIpvw"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, + `invalid CarMetadata: must contain either properties or error fields, not both`, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var roundtrip metadata.CarMetadata + require.ErrorContains(t, roundtrip.Deserialize([]byte(tc.byts)), tc.err) + }) + } +} From 6e72477b515e5c66bf90f468958a79e68b22c8f4 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 11 Aug 2023 17:32:35 +1000 Subject: [PATCH 2/2] fix: move metadata utils to lassie --- go.mod | 17 +----- go.sum | 31 ++-------- httpipfs.go | 27 +++------ metadata/metadata.go | 76 ------------------------ metadata/metadata.ipldsch | 31 ---------- metadata/metadata_test.go | 119 -------------------------------------- 6 files changed, 17 insertions(+), 284 deletions(-) delete mode 100644 metadata/metadata.go delete mode 100644 metadata/metadata.ipldsch delete mode 100644 metadata/metadata_test.go diff --git a/go.mod b/go.mod index 6ed4009..ec776a0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/dustin/go-humanize v1.0.1 - github.com/filecoin-project/lassie v0.12.2-0.20230614045620-19b6e938241c + github.com/filecoin-project/lassie v0.14.4-0.20230811073202-26954e00718d github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-graphsync v0.14.7 @@ -13,7 +13,7 @@ require ( github.com/ipfs/go-unixfsnode v1.7.3 github.com/ipld/go-car/v2 v2.10.1 github.com/ipld/go-codec-dagpb v1.6.0 - github.com/ipld/go-ipld-prime v0.21.1-0.20230810111002-bdf990edcdeb + github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de github.com/ipni/go-libipni v0.3.4 github.com/ipni/index-provider v0.13.5 github.com/libp2p/go-libp2p v0.29.2 @@ -33,13 +33,11 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect - github.com/felixge/httpsnoop v1.0.0 // indirect github.com/filecoin-project/go-address v1.1.0 // indirect github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect github.com/filecoin-project/go-cbor-util v0.0.1 // indirect @@ -59,7 +57,6 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -71,22 +68,18 @@ require ( github.com/hannahhoward/go-pubsub v1.0.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/huin/goupnp v1.2.0 // indirect - github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.10.2-0.20230629143123-2d3edc552442 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blocksutil v0.0.1 // indirect github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect - github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect + github.com/ipfs/go-unixfs v0.4.5 // indirect github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect @@ -102,9 +95,6 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-libp2p-gostream v0.6.0 // indirect github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect - github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.7.0 // indirect - github.com/libp2p/go-mplex v0.7.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect @@ -118,7 +108,6 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/go-server-timing v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect diff --git a/go.sum b/go.sum index c2ba7eb..149a6f9 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= -github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= -github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -78,8 +76,6 @@ github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/ github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.0 h1:gh8fMGz0rlOv/1WmRZm7OgncIOTsAj21iNJot48omJQ= -github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f/go.mod h1:+If3s2VxyjZn+KGGZIoRXBDSFQ9xL404JBJGf4WhEj0= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -121,8 +117,8 @@ github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNdofHZoGPjfNaAo5Q= github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= -github.com/filecoin-project/lassie v0.12.2-0.20230614045620-19b6e938241c h1:2E1hw30L6zKaHsNnqoqXc48A3EDJ/afnGMPFna8c4h0= -github.com/filecoin-project/lassie v0.12.2-0.20230614045620-19b6e938241c/go.mod h1:kbO6Ljk/2ug6CLJuHsgsd4c6EVrAmnVQKFAktmETSI4= +github.com/filecoin-project/lassie v0.14.4-0.20230811073202-26954e00718d h1:QC5m8PZxdJHBAqIp+XIMT8srIZnBpS2y9mFXyWQBIXU= +github.com/filecoin-project/lassie v0.14.4-0.20230811073202-26954e00718d/go.mod h1:4zIWVE9FSeGbx7xQTd9y+vFVt4UIdBxfVxnj4RCI9gM= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= @@ -158,8 +154,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= -github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -187,7 +181,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= @@ -230,16 +223,12 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= -github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.10.2-0.20230629143123-2d3edc552442 h1:SGbw381zt6c1VFf3QCBaJ+eVJ4AwD9fPaFKFp9U9Apk= -github.com/ipfs/boxo v0.10.2-0.20230629143123-2d3edc552442/go.mod h1:1qgKq45mPRCxf4ZPoJV2lnXxyxucigILMJOrQrVivv8= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -275,8 +264,6 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= -github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= @@ -314,10 +301,10 @@ github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= -github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= +github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= github.com/ipfs/go-unixfsnode v1.7.3 h1:giAxFq7CxAm2Z8h8yFAD7TOQUpf5XG7a2xrR143ci4Y= github.com/ipfs/go-unixfsnode v1.7.3/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= @@ -327,8 +314,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 h1:QAI/Ridj0+foHD6epbxmB4ugxz9B4vmNdYSmQLGa05E= github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:odxGcpiQZLzP5+yGu84Ljo8y3EzCvNAQKEodHNsHLXA= -github.com/ipld/go-ipld-prime v0.21.1-0.20230810111002-bdf990edcdeb h1:frdDF5813yKhOaEHAdEhgArLVk+uBQwygVtfg2jAGR4= -github.com/ipld/go-ipld-prime v0.21.1-0.20230810111002-bdf990edcdeb/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= +github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de h1:N6Wfk6dvcBjF4AJJDSmti6CkgHWZPDZ0fuqSQL+kKnU= +github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipni/go-libipni v0.3.4 h1:ZYgCE2TOZt/QJJcBZb+R63FaBLlA2suZGP2IH1fKv4A= github.com/ipni/go-libipni v0.3.4/go.mod h1:6EIUhN83pd1i6q7SCSCIuuUC3XgR7D/gjKkEnVyIQWE= @@ -390,13 +377,7 @@ github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qk github.com/libp2p/go-libp2p-gostream v0.6.0/go.mod h1:Nywu0gYZwfj7Jc91PQvbGU8dIpqbQQkjWgDuOrFaRdA= github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= -github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= -github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.7.0 h1:sirOYVD0wGWjkDwHZvinunIpaqPLBXkcnXApVHwZFGA= -github.com/libp2p/go-libp2p-routing-helpers v0.7.0/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= -github.com/libp2p/go-mplex v0.7.0 h1:BDhFZdlk5tbr0oyFq/xv/NPGfjbnrsDam1EvutpBDbY= -github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= @@ -437,8 +418,6 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE= -github.com/mitchellh/go-server-timing v1.0.1/go.mod h1:Mo6GKi9FSLwWFAMn3bqVPWe20y5ri5QGQuO9D9MCOxk= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= diff --git a/httpipfs.go b/httpipfs.go index 13536a1..27f2c85 100644 --- a/httpipfs.go +++ b/httpipfs.go @@ -9,11 +9,12 @@ import ( "sync" "time" - lassiehttp "github.com/filecoin-project/lassie/pkg/server/http" + lassiehttp "github.com/filecoin-project/lassie/pkg/httputil" + "github.com/filecoin-project/lassie/pkg/httputil/metadata" + "github.com/filecoin-project/lassie/pkg/types" lassietypes "github.com/filecoin-project/lassie/pkg/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" - "github.com/ipld/frisbii/metadata" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" mh "github.com/multiformats/go-multihash" @@ -89,7 +90,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { return } - includeDupes, err := lassiehttp.CheckFormat(req) + includeDupes, includeMeta, err := lassiehttp.CheckFormat(req) if err != nil { logError(http.StatusBadRequest, err) return @@ -116,8 +117,6 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { return } - includeMeta := req.URL.Query().Get("meta") == "eof" - if fileName == "" { fileName = fmt.Sprintf("%s%s", rootCid.String(), lassiehttp.FilenameExtCar) } @@ -130,7 +129,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName)) res.Header().Set("Accept-Ranges", lassiehttp.ResponseAcceptRangesHeader) res.Header().Set("Cache-Control", lassiehttp.ResponseCacheControlHeader) - res.Header().Set("Content-Type", lassiehttp.ResponseContentTypeHeader) + res.Header().Set("Content-Type", lassiehttp.RequestAcceptHeader) // instead of ResponseContentTypeHeader because we are doing meta=eof as well res.Header().Set("Etag", etag(rootCid, path.String(), dagScope, includeDupes)) res.Header().Set("X-Content-Type-Options", "nosniff") res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String()) @@ -174,7 +173,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { md.Error = &msg carErr = err } else { - md.Properties = &metadata.CarProperties{ + md.Properties = &types.CarProperties{ CarBytes: writer.(*checksumWriter).Count(), DataBytes: dataBytes, BlockCount: blockCount, @@ -183,17 +182,9 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) { } } - enc, err := metadata.CarMetadata{Metadata: &md}.Serialize() - if err != nil { - if carErr != nil { - carErr = err - } - } else { - if _, err := res.Write(enc); err != nil { - if carErr != nil { - carErr = err - } - } + err := metadata.CarMetadata{Metadata: &md}.Serialize(res) + if err != nil && carErr != nil { + carErr = err } } diff --git a/metadata/metadata.go b/metadata/metadata.go deleted file mode 100644 index 3a3aaf5..0000000 --- a/metadata/metadata.go +++ /dev/null @@ -1,76 +0,0 @@ -package metadata - -import ( - "fmt" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime/codec/dagjson" - bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry" - mh "github.com/multiformats/go-multihash" - - _ "embed" -) - -//go:embed metadata.ipldsch -var schema []byte - -var BindnodeRegistry = bindnoderegistry.NewRegistry() - -type CarMetadata struct { - Metadata *Metadata -} - -func (cm CarMetadata) Serialize() ([]byte, error) { - // TODO: do the same checks we do on Deserialize() - return BindnodeRegistry.TypeToBytes(&cm, dagjson.Encode) -} - -func (cm *CarMetadata) Deserialize(byts []byte) error { - cmIface, err := BindnodeRegistry.TypeFromBytes(byts, &CarMetadata{}, dagjson.Decode) - if err != nil { - return fmt.Errorf("invalid CarMetadata: %w", err) - } - cmm := cmIface.(*CarMetadata) // safe to assume type - if cmm.Metadata.Properties == nil && cmm.Metadata.Error == nil { - return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields") - } - if (cmm.Metadata.Properties == nil) == (cmm.Metadata.Error == nil) { - return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields, not both") - } - if cmm.Metadata.Properties != nil { - if _, err := mh.Decode(cmm.Metadata.Properties.ChecksumMultihash); err != nil { - return fmt.Errorf("invalid CarMetadata: checksum multihash: %w", err) - } - } - // TODO: parse and check EntityBytes format - *cm = *cmm - return nil -} - -type Metadata struct { - Request Request - Properties *CarProperties - Error *string -} - -type Request struct { - Root cid.Cid - Path *string - Scope types.DagScope - Duplicates bool - EntityBytes *string -} - -type CarProperties struct { - CarBytes int64 - DataBytes int64 - BlockCount int64 - ChecksumMultihash []byte -} - -func init() { - if err := BindnodeRegistry.RegisterType((*CarMetadata)(nil), string(schema), "CarMetadata"); err != nil { - panic(err.Error()) - } -} diff --git a/metadata/metadata.ipldsch b/metadata/metadata.ipldsch deleted file mode 100644 index ae27840..0000000 --- a/metadata/metadata.ipldsch +++ /dev/null @@ -1,31 +0,0 @@ -type CarMetadata union { - | Metadata "car-metadata/v1" -} representation keyed - -type Metadata struct { - request Request - # must contain either a properties or an error - properties optional CarProperties - error optional String -} - -type Request struct { - root &Any - path optional String - scope DagScope - duplicates Bool (rename "dups") - entityBytes optional String (rename "entity-bytes") # Must be a valid entity-bytes param: "from:to" -} - -type DagScope enum { - | all - | entity - | block -} - -type CarProperties struct { - carBytes Int (rename "car_bytes") - dataBytes Int (rename "data_bytes") - blockCount Int (rename "block_count") - checksumMultihash optional Bytes (rename "checksum") # Must be a valid multihash -} diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go deleted file mode 100644 index b7a2415..0000000 --- a/metadata/metadata_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package metadata_test - -import ( - "testing" - - "github.com/filecoin-project/lassie/pkg/types" - "github.com/ipfs/go-cid" - "github.com/ipld/frisbii/metadata" - "github.com/stretchr/testify/require" -) - -var testCid = cid.MustParse("bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4") - -func TestCarMetadataRoundtrip(t *testing.T) { - path := "/birb.mp4" - orig := metadata.CarMetadata{ - Metadata: &metadata.Metadata{ - Request: metadata.Request{ - Root: testCid, - Path: &path, - Scope: types.DagScopeAll, - Duplicates: true, - }, - Properties: &metadata.CarProperties{ - CarBytes: 202020, - DataBytes: 101010, - BlockCount: 303, - ChecksumMultihash: testCid.Hash(), - }, - }, - } - byts, err := orig.Serialize() - require.NoError(t, err) - - t.Log("metadata dag-json:", string(byts)) - - var roundtrip metadata.CarMetadata - err = roundtrip.Deserialize(byts) - require.NoError(t, err) - require.Equal(t, orig, roundtrip) - require.NotNil(t, roundtrip.Metadata) - require.Equal(t, testCid, roundtrip.Metadata.Request.Root) - require.NotNil(t, roundtrip.Metadata.Request.Path) - require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path) - require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope) - require.True(t, roundtrip.Metadata.Request.Duplicates) - require.NotNil(t, roundtrip.Metadata.Properties) - require.Nil(t, roundtrip.Metadata.Error) - require.Equal(t, int64(202020), roundtrip.Metadata.Properties.CarBytes) - require.Equal(t, int64(101010), roundtrip.Metadata.Properties.DataBytes) - require.Equal(t, int64(303), roundtrip.Metadata.Properties.BlockCount) - require.Equal(t, []byte(testCid.Hash()), roundtrip.Metadata.Properties.ChecksumMultihash) -} - -func TestCarMetadataErrorRoundtrip(t *testing.T) { - path := "/birb.mp4" - msg := "something bad happened" - orig := metadata.CarMetadata{ - Metadata: &metadata.Metadata{ - Request: metadata.Request{ - Root: testCid, - Path: &path, - Scope: types.DagScopeAll, - Duplicates: true, - }, - Error: &msg, - }, - } - byts, err := orig.Serialize() - require.NoError(t, err) - - t.Log("metadata dag-json:", string(byts)) - - var roundtrip metadata.CarMetadata - err = roundtrip.Deserialize(byts) - require.NoError(t, err) - require.Equal(t, orig, roundtrip) - require.NotNil(t, roundtrip.Metadata) - require.Equal(t, testCid, roundtrip.Metadata.Request.Root) - require.NotNil(t, roundtrip.Metadata.Request.Path) - require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path) - require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope) - require.True(t, roundtrip.Metadata.Request.Duplicates) - require.Nil(t, roundtrip.Metadata.Properties) - require.NotNil(t, roundtrip.Metadata.Error) - require.Equal(t, "something bad happened", *roundtrip.Metadata.Error) -} - -func TestBadMetadata(t *testing.T) { - testCases := []struct { - name string - byts string - err string - }{ - {"empty", `{}`, `union structure constraints for CarMetadata caused rejection: a union must have exactly one entry`}, - {"bad key", `{"not metadata":true}`, `union structure constraints for CarMetadata caused rejection: no member named "not metadata"`}, - { - "bad multihash", - `{"car-metadata/v1":{"properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"bm90IGEgbXVsdGloYXNo"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, - `invalid CarMetadata: checksum multihash:`, - }, - { - "no properties or error", - `{"car-metadata/v1":{"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, - `invalid CarMetadata: must contain either properties or error fields`, - }, - { - "both properties and error", - `{"car-metadata/v1":{"error":"something bad happened","properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"EiBd9neBCasGxUmysJN7nGza4ylHikmbsP2+nXs6BlIpvw"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, - `invalid CarMetadata: must contain either properties or error fields, not both`, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var roundtrip metadata.CarMetadata - require.ErrorContains(t, roundtrip.Deserialize([]byte(tc.byts)), tc.err) - }) - } -}