diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index dbd46074d..9bfb20263 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -47,6 +47,12 @@ jobs: # with: # go-version: ^1.21 # The Go version to download (if necessary) and use. + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Clone blobber uses: actions/checkout@v3 with: @@ -56,10 +62,9 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + mkdir -p /tmp/docker-config-${{ github.run_id }} + docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} # - name: Get changed files using defaults # id: changed-files @@ -96,6 +101,10 @@ jobs: docker tag ${BLOBBER_REGISTRY}:${TAG} ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA} docker push ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA} + - name: Clean up Docker Config + run: | + rm -rf /tmp/docker-config-${{ github.run_id }} + validator: timeout-minutes: 30 runs-on: [blobber-runner] @@ -136,11 +145,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + mkdir -p /tmp/docker-config-${{ github.run_id }} + docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Get changed files using defaults # id: changed-files @@ -178,6 +192,10 @@ jobs: docker tag ${VALIDATOR_REGISTRY}:${TAG} ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA} docker push ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA} + - name: Clean up Docker Config + run: | + rm -rf /tmp/docker-config-${{ github.run_id }} + system-tests: if: github.event_name != 'workflow_dispatch' needs: [blobber, validator] diff --git a/code/go/0chain.net/blobber/zcn.go b/code/go/0chain.net/blobber/zcn.go index 8417a81ea..e12444dcf 100644 --- a/code/go/0chain.net/blobber/zcn.go +++ b/code/go/0chain.net/blobber/zcn.go @@ -7,7 +7,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" - handleCommon "github.com/0chain/blobber/code/go/0chain.net/core/common/handler" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/gosdk/zboxcore/sdk" @@ -68,7 +67,8 @@ func registerOnChain() error { go setupWorkers(ctx) // go StartHealthCheck(ctx, common.ProviderTypeBlobber) - go handleCommon.StartHealthCheck(ctx, common.ProviderTypeBlobber) + // Health check transactions disabled - unnecessary for this deployment + // go handleCommon.StartHealthCheck(ctx, common.ProviderTypeBlobber) go startRefreshSettings(ctx) return err diff --git a/code/go/0chain.net/blobbercore/allocation/allocationchange.go b/code/go/0chain.net/blobbercore/allocation/allocationchange.go index 87669bb3d..635ba6d47 100644 --- a/code/go/0chain.net/blobbercore/allocation/allocationchange.go +++ b/code/go/0chain.net/blobbercore/allocation/allocationchange.go @@ -18,7 +18,6 @@ import ( "go.uber.org/zap" "gorm.io/gorm" - "gorm.io/gorm/clause" ) const ( @@ -32,8 +31,8 @@ const ( type AllocationChangeProcessor interface { CommitToFileStore(ctx context.Context, mut *sync.Mutex) error DeleteTempFile() error - ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, allocationRoot string, - ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) + ApplyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error GetPath() []string Marshal() (string, error) Unmarshal(string) error @@ -74,6 +73,7 @@ type AllocationChange struct { Input string `gorm:"column:input"` FilePath string `gorm:"-"` LookupHash string `gorm:"column:lookup_hash;size:64"` + AllocationID string `gorm:"-" json:"-"` datastore.ModelWithTS } @@ -156,7 +156,6 @@ func GetAllocationChanges(ctx context.Context, connectionID, allocationID, clien cc.ComputeProperties() // Load connection Obj size from memory cc.Size = GetConnectionObjSize(connectionID) - cc.Status = InProgressConnection return cc, nil } @@ -230,8 +229,8 @@ func (cc *AllocationChangeCollector) ComputeProperties() { switch change.Operation { case constants.FileOperationInsert: acp = new(UploadFileChanger) - case constants.FileOperationUpdate: - acp = new(UpdateFileChanger) + // case constants.FileOperationUpdate: + // acp = new(UpdateFileChanger) case constants.FileOperationDelete: acp = new(DeleteFileChange) case constants.FileOperationRename: @@ -254,29 +253,36 @@ func (cc *AllocationChangeCollector) ComputeProperties() { } } -func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, allocationRoot, prevAllocationRoot string, - ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - rootRef, err := cc.GetRootRef(ctx) - if err != nil { - return rootRef, err - } - if rootRef.Hash != prevAllocationRoot { - return rootRef, common.NewError("invalid_prev_root", "Invalid prev root") - } +func (cc *AllocationChangeCollector) ApplyChanges(ctx context.Context, + ts common.Timestamp, allocationVersion int64) error { + now := time.Now() + collector := reference.NewCollector(len(cc.Changes)) + timeoutctx, cancel := context.WithTimeout(ctx, time.Second*60) + defer cancel() + eg, egCtx := errgroup.WithContext(timeoutctx) + eg.SetLimit(10) for idx, change := range cc.Changes { - changeProcessor := cc.AllocationChanges[idx] - _, err := changeProcessor.ApplyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) - if err != nil { - return rootRef, err + select { + case <-egCtx.Done(): + return egCtx.Err() + default: + changeIndex := idx + eg.Go(func() error { + change.AllocationID = cc.AllocationID + changeProcessor := cc.AllocationChanges[changeIndex] + return changeProcessor.ApplyChange(ctx, ts, allocationVersion, collector) + }) } } - collector := reference.NewCollector(len(cc.Changes)) - _, err = rootRef.CalculateHash(ctx, true, collector) + err := eg.Wait() if err != nil { - return rootRef, err + return err } - err = collector.Finalize(ctx) - return rootRef, err + elapsedApplyChanges := time.Since(now) + err = collector.Finalize(ctx, cc.AllocationID, allocationVersion) + elapsedFinalize := time.Since(now) - elapsedApplyChanges + logging.Logger.Info("ApplyChanges", zap.String("allocation_id", cc.AllocationID), zap.Duration("apply_changes", elapsedApplyChanges), zap.Duration("finalize", elapsedFinalize), zap.Int("changes", len(cc.Changes))) + return err } func (a *AllocationChangeCollector) CommitToFileStore(ctx context.Context) error { @@ -304,154 +310,112 @@ func (a *AllocationChangeCollector) DeleteChanges(ctx context.Context) { } type Result struct { - Id string - ValidationRoot string - PrevValidationRoot string - ThumbnailHash string - PrevThumbnailHash string - FilestoreVersion int + LookupHash string } // TODO: Need to speed up this function -func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context) error { +func (a *AllocationChangeCollector) MoveToFilestore(ctx context.Context, allocationVersion int64) error { logging.Logger.Info("Move to filestore", zap.String("allocation_id", a.AllocationID)) - err := deleteFromFileStore(ctx, a.AllocationID) + var ( + refs []*reference.Ref + useRefCache bool + deletedRefs []*reference.Ref + ) + refCache := reference.GetRefCache(a.AllocationID) + defer reference.DeleteRefCache(a.AllocationID) + if refCache != nil && refCache.AllocationVersion == allocationVersion { + useRefCache = true + refs = refCache.CreatedRefs + deletedRefs = refCache.DeletedRefs + } else if refCache != nil && refCache.AllocationVersion != allocationVersion { + logging.Logger.Error("Ref cache is not valid", zap.String("allocation_id", a.AllocationID), zap.String("ref_cache_version", fmt.Sprintf("%d", refCache.AllocationVersion)), zap.String("allocation_version", fmt.Sprintf("%d", allocationVersion))) + } else { + logging.Logger.Error("Ref cache is nil", zap.String("allocation_id", a.AllocationID)) + } + err := deleteFromFileStore(a.AllocationID, deletedRefs, useRefCache) if err != nil { return err } - var refs []*Result - limitCh := make(chan struct{}, 10) - wg := &sync.WaitGroup{} - err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + limitCh := make(chan struct{}, 12) + wg := &sync.WaitGroup{} + if !useRefCache { tx := datastore.GetStore().GetTransaction(ctx) - err := tx.Model(&reference.Ref{}).Clauses(clause.Locking{Strength: "NO KEY UPDATE"}).Select("id", "validation_root", "thumbnail_hash", "prev_validation_root", "prev_thumbnail_hash", "filestore_version").Where("allocation_id=? AND is_precommit=? AND type=?", a.AllocationID, true, reference.FILE). - FindInBatches(&refs, 50, func(tx *gorm.DB, batch int) error { - - for _, ref := range refs { - - var count int64 - if ref.PrevValidationRoot != "" { - tx.Model(&reference.Ref{}). - Where("allocation_id=? AND validation_root=?", a.AllocationID, ref.PrevValidationRoot). - Count(&count) - } - - limitCh <- struct{}{} - wg.Add(1) - - go func(ref *Result) { - defer func() { - <-limitCh - wg.Done() - }() - - if count == 0 && ref.PrevValidationRoot != "" { - err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevValidationRoot, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", ref.ValidationRoot)) - } - } - err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ValidationRoot, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error()), - zap.String("validation_root", ref.ValidationRoot)) - } - - if ref.ThumbnailHash != "" && ref.ThumbnailHash != ref.PrevThumbnailHash { - if ref.PrevThumbnailHash != "" { - err := filestore.GetFileStore().DeleteFromFilestore(a.AllocationID, ref.PrevThumbnailHash, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail file: %s", err.Error()), - zap.String("thumbnail_hash", ref.ThumbnailHash)) - } - } - err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, ref.ThumbnailHash, ref.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while moving thumbnail file: %s", err.Error()), - zap.String("thumbnail_hash", ref.ThumbnailHash)) - } - } - - }(ref) - } - - return nil - }).Error - - wg.Wait() - + err = tx.Model(&reference.Ref{}).Select("lookup_hash").Where("allocation_id=? AND allocation_version=? AND type=?", a.AllocationID, allocationVersion, reference.FILE).Find(&refs).Error if err != nil { - logging.Logger.Error("Error while moving to filestore", zap.Error(err)) + logging.Logger.Error("Error while moving files to filestore", zap.Error(err)) return err } + } - return tx.Exec("UPDATE reference_objects SET is_precommit=?, prev_validation_root=validation_root, prev_thumbnail_hash=thumbnail_hash WHERE allocation_id=? AND is_precommit=? AND deleted_at is NULL", false, a.AllocationID, true).Error - }) - return err + for _, ref := range refs { + + limitCh <- struct{}{} + wg.Add(1) + refLookupHash := ref.LookupHash + go func() { + defer func() { + <-limitCh + wg.Done() + }() + err := filestore.GetFileStore().MoveToFilestore(a.AllocationID, refLookupHash, filestore.VERSION) + if err != nil { + logging.Logger.Error(fmt.Sprintf("Error while moving file: %s", err.Error())) + } + + }() + } + + wg.Wait() + err = filestore.GetFileStore().DeletePreCommitDir(a.AllocationID) + if err != nil { + logging.Logger.Error("Error while deleting precommit dir", zap.Error(err)) + return err + } + return nil } -func deleteFromFileStore(ctx context.Context, allocationID string) error { - limitCh := make(chan struct{}, 10) +func deleteFromFileStore(allocationID string, deletedRefs []*reference.Ref, useRefCache bool) error { + limitCh := make(chan struct{}, 12) wg := &sync.WaitGroup{} - var results []Result + var results []*reference.Ref + if useRefCache { + results = deletedRefs + } return datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { db := datastore.GetStore().GetTransaction(ctx) + if !useRefCache { + err := db.Model(&reference.Ref{}).Unscoped().Select("lookup_hash"). + Where("allocation_id=? AND type=? AND deleted_at is not NULL", allocationID, reference.FILE). + Find(&results).Error + if err != nil && err != gorm.ErrRecordNotFound { + logging.Logger.Error("DeleteFromFileStore", zap.Error(err)) + return err + } + } - err := db.Model(&reference.Ref{}).Unscoped().Select("id", "validation_root", "thumbnail_hash", "filestore_version"). - Where("allocation_id=? AND is_precommit=? AND type=? AND deleted_at is not NULL", allocationID, true, reference.FILE). - FindInBatches(&results, 100, func(tx *gorm.DB, batch int) error { - - for _, res := range results { - var count int64 - tx.Model(&reference.Ref{}). - Where("allocation_id=? AND validation_root=?", allocationID, res.ValidationRoot). - Count(&count) - - if count != 0 && res.ThumbnailHash == "" { - continue - } - - limitCh <- struct{}{} - wg.Add(1) - - go func(res Result, count int64) { - defer func() { - <-limitCh - wg.Done() - }() - - if count == 0 { - err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ValidationRoot, - res.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), - zap.String("validation_root", res.ValidationRoot)) - } - } - - if res.ThumbnailHash != "" { - err := filestore.GetFileStore().DeleteFromFilestore(allocationID, res.ThumbnailHash, res.FilestoreVersion) - if err != nil { - logging.Logger.Error(fmt.Sprintf("Error while deleting thumbnail: %s", err.Error()), - zap.String("thumbnail", res.ThumbnailHash)) - } - } - - }(res, count) - + for _, res := range results { + limitCh <- struct{}{} + wg.Add(1) + resLookupHash := res.LookupHash + go func() { + defer func() { + <-limitCh + wg.Done() + }() + + err := filestore.GetFileStore().DeleteFromFilestore(allocationID, resLookupHash, + filestore.VERSION) + if err != nil { + logging.Logger.Error(fmt.Sprintf("Error while deleting file: %s", err.Error()), + zap.String("lookup_hash", resLookupHash)) } - return nil - }).Error + }() - wg.Wait() - if err != nil && err != gorm.ErrRecordNotFound { - logging.Logger.Error("DeleteFromFileStore", zap.Error(err)) - return err } + wg.Wait() return db.Model(&reference.Ref{}).Unscoped(). Delete(&reference.Ref{}, diff --git a/code/go/0chain.net/blobbercore/allocation/common_test.go b/code/go/0chain.net/blobbercore/allocation/common_test.go index 372cbaee0..9cb6429fb 100644 --- a/code/go/0chain.net/blobbercore/allocation/common_test.go +++ b/code/go/0chain.net/blobbercore/allocation/common_test.go @@ -21,11 +21,9 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string, b := bytes.NewBuffer(make([]byte, 0)) n, _ := io.Copy(b, infile) return &filestore.FileOutputData{ - Name: fileData.Name, - Path: fileData.Path, - FixedMerkleRoot: "", - ValidationRoot: fileData.ValidationRoot, - Size: n, + Name: fileData.Name, + Path: fileData.Path, + Size: n, }, nil } diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index ef7eac62f..63fa7b764 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -68,10 +68,10 @@ func SaveFileChanger(connectionID string, fileChanger *BaseFileChanger) error { return common.NewError("connection_not_found", "connection not found") } connectionObj.lock.Lock() - if connectionObj.changes[fileChanger.PathHash] == nil { + if connectionObj.changes[fileChanger.LookupHash] == nil { return common.NewError("connection_change_not_found", "connection change not found") } - connectionObj.changes[fileChanger.PathHash].baseChanger = fileChanger + connectionObj.changes[fileChanger.LookupHash].baseChanger = fileChanger connectionObj.lock.Unlock() return nil } @@ -190,7 +190,7 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string if err != nil { return saveChange, err } - hasher := filestore.GetNewCommitHasher(contentSize) + hasher := filestore.NewCommitHasher(contentSize) change.hasher = hasher change.seqPQ = seqpriorityqueue.NewSeqPriorityQueue(contentSize) go hasher.Start(connectionObj.ctx, connectionID, connectionObj.AllocationID, fileName, pathHash, change.seqPQ) @@ -211,6 +211,12 @@ func SaveFileChange(ctx context.Context, connectionID, pathHash, fileName string DataBytes: dataWritten, }, contentSize) if addSize != 0 { + //check if reference exists and get the size + existingSize, err := reference.GetObjectSizeByLookupHash(ctx, pathHash) + if err != nil { + return saveChange, err + } + addSize -= existingSize UpdateConnectionObjSize(connectionID, addSize) } } else { @@ -240,12 +246,18 @@ func GetHasher(connectionID, pathHash string) *filestore.CommitHasher { // DeleteConnectionObjEntry remove the connectionID entry from map // If the given connectionID is not present, then it is no-op. func DeleteConnectionObjEntry(connectionID string) { + logging.Logger.Info("DeleteConnectionObjEntry", zap.String("connection_id", connectionID)) connectionObjMutex.Lock() connectionObj, ok := connectionProcessor[connectionID] if ok { connectionObj.cnclCtx() + for _, change := range connectionObj.changes { + if change.seqPQ != nil { + change.seqPQ.Done(seqpriorityqueue.UploadData{}, 1) + } + } + delete(connectionProcessor, connectionID) } - delete(connectionProcessor, connectionID) connectionObjMutex.Unlock() } @@ -256,6 +268,7 @@ func cleanConnectionObj() { for connectionID, connectionObj := range connectionProcessor { diff := time.Since(connectionObj.UpdatedAt) if diff >= ConnectionObjTimeout { + logging.Logger.Info("cleanConnectionObj", zap.String("connection_id", connectionID), zap.Duration("diff", diff)) // Stop the context and hash worker connectionObj.cnclCtx() for _, change := range connectionObj.changes { diff --git a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go index 9f849df4e..f119bf33a 100644 --- a/code/go/0chain.net/blobbercore/allocation/copyfilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/copyfilechange.go @@ -2,16 +2,17 @@ package allocation import ( "context" + "database/sql" "encoding/json" - "fmt" "path/filepath" "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) type CopyFileChange struct { @@ -19,122 +20,77 @@ type CopyFileChange struct { AllocationID string `json:"allocation_id"` SrcPath string `json:"path"` DestPath string `json:"dest_path"` + Type string `json:"type"` + CustomMeta string `json:"custom_meta"` } func (rf *CopyFileChange) DeleteTempFile() error { return nil } -func (rf *CopyFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - - totalRefs, err := reference.CountRefs(ctx, rf.AllocationID) - if err != nil { - return nil, err - } - - if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs { - return nil, common.NewErrorf("max_alloc_dir_files_reached", - "maximum files and directories already reached: %v", err) - } - - srcRef, err := rootRef.GetSrcPath(rf.SrcPath) - if err != nil { - return nil, err - } +func (rf *CopyFileChange) ApplyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true + var ( + srcRef *reference.Ref + err error + ) - dirRef := rootRef - fields, err := common.GetPathFields(rf.DestPath) - if err != nil { - return nil, err - } - - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - if child.Type == reference.DIRECTORY { - child.HashToBeComputed = true - dirRef = child - dirRef.UpdatedAt = ts - found = true - } else { - return nil, common.NewError("invalid_path", - fmt.Sprintf("%s is of file type", child.Path)) - } - } + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + srcRef, err = reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash) + if err != nil { + return err } - - if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir { - return nil, common.NewErrorf("max_objects_in_dir_reached", - "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir) + exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath) + if err != nil { + return err + } + if exist { + return common.NewError("invalid_reference_path", "file already exists") } - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = rf.AllocationID - newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in file ID meta", newRef.Path)) + rf.Type = srcRef.Type + if srcRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, srcRef.ID) + if err != nil { + return err + } + if !isEmpty { + return common.NewError("invalid_reference_path", "directory is not empty") } - newRef.FileID = fileID - newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/")) - newRef.Name = fields[i] - newRef.HashToBeComputed = true - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - dirRef.AddChild(newRef) - dirRef = newRef } + return nil + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return err } - _, err = rf.processCopyRefs(ctx, srcRef, dirRef, allocationRoot, ts, fileIDMeta) + parentDir, err := reference.Mkdir(ctx, rf.AllocationID, filepath.Dir(rf.DestPath), allocationVersion, ts, collector) if err != nil { - return nil, err + return err } - return rootRef, err -} - -func (rf *CopyFileChange) processCopyRefs( - ctx context.Context, srcRef, destRef *reference.Ref, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string, -) ( - fileRefs []*reference.Ref, err error, -) { - - newRef := *srcRef - newRef.ID = 0 - newRef.Path = filepath.Join(destRef.Path, srcRef.Name) - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID - newRef.ParentPath = destRef.Path - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - destRef.AddChild(&newRef) - if newRef.Type == reference.DIRECTORY { - for _, childRef := range srcRef.Children { - fRefs, err := rf.processCopyRefs(ctx, childRef, &newRef, allocationRoot, ts, fileIDMeta) - if err != nil { - return nil, err - } - fileRefs = append(fileRefs, fRefs...) - } - } else { - fileRefs = append(fileRefs, &newRef) + srcRef.ID = 0 + srcRef.ParentID = &parentDir.ID + srcRef.Path = rf.DestPath + srcRef.LookupHash = destLookUpHash + srcRef.CreatedAt = ts + srcRef.UpdatedAt = ts + srcRef.ParentPath = filepath.Dir(rf.DestPath) + srcRef.Name = filepath.Base(rf.DestPath) + srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/")) + srcRef.FileMetaHash = encryption.FastHash(srcRef.GetFileHashData()) + if rf.CustomMeta != "" { + srcRef.CustomMeta = rf.CustomMeta } + srcRef.AllocationVersion = allocationVersion + collector.CreateRefRecord(srcRef) - return + return nil } func (rf *CopyFileChange) Marshal() (string, error) { @@ -151,7 +107,12 @@ func (rf *CopyFileChange) Unmarshal(input string) error { } func (rf *CopyFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - return nil + if rf.Type == reference.DIRECTORY { + return nil + } + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) + return filestore.GetFileStore().CopyFile(rf.AllocationID, srcLookUpHash, destLookUpHash) } func (rf *CopyFileChange) GetPath() []string { diff --git a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go index 4caf0d1e3..18c4c7a31 100644 --- a/code/go/0chain.net/blobbercore/allocation/deletefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/deletefilechange.go @@ -3,7 +3,6 @@ package allocation import ( "context" "encoding/json" - "path/filepath" "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -20,18 +19,15 @@ type DeleteFileChange struct { Name string `json:"name"` Path string `json:"path"` Size int64 `json:"size"` - Hash string `json:"hash"` + LookupHash string `json:"lookup_hash"` + Type string `json:"type"` } -func (nf *DeleteFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { - - err := reference.DeleteObject(ctx, rootRef, nf.AllocationID, filepath.Clean(nf.Path), ts) - if err != nil { - return nil, err - } - - return nil, nil +func (nf *DeleteFileChange) ApplyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + collector.LockTransaction() + defer collector.UnlockTransaction() + return reference.DeleteObject(ctx, nf.AllocationID, nf.LookupHash, nf.Type, ts, allocationVersion, collector) } func (nf *DeleteFileChange) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/entity.go b/code/go/0chain.net/blobbercore/allocation/entity.go index 98e7db8c7..fdb19dadf 100644 --- a/code/go/0chain.net/blobbercore/allocation/entity.go +++ b/code/go/0chain.net/blobbercore/allocation/entity.go @@ -44,23 +44,26 @@ type Allocation struct { Tx string `gorm:"column:tx;size:64;not null;unique;index:idx_unique_allocations_tx,unique"` TotalSize int64 `gorm:"column:size;not null;default:0"` UsedSize int64 `gorm:"column:used_size;not null;default:0"` + PrevUsedSize int64 `gorm:"column:prev_used_size;not null;default:0"` OwnerID string `gorm:"column:owner_id;size:64;not null"` OwnerPublicKey string `gorm:"column:owner_public_key;size:512;not null"` RepairerID string `gorm:"column:repairer_id;size:64;not null"` Expiration common.Timestamp `gorm:"column:expiration_date;not null"` // AllocationRoot allcation_root of last write_marker - AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` - FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` - BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` - BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` - LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` - LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"` - IsRedeemRequired bool `gorm:"column:is_redeem_required"` - TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` - StartTime common.Timestamp `gorm:"column:start_time;not null"` + AllocationRoot string `gorm:"column:allocation_root;size:64;not null;default:''"` + FileMetaRoot string `gorm:"column:file_meta_root;size:64;not null;default:''"` + BlobberSize int64 `gorm:"column:blobber_size;not null;default:0"` + BlobberSizeUsed int64 `gorm:"column:blobber_size_used;not null;default:0"` + PrevBlobberSizeUsed int64 `gorm:"column:prev_blobber_size_used;not null;default:0"` + LatestRedeemedWM string `gorm:"column:latest_redeemed_write_marker;size:64"` + LastRedeemedSeq int64 `gorm:"column:last_redeemed_sequence;default:0"` + IsRedeemRequired bool `gorm:"column:is_redeem_required"` + TimeUnit time.Duration `gorm:"column:time_unit;not null;default:172800000000000"` + StartTime common.Timestamp `gorm:"column:start_time;not null"` // Ending and cleaning - CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` - Finalized bool `gorm:"column:finalized;not null;default:false"` + CleanedUp bool `gorm:"column:cleaned_up;not null;default:false"` + Finalized bool `gorm:"column:finalized;not null;default:false"` + AllocationVersion int64 `gorm:"column:allocation_version;not null;default:0"` // FileOptions to define file restrictions on an allocation for third-parties // default 00000000 for all crud operations suggesting only owner has the below listed abilities. diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go index 9c93a5bb2..f89725e14 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_base.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_base.go @@ -8,10 +8,11 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) -// swagger:model BaseFileChanger +// swagger:model BaseFileChanger // BaseFileChanger base file change processor type BaseFileChanger struct { //client side: unmarshal them from 'updateMeta'/'uploadMeta' @@ -34,16 +35,12 @@ type BaseFileChanger struct { //client side: MimeType string `json:"mimetype,omitempty"` //client side: - //client side: - FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"` //server side: update them by ChangeProcessor - AllocationID string `json:"allocation_id"` - //client side: - ValidationRootSignature string `json:"validation_root_signature,omitempty"` - //client side: - ValidationRoot string `json:"validation_root,omitempty"` - Size int64 `json:"size"` + AllocationID string `json:"allocation_id"` + DataHash string `json:"data_hash"` + DataHashSignature string `json:"data_hash_signature"` + Size int64 `json:"size"` //server side: ThumbnailHash string `json:"thumbnail_content_hash,omitempty"` ThumbnailSize int64 `json:"thumbnail_size"` @@ -60,16 +57,15 @@ type BaseFileChanger struct { ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of CompactMerkleTree ChunkHash string `json:"chunk_hash,omitempty"` UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to - PathHash string `json:"-"` // hash of path + CanUpdate bool `json:"can_update"` // can file be updated or not + LookupHash string `json:"-"` // hash of allocationID+path } // swagger:model UploadResult type UploadResult struct { - Filename string `json:"filename"` - Size int64 `json:"size"` - Hash string `json:"hash"` - ValidationRoot string `json:"validation_root"` - FixedMerkleRoot string `json:"fixed_merkle_root"` + Filename string `json:"filename"` + Size int64 `json:"size"` + Hash string `json:"hash"` // UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. UploadLength int64 `json:"upload_length"` @@ -108,7 +104,8 @@ func (fc *BaseFileChanger) DeleteTempFile() error { fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.Filename fileInputData.Path = fc.Path - fileInputData.ValidationRoot = fc.ValidationRoot + fileInputData.DataHash = fc.DataHash + fileInputData.LookupHash = fc.LookupHash err := filestore.GetFileStore().DeleteTempFile(fc.AllocationID, fc.ConnectionID, fileInputData) if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} @@ -122,11 +119,15 @@ func (fc *BaseFileChanger) DeleteTempFile() error { func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { + if fc.LookupHash == "" { + fc.LookupHash = reference.GetReferenceLookup(fc.AllocationID, fc.Path) + } if fc.ThumbnailSize > 0 { fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.ThumbnailFilename fileInputData.Path = fc.Path fileInputData.ThumbnailHash = fc.ThumbnailHash + fileInputData.LookupHash = fc.LookupHash fileInputData.ChunkSize = fc.ChunkSize fileInputData.IsThumbnail = true _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) @@ -137,13 +138,14 @@ func (fc *BaseFileChanger) CommitToFileStore(ctx context.Context, mut *sync.Mute fileInputData := &filestore.FileInputData{} fileInputData.Name = fc.Filename fileInputData.Path = fc.Path - fileInputData.ValidationRoot = fc.ValidationRoot - fileInputData.FixedMerkleRoot = fc.FixedMerkleRoot + fileInputData.DataHash = fc.DataHash + fileInputData.LookupHash = fc.LookupHash fileInputData.ChunkSize = fc.ChunkSize fileInputData.Size = fc.Size - fileInputData.Hasher = GetHasher(fc.ConnectionID, encryption.Hash(fc.Path)) + fileInputData.Hasher = GetHasher(fc.ConnectionID, fc.LookupHash) if fileInputData.Hasher == nil { - return common.NewError("invalid_parameters", "Invalid parameters. Error getting hasher for commit.") + logging.Logger.Error("CommitToFileStore: Error getting hasher", zap.String("connection_id", fc.ConnectionID), zap.String("lookup_hash", fc.LookupHash)) + return common.NewError("file_store_error", "Error getting hasher") } _, err := filestore.GetFileStore().CommitWrite(fc.AllocationID, fc.ConnectionID, fileInputData) if err != nil { diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go index e1e1282db..b8faf2677 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_update.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_update.go @@ -74,19 +74,14 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.HashToBeComputed = true nf.deleteHash = make(map[string]int) - if fileRef.ValidationRoot != "" && fileRef.ValidationRoot != nf.ValidationRoot { - nf.deleteHash[fileRef.ValidationRoot] = fileRef.FilestoreVersion - } - fileRef.ActualFileHash = nf.ActualHash fileRef.ActualFileHashSignature = nf.ActualFileHashSignature fileRef.ActualFileSize = nf.ActualSize fileRef.MimeType = nf.MimeType - fileRef.ValidationRootSignature = nf.ValidationRootSignature - fileRef.ValidationRoot = nf.ValidationRoot fileRef.CustomMeta = nf.CustomMeta - fileRef.FixedMerkleRoot = nf.FixedMerkleRoot - fileRef.AllocationRoot = allocationRoot + fileRef.DataHash = nf.DataHash + fileRef.DataHashSignature = nf.DataHashSignature + fileRef.LookupHash = nf.LookupHash fileRef.Size = nf.Size fileRef.ThumbnailHash = nf.ThumbnailHash fileRef.ThumbnailSize = nf.ThumbnailSize @@ -95,7 +90,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, rootRef *reference fileRef.EncryptedKey = nf.EncryptedKey fileRef.EncryptedKeyPoint = nf.EncryptedKeyPoint fileRef.ChunkSize = nf.ChunkSize - fileRef.IsPrecommit = true fileRef.FilestoreVersion = filestore.VERSION return rootRef, nil diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go index 3ad14d9db..50e11b0f4 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload.go @@ -2,18 +2,24 @@ package allocation import ( "context" + "database/sql" "encoding/json" - "fmt" + "math" "path/filepath" "strings" + "time" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" + "go.uber.org/zap" + "gorm.io/gorm" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" ) // swagger:model UploadFileChanger @@ -23,93 +29,27 @@ type UploadFileChanger struct { } // ApplyChange update references, and create a new FileRef -func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { +func (nf *UploadFileChanger) applyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { - totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) - if err != nil { - return nil, err - } - - if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs { - return nil, common.NewErrorf("max_alloc_dir_files_reached", - "maximum files and directories already reached: %v", err) - } - - fields, err := common.GetPathFields(filepath.Dir(nf.Path)) - if err != nil { - return nil, err - } - if rootRef.CreatedAt == 0 { - rootRef.CreatedAt = ts - } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - - dirRef := rootRef - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - if child.Type != reference.DIRECTORY { - return nil, common.NewError("invalid_reference_path", "Reference path has invalid ref type") - } - dirRef = child - dirRef.UpdatedAt = ts - dirRef.HashToBeComputed = true - found = true - } - } - - if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir { - return nil, common.NewErrorf("max_objects_in_dir_reached", - "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir) - } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = dirRef.AllocationID - newRef.Path = "/" + strings.Join(fields[:i+1], "/") - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID - newRef.ParentPath = "/" + strings.Join(fields[:i], "/") - newRef.Name = fields[i] - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - - dirRef.AddChild(newRef) - dirRef = newRef - } - } - - for _, child := range dirRef.Children { - if child.Name == nf.Filename { - return nil, common.NewError("duplicate_file", "File already exists") - } + if nf.AllocationID == "" { + return common.NewError("invalid_parameter", "allocation_id is required") } - + now := time.Now() + parentPath := filepath.Dir(nf.Path) + nf.LookupHash = reference.GetReferenceLookup(nf.AllocationID, nf.Path) newFile := &reference.Ref{ ActualFileHash: nf.ActualHash, ActualFileHashSignature: nf.ActualFileHashSignature, ActualFileSize: nf.ActualSize, - AllocationID: dirRef.AllocationID, - ValidationRoot: nf.ValidationRoot, - ValidationRootSignature: nf.ValidationRootSignature, + AllocationID: nf.AllocationID, CustomMeta: nf.CustomMeta, - FixedMerkleRoot: nf.FixedMerkleRoot, Name: nf.Filename, Path: nf.Path, - ParentPath: dirRef.Path, + ParentPath: parentPath, Type: reference.FILE, Size: nf.Size, MimeType: nf.MimeType, - AllocationRoot: allocationRoot, ThumbnailHash: nf.ThumbnailHash, ThumbnailSize: nf.ThumbnailSize, ActualThumbnailHash: nf.ActualThumbnailHash, @@ -119,21 +59,63 @@ func (nf *UploadFileChanger) applyChange(ctx context.Context, rootRef *reference ChunkSize: nf.ChunkSize, CreatedAt: ts, UpdatedAt: ts, - HashToBeComputed: true, - IsPrecommit: true, + LookupHash: nf.LookupHash, + DataHash: nf.DataHash, + DataHashSignature: nf.DataHashSignature, + PathLevel: len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")), + NumBlocks: int64(math.Ceil(float64(nf.Size*1.0) / float64(nf.ChunkSize))), FilestoreVersion: filestore.VERSION, + AllocationVersion: allocationVersion, + NumUpdates: 1, } - - fileID, ok := fileIDMeta[newFile.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newFile.Path)) + newFile.FileMetaHash = encryption.FastHash(newFile.GetFileMetaHashData()) + elapsedNewFile := time.Since(now) + // find if ref exists + var refResult struct { + ID int64 + Type string + NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` } - newFile.FileID = fileID - dirRef.AddChild(newFile) + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + tx := datastore.GetStore().GetTransaction(ctx) + return tx.Model(&reference.Ref{}).Select("id", "type", "num_of_updates").Where("lookup_hash = ?", newFile.LookupHash).Take(&refResult).Error + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil && err != gorm.ErrRecordNotFound { + return err + } - return rootRef, nil + if refResult.ID > 0 { + if !nf.CanUpdate { + return common.NewError("prohibited_allocation_file_options", "Cannot update data in this allocation.") + } + if refResult.Type != reference.FILE { + return common.NewError("invalid_reference_path", "Directory already exists with the same path") + } + deleteRecord := &reference.Ref{ + ID: refResult.ID, + LookupHash: newFile.LookupHash, + Type: refResult.Type, + } + collector.DeleteRefRecord(deleteRecord) + newFile.NumUpdates = refResult.NumUpdates + 1 + } + elapsedNewFileRecord := time.Since(now) - elapsedNewFile + // get parent id + parent := filepath.Dir(nf.Path) + // create or get parent directory + parentRef, err := reference.Mkdir(ctx, nf.AllocationID, parent, allocationVersion, ts, collector) + if err != nil { + return err + } + elapsedMkdir := time.Since(now) - elapsedNewFileRecord - elapsedNewFile + newFile.ParentID = &parentRef.ID + collector.CreateRefRecord(newFile) + elapsedCreateRefRecord := time.Since(now) - elapsedMkdir - elapsedNewFileRecord - elapsedNewFile + logging.Logger.Info("UploadFileChanger", zap.Duration("elapsedNewFile", elapsedNewFile), zap.Duration("elapsedNewFileRecord", elapsedNewFileRecord), zap.Duration("elapsedMkdir", elapsedMkdir), zap.Duration("elapsedCreateRefRecord", elapsedCreateRefRecord), zap.Duration("elapsedTotal", time.Since(now))) + return err } // Marshal marshal and change to persistent to postgres diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go index 5a41a40aa..7cb2fea21 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_main.go @@ -5,11 +5,12 @@ package allocation import ( "context" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -func (nf *UploadFileChanger) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - return nf.applyChange(ctx, rootRef, change, allocationRoot, ts, fileIDMeta) +func (nf *UploadFileChanger) ApplyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + return nf.applyChange(ctx, ts, allocationVersion, collector) } diff --git a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go index 4ea4e3bce..dced40d75 100644 --- a/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go +++ b/code/go/0chain.net/blobbercore/allocation/file_changer_upload_test.go @@ -106,7 +106,7 @@ func TestBlobberCore_FileChangerUpload(t *testing.T) { changes: make(map[string]*ConnectionChange), } connectionProcessor["connection_id"].changes[pathHash] = &ConnectionChange{ - hasher: filestore.GetNewCommitHasher(2310), + hasher: filestore.NewCommitHasher(2310), } change := &UploadFileChanger{ BaseFileChanger: BaseFileChanger{ diff --git a/code/go/0chain.net/blobbercore/allocation/movefilechange.go b/code/go/0chain.net/blobbercore/allocation/movefilechange.go index 7d3aa9c3e..52d587029 100644 --- a/code/go/0chain.net/blobbercore/allocation/movefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/movefilechange.go @@ -2,15 +2,17 @@ package allocation import ( "context" + "database/sql" "encoding/json" - "fmt" "path/filepath" "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" ) type MoveFileChange struct { @@ -18,140 +20,74 @@ type MoveFileChange struct { AllocationID string `json:"allocation_id"` SrcPath string `json:"path"` DestPath string `json:"dest_path"` + Type string `json:"type"` } func (rf *MoveFileChange) DeleteTempFile() error { return nil } -func (rf *MoveFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - - srcRef, err := rootRef.GetSrcPath(rf.SrcPath) - if err != nil { - return nil, err - } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - - srcParentPath, srcFileName := filepath.Split(rf.SrcPath) - srcFields, err := common.GetPathFields(srcParentPath) - if err != nil { - return nil, err - } - dirRef := rootRef - for i := 0; i < len(srcFields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == srcFields[i] { - dirRef = child - found = true - dirRef.HashToBeComputed = true - break - } +func (rf *MoveFileChange) ApplyChange(cctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + var err error + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) + var srcRef *reference.Ref + err = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + srcRef, err = reference.GetReferenceByLookupHash(ctx, rf.AllocationID, srcLookUpHash) + if err != nil { + return err } - if !found { - return nil, common.NewError("invalid_reference_path", - fmt.Sprintf("path %s does not exist", strings.Join(srcFields[:i+1], "/"))) + exist, err := reference.IsRefExist(ctx, rf.AllocationID, rf.DestPath) + if err != nil { + return err } - } - - var removed bool - for i, child := range dirRef.Children { - if child.Name == srcFileName { - dirRef.RemoveChild(i) - removed = true - break + if exist { + return common.NewError("invalid_reference_path", "file already exists") } - } - if !removed { - return nil, common.NewError("incomplete_move", - "move operation rejected as it cannot be completed") - } - - dirRef = rootRef - fields, err := common.GetPathFields(rf.DestPath) - if err != nil { - return nil, err - } - - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - if child.Type == reference.DIRECTORY { - child.HashToBeComputed = true - dirRef = child - dirRef.UpdatedAt = ts - found = true - } else { - return nil, common.NewError("invalid_path", - fmt.Sprintf("%s is of file type", child.Path)) - } + if srcRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(cctx, srcRef.ID) + if err != nil { + return err } - } - - if len(dirRef.Children) >= config.Configuration.MaxObjectsInDir { - return nil, common.NewErrorf("max_objects_in_dir_reached", - "maximum objects in directory %s reached: %v", dirRef.Path, config.Configuration.MaxObjectsInDir) - } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = rf.AllocationID - newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - newRef.ParentPath = filepath.Join("/", strings.Join(fields[:i], "/")) - newRef.Name = fields[i] - newRef.HashToBeComputed = true - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) + if !isEmpty { + return common.NewError("invalid_reference_path", "directory is not empty") } - newRef.FileID = fileID - dirRef.AddChild(newRef) - dirRef = newRef } + return nil + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return err } - fileRefs := rf.processMoveRefs(ctx, srcRef, dirRef, allocationRoot, ts, true) - - for _, fileRef := range fileRefs { - fileRef.IsPrecommit = true + rf.Type = srcRef.Type + parentDir, err := reference.Mkdir(cctx, rf.AllocationID, filepath.Dir(rf.DestPath), allocationVersion, ts, collector) + if err != nil { + return err } - return rootRef, nil -} -func (rf *MoveFileChange) processMoveRefs( - ctx context.Context, srcRef, destRef *reference.Ref, - allocationRoot string, ts common.Timestamp, toAdd bool) (fileRefs []*reference.Ref) { - if srcRef.Type == reference.DIRECTORY { - srcRef.Path = filepath.Join(destRef.Path, srcRef.Name) - srcRef.ParentPath = destRef.Path - srcRef.UpdatedAt = ts - srcRef.HashToBeComputed = true - if toAdd { - destRef.AddChild(srcRef) - } - - for _, childRef := range srcRef.Children { - fileRefs = append(fileRefs, rf.processMoveRefs(ctx, childRef, srcRef, allocationRoot, ts, false)...) - } - } else if srcRef.Type == reference.FILE { - srcRef.ParentPath = destRef.Path - srcRef.Path = filepath.Join(destRef.Path, srcRef.Name) - srcRef.UpdatedAt = ts - srcRef.HashToBeComputed = true - if toAdd { - destRef.AddChild(srcRef) - } - fileRefs = append(fileRefs, srcRef) + deleteRef := &reference.Ref{ + ID: srcRef.ID, + LookupHash: srcLookUpHash, + Type: srcRef.Type, } + collector.DeleteRefRecord(deleteRef) + + srcRef.ID = 0 + srcRef.ParentID = &parentDir.ID + srcRef.Path = rf.DestPath + srcRef.ParentPath = filepath.Dir(rf.DestPath) + srcRef.Name = filepath.Base(rf.DestPath) + srcRef.LookupHash = destLookUpHash + srcRef.CreatedAt = ts + srcRef.UpdatedAt = ts + srcRef.PathLevel = len(strings.Split(strings.TrimRight(rf.DestPath, "/"), "/")) + srcRef.FileMetaHash = encryption.FastHash(srcRef.GetFileHashData()) + srcRef.AllocationVersion = allocationVersion + collector.CreateRefRecord(srcRef) - return - + return nil } func (rf *MoveFileChange) Marshal() (string, error) { @@ -168,7 +104,12 @@ func (rf *MoveFileChange) Unmarshal(input string) error { } func (rf *MoveFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - return nil + if rf.Type == reference.DIRECTORY { + return nil + } + srcLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.SrcPath) + destLookUpHash := reference.GetReferenceLookup(rf.AllocationID, rf.DestPath) + return filestore.GetFileStore().CopyFile(rf.AllocationID, srcLookUpHash, destLookUpHash) } func (rf *MoveFileChange) GetPath() []string { diff --git a/code/go/0chain.net/blobbercore/allocation/newdirchange.go b/code/go/0chain.net/blobbercore/allocation/newdirchange.go index 48f434b71..e5bf4051e 100644 --- a/code/go/0chain.net/blobbercore/allocation/newdirchange.go +++ b/code/go/0chain.net/blobbercore/allocation/newdirchange.go @@ -3,16 +3,17 @@ package allocation import ( "context" "encoding/json" - "fmt" "path/filepath" "strings" "sync" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/util" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" + "gorm.io/gorm" ) type NewDir struct { @@ -20,72 +21,58 @@ type NewDir struct { Path string `json:"filepath" validation:"required"` AllocationID string `json:"allocation_id"` CustomMeta string `json:"custom_meta,omitempty"` + MimeType string `json:"mimetype,omitempty"` } -func (nf *NewDir) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, fileIDMeta map[string]string) (*reference.Ref, error) { - - totalRefs, err := reference.CountRefs(ctx, nf.AllocationID) - if err != nil { - return nil, err - } - - if int64(config.Configuration.MaxAllocationDirFiles) <= totalRefs { - return nil, common.NewErrorf("max_alloc_dir_files_reached", - "maximum files and directories already reached: %v", err) - } - - err = nf.Unmarshal(change.Input) - if err != nil { - return nil, err - } - - if rootRef.CreatedAt == 0 { - rootRef.CreatedAt = ts - } - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - fields, err := common.GetPathFields(nf.Path) - if err != nil { - return nil, err +func (nf *NewDir) ApplyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + parentPath := filepath.Dir(nf.Path) + parentPathLookup := reference.GetReferenceLookup(nf.AllocationID, parentPath) + parentRef, err := reference.GetFullReferenceByLookupHashWithNewTransaction(parentPathLookup) + if err != nil && err != gorm.ErrRecordNotFound { + return err } - dirRef := rootRef - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - dirRef = child - dirRef.HashToBeComputed = true - dirRef.UpdatedAt = ts - found = true - break - } + if parentRef == nil || parentRef.ID == 0 { + _, err = reference.Mkdir(ctx, nf.AllocationID, nf.Path, allocationVersion, ts, collector) + } else { + collector.LockTransaction() + defer collector.UnlockTransaction() + dirLookupHash := reference.GetReferenceLookup(nf.AllocationID, nf.Path) + dRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, nf.AllocationID, dirLookupHash, []string{"id"}) + if err != nil && err != gorm.ErrRecordNotFound { + logging.Logger.Error("ApplyChange:Newdir", zap.Error(err)) + return err } - - if !found { - newRef := reference.NewDirectoryRef() - newRef.AllocationID = nf.AllocationID - newRef.Path = filepath.Join("/", strings.Join(fields[:i+1], "/")) - newRef.PathLevel = len(fields) + 1 - newRef.ParentPath = filepath.Dir(newRef.Path) - newRef.Name = fields[i] - newRef.LookupHash = reference.GetReferenceLookup(nf.AllocationID, newRef.Path) - newRef.CreatedAt = ts - newRef.UpdatedAt = ts - newRef.HashToBeComputed = true - fileID, ok := fileIDMeta[newRef.Path] - if !ok || fileID == "" { - return nil, common.NewError("invalid_parameter", - fmt.Sprintf("file path %s has no entry in fileID meta", newRef.Path)) - } - newRef.FileID = fileID + err = nil + // already exists + if dRef != nil && dRef.ID != 0 { + return nil + } + parentIDRef := &parentRef.ID + newRef := reference.NewDirectoryRef() + newRef.AllocationID = nf.AllocationID + newRef.Path = nf.Path + if newRef.Path != "/" { + newRef.ParentPath = parentPath + } + newRef.Name = filepath.Base(nf.Path) + newRef.PathLevel = len(strings.Split(strings.TrimRight(nf.Path, "/"), "/")) + newRef.ParentID = parentIDRef + newRef.LookupHash = dirLookupHash + newRef.CreatedAt = ts + newRef.UpdatedAt = ts + newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData()) + if nf.CustomMeta != "" { newRef.CustomMeta = nf.CustomMeta - dirRef.AddChild(newRef) - dirRef = newRef } + if nf.MimeType != "" { + newRef.MimeType = nf.MimeType + } + newRef.AllocationVersion = allocationVersion + collector.CreateRefRecord(newRef) } - return rootRef, nil + return err } func (nd *NewDir) Marshal() (string, error) { diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go index fbcee3759..774027eee 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange.go @@ -6,31 +6,37 @@ import ( "path/filepath" "sync" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "go.uber.org/zap" ) type RenameFileChange struct { - ConnectionID string `json:"connection_id"` - AllocationID string `json:"allocation_id"` - Path string `json:"path"` - NewName string `json:"new_name"` - Name string `json:"name"` - Type string `json:"type"` + ConnectionID string `json:"connection_id"` + AllocationID string `json:"allocation_id"` + Path string `json:"path"` + NewName string `json:"new_name"` + Name string `json:"name"` + Type string `json:"type"` + CustomMeta string `json:"custom_meta"` + MimeType string `json:"mimetype"` + newLookupHash string `json:"-"` } func (rf *RenameFileChange) DeleteTempFile() error { return nil } -func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { - +func (rf *RenameFileChange) applyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { + collector.LockTransaction() + defer collector.UnlockTransaction() if rf.Path == "/" { - return nil, common.NewError("invalid_operation", "cannot rename root path") + return common.NewError("invalid_operation", "cannot rename root path") } newPath := filepath.Join(filepath.Dir(rf.Path), rf.NewName) @@ -40,79 +46,45 @@ func (rf *RenameFileChange) applyChange(ctx context.Context, rootRef *reference. } if isFilePresent { - return nil, common.NewError("invalid_reference_path", "file already exists") - } - - affectedRef, err := rootRef.GetSrcPath(rf.Path) - if err != nil { - return nil, err - } - affectedRef.HashToBeComputed = true - affectedRef.Name = rf.NewName - affectedRef.Path = newPath - affectedRef.UpdatedAt = ts - if affectedRef.Type == reference.FILE { - affectedRef.IsPrecommit = true - } else { - rf.processChildren(ctx, affectedRef, ts) + return common.NewError("invalid_reference_path", "file already exists") } - - parentPath := filepath.Dir(rf.Path) - fields, err := common.GetPathFields(parentPath) + oldFileLookupHash := reference.GetReferenceLookup(rf.AllocationID, rf.Path) + ref, err := reference.GetReferenceByLookupHash(ctx, rf.AllocationID, oldFileLookupHash) if err != nil { - return nil, err + return common.NewError("invalid_reference_path", err.Error()) } - - rootRef.UpdatedAt = ts - rootRef.HashToBeComputed = true - dirRef := rootRef - - for i := 0; i < len(fields); i++ { - found := false - for _, child := range dirRef.Children { - if child.Name == fields[i] { - dirRef = child - dirRef.UpdatedAt = ts - dirRef.HashToBeComputed = true - found = true - break - } + if ref.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, ref.ID) + if err != nil { + return common.NewError("invalid_reference_path", err.Error()) } - - if !found { - return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber") + if !isEmpty { + return common.NewError("invalid_reference_path", "Directory is not empty") } } - - found := false - for i, child := range dirRef.Children { - if child.Path == rf.Path { - dirRef.RemoveChild(i) - dirRef.AddChild(affectedRef) - found = true - break - } + rf.Type = ref.Type + deleteRef := &reference.Ref{ + ID: ref.ID, + LookupHash: oldFileLookupHash, + Type: ref.Type, } - if !found { - return nil, common.NewError("file_not_found", "File to rename not found in blobber") + collector.DeleteRefRecord(deleteRef) + ref.Name = rf.NewName + ref.Path = newPath + ref.ID = 0 + ref.LookupHash = reference.GetReferenceLookup(rf.AllocationID, newPath) + ref.UpdatedAt = ts + ref.FileMetaHash = encryption.Hash(ref.GetFileMetaHashData()) + if rf.CustomMeta != "" { + ref.CustomMeta = rf.CustomMeta } - - return rootRef, nil -} - -func (rf *RenameFileChange) processChildren(ctx context.Context, curRef *reference.Ref, ts common.Timestamp) { - for _, childRef := range curRef.Children { - childRef.UpdatedAt = ts - childRef.HashToBeComputed = true - newPath := filepath.Join(curRef.Path, childRef.Name) - childRef.UpdatePath(newPath, curRef.Path) - if childRef.Type == reference.FILE { - childRef.IsPrecommit = true - } - if childRef.Type == reference.DIRECTORY { - rf.processChildren(ctx, childRef, ts) - } + if rf.MimeType != "" { + ref.MimeType = rf.MimeType } + ref.AllocationVersion = allocationVersion + collector.CreateRefRecord(ref) + rf.newLookupHash = ref.LookupHash + return nil } func (rf *RenameFileChange) Marshal() (string, error) { @@ -129,7 +101,18 @@ func (rf *RenameFileChange) Unmarshal(input string) error { } func (rf *RenameFileChange) CommitToFileStore(ctx context.Context, mut *sync.Mutex) error { - return nil + if rf.Type == reference.DIRECTORY { + return nil + } + if rf.newLookupHash == "" { + return common.NewError("invalid_reference_path", "new lookup hash is empty") + } + oldFileLookupHash := reference.GetReferenceLookup(rf.AllocationID, rf.Path) + err := filestore.GetFileStore().CopyFile(rf.AllocationID, oldFileLookupHash, rf.newLookupHash) + if err != nil { + logging.Logger.Error("CommitToFileStore: CopyFile", zap.Error(err)) + } + return err } func (rf *RenameFileChange) GetPath() []string { diff --git a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go index 9c773cab7..424ffe7ab 100644 --- a/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go +++ b/code/go/0chain.net/blobbercore/allocation/renamefilechange_main.go @@ -10,8 +10,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/common" ) -func (rf *RenameFileChange) ApplyChange(ctx context.Context, rootRef *reference.Ref, change *AllocationChange, - allocationRoot string, ts common.Timestamp, _ map[string]string) (*reference.Ref, error) { +func (rf *RenameFileChange) ApplyChange(ctx context.Context, + ts common.Timestamp, allocationVersion int64, collector reference.QueryCollector) error { - return rf.applyChange(ctx, rootRef, change, allocationRoot, ts, nil) + return rf.applyChange(ctx, ts, allocationVersion, collector) } diff --git a/code/go/0chain.net/blobbercore/allocation/rollback.go b/code/go/0chain.net/blobbercore/allocation/rollback.go index fe25adaac..d3070ad96 100644 --- a/code/go/0chain.net/blobbercore/allocation/rollback.go +++ b/code/go/0chain.net/blobbercore/allocation/rollback.go @@ -8,23 +8,23 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" ) -func ApplyRollback(ctx context.Context, allocationID string) error { +func ApplyRollback(ctx context.Context, allocationID string, allocationVersion int64) error { db := datastore.GetStore().GetTransaction(ctx) - // delete all is_precommit rows + // delete all current allocation version rows err := db.Model(&reference.Ref{}).Unscoped(). Delete(&reference.Ref{}, - "allocation_id=? AND is_precommit=? AND deleted_at IS NULL", - allocationID, true).Error + "allocation_id=? AND allocation_version=? AND deleted_at IS NULL", + allocationID, allocationVersion).Error if err != nil { return err } // err = db.Exec("UPDATE file_stats SET deleted_at=NULL WHERE ref_id IN (SELECT id FROM reference_objects WHERE allocation_id=? AND deleted_at IS NOT NULL)", allocationID).Error // revive soft deleted ref rows - err = db.Exec("UPDATE reference_objects SET deleted_at=NULL,is_precommit=? WHERE allocation_id=? AND deleted_at IS NOT NULL", false, allocationID).Error + err = db.Exec("UPDATE reference_objects SET deleted_at=NULL WHERE allocation_id=? AND deleted_at IS NOT NULL", allocationID).Error return err } diff --git a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go index da41fcc62..5b4f4e162 100644 --- a/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go +++ b/code/go/0chain.net/blobbercore/allocation/updatefilechange_test.go @@ -344,7 +344,7 @@ func TestBlobberCore_UpdateFile(t *testing.T) { changes: make(map[string]*ConnectionChange), } connectionProcessor["connection_id"].changes[pathHash] = &ConnectionChange{ - hasher: filestore.GetNewCommitHasher(2310), + hasher: filestore.NewCommitHasher(2310), } change := &UpdateFileChanger{ BaseFileChanger: BaseFileChanger{ diff --git a/code/go/0chain.net/blobbercore/blobberhttp/response.go b/code/go/0chain.net/blobbercore/blobberhttp/response.go index c462fdb23..fb64be0da 100644 --- a/code/go/0chain.net/blobbercore/blobberhttp/response.go +++ b/code/go/0chain.net/blobbercore/blobberhttp/response.go @@ -35,7 +35,6 @@ type RefResult struct { OffsetPath string `json:"offset_path,omitempty"` //used for pagination; index for path is created in database OffsetDate common.Timestamp `json:"offset_date,omitempty"` //used for pagination; idex for updated_at is created in database Refs *[]reference.PaginatedRef `json:"refs"` - LatestWM *writemarker.WriteMarker `json:"latest_write_marker"` } // swagger:model RecentRefResult @@ -50,9 +49,9 @@ type ObjectPathResult struct { // swagger:model ListResult type ListResult struct { - AllocationRoot string `json:"allocation_root"` - Meta map[string]interface{} `json:"meta_data"` - Entities []map[string]interface{} `json:"list"` + AllocationVersion int64 `json:"allocation_version"` + Meta map[string]interface{} `json:"meta_data"` + Entities []map[string]interface{} `json:"list"` } // swagger:model DownloadResponse @@ -70,3 +69,7 @@ type LatestWriteMarkerResult struct { PrevWM *writemarker.WriteMarker `json:"prev_write_marker"` Version string `json:"version"` } + +type LatestVersionMarkerResult struct { + VersionMarker *writemarker.VersionMarker `json:"version_marker"` +} diff --git a/code/go/0chain.net/blobbercore/challenge/protocol.go b/code/go/0chain.net/blobbercore/challenge/protocol.go index 55b8940c0..65c0c84f6 100644 --- a/code/go/0chain.net/blobbercore/challenge/protocol.go +++ b/code/go/0chain.net/blobbercore/challenge/protocol.go @@ -109,9 +109,6 @@ func (cr *ChallengeEntity) LoadValidationTickets(ctx context.Context) error { blockNum := int64(0) var objectPath *reference.ObjectPath if rootRef != nil { - if rootRef.Hash != allocationObj.AllocationRoot { - logging.Logger.Error("root_mismatch", zap.Any("allocation_root", allocationObj.AllocationRoot), zap.Any("latest_write_marker", wms[len(wms)-1].WM.AllocationRoot), zap.Any("root_ref_hash", rootRef.Hash)) - } if rootRef.NumBlocks > 0 { r := rand.New(rand.NewSource(cr.RandomNumber)) blockNum = r.Int63n(rootRef.NumBlocks) diff --git a/code/go/0chain.net/blobbercore/challenge/worker.go b/code/go/0chain.net/blobbercore/challenge/worker.go index e6fbe14e2..b4ef094d0 100644 --- a/code/go/0chain.net/blobbercore/challenge/worker.go +++ b/code/go/0chain.net/blobbercore/challenge/worker.go @@ -54,8 +54,8 @@ const batchSize = 5 // SetupWorkers start challenge workers func SetupWorkers(ctx context.Context) { - go startPullWorker(ctx) - go startWorkers(ctx) + // go startPullWorker(ctx) + // go startWorkers(ctx) } func startPullWorker(ctx context.Context) { diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 7e9d2dccc..b45781495 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -32,9 +32,10 @@ func SetupDefaultConfig() { viper.SetDefault("rate_limiters.block_limit_request", 500) viper.SetDefault("rate_limiters.block_limit_monthly", 31250000) viper.SetDefault("rate_limiters.upload_limit_monthly", 31250000) - viper.SetDefault("rate_limiters.commit_limit_monthly", 30000) + viper.SetDefault("rate_limiters.commit_limit_monthly", 1000000000) viper.SetDefault("rate_limiters.commit_limit_daily", 1600) viper.SetDefault("rate_limiters.commit_zero_limit_daily", 400) + viper.SetDefault("rate_limiters.max_connection_changes", 100) viper.SetDefault("healthcheck.frequency", "60s") @@ -121,6 +122,7 @@ type Config struct { CommitLimitDaily int64 CommitZeroLimitDaily int64 ChallengeCleanupGap int64 + MaxConnectionChanges int HealthCheckWorkerFreq time.Duration @@ -292,7 +294,6 @@ func ReadConfig(deploymentMode int) { } else if Configuration.MinConfirmation > 100 { Configuration.MinConfirmation = 100 } - Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request") Configuration.BlockLimitMonthly = viper.GetInt64("rate_limiters.block_limit_monthly") @@ -301,6 +302,9 @@ func ReadConfig(deploymentMode int) { Configuration.CommitLimitDaily = viper.GetInt64("rate_limiters.commit_limit_daily") Configuration.CommitZeroLimitDaily = viper.GetInt64("rate_limiters.commit_zero_limit_daily") + Configuration.IsEnterprise = viper.GetBool("is_enterprise") + Configuration.MaxConnectionChanges = viper.GetInt("rate_limiters.max_connection_changes") + Configuration.IsEnterprise = viper.GetBool("is_enterprise") } diff --git a/code/go/0chain.net/blobbercore/convert/convert.go b/code/go/0chain.net/blobbercore/convert/convert.go index 8f3ca5c41..1c6ef1bb3 100644 --- a/code/go/0chain.net/blobbercore/convert/convert.go +++ b/code/go/0chain.net/blobbercore/convert/convert.go @@ -280,13 +280,9 @@ func convertFileRefToFileMetaDataGRPC(fileref *reference.Ref) *blobbergrpc.FileM LookupHash: fileref.LookupHash, Name: fileref.Name, Path: fileref.Path, - Hash: fileref.Hash, NumBlocks: fileref.NumBlocks, - PathHash: fileref.PathHash, CustomMeta: fileref.CustomMeta, - ValidationRoot: fileref.ValidationRoot, Size: fileref.Size, - FixedMerkleRoot: fileref.FixedMerkleRoot, ActualFileSize: fileref.ActualFileSize, ActualFileHash: fileref.ActualFileHash, MimeType: fileref.MimeType, @@ -306,9 +302,7 @@ func convertDirRefToDirMetaDataGRPC(dirref *reference.Ref) *blobbergrpc.DirMetaD LookupHash: dirref.LookupHash, Name: dirref.Name, Path: dirref.Path, - Hash: dirref.Hash, NumBlocks: dirref.NumBlocks, - PathHash: dirref.PathHash, Size: dirref.Size, CreatedAt: int64(dirref.CreatedAt), UpdatedAt: int64(dirref.UpdatedAt), @@ -336,13 +330,9 @@ func convertFileMetaDataGRPCToFileRef(metaData *blobbergrpc.FileMetaData) *refer LookupHash: metaData.LookupHash, Name: metaData.Name, Path: metaData.Path, - Hash: metaData.Hash, NumBlocks: metaData.NumBlocks, - PathHash: metaData.PathHash, CustomMeta: metaData.CustomMeta, - ValidationRoot: metaData.ValidationRoot, Size: metaData.Size, - FixedMerkleRoot: metaData.FixedMerkleRoot, ActualFileSize: metaData.ActualFileSize, ActualFileHash: metaData.ActualFileHash, MimeType: metaData.MimeType, @@ -362,9 +352,7 @@ func convertDirMetaDataGRPCToDirRef(dirref *blobbergrpc.DirMetaData) *reference. LookupHash: dirref.LookupHash, Name: dirref.Name, Path: dirref.Path, - Hash: dirref.Hash, NumBlocks: dirref.NumBlocks, - PathHash: dirref.PathHash, Size: dirref.Size, CreatedAt: common.Timestamp(dirref.CreatedAt), UpdatedAt: common.Timestamp(dirref.UpdatedAt), diff --git a/code/go/0chain.net/blobbercore/convert/response_creator.go b/code/go/0chain.net/blobbercore/convert/response_creator.go index 828bd2bcb..7e8dbbfdc 100644 --- a/code/go/0chain.net/blobbercore/convert/response_creator.go +++ b/code/go/0chain.net/blobbercore/convert/response_creator.go @@ -62,7 +62,6 @@ func ListEntitesResponseCreator(r interface{}) *blobbergrpc.ListEntitiesResponse } resp.MetaData = FileRefToFileRefGRPC(reference.ListingDataToRef(httpResp.Meta)) - resp.AllocationRoot = httpResp.AllocationRoot return &resp } @@ -157,12 +156,10 @@ func CopyObjectResponseCreator(r interface{}) *blobbergrpc.CopyObjectResponse { httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.CopyObjectResponse{ - Filename: httpResp.Filename, - Size: httpResp.Size, - ValidationRoot: httpResp.ValidationRoot, - FixedMerkleRoot: httpResp.FixedMerkleRoot, - UploadLength: httpResp.UploadLength, - UploadOffset: httpResp.UploadOffset, + Filename: httpResp.Filename, + Size: httpResp.Size, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } @@ -173,12 +170,10 @@ func RenameObjectResponseCreator(r interface{}) *blobbergrpc.RenameObjectRespons httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.RenameObjectResponse{ - Filename: httpResp.Filename, - Size: httpResp.Size, - ValidationRoot: httpResp.ValidationRoot, - FixedMerkleRoot: httpResp.FixedMerkleRoot, - UploadLength: httpResp.UploadLength, - UploadOffset: httpResp.UploadOffset, + Filename: httpResp.Filename, + Size: httpResp.Size, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } @@ -211,11 +206,9 @@ func UploadFileResponseCreator(r interface{}) *blobbergrpc.UploadFileResponse { httpResp, _ := r.(*allocation.UploadResult) return &blobbergrpc.UploadFileResponse{ - Filename: httpResp.Filename, - Size: httpResp.Size, - ValidationRoot: httpResp.ValidationRoot, - FixedMerkleRoot: httpResp.FixedMerkleRoot, - UploadLength: httpResp.UploadLength, - UploadOffset: httpResp.UploadOffset, + Filename: httpResp.Filename, + Size: httpResp.Size, + UploadLength: httpResp.UploadLength, + UploadOffset: httpResp.UploadOffset, } } diff --git a/code/go/0chain.net/blobbercore/convert/response_handler.go b/code/go/0chain.net/blobbercore/convert/response_handler.go index 3092c6b6d..8fb3374df 100644 --- a/code/go/0chain.net/blobbercore/convert/response_handler.go +++ b/code/go/0chain.net/blobbercore/convert/response_handler.go @@ -42,9 +42,8 @@ func ListEntitesResponseHandler(resp *blobbergrpc.ListEntitiesResponse) *blobber } return &blobberhttp.ListResult{ - AllocationRoot: resp.AllocationRoot, - Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx), - Entities: entities, + Meta: FileRefGRPCToFileRef(resp.MetaData).GetListingData(ctx), + Entities: entities, } } @@ -119,11 +118,9 @@ func GetCommitMetaTxnHandlerResponse(response *blobbergrpc.CommitMetaTxnResponse func CopyObjectResponseHandler(copyObjectResponse *blobbergrpc.CopyObjectResponse) *allocation.UploadResult { return &allocation.UploadResult{ - Filename: copyObjectResponse.Filename, - Size: copyObjectResponse.Size, - ValidationRoot: copyObjectResponse.ValidationRoot, - FixedMerkleRoot: copyObjectResponse.FixedMerkleRoot, - UploadLength: copyObjectResponse.UploadLength, - UploadOffset: copyObjectResponse.UploadOffset, + Filename: copyObjectResponse.Filename, + Size: copyObjectResponse.Size, + UploadLength: copyObjectResponse.UploadLength, + UploadOffset: copyObjectResponse.UploadOffset, } } diff --git a/code/go/0chain.net/blobbercore/datastore/mocket.go b/code/go/0chain.net/blobbercore/datastore/mocket.go index b560b0841..fb9b3d4cc 100644 --- a/code/go/0chain.net/blobbercore/datastore/mocket.go +++ b/code/go/0chain.net/blobbercore/datastore/mocket.go @@ -72,7 +72,7 @@ func (store *Mocket) Close() { } } -func (store *Mocket) CreateTransaction(ctx context.Context,opts ...*sql.TxOptions) context.Context { +func (store *Mocket) CreateTransaction(ctx context.Context, opts ...*sql.TxOptions) context.Context { db := store.db.Begin() return context.WithValue(ctx, ContextKeyTransaction, EnhanceDB(db)) } @@ -86,7 +86,7 @@ func (store *Mocket) GetTransaction(ctx context.Context) *EnhancedDB { return nil } -func (store *Mocket) WithNewTransaction(f func(ctx context.Context) error) error { +func (store *Mocket) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error { ctx := store.CreateTransaction(context.TODO()) defer ctx.Done() diff --git a/code/go/0chain.net/blobbercore/datastore/postgres.go b/code/go/0chain.net/blobbercore/datastore/postgres.go index 241701dfe..47e46623b 100644 --- a/code/go/0chain.net/blobbercore/datastore/postgres.go +++ b/code/go/0chain.net/blobbercore/datastore/postgres.go @@ -11,6 +11,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/logging" "gorm.io/driver/postgres" "gorm.io/gorm" + "moul.io/zapgorm2" ) // postgresStore store implementation for postgres @@ -46,6 +47,11 @@ func (p *postgresStore) GetPgDB() (*gorm.DB, error) { } func (store *postgresStore) Open() error { + gormLogger := zapgorm2.New(logging.Logger) + gormLogger.SlowThreshold = 100 * time.Millisecond + gormLogger.IgnoreRecordNotFoundError = true + gormLogger.SkipCallerLookup = true + gormLogger.SetAsDefault() db, err := gorm.Open(postgres.Open(fmt.Sprintf( "host=%v port=%v user=%v dbname=%v password=%v sslmode=disable", config.Configuration.DBHost, config.Configuration.DBPort, @@ -53,6 +59,7 @@ func (store *postgresStore) Open() error { config.Configuration.DBPassword)), &gorm.Config{ SkipDefaultTransaction: true, // https://gorm.io/docs/performance.html#Disable-Default-Transaction PrepareStmt: true, //https://gorm.io/docs/performance.html#Caches-Prepared-Statement + Logger: gormLogger, }) if err != nil { return common.NewErrorf("db_open_error", "Error opening the DB connection: %v", err) @@ -99,7 +106,7 @@ func (store *postgresStore) GetTransaction(ctx context.Context) *EnhancedDB { return nil } -func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error) error { +func (store *postgresStore) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error { timeoutctx, cancel := context.WithTimeout(context.TODO(), 45*time.Second) defer cancel() ctx := store.CreateTransaction(timeoutctx) diff --git a/code/go/0chain.net/blobbercore/datastore/sqlmock.go b/code/go/0chain.net/blobbercore/datastore/sqlmock.go index a7bcdeac5..354cc69e5 100644 --- a/code/go/0chain.net/blobbercore/datastore/sqlmock.go +++ b/code/go/0chain.net/blobbercore/datastore/sqlmock.go @@ -81,7 +81,7 @@ func (store *Sqlmock) GetTransaction(ctx context.Context) *EnhancedDB { return nil } -func (store *Sqlmock) WithNewTransaction(f func(ctx context.Context) error) error { +func (store *Sqlmock) WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error { ctx := store.CreateTransaction(context.TODO()) defer ctx.Done() diff --git a/code/go/0chain.net/blobbercore/datastore/store.go b/code/go/0chain.net/blobbercore/datastore/store.go index a097be572..bedc36896 100644 --- a/code/go/0chain.net/blobbercore/datastore/store.go +++ b/code/go/0chain.net/blobbercore/datastore/store.go @@ -45,7 +45,7 @@ type Store interface { CreateTransaction(ctx context.Context, opts ...*sql.TxOptions) context.Context // GetTransaction get transaction from context GetTransaction(ctx context.Context) *EnhancedDB - WithNewTransaction(f func(ctx context.Context) error) error + WithNewTransaction(f func(ctx context.Context) error, opts ...*sql.TxOptions) error WithTransaction(ctx context.Context, f func(ctx context.Context) error) error // Get db connection with user that creates roles and databases. Its dialactor does not contain database name GetPgDB() (*gorm.DB, error) diff --git a/code/go/0chain.net/blobbercore/filestore/state.go b/code/go/0chain.net/blobbercore/filestore/state.go index b098a34c8..f68b556f1 100644 --- a/code/go/0chain.net/blobbercore/filestore/state.go +++ b/code/go/0chain.net/blobbercore/filestore/state.go @@ -57,8 +57,6 @@ func (fs *FileStore) initMap() error { return errors.New("could not get db client") } - limitCh := make(chan struct{}, 50) - wg := &sync.WaitGroup{} var dbAllocations []*dbAllocation err := db.Model(&dbAllocation{}).FindInBatches(&dbAllocations, 1000, func(tx *gorm.DB, batch int) error { @@ -78,18 +76,12 @@ func (fs *FileStore) initMap() error { if err != nil { return err } - - limitCh <- struct{}{} - wg.Add(1) - go fs.getTemporaryStorageDetails(ctx, &a, dbAlloc.ID, limitCh, wg) - } fs.setAllocations(allocsMap) return nil }).Error - wg.Wait() return err }) return err diff --git a/code/go/0chain.net/blobbercore/filestore/storage.go b/code/go/0chain.net/blobbercore/filestore/storage.go index 63bd60e8a..4ec8826dd 100644 --- a/code/go/0chain.net/blobbercore/filestore/storage.go +++ b/code/go/0chain.net/blobbercore/filestore/storage.go @@ -46,6 +46,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/core/util" + "github.com/remeh/sizedwaitgroup" "go.uber.org/zap" "golang.org/x/crypto/sha3" "golang.org/x/sys/unix" @@ -57,10 +58,15 @@ const ( MerkleChunkSize = 64 ChunkSize = 64 * KB BufferSize = 80 * ChunkSize + ThumbnailSuffix = "_thumbnail" ) func (fs *FileStore) WriteFile(allocID, conID string, fileData *FileInputData, infile multipart.File) (*FileOutputData, error) { - tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileData.FilePathHash, conID) + fileHash := fileData.LookupHash + if fileData.IsThumbnail { + fileHash = fileData.LookupHash + ThumbnailSuffix + } + tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileHash, conID) var ( initialSize int64 ) @@ -157,6 +163,17 @@ func (fs *FileStore) MoveToFilestore(allocID, hash string, version int) error { } _ = os.Rename(preCommitPath, fPath) + logging.Logger.Info("move_to_filestore: ", zap.String("path", fPath), zap.String("hash", hash)) + // Check if thumbnail exists + thumbPath := fs.getPreCommitPathForFile(allocID, hash+ThumbnailSuffix, version) + if _, err := os.Stat(thumbPath); err == nil { + thumbFilePath, err := fs.GetPathForFile(allocID, hash+ThumbnailSuffix, version) + if err != nil { + return common.NewError("get_file_path_error", err.Error()) + } + _ = os.Rename(thumbPath, thumbFilePath) + } + return nil } @@ -179,29 +196,78 @@ func (fs *FileStore) DeleteFromFilestore(allocID, hash string, version int) erro } fs.incrDecrAllocFileSizeAndNumber(allocID, -stat.Size(), -1) + thumbPath, err := fs.GetPathForFile(allocID, hash+ThumbnailSuffix, version) + if err != nil { + return common.NewError("get_file_path_error", err.Error()) + } + if _, err := os.Stat(thumbPath); err == nil { + os.Remove(thumbPath) //nolint:errcheck + } + return nil } func (fs *FileStore) DeletePreCommitDir(allocID string) error { - + now := time.Now() preCommitDir := fs.getPreCommitDir(allocID) - err := os.RemoveAll(preCommitDir) + swg := sizedwaitgroup.New(5) + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + + entries, err := os.ReadDir(preCommitDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return common.NewError("pre_commit_dir_read_error", err.Error()) + } + + for _, entry := range entries { + entryPath := filepath.Join(preCommitDir, entry.Name()) + if entry.IsDir() { + swg.Add() + go func() { + select { + case <-ctx.Done(): + return + default: + err := os.RemoveAll(entryPath) + if err != nil { + logging.Logger.Error("failed to remove directory", zap.String("path", entryPath), zap.Error(err)) + cancel(err) + } + } + swg.Done() + }() + } else { + if err := os.Remove(entryPath); err != nil { + return fmt.Errorf("failed to remove file %s: %w", entryPath, err) + } + } + } + swg.Wait() + select { + case <-ctx.Done(): + return common.NewError("pre_commit_dir_deletion_error", ctx.Err().Error()) + default: + } + + err = os.Remove(preCommitDir) if err != nil { return common.NewError("pre_commit_dir_deletion_error", err.Error()) } + logging.Logger.Debug("pre_commit_dir_deleted", zap.String("allocation_id", allocID), zap.Duration("elapsed", time.Since(now))) return nil } func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) (_ bool, err error) { - - logging.Logger.Info("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) - filePathHash := encryption.Hash(fileData.Path) - tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, filePathHash, conID) - - fileHash := fileData.ValidationRoot + now := time.Now() + logging.Logger.Debug("Committing write", zap.String("allocation_id", allocID), zap.Any("file_data", fileData)) + fileHash := fileData.LookupHash if fileData.IsThumbnail { - fileHash = fileData.ThumbnailHash + fileHash = fileData.LookupHash + ThumbnailSuffix } + tempFilePath := fs.getTempPathForFile(allocID, fileData.Name, fileHash, conID) preCommitPath := fs.getPreCommitPathForFile(allocID, fileHash, VERSION) @@ -255,7 +321,7 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) return true, nil } - key := getKey(allocID, fileData.ValidationRoot) + key := getKey(allocID, fileData.LookupHash) l, _ := contentHashMapLock.GetLock(key) l.Lock() defer func() { @@ -270,41 +336,18 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) } fileSize := rStat.Size() - now := time.Now() ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() - err = fileData.Hasher.Wait(ctx, conID, allocID, fileData.Name, filePathHash) + err = fileData.Hasher.Wait(ctx) if err != nil { return false, common.NewError("hasher_wait_error", err.Error()) } - elapsedWait := time.Since(now) - _, err = r.Seek(fileSize, io.SeekStart) - if err != nil { - return false, common.NewError("seek_error", err.Error()) + md5Hash := fileData.Hasher.GetMd5Hash() + if md5Hash != fileData.DataHash { + return false, common.NewError("hash_mismatch", + fmt.Sprintf("calculated hash does not match with expected hash. Expected %s, got %s.", + fileData.DataHash, md5Hash)) } - fmtRootBytes, err := fileData.Hasher.fmt.CalculateRootAndStoreNodes(r) - if err != nil { - return false, common.NewError("fmt_hash_calculation_error", err.Error()) - } - - validationRootBytes, err := fileData.Hasher.vt.CalculateRootAndStoreNodes(r, fileSize) - if err != nil { - return false, common.NewError("validation_hash_calculation_error", err.Error()) - } - fmtRoot := hex.EncodeToString(fmtRootBytes) - validationRoot := hex.EncodeToString(validationRootBytes) - elapsedRoot := time.Since(now) - elapsedWait - if fmtRoot != fileData.FixedMerkleRoot { - err = common.NewError("fixed_merkle_root_mismatch", - fmt.Sprintf("Expected %s got %s", fileData.FixedMerkleRoot, fmtRoot)) - return false, err - } - if validationRoot != fileData.ValidationRoot { - err = common.NewError("validation_root_mismatch", - "calculated validation root does not match with client's validation root") - return false, err - } - err = os.Rename(tempFilePath, preCommitPath) if err != nil { return false, common.NewError("write_error", err.Error()) @@ -321,10 +364,81 @@ func (fs *FileStore) CommitWrite(allocID, conID string, fileData *FileInputData) // 5. Move: It is Copy + Delete. Delete will not delete file if ref exists in database. i.e. copy would create // ref that refers to this file therefore it will be skipped fs.incrDecrAllocFileSizeAndNumber(allocID, fileSize, 1) - logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.Duration("elapsed_wait", elapsedWait), zap.Duration("elapsed_root", elapsedRoot), zap.Duration("elapsed_total", time.Since(now))) + logging.Logger.Info("Committing write done", zap.String("file_path", fileData.Path), zap.String("lookup_hash", fileData.LookupHash), zap.Duration("elapsed_total", time.Since(now))) return true, nil } +func (fs *FileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error { + if oldFileLookupHash == newFileLookupHash { + return nil + } + var err error + oldObjectPath, err := fs.GetPathForFile(allocationID, oldFileLookupHash, VERSION) + if err != nil { + return common.NewError("get_file_path_error", err.Error()) + } + oldFile, err := os.Open(oldObjectPath) + if err != nil { + return common.NewError("file_open_error", err.Error()) + } + defer oldFile.Close() + stat, err := oldFile.Stat() + if err != nil { + return common.NewError("file_stat_error", err.Error()) + } + size := stat.Size() + + newObjectPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash, VERSION) + err = createDirs(filepath.Dir(newObjectPath)) + if err != nil { + return common.NewError("blob_object_precommit_dir_creation_error", err.Error()) + } + newFile, err := os.Create(newObjectPath) + if err != nil { + return common.NewError("file_create_error", err.Error()) + } + defer func() { + newFile.Close() + if err != nil { + os.Remove(newObjectPath) //nolint:errcheck + } + }() + bufSize := BufferSize + if size < int64(bufSize) { + bufSize = int(size) + } + copyBuf := make([]byte, bufSize) + _, err = io.CopyBuffer(newFile, oldFile, copyBuf) + if err != nil { + return common.NewError("file_copy_error", err.Error()) + } + // copy thumbnail if exists + oldThumbPath := fs.getPreCommitPathForFile(allocationID, oldFileLookupHash+ThumbnailSuffix, VERSION) + if _, err := os.Stat(oldThumbPath); err == nil { + newThumbPath := fs.getPreCommitPathForFile(allocationID, newFileLookupHash+ThumbnailSuffix, VERSION) + oldThumbFile, err := os.Open(oldThumbPath) + if err != nil { + return common.NewError("file_open_error", err.Error()) + } + defer oldThumbFile.Close() + newThumbFile, err := os.Create(newThumbPath) + if err != nil { + return common.NewError("file_create_error", err.Error()) + } + defer func() { + newThumbFile.Close() + if err != nil { + os.Remove(newThumbPath) //nolint:errcheck + } + }() + _, err = io.Copy(newThumbFile, oldThumbFile) + if err != nil { + return common.NewError("file_copy_error", err.Error()) + } + } + return nil +} + func (fs *FileStore) GetFilePathSize(allocID, filehash, thumbHash string, version int) (int64, int64, error) { filePath, err := fs.GetPathForFile(allocID, filehash, version) @@ -435,6 +549,7 @@ func (fs *FileStore) DeleteAllocation(allocID string) { func (fs *FileStore) GetFileThumbnail(readBlockIn *ReadBlockInput) (*FileDownloadResponse, error) { var fileObjectPath string var err error + readBlockIn.Hash += ThumbnailSuffix startBlock := readBlockIn.StartBlockNum if startBlock < 0 { return nil, common.NewError("invalid_block_number", "Invalid block number. Start block number cannot be negative") @@ -515,7 +630,6 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes } startBlock := readBlockIn.StartBlockNum - endBlock := readBlockIn.StartBlockNum + readBlockIn.NumBlocks - 1 if startBlock < 0 { return nil, common.NewError("invalid_block_number", "Invalid block number. Start block number cannot be negative") @@ -558,26 +672,6 @@ func (fs *FileStore) GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadRes nodesSize := getNodesSize(filesize, util.MaxMerkleLeavesSize) vmp := &FileDownloadResponse{} - if readBlockIn.VerifyDownload { - vpOffset := int64(FMTSize) - if readBlockIn.FilestoreVersion == 1 { - vpOffset += readBlockIn.FileSize - } - vp := validationTreeProof{ - dataSize: readBlockIn.FileSize, - offset: vpOffset, - } - - logging.Logger.Debug("calling GetMerkleProofOfMultipleIndexes", zap.Any("readBlockIn", readBlockIn), zap.Any("vmp", vmp)) - nodes, indexes, err := vp.GetMerkleProofOfMultipleIndexes(file, nodesSize, startBlock, endBlock) - if err != nil { - return nil, common.NewError("get_merkle_proof_error", err.Error()) - } - - vmp.Nodes = nodes - vmp.Indexes = indexes - } - logging.Logger.Info("filestore_version", zap.Int("version", readBlockIn.FilestoreVersion)) fileOffset := int64(startBlock) * ChunkSize if readBlockIn.FilestoreVersion == 1 { _, err = file.Seek(fileOffset, io.SeekStart) @@ -796,7 +890,7 @@ func (fs *FileStore) getAllocDir(allocID string) string { } func (fs *FileStore) GetPathForFile(allocID, hash string, version int) (string, error) { - if len(allocID) != 64 || len(hash) != 64 { + if len(allocID) != 64 { return "", errors.New("length of allocationID/hash must be 64") } var versionStr string @@ -830,7 +924,7 @@ func (fs *FileStore) getPreCommitDir(allocationID string) string { func (fs *FileStore) getTempPathForFile(allocId, fileName, pathHash, connectionID string) string { fileName = sanitizeFileName(fileName) - return filepath.Join(fs.getAllocTempDir(allocId), fileName+"."+pathHash+"."+connectionID) + return filepath.Join(fs.getAllocTempDir(allocId), getPartialPath(pathHash, getDirLevelsForFiles())+"."+connectionID) } func (fs *FileStore) getPreCommitPathForFile(allocID, hash string, version int) string { diff --git a/code/go/0chain.net/blobbercore/filestore/store.go b/code/go/0chain.net/blobbercore/filestore/store.go index 228e8de05..3e39ae39d 100644 --- a/code/go/0chain.net/blobbercore/filestore/store.go +++ b/code/go/0chain.net/blobbercore/filestore/store.go @@ -10,12 +10,11 @@ const ( ) type FileInputData struct { - Name string - Path string - ValidationRoot string - FixedMerkleRoot string - ThumbnailHash string - + Name string + Path string + DataHash string + ThumbnailHash string + LookupHash string // ChunkSize chunk size ChunkSize int64 //UploadLength indicates the size of the entire upload in bytes. The value MUST be a non-negative integer. @@ -23,19 +22,16 @@ type FileInputData struct { //Upload-Offset indicates a byte offset within a resource. The value MUST be a non-negative integer. UploadOffset int64 //IsFinal the request is final chunk - IsFinal bool - IsThumbnail bool - FilePathHash string - Size int64 - Hasher *CommitHasher + IsFinal bool + IsThumbnail bool + Size int64 + Hasher *CommitHasher } type FileOutputData struct { - Name string - Path string - ValidationRoot string - FixedMerkleRoot string - ThumbnailHash string + Name string + Path string + ThumbnailHash string // Size written size/chunk size Size int64 // ChunkUploaded the chunk is uploaded or not. @@ -57,6 +53,7 @@ type FileStorer interface { DeleteFromFilestore(allocID, hash string, version int) error DeletePreCommitDir(allocID string) error DeleteAllocation(allocID string) + CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error // GetFileBlock Get blocks of file starting from blockNum upto numBlocks. blockNum can't be less than 1. GetFileBlock(readBlockIn *ReadBlockInput) (*FileDownloadResponse, error) GetBlocksMerkleTreeForChallenge(cri *ChallengeReadBlockInput) (*ChallengeResponse, error) diff --git a/code/go/0chain.net/blobbercore/filestore/store_test.go b/code/go/0chain.net/blobbercore/filestore/store_test.go index ad8a74dc1..66ad81b25 100644 --- a/code/go/0chain.net/blobbercore/filestore/store_test.go +++ b/code/go/0chain.net/blobbercore/filestore/store_test.go @@ -251,16 +251,14 @@ func TestStoreStorageWriteAndCommit(t *testing.T) { validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) pathHash := encryption.Hash(test.remotePath) - hasher := GetNewCommitHasher(0) + hasher := NewCommitHasher(0) fid := &FileInputData{ - Name: test.fileName, - Path: test.remotePath, - ValidationRoot: validationRoot, - FixedMerkleRoot: fixedMerkleRoot, - ChunkSize: 64 * KB, - FilePathHash: pathHash, - Hasher: hasher, - Size: int64(size), + Name: test.fileName, + Path: test.remotePath, + ChunkSize: 64 * KB, + FilePathHash: pathHash, + Hasher: hasher, + Size: int64(size), } f, err := os.Open(fPath) @@ -338,7 +336,7 @@ func TestDeletePreCommitDir(t *testing.T) { validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) pathHash := encryption.Hash(remotePath) - hasher := GetNewCommitHasher(int64(size)) + hasher := NewCommitHasher(int64(size)) fid := &FileInputData{ Name: fileName, Path: remotePath, @@ -383,7 +381,7 @@ func TestDeletePreCommitDir(t *testing.T) { fid.ValidationRoot = validationRoot fid.FixedMerkleRoot = fixedMerkleRoot - hasher = GetNewCommitHasher(int64(size)) + hasher = NewCommitHasher(int64(size)) fid.Hasher = hasher // Write file to temp location @@ -446,7 +444,7 @@ func TestStorageUploadUpdate(t *testing.T) { validationRoot, fixedMerkleRoot, err := generateRandomData(fPath, int64(size)) require.Nil(t, err) pathHash := encryption.Hash(remotePath) - hasher := GetNewCommitHasher(int64(size)) + hasher := NewCommitHasher(int64(size)) fid := &FileInputData{ Name: fileName, Path: remotePath, @@ -834,7 +832,7 @@ func TestValidationRoot(t *testing.T) { fs, cleanUp := setupStorage(t) defer cleanUp() fPath := filepath.Join(fs.mp, randString(10)+".txt") - cH := GetNewCommitHasher(size) + cH := NewCommitHasher(size) _, err := cH.Write(thumbnailBytes) require.Nil(t, err) @@ -897,7 +895,7 @@ func generateRandomData(fPath string, size int64) (string, string, error) { } defer f.Close() - cH := GetNewCommitHasher(size) + cH := NewCommitHasher(size) _, err = cH.Write(p) if err != nil { return "", "", err @@ -938,7 +936,7 @@ func generateRandomDataAndStoreNodes(fPath string, size int64) (string, string, } defer f.Close() - cH := GetNewCommitHasher(size) + cH := NewCommitHasher(size) _, err = cH.Write(p) if err != nil { return "", "", err diff --git a/code/go/0chain.net/blobbercore/filestore/tree_validation.go b/code/go/0chain.net/blobbercore/filestore/tree_validation.go index 75856d485..1ceffce1e 100644 --- a/code/go/0chain.net/blobbercore/filestore/tree_validation.go +++ b/code/go/0chain.net/blobbercore/filestore/tree_validation.go @@ -5,9 +5,11 @@ package filestore import ( "context" + "crypto/md5" "encoding/hex" "errors" "fmt" + "hash" "io" "math" "os" @@ -406,16 +408,24 @@ func getNewValidationTree(dataSize int64) *validationTree { type CommitHasher struct { fmt *fixedMerkleTree vt *validationTree + md5hasher hash.Hash isInitialized bool doneChan chan struct{} hashErr error dataSize int64 } -func GetNewCommitHasher(dataSize int64) *CommitHasher { +var ( + md5Pool = &sync.Pool{ + New: func() interface{} { + return md5.New() + }, + } +) + +func NewCommitHasher(dataSize int64) *CommitHasher { c := new(CommitHasher) - c.fmt = getNewFixedMerkleTree() - c.vt = getNewValidationTree(dataSize) + c.md5hasher = md5Pool.Get().(hash.Hash) c.isInitialized = true c.doneChan = make(chan struct{}) c.dataSize = dataSize @@ -434,7 +444,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil defer f.Close() var toFinalize bool var totalWritten int64 - + logging.Logger.Info("hasher_start", zap.String("fileHash", filePathHash), zap.String("fileName", fileName), zap.String("tempFilePath", tempFilePath)) for { select { case <-ctx.Done(): @@ -456,7 +466,6 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil } else if pq.DataBytes == 0 { continue } - logging.Logger.Info("hasher_pop", zap.Int64("offset", pq.Offset), zap.Int64("dataBytes", pq.DataBytes), zap.Any("toFinalize", toFinalize), zap.Int64("dataSize", c.dataSize), zap.String("filename", fileName), zap.Int64("totalWritten", totalWritten)) bufSize := 2 * BufferSize if pq.DataBytes < int64(bufSize) { bufSize = int(pq.DataBytes) @@ -474,7 +483,7 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil pq.DataBytes -= int64(n) pq.Offset += int64(n) totalWritten += int64(n) - _, err = c.Write(buf[:n]) + _, err = c.md5hasher.Write(buf[:n]) if err != nil { logging.Logger.Error("hasher_write", zap.Error(err), zap.Int("n", n), zap.Int64("offset", pq.Offset), zap.Int64("dataBytes", pq.DataBytes), zap.Int64("dataSize", c.dataSize), zap.String("filename", fileName), zap.Int64("totalWritten", totalWritten)) c.hashErr = err @@ -483,13 +492,12 @@ func (c *CommitHasher) Start(ctx context.Context, connID, allocID, fileName, fil } buf = nil if toFinalize { - c.hashErr = c.Finalize() return } } } -func (c *CommitHasher) Wait(ctx context.Context, connID, allocID, fileName, filePathHash string) error { +func (c *CommitHasher) Wait(ctx context.Context) error { select { case <-c.doneChan: return c.hashErr @@ -568,3 +576,10 @@ func (c *CommitHasher) GetFixedMerkleRoot() string { func (c *CommitHasher) GetValidationMerkleRoot() string { return hex.EncodeToString(c.vt.GetValidationRoot()) } + +func (c *CommitHasher) GetMd5Hash() string { + hash := hex.EncodeToString(c.md5hasher.Sum(nil)) + c.md5hasher.Reset() + md5Pool.Put(c.md5hasher) + return hash +} diff --git a/code/go/0chain.net/blobbercore/handler/auth_ticket.go b/code/go/0chain.net/blobbercore/handler/auth_ticket.go index 89032880c..519c8c975 100644 --- a/code/go/0chain.net/blobbercore/handler/auth_ticket.go +++ b/code/go/0chain.net/blobbercore/handler/auth_ticket.go @@ -2,9 +2,12 @@ package handler import ( "context" + "fmt" + "net/http" + "github.com/0chain/blobber/code/go/0chain.net/core/node" "github.com/0chain/common/core/common" - "net/http" + "github.com/0chain/gosdk/core/encryption" ) // swagger:model AuthTicketResponse @@ -19,28 +22,33 @@ type AuthTicketResponse struct { // // parameters: // -// +name: Zbox-Signature -// in: header -// type: string -// description: Digital signature to verify that the sender is 0box service. -// +name: client_id -// type: string -// in: query -// description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key. +// +name: Zbox-Signature +// in: header +// type: string +// description: Digital signature to verify that the sender is 0box service. +// +name: client_id +// type: string +// in: query +// description: Client ID is used as a payload to the token generated. The token represents a signed version of this string by the blobber's private key. // // responses: -// 200: AuthTicketResponse +// +// 200: AuthTicketResponse func GenerateAuthTicket(ctx context.Context, r *http.Request) (interface{}, error) { + clientID := r.URL.Query().Get("client_id") if clientID == "" { return nil, common.NewError("missing_client_id", "client_id is required") } - signature, err := node.Self.Sign(clientID) + round := r.URL.Query().Get("round") + + payload := encryption.Hash(fmt.Sprintf("%s_%s", clientID, round)) + + signature, err := node.Self.Sign(payload) if err != nil { return nil, common.NewError("signature_failed", "signature failed") } - return &AuthTicketResponse{ AuthTicket: signature, }, nil diff --git a/code/go/0chain.net/blobbercore/handler/client_quota.go b/code/go/0chain.net/blobbercore/handler/client_quota.go index cee45a830..4335192d4 100644 --- a/code/go/0chain.net/blobbercore/handler/client_quota.go +++ b/code/go/0chain.net/blobbercore/handler/client_quota.go @@ -8,6 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -84,6 +86,7 @@ func AddWriteMarkerCount(clientID string, zeroSizeWM bool) { cs.TotalZeroWM++ } if cs.TotalZeroWM > config.Configuration.CommitZeroLimitDaily || cs.TotalWM > config.Configuration.CommitLimitDaily { + logging.Logger.Info("Client blacklisted", zap.String("client_id", clientID), zap.Int64("total_write_marker", cs.TotalWM), zap.Int64("total_zero_write_marker", cs.TotalZeroWM), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily), zap.Int64("commit_zero_limit_daily", config.Configuration.CommitZeroLimitDaily)) SetBlacklist(clientID) } } @@ -136,6 +139,7 @@ func saveClientStats() { } func startBlackListWorker(ctx context.Context) { + logging.Logger.Info("Starting black list worker", zap.Int64("upload_limit", config.Configuration.UploadLimitMonthly), zap.Int64("download_limit", config.Configuration.BlockLimitMonthly), zap.Int64("commit_limit", config.Configuration.CommitLimitMonthly), zap.Int64("commit_zero_limit", config.Configuration.CommitZeroLimitDaily), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily)) BlackListWorkerTime := 24 * time.Hour if config.Development() { BlackListWorkerTime = 10 * time.Second diff --git a/code/go/0chain.net/blobbercore/handler/file_command_delete.go b/code/go/0chain.net/blobbercore/handler/file_command_delete.go index 38497b283..aef760aaf 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_delete.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_delete.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "path/filepath" "github.com/0chain/gosdk/constants" "gorm.io/gorm" @@ -41,6 +42,10 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_parameters", "Invalid path") } + if filepath.Clean(path) != path { + return common.NewError("invalid_parameters", "Invalid path") + } + cmd.path = path connectionID, ok := common.GetField(req, "connection_id") @@ -50,13 +55,23 @@ func (cmd *DeleteFileCommand) IsValidated(ctx context.Context, req *http.Request cmd.connectionID = connectionID var err error lookUpHash := reference.GetReferenceLookup(allocationObj.ID, path) - cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "size", "hash", "fixed_merkle_root"}) + cmd.existingFileRef, err = reference.GetLimitedRefFieldsByLookupHashWith(ctx, allocationObj.ID, lookUpHash, []string{"path", "name", "type", "id", "size"}) if err != nil { if errors.Is(gorm.ErrRecordNotFound, err) { return common.ErrFileWasDeleted } return common.NewError("bad_db_operation", err.Error()) } + if cmd.existingFileRef.Type == reference.DIRECTORY { + // check if directory is empty + empty, err := reference.IsDirectoryEmpty(ctx, cmd.existingFileRef.ID) + if err != nil { + return common.NewError("bad_db_operation", err.Error()) + } + if !empty { + return common.NewError("invalid_operation", "Directory is not empty") + } + } cmd.existingFileRef.LookupHash = lookUpHash return nil } @@ -82,12 +97,10 @@ func (cmd *DeleteFileCommand) ProcessContent(_ context.Context, allocationObj *a connectionID := cmd.connectionID cmd.changeProcessor = &allocation.DeleteFileChange{ConnectionID: connectionID, AllocationID: allocationObj.ID, Name: cmd.existingFileRef.Name, - Hash: cmd.existingFileRef.Hash, Path: cmd.existingFileRef.Path, Size: deleteSize} + LookupHash: cmd.existingFileRef.LookupHash, Path: cmd.existingFileRef.Path, Size: deleteSize, Type: cmd.existingFileRef.Type} result := allocation.UploadResult{} result.Filename = cmd.existingFileRef.Name - result.ValidationRoot = cmd.existingFileRef.ValidationRoot - result.FixedMerkleRoot = cmd.existingFileRef.FixedMerkleRoot result.Size = cmd.existingFileRef.Size result.UpdateChange = true diff --git a/code/go/0chain.net/blobbercore/handler/file_command_update.go b/code/go/0chain.net/blobbercore/handler/file_command_update.go index 1b1daf5d4..aaadff53d 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_update.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_update.go @@ -14,7 +14,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" sdkConst "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" ) @@ -72,19 +71,19 @@ func (cmd *UpdateFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_connection", "Invalid connection id") } - cmd.fileChanger.PathHash = encryption.Hash(cmd.fileChanger.Path) + cmd.fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, cmd.fileChanger.Path) if cmd.fileChanger.ChunkSize <= 0 { cmd.fileChanger.ChunkSize = fileref.CHUNK_SIZE } - cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + cmd.existingFileRef = allocation.GetExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash) if cmd.existingFileRef == nil { cmd.existingFileRef, _ = reference.GetReference(ctx, allocationObj.ID, cmd.fileChanger.Path) if cmd.existingFileRef == nil { return common.NewError("invalid_file_update", "File at path does not exist for update") } - allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash, cmd.existingFileRef) //nolint:errcheck + allocation.SaveExistingRef(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash, cmd.existingFileRef) //nolint:errcheck } thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) @@ -112,7 +111,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj result.Filename = cmd.fileChanger.Filename defer cmd.contentFile.Close() - filePathHash := cmd.fileChanger.PathHash + filePathHash := cmd.fileChanger.LookupHash connID := cmd.fileChanger.ConnectionID fileInputData := &filestore.FileInputData{ @@ -120,7 +119,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj Path: cmd.fileChanger.Path, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, - FilePathHash: filePathHash, + LookupHash: filePathHash, Size: cmd.fileChanger.Size, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connID, fileInputData, cmd.contentFile) @@ -128,8 +127,6 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj return result, common.NewError("upload_error", "Failed to upload the file. "+err.Error()) } - result.ValidationRoot = fileOutputData.ValidationRoot - result.FixedMerkleRoot = fileOutputData.FixedMerkleRoot result.Size = fileOutputData.Size cmd.fileChanger.AllocationID = allocationObj.ID @@ -156,7 +153,7 @@ func (cmd *UpdateFileCommand) ProcessContent(ctx context.Context, allocationObj } } - saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size) + saveChange, err := allocation.SaveFileChange(ctx, connID, cmd.fileChanger.LookupHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size-cmd.existingFileRef.Size) if err != nil { return result, err } @@ -182,7 +179,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat connectionID := cmd.fileChanger.ConnectionID if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() - thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash} + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, LookupHash: cmd.fileChanger.LookupHash} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) @@ -196,7 +193,7 @@ func (cmd *UpdateFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat } func (cmd *UpdateFileCommand) reloadChange() { - changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash) if changer != nil && changer.ThumbnailHash != "" { cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize diff --git a/code/go/0chain.net/blobbercore/handler/file_command_upload.go b/code/go/0chain.net/blobbercore/handler/file_command_upload.go index 88fea77f9..d2859d69b 100644 --- a/code/go/0chain.net/blobbercore/handler/file_command_upload.go +++ b/code/go/0chain.net/blobbercore/handler/file_command_upload.go @@ -17,7 +17,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/core/common" - "github.com/0chain/blobber/code/go/0chain.net/core/encryption" "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/zboxcore/fileref" @@ -77,25 +76,15 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request return common.NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fileChanger.Path)) } + if filepath.Clean(fileChanger.Path) != fileChanger.Path { + return common.NewError("invalid_path", fmt.Sprintf("%v is not a clean path", fileChanger.Path)) + } + if fileChanger.ConnectionID == "" { return common.NewError("invalid_connection", "Invalid connection id") } - fileChanger.PathHash = encryption.Hash(fileChanger.Path) - - if fileChanger.UploadOffset == 0 { - isExist, err := reference.IsRefExist(ctx, allocationObj.ID, fileChanger.Path) - - if err != nil { - logging.Logger.Error(err.Error()) - return common.NewError("database_error", "Got db error while getting ref") - } - - if isExist { - msg := fmt.Sprintf("File at path :%s: already exists", fileChanger.Path) - return common.NewError("duplicate_file", msg) - } - } + fileChanger.LookupHash = reference.GetReferenceLookup(allocationObj.ID, fileChanger.Path) thumbFile, thumbHeader, _ := req.FormFile(UploadThumbnailFile) if thumbHeader != nil { @@ -111,6 +100,9 @@ func (cmd *UploadFileCommand) IsValidated(ctx context.Context, req *http.Request if fileChanger.ChunkSize <= 0 { fileChanger.ChunkSize = fileref.CHUNK_SIZE } + if allocationObj.CanUpdate() { + fileChanger.CanUpdate = true + } origfile, _, err := req.FormFile(UploadFile) if err != nil { @@ -135,7 +127,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj ChunkSize: cmd.fileChanger.ChunkSize, UploadOffset: cmd.fileChanger.UploadOffset, IsFinal: cmd.fileChanger.IsFinal, - FilePathHash: cmd.fileChanger.PathHash, + LookupHash: cmd.fileChanger.LookupHash, Size: cmd.fileChanger.Size, } fileOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, fileInputData, cmd.contentFile) @@ -144,10 +136,8 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj return result, common.NewError("upload_error", "Failed to write file. "+err.Error()) } result.Filename = cmd.fileChanger.Filename - result.ValidationRoot = fileOutputData.ValidationRoot result.Size = fileOutputData.Size - allocationSize := allocation.GetConnectionObjSize(connectionID) + cmd.fileChanger.Size cmd.fileChanger.AllocationID = allocationObj.ID cmd.allocationChange = &allocation.AllocationChange{} @@ -171,7 +161,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj } } - saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.PathHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size) + saveChange, err := allocation.SaveFileChange(ctx, connectionID, cmd.fileChanger.LookupHash, cmd.fileChanger.Filename, cmd, cmd.fileChanger.IsFinal, cmd.fileChanger.Size, cmd.fileChanger.UploadOffset, fileOutputData.Size, cmd.fileChanger.Size) if err != nil { logging.Logger.Error("UploadFileCommand.ProcessContent", zap.Error(err)) return result, err @@ -185,6 +175,7 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj return result, err } } + allocationSize := allocation.GetConnectionObjSize(connectionID) if allocationObj.BlobberSizeUsed+allocationSize > allocationObj.BlobberSize { return result, common.NewError("max_allocation_size", "Max size reached for the allocation with this blobber") @@ -195,12 +186,12 @@ func (cmd *UploadFileCommand) ProcessContent(ctx context.Context, allocationObj // ProcessThumbnail flush thumbnail file to FileStorage if it has. func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocation) error { - logging.Logger.Info("ProcessThumbnail: ", zap.String("allocationID: ", cmd.fileChanger.AllocationID)) + connectionID := cmd.fileChanger.ConnectionID if cmd.thumbHeader != nil { defer cmd.thumbFile.Close() - thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, FilePathHash: cmd.fileChanger.PathHash} + thumbInputData := &filestore.FileInputData{Name: cmd.thumbHeader.Filename, Path: cmd.fileChanger.Path, IsThumbnail: true, LookupHash: cmd.fileChanger.LookupHash} thumbOutputData, err := filestore.GetFileStore().WriteFile(allocationObj.ID, connectionID, thumbInputData, cmd.thumbFile) if err != nil { return common.NewError("upload_error", "Failed to upload the thumbnail. "+err.Error()) @@ -214,7 +205,7 @@ func (cmd *UploadFileCommand) ProcessThumbnail(allocationObj *allocation.Allocat } func (cmd *UploadFileCommand) reloadChange() { - changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.PathHash) + changer := allocation.GetFileChanger(cmd.fileChanger.ConnectionID, cmd.fileChanger.LookupHash) if changer != nil && changer.ThumbnailHash != "" { cmd.fileChanger.ThumbnailFilename = changer.ThumbnailFilename cmd.fileChanger.ThumbnailSize = changer.ThumbnailSize @@ -234,6 +225,7 @@ func (cmd *UploadFileCommand) AddChange(ctx context.Context) error { connectionInput, _ := cmd.fileChanger.Marshal() cmd.allocationChange.LookupHash = reference.GetReferenceLookup(cmd.fileChanger.AllocationID, cmd.fileChanger.Path) cmd.allocationChange.Input = connectionInput + logging.Logger.Info("AddChange: ", zap.String("connectionID", cmd.allocationChange.ConnectionID), zap.String("lookupHash", cmd.allocationChange.LookupHash)) return cmd.allocationChange.Create(ctx) } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index cc110b268..9b6f079b9 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -274,6 +274,10 @@ func setupHandlers(s *mux.Router) { RateLimitByGeneralRL(WithTxHandler(LoadPlaylistFile))). Methods(http.MethodGet, http.MethodOptions) + // Add at the end of the file, before any closing brackets + // Handler for /v1/auth/signature + s.HandleFunc("/v1/auth/getAuthTicketWithClientId", RateLimitByGeneralRL(common.ToJSONResponse(GenerateAuthSignature))) + } func WithReadOnlyConnection(handler common.JSONResponderF) common.JSONResponderF { @@ -2024,3 +2028,76 @@ func HTMLHeader(w http.ResponseWriter, title string) { func HTMLFooter(w http.ResponseWriter) { fmt.Fprintf(w, "") } + +// Add at the end of the file, before any closing brackets +// Handler for /v1/auth/signature +func GenerateAuthSignature(ctx context.Context, r *http.Request) (interface{}, error) { + type authTicketRequest struct { + PublicKey string `json:"public_key"` + PrivateKey string `json:"private_key"` + BlobberURL string `json:"blobber_url"` + ClientID string `json:"client_id"` + RoundExpiry int `json:"round_expiry"` + } + type authTicketResponse struct { + AuthTicket string `json:"auth_ticket,omitempty"` + ClientID string `json:"client_id,omitempty"` + Error string `json:"error,omitempty"` + } + var req authTicketRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return authTicketResponse{Error: "invalid request: " + err.Error()}, nil + } + + signatureScheme := zcncrypto.NewSignatureScheme("bls0chain") + _ = signatureScheme.SetPrivateKey(req.PrivateKey) + _ = signatureScheme.SetPublicKey(req.PublicKey) + + signature, err := signatureScheme.Sign(hex.EncodeToString([]byte(req.PublicKey))) + if err != nil { + return authTicketResponse{Error: err.Error()}, nil + } + + url := req.BlobberURL + "/v1/auth/generate?client_id=" + req.ClientID + "&round=" + strconv.FormatInt(int64(req.RoundExpiry), 10) + httpReq, err := http.NewRequest("GET", url, nil) + if err != nil { + return authTicketResponse{Error: err.Error()}, nil + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Zbox-Signature", signature) + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return authTicketResponse{Error: err.Error()}, nil + } + defer resp.Body.Close() + var responseMap map[string]string + err = json.NewDecoder(resp.Body).Decode(&responseMap) + if err != nil { + return authTicketResponse{Error: err.Error()}, nil + } + authTicket := responseMap["auth_ticket"] + if authTicket == "" { + return authTicketResponse{Error: "Error getting auth ticket from blobber"}, nil + } + + // Read client_id from blob_op_wallet.json just before returning + data, err := os.ReadFile("/var/0chain/blobber/blob_op_wallet.json") + if err != nil { + return authTicketResponse{Error: "read blob_op_wallet: " + err.Error()}, nil + } + type wallet struct { + ClientID string `json:"client_id"` + ClientKey string `json:"client_key"` + Keys []struct { + PublicKey string `json:"public_key"` + PrivateKey string `json:"private_key"` + } `json:"keys"` + } + var wlt wallet + if err := json.Unmarshal(data, &wlt); err != nil { + return authTicketResponse{Error: "parse blob_op_wallet: " + err.Error()}, nil + } + + return authTicketResponse{AuthTicket: authTicket, ClientID: wlt.ClientID}, nil +} diff --git a/code/go/0chain.net/blobbercore/handler/handler_common.go b/code/go/0chain.net/blobbercore/handler/handler_common.go index ebd6a9b64..8c9a46bbd 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_common.go +++ b/code/go/0chain.net/blobbercore/handler/handler_common.go @@ -8,8 +8,6 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" - "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/build" "github.com/0chain/blobber/code/go/0chain.net/core/chain" "github.com/0chain/blobber/code/go/0chain.net/core/common" @@ -127,10 +125,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu mutex := lock.GetMutex(allocation.Allocation{}.TableName(), allocationID) Logger.Info("Locking allocation", zap.String("allocation_id", allocationID)) - wmSet := writemarker.SetCommittingMarker(allocationID, true) - if !wmSet { - return nil, http.StatusBadRequest, common.NewError("pending_markers", "Committing marker set failed") - } mutex.Lock() defer mutex.Unlock() ctx = GetMetaDataStore().CreateTransaction(ctx) @@ -144,7 +138,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu if rollErr != nil { Logger.Error("couldn't rollback", zap.Error(err)) } - writemarker.SetCommittingMarker(allocationID, false) } }() @@ -160,14 +153,6 @@ func WithStatusConnectionForWM(handler common.StatusCodeResponderF) common.Statu } Logger.Info("commit_success", zap.String("allocation_id", allocationID), zap.Any("response", resp)) - - if blobberRes, ok := resp.(*blobberhttp.CommitResult); ok { - // Save the write marker data - writemarker.SaveMarkerData(allocationID, blobberRes.WriteMarker.WM.Timestamp, blobberRes.WriteMarker.WM.ChainLength) - } else { - Logger.Error("Invalid response type for commit handler") - return resp, http.StatusInternalServerError, common.NewError("invalid_response_type", "Invalid response type for commit handler") - } return } } diff --git a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go index 424b4c99f..bd073d88e 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_middlewares.go +++ b/code/go/0chain.net/blobbercore/handler/handler_middlewares.go @@ -41,7 +41,7 @@ func UseRecovery(h http.Handler) http.Handler { defer func() { if err := recover(); err != nil { escapedUrl := sanitizeString(r.URL.String()) - logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err)) + logging.Logger.Error("[recover]http", zap.String("url", escapedUrl), zap.Any("err", err), zap.Stack("recover_stack")) } }() diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index 968eaa2a1..442d41d54 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -88,14 +88,6 @@ func readPreRedeem( return } -func checkPendingMarkers(ctx context.Context, allocationID string) error { - pending := writemarker.CheckProcessingMarker(allocationID) - if pending { - return common.NewError("pending_markers", "previous marker is still pending to be redeemed") - } - return nil -} - func writePreRedeem(ctx context.Context, alloc *allocation.Allocation, writeMarker *writemarker.WriteMarker, payerID string) (err error) { // check out read pool tokens if read_price > 0 var ( @@ -462,22 +454,17 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewErrorf("download_file", "BlockNum or NumBlocks is too large to convert to int") } - fromPreCommit := false + fromPreCommit := alloc.AllocationVersion == fileref.AllocationVersion if downloadMode == DownloadContentThumb { - - if fileref.IsPrecommit { - fromPreCommit = fileref.ThumbnailHash != fileref.PrevThumbnailHash - } - rbi := &filestore.ReadBlockInput{ AllocationID: alloc.ID, FileSize: fileref.ThumbnailSize, - Hash: fileref.ThumbnailHash, + Hash: fileref.LookupHash, StartBlockNum: int(dr.BlockNum), NumBlocks: int(dr.NumBlocks), IsThumbnail: true, IsPrecommit: fromPreCommit, - FilestoreVersion: fileref.FilestoreVersion, + FilestoreVersion: filestore.VERSION, } logging.Logger.Info("calling GetFileBlock for thumb", zap.Any("rbi", rbi)) @@ -486,20 +473,15 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, common.NewErrorf("download_file", "couldn't get thumbnail block: %v", err) } } else { - - if fileref.IsPrecommit { - fromPreCommit = fileref.ValidationRoot != fileref.PrevValidationRoot - } - rbi := &filestore.ReadBlockInput{ AllocationID: alloc.ID, FileSize: fileref.Size, - Hash: fileref.ValidationRoot, + Hash: fileref.LookupHash, StartBlockNum: int(dr.BlockNum), NumBlocks: int(dr.NumBlocks), VerifyDownload: dr.VerifyDownload, IsPrecommit: fromPreCommit, - FilestoreVersion: fileref.FilestoreVersion, + FilestoreVersion: filestore.VERSION, } logging.Logger.Info("calling GetFileBlock", zap.Any("rbi", rbi)) fileDownloadResponse, err = filestore.GetFileStore().GetFileBlock(rbi) @@ -588,7 +570,6 @@ func (fsh *StorageHandler) CreateConnection(ctx context.Context, r *http.Request } func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*blobberhttp.CommitResult, error) { - var prevChainHash string startTime := time.Now() if r.Method == "GET" { return nil, common.NewError("invalid_method", "Invalid method used for the upload URL. Use POST instead") @@ -628,26 +609,24 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b } elapsedGetLock := time.Since(startTime) - elapsedAllocation - - err = checkPendingMarkers(ctx, allocationObj.ID) - if err != nil { - Logger.Error("Error checking pending markers", zap.Error(err)) - return nil, common.NewError("pending_markers", "previous marker is still pending to be redeemed") - } - + var result blobberhttp.CommitResult connectionObj, err := allocation.GetAllocationChanges(ctx, connectionID, allocationID, clientID) if err != nil { // might be good to check if blobber already has stored writemarker return nil, common.NewErrorf("invalid_parameters", "Invalid connection id. Connection id was not found: %v", err) } + if connectionObj.Status == allocation.CommittedConnection { + result.Success = true + return &result, nil + } if len(connectionObj.Changes) == 0 { - if connectionObj.Status == allocation.NewConnection { - return nil, common.NewError("invalid_parameters", - "Invalid connection id. Connection not found.") - } - return nil, common.NewError("invalid_parameters", - "Invalid connection id. Connection does not have any changes.") + logging.Logger.Info("commit_write_empty", zap.String("connection_id", connectionID)) + } + + if len(connectionObj.Changes) > config.Configuration.MaxConnectionChanges { + return nil, common.NewError("max_connection_changes", + "Max connection changes reached. A connection can only have "+fmt.Sprintf("%v", config.Configuration.MaxConnectionChanges)+" changes") } elapsedGetConnObj := time.Since(startTime) - elapsedAllocation - elapsedGetLock @@ -656,184 +635,86 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - writeMarkerString := r.FormValue("write_marker") - if writeMarkerString == "" { - return nil, common.NewError("invalid_parameters", "Invalid write marker passed") - } - writeMarker := writemarker.WriteMarker{} - err = json.Unmarshal([]byte(writeMarkerString), &writeMarker) - if err != nil { - return nil, common.NewErrorf("invalid_parameters", - "Invalid parameters. Error parsing the writemarker for commit: %v", - err) - } - - var result blobberhttp.CommitResult - var latestWriteMarkerEntity *writemarker.WriteMarkerEntity - if allocationObj.AllocationRoot == "" { - latestWriteMarkerEntity = nil - } else { - latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx, - allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewErrorf("latest_write_marker_read_error", - "Error reading the latest write marker for allocation: %v", err) - } - if latestWriteMarkerEntity.Status == writemarker.Failed { - return nil, common.NewError("latest_write_marker_failed", - "Latest write marker is in failed state") - } - - if latestWriteMarkerEntity.Status != writemarker.Committed { - writeMarker.ChainLength = latestWriteMarkerEntity.WM.ChainLength - } - prevChainHash = latestWriteMarkerEntity.WM.ChainHash - } - - writemarkerEntity := &writemarker.WriteMarkerEntity{} - writemarkerEntity.WM = writeMarker - writemarkerEntity.WM.ChainLength += 1 - if writemarkerEntity.WM.ChainLength > config.Configuration.MaxChainLength { - return nil, common.NewError("chain_length_exceeded", "Chain length exceeded") - } - - err = writemarkerEntity.VerifyMarker(ctx, allocationObj, connectionObj, latestWriteMarkerEntity) - if err != nil { - result.AllocationRoot = allocationObj.AllocationRoot - result.ErrorMessage = "Verification of write marker failed: " + err.Error() - result.Success = false - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity - } - Logger.Error("verify_writemarker_failed", zap.Error(err)) - return &result, common.NewError("write_marker_verification_failed", result.ErrorMessage) - } - - elapsedVerifyWM := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - - var clientIDForWriteRedeem = writeMarker.ClientID - - if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { - return nil, err + if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize { + return nil, common.NewError("max_allocation_size", + "Max size reached for the allocation with this blobber") } - elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj - elapsedVerifyWM - - fileIDMetaStr := r.FormValue("file_id_meta") - if fileIDMetaStr == "" { - return nil, common.NewError("invalid_parameters", "Invalid file ID meta passed") + versionMarkerStr := r.FormValue("version_marker") + if versionMarkerStr == "" { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed") } - fileIDMeta := make(map[string]string, 0) - err = json.Unmarshal([]byte(fileIDMetaStr), &fileIDMeta) + versionMarker := writemarker.VersionMarker{} + err = json.Unmarshal([]byte(versionMarkerStr), &versionMarker) if err != nil { - return nil, common.NewError("unmarshall_error", - fmt.Sprintf("Error while unmarshalling file ID meta data: %s", err.Error())) + return nil, common.NewError("unmarshall_error", fmt.Sprintf("Error while unmarshalling version marker: %s", err.Error())) } - // Move preCommitDir to finalDir - err = connectionObj.MoveToFilestore(ctx) + err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey) if err != nil { - return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error())) - } - - elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - - rootRef, err := connectionObj.ApplyChanges( - ctx, writeMarker.AllocationRoot, writeMarker.PreviousAllocationRoot, writeMarker.Timestamp, fileIDMeta) - if err != nil { - Logger.Error("Error applying changes", zap.Error(err)) return nil, err } - if !rootRef.IsPrecommit { - return nil, common.NewError("no_root_change", "No change in root ref") - } - connectionObj.Size = rootRef.Size - allocationObj.BlobberSizeUsed - - if writemarkerEntity.WM.Size != connectionObj.Size { - return nil, common.NewError("write_marker_validation_failed", fmt.Sprintf("Write Marker size %v does not match the connection size %v", writemarkerEntity.WM.Size, connectionObj.Size)) - } - - if allocationObj.BlobberSizeUsed+connectionObj.Size > allocationObj.BlobberSize { - return nil, common.NewError("max_allocation_size", - "Max size reached for the allocation with this blobber") - } - - if latestWriteMarkerEntity != nil && latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size != writeMarker.ChainSize { - return nil, common.NewErrorf("invalid_chain_size", - "Invalid chain size. expected:%v got %v", latestWriteMarkerEntity.WM.ChainSize+connectionObj.Size, writeMarker.ChainSize) - } else if latestWriteMarkerEntity == nil && connectionObj.Size != writeMarker.ChainSize { - return nil, common.NewErrorf("invalid_chain_size", - "Invalid chain size. expected:%v got %v", connectionObj.Size, writeMarker.ChainSize) - } - elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - - allocationRoot := rootRef.Hash - fileMetaRoot := rootRef.FileMetaHash - if allocationRoot != writeMarker.AllocationRoot { - result.AllocationRoot = allocationObj.AllocationRoot - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity + // Move preCommitDir to finalDir + if allocationObj.IsRedeemRequired { + err = connectionObj.MoveToFilestore(ctx, allocationObj.AllocationVersion) + if err != nil { + return nil, common.NewError("move_to_filestore_error", fmt.Sprintf("Error while moving to filestore: %s", err.Error())) } - result.Success = false - result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." + - " Expected hash: " + allocationRoot - return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) - } - - chainHash := writemarker.CalculateChainHash(prevChainHash, allocationRoot) - if chainHash != writeMarker.ChainHash { - return nil, common.NewError("chain_hash_mismatch", "Chain hash in the write marker does not match the calculated chain hash") } - if fileMetaRoot != writeMarker.FileMetaRoot { - // result.AllocationRoot = allocationObj.AllocationRoot - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity + elapsedMoveToFilestore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj + if len(connectionObj.Changes) > 0 { + err = connectionObj.ApplyChanges( + ctx, common.Now(), versionMarker.Version) + if err != nil { + Logger.Error("Error applying changes", zap.Error(err)) + return nil, err } - result.Success = false - result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." + - " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot - return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage) } - writemarkerEntity.ConnectionID = connectionObj.ID - writemarkerEntity.ClientPublicKey = clientKey + elapsedApplyChanges := time.Since(startTime) - elapsedAllocation - elapsedGetLock - + elapsedGetConnObj - elapsedMoveToFilestore db := datastore.GetStore().GetTransaction(ctx) - writemarkerEntity.Latest = true - if err = db.Create(writemarkerEntity).Error; err != nil { - return nil, common.NewError("write_marker_error", "Error persisting the write marker") + err = db.Create(&versionMarker).Error + if err != nil { + return nil, common.NewError("db_error", fmt.Sprintf("Error while saving version marker: %s", err.Error())) } - allocationObj.AllocationRoot = allocationRoot - allocationObj.FileMetaRoot = fileMetaRoot - allocationObj.IsRedeemRequired = true + allocationObj.PrevBlobberSizeUsed = allocationObj.BlobberSizeUsed + allocationObj.PrevUsedSize = allocationObj.UsedSize allocationObj.BlobberSizeUsed += connectionObj.Size allocationObj.UsedSize += connectionObj.Size + allocationObj.AllocationVersion = versionMarker.Version + if len(connectionObj.Changes) == 0 { + allocationObj.IsRedeemRequired = false + } else { + allocationObj.IsRedeemRequired = true + } updateMap := map[string]interface{}{ - "allocation_root": allocationRoot, - "file_meta_root": fileMetaRoot, - "used_size": allocationObj.UsedSize, - "blobber_size_used": allocationObj.BlobberSizeUsed, - "is_redeem_required": true, + "used_size": allocationObj.UsedSize, + "blobber_size_used": allocationObj.BlobberSizeUsed, + "is_redeem_required": allocationObj.IsRedeemRequired, + "allocation_version": versionMarker.Version, + "prev_used_size": allocationObj.PrevUsedSize, + "prev_blobber_size_used": allocationObj.PrevBlobberSizeUsed, } updateOption := func(a *allocation.Allocation) { - a.AllocationRoot = allocationRoot - a.FileMetaRoot = fileMetaRoot - a.IsRedeemRequired = true + a.IsRedeemRequired = allocationObj.IsRedeemRequired a.BlobberSizeUsed = allocationObj.BlobberSizeUsed a.UsedSize = allocationObj.UsedSize + a.AllocationVersion = allocationObj.AllocationVersion + a.PrevUsedSize = allocationObj.PrevUsedSize + a.PrevBlobberSizeUsed = allocationObj.PrevBlobberSizeUsed } if err = allocation.Repo.UpdateAllocation(ctx, allocationObj, updateMap, updateOption); err != nil { - return nil, common.NewError("allocation_write_error", "Error persisting the allocation object") + return nil, common.NewError("allocation_write_error", "Error persisting the allocation object "+err.Error()) } elapsedSaveAllocation := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - elapsedApplyChanges + elapsedGetConnObj - elapsedMoveToFilestore - elapsedApplyChanges err = connectionObj.CommitToFileStore(ctx) if err != nil { @@ -841,33 +722,37 @@ func (fsh *StorageHandler) CommitWrite(ctx context.Context, r *http.Request) (*b return nil, common.NewError("file_store_error", "Error committing to file store. "+err.Error()) } } - elapsedCommitStore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedVerifyWM - elapsedWritePreRedeem - elapsedApplyChanges - elapsedSaveAllocation - logging.Logger.Info("commit_filestore", zap.String("allocation_id", allocationId), zap.String("allocation_root", allocationRoot)) + elapsedCommitStore := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedGetConnObj - elapsedMoveToFilestore - elapsedApplyChanges - elapsedSaveAllocation + logging.Logger.Info("commit_filestore", zap.String("allocation_id", allocationId)) connectionObj.DeleteChanges(ctx) db.Model(connectionObj).Updates(allocation.AllocationChangeCollector{Status: allocation.CommittedConnection}) result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = writemarkerEntity result.Success = true result.ErrorMessage = "" - commitOperation := connectionObj.Changes[0].Operation - input := connectionObj.Changes[0].Input + var ( + commitOperation string + input string + ) + if len(connectionObj.Changes) > 0 { + commitOperation = connectionObj.Changes[0].Operation + input = connectionObj.Changes[0].Input + } else { + commitOperation = "[commit]empty" + input = "[commit]empty" + } //Delete connection object and its changes - db.Delete(connectionObj) go allocation.DeleteConnectionObjEntry(connectionID) go AddWriteMarkerCount(clientID, connectionObj.Size <= 0) Logger.Info("[commit]"+commitOperation, zap.String("alloc_id", allocationID), - zap.String("allocation_root", writeMarker.AllocationRoot), zap.String("input", input), zap.Duration("get_alloc", elapsedAllocation), zap.Duration("get-lock", elapsedGetLock), zap.Duration("get-conn-obj", elapsedGetConnObj), - zap.Duration("verify-wm", elapsedVerifyWM), - zap.Duration("write-pre-redeem", elapsedWritePreRedeem), zap.Duration("move-to-filestore", elapsedMoveToFilestore), zap.Duration("apply-changes", elapsedApplyChanges), zap.Duration("save-allocation", elapsedSaveAllocation), @@ -906,7 +791,10 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i if new_name == "" { return nil, common.NewError("invalid_parameters", "Invalid name") } - + if filepath.Base(new_name) != new_name { + logging.Logger.Error("invalid_parameters", zap.String("new_name", new_name), zap.String("base", filepath.Base(new_name))) + return nil, common.NewError("invalid_parameters", "Invalid name") + } pathHash, err := pathHashFromReq(r, allocationID) if err != nil { return nil, err @@ -926,7 +814,7 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return nil, common.NewError("meta_error", "Error reading metadata for connection") } - objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root", "type"}) + objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"}) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) @@ -936,6 +824,16 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i return nil, common.NewError("invalid_operation", "cannot rename root path") } + if objectRef.Type != reference.FILE { + isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID) + if err != nil { + return nil, common.NewError("invalid_operation", "Error checking if directory is empty") + } + if !isEmpty { + return nil, common.NewError("invalid_operation", "Directory is not empty") + } + } + allocationChange := &allocation.AllocationChange{} allocationChange.ConnectionID = connectionObj.ID allocationChange.Size = 0 @@ -954,9 +852,6 @@ func (fsh *StorageHandler) RenameObject(ctx context.Context, r *http.Request) (i result := &allocation.UploadResult{} result.Filename = new_name - result.Hash = objectRef.Hash - result.ValidationRoot = objectRef.ValidationRoot - result.FixedMerkleRoot = objectRef.FixedMerkleRoot result.Size = objectRef.Size return result, nil @@ -993,7 +888,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if destPath == "" { return nil, common.NewError("invalid_parameters", "Invalid destination for operation") } - + destPath = filepath.Clean(destPath) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { return nil, err @@ -1013,7 +908,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int return nil, common.NewError("meta_error", "Error reading metadata for connection") } - objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) + objectRef, err := reference.GetLimitedRefFieldsByLookupHash(ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"}) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) @@ -1021,7 +916,17 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int if objectRef.ParentPath == destPath || objectRef.Path == destPath { return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot copy to the same parent directory.") } + if objectRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID) + if err != nil { + return nil, common.NewError("invalid_operation", "Error checking if directory is empty") + } + if !isEmpty { + return nil, common.NewError("invalid_operation", "Directory is not empty") + } + } newPath := filepath.Join(destPath, objectRef.Name) + newPath = filepath.Clean(newPath) paths, err := common.GetParentPaths(newPath) if err != nil { return nil, err @@ -1052,8 +957,7 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int allocationChange.LookupHash = pathHash allocationChange.Operation = constants.FileOperationCopy dfc := &allocation.CopyFileChange{ConnectionID: connectionObj.ID, - AllocationID: connectionObj.AllocationID, DestPath: destPath} - dfc.SrcPath = objectRef.Path + AllocationID: connectionObj.AllocationID, DestPath: newPath, Type: objectRef.Type, SrcPath: objectRef.Path} allocation.UpdateConnectionObjSize(connectionID, allocationChange.Size) connectionObj.AddChange(allocationChange, dfc) @@ -1065,9 +969,6 @@ func (fsh *StorageHandler) CopyObject(ctx context.Context, r *http.Request) (int result := &allocation.UploadResult{} result.Filename = objectRef.Name - result.Hash = objectRef.Hash - result.ValidationRoot = objectRef.ValidationRoot - result.FixedMerkleRoot = objectRef.FixedMerkleRoot result.Size = objectRef.Size return result, nil } @@ -1103,6 +1004,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int if destPath == "" { return nil, common.NewError("invalid_parameters", "Invalid destination for operation") } + destPath = filepath.Clean(destPath) pathHash, err := pathHashFromReq(r, allocationID) if err != nil { @@ -1124,7 +1026,7 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int } objectRef, err := reference.GetLimitedRefFieldsByLookupHash( - ctx, allocationID, pathHash, []string{"id", "name", "path", "hash", "size", "validation_root", "fixed_merkle_root"}) + ctx, allocationID, pathHash, []string{"id", "name", "path", "size", "type"}) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid file path. "+err.Error()) @@ -1133,7 +1035,17 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int if objectRef.ParentPath == destPath { return nil, common.NewError("invalid_parameters", "Invalid destination path. Cannot move to the same parent directory.") } + if objectRef.Type == reference.DIRECTORY { + isEmpty, err := reference.IsDirectoryEmpty(ctx, objectRef.ID) + if err != nil { + return nil, common.NewError("invalid_operation", "Error checking if directory is empty") + } + if !isEmpty { + return nil, common.NewError("invalid_operation", "Directory is not empty") + } + } newPath := filepath.Join(destPath, objectRef.Name) + newPath = filepath.Clean(newPath) paths, err := common.GetParentPaths(newPath) if err != nil { return nil, err @@ -1167,7 +1079,8 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int ConnectionID: connectionObj.ID, AllocationID: connectionObj.AllocationID, SrcPath: objectRef.Path, - DestPath: destPath, + DestPath: newPath, + Type: objectRef.Type, } dfc.SrcPath = objectRef.Path connectionObj.AddChange(allocationChange, dfc) @@ -1180,9 +1093,6 @@ func (fsh *StorageHandler) MoveObject(ctx context.Context, r *http.Request) (int result := &allocation.UploadResult{} result.Filename = objectRef.Name - result.Hash = objectRef.Hash - result.ValidationRoot = objectRef.ValidationRoot - result.FixedMerkleRoot = objectRef.FixedMerkleRoot result.Size = objectRef.Size return result, nil } @@ -1194,7 +1104,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn return nil, common.NewError("invalid_parameters", "Invalid path") } fileRef, err := reference.GetLimitedRefFieldsByPath(ctx, connectionObj.AllocationID, path, - []string{"path", "name", "size", "hash", "validation_root", "fixed_merkle_root"}) + []string{"path", "name", "size"}) if err != nil { Logger.Error("invalid_file", zap.Error(err)) @@ -1209,7 +1119,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn allocationChange.Operation = constants.FileOperationDelete dfc := &allocation.DeleteFileChange{ConnectionID: connectionObj.ID, AllocationID: connectionObj.AllocationID, Name: fileRef.Name, - Hash: fileRef.Hash, Path: fileRef.Path, Size: deleteSize} + LookupHash: fileRef.LookupHash, Path: fileRef.Path, Size: deleteSize} allocation.UpdateConnectionObjSize(connectionObj.ID, allocationChange.Size) @@ -1217,9 +1127,7 @@ func (fsh *StorageHandler) DeleteFile(ctx context.Context, r *http.Request, conn result := &allocation.UploadResult{} result.Filename = fileRef.Name - result.Hash = fileRef.Hash - result.ValidationRoot = fileRef.ValidationRoot - result.FixedMerkleRoot = fileRef.FixedMerkleRoot + result.Hash = fileRef.LookupHash result.Size = fileRef.Size return result, nil @@ -1335,6 +1243,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*all allocationID := ctx.Value(constants.ContextKeyAllocationID).(string) allocationTx := ctx.Value(constants.ContextKeyAllocation).(string) clientID := ctx.Value(constants.ContextKeyClient).(string) + logging.Logger.Info("writeFile", zap.String("allocation_id", allocationID)) connectionID, ok := common.GetField(r, "connection_id") if !ok { logging.Logger.Error("no_connection_id", zap.String("alloc_id", allocationID)) @@ -1359,7 +1268,7 @@ func (fsh *StorageHandler) WriteFile(ctx context.Context, r *http.Request) (*all if err != nil { return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - + logging.Logger.Info("writeFileAllocation") if allocationObj.OwnerID != clientID && allocationObj.RepairerID != clientID { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner or the payer of the allocation") } @@ -1463,19 +1372,14 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return nil, common.NewError("invalid_parameters", "Invalid allocation id passed."+err.Error()) } - if allocationObj.AllocationRoot == "" { - Logger.Error("Allocation root is not set", zap.String("allocation_id", allocationObj.ID)) - return nil, common.NewError("invalid_parameters", "Allocation root is not set") + if allocationObj.AllocationVersion == 0 { + Logger.Error("Allocation version is 0", zap.String("allocation_id", allocationObj.ID)) + return nil, common.NewError("invalid_parameters", "Allocation version is not set") } elapsedAllocation := time.Since(startTime) allocationID := allocationObj.ID - connectionID, ok := common.GetField(r, "connection_id") - if !ok { - return nil, common.NewError("invalid_parameters", "Invalid connection id passed") - } - elapsedGetLock := time.Since(startTime) - elapsedAllocation if clientID == "" || clientKey == "" { @@ -1486,105 +1390,55 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") } - writeMarkerString := r.FormValue("write_marker") - writeMarker := writemarker.WriteMarker{} - err = json.Unmarshal([]byte(writeMarkerString), &writeMarker) + if !allocationObj.IsRedeemRequired { + return nil, common.NewError("invalid_operation", "Last commit is rollback") + } + + versionMarkerString := r.FormValue("version_marker") + versionMarker := writemarker.VersionMarker{} + err = json.Unmarshal([]byte(versionMarkerString), &versionMarker) if err != nil { return nil, common.NewErrorf("invalid_parameters", "Invalid parameters. Error parsing the writemarker for commit: %v", err) } - var result blobberhttp.CommitResult + if versionMarker.IsRepair { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Rollback marker cannot be a repair marker") + } - var latestWriteMarkerEntity *writemarker.WriteMarkerEntity - latestWriteMarkerEntity, err = writemarker.GetWriteMarkerEntity(ctx, - allocationObj.AllocationRoot) + var result blobberhttp.CommitResult + err = versionMarker.Verify(allocationID, allocationObj.OwnerPublicKey) if err != nil { - return nil, common.NewErrorf("latest_write_marker_read_error", - "Error reading the latest write marker for allocation: %v", err) - } - if latestWriteMarkerEntity == nil { - return nil, common.NewError("latest_write_marker_not_found", - "Latest write marker not found for allocation") + return nil, common.NewError("invalid_parameters", "Invalid version marker passed: "+err.Error()) } - writemarkerEntity := &writemarker.WriteMarkerEntity{} - writemarkerEntity.WM = writeMarker + if versionMarker.Version == allocationObj.AllocationVersion { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is same as the current version") + } - err = writemarkerEntity.VerifyRollbackMarker(ctx, allocationObj, latestWriteMarkerEntity) + currentVersionMarker, err := writemarker.GetCurrentVersion(ctx, allocationID) if err != nil { - return nil, common.NewError("write_marker_verification_failed", "Verification of the write marker failed: "+err.Error()) + return nil, common.NewError("invalid_parameters", "Error getting the current version marker") } - - if writemarkerEntity.WM.ChainLength > config.Configuration.MaxChainLength { - return nil, common.NewError("chain_length_exceeded", "Chain length exceeded") + if currentVersionMarker.IsRepair { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Allocation is in repair mode") } - - elapsedVerifyWM := time.Since(startTime) - elapsedAllocation - elapsedGetLock - - var clientIDForWriteRedeem = writeMarker.ClientID - - if err := writePreRedeem(ctx, allocationObj, &writeMarker, clientIDForWriteRedeem); err != nil { - return nil, err + if versionMarker.Version != currentVersionMarker.Version-1 { + return nil, common.NewError("invalid_parameters", "Invalid version marker passed. Version marker is not the previous version") } - elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM + elapsedWritePreRedeem := time.Since(startTime) - elapsedAllocation - elapsedGetLock timeoutCtx, cancel := context.WithTimeout(ctx, 45*time.Second) defer cancel() c := datastore.GetStore().CreateTransaction(timeoutCtx) txn := datastore.GetStore().GetTransaction(c) - err = allocation.ApplyRollback(c, allocationID) + err = allocation.ApplyRollback(c, allocationID, allocationObj.AllocationVersion) if err != nil { txn.Rollback() return nil, common.NewError("allocation_rollback_error", "Error applying the rollback for allocation: "+err.Error()) } - elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem - - //get allocation root and ref - rootRef, err := reference.GetLimitedRefFieldsByPath(c, allocationID, "/", []string{"hash", "file_meta_hash", "is_precommit"}) - if err != nil && err != gorm.ErrRecordNotFound { - txn.Rollback() - return nil, common.NewError("root_ref_read_error", "Error reading the root reference: "+err.Error()) - } - if err == gorm.ErrRecordNotFound { - rootRef = &reference.Ref{} - } - - Logger.Info("rollback_root_ref", zap.Any("root_ref", rootRef)) - allocationRoot := rootRef.Hash - fileMetaRoot := rootRef.FileMetaHash - - if allocationRoot != writeMarker.AllocationRoot { - result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = latestWriteMarkerEntity - result.Success = false - result.ErrorMessage = "Allocation root in the write marker does not match the calculated allocation root." + - " Expected hash: " + allocationRoot - txn.Rollback() - return &result, common.NewError("allocation_root_mismatch", result.ErrorMessage) - } - - chainHash := writemarker.CalculateChainHash(latestWriteMarkerEntity.WM.ChainHash, allocationRoot) - if chainHash != writeMarker.ChainHash { - txn.Rollback() - return nil, common.NewError("chain_hash_mismatch", "Chain hash in the write marker does not match the calculated chain hash") - } - - if fileMetaRoot != writeMarker.FileMetaRoot { - if latestWriteMarkerEntity != nil { - result.WriteMarker = latestWriteMarkerEntity - } - result.Success = false - result.ErrorMessage = "File meta root in the write marker does not match the calculated file meta root." + - " Expected hash: " + fileMetaRoot + "; Got: " + writeMarker.FileMetaRoot - txn.Rollback() - return &result, common.NewError("file_meta_root_mismatch", result.ErrorMessage) - } - - writemarkerEntity.ConnectionID = connectionID - writemarkerEntity.ClientPublicKey = clientKey - Logger.Info("rollback_writemarker", zap.Any("writemarker", writemarkerEntity.WM)) + elapsedApplyRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem alloc, err := allocation.Repo.GetByIdAndLock(c, allocationID) Logger.Info("[rollback]Lock Allocation", zap.Bool("is_redeem_required", alloc.IsRedeemRequired), zap.String("allocation_root", alloc.AllocationRoot), zap.String("latest_wm_redeemed", alloc.LatestRedeemedWM)) @@ -1593,28 +1447,25 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob return &result, common.NewError("allocation_read_error", "Error reading the allocation object") } - alloc.BlobberSizeUsed -= latestWriteMarkerEntity.WM.Size - alloc.UsedSize -= latestWriteMarkerEntity.WM.Size - alloc.AllocationRoot = allocationRoot - alloc.FileMetaRoot = fileMetaRoot - alloc.IsRedeemRequired = true + alloc.BlobberSizeUsed = alloc.PrevBlobberSizeUsed + alloc.UsedSize = alloc.PrevUsedSize + alloc.IsRedeemRequired = false + alloc.AllocationVersion = versionMarker.Version updateMap := map[string]interface{}{ "blobber_size_used": alloc.BlobberSizeUsed, "used_size": alloc.UsedSize, - "allocation_root": alloc.AllocationRoot, - "file_meta_root": alloc.FileMetaRoot, - "is_redeem_required": true, + "is_redeem_required": false, + "allocation_version": versionMarker.Version, } updateOption := func(a *allocation.Allocation) { a.BlobberSizeUsed = alloc.BlobberSizeUsed a.UsedSize = alloc.UsedSize - a.AllocationRoot = alloc.AllocationRoot + a.AllocationVersion = alloc.AllocationVersion a.FileMetaRoot = alloc.FileMetaRoot a.IsRedeemRequired = alloc.IsRedeemRequired } - writemarkerEntity.Latest = true - err = txn.Create(writemarkerEntity).Error + err = txn.Create(&versionMarker).Error if err != nil { txn.Rollback() return &result, common.NewError("write_marker_error", "Error persisting the write marker "+err.Error()) @@ -1633,9 +1484,7 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob Logger.Error("Error committing the rollback for allocation", zap.Error(err)) } - elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedVerifyWM - elapsedWritePreRedeem - result.AllocationRoot = allocationObj.AllocationRoot - result.WriteMarker = writemarkerEntity + elapsedCommitRollback := time.Since(startTime) - elapsedAllocation - elapsedGetLock - elapsedWritePreRedeem result.Success = true result.ErrorMessage = "" commitOperation := "rollback" @@ -1644,7 +1493,6 @@ func (fsh *StorageHandler) Rollback(ctx context.Context, r *http.Request) (*blob zap.String("alloc_id", allocationID), zap.Duration("get_alloc", elapsedAllocation), zap.Duration("get-lock", elapsedGetLock), - zap.Duration("verify-wm", elapsedVerifyWM), zap.Duration("write-pre-redeem", elapsedWritePreRedeem), zap.Duration("apply-rollback", elapsedApplyRollback), zap.Duration("total", time.Since(startTime)), diff --git a/code/go/0chain.net/blobbercore/handler/protocol.go b/code/go/0chain.net/blobbercore/handler/protocol.go index 976828558..8a5993e8b 100644 --- a/code/go/0chain.net/blobbercore/handler/protocol.go +++ b/code/go/0chain.net/blobbercore/handler/protocol.go @@ -67,6 +67,7 @@ func getStorageNode() (*transaction.StorageNode, error) { sn.StakePoolSettings.ServiceCharge = config.Configuration.ServiceCharge sn.IsEnterprise = config.Configuration.IsEnterprise + sn.StorageVersion = 1 return sn, nil } diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index ebe240393..56625bf28 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -21,6 +21,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/common" "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" "github.com/0chain/blobber/code/go/0chain.net/core/node" ) @@ -142,8 +143,15 @@ func (fsh *StorageHandler) GetFileMeta(ctx context.Context, r *http.Request) (in return nil, common.NewError("invalid_signature", "Invalid signature") } } - + if fileref.Type == reference.DIRECTORY { + fileref.IsEmpty, err = reference.IsDirectoryEmpty(ctx, fileref.ID) + if err != nil { + return nil, common.NewError("bad_db_operation", "Error checking if directory is empty. "+err.Error()) + } + } + fileref.AllocationVersion = alloc.AllocationVersion result := fileref.GetListingData(ctx) + Logger.Info("GetFileMeta", zap.Any("allocationResultVersion", result["allocation_version"]), zap.Int64("allocationVersion", alloc.AllocationVersion), zap.Any("path", result["path"])) if !isOwner && !isRepairer { var authTokenString = r.FormValue("auth_token") @@ -255,11 +263,6 @@ func (fsh *StorageHandler) GetFileStats(ctx context.Context, r *http.Request) (i if err != nil { return nil, common.NewError("bad_db_operation", "Error retrieving file stats. "+err.Error()) } - wm, _ := writemarker.GetWriteMarkerEntity(ctx, fileref.AllocationRoot) - if wm != nil && fileStats != nil { - fileStats.WriteMarkerRedeemTxn = wm.CloseTxnID - fileStats.OnChain = wm.OnChain() - } statsMap := make(map[string]interface{}) statsBytes, err := json.Marshal(fileStats) if err != nil { @@ -391,7 +394,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* if !ok { var listResult blobberhttp.ListResult - listResult.AllocationRoot = allocationObj.AllocationRoot + listResult.AllocationVersion = allocationObj.AllocationVersion if fileref == nil { fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID} } @@ -402,6 +405,9 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* } fileref = parent } + if path == "/" { + fileref.Size = allocationObj.BlobberSizeUsed + } listResult.Meta = fileref.GetListingData(ctx) if clientID != allocationObj.OwnerID { delete(listResult.Meta, "path") @@ -461,16 +467,22 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* dirref = parent } else { + if fileref == nil { + fileref = &reference.Ref{Type: reference.DIRECTORY, Path: path, AllocationID: allocationID} + } r, err := reference.GetRefWithChildren(ctx, fileref, allocationID, filePath, offset, pageLimit) if err != nil { return nil, common.NewError("invalid_parameters", "Invalid path. "+err.Error()) } dirref = r + if path == "/" { + dirref.Size = allocationObj.BlobberSizeUsed + } } var result blobberhttp.ListResult - result.AllocationRoot = allocationObj.AllocationRoot + result.AllocationVersion = allocationObj.AllocationVersion result.Meta = dirref.GetListingData(ctx) if clientID != allocationObj.OwnerID { delete(result.Meta, "path") @@ -490,7 +502,7 @@ func (fsh *StorageHandler) ListEntities(ctx context.Context, r *http.Request) (* return &result, nil } -func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Request) (*blobberhttp.LatestWriteMarkerResult, error) { +func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Request) (*blobberhttp.LatestVersionMarkerResult, error) { clientID := ctx.Value(constants.ContextKeyClient).(string) if clientID == "" { return nil, common.NewError("invalid_operation", "Operation needs to be performed by the owner of the allocation") @@ -511,41 +523,20 @@ func (fsh *StorageHandler) GetLatestWriteMarker(ctx context.Context, r *http.Req return nil, common.NewError("invalid_signature", "could not verify the allocation owner") } - var latestWM *writemarker.WriteMarkerEntity - var prevWM *writemarker.WriteMarkerEntity - if allocationObj.AllocationRoot == "" { - latestWM = nil - } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) + var vm *writemarker.VersionMarker + if allocationObj.AllocationVersion != 0 { + vm, err = writemarker.GetVersionMarker(ctx, allocationId, allocationObj.AllocationVersion) if err != nil { - Logger.Error("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID)) - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation. "+err.Error()) - } - if latestWM == nil { - Logger.Info("[latest_write_marker]", zap.String("allocation_root", allocationObj.AllocationRoot), zap.String("allocation_id", allocationObj.ID)) - return nil, common.NewError("latest_write_marker_read_error", "Latest write marker not found for allocation.") - } - if latestWM.WM.PreviousAllocationRoot != "" { - prevWM, err = writemarker.GetWriteMarkerEntity(ctx, latestWM.WM.PreviousAllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the previous write marker for allocation."+err.Error()) - } + return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) } + } else { + vm = &writemarker.VersionMarker{} } - var result blobberhttp.LatestWriteMarkerResult - result.Version = writemarker.MARKER_VERSION - if latestWM != nil { - if latestWM.Status == writemarker.Committed { - latestWM.WM.ChainLength = 0 // start a new chain - } - result.LatestWM = &latestWM.WM - } - if prevWM != nil { - result.PrevWM = &prevWM.WM + result := &blobberhttp.LatestVersionMarkerResult{ + VersionMarker: vm, } - - return &result, nil + return result, nil } func (fsh *StorageHandler) GetReferencePath(ctx context.Context, r *http.Request) (*blobberhttp.ReferencePathResult, error) { @@ -626,7 +617,8 @@ func (fsh *StorageHandler) getReferencePath(ctx context.Context, r *http.Request if allocationObj.AllocationRoot == "" { latestWM = nil } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, rootRef.Hash) + //TODO: remove latestWM + latestWM, err = writemarker.GetWriteMarkerEntity(ctx, rootRef.FileMetaHash) if err != nil { errCh <- common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) return @@ -834,15 +826,16 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb return nil, common.NewError("invalid_parameters", "empty path and authtoken") } - var pathRef *reference.Ref + var pathRef *reference.PaginatedRef switch { case path != "": pathHash = reference.GetReferenceLookup(allocationID, path) fallthrough case pathHash != "": - pathRef, err = reference.GetReferenceByLookupHash(ctx, allocationID, pathHash) + pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, pathHash) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { + logging.Logger.Error("GetRefs: GetPaginatedRefByLookupHash", zap.Error(err), zap.String("path", path), zap.String("pathHash", pathHash)) return nil, common.NewError("invalid_path", "") } return nil, err @@ -874,7 +867,7 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb } if pathRef == nil { - pathRef, err = reference.GetReferenceByLookupHash(ctx, allocationID, authToken.FilePathHash) + pathRef, err = reference.GetPaginatedRefByLookupHash(ctx, authToken.FilePathHash) if err != nil { return nil, fsh.convertGormError(err) } @@ -910,13 +903,16 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb pageLimit = o } } - + var offsetTime int offsetPath := r.FormValue("offsetPath") offsetDate := r.FormValue("offsetDate") updatedDate := r.FormValue("updatedDate") err = checkValidDate(offsetDate, OffsetDateLayout) if err != nil { - return nil, err + offsetTime, err = strconv.Atoi(offsetDate) + if err != nil { + return nil, err + } } err = checkValidDate(updatedDate, OffsetDateLayout) if err != nil { @@ -945,8 +941,11 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb switch { case refType == "regular": refs, totalPages, newOffsetPath, err = reference.GetRefs( - ctx, allocationID, path, offsetPath, fileType, level, pageLimit, + ctx, allocationID, path, offsetPath, fileType, level, pageLimit, offsetTime, pathRef, ) + if refs != nil { + logging.Logger.Info("GetRefs: regular", zap.Int("refs", len(*refs)), zap.Int("totalPages", totalPages), zap.String("newOffsetPath", newOffsetPath), zap.Error(err)) + } case refType == "updated": refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs( @@ -961,24 +960,11 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb if err != nil { return nil, err } - var latestWM *writemarker.WriteMarkerEntity - if allocationObj.AllocationRoot == "" { - latestWM = nil - } else { - latestWM, err = writemarker.GetWriteMarkerEntity(ctx, allocationObj.AllocationRoot) - if err != nil { - return nil, common.NewError("latest_write_marker_read_error", "Error reading the latest write marker for allocation."+err.Error()) - } - } - var refResult blobberhttp.RefResult refResult.Refs = refs refResult.TotalPages = totalPages refResult.OffsetPath = newOffsetPath refResult.OffsetDate = newOffsetDate - if latestWM != nil { - refResult.LatestWM = &latestWM.WM - } // Refs will be returned as it is and object tree will be build in client side return &refResult, nil } diff --git a/code/go/0chain.net/blobbercore/handler/tests_common_test.go b/code/go/0chain.net/blobbercore/handler/tests_common_test.go index 78fa2dcb7..6ea548768 100644 --- a/code/go/0chain.net/blobbercore/handler/tests_common_test.go +++ b/code/go/0chain.net/blobbercore/handler/tests_common_test.go @@ -76,11 +76,9 @@ func (mfs *MockFileStore) WriteFile(allocID, connID string, b := bytes.NewBuffer(make([]byte, 0)) n, _ := io.Copy(b, infile) return &filestore.FileOutputData{ - Name: fileData.Name, - Path: fileData.Path, - FixedMerkleRoot: "", - ValidationRoot: fileData.ValidationRoot, - Size: n, + Name: fileData.Name, + Path: fileData.Path, + Size: n, }, nil } @@ -119,6 +117,10 @@ func (mfs *MockFileStore) GetTempFilePath(allocID, connID, fileName, filePathHas return "" } +func (mfs *MockFileStore) CopyFile(allocationID, oldFileLookupHash, newFileLookupHash string) error { + return nil +} + func (mfs *MockFileStore) GetFileBlock(in *filestore.ReadBlockInput) (*filestore.FileDownloadResponse, error) { return &filestore.FileDownloadResponse{ Data: mockFileBlock, diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index b5881b4bb..a36ed4932 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -39,6 +39,10 @@ func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Alloca db := datastore.GetStore().GetTransaction(ctx) _ = filestore.GetFileStore().IterateObjects(allocationObj.ID, func(hash string, contentSize int64) { + // thumbnail suffix makes hash greater than 65 + if len(hash) > 65 { + return + } var refs []reference.Ref version := 0 if len(hash) > 64 { @@ -46,8 +50,7 @@ func cleanupAllocationFiles(ctx context.Context, allocationObj allocation.Alloca hash = hash[:64] } err := db.Table((reference.Ref{}).TableName()). - Where(reference.Ref{ValidationRoot: hash, Type: reference.FILE}). - Or(reference.Ref{ThumbnailHash: hash, Type: reference.FILE}). + Where(reference.Ref{LookupHash: hash, Type: reference.FILE}). Find(&refs).Error if err != nil { @@ -79,20 +82,26 @@ func cleanupTempFiles(ctx context.Context) { then := now.Add(time.Duration(-config.Configuration.OpenConnectionWorkerTolerance) * time.Second) var openConnectionsToDelete []allocation.AllocationChangeCollector - db.Table((&allocation.AllocationChangeCollector{}).TableName()).Where("updated_at < ? AND status IN (?,?)", then, allocation.NewConnection, allocation.InProgressConnection).Preload("Changes").Find(&openConnectionsToDelete) + db.Table((&allocation.AllocationChangeCollector{}).TableName()).Where("updated_at < ?", then).Preload("Changes").Find(&openConnectionsToDelete) for i := 0; i < len(openConnectionsToDelete); i++ { connection := &openConnectionsToDelete[i] logging.Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ID)) + processor := allocation.GetConnectionProcessor(connection.ID) + if processor != nil { + continue + } connection.ComputeProperties() nctx := datastore.GetStore().CreateTransaction(ctx) ndb := datastore.GetStore().GetTransaction(nctx) var errorOccurred bool - for _, changeProcessor := range connection.AllocationChanges { - if err := changeProcessor.DeleteTempFile(); err != nil { - errorOccurred = true - logging.Logger.Error("AllocationChangeProcessor_DeleteTempFile", zap.Error(err)) + if connection.Status == allocation.InProgressConnection || connection.Status == allocation.NewConnection { + for _, changeProcessor := range connection.AllocationChanges { + if err := changeProcessor.DeleteTempFile(); err != nil { + errorOccurred = true + logging.Logger.Error("AllocationChangeProcessor_DeleteTempFile", zap.Error(err)) + } } } diff --git a/code/go/0chain.net/blobbercore/reference/dbCollector.go b/code/go/0chain.net/blobbercore/reference/dbCollector.go index 37c3cc808..5531bcf60 100644 --- a/code/go/0chain.net/blobbercore/reference/dbCollector.go +++ b/code/go/0chain.net/blobbercore/reference/dbCollector.go @@ -2,48 +2,131 @@ package reference import ( "context" + "sync" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) type QueryCollector interface { CreateRefRecord(ref *Ref) DeleteRefRecord(ref *Ref) - Finalize(ctx context.Context) error + Finalize(ctx context.Context, allocationID string, allocationVersion int64) error + AddToCache(ref *Ref) + GetFromCache(lookupHash string) *Ref + DeleteLookupRefRecord(ref *Ref) + LockTransaction() + UnlockTransaction() } type dbCollector struct { createdRefs []*Ref deletedRefs []*Ref + refCache RefCache + refMap map[string]*Ref + txnLock sync.Mutex + lock sync.Mutex } +type RefCache struct { + AllocationVersion int64 + CreatedRefs []*Ref + DeletedRefs []*Ref +} + +var ( + cacheMap = make(map[string]*RefCache) + cacheMapLock sync.RWMutex +) + func NewCollector(changes int) QueryCollector { return &dbCollector{ - createdRefs: make([]*Ref, 0, changes*4), - deletedRefs: make([]*Ref, 0, changes*4), + createdRefs: make([]*Ref, 0, changes*2), + deletedRefs: make([]*Ref, 0, changes*2), + refCache: RefCache{ + CreatedRefs: make([]*Ref, 0, changes), + DeletedRefs: make([]*Ref, 0, changes), + }, + refMap: make(map[string]*Ref), } } func (dc *dbCollector) CreateRefRecord(ref *Ref) { + dc.lock.Lock() dc.createdRefs = append(dc.createdRefs, ref) + if ref.Type == FILE { + dc.refCache.CreatedRefs = append(dc.refCache.CreatedRefs, ref) + } + dc.lock.Unlock() } func (dc *dbCollector) DeleteRefRecord(ref *Ref) { + dc.lock.Lock() dc.deletedRefs = append(dc.deletedRefs, ref) + if ref.Type == FILE { + dc.refCache.DeletedRefs = append(dc.refCache.DeletedRefs, ref) + } + dc.lock.Unlock() } -func (dc *dbCollector) Finalize(ctx context.Context) error { +func (dc *dbCollector) DeleteLookupRefRecord(ref *Ref) { + dc.refCache.DeletedRefs = append(dc.refCache.DeletedRefs, ref) +} + +func (dc *dbCollector) Finalize(ctx context.Context, allocationID string, allocationVersion int64) error { db := datastore.GetStore().GetTransaction(ctx) if len(dc.deletedRefs) > 0 { - err := db.Delete(dc.deletedRefs).Error + err := db.Delete(&(dc.deletedRefs)).Error if err != nil { return err } } - err := db.Create(dc.createdRefs).Error - if err != nil { - return err + if len(dc.createdRefs) > 0 { + err := db.Create(&(dc.createdRefs)).Error + if err != nil { + for ind, ref := range dc.createdRefs { + logging.Logger.Error("create_ref_error", zap.String("lookup_hash", ref.LookupHash), zap.String("path", ref.Path), zap.Int("index", ind), zap.Int64("allocation_version", allocationVersion)) + } + return err + } } - + dc.refCache.AllocationVersion = allocationVersion + cacheMapLock.Lock() + cacheMap[allocationID] = &(dc.refCache) + logging.Logger.Info("Finalize", zap.Int("created", len(dc.createdRefs)), zap.Int("deleted", len(dc.deletedRefs)), zap.Int64("allocation_version", cacheMap[allocationID].AllocationVersion), zap.String("allocation_id", allocationID)) + cacheMapLock.Unlock() return nil } + +func (dc *dbCollector) AddToCache(ref *Ref) { + dc.lock.Lock() + dc.refMap[ref.LookupHash] = ref + dc.lock.Unlock() +} + +func (dc *dbCollector) GetFromCache(lookupHash string) *Ref { + dc.lock.Lock() + defer dc.lock.Unlock() + return dc.refMap[lookupHash] +} + +func GetRefCache(allocationID string) *RefCache { + cacheMapLock.RLock() + defer cacheMapLock.RUnlock() + return cacheMap[allocationID] +} + +func DeleteRefCache(allocationID string) { + cacheMapLock.Lock() + cacheMap[allocationID] = nil + cacheMapLock.Unlock() +} + +func (dc *dbCollector) LockTransaction() { + dc.txnLock.Lock() +} + +func (dc *dbCollector) UnlockTransaction() { + dc.txnLock.Unlock() +} diff --git a/code/go/0chain.net/blobbercore/reference/object.go b/code/go/0chain.net/blobbercore/reference/object.go index 31d2f0f87..f0a721ed0 100644 --- a/code/go/0chain.net/blobbercore/reference/object.go +++ b/code/go/0chain.net/blobbercore/reference/object.go @@ -2,74 +2,45 @@ package reference import ( "context" - "path/filepath" + "database/sql" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" ) -func DeleteObject(ctx context.Context, rootRef *Ref, allocationID, objPath string, ts common.Timestamp) error { - likePath := objPath + "/%" - if objPath == "/" { - likePath = "/%" - } - +func DeleteObject(ctx context.Context, allocationID, lookupHash, _type string, ts common.Timestamp, allocationVersion int64, collector QueryCollector) error { db := datastore.GetStore().GetTransaction(ctx) - - err := db.Exec("UPDATE reference_objects SET is_precommit=? WHERE allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", true, allocationID, "/", objPath, likePath).Error - if err != nil { - return err - } - - err = db.Delete(&Ref{}, "allocation_id=? AND path != ? AND (path=? OR path LIKE ?)", - allocationID, "/", objPath, likePath).Error - - if err != nil { - return err - } - if objPath == "/" { - rootRef.Children = nil - rootRef.HashToBeComputed = true - rootRef.childrenLoaded = true - rootRef.UpdatedAt = ts - return nil + if _type == DIRECTORY { + ref, err := GetLimitedRefFieldsByLookupHashWith(ctx, allocationID, lookupHash, []string{"id", "type"}) + if err != nil { + logging.Logger.Error("delete_object_error", zap.Error(err)) + return err + } + isEmpty, err := IsDirectoryEmpty(ctx, ref.ID) + if err != nil { + logging.Logger.Error("delete_object_error", zap.Error(err)) + return err + } + if !isEmpty { + return common.NewError("invalid_operation", "Directory is not empty") + } + _type = ref.Type } - parentPath, deleteFileName := filepath.Split(objPath) - rootRef.UpdatedAt = ts - fields, err := common.GetPathFields(parentPath) + err := db.Exec("UPDATE reference_objects SET deleted_at=? WHERE lookup_hash=?", sql.NullTime{ + Time: common.ToTime(ts), + Valid: true, + }, lookupHash).Error if err != nil { + logging.Logger.Error("delete_object_error", zap.Error(err)) return err } - - dirRef := rootRef - - for _, name := range fields { - var found bool - for _, ref := range dirRef.Children { - if ref.Name == name { - ref.HashToBeComputed = true - ref.childrenLoaded = true - ref.UpdatedAt = ts - found = true - dirRef = ref - break - } - } - - if !found { - return common.NewError("invalid_reference_path", "Reference path has invalid references") + if _type == FILE { + deletedRef := &Ref{ + LookupHash: lookupHash, } + collector.DeleteLookupRefRecord(deletedRef) } - - for i, child := range dirRef.Children { - basePath := filepath.Base(child.Path) - if basePath == deleteFileName || child.Path == objPath { - dirRef.RemoveChild(i) - break - } - } - - rootRef.HashToBeComputed = true - rootRef.childrenLoaded = true - return nil + return err } diff --git a/code/go/0chain.net/blobbercore/reference/objectpath.go b/code/go/0chain.net/blobbercore/reference/objectpath.go index d9d19fcb3..34d817b35 100644 --- a/code/go/0chain.net/blobbercore/reference/objectpath.go +++ b/code/go/0chain.net/blobbercore/reference/objectpath.go @@ -31,7 +31,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O if rootRef.NumBlocks == 0 { var retObj ObjectPath - retObj.RootHash = rootRef.Hash + // retObj.RootHash = rootRef.Hash retObj.FileBlockNum = 0 result := rootRef.GetListingData(ctx) list := make([]map[string]interface{}, len(rootRef.Children)) @@ -68,7 +68,7 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O break } curRef, err = GetRefWithSortedChildren(ctx, allocationID, child.Path) - if err != nil || curRef.Hash == "" { + if err != nil { return nil, common.NewError("failed_object_path", "Failed to get the object path") } curResult = list[idx] @@ -80,7 +80,6 @@ func GetObjectPath(ctx context.Context, allocationID string, blockNum int64) (*O } var retObj ObjectPath - retObj.RootHash = rootRef.Hash retObj.Meta = curRef.GetListingData(ctx) retObj.Path = result retObj.FileBlockNum = remainingBlocks diff --git a/code/go/0chain.net/blobbercore/reference/ref.go b/code/go/0chain.net/blobbercore/reference/ref.go index 6c50cf1b3..53df6c7da 100644 --- a/code/go/0chain.net/blobbercore/reference/ref.go +++ b/code/go/0chain.net/blobbercore/reference/ref.go @@ -2,7 +2,7 @@ package reference import ( "context" - "errors" + "database/sql" "fmt" "math" "path/filepath" @@ -39,41 +39,33 @@ func init() { field := refType.Field(i) dirListTag := field.Tag.Get(DIR_LIST_TAG) - if dirListTag != "" { + if dirListTag != "" && dirListTag != "is_empty" && dirListTag != "allocation_version" { dirListFields = append(dirListFields, dirListTag) } } - dirListFields = append(dirListFields, "parent_path") + dirListFields = append(dirListFields, "parent_path", "id") } type Ref struct { ID int64 `gorm:"column:id;primaryKey"` - FileID string `gorm:"column:file_id" dirlist:"file_id" filelist:"file_id"` + ParentID *int64 `gorm:"column:parent_id"` Type string `gorm:"column:type;size:1" dirlist:"type" filelist:"type"` AllocationID string `gorm:"column:allocation_id;size:64;not null;index:idx_path_alloc,priority:1;index:idx_parent_path_alloc,priority:1;index:idx_validation_alloc,priority:1" dirlist:"allocation_id" filelist:"allocation_id"` LookupHash string `gorm:"column:lookup_hash;size:64;not null;index:idx_lookup_hash" dirlist:"lookup_hash" filelist:"lookup_hash"` Name string `gorm:"column:name;size:100;not null;index:idx_name_gin" dirlist:"name" filelist:"name"` // uses GIN tsvector index for full-text search Path string `gorm:"column:path;size:1000;not null;index:idx_path_alloc,priority:2;index:path_idx;index:idx_path_gin_trgm" dirlist:"path" filelist:"path"` FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` - Hash string `gorm:"column:hash;size:64;not null" dirlist:"hash" filelist:"hash"` NumBlocks int64 `gorm:"column:num_of_blocks;not null;default:0" dirlist:"num_of_blocks" filelist:"num_of_blocks"` - PathHash string `gorm:"column:path_hash;size:64;not null" dirlist:"path_hash" filelist:"path_hash"` ParentPath string `gorm:"column:parent_path;size:999;index:idx_parent_path_alloc,priority:2"` PathLevel int `gorm:"column:level;not null;default:0"` CustomMeta string `gorm:"column:custom_meta;not null" filelist:"custom_meta" dirlist:"custom_meta"` - ValidationRoot string `gorm:"column:validation_root;size:64;not null;index:idx_validation_alloc,priority:2" filelist:"validation_root"` - PrevValidationRoot string `gorm:"column:prev_validation_root" filelist:"prev_validation_root" json:"prev_validation_root"` - ValidationRootSignature string `gorm:"column:validation_root_signature;size:64" filelist:"validation_root_signature" json:"validation_root_signature,omitempty"` Size int64 `gorm:"column:size;not null;default:0" dirlist:"size" filelist:"size"` - FixedMerkleRoot string `gorm:"column:fixed_merkle_root;size:64;not null" filelist:"fixed_merkle_root"` ActualFileSize int64 `gorm:"column:actual_file_size;not null;default:0" dirlist:"actual_file_size" filelist:"actual_file_size"` ActualFileHashSignature string `gorm:"column:actual_file_hash_signature;size:64" filelist:"actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"` ActualFileHash string `gorm:"column:actual_file_hash;size:64;not null" filelist:"actual_file_hash"` MimeType string `gorm:"column:mimetype;size:255;not null" filelist:"mimetype"` - AllocationRoot string `gorm:"column:allocation_root;size:64;not null"` ThumbnailSize int64 `gorm:"column:thumbnail_size;not null;default:0" filelist:"thumbnail_size"` ThumbnailHash string `gorm:"column:thumbnail_hash;size:64;not null" filelist:"thumbnail_hash"` - PrevThumbnailHash string `gorm:"column:prev_thumbnail_hash" filelist:"prev_thumbnail_hash"` ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size;not null;default:0" filelist:"actual_thumbnail_size"` ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash;size:64;not null" filelist:"actual_thumbnail_hash"` EncryptedKey string `gorm:"column:encrypted_key;size:64" filelist:"encrypted_key"` @@ -84,11 +76,14 @@ type Ref struct { UpdatedAt common.Timestamp `gorm:"column:updated_at;index:idx_updated_at,sort:desc;" dirlist:"updated_at" filelist:"updated_at"` DeletedAt gorm.DeletedAt `gorm:"column:deleted_at"` // soft deletion - IsPrecommit bool `gorm:"column:is_precommit;not null;default:false" filelist:"is_precommit" dirlist:"is_precommit"` ChunkSize int64 `gorm:"column:chunk_size;not null;default:65536" dirlist:"chunk_size" filelist:"chunk_size"` NumUpdates int64 `gorm:"column:num_of_updates" json:"num_of_updates"` NumBlockDownloads int64 `gorm:"column:num_of_block_downloads" json:"num_of_block_downloads"` FilestoreVersion int `gorm:"column:filestore_version" json:"-"` + DataHash string `gorm:"column:data_hash" filelist:"data_hash"` + DataHashSignature string `gorm:"column:data_hash_signature" filelist:"data_hash_signature"` + AllocationVersion int64 `gorm:"allocation_version" dirlist:"allocation_version" filelist:"allocation_version"` + IsEmpty bool `gorm:"-" dirlist:"is_empty"` HashToBeComputed bool `gorm:"-"` prevID int64 `gorm:"-"` } @@ -117,34 +112,27 @@ func (Ref) TableName() string { type PaginatedRef struct { //Gorm smart select fields. ID int64 `gorm:"column:id" json:"id,omitempty"` - FileID string `gorm:"file_id" json:"file_id"` Type string `gorm:"column:type" json:"type,omitempty"` AllocationID string `gorm:"column:allocation_id" json:"allocation_id,omitempty"` LookupHash string `gorm:"column:lookup_hash" json:"lookup_hash,omitempty"` Name string `gorm:"column:name" json:"name,omitempty"` Path string `gorm:"column:path" json:"path,omitempty"` - Hash string `gorm:"column:hash" json:"hash,omitempty"` - NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_blocks,omitempty"` - PathHash string `gorm:"column:path_hash" json:"path_hash,omitempty"` + NumBlocks int64 `gorm:"column:num_of_blocks" json:"num_of_blocks,omitempty"` ParentPath string `gorm:"column:parent_path" json:"parent_path,omitempty"` PathLevel int `gorm:"column:level" json:"level,omitempty"` CustomMeta string `gorm:"column:custom_meta" json:"custom_meta,omitempty"` - ValidationRootSignature string `gorm:"column:validation_root_signature" json:"validation_root_signature,omitempty"` - ValidationRoot string `gorm:"column:validation_root" json:"validation_root,omitempty"` Size int64 `gorm:"column:size" json:"size,omitempty"` - FixedMerkleRoot string `gorm:"column:fixed_merkle_root" json:"fixed_merkle_root,omitempty"` ActualFileSize int64 `gorm:"column:actual_file_size" json:"actual_file_size,omitempty"` ActualFileHashSignature string `gorm:"column:actual_file_hash_signature" json:"actual_file_hash_signature,omitempty"` ActualFileHash string `gorm:"column:actual_file_hash" json:"actual_file_hash,omitempty"` MimeType string `gorm:"column:mimetype" json:"mimetype,omitempty"` - AllocationRoot string `gorm:"column:allocation_root" json:"allocation_root,omitempty"` ThumbnailSize int64 `gorm:"column:thumbnail_size" json:"thumbnail_size,omitempty"` ThumbnailHash string `gorm:"column:thumbnail_hash" json:"thumbnail_hash,omitempty"` ActualThumbnailSize int64 `gorm:"column:actual_thumbnail_size" json:"actual_thumbnail_size,omitempty"` ActualThumbnailHash string `gorm:"column:actual_thumbnail_hash" json:"actual_thumbnail_hash,omitempty"` EncryptedKey string `gorm:"column:encrypted_key" json:"encrypted_key,omitempty"` EncryptedKeyPoint string `gorm:"column:encrypted_key_point" json:"encrypted_key_point,omitempty"` - FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" dirlist:"file_meta_hash" filelist:"file_meta_hash"` + FileMetaHash string `gorm:"column:file_meta_hash;size:64;not null" json:"file_meta_hash"` CreatedAt common.Timestamp `gorm:"column:created_at" json:"created_at,omitempty"` UpdatedAt common.Timestamp `gorm:"column:updated_at" json:"updated_at,omitempty"` @@ -158,49 +146,128 @@ func GetReferenceLookup(allocationID, path string) string { } func NewDirectoryRef() *Ref { - return &Ref{Type: DIRECTORY, IsPrecommit: true} + return &Ref{Type: DIRECTORY} } func NewFileRef() *Ref { - return &Ref{Type: FILE, IsPrecommit: true} + return &Ref{Type: FILE} } // Mkdir create dirs if they don't exits. do nothing if dir exists. last dir will be return without child -func Mkdir(ctx context.Context, allocationID, destpath string) (*Ref, error) { - var dirRef *Ref +func Mkdir(ctx context.Context, allocationID, destpath string, allocationVersion int64, ts common.Timestamp, collector QueryCollector) (*Ref, error) { + var err error db := datastore.GetStore().GetTransaction(ctx) - // cleaning path to avoid edge case issues: append '/' prefix if not added and removing suffix '/' if added - destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") - dirs := strings.Split(destpath, "/") - - for i := range dirs { - currentPath := filepath.Join("/", filepath.Join(dirs[:i+1]...)) - ref, err := GetReference(ctx, allocationID, currentPath) - if err == nil { - dirRef = ref - continue + if destpath != "/" { + destpath = strings.TrimSuffix(filepath.Clean("/"+destpath), "/") + } + destLookupHash := GetReferenceLookup(allocationID, destpath) + var destRef *Ref + cachedRef := collector.GetFromCache(destLookupHash) + if cachedRef != nil { + destRef = cachedRef + } else { + destRef, err = GetReferenceByLookupHashWithNewTransaction(destLookupHash) + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err } + if destRef != nil { + destRef.LookupHash = destLookupHash + defer collector.AddToCache(destRef) + } + } + if destRef != nil { + if destRef.Type != DIRECTORY { + return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") + } + return destRef, nil + } + fields, err := common.GetAllParentPaths(destpath) + if err != nil { + logging.Logger.Error("mkdir: failed to get all parent paths", zap.Error(err), zap.String("destpath", destpath)) + return nil, err + } + parentLookupHashes := make([]string, 0, len(fields)) + for i := 0; i < len(fields); i++ { + parentLookupHashes = append(parentLookupHashes, GetReferenceLookup(allocationID, fields[i])) + } + var parentRefs []*Ref + collector.LockTransaction() + defer collector.UnlockTransaction() + cachedRef = collector.GetFromCache(destLookupHash) + if cachedRef != nil { + if cachedRef.Type != DIRECTORY { + return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") + } + return cachedRef, nil + } else { + logging.Logger.Info("noEntryFound: ", zap.String("destLookupHash", destLookupHash), zap.String("destpath", destpath)) + } - if !errors.Is(err, gorm.ErrRecordNotFound) { - // unexpected sql error - return nil, err + tx := db.Model(&Ref{}).Select("id", "path", "type") + for i := 0; i < len(fields); i++ { + tx = tx.Or(Ref{LookupHash: parentLookupHashes[i]}) + } + err = tx.Order("path").Find(&parentRefs).Error + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } + var ( + parentID int64 + parentPath = "/" + ) + if len(parentRefs) > 0 { + parentID = parentRefs[len(parentRefs)-1].ID + parentPath = parentRefs[len(parentRefs)-1].Path + for i := 0; i < len(parentRefs); i++ { + if parentRefs[i].Type != DIRECTORY { + return nil, common.NewError("invalid_dir_tree", "parent path is not a directory") + } + if parentRefs[i].ID == 0 { + return nil, common.NewError("invalid_dir_tree", "parent path not found") + } } + } + if destpath != "/" { + fields = append(fields, destpath) + parentLookupHashes = append(parentLookupHashes, destLookupHash) + } - // dir doesn't exists , create it + for i := len(parentRefs); i < len(fields); i++ { + logging.Logger.Info("mkdir: creating directory", zap.String("path", fields[i]), zap.Int("parentID", int(parentID))) + var parentIDRef *int64 + if parentID > 0 { + parentIDRef = &parentID + } else if parentPath != "/" { + return nil, common.NewError("invalid_dir_tree", "parent path not found") + } newRef := NewDirectoryRef() newRef.AllocationID = allocationID - newRef.Path = currentPath - newRef.ParentPath = filepath.Join("/", filepath.Join(dirs[:i]...)) - newRef.Name = dirs[i] - newRef.Type = DIRECTORY + newRef.Path = fields[i] + if newRef.Path != "/" { + newRef.ParentPath = parentPath + } + newRef.Name = filepath.Base(fields[i]) newRef.PathLevel = i + 1 - newRef.LookupHash = GetReferenceLookup(allocationID, newRef.Path) + newRef.ParentID = parentIDRef + newRef.LookupHash = parentLookupHashes[i] + newRef.CreatedAt = ts + newRef.UpdatedAt = ts + newRef.FileMetaHash = encryption.FastHash(newRef.GetFileMetaHashData()) + newRef.AllocationVersion = allocationVersion err = db.Create(newRef).Error if err != nil { + logging.Logger.Error("mkdir: failed to create directory", zap.Error(err), zap.String("path", fields[i])) return nil, err } + collector.AddToCache(newRef) + parentID = newRef.ID + parentPath = newRef.Path + } - dirRef = newRef + dirRef := &Ref{ + AllocationID: allocationID, + ID: parentID, + Path: parentPath, } return dirRef, nil @@ -262,6 +329,16 @@ func GetReferenceByLookupHash(ctx context.Context, allocationID, pathHash string return ref, nil } +func GetPaginatedRefByLookupHash(ctx context.Context, pathHash string) (*PaginatedRef, error) { + ref := &PaginatedRef{} + db := datastore.GetStore().GetTransaction(ctx) + err := db.Model(&Ref{}).Where(&Ref{LookupHash: pathHash}).Take(ref).Error + if err != nil { + return nil, err + } + return ref, nil +} + func GetReferenceByLookupHashForDownload(ctx context.Context, allocationID, pathHash string) (*Ref, error) { ref := &Ref{} db := datastore.GetStore().GetTransaction(ctx) @@ -307,6 +384,22 @@ func IsRefExist(ctx context.Context, allocationID, path string) (bool, error) { return Found, nil } +func GetObjectSizeByLookupHash(ctx context.Context, lookupHash string) (int64, error) { + db := datastore.GetStore().GetTransaction(ctx) + var size int64 + err := db.Model(&Ref{}). + Select("size"). + Where("lookup_hash = ?", lookupHash). + Take(&size).Error + if err != nil { + if err == gorm.ErrRecordNotFound { + return 0, nil + } + return 0, err + } + return size, nil +} + // GetRefsTypeFromPaths Give list of paths it will return refs of respective path with only Type and Path selected in sql query func GetRefsTypeFromPaths(ctx context.Context, allocationID string, paths []string) (refs []*Ref, err error) { if len(paths) == 0 { @@ -342,7 +435,7 @@ func GetSubDirsFromPath(p string) []string { func GetRefWithChildren(ctx context.Context, parentRef *Ref, allocationID, path string, offset, pageLimit int) (*Ref, error) { var refs []*Ref t := datastore.GetStore().GetTransaction(ctx) - db := t.Where(Ref{ParentPath: path, AllocationID: allocationID}) + db := t.Where(Ref{ParentID: &parentRef.ID}) err := db.Order("path"). Offset(offset). Limit(pageLimit). @@ -415,43 +508,38 @@ func (r *Ref) GetFileMetaHashData() string { func (fr *Ref) GetFileHashData() string { return fmt.Sprintf( - "%s:%s:%s:%s:%d:%s:%s:%d:%s:%d:%s", + "%s:%s:%s:%s:%d:%d:%s:%d", fr.AllocationID, fr.Type, // don't need to add it as well fr.Name, // don't see any utility as fr.Path below has name in it fr.Path, fr.Size, - fr.ValidationRoot, - fr.FixedMerkleRoot, fr.ActualFileSize, fr.ActualFileHash, fr.ChunkSize, - fr.FileID, ) } func (r *Ref) GetHashData() string { - return fmt.Sprintf("%s:%s:%s", r.AllocationID, r.Path, r.FileID) + return fmt.Sprintf("%s:%s", r.AllocationID, r.Path) } func (fr *Ref) CalculateFileHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { fr.FileMetaHash = encryption.Hash(fr.GetFileMetaHashData()) - fr.Hash = encryption.Hash(fr.GetFileHashData()) fr.NumBlocks = int64(math.Ceil(float64(fr.Size*1.0) / float64(fr.ChunkSize))) fr.PathLevel = len(strings.Split(strings.TrimRight(fr.Path, "/"), "/")) fr.LookupHash = GetReferenceLookup(fr.AllocationID, fr.Path) - fr.PathHash = fr.LookupHash var err error if saveToDB && fr.HashToBeComputed { err = fr.SaveFileRef(ctx, collector) } - return fr.Hash, err + return fr.FileMetaHash, err } func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector QueryCollector) (h string, err error) { if !r.HashToBeComputed { - h = r.Hash + h = r.FileMetaHash return } @@ -464,9 +552,7 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que } }() - childHashes := make([]string, l) childFileMetaHashes := make([]string, l) - childPathHashes := make([]string, l) var refNumBlocks, size, actualSize int64 for i, childRef := range r.Children { @@ -478,22 +564,18 @@ func (r *Ref) CalculateDirHash(ctx context.Context, saveToDB bool, collector Que } childFileMetaHashes[i] = childRef.FileMetaHash - childHashes[i] = childRef.Hash - childPathHashes[i] = childRef.PathHash refNumBlocks += childRef.NumBlocks size += childRef.Size actualSize += childRef.ActualFileSize } r.FileMetaHash = encryption.Hash(r.Path + strings.Join(childFileMetaHashes, ":")) - r.Hash = encryption.Hash(r.GetHashData() + strings.Join(childHashes, ":")) - r.PathHash = encryption.Hash(strings.Join(childPathHashes, ":")) r.NumBlocks = refNumBlocks r.Size = size r.ActualFileSize = actualSize r.PathLevel = len(GetSubDirsFromPath(r.Path)) + 1 r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path) - return r.Hash, err + return r.FileMetaHash, err } func (r *Ref) CalculateHash(ctx context.Context, saveToDB bool, collector QueryCollector) (string, error) { @@ -549,17 +631,8 @@ func (r *Ref) UpdatePath(newPath, parentPath string) { r.LookupHash = GetReferenceLookup(r.AllocationID, r.Path) } -func DeleteReference(ctx context.Context, refID int64, pathHash string) error { - if refID <= 0 { - return common.NewError("invalid_ref_id", "Invalid reference ID to delete") - } - db := datastore.GetStore().GetTransaction(ctx) - return db.Where("path_hash = ?", pathHash).Delete(&Ref{ID: refID}).Error -} - func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error { r.prevID = r.ID - r.IsPrecommit = true r.NumUpdates += 1 if r.ID > 0 { deleteRef := &Ref{ID: r.ID} @@ -573,7 +646,6 @@ func (r *Ref) SaveFileRef(ctx context.Context, collector QueryCollector) error { func (r *Ref) SaveDirRef(ctx context.Context, collector QueryCollector) error { r.prevID = r.ID - r.IsPrecommit = true r.NumUpdates += 1 if r.ID > 0 { deleteRef := &Ref{ID: r.ID} @@ -675,3 +747,48 @@ func UpdateCustomMeta(ctx context.Context, ref *Ref, customMeta string) error { db := datastore.GetStore().GetTransaction(ctx) return db.Exec("UPDATE reference_objects SET custom_meta = ? WHERE id = ?", customMeta, ref.ID).Error } + +func IsDirectoryEmpty(ctx context.Context, id int64) (bool, error) { + db := datastore.GetStore().GetTransaction(ctx) + var ref Ref + err := db.Model(&Ref{}).Select("id").Where("parent_id = ?", &id).Take(&ref).Error + if err != nil { + if err == gorm.ErrRecordNotFound { + return true, nil + } + return false, err + } + if ref.ID > 0 { + return false, nil + } + + return true, nil +} + +func GetReferenceByLookupHashWithNewTransaction(lookupHash string) (*Ref, error) { + var ref *Ref + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + txn := datastore.GetStore().GetTransaction(ctx) + return txn.Model(&Ref{}).Select("id", "type").Where("lookup_hash = ?", lookupHash).Take(&ref).Error + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return nil, err + } + return ref, nil +} + +func GetFullReferenceByLookupHashWithNewTransaction(lookupHash string) (*Ref, error) { + var ref *Ref + err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + txn := datastore.GetStore().GetTransaction(ctx) + return txn.Model(&Ref{}).Where("lookup_hash = ?", lookupHash).Take(&ref).Error + }, &sql.TxOptions{ + ReadOnly: true, + }) + if err != nil { + return nil, err + } + return ref, nil +} diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 458fe9b24..f2bfc7830 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -238,7 +238,7 @@ func GetObjectTree(ctx context.Context, allocationID, path string) (*Ref, error) // Might need to consider covering index for efficient search https://blog.crunchydata.com/blog/why-covering-indexes-are-incredibly-helpful // To retrieve refs efficiently form pagination index is created in postgresql on path column so it can be used to paginate refs // very easily and effectively; Same case for offsetDate. -func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit int) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { +func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, level, pageLimit, offsetTime int, parentRef *PaginatedRef) (refs *[]PaginatedRef, totalPages int, newOffsetPath string, err error) { var ( pRefs = make([]PaginatedRef, 0, pageLimit/4) dbError error @@ -246,22 +246,40 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, ) path = filepath.Clean(path) tx := datastore.GetStore().GetTransaction(ctx) - pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + 1 - if pathLevel == level { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND parent_path = ? and level = ?", allocationID, path, level) + pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit)) + if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { + pRefs = append(pRefs, *parentRef) + refs = &pRefs + newOffsetPath = parentRef.Path + return + } + + if pathLevel+1 == level { + dbQuery = tx.Model(&Ref{}).Where("parent_id = ?", parentRef.ID) if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } dbQuery = dbQuery.Where("path > ?", offsetPath) + if offsetTime != 0 { + dbQuery = dbQuery.Where("created_at < ?", offsetTime) + } dbQuery = dbQuery.Order("path") } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND (path=? OR path LIKE ?)", allocationID, path, path+"%") + listPath := path + if path != "/" { + listPath = path + "/" + } + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } if level != 0 { dbQuery = dbQuery.Where("level = ?", level) } + if offsetTime != 0 { + dbQuery = dbQuery.Where("created_at < ?", offsetTime) + } dbQuery = dbQuery.Where("path > ?", offsetPath) diff --git a/code/go/0chain.net/blobbercore/writemarker/version_marker.go b/code/go/0chain.net/blobbercore/writemarker/version_marker.go new file mode 100644 index 000000000..757c7723c --- /dev/null +++ b/code/go/0chain.net/blobbercore/writemarker/version_marker.go @@ -0,0 +1,69 @@ +package writemarker + +import ( + "context" + "fmt" + + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" + "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/encryption" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" +) + +type VersionMarker struct { + ID int64 `gorm:"column:id;primaryKey"` + ClientID string `gorm:"client_id" json:"client_id"` + BlobberID string `gorm:"blobber_id" json:"blobber_id"` + AllocationID string `gorm:"allocation_id" json:"allocation_id"` + Version int64 `gorm:"version" json:"version"` + Timestamp int64 `gorm:"timestamp" json:"timestamp"` + Signature string `gorm:"signature" json:"signature"` + IsRepair bool `gorm:"is_repair" json:"is_repair"` + RepairVersion int64 `gorm:"repair_version" json:"repair_version"` + RepairOffset string `gorm:"repair_offset" json:"repair_offset"` +} + +func (VersionMarker) TableName() string { + return "version_markers" +} + +func GetCurrentVersion(ctx context.Context, allocationID string) (*VersionMarker, error) { + db := datastore.GetStore().GetTransaction(ctx) + var vm VersionMarker + err := db.Where("allocation_id = ?", allocationID).Order("id DESC").Take(&vm).Error + return &vm, err +} + +func GetVersionMarker(ctx context.Context, allocationID string, version int64) (*VersionMarker, error) { + db := datastore.GetStore().GetTransaction(ctx) + var vm VersionMarker + err := db.Where("allocation_id = ? and version = ?", allocationID, version).Order("id DESC").Take(&vm).Error + return &vm, err +} + +func (vm *VersionMarker) Verify(allocationID, clientPubKey string) error { + if vm.AllocationID != allocationID { + return common.NewError("version_marker_validation_failed", "Invalid allocation id") + } + + if vm.Signature == "" { + return common.NewError("version_marker_validation_failed", "Signature is missing") + } + + hashData := vm.GetHashData() + signatureHash := encryption.Hash(hashData) + sigOK, err := encryption.Verify(clientPubKey, vm.Signature, signatureHash) + if err != nil { + return common.NewError("version_marker_validation_failed", "Error during verifying signature. "+err.Error()) + } + if !sigOK { + logging.Logger.Error("write_marker_sig_error", zap.Any("vm", vm)) + return common.NewError("version_marker_validation_failed", "Version marker signature is not valid") + } + return nil +} + +func (vm *VersionMarker) GetHashData() string { + return fmt.Sprintf("%s:%s:%s:%d:%d", vm.AllocationID, vm.ClientID, vm.BlobberID, vm.Version, vm.Timestamp) +} diff --git a/code/go/0chain.net/blobbercore/writemarker/worker.go b/code/go/0chain.net/blobbercore/writemarker/worker.go index 7d5a3f924..7801a7c82 100644 --- a/code/go/0chain.net/blobbercore/writemarker/worker.go +++ b/code/go/0chain.net/blobbercore/writemarker/worker.go @@ -98,19 +98,19 @@ func deleteMarkerData(allocationID string) { // ) func SetupWorkers(ctx context.Context) { - var res []allocation.Res + // var res []allocation.Res - err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { - res = allocation.Repo.GetAllocationIds(ctx) - return nil - }) - if err != nil && err != gorm.ErrRecordNotFound { - logging.Logger.Error("error_getting_allocations_worker", - zap.Any("error", err)) - } + // err := datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + // res = allocation.Repo.GetAllocationIds(ctx) + // return nil + // }) + // if err != nil && err != gorm.ErrRecordNotFound { + // logging.Logger.Error("error_getting_allocations_worker", + // zap.Any("error", err)) + // } - startRedeem(ctx, res) - go startCollector(ctx) + // startRedeem(ctx, res) + // go startCollector(ctx) // go startCleanupWorker(ctx) } diff --git a/code/go/0chain.net/core/common/utils.go b/code/go/0chain.net/core/common/utils.go index d3ee7a82d..7aa8cdd2b 100644 --- a/code/go/0chain.net/core/common/utils.go +++ b/code/go/0chain.net/core/common/utils.go @@ -47,6 +47,29 @@ func GetParentPaths(fPath string) ([]string, error) { return paths[2:], nil } +func GetAllParentPaths(fPath string) ([]string, error) { + if fPath == "" { + return nil, nil + } + if fPath == "/" { + return []string{"/"}, nil + } + + fPath = filepath.Clean(fPath) + if !filepath.IsAbs(fPath) { + return nil, NewError("invalid_path", fmt.Sprintf("%v is not absolute path", fPath)) + } + splittedPaths := strings.Split(fPath, "/") + var paths []string + for i := 0; i < len(splittedPaths); i++ { + subPath := strings.Join(splittedPaths[0:i], "/") + paths = append(paths, subPath) + } + returnPaths := []string{"/"} + returnPaths = append(returnPaths, paths[2:]...) + return returnPaths, nil +} + // GetPathFields will return slice of fields of path. // For path /a/b/c/d/e/f.txt it will return [a, b, c, d, e, f.txt],nil func GetPathFields(p string) ([]string, error) { diff --git a/code/go/0chain.net/core/encryption/hash.go b/code/go/0chain.net/core/encryption/hash.go index 53f649343..b6859cb1b 100644 --- a/code/go/0chain.net/core/encryption/hash.go +++ b/code/go/0chain.net/core/encryption/hash.go @@ -3,6 +3,8 @@ package encryption import ( "crypto/sha1" "encoding/hex" + "hash" + "sync" "github.com/minio/sha256-simd" "golang.org/x/crypto/sha3" @@ -10,6 +12,12 @@ import ( const HASH_LENGTH = 32 +var sha3Pool = sync.Pool{ + New: func() interface{} { + return sha3.New256() + }, +} + type HashBytes [HASH_LENGTH]byte /*Hash - hash the given data and return the hash as hex string */ @@ -30,9 +38,12 @@ func RawHash(data interface{}) []byte { default: panic("unknown type") } - hash := sha3.New256() + hash := sha3Pool.Get().(hash.Hash) hash.Write(databuf) - return hash.Sum(nil) + res := hash.Sum(nil) + hash.Reset() + sha3Pool.Put(hash) + return res } func ShaHash(data interface{}) []byte { diff --git a/code/go/0chain.net/core/transaction/entity.go b/code/go/0chain.net/core/transaction/entity.go index 6a41ff980..00908352c 100644 --- a/code/go/0chain.net/core/transaction/entity.go +++ b/code/go/0chain.net/core/transaction/entity.go @@ -73,6 +73,7 @@ type StorageNode struct { PublicKey string `json:"-"` StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` IsEnterprise bool `json:"is_enterprise"` + StorageVersion int `json:"storage_version"` } type BlobberAllocation struct { diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index 55b5bcf37..758bab823 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -76,11 +76,13 @@ rate_limiters: # Max upload limit in a month for a client. Default is 2000GB(the value needs to be in blocks which is data/64KB) upload_limit_monthly: 31250000 # Max commit limit in a month for a client. Default is 30000 - commit_limit_monthly: 30000 + commit_limit_monthly: 1000000000 # Max commit limit in a day for a client. Default is 1600 commit_limit_daily: 1600 # Max commit limit with size zero or less in a day for a client. Default is 400 commit_zero_limit_daily: 400 + # Max connection changes in a batch. Default is 100 + max_connection_changes: 100 server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index fdfb40ee8..c874d4e2b 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -1,6 +1,6 @@ -# syntax=docker/dockerfile:1 -ARG DOCKER_IMAGE_BASE -FROM $DOCKER_IMAGE_BASE as blobber_build +# syntax=docker/dockerfile:1.6 +ARG DOCKER_IMAGE_BASE=golang:1.22-bookworm +FROM ${DOCKER_IMAGE_BASE} AS blobber_build LABEL zchain="blobber" ENV SRC_DIR=/0chain @@ -16,9 +16,17 @@ RUN cd $SRC_DIR/ && go mod download WORKDIR $SRC_DIR/code/go/0chain.net/blobber -ARG GIT_COMMIT +ARG GIT_COMMIT=unknown ENV GIT_COMMIT=$GIT_COMMIT -RUN CGO_ENABLED=1 go build -v -tags "bn256 development" -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=$GIT_COMMIT" + +RUN set -eux; \ + go version; \ + go env; \ + CGO_ENABLED=1 go build -x -v \ + -tags "bn256 development" \ + -ldflags "-X github.com/0chain/blobber/code/go/0chain.net/core/build.BuildTag=${GIT_COMMIT}" \ + -o blobber \ + . # Copy the build artifact into a minimal runtime image: FROM alpine:3.18 diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index 7c1313cc6..0ae0c0628 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 ARG DOCKER_IMAGE_BASE -FROM $DOCKER_IMAGE_BASE as validator_build +FROM $DOCKER_IMAGE_BASE AS validator_build LABEL zchain="validator" diff --git a/go.mod b/go.mod index 2d2edbbcd..3b0053de0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.17.0-RC1 + github.com/0chain/gosdk v1.17.0-RC6 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/didip/tollbooth/v6 v6.1.2 github.com/go-openapi/runtime v0.26.0 @@ -20,7 +20,7 @@ require ( github.com/selvatico/go-mocket v1.0.7 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 - go.uber.org/zap v1.24.0 + go.uber.org/zap v1.27.0 golang.org/x/crypto v0.17.0 golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 @@ -33,12 +33,12 @@ require ( gorm.io/datatypes v1.2.0 gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.5 + gorm.io/gorm v1.25.11 ) require ( github.com/lithammer/shortuuid/v3 v3.0.7 - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.7.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc ) @@ -53,6 +53,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.26.1 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.5 github.com/pressly/goose/v3 v3.13.4 + moul.io/zapgorm2 v1.3.0 ) require ( @@ -148,9 +149,8 @@ require ( go.dedis.ch/fixbuf v1.0.3 // indirect go.dedis.ch/kyber/v3 v3.1.0 // indirect go.mongodb.org/mongo-driver v1.11.3 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/text v0.16.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index b610d6491..ee79c73b0 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.17.0-RC1 h1:D3OwgmfkqW/2FR+JtjmDx+KhlCINXL5/ZWNMbbPh4lc= -github.com/0chain/gosdk v1.17.0-RC1/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg= +github.com/0chain/gosdk v1.17.0-RC6 h1:NB4pfu0VbVWfQQEfnFqn5inaiQuyhSaUW7VwgH3LW8U= +github.com/0chain/gosdk v1.17.0-RC6/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -107,7 +107,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNIC github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -840,6 +839,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= @@ -870,23 +870,24 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -942,8 +943,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -987,6 +989,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1013,8 +1016,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1073,9 +1076,11 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1094,8 +1099,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1164,8 +1169,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1330,9 +1336,10 @@ gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/driver/sqlserver v1.4.1/go.mod h1:DJ4P+MeZbc5rvY58PnmN1Lnyvb5gw5NPzGshHDnJLig= +gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= -gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1361,6 +1368,8 @@ modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk= +moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= diff --git a/goose/migrations/1698861371_full_db_snapshot.sql b/goose/migrations/1698861371_full_db_snapshot.sql index 950dd0e6a..9496a21b4 100644 --- a/goose/migrations/1698861371_full_db_snapshot.sql +++ b/goose/migrations/1698861371_full_db_snapshot.sql @@ -103,7 +103,10 @@ CREATE TABLE allocations ( cleaned_up boolean DEFAULT false NOT NULL, finalized boolean DEFAULT false NOT NULL, file_options integer DEFAULT 63 NOT NULL, - start_time bigint NOT NULL + start_time bigint NOT NULL, + allocation_version bigint DEFAULT 0 NOT NULL, + prev_used_size bigint DEFAULT 0 NOT NULL, + prev_blobber_size_used bigint DEFAULT 0 NOT NULL ); @@ -329,7 +332,6 @@ ALTER TABLE read_pools OWNER TO blobber_user; CREATE TABLE reference_objects ( id bigint NOT NULL, - file_id text, type character varying(1), allocation_id character varying(64) NOT NULL, lookup_hash character varying(64) NOT NULL, @@ -337,25 +339,17 @@ CREATE TABLE reference_objects ( thumbnail_filename text, path character varying(1000) NOT NULL COLLATE pg_catalog."POSIX", file_meta_hash character varying(64) NOT NULL, - hash character varying(64) NOT NULL, num_of_blocks bigint DEFAULT 0 NOT NULL, - path_hash character varying(64) NOT NULL, parent_path character varying(999), level bigint DEFAULT 0 NOT NULL, custom_meta text NOT NULL, - validation_root character varying(64) NOT NULL, - prev_validation_root text, - validation_root_signature character varying(64), size bigint DEFAULT 0 NOT NULL, - fixed_merkle_root character varying(64) NOT NULL, actual_file_size bigint DEFAULT 0 NOT NULL, actual_file_hash_signature character varying(64), actual_file_hash character varying(64) NOT NULL, mimetype character varying(255) NOT NULL, - allocation_root character varying(64) NOT NULL, thumbnail_size bigint DEFAULT 0 NOT NULL, thumbnail_hash character varying(64) NOT NULL, - prev_thumbnail_hash text, actual_thumbnail_size bigint DEFAULT 0 NOT NULL, actual_thumbnail_hash character varying(64) NOT NULL, encrypted_key character varying(64), @@ -363,10 +357,13 @@ CREATE TABLE reference_objects ( created_at bigint, updated_at bigint, deleted_at timestamp with time zone, - is_precommit boolean DEFAULT false NOT NULL, chunk_size bigint DEFAULT 65536 NOT NULL, num_of_updates bigint, - num_of_block_downloads bigint + num_of_block_downloads bigint, + data_hash character varying(64), + data_hash_signature character varying(64), + parent_id bigint DEFAULT NULL, + allocation_version bigint DEFAULT 0 NOT NULL ); @@ -575,8 +572,6 @@ ALTER TABLE ONLY marketplace_share_info ALTER COLUMN id SET DEFAULT nextval('mar ALTER TABLE ONLY reference_objects ALTER COLUMN id SET DEFAULT nextval('reference_objects_id_seq'::regclass); -ALTER TABLE ONLY reference_objects ADD CONSTRAINT path_commit UNIQUE(lookup_hash,is_precommit); - -- -- Name: terms id; Type: DEFAULT; Schema: public; Owner: blobber_user -- @@ -759,7 +754,7 @@ CREATE INDEX idx_created_at ON reference_objects USING btree (created_at DESC); -- Name: idx_lookup_hash; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_lookup_hash ON reference_objects USING btree (lookup_hash); +CREATE UNIQUE INDEX idx_lookup_hash_deleted ON reference_objects USING btree (lookup_hash,(deleted_at IS NULL)) INCLUDE(id,type,num_of_updates); -- @@ -787,14 +782,20 @@ CREATE INDEX idx_name_gin ON reference_objects USING gin (to_tsvector('english': -- Name: idx_parent_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path); +-- CREATE INDEX idx_parent_path_alloc ON reference_objects USING btree (allocation_id, parent_path) WHERE deleted_at IS NULL; + +-- +-- Name: idx_parent_id; Type: INDEX; Schema: public; Owner: blobber_user +-- + +CREATE INDEX idx_parent_id ON reference_objects USING btree (parent_id); -- -- Name: idx_path_alloc; Type: INDEX; Schema: public; Owner: blobber_user -- -CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path); +CREATE INDEX idx_path_alloc ON reference_objects USING btree (allocation_id, path) WHERE deleted_at IS NULL; -- @@ -839,13 +840,6 @@ CREATE INDEX idx_updated_at ON reference_objects USING btree (updated_at DESC); CREATE INDEX idx_write_pools_cab ON write_pools USING btree (allocation_id); --- --- Name: path_idx; Type: INDEX; Schema: public; Owner: blobber_user --- - -CREATE INDEX path_idx ON reference_objects USING btree (path); - - -- -- Name: allocation_changes fk_allocation_connections_changes; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user -- @@ -855,18 +849,18 @@ ALTER TABLE ONLY allocation_changes -- - -- Name: connection_id_index; Type: INDEX; Schema: public; Owner: blobber_user - -- + -- Name: fk_reference_objects; TYPE FK CONSTRAINT; Schema: public; Owner: blobber_user + -- -CREATE INDEX connection_id_index ON allocation_changes USING btree (connection_id); + ALTER TABLE ONLY reference_objects + ADD CONSTRAINT fk_reference_objects FOREIGN KEY (parent_id) REFERENCES reference_objects(id) ON DELETE CASCADE; --- --- Name: file_stats fk_file_stats_ref; Type: FK CONSTRAINT; Schema: public; Owner: blobber_user --- + -- + -- Name: connection_id_index; Type: INDEX; Schema: public; Owner: blobber_user + -- -ALTER TABLE ONLY file_stats - ADD CONSTRAINT fk_file_stats_ref FOREIGN KEY (ref_id) REFERENCES reference_objects(id) ON DELETE CASCADE; +CREATE INDEX connection_id_index ON allocation_changes USING btree (connection_id); -- diff --git a/goose/migrations/1717416291_change_lookuphash.sql b/goose/migrations/1717416291_change_lookuphash.sql index e2d8e71dd..f55ac0827 100644 --- a/goose/migrations/1717416291_change_lookuphash.sql +++ b/goose/migrations/1717416291_change_lookuphash.sql @@ -1,6 +1,6 @@ -- +goose Up -- +goose StatementBegin -ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64); +ALTER TABLE allocation_changes ADD COLUMN lookup_hash character varying(64); -- CREATE UNIQUE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash,connection_id); -- +goose StatementEnd \ No newline at end of file diff --git a/goose/migrations/1718188301_change_idx.sql b/goose/migrations/1718188301_change_idx.sql index 564e37891..14e6c3ba0 100644 --- a/goose/migrations/1718188301_change_idx.sql +++ b/goose/migrations/1718188301_change_idx.sql @@ -1,5 +1,10 @@ -- +goose Up -- +goose StatementBegin -CREATE INDEX idx_allocation_changes_lookup_hash ON allocation_changes USING HASH(lookup_hash); + -- + -- Name: connection_id_lookup_hash; Type: UNIQUE CONSTRAINT; Schema: public; Owner: blobber_user + -- + +ALTER TABLE ONLY allocation_changes ADD CONSTRAINT connection_id_lookup_hash UNIQUE(connection_id,lookup_hash); + -- +goose StatementEnd \ No newline at end of file diff --git a/goose/migrations/1718391849_ref_index.sql b/goose/migrations/1718391849_ref_index.sql index 4380bc232..6890115ea 100644 --- a/goose/migrations/1718391849_ref_index.sql +++ b/goose/migrations/1718391849_ref_index.sql @@ -1,4 +1,10 @@ -- +goose Up -- +goose StatementBegin -DROP INDEX idx_created_at,idx_updated_at,idx_lookup_hash,idx_path_gin_trgm,idx_name_gin,idx_allocation_changes_lookup_hash; +DROP INDEX idx_created_at,idx_updated_at,idx_path_gin_trgm,idx_name_gin; + +CREATE INDEX idx_is_allocation_version_deleted_at on reference_objects(allocation_id,allocation_version) WHERE type='f' AND deleted_at IS NULL; + +CREATE INDEX idx_is_deleted on reference_objects(allocation_id) WHERE deleted_at IS NOT NULL; + +CREATE INDEX idx_path_alloc_level ON reference_objects USING btree (allocation_id,level,type,path) WHERE deleted_at IS NULL; -- +goose StatementEnd \ No newline at end of file diff --git a/goose/migrations/1721021811_version_marker.sql b/goose/migrations/1721021811_version_marker.sql new file mode 100644 index 000000000..dcf563a9a --- /dev/null +++ b/goose/migrations/1721021811_version_marker.sql @@ -0,0 +1,61 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE TABLE version_markers( + id bigint NOT NULL, + allocation_id character varying(64) NOT NULL, + blobber_id character varying(64) NOT NULL, + client_id character varying(64) NOT NULL, + "version" bigint NOT NULL, + "timestamp" bigint NOT NULL, + signature character varying(64), + is_repair boolean NOT NULL DEFAULT false, + repair_version bigint, + repair_offset character varying(1000) +); + +ALTER TABLE version_markers OWNER TO blobber_user; + +-- +-- Name: version_markers_id_seq; Type: SEQUENCE; Schema: public; Owner: blobber_user +-- + +CREATE SEQUENCE version_markers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER TABLE version_markers_id_seq OWNER TO blobber_user; + + +-- +-- Name: version_markers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: blobber_user +-- + +ALTER SEQUENCE version_markers_id_seq OWNED BY version_markers.id; + + +-- +-- Name: version_markers version_markers_pkey; Type: CONSTRAINT; Schema: public; Owner: blobber_user +-- + +ALTER TABLE ONLY version_markers + ADD CONSTRAINT version_markers_pkey PRIMARY KEY (id); + + +-- +-- Name: version_markers id; Type: DEFAULT; Schema: public; Owner: blobber_user +-- + +ALTER TABLE ONLY version_markers ALTER COLUMN id SET DEFAULT nextval('version_markers_id_seq'::regclass); + + +-- +-- Name: version_markers_allocation_id_idx; Type: INDEX; Schema: public; Owner: blobber_user +-- + +CREATE INDEX version_markers_allocation_id_idx ON version_markers USING btree (allocation_id,version); + +-- +goose StatementEnd