From 089572df771c4fc0a75d2c3952a31937e9c731d6 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Tue, 25 Feb 2025 01:56:37 +0530 Subject: [PATCH 1/5] Revert "set tcp timeout to req timeout" This reverts commit a4e42c85bb1af63773cf8a4cf4f427608050adab. --- zboxcore/zboxutil/http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zboxcore/zboxutil/http.go b/zboxcore/zboxutil/http.go index 6ec61a5c5..d8a474fe0 100644 --- a/zboxcore/zboxutil/http.go +++ b/zboxcore/zboxutil/http.go @@ -159,10 +159,10 @@ func init() { DisableHeaderNamesNormalizing: true, // If you set the case on your headers correctly you can enable this DisablePathNormalizing: true, // increase DNS cache time to an hour instead of default minute - DialTimeout: (&fasthttp.TCPDialer{ + Dial: (&fasthttp.TCPDialer{ Concurrency: 4096, DNSCacheDuration: time.Hour, - }).DialTimeout, + }).Dial, ReadTimeout: 180 * time.Second, WriteTimeout: 180 * time.Second, MaxConnDuration: 45 * time.Second, From d45321a674902c57aaad3ce03bd8b9f6565516c3 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Thu, 20 Mar 2025 20:13:00 +0530 Subject: [PATCH 2/5] Fix --- core/version/version.go | 3 +- winsdk/sdk.go | 4 +- winsdk/storage.go | 183 +++++++++++++++++++++++++++++++++ zboxcore/sdk/downloadworker.go | 1 + 4 files changed, 189 insertions(+), 2 deletions(-) diff --git a/core/version/version.go b/core/version/version.go index fa0af25cc..89da8b642 100644 --- a/core/version/version.go +++ b/core/version/version.go @@ -2,4 +2,5 @@ package version -const VERSIONSTR = "v1.17.11-269-g7fd90660" +const VERSIONSTR = "v1.19.10-7-g881432d4 +const VERSIONSTR = "v1.19.10-7-g881432d4" diff --git a/winsdk/sdk.go b/winsdk/sdk.go index d26afb7fa..98c4c1ac9 100644 --- a/winsdk/sdk.go +++ b/winsdk/sdk.go @@ -108,12 +108,14 @@ func InitSDKs(configJson *C.char) *C.char { } err = client.Init(context.Background(), *configObj) - if err != nil { l.Logger.Error(err, configJs) return WithJSON(false, err) } + client.SetSdkInitialized(true) + client.SetSignatureScheme(configObj.SignatureScheme) + l.Logger.Info("InitZCNSDK success") l.Logger.Info(configObj.BlockWorker) l.Logger.Info(configObj.ChainID) diff --git a/winsdk/storage.go b/winsdk/storage.go index d8463c0f0..fa7656731 100644 --- a/winsdk/storage.go +++ b/winsdk/storage.go @@ -8,10 +8,15 @@ import ( ) import ( + "context" "encoding/json" "errors" "fmt" + l "github.com/0chain/gosdk/zboxcore/logger" "os" + "path/filepath" + "strconv" + "sync" "time" "github.com/0chain/gosdk/core/common" @@ -616,6 +621,184 @@ func DownloadSharedFileBlocks(localPath, authTicket *C.char, startBlock int64, e return WithJSON(info, nil) } +func DownloadFromAuthTicket(authTicket, fileName, lookupHash, downloadPath, taskID string) (string, string, error) { + var err error + authTicketObj := sdk.InitAuthTicket(authTicket) + if fileName == "" { + fileName, err = authTicketObj.GetFileName() + if err != nil { + return "", "", err + } + } + + fileName = lookupHash + "_" + taskID + filepath.Ext(fileName) + + // Check if directory exists, create only if it doesn't + if _, err = os.Stat(downloadPath); os.IsNotExist(err) { + if err = os.MkdirAll(downloadPath, os.ModePerm); err != nil { + return "", "", err + } + } + + alloc, err := sdk.GetAllocationFromAuthTicket(authTicket) + if err != nil { + return "", "", err + } + + localPath := filepath.Join(downloadPath, fileName) + f, err := os.Create(localPath) + if err != nil { + return "", "", err + } + defer f.Close() + + statusBar := NewStatusBar(statusDownload, lookupHash) + + wg := &sync.WaitGroup{} + wg.Add(1) + // Download file from allocation + err = alloc.DownloadFileToFileHandlerFromAuthTicket(f, authTicket, lookupHash, "", false, statusBar, true) + if err != nil { + return "", "", err + } + wg.Wait() + return localPath, fileName, nil +} + +func downloadFilesRecursively(authTicket, downloadPath string, listRes *sdk.ListResult, taskID string) (int64, error) { + l.Logger.Debug("download total size: ", downloadPath) + + totalSize := int64(0) + for _, file := range listRes.Children { + if file.Type == "f" { + if totalSize+file.Size >= 100*1024*1024 { + return 0, errors.New("download size exceeds 100MB") + } else { + totalSize += file.Size + } + + _, _, err := DownloadFromAuthTicket(authTicket, file.Name, file.LookupHash, downloadPath, taskID) + if err != nil { + return 0, err + } + + } else if file.Type == "d" { + // Recursively download files from subdirectories + subDirPath := filepath.Join(downloadPath, file.Name) + if _, err := os.Stat(subDirPath); os.IsNotExist(err) { + if err = os.MkdirAll(subDirPath, os.ModePerm); err != nil { + return 0, err + } + } + + subDirSize, err := downloadFilesRecursively(authTicket, subDirPath, file, taskID) + if err != nil { + return 0, err + } + totalSize += subDirSize + } + } + + l.Logger.Debug("download total size: ", totalSize) + return totalSize, nil +} + +// DownloadDirFromAuthTicket - download directory using auth ticket +// ## Inputs +// - authTicket +// - remotePath +// - downloadPath +// +// ## Outputs +// +// { +// "error":"", +// "result":"path where files were downloaded", +// } +// +//export DownloadDirFromAuthTicket +func DownloadDirFromAuthTicket(authTicket, lookupHash, downloadPath *C.char) *C.char { + defer func() { + if r := recover(); r != nil { + log.Error("win: crash ", r) + } + }() + + allocTicket := C.GoString(authTicket) + lHash := C.GoString(lookupHash) + dPath := C.GoString(downloadPath) + + alloc, err := sdk.GetAllocationFromAuthTicket(allocTicket) + if err != nil { + return WithJSON(nil, err) + } + + listRes, err := alloc.ListDirFromAuthTicket(allocTicket, lHash) + if err != nil || listRes == nil { + return WithJSON(nil, fmt.Errorf("failed to list directory: %w", err)) + } + + fullPath := filepath.Join(dPath, lHash) + + totalSize, err := downloadFilesRecursively(allocTicket, fullPath, listRes, strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return WithJSON(nil, err) + } + + l.Logger.Debug("Finished downloading total size: ", totalSize) + + if totalSize >= 100*1024*1024 { + return WithJSON(nil, errors.New("download size exceeds 100MB")) + } + + return WithJSON(map[string]string{"path": fullPath}, nil) +} + +// DownloadDirectory - downalod directory +// ## Inputs +// - allocationID +// - localPath +// - remotePath +// - verifyDownload +// - isFinal +// +// ## Outputs +// +// { +// "error":"", +// "result":"true", +// } +// +//export DownloadDirectory +func DownloadDirectory(allocationID, authTicket, localPath, remotePath *C.char) *C.char { + defer func() { + if r := recover(); r != nil { + log.Error("win: crash ", r) + } + }() + + allocID := C.GoString(allocationID) + + alloc, err := getAllocation(allocID) + if err != nil { + return WithJSON(false, err) + } + + r := C.GoString(remotePath) + l := C.GoString(localPath) + at := C.GoString(authTicket) + + lookupHash := getLookupHash(allocID, r) + statusBar := NewStatusBar(statusDownload, lookupHash) + + err = alloc.DownloadDirectory(context.Background(), l, r, at, statusBar) + if err != nil { + return WithJSON(false, err) + } + + return WithJSON(true, nil) +} + // GetDownloadStatus - get download status // ## Inputs // - key: lookuphash/lookuphash:thumbnail/lookuphash:startBlock-endBlock-numBlocks/lookuphash:startBlock-endBlock-numBlocks:thumbnail diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 013819c21..6c20d1d53 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -557,6 +557,7 @@ func (req *DownloadRequest) processDownload() { defer func() { l.Logger.Debug("Clearing download buffers: ", len(req.bufferMap)) for ind, rb := range req.bufferMap { + l.Logger.Debug("Hey, I am stuck here!!") rb.ClearBuffer() delete(req.bufferMap, ind) } From a124b02dfb5b90c3ece1eb15cc02c81a8c46948f Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Thu, 20 Mar 2025 20:13:53 +0530 Subject: [PATCH 3/5] Fix --- winsdk/storage.go | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/winsdk/storage.go b/winsdk/storage.go index fa7656731..fe26472c4 100644 --- a/winsdk/storage.go +++ b/winsdk/storage.go @@ -8,7 +8,6 @@ import ( ) import ( - "context" "encoding/json" "errors" "fmt" @@ -754,51 +753,6 @@ func DownloadDirFromAuthTicket(authTicket, lookupHash, downloadPath *C.char) *C. return WithJSON(map[string]string{"path": fullPath}, nil) } -// DownloadDirectory - downalod directory -// ## Inputs -// - allocationID -// - localPath -// - remotePath -// - verifyDownload -// - isFinal -// -// ## Outputs -// -// { -// "error":"", -// "result":"true", -// } -// -//export DownloadDirectory -func DownloadDirectory(allocationID, authTicket, localPath, remotePath *C.char) *C.char { - defer func() { - if r := recover(); r != nil { - log.Error("win: crash ", r) - } - }() - - allocID := C.GoString(allocationID) - - alloc, err := getAllocation(allocID) - if err != nil { - return WithJSON(false, err) - } - - r := C.GoString(remotePath) - l := C.GoString(localPath) - at := C.GoString(authTicket) - - lookupHash := getLookupHash(allocID, r) - statusBar := NewStatusBar(statusDownload, lookupHash) - - err = alloc.DownloadDirectory(context.Background(), l, r, at, statusBar) - if err != nil { - return WithJSON(false, err) - } - - return WithJSON(true, nil) -} - // GetDownloadStatus - get download status // ## Inputs // - key: lookuphash/lookuphash:thumbnail/lookuphash:startBlock-endBlock-numBlocks/lookuphash:startBlock-endBlock-numBlocks:thumbnail From 116daead177963c08c6d1c6e723ee0d05939bb1b Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Thu, 20 Mar 2025 20:30:05 +0530 Subject: [PATCH 4/5] Fix recursive download --- core/version/version.go | 4 ++-- winsdk/storage.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/core/version/version.go b/core/version/version.go index 89da8b642..b6d78ae5d 100644 --- a/core/version/version.go +++ b/core/version/version.go @@ -2,5 +2,5 @@ package version -const VERSIONSTR = "v1.19.10-7-g881432d4 -const VERSIONSTR = "v1.19.10-7-g881432d4" +const VERSIONSTR = "v1.19.10-9-ga124b02d +const VERSIONSTR = "v1.19.10-9-ga124b02d" diff --git a/winsdk/storage.go b/winsdk/storage.go index fe26472c4..7eea72f68 100644 --- a/winsdk/storage.go +++ b/winsdk/storage.go @@ -15,7 +15,6 @@ import ( "os" "path/filepath" "strconv" - "sync" "time" "github.com/0chain/gosdk/core/common" @@ -653,18 +652,15 @@ func DownloadFromAuthTicket(authTicket, fileName, lookupHash, downloadPath, task statusBar := NewStatusBar(statusDownload, lookupHash) - wg := &sync.WaitGroup{} - wg.Add(1) // Download file from allocation err = alloc.DownloadFileToFileHandlerFromAuthTicket(f, authTicket, lookupHash, "", false, statusBar, true) if err != nil { return "", "", err } - wg.Wait() return localPath, fileName, nil } -func downloadFilesRecursively(authTicket, downloadPath string, listRes *sdk.ListResult, taskID string) (int64, error) { +func downloadFilesRecursively(alloc *sdk.Allocation, authTicket, downloadPath string, listRes *sdk.ListResult, taskID string) (int64, error) { l.Logger.Debug("download total size: ", downloadPath) totalSize := int64(0) @@ -682,6 +678,11 @@ func downloadFilesRecursively(authTicket, downloadPath string, listRes *sdk.List } } else if file.Type == "d" { + listSubDirRes, err := alloc.ListDirFromAuthTicket(authTicket, file.LookupHash) + if err != nil || listSubDirRes == nil { + return 0, fmt.Errorf("failed to list directory: %w", err) + } + // Recursively download files from subdirectories subDirPath := filepath.Join(downloadPath, file.Name) if _, err := os.Stat(subDirPath); os.IsNotExist(err) { @@ -690,7 +691,7 @@ func downloadFilesRecursively(authTicket, downloadPath string, listRes *sdk.List } } - subDirSize, err := downloadFilesRecursively(authTicket, subDirPath, file, taskID) + subDirSize, err := downloadFilesRecursively(alloc, authTicket, subDirPath, listSubDirRes, taskID) if err != nil { return 0, err } @@ -739,7 +740,7 @@ func DownloadDirFromAuthTicket(authTicket, lookupHash, downloadPath *C.char) *C. fullPath := filepath.Join(dPath, lHash) - totalSize, err := downloadFilesRecursively(allocTicket, fullPath, listRes, strconv.FormatInt(time.Now().Unix(), 10)) + totalSize, err := downloadFilesRecursively(alloc, allocTicket, fullPath, listRes, strconv.FormatInt(time.Now().Unix(), 10)) if err != nil { return WithJSON(nil, err) } From d44f2489fbdd3f754871c86ee186a3a48463c5ce Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Fri, 21 Mar 2025 00:42:59 +0530 Subject: [PATCH 5/5] Add StatusCallback2 for improved download status handling --- core/version/version.go | 4 +-- winsdk/status2.go | 51 ++++++++++++++++++++++++++++++++++ winsdk/storage.go | 10 +++++-- zboxcore/sdk/downloadworker.go | 1 - 4 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 winsdk/status2.go diff --git a/core/version/version.go b/core/version/version.go index b6d78ae5d..344e36610 100644 --- a/core/version/version.go +++ b/core/version/version.go @@ -2,5 +2,5 @@ package version -const VERSIONSTR = "v1.19.10-9-ga124b02d -const VERSIONSTR = "v1.19.10-9-ga124b02d" +const VERSIONSTR = "v1.19.10-10-g116daead +const VERSIONSTR = "v1.19.10-10-g116daead" diff --git a/winsdk/status2.go b/winsdk/status2.go new file mode 100644 index 000000000..3d4e5f10a --- /dev/null +++ b/winsdk/status2.go @@ -0,0 +1,51 @@ +package main + +import ( + "sync" +) + +type StatusCallback2 struct { + wg *sync.WaitGroup + isRepair bool + success bool + err error +} + +func (cb *StatusCallback2) Started(allocationId, filePath string, op, totalBytes int) { + +} + +func (cb *StatusCallback2) InProgress(allocationId, filePath string, op, completedBytes int, data []byte) { +} + +func (cb *StatusCallback2) RepairCompleted(filesRepaired int) { + if cb.err == nil { + cb.success = true + } + cb.wg.Done() +} + +func (cb *StatusCallback2) Completed(allocationId, filePath, filename, mimetype string, size, op int) { + if !cb.isRepair { + cb.success = true + cb.wg.Done() + } +} + +func (cb *StatusCallback2) Error(allocationID, filePath string, op int, err error) { + cb.success = false + cb.err = err + if !cb.isRepair { + cb.wg.Done() + } +} + +func (cb *StatusCallback2) StatusError() error { + return cb.err +} + +func NewStatusCallback2(wg *sync.WaitGroup) *StatusCallback2 { + return &StatusCallback2{ + wg: wg, + } +} diff --git a/winsdk/storage.go b/winsdk/storage.go index 7eea72f68..fe5ac9cda 100644 --- a/winsdk/storage.go +++ b/winsdk/storage.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "time" "github.com/0chain/gosdk/core/common" @@ -650,13 +651,16 @@ func DownloadFromAuthTicket(authTicket, fileName, lookupHash, downloadPath, task } defer f.Close() - statusBar := NewStatusBar(statusDownload, lookupHash) - + wg := &sync.WaitGroup{} + wg.Add(1) + cb := NewStatusCallback2(wg) // Download file from allocation - err = alloc.DownloadFileToFileHandlerFromAuthTicket(f, authTicket, lookupHash, "", false, statusBar, true) + err = alloc.DownloadFileToFileHandlerFromAuthTicket(f, authTicket, lookupHash, "", false, cb, true) if err != nil { return "", "", err } + wg.Wait() + return localPath, fileName, nil } diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 6c20d1d53..013819c21 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -557,7 +557,6 @@ func (req *DownloadRequest) processDownload() { defer func() { l.Logger.Debug("Clearing download buffers: ", len(req.bufferMap)) for ind, rb := range req.bufferMap { - l.Logger.Debug("Hey, I am stuck here!!") rb.ClearBuffer() delete(req.bufferMap, ind) }