From a4321d7a1d12e2ab39d5a207d2c592e3acbacc91 Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 5 Feb 2025 20:01:49 +0545 Subject: [PATCH 01/19] updated for folder structure --- internal/api/util/config/config.go | 13 +- internal/cli/util/utils.go | 174 +++++++++++++++++- tests/cli_tests/0_dropboxmgrt_migrate_test.go | 164 ----------------- tests/cli_tests/0_gdrivemgrt_migrate_test.go | 144 --------------- tests/cli_tests/main_test.go | 61 ++---- .../s3migration_tests/1_google_drive_test.go | 109 +++++++++++ .../s3migration_tests/2_dropbox_test.go | 131 +++++++++++++ .../s3migration_tests/3_one_drive_test.go | 118 ++++++++++++ .../4_microsoft_azure_test.go | 150 +++++++++++++++ .../5_google_cloud_storage_test.go | 151 +++++++++++++++ .../cli_tests/s3migration_tests/main_test.go | 154 ++++++++++++++++ .../s3migration_tests/shared/main.go | 33 ++++ 12 files changed, 1042 insertions(+), 360 deletions(-) delete mode 100644 tests/cli_tests/0_dropboxmgrt_migrate_test.go delete mode 100644 tests/cli_tests/0_gdrivemgrt_migrate_test.go create mode 100644 tests/cli_tests/s3migration_tests/1_google_drive_test.go create mode 100644 tests/cli_tests/s3migration_tests/2_dropbox_test.go create mode 100644 tests/cli_tests/s3migration_tests/3_one_drive_test.go create mode 100644 tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go create mode 100644 tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go create mode 100644 tests/cli_tests/s3migration_tests/main_test.go create mode 100644 tests/cli_tests/s3migration_tests/shared/main.go diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 9dcaf33178..23e6f25389 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -37,7 +37,18 @@ type Config struct { BlobberOwnerWalletMnemonics string `yaml:"blobber_owner_wallet_mnemonics"` OwnerWalletMnemonics string `yaml:"owner_wallet_mnemonics"` DropboxAccessToken string `yaml:"dropboxAccessToken"` + DropboxRefreshToken string `yaml:"dropboxRefreshToken"` GdriveAccessToken string `yaml:"gdriveAccessToken"` + GdriveRefreshToken string `yaml:"gdriveRefreshToken"` + ConnectionString string `yaml:"connectionString"` + AccountName string `yaml:"accountName"` + ContainerName string `yaml:"container"` + GoogleCloudAccessToken string `yaml:"googleCloudAccessToken"` + GoogleCloudRefreshToken string `yaml:"googleCloudRefreshToken"` + GoogleClientId string `yaml:"googleClientId"` + GoogleClientSecret string `yaml:"googleClientSecret"` + OneDriveAccessToken string `yaml:"oneDriveAccessToken"` + OneDriveRefreshToken string `yaml:"oneDriveRefreshToken"` } func Parse(configPath string) *Config { @@ -91,4 +102,4 @@ func CreateFreeStorageMarker( require.Nil(t, err, "Could not marshal marker") return string(markerJson) -} +} \ No newline at end of file diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 56041fdfbe..2de6334c56 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -3,6 +3,8 @@ package cliutils import ( "bufio" "crypto/rand" + "encoding/json" + "errors" "fmt" "io" "log" @@ -10,13 +12,16 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "regexp" "strconv" "strings" + "sync" "testing" "time" "github.com/0chain/system_test/internal/api/util/test" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" "github.com/0chain/system_test/internal/cli/util/specific" @@ -26,6 +31,22 @@ import ( var Logger = getLogger() +var ( + WalletMutex sync.Mutex + Wallets []json.RawMessage + WalletIdx int64 +) + +func GetWallets() []json.RawMessage { + return Wallets +} + +func SetWallets(wallets []json.RawMessage) { + WalletMutex.Lock() + defer WalletMutex.Unlock() + Wallets = wallets +} + type Configuration struct { Server string HostPort string @@ -394,7 +415,7 @@ func LogOutput(stdout io.Reader, t *test.SystemTest) { } } -func GetAllocationID(path string) string { +func GetAllocationID(path string) (string, error) { file, err := os.Open(path) if err != nil { log.Printf("Error opening allocation.txt file: %v", err) @@ -404,7 +425,7 @@ func GetAllocationID(path string) string { scanner := bufio.NewScanner(file) scanner.Scan() allocationID := scanner.Text() - return allocationID + return allocationID, nil } func ReadFileMC(testSetup *testing.T) McConfiguration { @@ -441,3 +462,152 @@ func ReadFileMC(testSetup *testing.T) McConfiguration { config.Concurrent = strconv.FormatInt(int64(concurrent), 10) return config } + +func MigrateFromS3migration(t *test.SystemTest, params string) ([]string, error) { + commandGenerated := fmt.Sprintf("../s3migration migrate %s", params) + t.Log(commandGenerated) + return RunCommand(t, commandGenerated, 1, time.Hour*2) +} + +func EscapedTestName(t *test.SystemTest) string { + replacer := strings.NewReplacer("/", "-", "\"", "-", ":", "-", "(", "-", + ")", "-", "<", "LESS_THAN", ">", "GREATER_THAN", "|", "-", "*", "-", + "?", "-") + return replacer.Replace(t.Name()) +} + +func CreateParams(params map[string]interface{}) string { + var builder strings.Builder + + for k, v := range params { + if v == nil { + _, _ = builder.WriteString(fmt.Sprintf("--%s ", k)) + } else if reflect.TypeOf(v).String() == "bool" { + _, _ = builder.WriteString(fmt.Sprintf("--%s=%v ", k, v)) + } else { + _, _ = builder.WriteString(fmt.Sprintf("--%s %v ", k, v)) + } + } + return strings.TrimSpace(builder.String()) +} + +func ExecuteFaucetWithTokens(t *test.SystemTest, cliConfigFilename string, tokens float64) ([]string, error) { + return ExecuteFaucetWithTokensForWallet(t, EscapedTestName(t), cliConfigFilename, tokens) +} + +// ExecuteFaucetWithTokensForWallet executes faucet command with given tokens and wallet. +// Tokens greater than or equal to 10 are considered to be 1 token by the system. +func ExecuteFaucetWithTokensForWallet(t *test.SystemTest, wallet, cliConfigFilename string, tokens float64) ([]string, error) { + t.Logf("Executing faucet...") + return RunCommand(t, fmt.Sprintf("./zwallet faucet --methodName "+ + "pour --tokens %f --input {} --silent --wallet %s_wallet.json --configDir ./config --config %s", + tokens, + wallet, + cliConfigFilename, + ), 3, time.Second*5) +} + +func CreateWalletForName(rootPath, name string) { + walletPath := fmt.Sprintf("%s/config/%s_wallet.json", rootPath, name) + + // check if wallet already exists + if _, err := os.Stat(walletPath); err == nil { + return + } + WalletMutex.Lock() + + wallet := Wallets[WalletIdx] + + WalletIdx++ + WalletMutex.Unlock() + + err := os.WriteFile(walletPath, wallet, 0600) + if err != nil { + fmt.Printf("Error writing file %s: %v\n", walletPath, err) + } else { + fmt.Printf("File %s written successfully.\n", walletPath) + } +} + +func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { + return setupAllocationWithWallet(t, EscapedTestName(t), cliConfigFilename, rootPath, extraParams...) +} + +func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, rootPath, params string) ([]string, error) { + t.Log(cliConfigFilename, "configdir path") + return RunCommand(t, fmt.Sprintf( + "%s/zbox newallocation %s --silent --wallet %s --configDir %s --config %s --allocationFileName %s", + rootPath, + params, + wallet+"_wallet.json", + cliConfigFilename, + "config.yaml", + wallet+"_allocation.txt", + ), 3, time.Second*5) +} + +func setupAllocationWithWallet(t *test.SystemTest, walletName, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { + // Then create new allocation + options := map[string]interface{}{"size": "10000000", "lock": "5"} + + // Add additional parameters if available + // Overwrite with new parameters when available + for _, params := range extraParams { + for k, v := range params { + options[k] = v + } + } + // First create a wallet and run faucet command + CreateWalletForName(rootPath, walletName) + + output, err := CreateNewAllocationForWallet(t, walletName, cliConfigFilename, rootPath, CreateParams(options)) + defer func() { + fmt.Printf("err: %v\n", err) + }() + require.NoError(t, err, "create new allocation failed", strings.Join(output, "\n")) + require.Len(t, output, 1) + + // Get the allocation ID and return it + allocationID, err := getAllocationID(output[0]) + require.Nil(t, err, "could not get allocation ID", strings.Join(output, "\n")) + + return allocationID +} + +var ( + createAllocationRegex = regexp.MustCompile(`^Allocation created: (.+)$`) +) + +func getAllocationID(str string) (string, error) { + match := createAllocationRegex.FindStringSubmatch(str) + if len(match) < 2 { + return "", errors.New("allocation match not found") + } + return match[1], nil +} + +func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err error) { + pattern := `total count :: (\d+)` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(strings.Join(output, "\n")) + + pattern2 := `Total migrated objects :: (\d+)` + re2 := regexp.MustCompile(pattern2) + match_2 := re2.FindStringSubmatch(strings.Join(output, "\n")) + + if len(match) > 1 && len(match_2) > 1 { + totalCount, err = strconv.Atoi(match[1]) + if err != nil { + return + } + + totalMigrated, err = strconv.Atoi(match_2[1]) + if err != nil { + return + } + + return totalMigrated, totalCount, nil + } + + return 0, 0, errors.New("no match found") +} \ No newline at end of file diff --git a/tests/cli_tests/0_dropboxmgrt_migrate_test.go b/tests/cli_tests/0_dropboxmgrt_migrate_test.go deleted file mode 100644 index 2f357824b5..0000000000 --- a/tests/cli_tests/0_dropboxmgrt_migrate_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package cli_tests - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/0chain/system_test/internal/api/util/test" - cliutils "github.com/0chain/system_test/internal/cli/util" - "github.com/stretchr/testify/require" -) - -func Test0Dropbox(testSetup *testing.T) { - t := test.NewSystemTest(testSetup) - - if dropboxAccessToken == "" { - t.Skip("dropbox Access Token was missing") - } - - t.SetSmokeTests("Should migrate existing Dropbox folder and files successfully") - - t.RunSequentially("Should migrate existing Dropbox folder and files successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationId := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationId, - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "skip": 1, - })) - - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "skip": 1, - })) - - require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "source": "dropbox", - "config": configPath, - "configDir": configDir, - })) - - require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": "invalid", - "wallet": escapedTestName(t) + "_wallet.json", - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid_access_token/", "Output was not as expected", err) - }) - - t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "wallet": escapedTestName(t) + "_wallet.json", - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Missing Access Token", "Output was not as expected", strings.Join(output, "\n")) - }) - t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "wallet": escapedTestName(t) + "_wallet.json", - "source": "invalid", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - "access-token": dropboxAccessToken, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { - allocSize := int64(5 * KB) - func() { - defer func() { - // recover from panic if one occurred - if r := recover(); r != nil { - fmt.Println("Panic occurred:", r) // Log the panic - t.Log("Test passed even though a panic occurred") - // Set the test status to passed - require.Equal(t, "", "") - } - }() - - // Set up allocation with wallet - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - }() - }) -} - -func migrateFromDropbox(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { - t.Logf("Migrating Dropbox to Zus...") - t.Logf(fmt.Sprintf("params %v", params)) - t.Logf(fmt.Sprintf("cli %v", cliConfigFilename)) - t.Logf(fmt.Sprintf("./s3migration migrate %s", params)) - return cliutils.RunCommand(t, fmt.Sprintf("./s3migration migrate %s", params), 1, time.Hour*2) -} diff --git a/tests/cli_tests/0_gdrivemgrt_migrate_test.go b/tests/cli_tests/0_gdrivemgrt_migrate_test.go deleted file mode 100644 index 0a8d7fc4da..0000000000 --- a/tests/cli_tests/0_gdrivemgrt_migrate_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package cli_tests - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/0chain/system_test/internal/api/util/test" - cliutils "github.com/0chain/system_test/internal/cli/util" - "github.com/stretchr/testify/require" -) - -func Test0Gdrive(testSetup *testing.T) { - t := test.NewSystemTest(testSetup) - - if gdriveAccessToken == "" { - t.Skip("Gdrive Access Token was missing") - } - - t.SetSmokeTests("Should migrate existing Gdrive folder and files successfully") - - t.RunSequentially("Should migrate existing Gdrive folder and files successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": gdriveAccessToken, - "allocation": allocationID, - "source": "google_drive", - "wallet": escapedTestName(t) + "_wallet.json", - "config": configPath, - "configDir": configDir, - })) - require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": gdriveAccessToken, - "allocation": allocationID, - "source": "google_drive", - "wallet": escapedTestName(t) + "_wallet.json", - "config": configPath, - "configDir": configDir, - })) - - require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) - require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": gdriveAccessToken, - "allocation": allocationID, - "source": "google_drive", - "wallet": escapedTestName(t) + "_wallet.json", - "config": configPath, - "configDir": configDir, - })) - - require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) - require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "source": "google_drive", - "config": configPath, - "configDir": configDir, - })) - - require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": "invalid", - "wallet": escapedTestName(t) + "_wallet.json", - "source": "google_drive", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) - }) - - t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "wallet": escapedTestName(t) + "_wallet.json", - "source": "google_drive", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Missing Access Token", "Output was not as expected", strings.Join(output, "\n")) - }) -} - -func migrateFromGdrive(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { - t.Logf("Migrating Gdrive to Zus...") - t.Logf(fmt.Sprintf("params %v", params)) - t.Logf(fmt.Sprintf("cli %v", cliConfigFilename)) - t.Logf(fmt.Sprintf("./s3migration migrate %s", params)) - return cliutils.RunCommand(t, fmt.Sprintf("./s3migration migrate %s", params), 1, time.Hour*2) -} diff --git a/tests/cli_tests/main_test.go b/tests/cli_tests/main_test.go index 431748906e..05f7316b2f 100644 --- a/tests/cli_tests/main_test.go +++ b/tests/cli_tests/main_test.go @@ -20,9 +20,6 @@ import ( cliutils "github.com/0chain/system_test/internal/cli/util" "github.com/spf13/viper" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) @@ -54,12 +51,6 @@ func setupConfig() { parsedConfig := config.Parse(filepath.Join(".", path, "cli_tests_config.yaml")) defaultTestTimeout, err := time.ParseDuration(parsedConfig.DefaultTestCaseTimeout) - s3AccessKey = parsedConfig.S3AccessKey - s3SecretKey = parsedConfig.S3SecretKey - s3bucketName = parsedConfig.S3BucketName - s3BucketNameAlternate = parsedConfig.S3BucketNameAlternate - dropboxAccessToken = parsedConfig.DropboxAccessToken - gdriveAccessToken = parsedConfig.GdriveAccessToken if err != nil { log.Printf("Default test case timeout could not be parsed so has defaulted to [%v]", test.DefaultTestTimeout) @@ -109,8 +100,6 @@ var ( s3bucketName string s3BucketNameAlternate string S3Client *s3.S3 - dropboxAccessToken string - gdriveAccessToken string ) var ( @@ -192,44 +181,18 @@ func TestMain(m *testing.M) { //nolint:gocyclo } } - // Create a session with AWS - sess, err := session.NewSession(&aws.Config{ - Region: aws.String("us-east-2"), // Replace with your desired AWS region - Credentials: credentials.NewStaticCredentials(s3AccessKey, s3SecretKey, ""), - }) - - if err != nil { - log.Fatalln("Failed to create AWS session:", err) - return - } - - // Create a session with Dropbox - sess_dp, err_dp := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials( - dropboxAccessToken, "", ""), - }) - - if err_dp != nil { - log.Fatalln("Failed to create Dropbox session:", err_dp) - } - - sess_gd, err_gd := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials( - gdriveAccessToken, "", ""), - }) - - if err_gd != nil { - log.Fatalln("Failed to create Gdrive session:", err_dp) - } - // Create an S3 client - cloudService := os.Getenv("CLOUD_SERVICE") - - if cloudService == "dropbox" { - S3Client = s3.New(sess_dp) - } else if cloudService == "gdrive" { - S3Client = s3.New(sess_gd) - } else { - S3Client = s3.New(sess) + if tenderlyEnabled != "" { + err := tenderlyClient.InitBalance(ethereumAddress) + if err != nil { + cliutils.Logger.Error(err.Error()) + } else { + err = tenderlyClient.InitErc20Balance(tokenAddress, ethereumAddress) + if err != nil { + cliutils.Logger.Error(err.Error()) + } else { + tenderlyInitialized = true + } + } } walletMutex.Lock() diff --git a/tests/cli_tests/s3migration_tests/1_google_drive_test.go b/tests/cli_tests/s3migration_tests/1_google_drive_test.go new file mode 100644 index 0000000000..8d50dfbe0c --- /dev/null +++ b/tests/cli_tests/s3migration_tests/1_google_drive_test.go @@ -0,0 +1,109 @@ +package s3migration_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0Gdrive(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.GdriveAccessToken == "" { + t.Skip("Gdrive Access Token was missing") + } + t.SetSmokeTests("Should migrate existing Gdrive folder and files successfully") + + t.RunSequentially("Should migrate existing Gdrive folder and files successfully", func(t *test.SystemTest) { + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.GdriveAccessToken, + "secret-key": shared.ConfigData.GdriveRefreshToken, + "allocation": allocationID, + "source": "google_drive", + "wallet": EscapedTestName(t) + "_wallet.json", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.GdriveAccessToken, + "secret-key": shared.ConfigData.GdriveRefreshToken, + "allocation": shared.DefaultAllocationId, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.GdriveAccessToken, + "secret-key": shared.ConfigData.GdriveRefreshToken, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "allocation": shared.DefaultAllocationId, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "allocation": shared.DefaultAllocationId, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + t.Logf("EXpected log %v", strings.Join(output, "\n")) + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: access key, secret key") + }) +} diff --git a/tests/cli_tests/s3migration_tests/2_dropbox_test.go b/tests/cli_tests/s3migration_tests/2_dropbox_test.go new file mode 100644 index 0000000000..81772da7ff --- /dev/null +++ b/tests/cli_tests/s3migration_tests/2_dropbox_test.go @@ -0,0 +1,131 @@ +package s3migration_tests + +import ( + "fmt" + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0Dropbox(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.DropboxAccessToken == "" || shared.ConfigData.DropboxRefreshToken == "" { + t.Skip("Missing Required Tokens for Dropbox migration") + } + + t.SetSmokeTests("Should migrate existing Dropbox folder and files successfully") + + t.RunSequentially("Should migrate existing Dropbox folder and files successfully", func(t *test.SystemTest) { + allocSize := int64(50 * shared.MB) + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": allocSize, + }) + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.DropboxAccessToken, + "secret-key": shared.ConfigData.DropboxRefreshToken, + "allocation": allocationID, + "source": "dropbox", + "wallet": EscapedTestName(t) + "_wallet.json", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.DropboxAccessToken, + "secret-key": shared.ConfigData.DropboxRefreshToken, + "allocation": shared.DefaultAllocationId, + "source": "dropbox", + "wallet": shared.DefaultAllocationId, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.DropboxAccessToken, + "secret-key": shared.ConfigData.DropboxRefreshToken, + "source": "dropbox", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "allocation": shared.DefaultAllocationId, + "source": "dropbox", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid_access_token/", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "allocation": shared.DefaultAllocationId, + "source": "dropbox", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + t.Logf("EXpected log %v", strings.Join(output, "\n")) + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: access key, secret key") + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + allocSize := int64(5 * shared.KB) + var err error + defer func() { + require.Contains(t, err.Error(), "allocation match not found") + }() + _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + "size": allocSize, + }) + }) +} + +func setupAllocationWithWalletWithoutTest(t *test.SystemTest, walletName, cliConfigFilename string, extraParams ...map[string]interface{}) (string, error) { + options := map[string]interface{}{"size": "10000000", "lock": "5"} + + for _, params := range extraParams { + for k, v := range params { + options[k] = v + } + } + cli_utils.CreateWalletForName(shared.RootPath, walletName) + output, _ := cli_utils.CreateNewAllocationForWallet(t, walletName, cliConfigFilename, shared.RootPath, cli_utils.CreateParams(options)) + defer func() { + fmt.Printf("err: %v\n", output) + }() + return cli_utils.GetAllocationID(output[0]) +} diff --git a/tests/cli_tests/s3migration_tests/3_one_drive_test.go b/tests/cli_tests/s3migration_tests/3_one_drive_test.go new file mode 100644 index 0000000000..65d45a559f --- /dev/null +++ b/tests/cli_tests/s3migration_tests/3_one_drive_test.go @@ -0,0 +1,118 @@ +package s3migration_tests + +import ( + "fmt" + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0OneDrive(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.OneDriveAccessToken == "" || shared.ConfigData.OneDriveRefreshToken == "" { + t.Skip("Missing Required details for OneDrive migration") + } + + t.SetSmokeTests("Should migrate existing files and folder from Microsoft OneDrive successfully") + + t.RunSequentially("Should migrate existing Micrsoft OneDrive folder and files successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 1, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "wallet": shared.DefaultWallet, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 1, + })) + + fmt.Printf("Output: %v\n", output) + fmt.Printf("Error: %v\n", err) + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when token and refresh token is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Access token: InvalidAuthenticationToken", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "source": "invalid", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "skip": 1, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + allocSize := int64(5 * shared.KB) + var err error + defer func() { + require.Contains(t, err.Error(), "allocation match not found") + }() + _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + "size": allocSize, + }) + }) +} diff --git a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go new file mode 100644 index 0000000000..7d47ceb271 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go @@ -0,0 +1,150 @@ +package s3migration_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0MicrosoftAzure(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.AccountName == "" { + t.Skip("Missing Account name for required for migration") + } + + if shared.ConfigData.ConnectionString == "" { + t.Skip("Missing Connection String required for migration") + } + + t.SetSmokeTests("Should migrate existing files and folder from Microsoft Azure Container successfully") + + t.RunSequentially("Should migrate existing Microsoft Azure folder and files successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": shared.DefaultWallet, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when connection string is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "invalid", + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), " connection string is either blank or malformed", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when connection string is missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: connection string", "Output was not as expected", strings.Join(output, "\n")) + }) + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "invalid", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + size := int64(5 * shared.KB) + var err error + defer func() { + require.Contains(t, err.Error(), "allocation match not found") + }() + _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + "size": size, + }) + }) +} diff --git a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go new file mode 100644 index 0000000000..6da4697233 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go @@ -0,0 +1,151 @@ +package s3migration_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0GoogleCloudStorage(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.GoogleCloudAccessToken == "" || shared.ConfigData.GoogleCloudRefreshToken == "" || shared.ConfigData.GoogleClientId == "" || shared.ConfigData.GoogleClientSecret == "" { + t.Skip("Missing Required details for google cloud console") + } + + t.SetSmokeTests("Should migrate existing files and folder from Google Cloud successfully") + + t.RunSequentially("Should migrate existing Google Cloud folder and files successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + _, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, 0, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": shared.DefaultWallet, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when client credentials is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": shared.DefaultWallet, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid client credentials/", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when key is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": "invalid", + "secret-key": "invalid", + "wallet": shared.DefaultWallet, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Access key", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": shared.DefaultWallet, + "source": "invalid", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) +} diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go new file mode 100644 index 0000000000..b3857f6f9d --- /dev/null +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -0,0 +1,154 @@ +package s3migration_tests + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/config" + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/spf13/viper" +) + +func setupDefaultConfig() { + viper.SetDefault("nodes.miner01ID", "73ad5727612116c025bb4405bf3adb4a4a04867ae508c51cf885395bffc8a949") + viper.SetDefault("nodes.miner02ID", "3ec9a42db3355f33c35750ce589ed717c08787997b7f34a7f1f9fb0a03f2b17c") + viper.SetDefault("nodes.miner03ID", "c6f4b8ce5da386b278ba8c4e6cf98b24b32d15bc675b4d12c95e082079c91937") + viper.SetDefault("nodes.sharder01ID", "ea26431f8adb7061766f1d6bbcc3b292d70dd59960d857f04b8a75e6a5bbe04f") + viper.SetDefault("nodes.sharder02ID", "30001a01a888584772b7fee13934021ab8557e0ed471c0a3a454e9164180aef1") +} + +func setupConfig() { + setupDefaultConfig() + + dir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + + shared.ConfigPath = "config.yaml" + path := filepath.Clean(filepath.Join(dir, "config")) + shared.RootPath = dir + + viper.SetConfigName("nodes") + viper.SetConfigType("yaml") + viper.AddConfigPath(path) + + parsedConfig := config.Parse(filepath.Join(path, "cli_tests_config.yaml")) + + shared.SetupConfig(parsedConfig) +} + +func defaultData() { + t := testing.T{} + system_test := test.NewSystemTest(&t) + + defaultAllocationId := cli_utils.SetupAllocation(system_test, shared.ConfigData.ConnectionString, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + defaultWallet := "default" + cli_utils.CreateWalletForName(shared.RootPath, defaultWallet) + + fmt.Fprintf(os.Stdout, "Default allocation ID: %s\n", defaultAllocationId) + fmt.Fprintf(os.Stdout, "Default wallet: %s\n", defaultWallet) +} + +func EscapedTestName(t *test.SystemTest) string { + replacer := strings.NewReplacer("/", "-", "\"", "-", ":", "-", "(", "-", + ")", "-", "<", "LESS_THAN", ">", "GREATER_THAN", "|", "-", "*", "-", + "?", "-") + return replacer.Replace(t.Name()) +} + +func GetConfigDir() string { + curr, err := os.Getwd() + if err != nil { + log.Fatalln(err) + } + return filepath.Clean(filepath.Join(curr, "config")) +} + +func TestMain(m *testing.M) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-2"), // Replace with your desired AWS region + Credentials: credentials.NewStaticCredentials(shared.ConfigData.S3AccessKey, shared.ConfigData.S3SecretKey, ""), + }) + if err != nil { + log.Fatalln("Failed to create AWS session:", err) + return + } + + sess_dp, err_dp := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(shared.ConfigData.DropboxAccessToken, "", ""), + }) + if err_dp != nil { + log.Fatalln("Failed to create Dropbox session:", err_dp) + } + + sess_gd, err_gd := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(shared.ConfigData.GdriveAccessToken, "", ""), + }) + if err_gd != nil { + log.Fatalln("Failed to create GDrive session:", err_gd) + } + + cloudService := os.Getenv("CLOUD_SERVICE") + + if cloudService == "dropbox" { + shared.S3Client = s3.New(sess_dp) + } else if cloudService == "gdrive" { + shared.S3Client = s3.New(sess_gd) + } else { + shared.S3Client = s3.New(sess) + } + + shared.ConfigPath = os.Getenv("CONFIG_PATH") + shared.ConfigDir = os.Getenv("CONFIG_DIR") + shared.RootPath = os.Getenv("ROOT_PATH") + + if shared.ConfigDir == "" { + shared.ConfigDir = GetConfigDir() + } + + if shared.ConfigPath == "" { + shared.ConfigPath = "./config.yaml" + } + + shared.ConfigDir, _ = filepath.Abs(shared.ConfigDir) + + setupConfig() + + shared.WalletMutex.Lock() + fileContent, err := os.ReadFile("../config/wallets/wallets.json") + if err != nil { + log.Println("Error reading file:", err) + return + } + + err = json.Unmarshal(fileContent, &shared.Wallets) + if err != nil { + log.Println("Error decoding JSON:", err) + return + } + + cliutils.SetWallets(shared.Wallets) + shared.WalletIdx = 500 + shared.WalletMutex.Unlock() + + defaultData() + + exitRun := m.Run() + os.Exit(exitRun) +} diff --git a/tests/cli_tests/s3migration_tests/shared/main.go b/tests/cli_tests/s3migration_tests/shared/main.go new file mode 100644 index 0000000000..7c56851f21 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/shared/main.go @@ -0,0 +1,33 @@ +package shared + +import ( + "encoding/json" + "sync" + + util "github.com/0chain/system_test/internal/api/util/config" + "github.com/aws/aws-sdk-go/service/s3" +) + +var ( + ConfigData util.Config + WalletMutex sync.Mutex + Wallets []json.RawMessage + WalletIdx int64 + S3Client *s3.S3 + ConfigDir string + ConfigPath string + RootPath string + DefaultAllocationId string + DefaultWallet string +) + +const ( + KB = 1024 // kilobyte + MB = 1024 * KB // megabyte + GB = 1024 * MB // gigabyte + AllocSize = int64(50 * MB) +) + +func SetupConfig(parsedConfig *util.Config) { + ConfigData = *parsedConfig +} From c05f54780e519f287616604c3736b7082c09dc42 Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 5 Feb 2025 20:04:43 +0545 Subject: [PATCH 02/19] updated for lin --- internal/api/util/config/config.go | 2 +- internal/cli/util/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 23e6f25389..67a8bc8022 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -102,4 +102,4 @@ func CreateFreeStorageMarker( require.Nil(t, err, "Could not marshal marker") return string(markerJson) -} \ No newline at end of file +} diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 2de6334c56..67f1d02c29 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -610,4 +610,4 @@ func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err erro } return 0, 0, errors.New("no match found") -} \ No newline at end of file +} From 2bd26bdc750a0cc7b7901c085c2582827772c7dd Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 5 Feb 2025 21:41:25 +0545 Subject: [PATCH 03/19] updated for default allocation --- tests/cli_tests/config/cli_tests_config.yaml | 17 ++++++++++++++--- tests/cli_tests/s3migration_tests/main_test.go | 6 ++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 6867e623f2..51337e9c3b 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -1,7 +1,18 @@ default_test_case_timeout: 3m -s3_access_key: -s3_secret_key: +s3_access_key: +s3_secret_key: s3_bucket_name: s3_bucket_name_alternate: alternate-test-bucket -dropboxAccessToken: +dropboxAccessToken: +dropboxRefreshToken: gdriveAccessToken: +gdriveRefreshToken: +connectionString: +accountName: +container: +googleCloudAccessToken: +googleCloudRefreshToken: +googleClientId: +googleClientSecret: +oneDriveAccessToken: +oneDriveRefreshToken: diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go index b3857f6f9d..5a1411e4eb 100644 --- a/tests/cli_tests/s3migration_tests/main_test.go +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -2,7 +2,6 @@ package s3migration_tests import ( "encoding/json" - "fmt" "log" "os" "path/filepath" @@ -60,9 +59,8 @@ func defaultData() { defaultWallet := "default" cli_utils.CreateWalletForName(shared.RootPath, defaultWallet) - - fmt.Fprintf(os.Stdout, "Default allocation ID: %s\n", defaultAllocationId) - fmt.Fprintf(os.Stdout, "Default wallet: %s\n", defaultWallet) + shared.DefaultAllocationId = defaultAllocationId + shared.DefaultWallet = defaultWallet } func EscapedTestName(t *test.SystemTest) string { From eabb63d725b08b654f3cea92353461b9877316d5 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 11:12:30 +0545 Subject: [PATCH 04/19] path updated --- .../cli_tests/s3migration_tests/main_test.go | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go index 5a1411e4eb..38ab4f7de6 100644 --- a/tests/cli_tests/s3migration_tests/main_test.go +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -37,14 +37,15 @@ func setupConfig() { } shared.ConfigPath = "config.yaml" - path := filepath.Clean(filepath.Join(dir, "config")) - shared.RootPath = dir + rootPath := filepath.Dir(dir) + shared.RootPath = rootPath + configPath := filepath.Join(shared.RootPath,"config") viper.SetConfigName("nodes") viper.SetConfigType("yaml") - viper.AddConfigPath(path) + viper.AddConfigPath(configPath) - parsedConfig := config.Parse(filepath.Join(path, "cli_tests_config.yaml")) + parsedConfig := config.Parse(filepath.Join(configPath, "cli_tests_config.yaml")) shared.SetupConfig(parsedConfig) } @@ -52,15 +53,12 @@ func setupConfig() { func defaultData() { t := testing.T{} system_test := test.NewSystemTest(&t) - defaultAllocationId := cli_utils.SetupAllocation(system_test, shared.ConfigData.ConnectionString, shared.RootPath, map[string]interface{}{ "size": shared.AllocSize, }) - defaultWallet := "default" - cli_utils.CreateWalletForName(shared.RootPath, defaultWallet) shared.DefaultAllocationId = defaultAllocationId - shared.DefaultWallet = defaultWallet + shared.DefaultWallet = "default_wallet.json" } func EscapedTestName(t *test.SystemTest) string { @@ -75,7 +73,8 @@ func GetConfigDir() string { if err != nil { log.Fatalln(err) } - return filepath.Clean(filepath.Join(curr, "config")) + parentDirectory := filepath.Dir(curr) + return filepath.Clean(filepath.Join(parentDirectory, "config")) } func TestMain(m *testing.M) { @@ -121,15 +120,16 @@ func TestMain(m *testing.M) { } if shared.ConfigPath == "" { - shared.ConfigPath = "./config.yaml" + shared.ConfigPath = "config.yaml" } shared.ConfigDir, _ = filepath.Abs(shared.ConfigDir) + setupConfig() shared.WalletMutex.Lock() - fileContent, err := os.ReadFile("../config/wallets/wallets.json") + fileContent, err := os.ReadFile(shared.ConfigDir + "/wallets/wallets.json") if err != nil { log.Println("Error reading file:", err) return @@ -144,7 +144,6 @@ func TestMain(m *testing.M) { cliutils.SetWallets(shared.Wallets) shared.WalletIdx = 500 shared.WalletMutex.Unlock() - defaultData() exitRun := m.Run() From 10dda19dbfd094a07b74c797d6c6d7ba1cd16f7d Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 12:34:14 +0545 Subject: [PATCH 05/19] default wallet --- internal/cli/util/utils.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 67f1d02c29..860e3eb11b 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -508,6 +508,9 @@ func ExecuteFaucetWithTokensForWallet(t *test.SystemTest, wallet, cliConfigFilen } func CreateWalletForName(rootPath, name string) { + if name == "" { + name = "default" + } walletPath := fmt.Sprintf("%s/config/%s_wallet.json", rootPath, name) // check if wallet already exists @@ -529,13 +532,16 @@ func CreateWalletForName(rootPath, name string) { } } -func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { +func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}, ) string { return setupAllocationWithWallet(t, EscapedTestName(t), cliConfigFilename, rootPath, extraParams...) } func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, rootPath, params string) ([]string, error) { t.Log(cliConfigFilename, "configdir path") - return RunCommand(t, fmt.Sprintf( + if (wallet == "") { + wallet = "default" + } + command := fmt.Sprintf( "%s/zbox newallocation %s --silent --wallet %s --configDir %s --config %s --allocationFileName %s", rootPath, params, @@ -543,7 +549,8 @@ func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, cliConfigFilename, "config.yaml", wallet+"_allocation.txt", - ), 3, time.Second*5) + ) + return RunCommand(t, command, 3, time.Second*5) } func setupAllocationWithWallet(t *test.SystemTest, walletName, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { From 2c65a8f709e6e8fa0fed845732f04f6240cf5936 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi <53532703+YarikRevich@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:14:17 +0100 Subject: [PATCH 06/19] fix: lowered frequency of system tests (#1151) --- .github/workflows/nightly-staging-tenderly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-staging-tenderly.yml b/.github/workflows/nightly-staging-tenderly.yml index a9d0ba44d7..f070c7287f 100644 --- a/.github/workflows/nightly-staging-tenderly.yml +++ b/.github/workflows/nightly-staging-tenderly.yml @@ -4,8 +4,8 @@ concurrency: cancel-in-progress: true on: schedule: - # Runs every two days at 6am UTC - - cron: '0 6 */2 * *' + # Runs every five days at 6am UTC + - cron: '0 6 */5 * *' workflow_dispatch: jobs: From 976e0b0311f6be957ebb84b280551ba0922a2f87 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 22:24:03 +0545 Subject: [PATCH 07/19] s3migration --- internal/cli/util/utils.go | 26 +- .../s3migration_tests/1_google_drive_test.go | 15 +- .../s3migration_tests/2_dropbox_test.go | 14 +- .../s3migration_tests/3_one_drive_test.go | 16 +- .../4_microsoft_azure_test.go | 18 +- .../5_google_cloud_storage_test.go | 16 +- .../aws_s3_migration_tests/0_aws_s3_test.go} | 309 ++++++------------ .../aws_s3_migration_tests/1_aws_s3_test.go} | 258 +++++++-------- .../aws_s3_migration_tests/3_aws_s3_test.go} | 186 ++++------- .../cli_tests/s3migration_tests/main_test.go | 6 +- .../s3migration_tests/shared/main.go | 11 +- .../s3migration_tests/shared/utils.go | 151 +++++++++ 12 files changed, 490 insertions(+), 536 deletions(-) rename tests/cli_tests/{0_s3mgrt_migrate_alternate_test.go => s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go} (50%) rename tests/cli_tests/{0_s3mgrt_migrate_alternate2_test.go => s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go} (57%) rename tests/cli_tests/{0_s3mgrt_migrate_test.go => s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go} (57%) create mode 100644 tests/cli_tests/s3migration_tests/shared/utils.go diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 860e3eb11b..df4bea7bde 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -463,9 +463,8 @@ func ReadFileMC(testSetup *testing.T) McConfiguration { return config } -func MigrateFromS3migration(t *test.SystemTest, params string) ([]string, error) { - commandGenerated := fmt.Sprintf("../s3migration migrate %s", params) - t.Log(commandGenerated) +func MigrateFromCloud(t *test.SystemTest, params string) ([]string, error) { + commandGenerated := fmt.Sprintf("../s3mgrt migrate %s", params) return RunCommand(t, commandGenerated, 1, time.Hour*2) } @@ -532,13 +531,13 @@ func CreateWalletForName(rootPath, name string) { } } -func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}, ) string { +func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { return setupAllocationWithWallet(t, EscapedTestName(t), cliConfigFilename, rootPath, extraParams...) } func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, rootPath, params string) ([]string, error) { t.Log(cliConfigFilename, "configdir path") - if (wallet == "") { + if wallet == "" { wallet = "default" } command := fmt.Sprintf( @@ -594,7 +593,7 @@ func getAllocationID(str string) (string, error) { } func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err error) { - pattern := `total count :: (\d+)` + pattern := `Total files ::(\d+)` re := regexp.MustCompile(pattern) match := re.FindStringSubmatch(strings.Join(output, "\n")) @@ -618,3 +617,18 @@ func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err erro return 0, 0, errors.New("no match found") } + +func GetFileStats(t *test.SystemTest, cliConfigFilename, param string, retry bool) ([]string, error) { + t.Logf("Getting file stats...") + cmd := fmt.Sprintf( + "./zbox stats %s --silent --wallet %s --configDir ./config --config %s", + param, + EscapedTestName(t)+"_wallet.json", + cliConfigFilename, + ) + if retry { + return RunCommand(t, cmd, 3, time.Second*2) + } else { + return RunCommandWithoutRetry(cmd) + } +} diff --git a/tests/cli_tests/s3migration_tests/1_google_drive_test.go b/tests/cli_tests/s3migration_tests/1_google_drive_test.go index 8d50dfbe0c..e260847d21 100644 --- a/tests/cli_tests/s3migration_tests/1_google_drive_test.go +++ b/tests/cli_tests/s3migration_tests/1_google_drive_test.go @@ -12,7 +12,7 @@ import ( func Test0Gdrive(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - + t.Log(shared.ConfigData.GdriveAccessToken) if shared.ConfigData.GdriveAccessToken == "" { t.Skip("Gdrive Access Token was missing") } @@ -22,12 +22,12 @@ func Test0Gdrive(testSetup *testing.T) { allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.GdriveAccessToken, "secret-key": shared.ConfigData.GdriveRefreshToken, "allocation": allocationID, "source": "google_drive", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "config": shared.ConfigPath, "configDir": shared.ConfigDir, "skip": 0, @@ -35,7 +35,6 @@ func Test0Gdrive(testSetup *testing.T) { require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) - if err != nil { t.Fatal(err) } @@ -44,7 +43,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.GdriveAccessToken, "secret-key": shared.ConfigData.GdriveRefreshToken, "allocation": shared.DefaultAllocationId, @@ -61,7 +60,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.GdriveAccessToken, "secret-key": shared.ConfigData.GdriveRefreshToken, "source": "google_drive", @@ -75,7 +74,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", "allocation": shared.DefaultAllocationId, @@ -92,7 +91,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "allocation": shared.DefaultAllocationId, "source": "google_drive", "wallet": shared.DefaultWallet, diff --git a/tests/cli_tests/s3migration_tests/2_dropbox_test.go b/tests/cli_tests/s3migration_tests/2_dropbox_test.go index 81772da7ff..add377a523 100644 --- a/tests/cli_tests/s3migration_tests/2_dropbox_test.go +++ b/tests/cli_tests/s3migration_tests/2_dropbox_test.go @@ -25,12 +25,12 @@ func Test0Dropbox(testSetup *testing.T) { allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": allocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.DropboxAccessToken, "secret-key": shared.ConfigData.DropboxRefreshToken, "allocation": allocationID, "source": "dropbox", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "config": shared.ConfigPath, "configDir": shared.ConfigDir, "skip": 0, @@ -40,7 +40,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.DropboxAccessToken, "secret-key": shared.ConfigData.DropboxRefreshToken, "allocation": shared.DefaultAllocationId, @@ -56,7 +56,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.DropboxAccessToken, "secret-key": shared.ConfigData.DropboxRefreshToken, "source": "dropbox", @@ -70,7 +70,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", "allocation": shared.DefaultAllocationId, @@ -87,7 +87,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "allocation": shared.DefaultAllocationId, "source": "dropbox", "wallet": shared.DefaultWallet, @@ -108,7 +108,7 @@ func Test0Dropbox(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) diff --git a/tests/cli_tests/s3migration_tests/3_one_drive_test.go b/tests/cli_tests/s3migration_tests/3_one_drive_test.go index 65d45a559f..c7aef394dd 100644 --- a/tests/cli_tests/s3migration_tests/3_one_drive_test.go +++ b/tests/cli_tests/s3migration_tests/3_one_drive_test.go @@ -25,10 +25,10 @@ func Test0OneDrive(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "onedrive", "config": shared.ConfigPath, @@ -40,10 +40,10 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": shared.DefaultAllocationId, "source": "onedrive", "config": shared.ConfigPath, @@ -56,7 +56,7 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", "wallet": shared.DefaultWallet, @@ -72,7 +72,7 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when token and refresh token is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", "wallet": shared.DefaultWallet, @@ -89,7 +89,7 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", "source": "invalid", @@ -111,7 +111,7 @@ func Test0OneDrive(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) diff --git a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go index 7d47ceb271..2495d58a5f 100644 --- a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go +++ b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go @@ -28,11 +28,11 @@ func Test0MicrosoftAzure(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "azure", "config": shared.ConfigPath, @@ -54,11 +54,11 @@ func Test0MicrosoftAzure(testSetup *testing.T) { "size": shared.AllocSize, }) - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "azure", "config": shared.ConfigPath, @@ -71,7 +71,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", @@ -86,7 +86,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) t.RunSequentially("Should fail when connection string is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "invalid", @@ -104,7 +104,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) t.RunSequentially("Should fail when connection string is missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "wallet": shared.DefaultWallet, @@ -120,7 +120,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { require.Contains(t, strings.Join(output, "\n"), "Missing fields: connection string", "Output was not as expected", strings.Join(output, "\n")) }) t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", @@ -143,7 +143,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": size, }) }) diff --git a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go index 6da4697233..43d5bd39f4 100644 --- a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go +++ b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go @@ -24,12 +24,12 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, "secret-key": shared.ConfigData.GoogleCloudRefreshToken, - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "google_cloud_storage", "config": shared.ConfigPath, @@ -52,12 +52,12 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, "secret-key": shared.ConfigData.GoogleCloudRefreshToken, - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "google_cloud_storage", "config": shared.ConfigPath, @@ -76,7 +76,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, @@ -93,7 +93,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when client credentials is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, @@ -112,7 +112,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when key is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": "invalid", @@ -131,7 +131,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, diff --git a/tests/cli_tests/0_s3mgrt_migrate_alternate_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go similarity index 50% rename from tests/cli_tests/0_s3mgrt_migrate_alternate_test.go rename to tests/cli_tests/s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go index 87f464e3e0..d8e91af3a4 100644 --- a/tests/cli_tests/0_s3mgrt_migrate_alternate_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go @@ -1,45 +1,31 @@ -package cli_tests +package s3migration_tests import ( "bytes" - "crypto/rand" - "encoding/json" - "fmt" - "math" - "math/big" "os" "path" - "path/filepath" "strings" "testing" - "time" "github.com/0chain/system_test/internal/api/util/test" - climodel "github.com/0chain/system_test/internal/cli/model" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/require" - "golang.org/x/crypto/sha3" -) - -const chunksize = 64 * 1024 - -const ( - dirPrefix = "dir" - dirMaxRand = 1000 ) func Test0S3MigrationAlternate(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - if s3SecretKey == "" || s3AccessKey == "" { - t.Skip("s3SecretKey or s3AccessKey was missing") + if shared.ConfigData.S3SecretKey == "" || shared.ConfigData.S3AccessKey == "" { + t.Skip("shared.ConfigData.S3SecretKey or shared.ConfigData.S3AccessKey was missing") } fileKey := "sdfg" + ".txt" t.TestSetup("Setup s3 bucket with relevant file", func() { // Cleanup bucket before test - err := cleanupBucket(S3Client, s3BucketNameAlternate) + err := shared.CleanupBucket(shared.S3Client, shared.ConfigData.S3BucketNameAlternate) if err != nil { t.Log("Failed to cleanup bucket: ", err) } @@ -47,8 +33,8 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { fileContents := []byte("Hello, World!") // Upload the file to S3 - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKey), Body: bytes.NewReader(fileContents), }) @@ -58,16 +44,15 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { }) t.RunSequentially("Should migrate existing bucket successfully with skip 0 and replace existing file", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 0, })) @@ -77,25 +62,24 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, remotepath) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket to specified path successfully with encryption on", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) remotepath := "/root2" - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "migrate-to": remotepath, })) @@ -105,24 +89,23 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remoteFilePath := path.Join(remotepath, "/") - remoteFilePath = path.Join(remoteFilePath, s3BucketNameAlternate) + remoteFilePath = path.Join(remoteFilePath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, "/") remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate as copy bucket successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) @@ -130,11 +113,11 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - output, err = migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err = cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 2, "dup-suffix": "_copy", @@ -149,21 +132,20 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { // parts := strings.Split(fileKey, ".") // fileKey_modified := parts[0]+ "_modified." + parts[1] // remoteFilePath := path.Join(remotepath, fileKey_modified) - // uploadStats := checkStats(t, remoteFilePath, fileKey_modified, allocationID) + // uploadStats := shared.CheckStats(t, remoteFilePath, fileKey_modified, allocationID) // require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with encryption on", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "encrypt": "true", })) @@ -173,23 +155,22 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, true) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, true) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should skip migration with skip flag == 1 and migartion should be skipped", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 1, })) @@ -200,16 +181,15 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { }) t.RunSequentially("Should migrate successfully with duplicate files with skip flag == 2", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 2, })) @@ -222,23 +202,22 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { // FIXME : copy extension is not there // parts := strings.Split(fileKey, ".") // fileKey_modified := parts[0]+ "_copy." + parts[1] - remotepath = path.Join(remotepath, s3BucketNameAlternate) + remotepath = path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath := path.Join(remotepath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate successfully with duplication files with skip flag == 2 and dup-suffix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 2, "dup-suffix": "_modified", @@ -252,28 +231,27 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { // FIXME : dupl suffix is not working // parts := strings.Split(fileKey, ".") // fileKey_modified := parts[0]+ "_modified." + parts[1] - remotepath = path.Join(remotepath, s3BucketNameAlternate) + remotepath = path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath := path.Join(remotepath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate successfully and delete the s3 bucket file and use custom workdir", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - workingDirName := createDirectoryForTestname(t) + workingDirName := shared.CreateDirectoryForTestname(t) // remove the dir after use defer func() { _ = os.RemoveAll(workingDirName) }() - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 0, "delete-source": true, @@ -285,19 +263,18 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should error out if workdir is not default and not empty", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - workingDirName := createDirectoryForTestname(t) + workingDirName := shared.CreateDirectoryForTestname(t) file, _ := os.CreateTemp(workingDirName, "prefix") // remove the dir after use defer func() { @@ -306,11 +283,11 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { }() remotepath := "/root3" - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "migrate-to": remotepath, "wd": workingDirName, @@ -321,107 +298,3 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, output[0], "working directory not empty", "Output was not as expected", strings.Join(output, "\n")) }) } - -func checkStats(t *test.SystemTest, remoteFilePath, fname, allocationID string, encrypted bool) bool { - t.Log("remotepath: ", remoteFilePath) - output, err := getFileStats(t, configPath, createParams(map[string]interface{}{ - "allocation": allocationID, - "remotepath": remoteFilePath, - "json": "true", - }), true) - require.Nil(t, err, strings.Join(output, "\n")) - require.Len(t, output, 1) - - var stats map[string]*climodel.FileStats - t.Log(output[0]) - err = json.Unmarshal([]byte(output[0]), &stats) - require.Nil(t, err) - - if len(stats) == 0 { - t.Logf("0. zero no files") - return false - } - - for _, data := range stats { - if fname != data.Name { - t.Logf("1. %s != %s", fname, data.Name) - return false - } - if remoteFilePath != data.Path { - t.Logf("2. %s != %s", remoteFilePath, data.Path) - return false - } - hash := fmt.Sprintf("%x", sha3.Sum256([]byte(allocationID+":"+remoteFilePath))) - if hash != data.PathHash { - t.Logf("3. %s != %s", hash, data.PathHash) - return false - } - if int64(0) != data.NumOfBlockDownloads { - t.Logf("4. %d != %d", int64(0), data.NumOfBlockDownloads) - return false - } - if int64(1) != data.NumOfUpdates { - t.Logf("5. %d != %d", int64(1), data.NumOfUpdates) - return false - } - if float64(data.NumOfBlocks) != math.Ceil(float64(data.Size)/float64(chunksize)) { - t.Logf("6. %f != %f", float64(data.NumOfBlocks), math.Ceil(float64(data.Size)/float64(chunksize))) - return false - } - if data.WriteMarkerTxn == "" { - if data.BlockchainAware != false { - t.Logf("7. %t", data.BlockchainAware) - return false - } - } else { - if data.BlockchainAware != true { - t.Logf("8. %t", data.BlockchainAware) - return false - } - } - } - return true -} - -func createDirectoryForTestname(t *test.SystemTest) (fullPath string) { - randomBigInt, err := rand.Int(rand.Reader, big.NewInt(int64(dirMaxRand))) - require.Nil(t, err) - - randomNumber := int(randomBigInt.Int64()) - - // Generate a unique directory name based on the random number and current timestamp - dirName := fmt.Sprintf("%s%d_%d", dirPrefix, randomNumber, time.Now().UnixNano()) - - fullPath, err = filepath.Abs(dirName) - require.Nil(t, err) - - err = os.MkdirAll(fullPath, os.ModePerm) - require.Nil(t, err) - - t.Log("Directory created successfully: ", fullPath) - - return fullPath -} - -func cleanupBucket(svc *s3.S3, s3BucketNameAlternate string) error { - // List all objects within the bucket - resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: aws.String(s3BucketNameAlternate), - }) - if err != nil { - return err - } - - // Delete each object in the bucket - for _, obj := range resp.Contents { - _, err := svc.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), - Key: obj.Key, - }) - if err != nil { - return err - } - } - - return nil -} diff --git a/tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go similarity index 57% rename from tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go rename to tests/cli_tests/s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go index fe6ee6696d..2fd428ff8d 100644 --- a/tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go @@ -1,9 +1,7 @@ -package cli_tests +package s3migration_tests import ( - "bytes" - "crypto/sha1" // #nosec - "encoding/hex" + "bytes" // #nosec "fmt" "os" "path" @@ -13,7 +11,9 @@ import ( "time" "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/require" @@ -22,14 +22,14 @@ import ( func Test0S3MigrationAlternatePart2(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - if s3SecretKey == "" || s3AccessKey == "" { - t.Skip("s3SecretKey or s3AccessKey was missing") + if shared.ConfigData.S3SecretKey == "" || shared.ConfigData.S3AccessKey == "" { + t.Skip("shared.ConfigData.S3SecretKey or shared.ConfigData.S3AccessKey was missing") } fileKey := "OneMinNew" + ".txt" t.TestSetup("Setup s3 bucket with relevant file", func() { // Cleanup before test - err := cleanupBucket(S3Client, s3BucketNameAlternate) + err := shared.CleanupBucket(shared.S3Client, shared.ConfigData.S3BucketNameAlternate) if err != nil { t.Log("Failed to cleanup bucket: ", err) } @@ -37,8 +37,8 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { fileContents := []byte("Hello, World!") // Upload the file to S3 - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKey), Body: bytes.NewReader(fileContents), }) @@ -48,21 +48,20 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { }) t.RunSequentially("Should migrate existing bucket successfully with concurrency==4 and working dir current dir", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - workingDirName := createDirectoryForTestname(t) + workingDirName := shared.CreateDirectoryForTestname(t) // remove the dir after use defer func() { _ = os.RemoveAll(workingDirName) }() - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "concurrency": 4, "wd": workingDirName, @@ -73,32 +72,31 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with newer than prefix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "oneMinOld" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "newer-than": time.Now().Unix() - 60, // start timestamp })) @@ -107,22 +105,21 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with older than prefix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) olderThanFileKey := "olderThanFile" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(olderThanFileKey), Body: bytes.NewReader(fileContents), }) @@ -130,11 +127,11 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { cliutils.Wait(t, 70*time.Second) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "older-than": time.Now().Unix() - 60, // end timestamp })) @@ -144,39 +141,38 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, olderThanFileKey) - uploadStats := checkStats(t, remoteFilePath, olderThanFileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, olderThanFileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with files staring with given prefix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyToBemigrated := "mgrt" + ".txt" fileKeyNotToBeMigrated := "noMgrt" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyToBemigrated), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNotToBeMigrated), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "prefix": "mgrt", })) @@ -185,25 +181,24 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePathPos := path.Join(remoteFilePath, fileKeyToBemigrated) remoteFilePathNeg := path.Join(remoteFilePath, fileKeyNotToBeMigrated) - uploadStats := checkStats(t, remoteFilePathPos, fileKeyToBemigrated, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePathPos, fileKeyToBemigrated, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") - uploadStats = checkStats(t, remoteFilePathNeg, fileKeyNotToBeMigrated, allocationID, false) + uploadStats = shared.CheckStats(t, remoteFilePathNeg, fileKeyNotToBeMigrated, allocationID, false) require.Equal(t, false, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass when allocation flag missing but allocation path is given", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForAllocPath" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) @@ -215,11 +210,11 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { err = os.WriteFile(allocPath, []byte(allocationID), 0644) //nolint:gosec require.Nil(t, err, "allocation file is not written properly") - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "alloc-path": allocPath, })) @@ -228,21 +223,20 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass when access key and secret key is missing but aws-cred-path path is given", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForAwsCredPath" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) @@ -252,8 +246,8 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Nil(t, err, "can't get current dir") awsCredPath := filepath.Join(currentDir, "awsCredPathForTestS3.txt") lines := []string{ - fmt.Sprintf(`aws_access_key: "%v"`, s3AccessKey), - fmt.Sprintf(`aws_secret_key: "%v"`, s3SecretKey), + fmt.Sprintf(`aws_access_key: "%v"`, shared.ConfigData.S3AccessKey), + fmt.Sprintf(`aws_secret_key: "%v"`, shared.ConfigData.S3SecretKey), } file, err := os.OpenFile(awsCredPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) require.Nil(t, err, "file is not created properly") @@ -264,10 +258,10 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Nil(t, err, "failed to write file") } - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "bucket": s3BucketNameAlternate, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "bucket": shared.ConfigData.S3BucketNameAlternate, "allocation": allocationID, - "wallet": escapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "aws-cred-path": awsCredPath, })) @@ -276,32 +270,31 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass concurrency flag is set to 20", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForConCurrTest" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "concurrency": 20, })) @@ -311,32 +304,31 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass retry flag is set to 4", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForRetryTest" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "retry": 4, })) @@ -346,40 +338,39 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass when filename size is more than 100 character with renamed file names", func(t *test.SystemTest) { // Cleanup before test - err := cleanupBucket(S3Client, s3BucketNameAlternate) + err := shared.CleanupBucket(shared.S3Client, shared.ConfigData.S3BucketNameAlternate) if err != nil { t.Log("Failed to cleanup bucket: ", err) } - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) // As per the current logic in s3-migration even the longer file names would be migrated with file names // trimmed to 100 chars. fileKeyNew := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbb.txt" fileContents := []byte("Hello, World!") - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) // mssg can be changed @@ -388,35 +379,10 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) - shortFileKey := getUniqueShortObjKey(fileKeyNew) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) + shortFileKey := shared.GetUniqueShortObjKey(fileKeyNew) remoteFilePath = path.Join(remoteFilePath, shortFileKey) - uploadStats := checkStats(t, remoteFilePath, shortFileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, shortFileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated does not match with expected file") }) } - -// This func is a copy of new file name generation logic in s3-migration from migration/migrate.go -// This func needs to be in sync with the original func. -func getUniqueShortObjKey(objectKey string) string { - // Max length to which objectKey would be trimmed to. - // Keeping this less than 100 chars to prevent longer name in case of uploading duplicate - // files with `_copy` suffixes. - const maxLength = 90 - - if len(objectKey) > maxLength { - // Generate a SHA-1 hash of the object key - hash := sha1.New() // #nosec - _, _ = hash.Write([]byte(objectKey)) - hashSum := hash.Sum(nil) - - // Convert the hash to a hexadecimal string - hashString := hex.EncodeToString(hashSum) - - // Combine the first 10 characters of the hash with a truncated object key - shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:]) - return shortKey - } - - return objectKey -} diff --git a/tests/cli_tests/0_s3mgrt_migrate_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go similarity index 57% rename from tests/cli_tests/0_s3mgrt_migrate_test.go rename to tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go index 5e9f992e0b..d832f914ff 100644 --- a/tests/cli_tests/0_s3mgrt_migrate_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go @@ -1,36 +1,35 @@ -package cli_tests +package s3migration_tests import ( - "fmt" "strings" "testing" - "time" "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/stretchr/testify/require" ) func Test0S3Migration(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - if s3SecretKey == "" || s3AccessKey == "" { - t.Skip("s3SecretKey or s3AccessKey was missing") + if shared.ConfigData.S3SecretKey == "" || shared.ConfigData.S3AccessKey == "" { + t.Skip("shared.ConfigData.S3SecretKey or shared.ConfigData.S3AccessKey was missing") } t.SetSmokeTests("Should migrate existing bucket successfully") t.RunSequentially("Should migrate existing bucket successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) @@ -40,16 +39,15 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should migrate empty bucket successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, "bucket": "system-tests-empty", - "wallet": escapedTestName(t) + "_wallet.json", + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) @@ -59,16 +57,16 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when bucket too large for allocation", func(t *test.SystemTest) { - allocSize := int64(64 * KB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ + allocSize := int64(64 * shared.KB) + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": allocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": shared.DefaultWallet, "allocation": allocationID, })) @@ -78,17 +76,12 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when bucket does not exist", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, "bucket": "invalid", - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -97,16 +90,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when bucket flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -115,16 +103,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": shared.DefaultWallet, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -133,17 +116,13 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -152,16 +131,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -170,17 +144,12 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when secret key invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, "secret-key": "invalid", - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -189,16 +158,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when secret key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -207,17 +171,13 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access and secret key invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -226,15 +186,10 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access and secret key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -242,8 +197,3 @@ func Test0S3Migration(testSetup *testing.T) { require.Equal(t, output[0], "Error: aws credentials missing", "Output was not as expected", strings.Join(output, "\n")) }) } - -func migrateFromS3(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { - t.Logf("Migrating S3 bucket to Zus...") - return cliutils.RunCommand(t, fmt.Sprintf("./s3mgrt migrate --silent --configDir ./config --config %s --network %s %s", cliConfigFilename, cliConfigFilename, params), 1, time.Second*2) -} diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go index 38ab4f7de6..1e3e9d86f7 100644 --- a/tests/cli_tests/s3migration_tests/main_test.go +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -10,7 +10,6 @@ import ( "github.com/0chain/system_test/internal/api/util/config" "github.com/0chain/system_test/internal/api/util/test" - cli_utils "github.com/0chain/system_test/internal/cli/util" cliutils "github.com/0chain/system_test/internal/cli/util" "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/aws/aws-sdk-go/aws" @@ -39,7 +38,7 @@ func setupConfig() { shared.ConfigPath = "config.yaml" rootPath := filepath.Dir(dir) shared.RootPath = rootPath - configPath := filepath.Join(shared.RootPath,"config") + configPath := filepath.Join(shared.RootPath, "config") viper.SetConfigName("nodes") viper.SetConfigType("yaml") @@ -53,7 +52,7 @@ func setupConfig() { func defaultData() { t := testing.T{} system_test := test.NewSystemTest(&t) - defaultAllocationId := cli_utils.SetupAllocation(system_test, shared.ConfigData.ConnectionString, shared.RootPath, map[string]interface{}{ + defaultAllocationId := cliutils.SetupAllocation(system_test, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": shared.AllocSize, }) @@ -125,7 +124,6 @@ func TestMain(m *testing.M) { shared.ConfigDir, _ = filepath.Abs(shared.ConfigDir) - setupConfig() shared.WalletMutex.Lock() diff --git a/tests/cli_tests/s3migration_tests/shared/main.go b/tests/cli_tests/s3migration_tests/shared/main.go index 7c56851f21..1db4138f28 100644 --- a/tests/cli_tests/s3migration_tests/shared/main.go +++ b/tests/cli_tests/s3migration_tests/shared/main.go @@ -22,10 +22,13 @@ var ( ) const ( - KB = 1024 // kilobyte - MB = 1024 * KB // megabyte - GB = 1024 * MB // gigabyte - AllocSize = int64(50 * MB) + KB = 1024 // kilobyte + MB = 1024 * KB // megabyte + GB = 1024 * MB // gigabyte + AllocSize = int64(50 * MB) + Chunksize = 64 * 1024 + DirPrefix = "dir" + DirMaxRand = 1000 ) func SetupConfig(parsedConfig *util.Config) { diff --git a/tests/cli_tests/s3migration_tests/shared/utils.go b/tests/cli_tests/s3migration_tests/shared/utils.go new file mode 100644 index 0000000000..7aecc8e80a --- /dev/null +++ b/tests/cli_tests/s3migration_tests/shared/utils.go @@ -0,0 +1,151 @@ +package shared + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "math/big" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/sha3" + + "github.com/0chain/system_test/internal/api/util/test" + climodel "github.com/0chain/system_test/internal/cli/model" + cli_utils "github.com/0chain/system_test/internal/cli/util" +) + +func CleanupBucket(svc *s3.S3, s3BucketNameAlternate string) error { + // List all objects within the bucket + resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(s3BucketNameAlternate), + }) + if err != nil { + return err + } + + // Delete each object in the bucket + for _, obj := range resp.Contents { + _, err := svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(s3BucketNameAlternate), + Key: obj.Key, + }) + if err != nil { + return err + } + } + + return nil +} + +func CheckStats(t *test.SystemTest, remoteFilePath, fname, allocationID string, encrypted bool) bool { + t.Log("remotepath: ", remoteFilePath) + output, err := cli_utils.GetFileStats(t, ConfigPath, cli_utils.CreateParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remoteFilePath, + "json": "true", + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) + + var stats map[string]*climodel.FileStats + t.Log(output[0]) + err = json.Unmarshal([]byte(output[0]), &stats) + require.Nil(t, err) + + if len(stats) == 0 { + t.Logf("0. zero no files") + return false + } + + for _, data := range stats { + if fname != data.Name { + t.Logf("1. %s != %s", fname, data.Name) + return false + } + if remoteFilePath != data.Path { + t.Logf("2. %s != %s", remoteFilePath, data.Path) + return false + } + hash := fmt.Sprintf("%x", sha3.Sum256([]byte(allocationID+":"+remoteFilePath))) + if hash != data.PathHash { + t.Logf("3. %s != %s", hash, data.PathHash) + return false + } + if int64(0) != data.NumOfBlockDownloads { + t.Logf("4. %d != %d", int64(0), data.NumOfBlockDownloads) + return false + } + if int64(1) != data.NumOfUpdates { + t.Logf("5. %d != %d", int64(1), data.NumOfUpdates) + return false + } + if float64(data.NumOfBlocks) != math.Ceil(float64(data.Size)/float64(Chunksize)) { + t.Logf("6. %f != %f", float64(data.NumOfBlocks), math.Ceil(float64(data.Size)/float64(Chunksize))) + return false + } + if data.WriteMarkerTxn == "" { + if data.BlockchainAware != false { + t.Logf("7. %t", data.BlockchainAware) + return false + } + } else { + if data.BlockchainAware != true { + t.Logf("8. %t", data.BlockchainAware) + return false + } + } + } + return true +} + +func CreateDirectoryForTestname(t *test.SystemTest) (fullPath string) { + randomBigInt, err := rand.Int(rand.Reader, big.NewInt(int64(DirMaxRand))) + require.Nil(t, err) + + randomNumber := int(randomBigInt.Int64()) + + // Generate a unique directory name based on the random number and current timestamp + dirName := fmt.Sprintf("%s%d_%d", DirPrefix, randomNumber, time.Now().UnixNano()) + + fullPath, err = filepath.Abs(dirName) + require.Nil(t, err) + + err = os.MkdirAll(fullPath, os.ModePerm) + require.Nil(t, err) + + t.Log("Directory created successfully: ", fullPath) + + return fullPath +} + +func GetUniqueShortObjKey(objectKey string) string { + // Max length to which objectKey would be trimmed to. + // Keeping this less than 100 chars to prevent longer name in case of uploading duplicate + // files with `_copy` suffixes. + const maxLength = 90 + + if len(objectKey) > maxLength { + // Generate a SHA-1 hash of the object key + hash := sha1.New() // #nosec + _, _ = hash.Write([]byte(objectKey)) + hashSum := hash.Sum(nil) + + // Convert the hash to a hexadecimal string + hashString := hex.EncodeToString(hashSum) + + // Combine the first 10 characters of the hash with a truncated object key + shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:]) + return shortKey + } + + return objectKey +} From 9ce43323bac74e676df30de17c64eeffa0f3c655 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 22:48:08 +0545 Subject: [PATCH 08/19] config --- .../s3migration_tests/config/config.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/cli_tests/s3migration_tests/config/config.yaml diff --git a/tests/cli_tests/s3migration_tests/config/config.yaml b/tests/cli_tests/s3migration_tests/config/config.yaml new file mode 100644 index 0000000000..528e913f47 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/config/config.yaml @@ -0,0 +1,15 @@ +block_worker: https://test.zus.network/dns +confirmation_chain_length: 3 +ethereum_node_url: "https://rpc.tenderly.co/fork/5b7ffac9-50cc-4169-b0ca-6fd203d26ef6" +min_confirmation: 50 +min_submit: 50 +signature_scheme: bls0chain +store_unlock_duration_sec: 2 + +bridge: + bridge_address: 0x7700D773022b19622095118Fadf46f7B9448Be9b + token_address: 0xb9EF770B6A5e12E45983C5D80545258aA38F3B78 + authorizers_address: 0x481daB4407b9880DE0A68dc62E6aF611c4949E42 + uniswap_address: 0x4c12C2FeEDD86267d17dB64BaB2cFD12cD8611f5 + ethereum_address: 0x8E25cfd9bd6c0ca67a5522cd920b3c66D39d6E97 + password: "12345678" From e5a929cba0363b4a198a12b6ba06178f6d44ce4a Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 23:00:36 +0545 Subject: [PATCH 09/19] updated for lint --- tests/cli_tests/main_test.go | 4 ---- .../s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go | 2 -- tests/cli_tests/s3migration_tests/shared/utils.go | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/cli_tests/main_test.go b/tests/cli_tests/main_test.go index 05f7316b2f..57de26d4e1 100644 --- a/tests/cli_tests/main_test.go +++ b/tests/cli_tests/main_test.go @@ -95,10 +95,6 @@ var ( ethereumNodeURL string tokenAddress string ethereumAddress string - s3SecretKey string - s3AccessKey string - s3bucketName string - s3BucketNameAlternate string S3Client *s3.S3 ) diff --git a/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go index d832f914ff..8cc4426d9e 100644 --- a/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go @@ -116,7 +116,6 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": shared.ConfigData.S3SecretKey, @@ -171,7 +170,6 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access and secret key invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", diff --git a/tests/cli_tests/s3migration_tests/shared/utils.go b/tests/cli_tests/s3migration_tests/shared/utils.go index 7aecc8e80a..0800d42101 100644 --- a/tests/cli_tests/s3migration_tests/shared/utils.go +++ b/tests/cli_tests/s3migration_tests/shared/utils.go @@ -2,7 +2,7 @@ package shared import ( "crypto/rand" - "crypto/sha1" + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -135,7 +135,7 @@ func GetUniqueShortObjKey(objectKey string) string { if len(objectKey) > maxLength { // Generate a SHA-1 hash of the object key - hash := sha1.New() // #nosec + hash := sha256.New() // #nosec _, _ = hash.Write([]byte(objectKey)) hashSum := hash.Sum(nil) From 6c26c3cd9c2afc6d1ea8735ef7873c192cdffeaf Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 01:29:52 +0545 Subject: [PATCH 10/19] added for system test --- internal/api/util/config/config.go | 4 + .../1_general_test.go} | 4 +- .../box_tests/1_general_test.go | 116 ++++++++++++++++++ .../1_general_test.go} | 21 +--- .../1_general_test.go} | 2 +- .../1_general_test.go} | 0 .../1_general_test.go} | 4 +- .../s3migration_tests/shared/main.go | 19 +++ 8 files changed, 146 insertions(+), 24 deletions(-) rename tests/cli_tests/s3migration_tests/{4_microsoft_azure_test.go => azure_tests/1_general_test.go} (97%) create mode 100644 tests/cli_tests/s3migration_tests/box_tests/1_general_test.go rename tests/cli_tests/s3migration_tests/{2_dropbox_test.go => dropbox_tests/1_general_test.go} (85%) rename tests/cli_tests/s3migration_tests/{1_google_drive_test.go => gdrive_tests/1_general_test.go} (99%) rename tests/cli_tests/s3migration_tests/{5_google_cloud_storage_test.go => google_cloud_storage_tests/1_general_test.go} (100%) rename tests/cli_tests/s3migration_tests/{3_one_drive_test.go => onedrive_tests/1_general_test.go} (96%) diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 67a8bc8022..a90ba001ba 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -49,6 +49,10 @@ type Config struct { GoogleClientSecret string `yaml:"googleClientSecret"` OneDriveAccessToken string `yaml:"oneDriveAccessToken"` OneDriveRefreshToken string `yaml:"oneDriveRefreshToken"` + BoxAccessToken string `yaml:"boxAccessToken"` + BoxRefreshToken string `yaml:"boxRefreshToken"` + BoxClientId string `yaml:"boxClientId"` + BoxClientSecret string `yaml:"boxClientSecret"` } func Parse(configPath string) *Config { diff --git a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go similarity index 97% rename from tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go rename to tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go index 2495d58a5f..55e5a0afc2 100644 --- a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go +++ b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go @@ -1,4 +1,4 @@ -package s3migration_tests +package azure_tests import ( "strings" @@ -143,7 +143,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = shared.SetupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": size, }) }) diff --git a/tests/cli_tests/s3migration_tests/box_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/box_tests/1_general_test.go new file mode 100644 index 0000000000..3dcf6cc3c6 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/box_tests/1_general_test.go @@ -0,0 +1,116 @@ +package box_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0Box(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + t.Log(shared.ConfigData.BoxAccessToken) + if shared.ConfigData.BoxAccessToken == "" || shared.ConfigData.BoxRefreshToken == "" || shared.ConfigData.BoxClientId == "" || shared.ConfigData.BoxClientSecret == "" { + t.Skip("Box Credentials were missing") + } + t.SetSmokeTests("Should migrate existing Box folder and files successfully") + + t.RunSequentially("Should migrate existing box folder and files successfully", func(t *test.SystemTest) { + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.BoxAccessToken, + "secret-key": shared.ConfigData.BoxRefreshToken, + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "allocation": allocationID, + "source": "box", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.BoxAccessToken, + "secret-key": shared.ConfigData.BoxRefreshToken, + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "allocation": shared.DefaultAllocationId, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.BoxAccessToken, + "secret-key": shared.ConfigData.BoxRefreshToken, + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "allocation": shared.DefaultAllocationId, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "allocation": shared.DefaultAllocationId, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + t.Logf("EXpected log %v", strings.Join(output, "\n")) + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: access key, secret key") + }) +} diff --git a/tests/cli_tests/s3migration_tests/2_dropbox_test.go b/tests/cli_tests/s3migration_tests/dropbox_tests/1_general_test.go similarity index 85% rename from tests/cli_tests/s3migration_tests/2_dropbox_test.go rename to tests/cli_tests/s3migration_tests/dropbox_tests/1_general_test.go index add377a523..142f5e9a17 100644 --- a/tests/cli_tests/s3migration_tests/2_dropbox_test.go +++ b/tests/cli_tests/s3migration_tests/dropbox_tests/1_general_test.go @@ -1,7 +1,6 @@ -package s3migration_tests +package dropbox_tests import ( - "fmt" "strings" "testing" @@ -108,24 +107,8 @@ func Test0Dropbox(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = shared.SetupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) } - -func setupAllocationWithWalletWithoutTest(t *test.SystemTest, walletName, cliConfigFilename string, extraParams ...map[string]interface{}) (string, error) { - options := map[string]interface{}{"size": "10000000", "lock": "5"} - - for _, params := range extraParams { - for k, v := range params { - options[k] = v - } - } - cli_utils.CreateWalletForName(shared.RootPath, walletName) - output, _ := cli_utils.CreateNewAllocationForWallet(t, walletName, cliConfigFilename, shared.RootPath, cli_utils.CreateParams(options)) - defer func() { - fmt.Printf("err: %v\n", output) - }() - return cli_utils.GetAllocationID(output[0]) -} diff --git a/tests/cli_tests/s3migration_tests/1_google_drive_test.go b/tests/cli_tests/s3migration_tests/gdrive_tests/1_general_test.go similarity index 99% rename from tests/cli_tests/s3migration_tests/1_google_drive_test.go rename to tests/cli_tests/s3migration_tests/gdrive_tests/1_general_test.go index e260847d21..17b12974c0 100644 --- a/tests/cli_tests/s3migration_tests/1_google_drive_test.go +++ b/tests/cli_tests/s3migration_tests/gdrive_tests/1_general_test.go @@ -1,4 +1,4 @@ -package s3migration_tests +package gdrive_tests import ( "strings" diff --git a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go b/tests/cli_tests/s3migration_tests/google_cloud_storage_tests/1_general_test.go similarity index 100% rename from tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go rename to tests/cli_tests/s3migration_tests/google_cloud_storage_tests/1_general_test.go diff --git a/tests/cli_tests/s3migration_tests/3_one_drive_test.go b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go similarity index 96% rename from tests/cli_tests/s3migration_tests/3_one_drive_test.go rename to tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go index c7aef394dd..c6c0168623 100644 --- a/tests/cli_tests/s3migration_tests/3_one_drive_test.go +++ b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go @@ -1,4 +1,4 @@ -package s3migration_tests +package onedrive_tests import ( "fmt" @@ -111,7 +111,7 @@ func Test0OneDrive(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = shared.SetupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) diff --git a/tests/cli_tests/s3migration_tests/shared/main.go b/tests/cli_tests/s3migration_tests/shared/main.go index 1db4138f28..4b0a054e3c 100644 --- a/tests/cli_tests/s3migration_tests/shared/main.go +++ b/tests/cli_tests/s3migration_tests/shared/main.go @@ -2,9 +2,12 @@ package shared import ( "encoding/json" + "fmt" "sync" util "github.com/0chain/system_test/internal/api/util/config" + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" "github.com/aws/aws-sdk-go/service/s3" ) @@ -34,3 +37,19 @@ const ( func SetupConfig(parsedConfig *util.Config) { ConfigData = *parsedConfig } + +func SetupAllocationWithWalletWithoutTest(t *test.SystemTest, walletName, cliConfigFilename string, extraParams ...map[string]interface{}) (string, error) { + options := map[string]interface{}{"size": "10000000", "lock": "5"} + + for _, params := range extraParams { + for k, v := range params { + options[k] = v + } + } + cli_utils.CreateWalletForName(RootPath, walletName) + output, _ := cli_utils.CreateNewAllocationForWallet(t, walletName, cliConfigFilename, RootPath, cli_utils.CreateParams(options)) + defer func() { + fmt.Printf("err: %v\n", output) + }() + return cli_utils.GetAllocationID(output[0]) +} From 66dfd86d40f57284d8dec792d64e8a896eb4ff2d Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 01:30:41 +0545 Subject: [PATCH 11/19] added for system test --- .../s3migration_tests/onedrive_tests/1_general_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go index c6c0168623..5d962b00cf 100644 --- a/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go +++ b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" ) -func Test0OneDrive(testSetup *testing.T) { +func TestOneDrive(testSetup *testing.T) { t := test.NewSystemTest(testSetup) if shared.ConfigData.OneDriveAccessToken == "" || shared.ConfigData.OneDriveRefreshToken == "" { From 83b7551740e178f4fdde9fb62bc27dfb3bcf8687 Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 02:15:27 +0545 Subject: [PATCH 12/19] updated config yaml file --- tests/cli_tests/config/cli_tests_config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 51337e9c3b..71fb6c448d 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -16,3 +16,7 @@ googleClientId: googleClientSecret: oneDriveAccessToken: oneDriveRefreshToken: +boxAccessToken: +boxRefreshToken: +boxClientId: +boxClientSecret: From b4de58c46142cd478f260c0e8850e72e9f31d64e Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 02:16:27 +0545 Subject: [PATCH 13/19] ignore s3mgrt --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fdb1965f0c..bfc7fc3edc 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ warp-*_output.txt warp minio mc -store \ No newline at end of file +store +s3mgrt \ No newline at end of file From 9e2cc4eec64641a680d2a01939ac18c08df18ea1 Mon Sep 17 00:00:00 2001 From: Maulik Singhal <69348454+smaulik13@users.noreply.github.com> Date: Fri, 14 Mar 2025 00:15:23 +0530 Subject: [PATCH 14/19] updated: added test cases for storage version (#1135) * updated: added test cases for storage version * updated: resolved comments * updated: added mofied code * updated: added allocagion test * updated: * updated: * updated: removed managing wallet from allocation tc * updated: added update blobber version in api test * updated:modified code * updated: added kill blobber * updated: added kill blobber * updated: added kill blobber * updated: added kill blobber tc * updated: resolved comments * updated: modified register blobber func for id verification * updated: reveretd create wallet * updated: modified register blobber * updated: default storage version to 1 * updated: added get blobber to verify * updated: urls reverted * updated: * updated: comments resolved * updated: comments resolved * updated: comments resolved * lint fix * sys_test * go update * lint fix * lint fix * uncomment * fixeS * expire alloc test fix --------- Co-authored-by: Akhilesh Mahajan Co-authored-by: Jayash Satolia <73050737+Jayashsatolia403@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- internal/api/model/api.go | 18 +++++ internal/api/model/helper.go | 1 + internal/api/util/client/api_client.go | 68 ++++++++++++++++++- internal/cli/model/model.go | 7 +- tests/api_tests/config/sc_owner_wallet.json | 13 ++++ tests/api_tests/main_test.go | 39 +++++++++++ tests/api_tests/register_blobber_test.go | 54 +++++++++++++-- tests/api_tests/update_blobber_test.go | 51 ++++++++++++++ tests/cli_tests/0_expired_allocation_test.go | 10 ++- tests/cli_tests/config/config.yaml | 2 +- tests/cli_tests/config/zbox_config.yaml | 2 +- .../zboxcli_blobber_config_update_test.go | 52 ++++++++++++++ .../zboxcli_create_allocation_test.go | 25 +++++++ 15 files changed, 328 insertions(+), 20 deletions(-) create mode 100644 tests/api_tests/config/sc_owner_wallet.json diff --git a/go.mod b/go.mod index 147acfb80a..8b3b8f7da3 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.18.13-0.20241209154844-07366a2c2666 + github.com/0chain/gosdk v1.18.14 github.com/go-resty/resty/v2 v2.7.0 github.com/herumi/bls-go-binary v1.31.0 github.com/shopspring/decimal v1.3.1 diff --git a/go.sum b/go.sum index 4747d95043..72e7f1fb90 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI= github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.18.13-0.20241209154844-07366a2c2666 h1:xnvnaDeuMFadWR4bM0iiXm7CArjt8V9aBsdx3xcevDs= -github.com/0chain/gosdk v1.18.13-0.20241209154844-07366a2c2666/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw= +github.com/0chain/gosdk v1.18.14 h1:N/PwLmUVQe1qM0eDuEDSQRB97a03Fp+I5W67cF0Zgsw= +github.com/0chain/gosdk v1.18.14/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= diff --git a/internal/api/model/api.go b/internal/api/model/api.go index eeafe345b9..84afbd7f49 100644 --- a/internal/api/model/api.go +++ b/internal/api/model/api.go @@ -176,6 +176,13 @@ func NewRegisterBlobberTransactionData(scRegisterBlobberRequest *StorageNode) Tr } } +func NewKillBlobberTransactionData(killBlobberRequest *KillBlobberRequest) TransactionData { + return TransactionData{ + Name: "kill_blobber", + Input: *killBlobberRequest, + } +} + func NewCreateFreeAllocationTransactionData(scRestGetFreeAllocationBlobbersResponse *SCRestGetFreeAllocationBlobbersResponse) TransactionData { return TransactionData{ Name: "free_allocation_request", @@ -507,6 +514,7 @@ type BlobberRequirements struct { ExpirationDate int64 `json:"expiration_date"` ReadPriceRange PriceRange `json:"read_price_range"` WritePriceRange PriceRange `json:"write_price_range"` + StorageVersion int64 `json:"storage_version"` } type PriceRange struct { @@ -630,6 +638,10 @@ type SCRestGetBlobberResponse struct { ReadData int64 `json:"read_data"` ChallengesPassed int64 `json:"challenges_passed"` ChallengesCompleted int64 `json:"challenges_completed"` + + //todo: + StorageVersion int64 `json:"storage_version"` + ManagingWallet string `json:"managing_wallet"` } type SCRestGetBlobbersResponse struct { @@ -670,6 +682,10 @@ type FreeAllocationRequest struct { Marker string `json:"marker,omitempty"` } +type KillBlobberRequest struct { + ProviderID string `json:"provider_id,omitempty"` +} + type SCRestGetAllocationResponse struct { ID string `json:"id"` Tx string `json:"tx"` @@ -712,6 +728,8 @@ type StorageNode struct { LastHealthCheck int64 `json:"last_health_check"` PublicKey string `json:"-"` StakePoolSettings StakePoolSettings `json:"stake_pool_settings"` + StorageVersion int64 `json:"storage_version"` + ManagingWallet string `json:"managing_wallet"` } type StorageNodeGeolocation struct { diff --git a/internal/api/model/helper.go b/internal/api/model/helper.go index 92a7a9f7ca..9450fd8f62 100644 --- a/internal/api/model/helper.go +++ b/internal/api/model/helper.go @@ -18,5 +18,6 @@ func DefaultBlobberRequirements(id, publicKey string) BlobberRequirements { }, OwnerId: id, OwnerPublicKey: publicKey, + StorageVersion: 1, } } diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index 56b7473a9a..7335bd9393 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -914,7 +914,8 @@ func (c *APIClient) RegisterBlobber(t *test.SystemTest, wallet *model.Wallet, storageNode *model.StorageNode, requiredTransactionStatus int, - expectedResponse string) string { + expectedResponse string, + requireIdVerification bool) string { t.Log("Registering blobber...") registerBlobberTransactionPutResponse, resp, err := c.V1TransactionPut( @@ -954,6 +955,19 @@ func (c *APIClient) RegisterBlobber(t *test.SystemTest, return false } + if requireIdVerification { + var storageNode model.StorageNode + + // Unmarshal the JSON string into the StorageNode struct + err := json.Unmarshal([]byte(registerBlobberTransactionGetConfirmationResponse.Transaction.TransactionOutput), &storageNode) + if err != nil { + t.Log("Error unmarshalling JSON:", err) + return false + } + + return registerBlobberTransactionGetConfirmationResponse.Status == requiredTransactionStatus && storageNode.ID == expectedResponse + } + return registerBlobberTransactionGetConfirmationResponse.Status == requiredTransactionStatus && registerBlobberTransactionGetConfirmationResponse.Transaction.TransactionOutput == expectedResponse }) @@ -962,6 +976,58 @@ func (c *APIClient) RegisterBlobber(t *test.SystemTest, return registerBlobberTransactionPutResponse.Entity.Hash } +func (c *APIClient) KillBlobber(t *test.SystemTest, + wallet *model.Wallet, + killBlobberRequest *model.KillBlobberRequest, + requiredTransactionStatus int) string { + t.Log("Killing blobber...") + + killBlobberTransactionPutResponse, resp, err := c.V1TransactionPut( + t, + model.InternalTransactionPutRequest{ + Wallet: wallet, + ToClientID: StorageSmartContractAddress, + TransactionData: model.NewKillBlobberTransactionData(killBlobberRequest), + Value: tokenomics.IntToZCN(0), + TxnType: SCTxType, + }, + HttpOkStatus) + require.Nil(t, err) + require.NotNil(t, resp) + require.NotNil(t, killBlobberTransactionPutResponse) + + var killBlobberTransactionGetConfirmationResponse *model.TransactionGetConfirmationResponse + + wait.PoolImmediately(t, time.Minute*2, func() bool { + killBlobberTransactionGetConfirmationResponse, resp, err = c.V1TransactionGetConfirmation( + t, + model.TransactionGetConfirmationRequest{ + Hash: killBlobberTransactionPutResponse.Entity.Hash, + }, + HttpOkStatus) + + if err != nil { + t.Log("Error killing blobber : ", err) + return false + } + + if resp == nil { + fmt.Println("got nil response : ", resp) + return false + } + + if killBlobberTransactionGetConfirmationResponse == nil { + fmt.Println("got nil txn confirmation response : ", killBlobberTransactionGetConfirmationResponse) + return false + } + + return killBlobberTransactionGetConfirmationResponse.Status == requiredTransactionStatus + }) + + wallet.IncNonce() + return killBlobberTransactionPutResponse.Entity.Hash +} + func (c *APIClient) CreateFreeAllocation(t *test.SystemTest, wallet *model.Wallet, scRestGetFreeAllocationBlobbersResponse *model.SCRestGetFreeAllocationBlobbersResponse, diff --git a/internal/cli/model/model.go b/internal/cli/model/model.go index bd748ecfd2..738dc8f89c 100644 --- a/internal/cli/model/model.go +++ b/internal/cli/model/model.go @@ -158,12 +158,6 @@ type Terms struct { WritePrice int64 `json:"write_price"` } -type Settings struct { - Delegate_wallet string `json:"delegate_wallet"` - Num_delegates int `json:"num_delegates"` - Service_charge float64 `json:"service_charge"` -} - type BlobberInfo struct { Id string `json:"id"` Url string `json:"url"` @@ -380,6 +374,7 @@ type BlobberDetails struct { IsShutdown bool `json:"is_shutdown"` IsRestricted bool `json:"is_restricted"` NotAvailable bool `json:"not_available"` + StorageVersion int64 `json:"storage_version"` } type Validator struct { diff --git a/tests/api_tests/config/sc_owner_wallet.json b/tests/api_tests/config/sc_owner_wallet.json new file mode 100644 index 0000000000..2a9afb47bd --- /dev/null +++ b/tests/api_tests/config/sc_owner_wallet.json @@ -0,0 +1,13 @@ +{ + "client_id": "1746b06bb09f55ee01b33b5e2e055d6cc7a900cb57c0a3a5eaabb8a0e7745802", + "client_key": "7b630ba670dac2f22d43c2399b70eff378689a53ee03ea20957bb7e73df016200fea410ba5102558b0c39617e5afd2c1843b161a1dedec15e1ab40543a78a518", + "keys": [ + { + "public_key": "7b630ba670dac2f22d43c2399b70eff378689a53ee03ea20957bb7e73df016200fea410ba5102558b0c39617e5afd2c1843b161a1dedec15e1ab40543a78a518", + "private_key": "c06b6f6945ba02d5a3be86b8779deca63bb636ce7e46804a479c50e53c864915" + } + ], + "mnemonics": "cactus panther essence ability copper fox wise actual need cousin boat uncover ride diamond group jacket anchor current float rely tragic omit child payment", + "version": "1.0", + "date_created": "2023-11-26T01:23:58Z" +} \ No newline at end of file diff --git a/tests/api_tests/main_test.go b/tests/api_tests/main_test.go index d4b58c278e..055d18b9bb 100644 --- a/tests/api_tests/main_test.go +++ b/tests/api_tests/main_test.go @@ -130,6 +130,45 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +func initialiseSCWallet() *model.Wallet { + // read the file sc_owner_wallet.json + fileContent, err := os.ReadFile("./config/sc_owner_wallet.json") + if err != nil { + log.Println("Error reading file:", err) + return nil + } + + fileWallet := WalletFile{} + + // Parse the JSON data into a list of strings + err = json.Unmarshal(fileContent, &fileWallet) + + if err != nil { + log.Println("Error decoding JSON:", err) + return nil + } + + wallet := &model.Wallet{ + Id: fileWallet.ClientId, + Version: fileWallet.Version, + PublicKey: fileWallet.Keys[0].PublicKey, + Nonce: 0, + Keys: &model.KeyPair{}, + Mnemonics: fileWallet.Mnemonics, + } + + err = wallet.Keys.PublicKey.DeserializeHexStr(fileWallet.Keys[0].PublicKey) + if err != nil { + log.Println("Error decoding JSON:", err) + } + err = wallet.Keys.PrivateKey.DeserializeHexStr(fileWallet.Keys[0].PrivateKey) + if err != nil { + log.Println("Error decoding JSON:", err) + } + + return wallet +} + type WalletFile struct { ClientId string `json:"client_id"` ClientKey string `json:"client_key"` diff --git a/tests/api_tests/register_blobber_test.go b/tests/api_tests/register_blobber_test.go index 548b4306d1..fdcab759af 100644 --- a/tests/api_tests/register_blobber_test.go +++ b/tests/api_tests/register_blobber_test.go @@ -17,6 +17,36 @@ func TestRegisterBlobber(testSetup *testing.T) { t := test.NewSystemTest(testSetup) t.Parallel() + // write a test case to register a blobber with storage version + t.Run("Register blobber with storage version", func(t *test.SystemTest) { + wallet := createWallet(t) + + defer killBlobber(t, wallet.Id) + + walletBalance := apiClient.GetWalletBalance(t, wallet, client.HttpOkStatus) + t.Logf("wallet balance: %v", wallet) + wallet.Nonce = int(walletBalance.Nonce) + + sn := &model.StorageNode{} + + sn.ID = uuid.New().String() + sn.BaseURL = generateRandomURL() + + sn.Capacity = 10240 * GB + sn.Terms.ReadPrice = 1000000000 + sn.Terms.WritePrice = 1000000000 + + sn.StakePoolSettings.DelegateWallet = "config.Configuration.DelegateWallet" + sn.StakePoolSettings.NumDelegates = 2 + sn.StakePoolSettings.ServiceCharge = 0.2 + + //todo: make check to this + sn.StorageVersion = 2 + sn.ManagingWallet = wallet.Id + + apiClient.RegisterBlobber(t, wallet, sn, 1, wallet.Id, true) + }) + t.Run("Write price lower than min_write_price should not allow register", func(t *test.SystemTest) { wallet := createWallet(t) @@ -36,7 +66,7 @@ func TestRegisterBlobber(testSetup *testing.T) { sn.StakePoolSettings.NumDelegates = 2 sn.StakePoolSettings.ServiceCharge = 0.2 - apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: write_price is less than min_write_price allowed") + apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: write_price is less than min_write_price allowed", false) }) t.Run("Write price higher than max_write_price should not allow register", func(t *test.SystemTest) { @@ -58,7 +88,7 @@ func TestRegisterBlobber(testSetup *testing.T) { sn.StakePoolSettings.NumDelegates = 2 sn.StakePoolSettings.ServiceCharge = 0.2 - apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: write_price is greater than max_write_price allowed") + apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: write_price is greater than max_write_price allowed", false) }) t.Run("Read price higher than max_read_price should not allow register", func(t *test.SystemTest) { @@ -80,7 +110,7 @@ func TestRegisterBlobber(testSetup *testing.T) { sn.StakePoolSettings.NumDelegates = 2 sn.StakePoolSettings.ServiceCharge = 0.2 - apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: read_price is greater than max_read_price allowed") + apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: read_price is greater than max_read_price allowed", false) }) t.Run("Service charge higher than max_service_charge should not allow register", func(t *test.SystemTest) { @@ -102,7 +132,7 @@ func TestRegisterBlobber(testSetup *testing.T) { sn.StakePoolSettings.NumDelegates = 2 sn.StakePoolSettings.ServiceCharge = 0.6 - apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: creating stake pool: invalid stake_pool settings: service_charge (0.600000) is greater than max allowed by SC (0.500000)") + apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: creating stake pool: invalid stake_pool settings: service_charge (0.600000) is greater than max allowed by SC (0.500000)", false) }) t.Run("Capacity lower than min_blobber_capacity should not allow register", func(t *test.SystemTest) { @@ -124,7 +154,7 @@ func TestRegisterBlobber(testSetup *testing.T) { sn.StakePoolSettings.NumDelegates = 2 sn.StakePoolSettings.ServiceCharge = 0.2 - apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: insufficient blobber capacity") + apiClient.RegisterBlobber(t, wallet, sn, 2, "add_or_update_blobber_failed: invalid blobber params: insufficient blobber capacity", false) }) } @@ -145,3 +175,17 @@ func generateRandomString(length int) string { func generateRandomURL() string { return fmt.Sprintf("http://%s.com/%s", generateRandomString(10), generateRandomString(8)) } + +func killBlobber(t *test.SystemTest, providerId string) { + var killBlobberReq = &model.KillBlobberRequest{ + ProviderID: providerId, + } + + scWallet := initialiseSCWallet() + + // get wallet balance + walletBalance := apiClient.GetWalletBalance(t, scWallet, client.HttpOkStatus) + scWallet.Nonce = int(walletBalance.Nonce) + + apiClient.KillBlobber(t, scWallet, killBlobberReq, 1) +} diff --git a/tests/api_tests/update_blobber_test.go b/tests/api_tests/update_blobber_test.go index c22b545364..617aefd281 100644 --- a/tests/api_tests/update_blobber_test.go +++ b/tests/api_tests/update_blobber_test.go @@ -17,6 +17,57 @@ func TestUpdateBlobber(testSetup *testing.T) { t.Parallel() + t.Run("update blobber version should work", func(t *test.SystemTest) { + wallet := createWallet(t) + + blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey) + allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus) + allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus) + + allocation := apiClient.GetAllocation(t, allocationID, client.HttpOkStatus) + + blobberID := getFirstUsedStorageNodeID(allocationBlobbers.Blobbers, allocation.Blobbers) + require.NotZero(t, blobberID) + + blobber := apiClient.GetBlobber(t, blobberID, client.HttpOkStatus) + require.NotEqual(t, wallet.Id, blobber.StakePoolSettings.DelegateWallet) + + blobber.StorageVersion = 1 + + apiClient.UpdateBlobber(t, wallet, blobber, client.TxUnsuccessfulStatus) + + blobber = apiClient.GetBlobber(t, blobberID, client.HttpOkStatus) + require.NotEqual(t, wallet.Id, blobber.StakePoolSettings.DelegateWallet) + + require.Equal(t, int64(1), blobber.StorageVersion) + }) + + t.Run("update blobber: degrade version should not work", func(t *test.SystemTest) { + wallet := createWallet(t) + + blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey) + allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus) + + allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus) + + allocation := apiClient.GetAllocation(t, allocationID, client.HttpOkStatus) + + blobberID := getFirstUsedStorageNodeID(allocationBlobbers.Blobbers, allocation.Blobbers) + require.NotZero(t, blobberID) + + blobber := apiClient.GetBlobber(t, blobberID, client.HttpOkStatus) + require.NotEqual(t, wallet.Id, blobber.StakePoolSettings.DelegateWallet) + + blobber.StorageVersion = 0 + + apiClient.UpdateBlobber(t, wallet, blobber, client.TxUnsuccessfulStatus) + + blobber = apiClient.GetBlobber(t, blobberID, client.HttpOkStatus) + require.NotEqual(t, wallet.Id, blobber.StakePoolSettings.DelegateWallet) + + require.Equal(t, int64(1), blobber.StorageVersion) + }) + t.Run("Update blobber in allocation without correct delegated client, shouldn't work", func(t *test.SystemTest) { wallet := createWallet(t) diff --git a/tests/cli_tests/0_expired_allocation_test.go b/tests/cli_tests/0_expired_allocation_test.go index 596690ccc7..48a939e79a 100644 --- a/tests/cli_tests/0_expired_allocation_test.go +++ b/tests/cli_tests/0_expired_allocation_test.go @@ -182,9 +182,13 @@ func TestExpiredAllocation(testSetup *testing.T) { cliutils.Wait(t, time.Minute*1) output, err = finalizeAllocation(t, configPath, allocationID, true) - require.Nil(t, err, "unexpected error updating allocation", strings.Join(output, "\n")) - require.True(t, len(output) > 0, "expected output length be at least 1", strings.Join(output, "\n")) - require.Regexp(t, regexp.MustCompile("Allocation finalized with txId .*$"), output[0]) + if err != nil { + require.Contains(t, err.Error(), "already finalized", "unexpected error: %s", err.Error()) + } else { + require.Nil(t, err, "unexpected error updating allocation", strings.Join(output, "\n")) + require.True(t, len(output) > 0, "expected output length be at least 1", strings.Join(output, "\n")) + require.Regexp(t, regexp.MustCompile("Allocation finalized with txId .*$"), output[0]) + } // get balance after unlock balanceAfterFinalize, err := getBalanceZCN(t, configPath) diff --git a/tests/cli_tests/config/config.yaml b/tests/cli_tests/config/config.yaml index 3d36109c59..a00e661cf2 100644 --- a/tests/cli_tests/config/config.yaml +++ b/tests/cli_tests/config/config.yaml @@ -1,4 +1,4 @@ -block_worker: https://dev.zus.network/dns +block_worker: https://dev-st.devnet-0chain.net/dns confirmation_chain_length: 3 ethereum_node_url: "https://rpc.tenderly.co/fork/5b7ffac9-50cc-4169-b0ca-6fd203d26ef6" min_confirmation: 50 diff --git a/tests/cli_tests/config/zbox_config.yaml b/tests/cli_tests/config/zbox_config.yaml index e93c92478a..a00e661cf2 100644 --- a/tests/cli_tests/config/zbox_config.yaml +++ b/tests/cli_tests/config/zbox_config.yaml @@ -1,6 +1,6 @@ block_worker: https://dev-st.devnet-0chain.net/dns confirmation_chain_length: 3 -ethereum_node_url: "https://virtual.mainnet.rpc.tenderly.co/1b2b3323-9a80-49c3-a2ff-50efb7afb359" +ethereum_node_url: "https://rpc.tenderly.co/fork/5b7ffac9-50cc-4169-b0ca-6fd203d26ef6" min_confirmation: 50 min_submit: 50 signature_scheme: bls0chain diff --git a/tests/cli_tests/zboxcli_blobber_config_update_test.go b/tests/cli_tests/zboxcli_blobber_config_update_test.go index 413bd09efd..d54dab551d 100644 --- a/tests/cli_tests/zboxcli_blobber_config_update_test.go +++ b/tests/cli_tests/zboxcli_blobber_config_update_test.go @@ -64,6 +64,57 @@ func TestBlobberConfigUpdate(testSetup *testing.T) { require.Nil(t, err, strings.Join(output, "\n")) }) + // update blobber: managing wallet should be able to udpate delegate wallet + t.RunSequentially("update blobber managing wallet should be able to update delegate wallet", func(t *test.SystemTest) { + createWallet(t) + + fmt.Println("delegate wallet: ", intialBlobberInfo.StakePoolSettings.DelegateWallet) + // create a delegate wallet + createWalletForName(escapedTestName(t) + "_delegate") + delegateWallet, err := getWalletForName(t, configPath, escapedTestName(t)+"_delegate") + require.Nil(t, err, "error occurred when getting delegate wallet") + + output, err := updateBlobberInfo(t, configPath, createParams(map[string]interface{}{ + "blobber_id": intialBlobberInfo.ID, + "delegate_wallet": delegateWallet.ClientID, + })) + + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) + + updatedOutput, err := getBlobberInfo(t, configPath, createParams(map[string]interface{}{"json": "", "blobber_id": intialBlobberInfo.ID})) + require.Nil(t, err, strings.Join(updatedOutput, "\n")) + require.Len(t, updatedOutput, 1) + + var finalBlobberInfo climodel.BlobberDetails + err = json.Unmarshal([]byte(updatedOutput[0]), &finalBlobberInfo) + + require.Nil(t, err, strings.Join(updatedOutput, "\n")) + + fmt.Println("upadted output : \n\n", finalBlobberInfo) + + require.Equal(t, delegateWallet.ClientID, finalBlobberInfo.StakePoolSettings.DelegateWallet) + + // revert back the delegate wallet id to original one + t.Cleanup(func() { + output, err := updateBlobberInfo(t, configPath, createParams(map[string]interface{}{ + "blobber_id": intialBlobberInfo.ID, + "delegate_wallet": intialBlobberInfo.StakePoolSettings.DelegateWallet, + })) + require.Nil(t, err, strings.Join(output, "\n")) + + output, err = getBlobberInfo(t, configPath, createParams(map[string]interface{}{"json": "", "blobber_id": intialBlobberInfo.ID})) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) + + var finalBlobberInfo climodel.BlobberDetails + err = json.Unmarshal([]byte(output[0]), &finalBlobberInfo) + require.Nil(t, err, strings.Join(output, "\n")) + + require.Equal(t, intialBlobberInfo.StakePoolSettings.DelegateWallet, finalBlobberInfo.StakePoolSettings.DelegateWallet) + }) + }) + t.RunSequentially("update blobber capacity should work", func(t *test.SystemTest) { // create wallet for normal user createWallet(t) @@ -253,6 +304,7 @@ func TestBlobberConfigUpdate(testSetup *testing.T) { require.Equal(t, newNotAvailable, finalBlobberInfo.NotAvailable) require.Equal(t, url, finalBlobberInfo.BaseURL) }) + t.RunSequentially("update base_url should work", func(t *test.SystemTest) { createWallet(t) diff --git a/tests/cli_tests/zboxcli_create_allocation_test.go b/tests/cli_tests/zboxcli_create_allocation_test.go index a87ca59ea2..f83dd0912f 100644 --- a/tests/cli_tests/zboxcli_create_allocation_test.go +++ b/tests/cli_tests/zboxcli_create_allocation_test.go @@ -21,6 +21,30 @@ func TestCreateAllocation(testSetup *testing.T) { t.Parallel() + // write a test case to create the allocation with storage version + t.Run("Create allocation with storage version", func(t *test.SystemTest) { + _ = setupWallet(t, configPath) + + options := map[string]interface{}{ + "lock": "0.5", + "size": "1024", + "read_price": "0-1", + "write_price": "0-1", + "storage_version": 1, + } + + output, err := createNewAllocation(t, configPath, createParams(options)) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) + + require.Regexp(t, regexp.MustCompile("^Allocation created: [0-9a-fA-F]{64}$"), output[0], strings.Join(output, "\n")) + + allocationID, err := getAllocationID(output[0]) + require.Nil(t, err, "could not get allocation ID", strings.Join(output, "\n")) + + createAllocationTestTeardown(t, allocationID) + }) + t.Run("Create allocation for locking cost equal to the cost calculated should work", func(t *test.SystemTest) { _ = setupWallet(t, configPath) @@ -453,6 +477,7 @@ func createNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, } func createNewAllocationWithoutRetry(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { + t.Logf("params values %v...", params) return cliutils.RunCommandWithoutRetry(fmt.Sprintf( "./zbox newallocation %s --silent --wallet %s --configDir ./config --config %s --allocationFileName %s", params, From 464b11e64aee2ff0e27b60448ffa3d777055ae94 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 14 Mar 2025 23:28:38 +0545 Subject: [PATCH 15/19] updated for azure --- internal/api/util/config/config.go | 1 + tests/cli_tests/config/cli_tests_config.yaml | 1 + .../s3migration_tests/azure_tests/1_general_test.go | 9 ++++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index a90ba001ba..0fcaca1128 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -43,6 +43,7 @@ type Config struct { ConnectionString string `yaml:"connectionString"` AccountName string `yaml:"accountName"` ContainerName string `yaml:"container"` + AzureAccessKey string `yaml:"azureAccessKey"` GoogleCloudAccessToken string `yaml:"googleCloudAccessToken"` GoogleCloudRefreshToken string `yaml:"googleCloudRefreshToken"` GoogleClientId string `yaml:"googleClientId"` diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 71fb6c448d..256b19804a 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -10,6 +10,7 @@ gdriveRefreshToken: connectionString: accountName: container: +azureAccessKey: googleCloudAccessToken: googleCloudRefreshToken: googleClientId: diff --git a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go index 55e5a0afc2..beed9272df 100644 --- a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go +++ b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go @@ -17,10 +17,13 @@ func Test0MicrosoftAzure(testSetup *testing.T) { t.Skip("Missing Account name for required for migration") } - if shared.ConfigData.ConnectionString == "" { - t.Skip("Missing Connection String required for migration") + if shared.ConfigData.ContainerName == "" { + t.Skip("Missing Container Name required for migration") } + if shared.ConfigData.AzureAccessKey == "" { + t.Skip("Missing Access Key required for migration") + } t.SetSmokeTests("Should migrate existing files and folder from Microsoft Azure Container successfully") t.RunSequentially("Should migrate existing Microsoft Azure folder and files successfully", func(t *test.SystemTest) { @@ -31,7 +34,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, - "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "access-key": shared.ConfigData.AzureAccessKey, "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "azure", From 4cd3a520fb336eb42962664e86be52fb963169c0 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 14 Mar 2025 23:38:19 +0545 Subject: [PATCH 16/19] box format --- .../azure_tests/1_general_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go index beed9272df..e3ff5ef1b0 100644 --- a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go +++ b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go @@ -32,15 +32,15 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ - "account-name": shared.ConfigData.AccountName, - "container": shared.ConfigData.ContainerName, - "access-key": shared.ConfigData.AzureAccessKey, - "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", - "allocation": allocationId, - "source": "azure", - "config": shared.ConfigPath, - "configDir": shared.ConfigDir, - "skip": 0, + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "access-key": shared.ConfigData.AzureAccessKey, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, })) totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) From 62c83c436cb1b62cbdb872607c07273c4b2dda8b Mon Sep 17 00:00:00 2001 From: pewssh Date: Tue, 18 Mar 2025 03:59:02 +0800 Subject: [PATCH 17/19] NTR --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9c15c694e..c9c0c7a4e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,8 +122,8 @@ jobs: svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} deploy_report_page: true archive_results: true - run_flaky_tests: true - run_api_system_tests: true + run_flaky_tests: false + run_api_system_tests: false run_cli_system_tests: true run_tenderly_tests: false run_tokenomics_system_tests: false From 35eb5d414abd1cbc7eefc2b9b1045e1749c704b8 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 8 Apr 2025 01:51:33 +0530 Subject: [PATCH 18/19] dummy --- internal/api/util/client/api_client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index 7335bd9393..71b879d07f 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -964,7 +964,6 @@ func (c *APIClient) RegisterBlobber(t *test.SystemTest, t.Log("Error unmarshalling JSON:", err) return false } - return registerBlobberTransactionGetConfirmationResponse.Status == requiredTransactionStatus && storageNode.ID == expectedResponse } From 58b1712f25dc60cebbf12fffef6c9226b633fbee Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 8 Apr 2025 17:27:12 +0530 Subject: [PATCH 19/19] fix --- tests/cli_tests/0_expired_allocation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_tests/0_expired_allocation_test.go b/tests/cli_tests/0_expired_allocation_test.go index 48a939e79a..cea14ad514 100644 --- a/tests/cli_tests/0_expired_allocation_test.go +++ b/tests/cli_tests/0_expired_allocation_test.go @@ -183,7 +183,7 @@ func TestExpiredAllocation(testSetup *testing.T) { output, err = finalizeAllocation(t, configPath, allocationID, true) if err != nil { - require.Contains(t, err.Error(), "already finalized", "unexpected error: %s", err.Error()) + require.Contains(t, output[0], "already finalized", "unexpected error: %s", err.Error()) } else { require.Nil(t, err, "unexpected error updating allocation", strings.Join(output, "\n")) require.True(t, len(output) > 0, "expected output length be at least 1", strings.Join(output, "\n"))