From 2775175624a210b7de3977f6319203ef6d306192 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Thu, 2 Jan 2025 19:32:27 +0530 Subject: [PATCH 01/27] Empty commit From 7a3b6539a374278fea217af66ae6d31921b0d2ee Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Sun, 5 Jan 2025 01:12:00 +0530 Subject: [PATCH 02/27] Fix restricted blobber test --- tests/cli_tests/zboxcli_restricted_blobber_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli_tests/zboxcli_restricted_blobber_test.go b/tests/cli_tests/zboxcli_restricted_blobber_test.go index 348b519cbc..5ce518eefe 100644 --- a/tests/cli_tests/zboxcli_restricted_blobber_test.go +++ b/tests/cli_tests/zboxcli_restricted_blobber_test.go @@ -3,7 +3,6 @@ package cli_tests import ( "encoding/hex" "encoding/json" - "github.com/0chain/errors" "io/ioutil" "net/http" "os" @@ -14,6 +13,8 @@ import ( "testing" "time" + "github.com/0chain/errors" + "github.com/0chain/common/core/common" "github.com/0chain/gosdk/core/zcncrypto" From 37cea1c69b162aea0283c400cff075751fe17e4f Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Mon, 13 Jan 2025 19:44:48 +0530 Subject: [PATCH 03/27] Fix restricted blobber tests --- tests/cli_tests/zboxcli_restricted_blobber_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_tests/zboxcli_restricted_blobber_test.go b/tests/cli_tests/zboxcli_restricted_blobber_test.go index 5ce518eefe..e3d0ae77fc 100644 --- a/tests/cli_tests/zboxcli_restricted_blobber_test.go +++ b/tests/cli_tests/zboxcli_restricted_blobber_test.go @@ -91,7 +91,7 @@ func TestRestrictedBlobbers(testSetup *testing.T) { // Setup wallet and create allocation _ = setupWallet(t, configPath) - options := map[string]interface{}{"size": "1024", "data": "3", "parity": "3", "lock": "0.5", "force": "true"} + options := map[string]interface{}{"size": "1024", "data": "3", "parity": "3", "lock": "0.5", "force": "true", "auth_round_expiry": 1000000000000000} output, err = createNewAllocationWithoutRetry(t, configPath, createParams(options)) require.NotNil(t, err) require.True(t, len(output) > 0, "expected output length be at least 1", strings.Join(output, "\n")) From 8b56dcd4095d1aa758cc32549164371de7da4dab Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Wed, 15 Jan 2025 23:40:10 +0530 Subject: [PATCH 04/27] Debug allocation tokenomics test --- tests/tokenomics_tests/allocation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tokenomics_tests/allocation_test.go b/tests/tokenomics_tests/allocation_test.go index 6dab1dbc78..c9a82f1a3f 100644 --- a/tests/tokenomics_tests/allocation_test.go +++ b/tests/tokenomics_tests/allocation_test.go @@ -327,7 +327,7 @@ func TestAllocationRewards(testSetup *testing.T) { require.Nil(t, err) // sleep for 5 minutes - time.Sleep(10 * time.Minute) + time.Sleep(11 * time.Minute) alloc = utils.GetAllocation(t, allocationId) require.Greater(t, alloc.MovedToChallenge, movedToChallengePool, "MovedToChallenge should increase") From 060952e94a0fb1d2223fc3016eac4a0f717b2f7d Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Thu, 16 Jan 2025 17:03:11 +0530 Subject: [PATCH 05/27] Add error check in update config defer function --- tests/cli_tests/zwalletcli_update_global_config_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/cli_tests/zwalletcli_update_global_config_test.go b/tests/cli_tests/zwalletcli_update_global_config_test.go index 43e30c7838..0fb2be38df 100644 --- a/tests/cli_tests/zwalletcli_update_global_config_test.go +++ b/tests/cli_tests/zwalletcli_update_global_config_test.go @@ -55,10 +55,12 @@ func TestUpdateGlobalConfig(testSetup *testing.T) { // ensure revert in config is run regardless of test result defer func() { - _, _ = updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ + output, err := updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ "keys": configKey, "values": oldValue, }, true) + + require.Nil(t, err, strings.Join(output, "\n")) }() output, err := updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ @@ -100,10 +102,12 @@ func TestUpdateGlobalConfig(testSetup *testing.T) { // ensure revert in config is run regardless of test result defer func() { - _, _ = updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ + output, err := updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ "keys": configKey1 + "," + configKey2, "values": oldValue1 + "," + oldValue2, }, true) + + require.Nil(t, err, strings.Join(output, "\n")) }() output, err := updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ From c4952b8ba8b67b216708cf61f3dece3df1eb86ad Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Thu, 16 Jan 2025 21:18:19 +0530 Subject: [PATCH 06/27] Fix allocation tokenomics test --- tests/tokenomics_tests/allocation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tokenomics_tests/allocation_test.go b/tests/tokenomics_tests/allocation_test.go index c9a82f1a3f..ac4be3d61c 100644 --- a/tests/tokenomics_tests/allocation_test.go +++ b/tests/tokenomics_tests/allocation_test.go @@ -136,7 +136,7 @@ func TestAllocationRewards(testSetup *testing.T) { require.Nil(t, err, "Error updating allocation", strings.Join(output, "\n")) // sleep for 10 minutes - time.Sleep(10 * time.Minute) + time.Sleep(11 * time.Minute) alloc = utils.GetAllocation(t, allocationId) require.Equal(t, true, alloc.Finalized, "Allocation should be finalized : ", alloc.ExpirationDate) From 145d266ffbe4980e4370f44643f03c1225eda0e5 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Fri, 17 Jan 2025 01:06:31 +0530 Subject: [PATCH 07/27] Fix restricted blobber test --- tests/cli_tests/zboxcli_restricted_blobber_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_tests/zboxcli_restricted_blobber_test.go b/tests/cli_tests/zboxcli_restricted_blobber_test.go index e3d0ae77fc..f9e7e16c28 100644 --- a/tests/cli_tests/zboxcli_restricted_blobber_test.go +++ b/tests/cli_tests/zboxcli_restricted_blobber_test.go @@ -91,7 +91,7 @@ func TestRestrictedBlobbers(testSetup *testing.T) { // Setup wallet and create allocation _ = setupWallet(t, configPath) - options := map[string]interface{}{"size": "1024", "data": "3", "parity": "3", "lock": "0.5", "force": "true", "auth_round_expiry": 1000000000000000} + options := map[string]interface{}{"size": "1024", "data": "3", "parity": "3", "lock": "0.5", "force": "true", "auth_round_expiry": 1000000000} output, err = createNewAllocationWithoutRetry(t, configPath, createParams(options)) require.NotNil(t, err) require.True(t, len(output) > 0, "expected output length be at least 1", strings.Join(output, "\n")) From 5d80b171204034c795587bad406971c197aaa730 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Sat, 18 Jan 2025 01:03:10 +0530 Subject: [PATCH 08/27] Comment miner stats blocks finalized check --- tests/api_tests/get_scstats_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/api_tests/get_scstats_test.go b/tests/api_tests/get_scstats_test.go index fd594eab6b..be0e544a4c 100644 --- a/tests/api_tests/get_scstats_test.go +++ b/tests/api_tests/get_scstats_test.go @@ -16,13 +16,14 @@ func TestGetSCStats(testSetup *testing.T) { t.Parallel() t.Run("Get miner stats call should return successfully", func(t *test.SystemTest) { + // TODO: fix miner stats blocks finalised minerGetStatsResponse, resp, err := apiClient.V1MinerGetStats(t, client.HttpOkStatus) require.Nil(t, err) require.NotNil(t, resp) require.NotNil(t, minerGetStatsResponse) require.NotZero(t, minerGetStatsResponse.BlockFinality) require.NotZero(t, minerGetStatsResponse.LastFinalizedRound) - require.NotZero(t, minerGetStatsResponse.BlocksFinalized) + //require.NotZero(t, minerGetStatsResponse.BlocksFinalized) require.GreaterOrEqual(t, minerGetStatsResponse.StateHealth, int64(-1)) require.NotZero(t, minerGetStatsResponse.CurrentRound) require.GreaterOrEqual(t, minerGetStatsResponse.RoundTimeout, int64(0)) From 5537ac5ba6873ed2556627e3c51db94254cafb7d Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Tue, 21 Jan 2025 01:32:22 +0530 Subject: [PATCH 09/27] Check reversed values in global config update are actual --- tests/cli_tests/zwalletcli_update_global_config_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/cli_tests/zwalletcli_update_global_config_test.go b/tests/cli_tests/zwalletcli_update_global_config_test.go index 0fb2be38df..4e36fc66a7 100644 --- a/tests/cli_tests/zwalletcli_update_global_config_test.go +++ b/tests/cli_tests/zwalletcli_update_global_config_test.go @@ -61,6 +61,9 @@ func TestUpdateGlobalConfig(testSetup *testing.T) { }, true) require.Nil(t, err, strings.Join(output, "\n")) + + config := getGlobalConfiguration(t, true) + require.Equal(t, oldValue, config[configKey], "old value %s for config was not set", oldValue, configKey) }() output, err := updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ @@ -108,6 +111,10 @@ func TestUpdateGlobalConfig(testSetup *testing.T) { }, true) require.Nil(t, err, strings.Join(output, "\n")) + + config := getGlobalConfiguration(t, true) + require.Equal(t, oldValue1, config[configKey1], "old value %s for config was not set", oldValue1, configKey1) + require.Equal(t, oldValue2, config[configKey2], "old value %s for config was not set", oldValue2, configKey2) }() output, err := updateGlobalConfigWithWallet(t, scOwnerWallet, map[string]interface{}{ From 431b84bcd35cda5ab4e27e8237ed0169b7052ab6 Mon Sep 17 00:00:00 2001 From: Jayash Satolia Date: Tue, 21 Jan 2025 01:46:18 +0530 Subject: [PATCH 10/27] Remove read rewards tests --- .github/workflows/tokenomics_ci.yml | 50 ---- tests/tokenomics_tests/allocation_test.go | 4 + tests/tokenomics_tests/blobber_read_test.go | 258 -------------------- 3 files changed, 4 insertions(+), 308 deletions(-) delete mode 100644 tests/tokenomics_tests/blobber_read_test.go diff --git a/.github/workflows/tokenomics_ci.yml b/.github/workflows/tokenomics_ci.yml index 914350a470..efa95ddfc4 100644 --- a/.github/workflows/tokenomics_ci.yml +++ b/.github/workflows/tokenomics_ci.yml @@ -169,24 +169,6 @@ jobs: graphnode_ethereum_node_url: https://rpc.tenderly.co/fork/"" svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} - - name: "Run Read Rewards System tests" - uses: 0chain/actions/run-system-tests-tokenomics@master - with: - repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}" - network: ${{ env.NETWORK_URL }} - svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} - deploy_report_page: true - archive_results: true - run_flaky_tests: false - run_api_system_tests: false - run_cli_system_tests: false - run_tokenomics_system_tests: true - tokenomics_test_filter: ${{ env.TOKENOMICS_READ_REWARD_TESTS }} - TENDERLY_VIRTUAL_TESTNET_ID: "" - run_smoke_tests: ${{ inputs.run_smoke_tests }} - S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - - name: "Run Slash Rewards System tests" uses: 0chain/actions/run-system-tests-tokenomics@master with: @@ -237,38 +219,6 @@ jobs: S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} -# - name: "Deploy 0Chain" -# if: github.event_name == 'push' || github.event.inputs.existing_network == '' -# uses: 0chain/actions/deploy-0chain-fileops-limits@master -# with: -# repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}" -# kube_config: ${{ secrets[format('DEV{0}KC', env.RUNNER_NUMBER)] }} -# teardown_condition: "TESTS_PASSED" -# SUBGRAPH_API_URL: ${{ secrets.SUBGRAPH_API_URL }} -# TENDERLY_VIRTUAL_TESTNET_ID: "" -# graphnode_sc: ${{ secrets.GRAPHNODE_SC }} -# graphnode_network: ${{ secrets.GRAPHNODE_NETWORK }} -# graphnode_ethereum_node_url: https://rpc.tenderly.co/fork/"" -# svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} -# -# - name: "Run Client throttling tests " -# uses: 0chain/actions/run-system-tests-tokenomics@master -# with: -# repo_snapshots_branch: "${{ env.REPO_SNAPSHOTS_BRANCH }}" -# network: ${{ env.NETWORK_URL }} -# svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} -# deploy_report_page: true -# archive_results: true -# run_flaky_tests: false -# run_api_system_tests: false -# run_cli_system_tests: false -# run_tokenomics_system_tests: true -# tokenomics_test_filter: ${{ env.CLIENT_THROTTLING }} -# TENDERLY_VIRTUAL_TESTNET_ID: "" -# run_smoke_tests: ${{ inputs.run_smoke_tests }} -# S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} -# S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} - notify_slack_on_failure: diff --git a/tests/tokenomics_tests/allocation_test.go b/tests/tokenomics_tests/allocation_test.go index ac4be3d61c..c49f6703d1 100644 --- a/tests/tokenomics_tests/allocation_test.go +++ b/tests/tokenomics_tests/allocation_test.go @@ -855,3 +855,7 @@ func unstakeTokensForBlobbersAndValidatorsForWallet(t *test.SystemTest, blobbers } } } + +func sizeInGB(size int64) float64 { + return float64(size) / float64(GB) +} diff --git a/tests/tokenomics_tests/blobber_read_test.go b/tests/tokenomics_tests/blobber_read_test.go deleted file mode 100644 index 1046743c3d..0000000000 --- a/tests/tokenomics_tests/blobber_read_test.go +++ /dev/null @@ -1,258 +0,0 @@ -package tokenomics_tests - -import ( - "encoding/json" - "fmt" - "io" - "math" - "net/http" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/0chain/system_test/internal/api/util/test" - climodel "github.com/0chain/system_test/internal/cli/model" - "github.com/0chain/system_test/tests/tokenomics_tests/utils" - "github.com/stretchr/testify/require" -) - -func TestBlobberReadReward(testSetup *testing.T) { - t := test.NewSystemTest(testSetup) - t.Skip() - - t.TestSetup("set storage config to use time_unit as 5 minutes", func() { - output, err := utils.UpdateStorageSCConfig(t, scOwnerWallet, map[string]string{ - "time_unit": "10m", - }, true) - require.Nil(t, err, strings.Join(output, "\n")) - }) - - t.Cleanup(func() { - output, err := utils.UpdateStorageSCConfig(t, scOwnerWallet, map[string]string{ - "time_unit": "1h", - }, true) - require.Nil(t, err, strings.Join(output, "\n")) - }) - - output, err := utils.CreateWallet(t, configPath) - require.Nil(t, err, "Error registering wallet", strings.Join(output, "\n")) - - var blobberList []climodel.BlobberInfo - output, err = utils.ListBlobbers(t, configPath, "--json") - require.Nil(t, err, "Error listing blobbers", strings.Join(output, "\n")) - require.Len(t, output, 1) - - err = json.Unmarshal([]byte(output[0]), &blobberList) - require.Nil(t, err, "Error unmarshalling blobber list", strings.Join(output, "\n")) - require.True(t, len(blobberList) > 0, "No blobbers found in blobber list") - - var blobberListString []string - for _, blobber := range blobberList { - blobberListString = append(blobberListString, blobber.Id) - } - - var validatorList []climodel.Validator - output, err = utils.ListValidators(t, configPath, "--json") - require.Nil(t, err, "Error listing validators", strings.Join(output, "\n")) - require.Len(t, output, 1) - - err = json.Unmarshal([]byte(output[0]), &validatorList) - require.Nil(t, err, "Error unmarshalling validator list", strings.Join(output, "\n")) - require.True(t, len(validatorList) > 0, "No validators found in validator list") - - var validatorListString []string - for _, validator := range validatorList { - validatorListString = append(validatorListString, validator.ID) - } - - blobber1 := blobberListString[0] - blobber2 := blobberListString[1] - - stakeTokensToBlobbersAndValidators(t, blobberListString, validatorListString, configPath, []float64{ - 1, 1, 1, 1, - }, 1) - - t.RunSequentiallyWithTimeout("download one time, equal from both blobbers", 30*time.Minute, func(t *test.SystemTest) { - output, err := utils.CreateWallet(t, configPath) - require.Nil(t, err, "Error registering wallet", strings.Join(output, "\n")) - - // 1. Create an allocation with 1 data shard and 1 parity shard. - allocationId := utils.SetupAllocation(t, configPath, map[string]interface{}{ - "size": 500 * MB, - "tokens": 1, - "data": 1, - "parity": 1, - }) - - remotepath := "/dir/" - filesize := 50 * MB - filename := utils.GenerateRandomTestFileName(t) - - err = utils.CreateFileWithSize(filename, int64(filesize)) - require.Nil(t, err) - - output, err = utils.UploadFile(t, configPath, map[string]interface{}{ - "allocation": allocationId, - "remotepath": remotepath + filepath.Base(filename), - "localpath": filename, - }, true) - require.Nil(t, err, "error uploading file", strings.Join(output, "\n")) - - err = os.Remove(filename) - require.Nil(t, err) - - remoteFilepath := remotepath + filepath.Base(filename) - - output, err = utils.DownloadFile(t, configPath, utils.CreateParams(map[string]interface{}{ - "allocation": allocationId, - "remotepath": remoteFilepath, - "localpath": os.TempDir() + string(os.PathSeparator), - }), true) - require.Nil(t, err, "error downloading file", strings.Join(output, "\n")) - - time.Sleep(30 * time.Second) - - downloadCost := sizeInGB(int64(filesize)) * math.Pow10(8) * 2 - - downloadRewards, err := getReadRewards(t, allocationId) - require.Nil(t, err, "error getting read rewards") - - blobber1DownloadRewards := downloadRewards[blobber1].Amount - blobber2DownloadRewards := downloadRewards[blobber2].Amount - blobber1DelegatesDownloadRewards := downloadRewards[blobber1].Total - blobber1DownloadRewards - blobber2DelegatesDownloadRewards := downloadRewards[blobber2].Total - blobber2DownloadRewards - blobber1TotalDownloadRewards := downloadRewards[blobber1].Total - blobber2TotalDownloadRewards := downloadRewards[blobber2].Total - - totalDownloadRewards := blobber1TotalDownloadRewards + blobber2TotalDownloadRewards - - // log all the values - t.Log("downloadCost", downloadCost) - t.Log("blobber1DownloadRewards", blobber1DownloadRewards) - t.Log("blobber2DownloadRewards", blobber2DownloadRewards) - t.Log("blobber1Delegate1DownloadRewards", blobber1DelegatesDownloadRewards) - t.Log("blobber2Delegate1DownloadRewards", blobber2DelegatesDownloadRewards) - t.Log("blobber1TotalDownloadRewards", blobber1TotalDownloadRewards) - t.Log("blobber2TotalDownloadRewards", blobber2TotalDownloadRewards) - t.Log("totalDownloadRewards", totalDownloadRewards) - - require.InEpsilon(t, downloadCost, totalDownloadRewards, 0.05, "Download cost and total download rewards are not equal") - require.InEpsilon(t, blobber1DownloadRewards, blobber2DownloadRewards, 0.05, "Blobber 1 and Blobber 2 download rewards are not equal") - require.InEpsilon(t, blobber1DelegatesDownloadRewards, blobber2DelegatesDownloadRewards, 0.05, "Blobber 1 delegate 1 and Blobber 2 delegate 1 download rewards are not equal") - require.InEpsilon(t, blobber1TotalDownloadRewards, blobber2TotalDownloadRewards, 0.05, "Blobber 1 total download rewards and Blobber 2 total download rewards are not equal") - }) - - t.RunSequentiallyWithTimeout("test download rewards and checking if downloading fails after allocation expiry", 30*time.Minute, func(t *test.SystemTest) { - output, err := utils.CreateWallet(t, configPath) - require.Nil(t, err, "Error registering wallet", strings.Join(output, "\n")) - - // 1. Create an allocation with 1 data shard and 1 parity shard. - allocationId := utils.SetupAllocation(t, configPath, map[string]interface{}{ - "size": 500 * MB, - "tokens": 1, - "data": 1, - "parity": 1, - }) - - remotepath := "/dir/" - filesize := 50 * MB - filename := utils.GenerateRandomTestFileName(t) - - err = utils.CreateFileWithSize(filename, int64(filesize)) - require.Nil(t, err) - - output, err = utils.UploadFile(t, configPath, map[string]interface{}{ - "allocation": allocationId, - "remotepath": remotepath + filepath.Base(filename), - "localpath": filename, - }, true) - require.Nil(t, err, "error uploading file", strings.Join(output, "\n")) - - err = os.Remove(filename) - require.Nil(t, err) - - remoteFilepath := remotepath + filepath.Base(filename) - - output, err = utils.DownloadFile(t, configPath, utils.CreateParams(map[string]interface{}{ - "allocation": allocationId, - "remotepath": remoteFilepath, - "localpath": os.TempDir() + string(os.PathSeparator), - }), true) - require.Nil(t, err, "error downloading file", strings.Join(output, "\n")) - - time.Sleep(30 * time.Second) - - downloadCost := sizeInGB(int64(filesize)) * math.Pow10(8) * 2 - - downloadRewards, err := getReadRewards(t, allocationId) - require.Nil(t, err, "error getting read rewards") - - blobber1DownloadRewards := downloadRewards[blobber1].Amount - blobber2DownloadRewards := downloadRewards[blobber2].Amount - blobber1DelegatesDownloadRewards := downloadRewards[blobber1].Total - blobber1DownloadRewards - blobber2DelegatesDownloadRewards := downloadRewards[blobber2].Total - blobber2DownloadRewards - blobber1TotalDownloadRewards := downloadRewards[blobber1].Total - blobber2TotalDownloadRewards := downloadRewards[blobber2].Total - - totalDownloadRewards := blobber1TotalDownloadRewards + blobber2TotalDownloadRewards - - // log all the values - t.Log("downloadCost", downloadCost) - t.Log("blobber1DownloadRewards", blobber1DownloadRewards) - t.Log("blobber2DownloadRewards", blobber2DownloadRewards) - t.Log("blobber1Delegate1DownloadRewards", blobber1DelegatesDownloadRewards) - t.Log("blobber2Delegate1DownloadRewards", blobber2DelegatesDownloadRewards) - t.Log("blobber1TotalDownloadRewards", blobber1TotalDownloadRewards) - t.Log("blobber2TotalDownloadRewards", blobber2TotalDownloadRewards) - t.Log("totalDownloadRewards", totalDownloadRewards) - - require.InEpsilon(t, downloadCost, totalDownloadRewards, 0.05, "Download cost and total download rewards are not equal") - require.InEpsilon(t, blobber1DownloadRewards, blobber2DownloadRewards, 0.05, "Blobber 1 and Blobber 2 download rewards are not equal") - require.InEpsilon(t, blobber1DelegatesDownloadRewards, blobber2DelegatesDownloadRewards, 0.05, "Blobber 1 delegate 1 and Blobber 2 delegate 1 download rewards are not equal") - require.InEpsilon(t, blobber1TotalDownloadRewards, blobber2TotalDownloadRewards, 0.05, "Blobber 1 total download rewards and Blobber 2 total download rewards are not equal") - - // Sleep for 10 minutes - time.Sleep(10 * time.Minute) - - err = os.Remove(filename) - require.Nil(t, err) - - remoteFilepath = remotepath + filepath.Base(filename) - - output, err = utils.DownloadFile(t, configPath, utils.CreateParams(map[string]interface{}{ - "allocation": allocationId, - "remotepath": remoteFilepath, - "localpath": os.TempDir() + string(os.PathSeparator), - }), true) - require.NotNil(t, err, "File should not be downloaded from expired allocation", strings.Join(output, "\n")) - }) -} - -func getReadRewards(t *test.SystemTest, allocationID string) (map[string]ProviderAllocationRewards, error) { - var result map[string]ProviderAllocationRewards - - StorageScAddress := "6dba10422e368813802877a85039d3985d96760ed844092319743fb3a76712d7" - sharderBaseUrl := utils.GetSharderUrl(t) - url := fmt.Sprintf(sharderBaseUrl + "/v1/screst/" + StorageScAddress + "/read-rewards?allocation_id=" + allocationID) - - t.Log("Allocation challenge rewards url: ", url) - - res, _ := http.Get(url) //nolint:gosec - defer func(Body io.ReadCloser) { - err := Body.Close() - if err != nil { - t.Log("Error closing response body") - } - }(res.Body) - body, _ := io.ReadAll(res.Body) - err := json.Unmarshal(body, &result) - if err != nil { - return nil, err - } - return result, nil -} -func sizeInGB(size int64) float64 { - return float64(size) / float64(GB) -} From 58025b6a6e957680ba75a960a40e4399452b02e7 Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 5 Feb 2025 20:01:49 +0545 Subject: [PATCH 11/27] updated for folder structure --- internal/api/util/config/config.go | 13 +- internal/cli/util/utils.go | 174 +++++++++++++++++- tests/cli_tests/0_dropboxmgrt_migrate_test.go | 164 ----------------- tests/cli_tests/0_gdrivemgrt_migrate_test.go | 144 --------------- tests/cli_tests/main_test.go | 61 ++---- .../s3migration_tests/1_google_drive_test.go | 109 +++++++++++ .../s3migration_tests/2_dropbox_test.go | 131 +++++++++++++ .../s3migration_tests/3_one_drive_test.go | 118 ++++++++++++ .../4_microsoft_azure_test.go | 150 +++++++++++++++ .../5_google_cloud_storage_test.go | 151 +++++++++++++++ .../cli_tests/s3migration_tests/main_test.go | 154 ++++++++++++++++ .../s3migration_tests/shared/main.go | 33 ++++ 12 files changed, 1042 insertions(+), 360 deletions(-) delete mode 100644 tests/cli_tests/0_dropboxmgrt_migrate_test.go delete mode 100644 tests/cli_tests/0_gdrivemgrt_migrate_test.go create mode 100644 tests/cli_tests/s3migration_tests/1_google_drive_test.go create mode 100644 tests/cli_tests/s3migration_tests/2_dropbox_test.go create mode 100644 tests/cli_tests/s3migration_tests/3_one_drive_test.go create mode 100644 tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go create mode 100644 tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go create mode 100644 tests/cli_tests/s3migration_tests/main_test.go create mode 100644 tests/cli_tests/s3migration_tests/shared/main.go diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 9dcaf33178..23e6f25389 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -37,7 +37,18 @@ type Config struct { BlobberOwnerWalletMnemonics string `yaml:"blobber_owner_wallet_mnemonics"` OwnerWalletMnemonics string `yaml:"owner_wallet_mnemonics"` DropboxAccessToken string `yaml:"dropboxAccessToken"` + DropboxRefreshToken string `yaml:"dropboxRefreshToken"` GdriveAccessToken string `yaml:"gdriveAccessToken"` + GdriveRefreshToken string `yaml:"gdriveRefreshToken"` + ConnectionString string `yaml:"connectionString"` + AccountName string `yaml:"accountName"` + ContainerName string `yaml:"container"` + GoogleCloudAccessToken string `yaml:"googleCloudAccessToken"` + GoogleCloudRefreshToken string `yaml:"googleCloudRefreshToken"` + GoogleClientId string `yaml:"googleClientId"` + GoogleClientSecret string `yaml:"googleClientSecret"` + OneDriveAccessToken string `yaml:"oneDriveAccessToken"` + OneDriveRefreshToken string `yaml:"oneDriveRefreshToken"` } func Parse(configPath string) *Config { @@ -91,4 +102,4 @@ func CreateFreeStorageMarker( require.Nil(t, err, "Could not marshal marker") return string(markerJson) -} +} \ No newline at end of file diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 56041fdfbe..2de6334c56 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -3,6 +3,8 @@ package cliutils import ( "bufio" "crypto/rand" + "encoding/json" + "errors" "fmt" "io" "log" @@ -10,13 +12,16 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "regexp" "strconv" "strings" + "sync" "testing" "time" "github.com/0chain/system_test/internal/api/util/test" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" "github.com/0chain/system_test/internal/cli/util/specific" @@ -26,6 +31,22 @@ import ( var Logger = getLogger() +var ( + WalletMutex sync.Mutex + Wallets []json.RawMessage + WalletIdx int64 +) + +func GetWallets() []json.RawMessage { + return Wallets +} + +func SetWallets(wallets []json.RawMessage) { + WalletMutex.Lock() + defer WalletMutex.Unlock() + Wallets = wallets +} + type Configuration struct { Server string HostPort string @@ -394,7 +415,7 @@ func LogOutput(stdout io.Reader, t *test.SystemTest) { } } -func GetAllocationID(path string) string { +func GetAllocationID(path string) (string, error) { file, err := os.Open(path) if err != nil { log.Printf("Error opening allocation.txt file: %v", err) @@ -404,7 +425,7 @@ func GetAllocationID(path string) string { scanner := bufio.NewScanner(file) scanner.Scan() allocationID := scanner.Text() - return allocationID + return allocationID, nil } func ReadFileMC(testSetup *testing.T) McConfiguration { @@ -441,3 +462,152 @@ func ReadFileMC(testSetup *testing.T) McConfiguration { config.Concurrent = strconv.FormatInt(int64(concurrent), 10) return config } + +func MigrateFromS3migration(t *test.SystemTest, params string) ([]string, error) { + commandGenerated := fmt.Sprintf("../s3migration migrate %s", params) + t.Log(commandGenerated) + return RunCommand(t, commandGenerated, 1, time.Hour*2) +} + +func EscapedTestName(t *test.SystemTest) string { + replacer := strings.NewReplacer("/", "-", "\"", "-", ":", "-", "(", "-", + ")", "-", "<", "LESS_THAN", ">", "GREATER_THAN", "|", "-", "*", "-", + "?", "-") + return replacer.Replace(t.Name()) +} + +func CreateParams(params map[string]interface{}) string { + var builder strings.Builder + + for k, v := range params { + if v == nil { + _, _ = builder.WriteString(fmt.Sprintf("--%s ", k)) + } else if reflect.TypeOf(v).String() == "bool" { + _, _ = builder.WriteString(fmt.Sprintf("--%s=%v ", k, v)) + } else { + _, _ = builder.WriteString(fmt.Sprintf("--%s %v ", k, v)) + } + } + return strings.TrimSpace(builder.String()) +} + +func ExecuteFaucetWithTokens(t *test.SystemTest, cliConfigFilename string, tokens float64) ([]string, error) { + return ExecuteFaucetWithTokensForWallet(t, EscapedTestName(t), cliConfigFilename, tokens) +} + +// ExecuteFaucetWithTokensForWallet executes faucet command with given tokens and wallet. +// Tokens greater than or equal to 10 are considered to be 1 token by the system. +func ExecuteFaucetWithTokensForWallet(t *test.SystemTest, wallet, cliConfigFilename string, tokens float64) ([]string, error) { + t.Logf("Executing faucet...") + return RunCommand(t, fmt.Sprintf("./zwallet faucet --methodName "+ + "pour --tokens %f --input {} --silent --wallet %s_wallet.json --configDir ./config --config %s", + tokens, + wallet, + cliConfigFilename, + ), 3, time.Second*5) +} + +func CreateWalletForName(rootPath, name string) { + walletPath := fmt.Sprintf("%s/config/%s_wallet.json", rootPath, name) + + // check if wallet already exists + if _, err := os.Stat(walletPath); err == nil { + return + } + WalletMutex.Lock() + + wallet := Wallets[WalletIdx] + + WalletIdx++ + WalletMutex.Unlock() + + err := os.WriteFile(walletPath, wallet, 0600) + if err != nil { + fmt.Printf("Error writing file %s: %v\n", walletPath, err) + } else { + fmt.Printf("File %s written successfully.\n", walletPath) + } +} + +func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { + return setupAllocationWithWallet(t, EscapedTestName(t), cliConfigFilename, rootPath, extraParams...) +} + +func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, rootPath, params string) ([]string, error) { + t.Log(cliConfigFilename, "configdir path") + return RunCommand(t, fmt.Sprintf( + "%s/zbox newallocation %s --silent --wallet %s --configDir %s --config %s --allocationFileName %s", + rootPath, + params, + wallet+"_wallet.json", + cliConfigFilename, + "config.yaml", + wallet+"_allocation.txt", + ), 3, time.Second*5) +} + +func setupAllocationWithWallet(t *test.SystemTest, walletName, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { + // Then create new allocation + options := map[string]interface{}{"size": "10000000", "lock": "5"} + + // Add additional parameters if available + // Overwrite with new parameters when available + for _, params := range extraParams { + for k, v := range params { + options[k] = v + } + } + // First create a wallet and run faucet command + CreateWalletForName(rootPath, walletName) + + output, err := CreateNewAllocationForWallet(t, walletName, cliConfigFilename, rootPath, CreateParams(options)) + defer func() { + fmt.Printf("err: %v\n", err) + }() + require.NoError(t, err, "create new allocation failed", strings.Join(output, "\n")) + require.Len(t, output, 1) + + // Get the allocation ID and return it + allocationID, err := getAllocationID(output[0]) + require.Nil(t, err, "could not get allocation ID", strings.Join(output, "\n")) + + return allocationID +} + +var ( + createAllocationRegex = regexp.MustCompile(`^Allocation created: (.+)$`) +) + +func getAllocationID(str string) (string, error) { + match := createAllocationRegex.FindStringSubmatch(str) + if len(match) < 2 { + return "", errors.New("allocation match not found") + } + return match[1], nil +} + +func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err error) { + pattern := `total count :: (\d+)` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(strings.Join(output, "\n")) + + pattern2 := `Total migrated objects :: (\d+)` + re2 := regexp.MustCompile(pattern2) + match_2 := re2.FindStringSubmatch(strings.Join(output, "\n")) + + if len(match) > 1 && len(match_2) > 1 { + totalCount, err = strconv.Atoi(match[1]) + if err != nil { + return + } + + totalMigrated, err = strconv.Atoi(match_2[1]) + if err != nil { + return + } + + return totalMigrated, totalCount, nil + } + + return 0, 0, errors.New("no match found") +} \ No newline at end of file diff --git a/tests/cli_tests/0_dropboxmgrt_migrate_test.go b/tests/cli_tests/0_dropboxmgrt_migrate_test.go deleted file mode 100644 index 2f357824b5..0000000000 --- a/tests/cli_tests/0_dropboxmgrt_migrate_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package cli_tests - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/0chain/system_test/internal/api/util/test" - cliutils "github.com/0chain/system_test/internal/cli/util" - "github.com/stretchr/testify/require" -) - -func Test0Dropbox(testSetup *testing.T) { - t := test.NewSystemTest(testSetup) - - if dropboxAccessToken == "" { - t.Skip("dropbox Access Token was missing") - } - - t.SetSmokeTests("Should migrate existing Dropbox folder and files successfully") - - t.RunSequentially("Should migrate existing Dropbox folder and files successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationId := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationId, - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "skip": 1, - })) - - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "skip": 1, - })) - - require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "source": "dropbox", - "config": configPath, - "configDir": configDir, - })) - - require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "access-token": "invalid", - "wallet": escapedTestName(t) + "_wallet.json", - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid_access_token/", "Output was not as expected", err) - }) - - t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "wallet": escapedTestName(t) + "_wallet.json", - "source": "dropbox", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Missing Access Token", "Output was not as expected", strings.Join(output, "\n")) - }) - t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ - "wallet": escapedTestName(t) + "_wallet.json", - "source": "invalid", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - "access-token": dropboxAccessToken, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { - allocSize := int64(5 * KB) - func() { - defer func() { - // recover from panic if one occurred - if r := recover(); r != nil { - fmt.Println("Panic occurred:", r) // Log the panic - t.Log("Test passed even though a panic occurred") - // Set the test status to passed - require.Equal(t, "", "") - } - }() - - // Set up allocation with wallet - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - }() - }) -} - -func migrateFromDropbox(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { - t.Logf("Migrating Dropbox to Zus...") - t.Logf(fmt.Sprintf("params %v", params)) - t.Logf(fmt.Sprintf("cli %v", cliConfigFilename)) - t.Logf(fmt.Sprintf("./s3migration migrate %s", params)) - return cliutils.RunCommand(t, fmt.Sprintf("./s3migration migrate %s", params), 1, time.Hour*2) -} diff --git a/tests/cli_tests/0_gdrivemgrt_migrate_test.go b/tests/cli_tests/0_gdrivemgrt_migrate_test.go deleted file mode 100644 index 0a8d7fc4da..0000000000 --- a/tests/cli_tests/0_gdrivemgrt_migrate_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package cli_tests - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/0chain/system_test/internal/api/util/test" - cliutils "github.com/0chain/system_test/internal/cli/util" - "github.com/stretchr/testify/require" -) - -func Test0Gdrive(testSetup *testing.T) { - t := test.NewSystemTest(testSetup) - - if gdriveAccessToken == "" { - t.Skip("Gdrive Access Token was missing") - } - - t.SetSmokeTests("Should migrate existing Gdrive folder and files successfully") - - t.RunSequentially("Should migrate existing Gdrive folder and files successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": gdriveAccessToken, - "allocation": allocationID, - "source": "google_drive", - "wallet": escapedTestName(t) + "_wallet.json", - "config": configPath, - "configDir": configDir, - })) - require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": gdriveAccessToken, - "allocation": allocationID, - "source": "google_drive", - "wallet": escapedTestName(t) + "_wallet.json", - "config": configPath, - "configDir": configDir, - })) - - require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) - require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": gdriveAccessToken, - "allocation": allocationID, - "source": "google_drive", - "wallet": escapedTestName(t) + "_wallet.json", - "config": configPath, - "configDir": configDir, - })) - - require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) - require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, _ := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": dropboxAccessToken, - "wallet": escapedTestName(t) + "_wallet.json", - "source": "google_drive", - "config": configPath, - "configDir": configDir, - })) - - require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) - }) - - t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "access-token": "invalid", - "wallet": escapedTestName(t) + "_wallet.json", - "source": "google_drive", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) - }) - - t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ - "wallet": escapedTestName(t) + "_wallet.json", - "source": "google_drive", - "config": configPath, - "configDir": configDir, - "allocation": allocationID, - })) - - require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) - require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) - require.Contains(t, strings.Join(output, "\n"), "Missing Access Token", "Output was not as expected", strings.Join(output, "\n")) - }) -} - -func migrateFromGdrive(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { - t.Logf("Migrating Gdrive to Zus...") - t.Logf(fmt.Sprintf("params %v", params)) - t.Logf(fmt.Sprintf("cli %v", cliConfigFilename)) - t.Logf(fmt.Sprintf("./s3migration migrate %s", params)) - return cliutils.RunCommand(t, fmt.Sprintf("./s3migration migrate %s", params), 1, time.Hour*2) -} diff --git a/tests/cli_tests/main_test.go b/tests/cli_tests/main_test.go index 431748906e..05f7316b2f 100644 --- a/tests/cli_tests/main_test.go +++ b/tests/cli_tests/main_test.go @@ -20,9 +20,6 @@ import ( cliutils "github.com/0chain/system_test/internal/cli/util" "github.com/spf13/viper" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) @@ -54,12 +51,6 @@ func setupConfig() { parsedConfig := config.Parse(filepath.Join(".", path, "cli_tests_config.yaml")) defaultTestTimeout, err := time.ParseDuration(parsedConfig.DefaultTestCaseTimeout) - s3AccessKey = parsedConfig.S3AccessKey - s3SecretKey = parsedConfig.S3SecretKey - s3bucketName = parsedConfig.S3BucketName - s3BucketNameAlternate = parsedConfig.S3BucketNameAlternate - dropboxAccessToken = parsedConfig.DropboxAccessToken - gdriveAccessToken = parsedConfig.GdriveAccessToken if err != nil { log.Printf("Default test case timeout could not be parsed so has defaulted to [%v]", test.DefaultTestTimeout) @@ -109,8 +100,6 @@ var ( s3bucketName string s3BucketNameAlternate string S3Client *s3.S3 - dropboxAccessToken string - gdriveAccessToken string ) var ( @@ -192,44 +181,18 @@ func TestMain(m *testing.M) { //nolint:gocyclo } } - // Create a session with AWS - sess, err := session.NewSession(&aws.Config{ - Region: aws.String("us-east-2"), // Replace with your desired AWS region - Credentials: credentials.NewStaticCredentials(s3AccessKey, s3SecretKey, ""), - }) - - if err != nil { - log.Fatalln("Failed to create AWS session:", err) - return - } - - // Create a session with Dropbox - sess_dp, err_dp := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials( - dropboxAccessToken, "", ""), - }) - - if err_dp != nil { - log.Fatalln("Failed to create Dropbox session:", err_dp) - } - - sess_gd, err_gd := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials( - gdriveAccessToken, "", ""), - }) - - if err_gd != nil { - log.Fatalln("Failed to create Gdrive session:", err_dp) - } - // Create an S3 client - cloudService := os.Getenv("CLOUD_SERVICE") - - if cloudService == "dropbox" { - S3Client = s3.New(sess_dp) - } else if cloudService == "gdrive" { - S3Client = s3.New(sess_gd) - } else { - S3Client = s3.New(sess) + if tenderlyEnabled != "" { + err := tenderlyClient.InitBalance(ethereumAddress) + if err != nil { + cliutils.Logger.Error(err.Error()) + } else { + err = tenderlyClient.InitErc20Balance(tokenAddress, ethereumAddress) + if err != nil { + cliutils.Logger.Error(err.Error()) + } else { + tenderlyInitialized = true + } + } } walletMutex.Lock() diff --git a/tests/cli_tests/s3migration_tests/1_google_drive_test.go b/tests/cli_tests/s3migration_tests/1_google_drive_test.go new file mode 100644 index 0000000000..8d50dfbe0c --- /dev/null +++ b/tests/cli_tests/s3migration_tests/1_google_drive_test.go @@ -0,0 +1,109 @@ +package s3migration_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0Gdrive(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.GdriveAccessToken == "" { + t.Skip("Gdrive Access Token was missing") + } + t.SetSmokeTests("Should migrate existing Gdrive folder and files successfully") + + t.RunSequentially("Should migrate existing Gdrive folder and files successfully", func(t *test.SystemTest) { + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.GdriveAccessToken, + "secret-key": shared.ConfigData.GdriveRefreshToken, + "allocation": allocationID, + "source": "google_drive", + "wallet": EscapedTestName(t) + "_wallet.json", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.GdriveAccessToken, + "secret-key": shared.ConfigData.GdriveRefreshToken, + "allocation": shared.DefaultAllocationId, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.GdriveAccessToken, + "secret-key": shared.ConfigData.GdriveRefreshToken, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "allocation": shared.DefaultAllocationId, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "allocation": shared.DefaultAllocationId, + "source": "google_drive", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + t.Logf("EXpected log %v", strings.Join(output, "\n")) + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: access key, secret key") + }) +} diff --git a/tests/cli_tests/s3migration_tests/2_dropbox_test.go b/tests/cli_tests/s3migration_tests/2_dropbox_test.go new file mode 100644 index 0000000000..81772da7ff --- /dev/null +++ b/tests/cli_tests/s3migration_tests/2_dropbox_test.go @@ -0,0 +1,131 @@ +package s3migration_tests + +import ( + "fmt" + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0Dropbox(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.DropboxAccessToken == "" || shared.ConfigData.DropboxRefreshToken == "" { + t.Skip("Missing Required Tokens for Dropbox migration") + } + + t.SetSmokeTests("Should migrate existing Dropbox folder and files successfully") + + t.RunSequentially("Should migrate existing Dropbox folder and files successfully", func(t *test.SystemTest) { + allocSize := int64(50 * shared.MB) + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": allocSize, + }) + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.DropboxAccessToken, + "secret-key": shared.ConfigData.DropboxRefreshToken, + "allocation": allocationID, + "source": "dropbox", + "wallet": EscapedTestName(t) + "_wallet.json", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.DropboxAccessToken, + "secret-key": shared.ConfigData.DropboxRefreshToken, + "allocation": shared.DefaultAllocationId, + "source": "dropbox", + "wallet": shared.DefaultAllocationId, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.DropboxAccessToken, + "secret-key": shared.ConfigData.DropboxRefreshToken, + "source": "dropbox", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "allocation": shared.DefaultAllocationId, + "source": "dropbox", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid_access_token/", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "allocation": shared.DefaultAllocationId, + "source": "dropbox", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + t.Logf("EXpected log %v", strings.Join(output, "\n")) + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: access key, secret key") + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + allocSize := int64(5 * shared.KB) + var err error + defer func() { + require.Contains(t, err.Error(), "allocation match not found") + }() + _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + "size": allocSize, + }) + }) +} + +func setupAllocationWithWalletWithoutTest(t *test.SystemTest, walletName, cliConfigFilename string, extraParams ...map[string]interface{}) (string, error) { + options := map[string]interface{}{"size": "10000000", "lock": "5"} + + for _, params := range extraParams { + for k, v := range params { + options[k] = v + } + } + cli_utils.CreateWalletForName(shared.RootPath, walletName) + output, _ := cli_utils.CreateNewAllocationForWallet(t, walletName, cliConfigFilename, shared.RootPath, cli_utils.CreateParams(options)) + defer func() { + fmt.Printf("err: %v\n", output) + }() + return cli_utils.GetAllocationID(output[0]) +} diff --git a/tests/cli_tests/s3migration_tests/3_one_drive_test.go b/tests/cli_tests/s3migration_tests/3_one_drive_test.go new file mode 100644 index 0000000000..65d45a559f --- /dev/null +++ b/tests/cli_tests/s3migration_tests/3_one_drive_test.go @@ -0,0 +1,118 @@ +package s3migration_tests + +import ( + "fmt" + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0OneDrive(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.OneDriveAccessToken == "" || shared.ConfigData.OneDriveRefreshToken == "" { + t.Skip("Missing Required details for OneDrive migration") + } + + t.SetSmokeTests("Should migrate existing files and folder from Microsoft OneDrive successfully") + + t.RunSequentially("Should migrate existing Micrsoft OneDrive folder and files successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 1, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "wallet": shared.DefaultWallet, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 1, + })) + + fmt.Printf("Output: %v\n", output) + fmt.Printf("Error: %v\n", err) + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when token and refresh token is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "onedrive", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Access token: InvalidAuthenticationToken", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", + "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", + "source": "invalid", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "skip": 1, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + allocSize := int64(5 * shared.KB) + var err error + defer func() { + require.Contains(t, err.Error(), "allocation match not found") + }() + _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + "size": allocSize, + }) + }) +} diff --git a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go new file mode 100644 index 0000000000..7d47ceb271 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go @@ -0,0 +1,150 @@ +package s3migration_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0MicrosoftAzure(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.AccountName == "" { + t.Skip("Missing Account name for required for migration") + } + + if shared.ConfigData.ConnectionString == "" { + t.Skip("Missing Connection String required for migration") + } + + t.SetSmokeTests("Should migrate existing files and folder from Microsoft Azure Container successfully") + + t.RunSequentially("Should migrate existing Microsoft Azure folder and files successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": shared.DefaultWallet, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when connection string is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "invalid", + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), " connection string is either blank or malformed", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when connection string is missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: connection string", "Output was not as expected", strings.Join(output, "\n")) + }) + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, + "source": "invalid", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + size := int64(5 * shared.KB) + var err error + defer func() { + require.Contains(t, err.Error(), "allocation match not found") + }() + _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + "size": size, + }) + }) +} diff --git a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go new file mode 100644 index 0000000000..6da4697233 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go @@ -0,0 +1,151 @@ +package s3migration_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0GoogleCloudStorage(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if shared.ConfigData.GoogleCloudAccessToken == "" || shared.ConfigData.GoogleCloudRefreshToken == "" || shared.ConfigData.GoogleClientId == "" || shared.ConfigData.GoogleClientSecret == "" { + t.Skip("Missing Required details for google cloud console") + } + + t.SetSmokeTests("Should migrate existing files and folder from Google Cloud successfully") + + t.RunSequentially("Should migrate existing Google Cloud folder and files successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + allocationId := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + _, totalMigrated, err := cli_utils.GetmigratedDataID(output) + + if err != nil { + t.Fatal(err) + } + require.Equal(t, 0, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": shared.DefaultWallet, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when client credentials is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": shared.DefaultWallet, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid client credentials/", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when key is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": "invalid", + "secret-key": "invalid", + "wallet": shared.DefaultWallet, + "source": "google_cloud_storage", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Access key", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + "client-id": shared.ConfigData.GoogleClientId, + "client-secret": shared.ConfigData.GoogleClientSecret, + "access-key": shared.ConfigData.GoogleCloudAccessToken, + "secret-key": shared.ConfigData.GoogleCloudRefreshToken, + "wallet": shared.DefaultWallet, + "source": "invalid", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + "wd": "0chainmigration", + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) +} diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go new file mode 100644 index 0000000000..b3857f6f9d --- /dev/null +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -0,0 +1,154 @@ +package s3migration_tests + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/config" + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/spf13/viper" +) + +func setupDefaultConfig() { + viper.SetDefault("nodes.miner01ID", "73ad5727612116c025bb4405bf3adb4a4a04867ae508c51cf885395bffc8a949") + viper.SetDefault("nodes.miner02ID", "3ec9a42db3355f33c35750ce589ed717c08787997b7f34a7f1f9fb0a03f2b17c") + viper.SetDefault("nodes.miner03ID", "c6f4b8ce5da386b278ba8c4e6cf98b24b32d15bc675b4d12c95e082079c91937") + viper.SetDefault("nodes.sharder01ID", "ea26431f8adb7061766f1d6bbcc3b292d70dd59960d857f04b8a75e6a5bbe04f") + viper.SetDefault("nodes.sharder02ID", "30001a01a888584772b7fee13934021ab8557e0ed471c0a3a454e9164180aef1") +} + +func setupConfig() { + setupDefaultConfig() + + dir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + + shared.ConfigPath = "config.yaml" + path := filepath.Clean(filepath.Join(dir, "config")) + shared.RootPath = dir + + viper.SetConfigName("nodes") + viper.SetConfigType("yaml") + viper.AddConfigPath(path) + + parsedConfig := config.Parse(filepath.Join(path, "cli_tests_config.yaml")) + + shared.SetupConfig(parsedConfig) +} + +func defaultData() { + t := testing.T{} + system_test := test.NewSystemTest(&t) + + defaultAllocationId := cli_utils.SetupAllocation(system_test, shared.ConfigData.ConnectionString, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + + defaultWallet := "default" + cli_utils.CreateWalletForName(shared.RootPath, defaultWallet) + + fmt.Fprintf(os.Stdout, "Default allocation ID: %s\n", defaultAllocationId) + fmt.Fprintf(os.Stdout, "Default wallet: %s\n", defaultWallet) +} + +func EscapedTestName(t *test.SystemTest) string { + replacer := strings.NewReplacer("/", "-", "\"", "-", ":", "-", "(", "-", + ")", "-", "<", "LESS_THAN", ">", "GREATER_THAN", "|", "-", "*", "-", + "?", "-") + return replacer.Replace(t.Name()) +} + +func GetConfigDir() string { + curr, err := os.Getwd() + if err != nil { + log.Fatalln(err) + } + return filepath.Clean(filepath.Join(curr, "config")) +} + +func TestMain(m *testing.M) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-2"), // Replace with your desired AWS region + Credentials: credentials.NewStaticCredentials(shared.ConfigData.S3AccessKey, shared.ConfigData.S3SecretKey, ""), + }) + if err != nil { + log.Fatalln("Failed to create AWS session:", err) + return + } + + sess_dp, err_dp := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(shared.ConfigData.DropboxAccessToken, "", ""), + }) + if err_dp != nil { + log.Fatalln("Failed to create Dropbox session:", err_dp) + } + + sess_gd, err_gd := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(shared.ConfigData.GdriveAccessToken, "", ""), + }) + if err_gd != nil { + log.Fatalln("Failed to create GDrive session:", err_gd) + } + + cloudService := os.Getenv("CLOUD_SERVICE") + + if cloudService == "dropbox" { + shared.S3Client = s3.New(sess_dp) + } else if cloudService == "gdrive" { + shared.S3Client = s3.New(sess_gd) + } else { + shared.S3Client = s3.New(sess) + } + + shared.ConfigPath = os.Getenv("CONFIG_PATH") + shared.ConfigDir = os.Getenv("CONFIG_DIR") + shared.RootPath = os.Getenv("ROOT_PATH") + + if shared.ConfigDir == "" { + shared.ConfigDir = GetConfigDir() + } + + if shared.ConfigPath == "" { + shared.ConfigPath = "./config.yaml" + } + + shared.ConfigDir, _ = filepath.Abs(shared.ConfigDir) + + setupConfig() + + shared.WalletMutex.Lock() + fileContent, err := os.ReadFile("../config/wallets/wallets.json") + if err != nil { + log.Println("Error reading file:", err) + return + } + + err = json.Unmarshal(fileContent, &shared.Wallets) + if err != nil { + log.Println("Error decoding JSON:", err) + return + } + + cliutils.SetWallets(shared.Wallets) + shared.WalletIdx = 500 + shared.WalletMutex.Unlock() + + defaultData() + + exitRun := m.Run() + os.Exit(exitRun) +} diff --git a/tests/cli_tests/s3migration_tests/shared/main.go b/tests/cli_tests/s3migration_tests/shared/main.go new file mode 100644 index 0000000000..7c56851f21 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/shared/main.go @@ -0,0 +1,33 @@ +package shared + +import ( + "encoding/json" + "sync" + + util "github.com/0chain/system_test/internal/api/util/config" + "github.com/aws/aws-sdk-go/service/s3" +) + +var ( + ConfigData util.Config + WalletMutex sync.Mutex + Wallets []json.RawMessage + WalletIdx int64 + S3Client *s3.S3 + ConfigDir string + ConfigPath string + RootPath string + DefaultAllocationId string + DefaultWallet string +) + +const ( + KB = 1024 // kilobyte + MB = 1024 * KB // megabyte + GB = 1024 * MB // gigabyte + AllocSize = int64(50 * MB) +) + +func SetupConfig(parsedConfig *util.Config) { + ConfigData = *parsedConfig +} From d7ec1495c8a6afd4037c5703fcade4e8f120c086 Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 5 Feb 2025 20:04:43 +0545 Subject: [PATCH 12/27] updated for lin --- internal/api/util/config/config.go | 2 +- internal/cli/util/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 23e6f25389..67a8bc8022 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -102,4 +102,4 @@ func CreateFreeStorageMarker( require.Nil(t, err, "Could not marshal marker") return string(markerJson) -} \ No newline at end of file +} diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 2de6334c56..67f1d02c29 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -610,4 +610,4 @@ func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err erro } return 0, 0, errors.New("no match found") -} \ No newline at end of file +} From bbc847bdd78030472c7979137c34b625395943c8 Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 5 Feb 2025 21:41:25 +0545 Subject: [PATCH 13/27] updated for default allocation --- tests/cli_tests/config/cli_tests_config.yaml | 17 ++++++++++++++--- tests/cli_tests/s3migration_tests/main_test.go | 6 ++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 6867e623f2..51337e9c3b 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -1,7 +1,18 @@ default_test_case_timeout: 3m -s3_access_key: -s3_secret_key: +s3_access_key: +s3_secret_key: s3_bucket_name: s3_bucket_name_alternate: alternate-test-bucket -dropboxAccessToken: +dropboxAccessToken: +dropboxRefreshToken: gdriveAccessToken: +gdriveRefreshToken: +connectionString: +accountName: +container: +googleCloudAccessToken: +googleCloudRefreshToken: +googleClientId: +googleClientSecret: +oneDriveAccessToken: +oneDriveRefreshToken: diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go index b3857f6f9d..5a1411e4eb 100644 --- a/tests/cli_tests/s3migration_tests/main_test.go +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -2,7 +2,6 @@ package s3migration_tests import ( "encoding/json" - "fmt" "log" "os" "path/filepath" @@ -60,9 +59,8 @@ func defaultData() { defaultWallet := "default" cli_utils.CreateWalletForName(shared.RootPath, defaultWallet) - - fmt.Fprintf(os.Stdout, "Default allocation ID: %s\n", defaultAllocationId) - fmt.Fprintf(os.Stdout, "Default wallet: %s\n", defaultWallet) + shared.DefaultAllocationId = defaultAllocationId + shared.DefaultWallet = defaultWallet } func EscapedTestName(t *test.SystemTest) string { From 58adf09b400481eae717d2beeb3a38ace1ff138a Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 11:12:30 +0545 Subject: [PATCH 14/27] path updated --- .../cli_tests/s3migration_tests/main_test.go | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go index 5a1411e4eb..38ab4f7de6 100644 --- a/tests/cli_tests/s3migration_tests/main_test.go +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -37,14 +37,15 @@ func setupConfig() { } shared.ConfigPath = "config.yaml" - path := filepath.Clean(filepath.Join(dir, "config")) - shared.RootPath = dir + rootPath := filepath.Dir(dir) + shared.RootPath = rootPath + configPath := filepath.Join(shared.RootPath,"config") viper.SetConfigName("nodes") viper.SetConfigType("yaml") - viper.AddConfigPath(path) + viper.AddConfigPath(configPath) - parsedConfig := config.Parse(filepath.Join(path, "cli_tests_config.yaml")) + parsedConfig := config.Parse(filepath.Join(configPath, "cli_tests_config.yaml")) shared.SetupConfig(parsedConfig) } @@ -52,15 +53,12 @@ func setupConfig() { func defaultData() { t := testing.T{} system_test := test.NewSystemTest(&t) - defaultAllocationId := cli_utils.SetupAllocation(system_test, shared.ConfigData.ConnectionString, shared.RootPath, map[string]interface{}{ "size": shared.AllocSize, }) - defaultWallet := "default" - cli_utils.CreateWalletForName(shared.RootPath, defaultWallet) shared.DefaultAllocationId = defaultAllocationId - shared.DefaultWallet = defaultWallet + shared.DefaultWallet = "default_wallet.json" } func EscapedTestName(t *test.SystemTest) string { @@ -75,7 +73,8 @@ func GetConfigDir() string { if err != nil { log.Fatalln(err) } - return filepath.Clean(filepath.Join(curr, "config")) + parentDirectory := filepath.Dir(curr) + return filepath.Clean(filepath.Join(parentDirectory, "config")) } func TestMain(m *testing.M) { @@ -121,15 +120,16 @@ func TestMain(m *testing.M) { } if shared.ConfigPath == "" { - shared.ConfigPath = "./config.yaml" + shared.ConfigPath = "config.yaml" } shared.ConfigDir, _ = filepath.Abs(shared.ConfigDir) + setupConfig() shared.WalletMutex.Lock() - fileContent, err := os.ReadFile("../config/wallets/wallets.json") + fileContent, err := os.ReadFile(shared.ConfigDir + "/wallets/wallets.json") if err != nil { log.Println("Error reading file:", err) return @@ -144,7 +144,6 @@ func TestMain(m *testing.M) { cliutils.SetWallets(shared.Wallets) shared.WalletIdx = 500 shared.WalletMutex.Unlock() - defaultData() exitRun := m.Run() From 760d81b324c6499e814107aba534bd74c89b4d40 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 12:34:14 +0545 Subject: [PATCH 15/27] default wallet --- internal/cli/util/utils.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 67f1d02c29..860e3eb11b 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -508,6 +508,9 @@ func ExecuteFaucetWithTokensForWallet(t *test.SystemTest, wallet, cliConfigFilen } func CreateWalletForName(rootPath, name string) { + if name == "" { + name = "default" + } walletPath := fmt.Sprintf("%s/config/%s_wallet.json", rootPath, name) // check if wallet already exists @@ -529,13 +532,16 @@ func CreateWalletForName(rootPath, name string) { } } -func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { +func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}, ) string { return setupAllocationWithWallet(t, EscapedTestName(t), cliConfigFilename, rootPath, extraParams...) } func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, rootPath, params string) ([]string, error) { t.Log(cliConfigFilename, "configdir path") - return RunCommand(t, fmt.Sprintf( + if (wallet == "") { + wallet = "default" + } + command := fmt.Sprintf( "%s/zbox newallocation %s --silent --wallet %s --configDir %s --config %s --allocationFileName %s", rootPath, params, @@ -543,7 +549,8 @@ func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, cliConfigFilename, "config.yaml", wallet+"_allocation.txt", - ), 3, time.Second*5) + ) + return RunCommand(t, command, 3, time.Second*5) } func setupAllocationWithWallet(t *test.SystemTest, walletName, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { From c9b676dcbb93df9ccea177e96c9a4f56d0104fd0 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 22:24:03 +0545 Subject: [PATCH 16/27] s3migration --- internal/cli/util/utils.go | 26 +- .../s3migration_tests/1_google_drive_test.go | 15 +- .../s3migration_tests/2_dropbox_test.go | 14 +- .../s3migration_tests/3_one_drive_test.go | 16 +- .../4_microsoft_azure_test.go | 18 +- .../5_google_cloud_storage_test.go | 16 +- .../aws_s3_migration_tests/0_aws_s3_test.go} | 309 ++++++------------ .../aws_s3_migration_tests/1_aws_s3_test.go} | 258 +++++++-------- .../aws_s3_migration_tests/3_aws_s3_test.go} | 186 ++++------- .../cli_tests/s3migration_tests/main_test.go | 6 +- .../s3migration_tests/shared/main.go | 11 +- .../s3migration_tests/shared/utils.go | 151 +++++++++ 12 files changed, 490 insertions(+), 536 deletions(-) rename tests/cli_tests/{0_s3mgrt_migrate_alternate_test.go => s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go} (50%) rename tests/cli_tests/{0_s3mgrt_migrate_alternate2_test.go => s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go} (57%) rename tests/cli_tests/{0_s3mgrt_migrate_test.go => s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go} (57%) create mode 100644 tests/cli_tests/s3migration_tests/shared/utils.go diff --git a/internal/cli/util/utils.go b/internal/cli/util/utils.go index 860e3eb11b..df4bea7bde 100644 --- a/internal/cli/util/utils.go +++ b/internal/cli/util/utils.go @@ -463,9 +463,8 @@ func ReadFileMC(testSetup *testing.T) McConfiguration { return config } -func MigrateFromS3migration(t *test.SystemTest, params string) ([]string, error) { - commandGenerated := fmt.Sprintf("../s3migration migrate %s", params) - t.Log(commandGenerated) +func MigrateFromCloud(t *test.SystemTest, params string) ([]string, error) { + commandGenerated := fmt.Sprintf("../s3mgrt migrate %s", params) return RunCommand(t, commandGenerated, 1, time.Hour*2) } @@ -532,13 +531,13 @@ func CreateWalletForName(rootPath, name string) { } } -func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}, ) string { +func SetupAllocation(t *test.SystemTest, cliConfigFilename, rootPath string, extraParams ...map[string]interface{}) string { return setupAllocationWithWallet(t, EscapedTestName(t), cliConfigFilename, rootPath, extraParams...) } func CreateNewAllocationForWallet(t *test.SystemTest, wallet, cliConfigFilename, rootPath, params string) ([]string, error) { t.Log(cliConfigFilename, "configdir path") - if (wallet == "") { + if wallet == "" { wallet = "default" } command := fmt.Sprintf( @@ -594,7 +593,7 @@ func getAllocationID(str string) (string, error) { } func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err error) { - pattern := `total count :: (\d+)` + pattern := `Total files ::(\d+)` re := regexp.MustCompile(pattern) match := re.FindStringSubmatch(strings.Join(output, "\n")) @@ -618,3 +617,18 @@ func GetmigratedDataID(output []string) (totalMigrated, totalCount int, err erro return 0, 0, errors.New("no match found") } + +func GetFileStats(t *test.SystemTest, cliConfigFilename, param string, retry bool) ([]string, error) { + t.Logf("Getting file stats...") + cmd := fmt.Sprintf( + "./zbox stats %s --silent --wallet %s --configDir ./config --config %s", + param, + EscapedTestName(t)+"_wallet.json", + cliConfigFilename, + ) + if retry { + return RunCommand(t, cmd, 3, time.Second*2) + } else { + return RunCommandWithoutRetry(cmd) + } +} diff --git a/tests/cli_tests/s3migration_tests/1_google_drive_test.go b/tests/cli_tests/s3migration_tests/1_google_drive_test.go index 8d50dfbe0c..e260847d21 100644 --- a/tests/cli_tests/s3migration_tests/1_google_drive_test.go +++ b/tests/cli_tests/s3migration_tests/1_google_drive_test.go @@ -12,7 +12,7 @@ import ( func Test0Gdrive(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - + t.Log(shared.ConfigData.GdriveAccessToken) if shared.ConfigData.GdriveAccessToken == "" { t.Skip("Gdrive Access Token was missing") } @@ -22,12 +22,12 @@ func Test0Gdrive(testSetup *testing.T) { allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.GdriveAccessToken, "secret-key": shared.ConfigData.GdriveRefreshToken, "allocation": allocationID, "source": "google_drive", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "config": shared.ConfigPath, "configDir": shared.ConfigDir, "skip": 0, @@ -35,7 +35,6 @@ func Test0Gdrive(testSetup *testing.T) { require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) - if err != nil { t.Fatal(err) } @@ -44,7 +43,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.GdriveAccessToken, "secret-key": shared.ConfigData.GdriveRefreshToken, "allocation": shared.DefaultAllocationId, @@ -61,7 +60,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.GdriveAccessToken, "secret-key": shared.ConfigData.GdriveRefreshToken, "source": "google_drive", @@ -75,7 +74,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", "allocation": shared.DefaultAllocationId, @@ -92,7 +91,7 @@ func Test0Gdrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "allocation": shared.DefaultAllocationId, "source": "google_drive", "wallet": shared.DefaultWallet, diff --git a/tests/cli_tests/s3migration_tests/2_dropbox_test.go b/tests/cli_tests/s3migration_tests/2_dropbox_test.go index 81772da7ff..add377a523 100644 --- a/tests/cli_tests/s3migration_tests/2_dropbox_test.go +++ b/tests/cli_tests/s3migration_tests/2_dropbox_test.go @@ -25,12 +25,12 @@ func Test0Dropbox(testSetup *testing.T) { allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": allocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.DropboxAccessToken, "secret-key": shared.ConfigData.DropboxRefreshToken, "allocation": allocationID, "source": "dropbox", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "config": shared.ConfigPath, "configDir": shared.ConfigDir, "skip": 0, @@ -40,7 +40,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.DropboxAccessToken, "secret-key": shared.ConfigData.DropboxRefreshToken, "allocation": shared.DefaultAllocationId, @@ -56,7 +56,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": shared.ConfigData.DropboxAccessToken, "secret-key": shared.ConfigData.DropboxRefreshToken, "source": "dropbox", @@ -70,7 +70,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", "allocation": shared.DefaultAllocationId, @@ -87,7 +87,7 @@ func Test0Dropbox(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "allocation": shared.DefaultAllocationId, "source": "dropbox", "wallet": shared.DefaultWallet, @@ -108,7 +108,7 @@ func Test0Dropbox(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) diff --git a/tests/cli_tests/s3migration_tests/3_one_drive_test.go b/tests/cli_tests/s3migration_tests/3_one_drive_test.go index 65d45a559f..c7aef394dd 100644 --- a/tests/cli_tests/s3migration_tests/3_one_drive_test.go +++ b/tests/cli_tests/s3migration_tests/3_one_drive_test.go @@ -25,10 +25,10 @@ func Test0OneDrive(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "onedrive", "config": shared.ConfigPath, @@ -40,10 +40,10 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": shared.DefaultAllocationId, "source": "onedrive", "config": shared.ConfigPath, @@ -56,7 +56,7 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", "wallet": shared.DefaultWallet, @@ -72,7 +72,7 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when token and refresh token is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", "wallet": shared.DefaultWallet, @@ -89,7 +89,7 @@ func Test0OneDrive(testSetup *testing.T) { }) t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "'" + shared.ConfigData.OneDriveAccessToken + "'", "secret-key": "'" + shared.ConfigData.OneDriveRefreshToken + "'", "source": "invalid", @@ -111,7 +111,7 @@ func Test0OneDrive(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) diff --git a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go index 7d47ceb271..2495d58a5f 100644 --- a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go +++ b/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go @@ -28,11 +28,11 @@ func Test0MicrosoftAzure(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "azure", "config": shared.ConfigPath, @@ -54,11 +54,11 @@ func Test0MicrosoftAzure(testSetup *testing.T) { "size": shared.AllocSize, }) - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "azure", "config": shared.ConfigPath, @@ -71,7 +71,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", @@ -86,7 +86,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) t.RunSequentially("Should fail when connection string is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "invalid", @@ -104,7 +104,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) t.RunSequentially("Should fail when connection string is missing", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "wallet": shared.DefaultWallet, @@ -120,7 +120,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { require.Contains(t, strings.Join(output, "\n"), "Missing fields: connection string", "Output was not as expected", strings.Join(output, "\n")) }) t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, "connection-string": "'" + shared.ConfigData.ConnectionString + "'", @@ -143,7 +143,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": size, }) }) diff --git a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go index 6da4697233..43d5bd39f4 100644 --- a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go +++ b/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go @@ -24,12 +24,12 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, "secret-key": shared.ConfigData.GoogleCloudRefreshToken, - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "google_cloud_storage", "config": shared.ConfigPath, @@ -52,12 +52,12 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { "size": shared.AllocSize, }) - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, "secret-key": shared.ConfigData.GoogleCloudRefreshToken, - "wallet": EscapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "google_cloud_storage", "config": shared.ConfigPath, @@ -76,7 +76,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - output, _ := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, @@ -93,7 +93,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when client credentials is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, @@ -112,7 +112,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when key is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": "invalid", @@ -131,7 +131,7 @@ func Test0GoogleCloudStorage(testSetup *testing.T) { }) t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromS3migration(t, cli_utils.CreateParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "client-id": shared.ConfigData.GoogleClientId, "client-secret": shared.ConfigData.GoogleClientSecret, "access-key": shared.ConfigData.GoogleCloudAccessToken, diff --git a/tests/cli_tests/0_s3mgrt_migrate_alternate_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go similarity index 50% rename from tests/cli_tests/0_s3mgrt_migrate_alternate_test.go rename to tests/cli_tests/s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go index 87f464e3e0..d8e91af3a4 100644 --- a/tests/cli_tests/0_s3mgrt_migrate_alternate_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/0_aws_s3_test.go @@ -1,45 +1,31 @@ -package cli_tests +package s3migration_tests import ( "bytes" - "crypto/rand" - "encoding/json" - "fmt" - "math" - "math/big" "os" "path" - "path/filepath" "strings" "testing" - "time" "github.com/0chain/system_test/internal/api/util/test" - climodel "github.com/0chain/system_test/internal/cli/model" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/require" - "golang.org/x/crypto/sha3" -) - -const chunksize = 64 * 1024 - -const ( - dirPrefix = "dir" - dirMaxRand = 1000 ) func Test0S3MigrationAlternate(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - if s3SecretKey == "" || s3AccessKey == "" { - t.Skip("s3SecretKey or s3AccessKey was missing") + if shared.ConfigData.S3SecretKey == "" || shared.ConfigData.S3AccessKey == "" { + t.Skip("shared.ConfigData.S3SecretKey or shared.ConfigData.S3AccessKey was missing") } fileKey := "sdfg" + ".txt" t.TestSetup("Setup s3 bucket with relevant file", func() { // Cleanup bucket before test - err := cleanupBucket(S3Client, s3BucketNameAlternate) + err := shared.CleanupBucket(shared.S3Client, shared.ConfigData.S3BucketNameAlternate) if err != nil { t.Log("Failed to cleanup bucket: ", err) } @@ -47,8 +33,8 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { fileContents := []byte("Hello, World!") // Upload the file to S3 - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKey), Body: bytes.NewReader(fileContents), }) @@ -58,16 +44,15 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { }) t.RunSequentially("Should migrate existing bucket successfully with skip 0 and replace existing file", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 0, })) @@ -77,25 +62,24 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, remotepath) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket to specified path successfully with encryption on", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) remotepath := "/root2" - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "migrate-to": remotepath, })) @@ -105,24 +89,23 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remoteFilePath := path.Join(remotepath, "/") - remoteFilePath = path.Join(remoteFilePath, s3BucketNameAlternate) + remoteFilePath = path.Join(remoteFilePath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, "/") remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate as copy bucket successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) @@ -130,11 +113,11 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - output, err = migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err = cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 2, "dup-suffix": "_copy", @@ -149,21 +132,20 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { // parts := strings.Split(fileKey, ".") // fileKey_modified := parts[0]+ "_modified." + parts[1] // remoteFilePath := path.Join(remotepath, fileKey_modified) - // uploadStats := checkStats(t, remoteFilePath, fileKey_modified, allocationID) + // uploadStats := shared.CheckStats(t, remoteFilePath, fileKey_modified, allocationID) // require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with encryption on", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "encrypt": "true", })) @@ -173,23 +155,22 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, true) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, true) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should skip migration with skip flag == 1 and migartion should be skipped", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 1, })) @@ -200,16 +181,15 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { }) t.RunSequentially("Should migrate successfully with duplicate files with skip flag == 2", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 2, })) @@ -222,23 +202,22 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { // FIXME : copy extension is not there // parts := strings.Split(fileKey, ".") // fileKey_modified := parts[0]+ "_copy." + parts[1] - remotepath = path.Join(remotepath, s3BucketNameAlternate) + remotepath = path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath := path.Join(remotepath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate successfully with duplication files with skip flag == 2 and dup-suffix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 2, "dup-suffix": "_modified", @@ -252,28 +231,27 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { // FIXME : dupl suffix is not working // parts := strings.Split(fileKey, ".") // fileKey_modified := parts[0]+ "_modified." + parts[1] - remotepath = path.Join(remotepath, s3BucketNameAlternate) + remotepath = path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath := path.Join(remotepath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate successfully and delete the s3 bucket file and use custom workdir", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - workingDirName := createDirectoryForTestname(t) + workingDirName := shared.CreateDirectoryForTestname(t) // remove the dir after use defer func() { _ = os.RemoveAll(workingDirName) }() - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "skip": 0, "delete-source": true, @@ -285,19 +263,18 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should error out if workdir is not default and not empty", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - workingDirName := createDirectoryForTestname(t) + workingDirName := shared.CreateDirectoryForTestname(t) file, _ := os.CreateTemp(workingDirName, "prefix") // remove the dir after use defer func() { @@ -306,11 +283,11 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { }() remotepath := "/root3" - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "migrate-to": remotepath, "wd": workingDirName, @@ -321,107 +298,3 @@ func Test0S3MigrationAlternate(testSetup *testing.T) { require.Contains(t, output[0], "working directory not empty", "Output was not as expected", strings.Join(output, "\n")) }) } - -func checkStats(t *test.SystemTest, remoteFilePath, fname, allocationID string, encrypted bool) bool { - t.Log("remotepath: ", remoteFilePath) - output, err := getFileStats(t, configPath, createParams(map[string]interface{}{ - "allocation": allocationID, - "remotepath": remoteFilePath, - "json": "true", - }), true) - require.Nil(t, err, strings.Join(output, "\n")) - require.Len(t, output, 1) - - var stats map[string]*climodel.FileStats - t.Log(output[0]) - err = json.Unmarshal([]byte(output[0]), &stats) - require.Nil(t, err) - - if len(stats) == 0 { - t.Logf("0. zero no files") - return false - } - - for _, data := range stats { - if fname != data.Name { - t.Logf("1. %s != %s", fname, data.Name) - return false - } - if remoteFilePath != data.Path { - t.Logf("2. %s != %s", remoteFilePath, data.Path) - return false - } - hash := fmt.Sprintf("%x", sha3.Sum256([]byte(allocationID+":"+remoteFilePath))) - if hash != data.PathHash { - t.Logf("3. %s != %s", hash, data.PathHash) - return false - } - if int64(0) != data.NumOfBlockDownloads { - t.Logf("4. %d != %d", int64(0), data.NumOfBlockDownloads) - return false - } - if int64(1) != data.NumOfUpdates { - t.Logf("5. %d != %d", int64(1), data.NumOfUpdates) - return false - } - if float64(data.NumOfBlocks) != math.Ceil(float64(data.Size)/float64(chunksize)) { - t.Logf("6. %f != %f", float64(data.NumOfBlocks), math.Ceil(float64(data.Size)/float64(chunksize))) - return false - } - if data.WriteMarkerTxn == "" { - if data.BlockchainAware != false { - t.Logf("7. %t", data.BlockchainAware) - return false - } - } else { - if data.BlockchainAware != true { - t.Logf("8. %t", data.BlockchainAware) - return false - } - } - } - return true -} - -func createDirectoryForTestname(t *test.SystemTest) (fullPath string) { - randomBigInt, err := rand.Int(rand.Reader, big.NewInt(int64(dirMaxRand))) - require.Nil(t, err) - - randomNumber := int(randomBigInt.Int64()) - - // Generate a unique directory name based on the random number and current timestamp - dirName := fmt.Sprintf("%s%d_%d", dirPrefix, randomNumber, time.Now().UnixNano()) - - fullPath, err = filepath.Abs(dirName) - require.Nil(t, err) - - err = os.MkdirAll(fullPath, os.ModePerm) - require.Nil(t, err) - - t.Log("Directory created successfully: ", fullPath) - - return fullPath -} - -func cleanupBucket(svc *s3.S3, s3BucketNameAlternate string) error { - // List all objects within the bucket - resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: aws.String(s3BucketNameAlternate), - }) - if err != nil { - return err - } - - // Delete each object in the bucket - for _, obj := range resp.Contents { - _, err := svc.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), - Key: obj.Key, - }) - if err != nil { - return err - } - } - - return nil -} diff --git a/tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go similarity index 57% rename from tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go rename to tests/cli_tests/s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go index fe6ee6696d..2fd428ff8d 100644 --- a/tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/1_aws_s3_test.go @@ -1,9 +1,7 @@ -package cli_tests +package s3migration_tests import ( - "bytes" - "crypto/sha1" // #nosec - "encoding/hex" + "bytes" // #nosec "fmt" "os" "path" @@ -13,7 +11,9 @@ import ( "time" "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/require" @@ -22,14 +22,14 @@ import ( func Test0S3MigrationAlternatePart2(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - if s3SecretKey == "" || s3AccessKey == "" { - t.Skip("s3SecretKey or s3AccessKey was missing") + if shared.ConfigData.S3SecretKey == "" || shared.ConfigData.S3AccessKey == "" { + t.Skip("shared.ConfigData.S3SecretKey or shared.ConfigData.S3AccessKey was missing") } fileKey := "OneMinNew" + ".txt" t.TestSetup("Setup s3 bucket with relevant file", func() { // Cleanup before test - err := cleanupBucket(S3Client, s3BucketNameAlternate) + err := shared.CleanupBucket(shared.S3Client, shared.ConfigData.S3BucketNameAlternate) if err != nil { t.Log("Failed to cleanup bucket: ", err) } @@ -37,8 +37,8 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { fileContents := []byte("Hello, World!") // Upload the file to S3 - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKey), Body: bytes.NewReader(fileContents), }) @@ -48,21 +48,20 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { }) t.RunSequentially("Should migrate existing bucket successfully with concurrency==4 and working dir current dir", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - workingDirName := createDirectoryForTestname(t) + workingDirName := shared.CreateDirectoryForTestname(t) // remove the dir after use defer func() { _ = os.RemoveAll(workingDirName) }() - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "concurrency": 4, "wd": workingDirName, @@ -73,32 +72,31 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKey) - uploadStats := checkStats(t, remoteFilePath, fileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with newer than prefix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "oneMinOld" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "newer-than": time.Now().Unix() - 60, // start timestamp })) @@ -107,22 +105,21 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with older than prefix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) olderThanFileKey := "olderThanFile" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(olderThanFileKey), Body: bytes.NewReader(fileContents), }) @@ -130,11 +127,11 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { cliutils.Wait(t, 70*time.Second) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "older-than": time.Now().Unix() - 60, // end timestamp })) @@ -144,39 +141,38 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, olderThanFileKey) - uploadStats := checkStats(t, remoteFilePath, olderThanFileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, olderThanFileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should migrate existing bucket successfully with files staring with given prefix", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyToBemigrated := "mgrt" + ".txt" fileKeyNotToBeMigrated := "noMgrt" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyToBemigrated), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNotToBeMigrated), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "prefix": "mgrt", })) @@ -185,25 +181,24 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n")) - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePathPos := path.Join(remoteFilePath, fileKeyToBemigrated) remoteFilePathNeg := path.Join(remoteFilePath, fileKeyNotToBeMigrated) - uploadStats := checkStats(t, remoteFilePathPos, fileKeyToBemigrated, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePathPos, fileKeyToBemigrated, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") - uploadStats = checkStats(t, remoteFilePathNeg, fileKeyNotToBeMigrated, allocationID, false) + uploadStats = shared.CheckStats(t, remoteFilePathNeg, fileKeyNotToBeMigrated, allocationID, false) require.Equal(t, false, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass when allocation flag missing but allocation path is given", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForAllocPath" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) @@ -215,11 +210,11 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { err = os.WriteFile(allocPath, []byte(allocationID), 0644) //nolint:gosec require.Nil(t, err, "allocation file is not written properly") - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "alloc-path": allocPath, })) @@ -228,21 +223,20 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass when access key and secret key is missing but aws-cred-path path is given", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForAwsCredPath" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) @@ -252,8 +246,8 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Nil(t, err, "can't get current dir") awsCredPath := filepath.Join(currentDir, "awsCredPathForTestS3.txt") lines := []string{ - fmt.Sprintf(`aws_access_key: "%v"`, s3AccessKey), - fmt.Sprintf(`aws_secret_key: "%v"`, s3SecretKey), + fmt.Sprintf(`aws_access_key: "%v"`, shared.ConfigData.S3AccessKey), + fmt.Sprintf(`aws_secret_key: "%v"`, shared.ConfigData.S3SecretKey), } file, err := os.OpenFile(awsCredPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) require.Nil(t, err, "file is not created properly") @@ -264,10 +258,10 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Nil(t, err, "failed to write file") } - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "bucket": s3BucketNameAlternate, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "bucket": shared.ConfigData.S3BucketNameAlternate, "allocation": allocationID, - "wallet": escapedTestName(t) + "_wallet.json", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "aws-cred-path": awsCredPath, })) @@ -276,32 +270,31 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass concurrency flag is set to 20", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForConCurrTest" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "concurrency": 20, })) @@ -311,32 +304,31 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass retry flag is set to 4", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) fileKeyNew := "fileForRetryTest" + ".txt" fileContents := []byte("Hello, World!") - _, err := S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err := shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, "retry": 4, })) @@ -346,40 +338,39 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Contains(t, output[0], "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) remoteFilePath = path.Join(remoteFilePath, fileKeyNew) - uploadStats := checkStats(t, remoteFilePath, fileKeyNew, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, fileKeyNew, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file") }) t.RunSequentially("Should pass when filename size is more than 100 character with renamed file names", func(t *test.SystemTest) { // Cleanup before test - err := cleanupBucket(S3Client, s3BucketNameAlternate) + err := shared.CleanupBucket(shared.S3Client, shared.ConfigData.S3BucketNameAlternate) if err != nil { t.Log("Failed to cleanup bucket: ", err) } - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) // As per the current logic in s3-migration even the longer file names would be migrated with file names // trimmed to 100 chars. fileKeyNew := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbb.txt" fileContents := []byte("Hello, World!") - _, err = S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s3BucketNameAlternate), + _, err = shared.S3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(shared.ConfigData.S3BucketNameAlternate), Key: aws.String(fileKeyNew), Body: bytes.NewReader(fileContents), }) require.Nil(t, err) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3BucketNameAlternate, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketNameAlternate, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) // mssg can be changed @@ -388,35 +379,10 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) { require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) remotepath := "/" - remoteFilePath := path.Join(remotepath, s3BucketNameAlternate) - shortFileKey := getUniqueShortObjKey(fileKeyNew) + remoteFilePath := path.Join(remotepath, shared.ConfigData.S3BucketNameAlternate) + shortFileKey := shared.GetUniqueShortObjKey(fileKeyNew) remoteFilePath = path.Join(remoteFilePath, shortFileKey) - uploadStats := checkStats(t, remoteFilePath, shortFileKey, allocationID, false) + uploadStats := shared.CheckStats(t, remoteFilePath, shortFileKey, allocationID, false) require.Equal(t, true, uploadStats, "The file migrated does not match with expected file") }) } - -// This func is a copy of new file name generation logic in s3-migration from migration/migrate.go -// This func needs to be in sync with the original func. -func getUniqueShortObjKey(objectKey string) string { - // Max length to which objectKey would be trimmed to. - // Keeping this less than 100 chars to prevent longer name in case of uploading duplicate - // files with `_copy` suffixes. - const maxLength = 90 - - if len(objectKey) > maxLength { - // Generate a SHA-1 hash of the object key - hash := sha1.New() // #nosec - _, _ = hash.Write([]byte(objectKey)) - hashSum := hash.Sum(nil) - - // Convert the hash to a hexadecimal string - hashString := hex.EncodeToString(hashSum) - - // Combine the first 10 characters of the hash with a truncated object key - shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:]) - return shortKey - } - - return objectKey -} diff --git a/tests/cli_tests/0_s3mgrt_migrate_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go similarity index 57% rename from tests/cli_tests/0_s3mgrt_migrate_test.go rename to tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go index 5e9f992e0b..d832f914ff 100644 --- a/tests/cli_tests/0_s3mgrt_migrate_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go @@ -1,36 +1,35 @@ -package cli_tests +package s3migration_tests import ( - "fmt" "strings" "testing" - "time" "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/stretchr/testify/require" ) func Test0S3Migration(testSetup *testing.T) { t := test.NewSystemTest(testSetup) - if s3SecretKey == "" || s3AccessKey == "" { - t.Skip("s3SecretKey or s3AccessKey was missing") + if shared.ConfigData.S3SecretKey == "" || shared.ConfigData.S3AccessKey == "" { + t.Skip("shared.ConfigData.S3SecretKey or shared.ConfigData.S3AccessKey was missing") } t.SetSmokeTests("Should migrate existing bucket successfully") t.RunSequentially("Should migrate existing bucket successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) @@ -40,16 +39,15 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should migrate empty bucket successfully", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, "bucket": "system-tests-empty", - "wallet": escapedTestName(t) + "_wallet.json", + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", "allocation": allocationID, })) @@ -59,16 +57,16 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when bucket too large for allocation", func(t *test.SystemTest) { - allocSize := int64(64 * KB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ + allocSize := int64(64 * shared.KB) + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": allocSize, }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": shared.DefaultWallet, "allocation": allocationID, })) @@ -78,17 +76,12 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when bucket does not exist", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, "bucket": "invalid", - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -97,16 +90,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when bucket flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "wallet": shared.DefaultWallet, + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -115,16 +103,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - _ = setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": shared.DefaultWallet, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -133,17 +116,13 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -152,16 +131,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "secret-key": s3SecretKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "secret-key": shared.ConfigData.S3SecretKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -170,17 +144,12 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when secret key invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, "secret-key": "invalid", - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -189,16 +158,11 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when secret key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "access-key": s3AccessKey, - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.S3AccessKey, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -207,17 +171,13 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access and secret key invalid", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -226,15 +186,10 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access and secret key missing", func(t *test.SystemTest) { - allocSize := int64(50 * MB) - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) - - output, err := migrateFromS3(t, configPath, createParams(map[string]interface{}{ - "bucket": s3bucketName, - "wallet": escapedTestName(t) + "_wallet.json", - "allocation": allocationID, + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "bucket": shared.ConfigData.S3BucketName, + "wallet": cliutils.EscapedTestName(t) + "_wallet.json", + "allocation": shared.DefaultAllocationId, })) require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) @@ -242,8 +197,3 @@ func Test0S3Migration(testSetup *testing.T) { require.Equal(t, output[0], "Error: aws credentials missing", "Output was not as expected", strings.Join(output, "\n")) }) } - -func migrateFromS3(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { - t.Logf("Migrating S3 bucket to Zus...") - return cliutils.RunCommand(t, fmt.Sprintf("./s3mgrt migrate --silent --configDir ./config --config %s --network %s %s", cliConfigFilename, cliConfigFilename, params), 1, time.Second*2) -} diff --git a/tests/cli_tests/s3migration_tests/main_test.go b/tests/cli_tests/s3migration_tests/main_test.go index 38ab4f7de6..1e3e9d86f7 100644 --- a/tests/cli_tests/s3migration_tests/main_test.go +++ b/tests/cli_tests/s3migration_tests/main_test.go @@ -10,7 +10,6 @@ import ( "github.com/0chain/system_test/internal/api/util/config" "github.com/0chain/system_test/internal/api/util/test" - cli_utils "github.com/0chain/system_test/internal/cli/util" cliutils "github.com/0chain/system_test/internal/cli/util" "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" "github.com/aws/aws-sdk-go/aws" @@ -39,7 +38,7 @@ func setupConfig() { shared.ConfigPath = "config.yaml" rootPath := filepath.Dir(dir) shared.RootPath = rootPath - configPath := filepath.Join(shared.RootPath,"config") + configPath := filepath.Join(shared.RootPath, "config") viper.SetConfigName("nodes") viper.SetConfigType("yaml") @@ -53,7 +52,7 @@ func setupConfig() { func defaultData() { t := testing.T{} system_test := test.NewSystemTest(&t) - defaultAllocationId := cli_utils.SetupAllocation(system_test, shared.ConfigData.ConnectionString, shared.RootPath, map[string]interface{}{ + defaultAllocationId := cliutils.SetupAllocation(system_test, shared.ConfigDir, shared.RootPath, map[string]interface{}{ "size": shared.AllocSize, }) @@ -125,7 +124,6 @@ func TestMain(m *testing.M) { shared.ConfigDir, _ = filepath.Abs(shared.ConfigDir) - setupConfig() shared.WalletMutex.Lock() diff --git a/tests/cli_tests/s3migration_tests/shared/main.go b/tests/cli_tests/s3migration_tests/shared/main.go index 7c56851f21..1db4138f28 100644 --- a/tests/cli_tests/s3migration_tests/shared/main.go +++ b/tests/cli_tests/s3migration_tests/shared/main.go @@ -22,10 +22,13 @@ var ( ) const ( - KB = 1024 // kilobyte - MB = 1024 * KB // megabyte - GB = 1024 * MB // gigabyte - AllocSize = int64(50 * MB) + KB = 1024 // kilobyte + MB = 1024 * KB // megabyte + GB = 1024 * MB // gigabyte + AllocSize = int64(50 * MB) + Chunksize = 64 * 1024 + DirPrefix = "dir" + DirMaxRand = 1000 ) func SetupConfig(parsedConfig *util.Config) { diff --git a/tests/cli_tests/s3migration_tests/shared/utils.go b/tests/cli_tests/s3migration_tests/shared/utils.go new file mode 100644 index 0000000000..7aecc8e80a --- /dev/null +++ b/tests/cli_tests/s3migration_tests/shared/utils.go @@ -0,0 +1,151 @@ +package shared + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "math/big" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/sha3" + + "github.com/0chain/system_test/internal/api/util/test" + climodel "github.com/0chain/system_test/internal/cli/model" + cli_utils "github.com/0chain/system_test/internal/cli/util" +) + +func CleanupBucket(svc *s3.S3, s3BucketNameAlternate string) error { + // List all objects within the bucket + resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(s3BucketNameAlternate), + }) + if err != nil { + return err + } + + // Delete each object in the bucket + for _, obj := range resp.Contents { + _, err := svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(s3BucketNameAlternate), + Key: obj.Key, + }) + if err != nil { + return err + } + } + + return nil +} + +func CheckStats(t *test.SystemTest, remoteFilePath, fname, allocationID string, encrypted bool) bool { + t.Log("remotepath: ", remoteFilePath) + output, err := cli_utils.GetFileStats(t, ConfigPath, cli_utils.CreateParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remoteFilePath, + "json": "true", + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) + + var stats map[string]*climodel.FileStats + t.Log(output[0]) + err = json.Unmarshal([]byte(output[0]), &stats) + require.Nil(t, err) + + if len(stats) == 0 { + t.Logf("0. zero no files") + return false + } + + for _, data := range stats { + if fname != data.Name { + t.Logf("1. %s != %s", fname, data.Name) + return false + } + if remoteFilePath != data.Path { + t.Logf("2. %s != %s", remoteFilePath, data.Path) + return false + } + hash := fmt.Sprintf("%x", sha3.Sum256([]byte(allocationID+":"+remoteFilePath))) + if hash != data.PathHash { + t.Logf("3. %s != %s", hash, data.PathHash) + return false + } + if int64(0) != data.NumOfBlockDownloads { + t.Logf("4. %d != %d", int64(0), data.NumOfBlockDownloads) + return false + } + if int64(1) != data.NumOfUpdates { + t.Logf("5. %d != %d", int64(1), data.NumOfUpdates) + return false + } + if float64(data.NumOfBlocks) != math.Ceil(float64(data.Size)/float64(Chunksize)) { + t.Logf("6. %f != %f", float64(data.NumOfBlocks), math.Ceil(float64(data.Size)/float64(Chunksize))) + return false + } + if data.WriteMarkerTxn == "" { + if data.BlockchainAware != false { + t.Logf("7. %t", data.BlockchainAware) + return false + } + } else { + if data.BlockchainAware != true { + t.Logf("8. %t", data.BlockchainAware) + return false + } + } + } + return true +} + +func CreateDirectoryForTestname(t *test.SystemTest) (fullPath string) { + randomBigInt, err := rand.Int(rand.Reader, big.NewInt(int64(DirMaxRand))) + require.Nil(t, err) + + randomNumber := int(randomBigInt.Int64()) + + // Generate a unique directory name based on the random number and current timestamp + dirName := fmt.Sprintf("%s%d_%d", DirPrefix, randomNumber, time.Now().UnixNano()) + + fullPath, err = filepath.Abs(dirName) + require.Nil(t, err) + + err = os.MkdirAll(fullPath, os.ModePerm) + require.Nil(t, err) + + t.Log("Directory created successfully: ", fullPath) + + return fullPath +} + +func GetUniqueShortObjKey(objectKey string) string { + // Max length to which objectKey would be trimmed to. + // Keeping this less than 100 chars to prevent longer name in case of uploading duplicate + // files with `_copy` suffixes. + const maxLength = 90 + + if len(objectKey) > maxLength { + // Generate a SHA-1 hash of the object key + hash := sha1.New() // #nosec + _, _ = hash.Write([]byte(objectKey)) + hashSum := hash.Sum(nil) + + // Convert the hash to a hexadecimal string + hashString := hex.EncodeToString(hashSum) + + // Combine the first 10 characters of the hash with a truncated object key + shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:]) + return shortKey + } + + return objectKey +} From e7ca4089e0cf2f2e433125ea245c7cded8cd44b7 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 22:48:08 +0545 Subject: [PATCH 17/27] config --- .../s3migration_tests/config/config.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/cli_tests/s3migration_tests/config/config.yaml diff --git a/tests/cli_tests/s3migration_tests/config/config.yaml b/tests/cli_tests/s3migration_tests/config/config.yaml new file mode 100644 index 0000000000..528e913f47 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/config/config.yaml @@ -0,0 +1,15 @@ +block_worker: https://test.zus.network/dns +confirmation_chain_length: 3 +ethereum_node_url: "https://rpc.tenderly.co/fork/5b7ffac9-50cc-4169-b0ca-6fd203d26ef6" +min_confirmation: 50 +min_submit: 50 +signature_scheme: bls0chain +store_unlock_duration_sec: 2 + +bridge: + bridge_address: 0x7700D773022b19622095118Fadf46f7B9448Be9b + token_address: 0xb9EF770B6A5e12E45983C5D80545258aA38F3B78 + authorizers_address: 0x481daB4407b9880DE0A68dc62E6aF611c4949E42 + uniswap_address: 0x4c12C2FeEDD86267d17dB64BaB2cFD12cD8611f5 + ethereum_address: 0x8E25cfd9bd6c0ca67a5522cd920b3c66D39d6E97 + password: "12345678" From f373e82f75cb3ea86e7cbd29e9966ac7da193c92 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 7 Feb 2025 23:00:36 +0545 Subject: [PATCH 18/27] updated for lint --- tests/cli_tests/main_test.go | 4 ---- .../s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go | 2 -- tests/cli_tests/s3migration_tests/shared/utils.go | 4 ++-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/cli_tests/main_test.go b/tests/cli_tests/main_test.go index 05f7316b2f..57de26d4e1 100644 --- a/tests/cli_tests/main_test.go +++ b/tests/cli_tests/main_test.go @@ -95,10 +95,6 @@ var ( ethereumNodeURL string tokenAddress string ethereumAddress string - s3SecretKey string - s3AccessKey string - s3bucketName string - s3BucketNameAlternate string S3Client *s3.S3 ) diff --git a/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go index d832f914ff..8cc4426d9e 100644 --- a/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go +++ b/tests/cli_tests/s3migration_tests/aws_s3_migration_tests/3_aws_s3_test.go @@ -116,7 +116,6 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access key invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": shared.ConfigData.S3SecretKey, @@ -171,7 +170,6 @@ func Test0S3Migration(testSetup *testing.T) { }) t.RunSequentially("Should fail when access and secret key invalid", func(t *test.SystemTest) { - output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "access-key": "invalid", "secret-key": "invalid", diff --git a/tests/cli_tests/s3migration_tests/shared/utils.go b/tests/cli_tests/s3migration_tests/shared/utils.go index 7aecc8e80a..0800d42101 100644 --- a/tests/cli_tests/s3migration_tests/shared/utils.go +++ b/tests/cli_tests/s3migration_tests/shared/utils.go @@ -2,7 +2,7 @@ package shared import ( "crypto/rand" - "crypto/sha1" + "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -135,7 +135,7 @@ func GetUniqueShortObjKey(objectKey string) string { if len(objectKey) > maxLength { // Generate a SHA-1 hash of the object key - hash := sha1.New() // #nosec + hash := sha256.New() // #nosec _, _ = hash.Write([]byte(objectKey)) hashSum := hash.Sum(nil) From 75fed5a8b749328baee42057cc968db8d0332046 Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 01:29:52 +0545 Subject: [PATCH 19/27] added for system test --- internal/api/util/config/config.go | 4 + .../1_general_test.go} | 4 +- .../box_tests/1_general_test.go | 116 ++++++++++++++++++ .../1_general_test.go} | 21 +--- .../1_general_test.go} | 2 +- .../1_general_test.go} | 0 .../1_general_test.go} | 4 +- .../s3migration_tests/shared/main.go | 19 +++ 8 files changed, 146 insertions(+), 24 deletions(-) rename tests/cli_tests/s3migration_tests/{4_microsoft_azure_test.go => azure_tests/1_general_test.go} (97%) create mode 100644 tests/cli_tests/s3migration_tests/box_tests/1_general_test.go rename tests/cli_tests/s3migration_tests/{2_dropbox_test.go => dropbox_tests/1_general_test.go} (85%) rename tests/cli_tests/s3migration_tests/{1_google_drive_test.go => gdrive_tests/1_general_test.go} (99%) rename tests/cli_tests/s3migration_tests/{5_google_cloud_storage_test.go => google_cloud_storage_tests/1_general_test.go} (100%) rename tests/cli_tests/s3migration_tests/{3_one_drive_test.go => onedrive_tests/1_general_test.go} (96%) diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 67a8bc8022..a90ba001ba 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -49,6 +49,10 @@ type Config struct { GoogleClientSecret string `yaml:"googleClientSecret"` OneDriveAccessToken string `yaml:"oneDriveAccessToken"` OneDriveRefreshToken string `yaml:"oneDriveRefreshToken"` + BoxAccessToken string `yaml:"boxAccessToken"` + BoxRefreshToken string `yaml:"boxRefreshToken"` + BoxClientId string `yaml:"boxClientId"` + BoxClientSecret string `yaml:"boxClientSecret"` } func Parse(configPath string) *Config { diff --git a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go similarity index 97% rename from tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go rename to tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go index 2495d58a5f..55e5a0afc2 100644 --- a/tests/cli_tests/s3migration_tests/4_microsoft_azure_test.go +++ b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go @@ -1,4 +1,4 @@ -package s3migration_tests +package azure_tests import ( "strings" @@ -143,7 +143,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = shared.SetupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": size, }) }) diff --git a/tests/cli_tests/s3migration_tests/box_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/box_tests/1_general_test.go new file mode 100644 index 0000000000..3dcf6cc3c6 --- /dev/null +++ b/tests/cli_tests/s3migration_tests/box_tests/1_general_test.go @@ -0,0 +1,116 @@ +package box_tests + +import ( + "strings" + "testing" + + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" + "github.com/0chain/system_test/tests/cli_tests/s3migration_tests/shared" + "github.com/stretchr/testify/require" +) + +func Test0Box(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + t.Log(shared.ConfigData.BoxAccessToken) + if shared.ConfigData.BoxAccessToken == "" || shared.ConfigData.BoxRefreshToken == "" || shared.ConfigData.BoxClientId == "" || shared.ConfigData.BoxClientSecret == "" { + t.Skip("Box Credentials were missing") + } + t.SetSmokeTests("Should migrate existing Box folder and files successfully") + + t.RunSequentially("Should migrate existing box folder and files successfully", func(t *test.SystemTest) { + allocationID := cli_utils.SetupAllocation(t, shared.ConfigDir, shared.RootPath, map[string]interface{}{ + "size": shared.AllocSize, + }) + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.BoxAccessToken, + "secret-key": shared.ConfigData.BoxRefreshToken, + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "allocation": allocationID, + "source": "box", + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + + totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) + if err != nil { + t.Fatal(err) + } + require.Equal(t, totalCount, totalMigrated, "Total count of migrated files is not equal to total migrated files") + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.BoxAccessToken, + "secret-key": shared.ConfigData.BoxRefreshToken, + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "allocation": shared.DefaultAllocationId, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": shared.ConfigData.BoxAccessToken, + "secret-key": shared.ConfigData.BoxRefreshToken, + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "access-key": "invalid", + "secret-key": "invalid", + "client-id": shared.ConfigData.BoxClientId, + "client-secret": shared.ConfigData.BoxClientSecret, + "allocation": shared.DefaultAllocationId, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + output, err := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ + "allocation": shared.DefaultAllocationId, + "source": "box", + "wallet": shared.DefaultWallet, + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, + })) + + t.Logf("EXpected log %v", strings.Join(output, "\n")) + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing fields: access key, secret key") + }) +} diff --git a/tests/cli_tests/s3migration_tests/2_dropbox_test.go b/tests/cli_tests/s3migration_tests/dropbox_tests/1_general_test.go similarity index 85% rename from tests/cli_tests/s3migration_tests/2_dropbox_test.go rename to tests/cli_tests/s3migration_tests/dropbox_tests/1_general_test.go index add377a523..142f5e9a17 100644 --- a/tests/cli_tests/s3migration_tests/2_dropbox_test.go +++ b/tests/cli_tests/s3migration_tests/dropbox_tests/1_general_test.go @@ -1,7 +1,6 @@ -package s3migration_tests +package dropbox_tests import ( - "fmt" "strings" "testing" @@ -108,24 +107,8 @@ func Test0Dropbox(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = shared.SetupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) } - -func setupAllocationWithWalletWithoutTest(t *test.SystemTest, walletName, cliConfigFilename string, extraParams ...map[string]interface{}) (string, error) { - options := map[string]interface{}{"size": "10000000", "lock": "5"} - - for _, params := range extraParams { - for k, v := range params { - options[k] = v - } - } - cli_utils.CreateWalletForName(shared.RootPath, walletName) - output, _ := cli_utils.CreateNewAllocationForWallet(t, walletName, cliConfigFilename, shared.RootPath, cli_utils.CreateParams(options)) - defer func() { - fmt.Printf("err: %v\n", output) - }() - return cli_utils.GetAllocationID(output[0]) -} diff --git a/tests/cli_tests/s3migration_tests/1_google_drive_test.go b/tests/cli_tests/s3migration_tests/gdrive_tests/1_general_test.go similarity index 99% rename from tests/cli_tests/s3migration_tests/1_google_drive_test.go rename to tests/cli_tests/s3migration_tests/gdrive_tests/1_general_test.go index e260847d21..17b12974c0 100644 --- a/tests/cli_tests/s3migration_tests/1_google_drive_test.go +++ b/tests/cli_tests/s3migration_tests/gdrive_tests/1_general_test.go @@ -1,4 +1,4 @@ -package s3migration_tests +package gdrive_tests import ( "strings" diff --git a/tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go b/tests/cli_tests/s3migration_tests/google_cloud_storage_tests/1_general_test.go similarity index 100% rename from tests/cli_tests/s3migration_tests/5_google_cloud_storage_test.go rename to tests/cli_tests/s3migration_tests/google_cloud_storage_tests/1_general_test.go diff --git a/tests/cli_tests/s3migration_tests/3_one_drive_test.go b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go similarity index 96% rename from tests/cli_tests/s3migration_tests/3_one_drive_test.go rename to tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go index c7aef394dd..c6c0168623 100644 --- a/tests/cli_tests/s3migration_tests/3_one_drive_test.go +++ b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go @@ -1,4 +1,4 @@ -package s3migration_tests +package onedrive_tests import ( "fmt" @@ -111,7 +111,7 @@ func Test0OneDrive(testSetup *testing.T) { defer func() { require.Contains(t, err.Error(), "allocation match not found") }() - _, err = setupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ + _, err = shared.SetupAllocationWithWalletWithoutTest(t, cli_utils.EscapedTestName(t)+"_wallet.json", shared.ConfigPath, map[string]interface{}{ "size": allocSize, }) }) diff --git a/tests/cli_tests/s3migration_tests/shared/main.go b/tests/cli_tests/s3migration_tests/shared/main.go index 1db4138f28..4b0a054e3c 100644 --- a/tests/cli_tests/s3migration_tests/shared/main.go +++ b/tests/cli_tests/s3migration_tests/shared/main.go @@ -2,9 +2,12 @@ package shared import ( "encoding/json" + "fmt" "sync" util "github.com/0chain/system_test/internal/api/util/config" + "github.com/0chain/system_test/internal/api/util/test" + cli_utils "github.com/0chain/system_test/internal/cli/util" "github.com/aws/aws-sdk-go/service/s3" ) @@ -34,3 +37,19 @@ const ( func SetupConfig(parsedConfig *util.Config) { ConfigData = *parsedConfig } + +func SetupAllocationWithWalletWithoutTest(t *test.SystemTest, walletName, cliConfigFilename string, extraParams ...map[string]interface{}) (string, error) { + options := map[string]interface{}{"size": "10000000", "lock": "5"} + + for _, params := range extraParams { + for k, v := range params { + options[k] = v + } + } + cli_utils.CreateWalletForName(RootPath, walletName) + output, _ := cli_utils.CreateNewAllocationForWallet(t, walletName, cliConfigFilename, RootPath, cli_utils.CreateParams(options)) + defer func() { + fmt.Printf("err: %v\n", output) + }() + return cli_utils.GetAllocationID(output[0]) +} From 4c9b2c031f53926e6a9ec906bbefc6f8f8018a6e Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 01:30:41 +0545 Subject: [PATCH 20/27] added for system test --- .../s3migration_tests/onedrive_tests/1_general_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go index c6c0168623..5d962b00cf 100644 --- a/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go +++ b/tests/cli_tests/s3migration_tests/onedrive_tests/1_general_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" ) -func Test0OneDrive(testSetup *testing.T) { +func TestOneDrive(testSetup *testing.T) { t := test.NewSystemTest(testSetup) if shared.ConfigData.OneDriveAccessToken == "" || shared.ConfigData.OneDriveRefreshToken == "" { From 6c11f86426a720fef77a019bc1bb9f3980124c74 Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 02:15:27 +0545 Subject: [PATCH 21/27] updated config yaml file --- tests/cli_tests/config/cli_tests_config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 51337e9c3b..71fb6c448d 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -16,3 +16,7 @@ googleClientId: googleClientSecret: oneDriveAccessToken: oneDriveRefreshToken: +boxAccessToken: +boxRefreshToken: +boxClientId: +boxClientSecret: From 63eb47e556c8a1cddc276ab2d697a076b6003a9b Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 3 Mar 2025 02:16:27 +0545 Subject: [PATCH 22/27] ignore s3mgrt --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fdb1965f0c..bfc7fc3edc 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ warp-*_output.txt warp minio mc -store \ No newline at end of file +store +s3mgrt \ No newline at end of file From d60b3b243583f953ccc973edc3077d4f92114a54 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 14 Mar 2025 23:28:38 +0545 Subject: [PATCH 23/27] updated for azure --- internal/api/util/config/config.go | 1 + tests/cli_tests/config/cli_tests_config.yaml | 1 + .../s3migration_tests/azure_tests/1_general_test.go | 9 ++++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index a90ba001ba..0fcaca1128 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -43,6 +43,7 @@ type Config struct { ConnectionString string `yaml:"connectionString"` AccountName string `yaml:"accountName"` ContainerName string `yaml:"container"` + AzureAccessKey string `yaml:"azureAccessKey"` GoogleCloudAccessToken string `yaml:"googleCloudAccessToken"` GoogleCloudRefreshToken string `yaml:"googleCloudRefreshToken"` GoogleClientId string `yaml:"googleClientId"` diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 71fb6c448d..256b19804a 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -10,6 +10,7 @@ gdriveRefreshToken: connectionString: accountName: container: +azureAccessKey: googleCloudAccessToken: googleCloudRefreshToken: googleClientId: diff --git a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go index 55e5a0afc2..beed9272df 100644 --- a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go +++ b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go @@ -17,10 +17,13 @@ func Test0MicrosoftAzure(testSetup *testing.T) { t.Skip("Missing Account name for required for migration") } - if shared.ConfigData.ConnectionString == "" { - t.Skip("Missing Connection String required for migration") + if shared.ConfigData.ContainerName == "" { + t.Skip("Missing Container Name required for migration") } + if shared.ConfigData.AzureAccessKey == "" { + t.Skip("Missing Access Key required for migration") + } t.SetSmokeTests("Should migrate existing files and folder from Microsoft Azure Container successfully") t.RunSequentially("Should migrate existing Microsoft Azure folder and files successfully", func(t *test.SystemTest) { @@ -31,7 +34,7 @@ func Test0MicrosoftAzure(testSetup *testing.T) { output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ "account-name": shared.ConfigData.AccountName, "container": shared.ConfigData.ContainerName, - "connection-string": "'" + shared.ConfigData.ConnectionString + "'", + "access-key": shared.ConfigData.AzureAccessKey, "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", "allocation": allocationId, "source": "azure", From 6c7e007d29362e7bf741da7bc59ed14bbcddf834 Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 14 Mar 2025 23:38:19 +0545 Subject: [PATCH 24/27] box format --- .../azure_tests/1_general_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go index beed9272df..e3ff5ef1b0 100644 --- a/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go +++ b/tests/cli_tests/s3migration_tests/azure_tests/1_general_test.go @@ -32,15 +32,15 @@ func Test0MicrosoftAzure(testSetup *testing.T) { }) output, _ := cli_utils.MigrateFromCloud(t, cli_utils.CreateParams(map[string]interface{}{ - "account-name": shared.ConfigData.AccountName, - "container": shared.ConfigData.ContainerName, - "access-key": shared.ConfigData.AzureAccessKey, - "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", - "allocation": allocationId, - "source": "azure", - "config": shared.ConfigPath, - "configDir": shared.ConfigDir, - "skip": 0, + "account-name": shared.ConfigData.AccountName, + "container": shared.ConfigData.ContainerName, + "access-key": shared.ConfigData.AzureAccessKey, + "wallet": cli_utils.EscapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "azure", + "config": shared.ConfigPath, + "configDir": shared.ConfigDir, + "skip": 0, })) totalCount, totalMigrated, err := cli_utils.GetmigratedDataID(output) From 935db55aa939dc39d2d605785e49b244b6bac082 Mon Sep 17 00:00:00 2001 From: pewssh Date: Tue, 18 Mar 2025 03:59:02 +0800 Subject: [PATCH 25/27] NTR --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9c15c694e..c9c0c7a4e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,8 +122,8 @@ jobs: svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} deploy_report_page: true archive_results: true - run_flaky_tests: true - run_api_system_tests: true + run_flaky_tests: false + run_api_system_tests: false run_cli_system_tests: true run_tenderly_tests: false run_tokenomics_system_tests: false From 68a8ad8a618bb28bd773cad93c7622fa81668f3c Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 8 Apr 2025 01:51:33 +0530 Subject: [PATCH 26/27] dummy --- internal/api/util/client/api_client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/api/util/client/api_client.go b/internal/api/util/client/api_client.go index 7335bd9393..71b879d07f 100644 --- a/internal/api/util/client/api_client.go +++ b/internal/api/util/client/api_client.go @@ -964,7 +964,6 @@ func (c *APIClient) RegisterBlobber(t *test.SystemTest, t.Log("Error unmarshalling JSON:", err) return false } - return registerBlobberTransactionGetConfirmationResponse.Status == requiredTransactionStatus && storageNode.ID == expectedResponse } From 66867975787922d65d0d40eefc73118a4370f840 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 8 Apr 2025 17:27:12 +0530 Subject: [PATCH 27/27] fix --- tests/cli_tests/0_expired_allocation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_tests/0_expired_allocation_test.go b/tests/cli_tests/0_expired_allocation_test.go index 48a939e79a..cea14ad514 100644 --- a/tests/cli_tests/0_expired_allocation_test.go +++ b/tests/cli_tests/0_expired_allocation_test.go @@ -183,7 +183,7 @@ func TestExpiredAllocation(testSetup *testing.T) { output, err = finalizeAllocation(t, configPath, allocationID, true) if err != nil { - require.Contains(t, err.Error(), "already finalized", "unexpected error: %s", err.Error()) + require.Contains(t, output[0], "already finalized", "unexpected error: %s", err.Error()) } else { require.Nil(t, err, "unexpected error updating allocation", strings.Join(output, "\n")) require.True(t, len(output) > 0, "expected output length be at least 1", strings.Join(output, "\n"))