From 9d7f0dac2e08e457aa012390312fd95c4a5d2b86 Mon Sep 17 00:00:00 2001 From: Pankaj Keswani Date: Thu, 28 Mar 2019 22:40:06 +0530 Subject: [PATCH 1/9] Update package.json (#59) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 85c858e..206683c 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "OST Storage", "DynamoDB" ], - "author": "OST.COM LTD.", + "author": "OST.com Inc.", "license": "Apache-2.0", "bugs": { "url": "https://github.com/ostdotcom/storage/issues" From c5ca43d054eebb737e25464022043fba835d374e Mon Sep 17 00:00:00 2001 From: Puneet Khushwani Date: Wed, 3 Apr 2019 21:14:38 +0530 Subject: [PATCH 2/9] logging changes + retry batch requests in case of provisionedThroughput errors from DDb + version bump --- config/dynamoFactory.js | 2 ++ package.json | 2 +- services/dynamodb/Base.js | 6 ++---- services/dynamodb/BatchGet.js | 10 +++++----- services/dynamodb/BatchWrite.js | 15 ++++++++++----- services/dynamodb/WaitFor.js | 2 -- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/config/dynamoFactory.js b/config/dynamoFactory.js index 4c6a703..dbf3823 100644 --- a/config/dynamoFactory.js +++ b/config/dynamoFactory.js @@ -8,6 +8,7 @@ const AWS = require('aws-sdk'), AWSDaxClient = require('amazon-dax-client'), OSTBase = require('@ostdotcom/base'), coreConstant = require(rootPrefix + '/config/coreConstant'), + logger = require(rootPrefix + '/lib/logger/customConsoleLogger'), util = require(rootPrefix + '/lib/util'); const InstanceComposer = OSTBase.InstanceComposer; @@ -74,6 +75,7 @@ DynamoConfigFactory.prototype = { logger: configStrategies.storage.enableLogging == 1 ? console : '', retryDelayOptions: { customBackoff: function(retryCount) { + logger.debug(`DDB Retry customBackoff called for retryCount: ${retryCount}`); return coreConstant.fixedRetryAfterTime() + retryCount * coreConstant.variableRetryAfterTime(); } }, diff --git a/package.json b/package.json index 206683c..1c966f9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ostdotcom/storage", - "version": "1.0.3", + "version": "1.0.4-beta.1", "description": "OST storage provides data storage and sharding services.", "main": "index.js", "scripts": { diff --git a/services/dynamodb/Base.js b/services/dynamodb/Base.js index 9acb31b..87ac539 100644 --- a/services/dynamodb/Base.js +++ b/services/dynamodb/Base.js @@ -64,13 +64,11 @@ DDBServiceBase.prototype = { const oThis = this; let r = oThis.validateParams(); - logger.debug('=======Base.validateParams.result======='); - logger.debug(r); + if (r.isFailure()) return r; r = await oThis.executeDdbRequest(); - logger.debug('=======Base.executeDdbRequest.result======='); - logger.debug(r); + return r; }, diff --git a/services/dynamodb/BatchGet.js b/services/dynamodb/BatchGet.js index 6335ac8..09ce899 100644 --- a/services/dynamodb/BatchGet.js +++ b/services/dynamodb/BatchGet.js @@ -80,9 +80,12 @@ const batchGetPrototype = { localResponse = await oThis.batchGetItemAfterWait(batchGetParams, waitTime); if (!localResponse.isSuccess()) { - if (localResponse.internalErrorCode.includes('ResourceNotFoundException')) { + if ( + localResponse.internalErrorCode.includes('ResourceNotFoundException') || + localResponse.internalErrorCode.includes('ProvisionedThroughputExceededException') + ) { logger.error( - 'services/dynamodb/BatchGet.js:executeDdbRequest, ResourceNotFoundException : attemptNo: ', + `services/dynamodb/BatchGet.js:executeDdbRequest, ${localResponse.internalErrorCode} : attemptNo: `, attemptNo ); localResponse.data['UnprocessedKeys'] = batchGetParams['RequestItems']; @@ -161,9 +164,6 @@ const batchGetPrototype = { } } - logger.debug('=======Base.perform.result======='); - logger.debug(globalResponse); - return globalResponse; } catch (err) { logger.error('services/dynamodb/BatchGet.js:executeDdbRequest inside catch ', err); diff --git a/services/dynamodb/BatchWrite.js b/services/dynamodb/BatchWrite.js index a3ca526..4c7448b 100644 --- a/services/dynamodb/BatchWrite.js +++ b/services/dynamodb/BatchWrite.js @@ -74,9 +74,12 @@ const batchWritePrototype = { response = await oThis.batchWriteItemAfterWait(batchWriteParams, waitTime); if (!response.isSuccess()) { - if (response.internalErrorCode.includes('ResourceNotFoundException')) { + if ( + response.internalErrorCode.includes('ResourceNotFoundException') || + response.internalErrorCode.includes('ProvisionedThroughputExceededException') + ) { logger.error( - 'services/dynamodb/BatchWrite.js:executeDdbRequest, ResourceNotFoundException : attemptNo: ', + `services/dynamodb/BatchWrite.js:executeDdbRequest, ${response.internalErrorCode} : attemptNo: `, attemptNo ); response.data['UnprocessedItems'] = batchWriteParams['RequestItems']; @@ -139,8 +142,6 @@ const batchWritePrototype = { } } - logger.debug('=======Base.perform.result======='); - logger.debug(response); return response; } catch (err) { logger.error('services/dynamodb/BatchWrite.js:executeDdbRequest inside catch ', err); @@ -177,6 +178,10 @@ const batchWritePrototype = { Object.assign(DDBServiceBatchWriteItem.prototype, batchWritePrototype); DDBServiceBatchWriteItem.prototype.constructor = batchWritePrototype; -InstanceComposer.registerAsShadowableClass(DDBServiceBatchWriteItem, coreConstant.icNameSpace, 'DDBServiceBatchWriteItem'); +InstanceComposer.registerAsShadowableClass( + DDBServiceBatchWriteItem, + coreConstant.icNameSpace, + 'DDBServiceBatchWriteItem' +); module.exports = DDBServiceBatchWriteItem; diff --git a/services/dynamodb/WaitFor.js b/services/dynamodb/WaitFor.js index 0ea499f..15698aa 100644 --- a/services/dynamodb/WaitFor.js +++ b/services/dynamodb/WaitFor.js @@ -68,8 +68,6 @@ const waitForPrototype = { .ic() .getInstanceFor(coreConstant.icNameSpace, 'libDynamoDBBase') .queryDdb(oThis.methodName, 'raw', oThis.waitForMethod, oThis.params); - logger.debug('=======Base.perform.result======='); - logger.debug(r); return r; } catch (err) { logger.error('services/dynamodb/Base.js:executeDdbRequest inside catch ', err); From 127ad4e99bfb7208086d21f357dae1e7eaab03ce Mon Sep 17 00:00:00 2001 From: AnaghaMurtarkar Date: Tue, 22 Oct 2019 19:07:51 +0530 Subject: [PATCH 3/9] Updated ost-base package. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1c966f9..d029f68 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ }, "homepage": "https://github.com/ostdotcom/storage#readme", "dependencies": { - "@ostdotcom/base": "1.0.0", + "@ostdotcom/base": "1.0.1-beta.1", "aws-sdk": "2.263.1", "amazon-dax-client": "1.1.3", "bignumber.js": "4.1.0" From 8eb5ae2d91c6d9ab9aed870e2b8e445e4fd00c14 Mon Sep 17 00:00:00 2001 From: AnaghaMurtarkar Date: Tue, 22 Oct 2019 19:09:06 +0530 Subject: [PATCH 4/9] Updated travis.yml for node 10 and 12. --- .travis.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 273dec1..8cc83d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,15 +17,16 @@ env: - OS_CACHING_ENGINE=memcached - OST_MEMCACHE_SERVERS=127.0.0.1:11211 node_js: - - "8" + - "12" + - "10" services: - memcached before_install: - - sudo apt-get update - - sudo apt-get install nodejs - - sudo apt-get install npm + - sudo apt-get update + - sudo apt-get install nodejs + - sudo apt-get install npm install: - - npm install + - npm install before_script: - mkdir -p dynamodb - wget https://s3.ap-south-1.amazonaws.com/dynamodb-local-mumbai/dynamodb_local_latest.zip -O ./dynamodb/dynamodb_local_latest.zip @@ -39,4 +40,4 @@ script: after_script: - kill $(ps aux | grep 'DynamoDBLocal.jar' | awk '{print $2}') - rm -rf dynamodb - - rm -rf dynamodb_instance_2 \ No newline at end of file + - rm -rf dynamodb_instance_2 From d44bba7ef419b25a4fad81342785c257454c8395 Mon Sep 17 00:00:00 2001 From: AnaghaMurtarkar Date: Tue, 22 Oct 2019 19:16:23 +0530 Subject: [PATCH 5/9] Version bump. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d029f68..43006f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ostdotcom/storage", - "version": "1.0.4-beta.1", + "version": "1.0.4-beta.2", "description": "OST storage provides data storage and sharding services.", "main": "index.js", "scripts": { From 75e71da92d284ad8b8b85838e57c44612e7703b2 Mon Sep 17 00:00:00 2001 From: Shlok Gilda Date: Fri, 8 Nov 2019 16:08:58 +0530 Subject: [PATCH 6/9] Update package.json --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 43006f3..ee5c962 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ostdotcom/storage", - "version": "1.0.4-beta.2", + "version": "1.0.4-beta.3", "description": "OST storage provides data storage and sharding services.", "main": "index.js", "scripts": { @@ -24,7 +24,7 @@ }, "homepage": "https://github.com/ostdotcom/storage#readme", "dependencies": { - "@ostdotcom/base": "1.0.1-beta.1", + "@ostdotcom/base": "^2.0.0", "aws-sdk": "2.263.1", "amazon-dax-client": "1.1.3", "bignumber.js": "4.1.0" From 651d7a8d4f275189d8d2b2bf3bf338964cc89630 Mon Sep 17 00:00:00 2001 From: Shlok Gilda Date: Fri, 8 Nov 2019 16:09:31 +0530 Subject: [PATCH 7/9] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 21e8796..ee90284 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.3 +1.0.4 From 9219e56840aa9f0c0531d7e10b9f78747192086f Mon Sep 17 00:00:00 2001 From: Kedar Chandrayan Date: Fri, 8 Nov 2019 16:17:40 +0530 Subject: [PATCH 8/9] Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8efd0e1..bd3e15d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Storage v1.0.4 +- Upgraded node version to 10.x +- Version bump for dependencies. + ## Storage v1.0.3 - Integrated with new Instance Composer. - Migrated to ES6. @@ -23,4 +27,4 @@ - Auto Scale services to scale read/write capacity of DynamoDB tables. - Cache layer on top of Shard management services. - Model layer for token_balances and transaction_logs to support respective queries to DynamoDB. -- Cache layer on top of token_balances. \ No newline at end of file +- Cache layer on top of token_balances. From 65324ddf3d2c34112467d9e031cb322c6ba938ad Mon Sep 17 00:00:00 2001 From: Kedar Chandrayan Date: Fri, 8 Nov 2019 16:20:07 +0530 Subject: [PATCH 9/9] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ee5c962..01b8bcf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ostdotcom/storage", - "version": "1.0.4-beta.3", + "version": "1.0.4", "description": "OST storage provides data storage and sharding services.", "main": "index.js", "scripts": {