From b81d4559c659f4011ac31ac5b9c3ff8c54e5525f Mon Sep 17 00:00:00 2001 From: nkokkera Date: Tue, 29 Aug 2023 18:15:07 -0400 Subject: [PATCH 1/6] EGDM-3362 - Datapull Jobs fails on duplicate subnet id --- .../java/com/homeaway/datapullclient/process/DataPullTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java b/api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java index b078d27..0f2ebd7 100644 --- a/api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java +++ b/api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java @@ -397,7 +397,7 @@ private JobFlowInstancesConfig getJobFlowInstancesConfig(EMRProperties emrProper .withInstanceTypeConfigs(workerInstanceTypeConfig) .withTargetOnDemandCapacity(count); - List subnetIds= new ArrayList<>(); + Set subnetIds= new HashSet<>(); subnetIds.addAll(toList(new String[]{dataPullProperties.getApplicationSubnet1(), dataPullProperties.getApplicationSubnet2()})); From 23b93e7b9be0d20f02b32dfa78605d1a4d87945f Mon Sep 17 00:00:00 2001 From: nkokkera Date: Mon, 4 Sep 2023 21:51:54 -0400 Subject: [PATCH 2/6] EGDM-3362 - Datapull Jobs fails on duplicate subnet id --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e83f8..a5ac11a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [0.1.70] - 2023-09-04 +Fixed bug of Jobs fails on duplicate subnet id + ## [0.1.69] - 2023-06-07 we have added partitions for every batch we write to s3 to avoid file already existing errors. and added logging for run job flow request response. From 06c85800d585e39ee293c167aaa9e0c0c1e9e3f3 Mon Sep 17 00:00:00 2001 From: nkokkera Date: Wed, 6 Sep 2023 09:45:11 -0400 Subject: [PATCH 3/6] EGDM-3374-Unable to update customer data with Data Pull. --- core/src/main/scala/core/DataFrameFromTo.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/core/DataFrameFromTo.scala b/core/src/main/scala/core/DataFrameFromTo.scala index cbddcfe..5755846 100644 --- a/core/src/main/scala/core/DataFrameFromTo.scala +++ b/core/src/main/scala/core/DataFrameFromTo.scala @@ -1277,7 +1277,7 @@ class DataFrameFromTo(appConfig: AppConfig, pipeline: String) extends Serializab // create the statement, and run the command val statement = connection.createStatement() - if (colType != null) { + if (colType.isDefined) { resultSet= statement.executeQuery(sql_command) } From fc3380f1265ba7bcfff1c6de33729b1913524135 Mon Sep 17 00:00:00 2001 From: nkokkera Date: Tue, 12 Sep 2023 20:25:27 -0400 Subject: [PATCH 4/6] EGDM-3362 - Datapull Jobs fails on duplicate subnet id --- core/src/main/scala/core/DataFrameFromTo.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/core/DataFrameFromTo.scala b/core/src/main/scala/core/DataFrameFromTo.scala index 5755846..cbddcfe 100644 --- a/core/src/main/scala/core/DataFrameFromTo.scala +++ b/core/src/main/scala/core/DataFrameFromTo.scala @@ -1277,7 +1277,7 @@ class DataFrameFromTo(appConfig: AppConfig, pipeline: String) extends Serializab // create the statement, and run the command val statement = connection.createStatement() - if (colType.isDefined) { + if (colType != null) { resultSet= statement.executeQuery(sql_command) } From 81d984037f54d2212d0d2173a03e349d5ff5d0f5 Mon Sep 17 00:00:00 2001 From: nkokkera Date: Tue, 12 Sep 2023 20:37:11 -0400 Subject: [PATCH 5/6] EGDM-3374 - Unable to update customer data with Data Pull. --- core/src/main/scala/core/DataFrameFromTo.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/core/DataFrameFromTo.scala b/core/src/main/scala/core/DataFrameFromTo.scala index cbddcfe..5755846 100644 --- a/core/src/main/scala/core/DataFrameFromTo.scala +++ b/core/src/main/scala/core/DataFrameFromTo.scala @@ -1277,7 +1277,7 @@ class DataFrameFromTo(appConfig: AppConfig, pipeline: String) extends Serializab // create the statement, and run the command val statement = connection.createStatement() - if (colType != null) { + if (colType.isDefined) { resultSet= statement.executeQuery(sql_command) } From 3adbeb3bd649aa1d8874ff216296c8047d6ae8bc Mon Sep 17 00:00:00 2001 From: nkokkera Date: Tue, 12 Sep 2023 20:41:39 -0400 Subject: [PATCH 6/6] EGDM-3374 - Unable to update customer data with Data Pull. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ac11a..17a603f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html) +## [0.1.71] - 2023-09-12 +Fixed bug of Unable to update customer data with Data Pull. + ## [0.1.70] - 2023-09-04 Fixed bug of Jobs fails on duplicate subnet id