From 2088b8fcedf37dbbb3d2d753f4dff55bbffc7af0 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 1 Dec 2025 12:36:33 +0000 Subject: [PATCH 01/11] Construct the test set up for v2 --- atom-publisher-lib/build.sbt | 3 +- .../com/gu/atom/data/AtomSerializer.scala | 13 ++ .../com/gu/atom/data/DynamoDataStore.scala | 6 - .../com/gu/atom/data/DynamoDataStoreV2.scala | 177 ++++++++++++++++++ .../gu/atom/data/DynamoDataStoreV2Spec.scala | 143 ++++++++++++++ .../com/gu/atom/data/LocalDynamoDBV2.scala | 52 +++++ 6 files changed, 387 insertions(+), 7 deletions(-) create mode 100644 atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala create mode 100644 atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala create mode 100644 atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala create mode 100644 atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala diff --git a/atom-publisher-lib/build.sbt b/atom-publisher-lib/build.sbt index 6a02e06..0899704 100644 --- a/atom-publisher-lib/build.sbt +++ b/atom-publisher-lib/build.sbt @@ -26,5 +26,6 @@ libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % Test, "org.scalatestplus" %% "mockito-4-6" % "3.2.14.0" % Test, "org.scalatest" %% "scalatest" % "3.2.14" % Test, - "software.amazon.awssdk" % "kinesis" % "2.39.4" + "software.amazon.awssdk" % "kinesis" % "2.39.4", + "software.amazon.awssdk" % "dynamodb" % "2.39.4" ) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala new file mode 100644 index 0000000..c28368e --- /dev/null +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala @@ -0,0 +1,13 @@ +package com.gu.atom.data + +import com.gu.contentatom.thrift.Atom +import io.circe.Json +import io.circe.syntax._ +import com.gu.fezziwig.CirceScroogeMacros.encodeThriftStruct +import io.circe.syntax._ +import com.gu.fezziwig.CirceScroogeMacros.{encodeThriftStruct, encodeThriftUnion} +import com.gu.atom.util.JsonSupport.{backwardsCompatibleAtomDecoder, thriftEnumEncoder} +object AtomSerializer { + + def toJson(newAtom: Atom): Json = newAtom.asJson +} \ No newline at end of file diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStore.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStore.scala index 38a1bcf..59bb18a 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStore.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStore.scala @@ -8,7 +8,6 @@ import com.amazonaws.services.dynamodbv2.model.{ConditionalCheckFailedException, import com.amazonaws.{AmazonClientException, AmazonServiceException} import com.gu.contentatom.thrift.Atom import cats.implicits._ -import cats.syntax.either._ import io.circe._ import io.circe.syntax._ import com.gu.fezziwig.CirceScroogeMacros.{encodeThriftStruct, encodeThriftUnion} @@ -17,11 +16,6 @@ import com.gu.atom.util.JsonSupport.{backwardsCompatibleAtomDecoder, thriftEnumE import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} -object AtomSerializer { - - def toJson(newAtom: Atom): Json = newAtom.asJson -} - abstract class DynamoDataStore (dynamo: AmazonDynamoDB, tableName: String) extends AtomDataStore { diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala new file mode 100644 index 0000000..842cbdf --- /dev/null +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -0,0 +1,177 @@ +package com.gu.atom.data + +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient +import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse +import software.amazon.awssdk.services.dynamodb.model.ItemResponse +import software.amazon.awssdk.awscore.exception.AwsServiceException +import com.gu.contentatom.thrift.Atom +import cats.implicits._ +import io.circe._ +import com.gu.atom.util.JsonSupport.backwardsCompatibleAtomDecoder +import software.amazon.awssdk.core.exception.SdkException + +abstract class DynamoDataStoreV2 + (dynamo: DynamoDbAsyncClient, tableName: String) + extends AtomDataStore { + +// private val dynamoDB = DynamoDbClient.builder().build() +// private val table = dynamo.(tableName) + + private val SimpleKeyName = "id" + private object CompositeKey { + val partitionKey = "atomType" + val sortKey = "id" + } + + import AtomSerializer._ + + protected def get(key: DynamoCompositeKey): DataStoreResult[Json] = { +// Try { +// val result = table.getItem(uniqueKey(key)) +// Option(result) //null if not found +// +// } match { +// case Success(Some(item)) => parseJson(item.toJSON) +// case Success(None) => Left(IDNotFound) +// case Failure(e) => Left(handleException(e)) +// } + ??? + } + + protected def put(json: Json): DataStoreResult[Json] = { +// Try(table.putItem(jsonToItem(json))) match { +// case Success(_) => Right(json) +// case Failure(e) => Left(handleException(e)) +// } + ??? + } + + /** + * Conditional put, ensuring passed revision is higher than the value in dynamo + */ + protected def put(json: Json, revision: Long): DataStoreResult[Json] = { +// val expressionAttributeValues = new util.HashMap[String, Object]() +// expressionAttributeValues.put(":revision", revision.asInstanceOf[Object]) +// +// Try { +// table.putItem( +// jsonToItem(json), +// "contentChangeDetails.revision < :revision", +// null, +// expressionAttributeValues +// ) +// } match { +// case Success(_) => Right(json) +// case Failure(conditionError: ConditionalCheckFailedException) => +// Left(VersionConflictError(revision)) +// case Failure(e) => Left(handleException(e)) +// } + ??? + } + + protected def delete(key: DynamoCompositeKey): DataStoreResult[DeleteItemResponse] = { +// Try { +// key match { +// case DynamoCompositeKey(partitionKey, None) => +// table.deleteItem(SimpleKeyName, partitionKey) +// case DynamoCompositeKey(partitionKey, Some(sortKey)) => +// table.deleteItem(CompositeKey.partitionKey, partitionKey, CompositeKey.sortKey, sortKey) +// } +// } match { +// case Success(outcome) => Right(outcome.getDeleteItemResult) +// case Failure(e) => Left(handleException(e)) +// } + ??? + } + + protected def scan: DataStoreResult[List[Json]] = { +// Try { +// table.scan().iterator.asScala.toList +// +// } match { +// case Success(items) => items.traverse(item => parseJson(item.toJSON)) +// case Failure(e) => Left(DynamoError(e.getMessage)) +// } + ??? + } + + private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey) = ??? +// dynamoCompositeKey match { +// case DynamoCompositeKey(partitionKey, None) => +// new PrimaryKey(SimpleKeyName, partitionKey) +// case DynamoCompositeKey(partitionKey, Some(sortKey)) => +// new PrimaryKey(CompositeKey.partitionKey, partitionKey, CompositeKey.sortKey, sortKey) +// } + + def parseJson(s: String): DataStoreResult[Json] = + parser.parse(s).leftMap(parsingFailure => DynamoError(parsingFailure.getMessage)) + + def jsonToAtom(json: Json): DataStoreResult[Atom] = + json.as[Atom](backwardsCompatibleAtomDecoder).leftMap(error => DecoderError(error.message)) + + def jsonToItem(json: Json): ItemResponse = { +// val item = new Item() +// json.asObject.foreach { obj => +// obj.toMap.map { case (key, value) => item.withJSON(key, value.noSpaces) } +// } +// item + ??? + } + + private def handleException(e: Throwable) = e match { + case serviceError: AwsServiceException => DynamoError(serviceError.awsErrorDetails().errorMessage) + case clientError: SdkException => ClientError(clientError.getMessage) + case other => ReadError + } + + def getAtom(id: String): DataStoreResult[Atom] = getAtom(DynamoCompositeKey(id)) + + def getAtom(dynamoCompositeKey: DynamoCompositeKey): DataStoreResult[Atom] = + get(dynamoCompositeKey) flatMap jsonToAtom + + def createAtom(atom: Atom): DataStoreResult[Atom] = createAtom(DynamoCompositeKey(atom.id), atom) + + def createAtom(dynamoCompositeKey: DynamoCompositeKey, atom: Atom): DataStoreResult[Atom] = { + getAtom(dynamoCompositeKey) match { + case Right(_) => + Left(IDConflictError) + case Left(error) => + put(toJson(atom)).map(_ => atom) + } + } + + def deleteAtom(id: String): DataStoreResult[Atom] = deleteAtom(DynamoCompositeKey(id)) + + def deleteAtom(dynamoCompositeKey: DynamoCompositeKey): DataStoreResult[Atom] = + getAtom(dynamoCompositeKey).flatMap { atom => + delete(dynamoCompositeKey).map(_ => atom) + } + + private def findAtoms(tableName: String): DataStoreResult[List[Atom]] = + scan.flatMap(_.traverse(jsonToAtom)) + + def listAtoms: DataStoreResult[List[Atom]] = findAtoms(tableName) + +} + +class PreviewDynamoDataStoreV2 +(dynamo: DynamoDbAsyncClient, tableName: String) + extends DynamoDataStoreV2(dynamo, tableName) + with PreviewDataStore { + + import AtomSerializer._ + + def updateAtom(newAtom: Atom) = + put(toJson(newAtom), newAtom.contentChangeDetails.revision).map(_ => newAtom) +} + +class PublishedDynamoDataStoreV2 +(dynamo: DynamoDbAsyncClient, tableName: String) + extends DynamoDataStoreV2(dynamo, tableName) + with PublishedDataStore { + + import AtomSerializer._ + + def updateAtom(newAtom: Atom) = put(toJson(newAtom)).map(_ => newAtom) +} + diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala new file mode 100644 index 0000000..2675bc6 --- /dev/null +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -0,0 +1,143 @@ +package com.gu.atom.data + +import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType._ +import com.gu.atom.TestData._ +import com.gu.atom.util.{AtomImplicitsGeneral, JsonSupport} +import com.gu.contentatom.thrift.Atom +import org.scalatest.funspec.FixtureAnyFunSpec +import org.scalatest.matchers.should._ +import org.scalatest.{BeforeAndAfterAll, OptionValues} +import software.amazon.awssdk.services.dynamodb.model.{KeyType, ScalarAttributeType} + +class DynamoDataStoreV2Spec + extends FixtureAnyFunSpec + with Matchers + with OptionValues + with BeforeAndAfterAll + with AtomImplicitsGeneral { + + val tableName = "atom-test-table" + val publishedTableName = "published-atom-test-table" + val compositeKeyTableName = "composite-key-table" + + case class DataStoresV2(preview: PreviewDynamoDataStoreV2, + published: PublishedDynamoDataStoreV2, + compositeKey: PreviewDynamoDataStoreV2 + ) + + type FixtureParam = DataStoresV2 + + def withFixture(test: OneArgTest) = { + val previewDb = new PreviewDynamoDataStoreV2(LocalDynamoDBV2.client(), tableName) + val compositeKeyDb = new PreviewDynamoDataStoreV2(LocalDynamoDBV2.client(), compositeKeyTableName) + val publishedDb = new PublishedDynamoDataStoreV2(LocalDynamoDBV2.client(), publishedTableName) + super.withFixture(test.toNoArgTest(DataStoresV2(previewDb, publishedDb, compositeKeyDb))) + } + + describe("DynamoDataStore") { + it("should create a new atom") { dataStores => + dataStores.preview.createAtom(testAtom) should equal(Right(testAtom)) + } + + it("should list all atoms of all types") { dataStores => + dataStores.preview.createAtom(testAtoms(1)) + dataStores.preview.createAtom(testAtoms(2)) + dataStores.preview.listAtoms.map(_.toList).fold(identity, res => res should contain theSameElementsAs testAtoms) + } + + it("should return the atom") { dataStores => + dataStores.preview.getAtom(testAtom.id) should equal(Right(testAtom)) + } + + it("should update the atom") { dataStores => + val updated = testAtom + .copy(defaultHtml = "
updated
") + .bumpRevision + + dataStores.preview.updateAtom(updated) should equal(Right(updated)) + dataStores.preview.getAtom(testAtom.id) should equal(Right(updated)) + } + + it("should update a published atom") { dataStores => + val updated = testAtom + .copy() + .withRevision(1) + + dataStores.published.updateAtom(updated) should equal(Right(updated)) + dataStores.published.getAtom(testAtom.id) should equal(Right(updated)) + } + + it("should create the atom with composite key") { dataStores => + dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) + } + + it("should return the atom with composite key") { dataStores => + dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) + } + + it("should update an atom with composite key") { dataStores => + val updated = testAtom + .copy(defaultHtml = "
updated
") + .bumpRevision + + dataStores.compositeKey.updateAtom(updated) should equal(Right(updated)) + dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(updated)) + } + + it("should delete an atom if it exists in the table") { dataStores => + dataStores.preview.createAtom(testAtomForDeletion) should equal(Right(testAtomForDeletion)) + dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal(Right(testAtomForDeletion)) + } + + it("should delete an atom with composite key if it exists in the table") { dataStores => + val key = DynamoCompositeKey(testAtomForDeletion.atomType.toString, Some(testAtomForDeletion.id)) + dataStores.compositeKey.createAtom(key, testAtomForDeletion) should equal(Right(testAtomForDeletion)) + dataStores.compositeKey.deleteAtom(key) should equal(Right(testAtomForDeletion)) + } + + it("should decode the old format from dynamo") { dataStores => + val json = dataStores.published.parseJson( + """ + |{ + | "defaultHtml" : "
", + | "data" : { + | "assets" : [ + | { + | "id" : "xyzzy", + | "version" : 1, + | "platform" : "Youtube", + | "assetType" : "Video" + | }, + | { + | "id" : "fizzbuzz", + | "version" : 2, + | "platform" : "Youtube", + | "assetType" : "Video" + | } + | ], + | "activeVersion" : 2, + | "title" : "Test atom 1", + | "category" : "News" + | }, + | "contentChangeDetails" : { + | "revision" : 1 + | }, + | "id" : "1", + | "atomType" : "Media", + | "labels" : [ + | ] + |} + """.stripMargin).toOption.get + + val atom = json.as[Atom](JsonSupport.backwardsCompatibleAtomDecoder) + atom should equal(Right(testAtom)) + } + } + + override def beforeAll() = { + val client = LocalDynamoDBV2.client() + LocalDynamoDBV2.createTable(client)(tableName)(KeyType.HASH -> "id") + LocalDynamoDBV2.createTable(client)(publishedTableName)(KeyType.HASH -> "id") + LocalDynamoDBV2.createTable(client)(compositeKeyTableName)(KeyType.HASH -> "atomType", KeyType.RANGE -> "id") + } +} diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala new file mode 100644 index 0000000..2932f01 --- /dev/null +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala @@ -0,0 +1,52 @@ +package com.gu.atom.data + +import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient +import software.amazon.awssdk.services.dynamodb.model.{AttributeDefinition, CreateTableRequest, KeySchemaElement, KeyType, ScalarAttributeType} + +import java.net.URI +import scala.jdk.CollectionConverters._ + +/* + * copied from: + * https://github.com/guardian/scanamo/blob/master/src/test/scala/com/gu/scanamo/LocalDynamoDB.scala + */ + +object LocalDynamoDBV2 { + def client() = { + DynamoDbAsyncClient.builder() + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("key", "secret") + )) + .endpointOverride(URI.create("http://localhost:8000")) + .region(Region.EU_WEST_1) + .build() + } + + def createTable(client: DynamoDbAsyncClient)(tableName: String)(attributes: (KeyType, String)*) = { + val attrs = attributes.toList.map { case(kt, attrName) => KeySchemaElement.builder().keyType(kt).attributeName(attrName).build()} + val attributeDefinitions = attributes.toList.map { case(at, attrName) => AttributeDefinition.builder().attributeType(ScalarAttributeType.S).attributeName(attrName).build() } + + val createTableRequest = CreateTableRequest.builder() + .tableName(tableName) + .keySchema(attrs.asJava) + .attributeDefinitions(attributeDefinitions.asJava) + .build() + client.createTable(createTableRequest) + } + + + +// private def keySchema(attributes: Seq[(Symbol, ScalarAttributeType)]) = { +// val hashKeyWithType :: rangeKeyWithType = attributes.toList +// val keySchemas = hashKeyWithType._1 -> KeyType.HASH :: rangeKeyWithType.map(_._1 -> KeyType.RANGE) +// keySchemas.map{ case (symbol, keyType) => new KeySchemaElement(symbol.name, keyType)}.asJava +// } +// +// private def attributeDefinitions(attributes: Seq[(Symbol, ScalarAttributeType)]) = { +// attributes.map{ case (symbol, attributeType) => new AttributeDefinition(symbol.name, attributeType)}.asJava +// } +// +// private val arbitraryThroughputThatIsIgnoredByDynamoDBLocal = new ProvisionedThroughput(1L, 1L) +} From a4567ca577a4580859fcf3549ea49eda0cc81082 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 1 Dec 2025 19:16:44 +0000 Subject: [PATCH 02/11] Use enhanced client to get the doc --- atom-publisher-lib/build.sbt | 3 +- .../com/gu/atom/data/DynamoDataStoreV2.scala | 94 ++++++--- .../gu/atom/data/DynamoDataStoreV2Spec.scala | 194 +++++++++--------- .../com/gu/atom/data/LocalDynamoDBV2.scala | 14 +- 4 files changed, 179 insertions(+), 126 deletions(-) diff --git a/atom-publisher-lib/build.sbt b/atom-publisher-lib/build.sbt index 0899704..44db2d3 100644 --- a/atom-publisher-lib/build.sbt +++ b/atom-publisher-lib/build.sbt @@ -27,5 +27,6 @@ libraryDependencies ++= Seq( "org.scalatestplus" %% "mockito-4-6" % "3.2.14.0" % Test, "org.scalatest" %% "scalatest" % "3.2.14" % Test, "software.amazon.awssdk" % "kinesis" % "2.39.4", - "software.amazon.awssdk" % "dynamodb" % "2.39.4" + "software.amazon.awssdk" % "dynamodb" % "2.39.6", + "software.amazon.awssdk" % "dynamodb-enhanced" % "2.39.6", ) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index 842cbdf..981740d 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -1,21 +1,41 @@ package com.gu.atom.data -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.dynamodb.model.DeleteItemResponse -import software.amazon.awssdk.services.dynamodb.model.ItemResponse +import software.amazon.awssdk.services.dynamodb.{DynamoDbAsyncClient, DynamoDbClient} +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, DeleteItemResponse, DescribeTableRequest, GetItemRequest, ItemResponse} import software.amazon.awssdk.awscore.exception.AwsServiceException import com.gu.contentatom.thrift.Atom import cats.implicits._ import io.circe._ import com.gu.atom.util.JsonSupport.backwardsCompatibleAtomDecoder +import io.circe.syntax.EncoderOps import software.amazon.awssdk.core.exception.SdkException +import software.amazon.awssdk.enhanced.dynamodb.TableMetadata.primaryIndexName +import software.amazon.awssdk.enhanced.dynamodb.model.GetItemEnhancedRequest +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} + +import scala.jdk.CollectionConverters.MapHasAsJava +import scala.util.{Failure, Success, Try} abstract class DynamoDataStoreV2 - (dynamo: DynamoDbAsyncClient, tableName: String) + (dynamo: DynamoDbClient, tableName: String) extends AtomDataStore { -// private val dynamoDB = DynamoDbClient.builder().build() -// private val table = dynamo.(tableName) + lazy val ddb: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(dynamo).build() + + lazy val tableSchema1 = TableSchema.documentSchemaBuilder() + .addIndexPartitionKey(TableMetadata.primaryIndexName(), "id", AttributeValueType.S) + .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) + .build() + + val table1 = ddb.table(tableName, tableSchema1) + + lazy val tableSchema2 = TableSchema.documentSchemaBuilder() + .addIndexPartitionKey(TableMetadata.primaryIndexName(), CompositeKey.partitionKey, AttributeValueType.S) + .addIndexSortKey("commission-index", CompositeKey.sortKey, AttributeValueType.S) + .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) + .build() + + val table2 = ddb.table(tableName, tableSchema2) private val SimpleKeyName = "id" private object CompositeKey { @@ -26,16 +46,15 @@ abstract class DynamoDataStoreV2 import AtomSerializer._ protected def get(key: DynamoCompositeKey): DataStoreResult[Json] = { -// Try { -// val result = table.getItem(uniqueKey(key)) -// Option(result) //null if not found -// -// } match { -// case Success(Some(item)) => parseJson(item.toJSON) -// case Success(None) => Left(IDNotFound) -// case Failure(e) => Left(handleException(e)) -// } - ??? + Try { + Option(getTableToQuery(key).getItem(uniqueKey(key))) + } match { + case Success(Some(item)) => parseJson(item.toJson) + case Success(None) => Left(IDNotFound) + case Failure(e) => Left(handleException(e)) + } + + } protected def put(json: Json): DataStoreResult[Json] = { @@ -95,13 +114,33 @@ abstract class DynamoDataStoreV2 ??? } - private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey) = ??? -// dynamoCompositeKey match { -// case DynamoCompositeKey(partitionKey, None) => -// new PrimaryKey(SimpleKeyName, partitionKey) -// case DynamoCompositeKey(partitionKey, Some(sortKey)) => -// new PrimaryKey(CompositeKey.partitionKey, partitionKey, CompositeKey.sortKey, sortKey) + +// private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Map[String, AttributeValue] = dynamoCompositeKey match { +// case DynamoCompositeKey(partitionKey, None) => { +// Map(SimpleKeyName -> AttributeValue.builder().s(partitionKey).build()) +// } +// +// case DynamoCompositeKey(partitionKey, Some(sortKey)) =>{ +// Map( +// CompositeKey.partitionKey -> AttributeValue.builder().s(partitionKey).build(), +// CompositeKey.sortKey -> AttributeValue.builder().s(sortKey).build() +// ) +// } // } + private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Key = dynamoCompositeKey match { + case DynamoCompositeKey(partitionKey, None) => { + Key.builder().partitionValue(partitionKey).build() + } + + case DynamoCompositeKey(partitionKey, Some(sortKey)) =>{ + Key.builder().partitionValue(partitionKey).addSortValue(sortKey).build() + } + } + + private def getTableToQuery(dynamoCompositeKey: DynamoCompositeKey) = dynamoCompositeKey match { + case DynamoCompositeKey(_, None) => table1 + case DynamoCompositeKey(_, Some(_)) => table2 + } def parseJson(s: String): DataStoreResult[Json] = parser.parse(s).leftMap(parsingFailure => DynamoError(parsingFailure.getMessage)) @@ -120,14 +159,17 @@ abstract class DynamoDataStoreV2 private def handleException(e: Throwable) = e match { case serviceError: AwsServiceException => DynamoError(serviceError.awsErrorDetails().errorMessage) - case clientError: SdkException => ClientError(clientError.getMessage) + case clientError: SdkException => { + ClientError(clientError.getMessage) + } case other => ReadError } def getAtom(id: String): DataStoreResult[Atom] = getAtom(DynamoCompositeKey(id)) - def getAtom(dynamoCompositeKey: DynamoCompositeKey): DataStoreResult[Atom] = + def getAtom(dynamoCompositeKey: DynamoCompositeKey): DataStoreResult[Atom] = { get(dynamoCompositeKey) flatMap jsonToAtom + } def createAtom(atom: Atom): DataStoreResult[Atom] = createAtom(DynamoCompositeKey(atom.id), atom) @@ -155,7 +197,7 @@ abstract class DynamoDataStoreV2 } class PreviewDynamoDataStoreV2 -(dynamo: DynamoDbAsyncClient, tableName: String) +(dynamo: DynamoDbClient, tableName: String) extends DynamoDataStoreV2(dynamo, tableName) with PreviewDataStore { @@ -166,7 +208,7 @@ class PreviewDynamoDataStoreV2 } class PublishedDynamoDataStoreV2 -(dynamo: DynamoDbAsyncClient, tableName: String) +(dynamo: DynamoDbClient, tableName: String) extends DynamoDataStoreV2(dynamo, tableName) with PublishedDataStore { diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala index 2675bc6..adf7f73 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -1,6 +1,5 @@ package com.gu.atom.data -import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType._ import com.gu.atom.TestData._ import com.gu.atom.util.{AtomImplicitsGeneral, JsonSupport} import com.gu.contentatom.thrift.Atom @@ -35,109 +34,114 @@ class DynamoDataStoreV2Spec } describe("DynamoDataStore") { - it("should create a new atom") { dataStores => - dataStores.preview.createAtom(testAtom) should equal(Right(testAtom)) - } - - it("should list all atoms of all types") { dataStores => - dataStores.preview.createAtom(testAtoms(1)) - dataStores.preview.createAtom(testAtoms(2)) - dataStores.preview.listAtoms.map(_.toList).fold(identity, res => res should contain theSameElementsAs testAtoms) - } +// it("should create a new atom") { dataStores => +// dataStores.preview.createAtom(testAtom) should equal(Right(testAtom)) +// } +// +// it("should list all atoms of all types") { dataStores => +// dataStores.preview.createAtom(testAtoms(1)) +// dataStores.preview.createAtom(testAtoms(2)) +// dataStores.preview.listAtoms.map(_.toList).fold(identity, res => res should contain theSameElementsAs testAtoms) +// } it("should return the atom") { dataStores => dataStores.preview.getAtom(testAtom.id) should equal(Right(testAtom)) } - it("should update the atom") { dataStores => - val updated = testAtom - .copy(defaultHtml = "
updated
") - .bumpRevision - - dataStores.preview.updateAtom(updated) should equal(Right(updated)) - dataStores.preview.getAtom(testAtom.id) should equal(Right(updated)) - } - - it("should update a published atom") { dataStores => - val updated = testAtom - .copy() - .withRevision(1) - - dataStores.published.updateAtom(updated) should equal(Right(updated)) - dataStores.published.getAtom(testAtom.id) should equal(Right(updated)) - } - - it("should create the atom with composite key") { dataStores => - dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) - } - - it("should return the atom with composite key") { dataStores => - dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) - } - - it("should update an atom with composite key") { dataStores => - val updated = testAtom - .copy(defaultHtml = "
updated
") - .bumpRevision - - dataStores.compositeKey.updateAtom(updated) should equal(Right(updated)) - dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(updated)) - } - - it("should delete an atom if it exists in the table") { dataStores => - dataStores.preview.createAtom(testAtomForDeletion) should equal(Right(testAtomForDeletion)) - dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal(Right(testAtomForDeletion)) - } - - it("should delete an atom with composite key if it exists in the table") { dataStores => - val key = DynamoCompositeKey(testAtomForDeletion.atomType.toString, Some(testAtomForDeletion.id)) - dataStores.compositeKey.createAtom(key, testAtomForDeletion) should equal(Right(testAtomForDeletion)) - dataStores.compositeKey.deleteAtom(key) should equal(Right(testAtomForDeletion)) - } - - it("should decode the old format from dynamo") { dataStores => - val json = dataStores.published.parseJson( - """ - |{ - | "defaultHtml" : "
", - | "data" : { - | "assets" : [ - | { - | "id" : "xyzzy", - | "version" : 1, - | "platform" : "Youtube", - | "assetType" : "Video" - | }, - | { - | "id" : "fizzbuzz", - | "version" : 2, - | "platform" : "Youtube", - | "assetType" : "Video" - | } - | ], - | "activeVersion" : 2, - | "title" : "Test atom 1", - | "category" : "News" - | }, - | "contentChangeDetails" : { - | "revision" : 1 - | }, - | "id" : "1", - | "atomType" : "Media", - | "labels" : [ - | ] - |} - """.stripMargin).toOption.get - - val atom = json.as[Atom](JsonSupport.backwardsCompatibleAtomDecoder) - atom should equal(Right(testAtom)) - } +// it("should update the atom") { dataStores => +// val updated = testAtom +// .copy(defaultHtml = "
updated
") +// .bumpRevision +// +// dataStores.preview.updateAtom(updated) should equal(Right(updated)) +// dataStores.preview.getAtom(testAtom.id) should equal(Right(updated)) +// } +// +// it("should update a published atom") { dataStores => +// val updated = testAtom +// .copy() +// .withRevision(1) +// +// dataStores.published.updateAtom(updated) should equal(Right(updated)) +// dataStores.published.getAtom(testAtom.id) should equal(Right(updated)) +// } +// +// it("should create the atom with composite key") { dataStores => +// dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) +// } +// +// it("should return the atom with composite key") { dataStores => +// dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) +// } +// +// it("should update an atom with composite key") { dataStores => +// val updated = testAtom +// .copy(defaultHtml = "
updated
") +// .bumpRevision +// +// dataStores.compositeKey.updateAtom(updated) should equal(Right(updated)) +// dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(updated)) +// } +// +// it("should delete an atom if it exists in the table") { dataStores => +// dataStores.preview.createAtom(testAtomForDeletion) should equal(Right(testAtomForDeletion)) +// dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal(Right(testAtomForDeletion)) +// } +// +// it("should delete an atom with composite key if it exists in the table") { dataStores => +// val key = DynamoCompositeKey(testAtomForDeletion.atomType.toString, Some(testAtomForDeletion.id)) +// dataStores.compositeKey.createAtom(key, testAtomForDeletion) should equal(Right(testAtomForDeletion)) +// dataStores.compositeKey.deleteAtom(key) should equal(Right(testAtomForDeletion)) +// } +// +// it("should decode the old format from dynamo") { dataStores => +// val json = dataStores.published.parseJson( +// """ +// |{ +// | "defaultHtml" : "
", +// | "data" : { +// | "assets" : [ +// | { +// | "id" : "xyzzy", +// | "version" : 1, +// | "platform" : "Youtube", +// | "assetType" : "Video" +// | }, +// | { +// | "id" : "fizzbuzz", +// | "version" : 2, +// | "platform" : "Youtube", +// | "assetType" : "Video" +// | } +// | ], +// | "activeVersion" : 2, +// | "title" : "Test atom 1", +// | "category" : "News" +// | }, +// | "contentChangeDetails" : { +// | "revision" : 1 +// | }, +// | "id" : "1", +// | "atomType" : "Media", +// | "labels" : [ +// | ] +// |} +// """.stripMargin).toOption.get +// +// val atom = json.as[Atom](JsonSupport.backwardsCompatibleAtomDecoder) +// atom should equal(Right(testAtom)) +// } } - + val client = LocalDynamoDBV2.client() override def beforeAll() = { - val client = LocalDynamoDBV2.client() LocalDynamoDBV2.createTable(client)(tableName)(KeyType.HASH -> "id") LocalDynamoDBV2.createTable(client)(publishedTableName)(KeyType.HASH -> "id") LocalDynamoDBV2.createTable(client)(compositeKeyTableName)(KeyType.HASH -> "atomType", KeyType.RANGE -> "id") } + + override def afterAll(): Unit = { + LocalDynamoDBV2.deleteTable(client)(tableName) + LocalDynamoDBV2.deleteTable(client)(publishedTableName) + LocalDynamoDBV2.deleteTable(client)(compositeKeyTableName) + } } diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala index 2932f01..4f8fa01 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala @@ -1,9 +1,10 @@ package com.gu.atom.data +import net.bytebuddy.TypeCache.SimpleKey import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient -import software.amazon.awssdk.services.dynamodb.model.{AttributeDefinition, CreateTableRequest, KeySchemaElement, KeyType, ScalarAttributeType} +import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.{AttributeDefinition, CreateTableRequest, DeleteTableRequest, DeleteTableResponse, KeySchemaElement, KeyType, ProvisionedThroughput, ScalarAttributeType} import java.net.URI import scala.jdk.CollectionConverters._ @@ -15,7 +16,7 @@ import scala.jdk.CollectionConverters._ object LocalDynamoDBV2 { def client() = { - DynamoDbAsyncClient.builder() + DynamoDbClient.builder() .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create("key", "secret") )) @@ -24,7 +25,7 @@ object LocalDynamoDBV2 { .build() } - def createTable(client: DynamoDbAsyncClient)(tableName: String)(attributes: (KeyType, String)*) = { + def createTable(client: DynamoDbClient)(tableName: String)(attributes: (KeyType, String)*) = { val attrs = attributes.toList.map { case(kt, attrName) => KeySchemaElement.builder().keyType(kt).attributeName(attrName).build()} val attributeDefinitions = attributes.toList.map { case(at, attrName) => AttributeDefinition.builder().attributeType(ScalarAttributeType.S).attributeName(attrName).build() } @@ -32,10 +33,15 @@ object LocalDynamoDBV2 { .tableName(tableName) .keySchema(attrs.asJava) .attributeDefinitions(attributeDefinitions.asJava) + .provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(1L).writeCapacityUnits(1L).build()) .build() client.createTable(createTableRequest) } + def deleteTable(client: DynamoDbClient)(tableName: String): DeleteTableResponse = { + client.deleteTable(DeleteTableRequest.builder().tableName(tableName).build()) + } + // private def keySchema(attributes: Seq[(Symbol, ScalarAttributeType)]) = { From 29c07332dbc55b468ed024ac7c3d30d702bf0fdb Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 2 Dec 2025 12:22:33 +0000 Subject: [PATCH 03/11] Add in a scan simple method --- .../com/gu/atom/data/DynamoDataStoreV2.scala | 35 +++++++++++++------ .../gu/atom/data/DynamoDataStoreV2Spec.scala | 20 ++++++----- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index 981740d..e4f812b 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -7,13 +7,11 @@ import com.gu.contentatom.thrift.Atom import cats.implicits._ import io.circe._ import com.gu.atom.util.JsonSupport.backwardsCompatibleAtomDecoder -import io.circe.syntax.EncoderOps import software.amazon.awssdk.core.exception.SdkException -import software.amazon.awssdk.enhanced.dynamodb.TableMetadata.primaryIndexName -import software.amazon.awssdk.enhanced.dynamodb.model.GetItemEnhancedRequest +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} -import scala.jdk.CollectionConverters.MapHasAsJava +import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala} import scala.util.{Failure, Success, Try} abstract class DynamoDataStoreV2 @@ -58,10 +56,20 @@ abstract class DynamoDataStoreV2 } protected def put(json: Json): DataStoreResult[Json] = { -// Try(table.putItem(jsonToItem(json))) match { -// case Success(_) => Right(json) -// case Failure(e) => Left(handleException(e)) -// } + ??? + } + + + protected def putSimple(json: Json): DataStoreResult[Json] = { + Try(table1.putItem( + EnhancedDocument.builder().json(json.spaces2).build() + )) match { + case Success(_) => Right(json) + case Failure(e) => Left(handleException(e)) + } + } + + protected def putComposite(json: Json): DataStoreResult[Json] = { ??? } @@ -114,6 +122,13 @@ abstract class DynamoDataStoreV2 ??? } + protected def scanSimple: DataStoreResult[List[Json]] = { + Try { + table1.scan().iterator().asScala.toList + } match { + case Success(page) => page.flatMap(p => p.items().asScala.map(i => parseJson(i.toJson))).sequence + } + } // private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Map[String, AttributeValue] = dynamoCompositeKey match { // case DynamoCompositeKey(partitionKey, None) => { @@ -178,7 +193,7 @@ abstract class DynamoDataStoreV2 case Right(_) => Left(IDConflictError) case Left(error) => - put(toJson(atom)).map(_ => atom) + putSimple(toJson(atom)).map(_ => atom) } } @@ -190,7 +205,7 @@ abstract class DynamoDataStoreV2 } private def findAtoms(tableName: String): DataStoreResult[List[Atom]] = - scan.flatMap(_.traverse(jsonToAtom)) + scanSimple.flatMap(_.traverse(jsonToAtom)) def listAtoms: DataStoreResult[List[Atom]] = findAtoms(tableName) diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala index adf7f73..5171fde 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -34,15 +34,17 @@ class DynamoDataStoreV2Spec } describe("DynamoDataStore") { -// it("should create a new atom") { dataStores => -// dataStores.preview.createAtom(testAtom) should equal(Right(testAtom)) -// } -// -// it("should list all atoms of all types") { dataStores => -// dataStores.preview.createAtom(testAtoms(1)) -// dataStores.preview.createAtom(testAtoms(2)) -// dataStores.preview.listAtoms.map(_.toList).fold(identity, res => res should contain theSameElementsAs testAtoms) -// } + it("should create a new atom") { dataStores => + val atomCreated = dataStores.preview.createAtom(testAtom) + println(dataStores.preview.listAtoms.map(as => as.map(a => a.id))) + atomCreated should equal(Right(testAtom)) + } + + it("should list all atoms of all types") { dataStores => + dataStores.preview.createAtom(testAtoms(1)) + dataStores.preview.createAtom(testAtoms(2)) + dataStores.preview.listAtoms.map(_.toList).fold(identity, res => res should contain theSameElementsAs testAtoms) + } it("should return the atom") { dataStores => dataStores.preview.getAtom(testAtom.id) should equal(Right(testAtom)) From ceab6419f40fd0bd76015f5b9bcb074940f3c71a Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 2 Dec 2025 15:09:50 +0000 Subject: [PATCH 04/11] Add an update method --- .../com/gu/atom/data/DynamoDataStoreV2.scala | 48 +++++++++---------- .../gu/atom/data/DynamoDataStoreV2Spec.scala | 16 +++---- .../com/gu/atom/data/LocalDynamoDBV2.scala | 14 ------ 3 files changed, 32 insertions(+), 46 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index e4f812b..ee53e70 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -1,7 +1,7 @@ package com.gu.atom.data -import software.amazon.awssdk.services.dynamodb.{DynamoDbAsyncClient, DynamoDbClient} -import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, DeleteItemResponse, DescribeTableRequest, GetItemRequest, ItemResponse} +import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, DeleteItemResponse, ItemResponse, ConditionalCheckFailedException} import software.amazon.awssdk.awscore.exception.AwsServiceException import com.gu.contentatom.thrift.Atom import cats.implicits._ @@ -9,9 +9,11 @@ import io.circe._ import com.gu.atom.util.JsonSupport.backwardsCompatibleAtomDecoder import software.amazon.awssdk.core.exception.SdkException import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} +import software.amazon.awssdk.enhanced.dynamodb.Expression -import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala} +import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala, MapHasAsJava} import scala.util.{Failure, Success, Try} abstract class DynamoDataStoreV2 @@ -51,8 +53,6 @@ abstract class DynamoDataStoreV2 case Success(None) => Left(IDNotFound) case Failure(e) => Left(handleException(e)) } - - } protected def put(json: Json): DataStoreResult[Json] = { @@ -76,24 +76,24 @@ abstract class DynamoDataStoreV2 /** * Conditional put, ensuring passed revision is higher than the value in dynamo */ - protected def put(json: Json, revision: Long): DataStoreResult[Json] = { -// val expressionAttributeValues = new util.HashMap[String, Object]() -// expressionAttributeValues.put(":revision", revision.asInstanceOf[Object]) -// -// Try { -// table.putItem( -// jsonToItem(json), -// "contentChangeDetails.revision < :revision", -// null, -// expressionAttributeValues -// ) -// } match { -// case Success(_) => Right(json) -// case Failure(conditionError: ConditionalCheckFailedException) => -// Left(VersionConflictError(revision)) -// case Failure(e) => Left(handleException(e)) -// } - ??? + protected def putSimple(json: Json, revision: Long): DataStoreResult[Json] = { + val expressionAttrValues = Map[String, AttributeValue](":revision" -> AttributeValue.builder().n(revision.toString).build()) + val expression = Expression.builder().expression("contentChangeDetails.revision < :revision").expressionValues(expressionAttrValues.asJava).build() + val doc = EnhancedDocument.fromJson(json.spaces2) + val putItemRequest = PutItemEnhancedRequest + .builder(classOf[EnhancedDocument]) + .item(doc) + .conditionExpression(expression) + .build() + Try { + table1.putItem(putItemRequest) + } + match { + case Success(item) => Right(json) + case Failure(conditionError: ConditionalCheckFailedException) => + Left(VersionConflictError(revision)) + case Failure(e) => Left(handleException(e)) + } } protected def delete(key: DynamoCompositeKey): DataStoreResult[DeleteItemResponse] = { @@ -219,7 +219,7 @@ class PreviewDynamoDataStoreV2 import AtomSerializer._ def updateAtom(newAtom: Atom) = - put(toJson(newAtom), newAtom.contentChangeDetails.revision).map(_ => newAtom) + putSimple(toJson(newAtom), newAtom.contentChangeDetails.revision).map(_ => newAtom) } class PublishedDynamoDataStoreV2 diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala index 5171fde..a582c00 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -50,14 +50,14 @@ class DynamoDataStoreV2Spec dataStores.preview.getAtom(testAtom.id) should equal(Right(testAtom)) } -// it("should update the atom") { dataStores => -// val updated = testAtom -// .copy(defaultHtml = "
updated
") -// .bumpRevision -// -// dataStores.preview.updateAtom(updated) should equal(Right(updated)) -// dataStores.preview.getAtom(testAtom.id) should equal(Right(updated)) -// } + it("should update the atom") { dataStores => + val updated = testAtom + .copy(defaultHtml = "
updated
") + .bumpRevision + + dataStores.preview.updateAtom(updated) should equal(Right(updated)) + dataStores.preview.getAtom(testAtom.id) should equal(Right(updated)) + } // // it("should update a published atom") { dataStores => // val updated = testAtom diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala index 4f8fa01..359f4eb 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala @@ -1,6 +1,5 @@ package com.gu.atom.data -import net.bytebuddy.TypeCache.SimpleKey import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.dynamodb.DynamoDbClient @@ -42,17 +41,4 @@ object LocalDynamoDBV2 { client.deleteTable(DeleteTableRequest.builder().tableName(tableName).build()) } - - -// private def keySchema(attributes: Seq[(Symbol, ScalarAttributeType)]) = { -// val hashKeyWithType :: rangeKeyWithType = attributes.toList -// val keySchemas = hashKeyWithType._1 -> KeyType.HASH :: rangeKeyWithType.map(_._1 -> KeyType.RANGE) -// keySchemas.map{ case (symbol, keyType) => new KeySchemaElement(symbol.name, keyType)}.asJava -// } -// -// private def attributeDefinitions(attributes: Seq[(Symbol, ScalarAttributeType)]) = { -// attributes.map{ case (symbol, attributeType) => new AttributeDefinition(symbol.name, attributeType)}.asJava -// } -// -// private val arbitraryThroughputThatIsIgnoredByDynamoDBLocal = new ProvisionedThroughput(1L, 1L) } From 5f6c88a66d7b714dd69e9a049b9b19d0e932fa0a Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 2 Dec 2025 15:25:43 +0000 Subject: [PATCH 05/11] Add the ability to create an atom with a composite key --- .../com/gu/atom/data/DynamoDataStoreV2.scala | 25 +++++++++--------- .../gu/atom/data/DynamoDataStoreV2Spec.scala | 26 +++++++++---------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index ee53e70..095f302 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -1,7 +1,7 @@ package com.gu.atom.data import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, DeleteItemResponse, ItemResponse, ConditionalCheckFailedException} +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, ConditionalCheckFailedException, DeleteItemResponse, ItemResponse} import software.amazon.awssdk.awscore.exception.AwsServiceException import com.gu.contentatom.thrift.Atom import cats.implicits._ @@ -10,8 +10,7 @@ import com.gu.atom.util.JsonSupport.backwardsCompatibleAtomDecoder import software.amazon.awssdk.core.exception.SdkException import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest -import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} -import software.amazon.awssdk.enhanced.dynamodb.Expression +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, DynamoDbTable, Expression, Key, TableMetadata, TableSchema} import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala, MapHasAsJava} import scala.util.{Failure, Success, Try} @@ -55,13 +54,8 @@ abstract class DynamoDataStoreV2 } } - protected def put(json: Json): DataStoreResult[Json] = { - ??? - } - - - protected def putSimple(json: Json): DataStoreResult[Json] = { - Try(table1.putItem( + protected def put(json: Json, table: DynamoDbTable[EnhancedDocument]): DataStoreResult[Json] = { + Try(table.putItem( EnhancedDocument.builder().json(json.spaces2).build() )) match { case Success(_) => Right(json) @@ -69,6 +63,12 @@ abstract class DynamoDataStoreV2 } } + protected def putSimple(json: Json): DataStoreResult[Json] = { + put(json, table1) + } + + + protected def putComposite(json: Json): DataStoreResult[Json] = { ??? } @@ -193,7 +193,8 @@ abstract class DynamoDataStoreV2 case Right(_) => Left(IDConflictError) case Left(error) => - putSimple(toJson(atom)).map(_ => atom) + val table = getTableToQuery(dynamoCompositeKey) + put(toJson(atom), table).map(_ => atom) } } @@ -229,6 +230,6 @@ class PublishedDynamoDataStoreV2 import AtomSerializer._ - def updateAtom(newAtom: Atom) = put(toJson(newAtom)).map(_ => newAtom) + def updateAtom(newAtom: Atom) = putSimple(toJson(newAtom)).map(_ => newAtom) } diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala index a582c00..0006470 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -58,19 +58,19 @@ class DynamoDataStoreV2Spec dataStores.preview.updateAtom(updated) should equal(Right(updated)) dataStores.preview.getAtom(testAtom.id) should equal(Right(updated)) } -// -// it("should update a published atom") { dataStores => -// val updated = testAtom -// .copy() -// .withRevision(1) -// -// dataStores.published.updateAtom(updated) should equal(Right(updated)) -// dataStores.published.getAtom(testAtom.id) should equal(Right(updated)) -// } -// -// it("should create the atom with composite key") { dataStores => -// dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) -// } + + it("should update a published atom") { dataStores => + val updated = testAtom + .copy() + .withRevision(1) + + dataStores.published.updateAtom(updated) should equal(Right(updated)) + dataStores.published.getAtom(testAtom.id) should equal(Right(updated)) + } + + it("should create the atom with composite key") { dataStores => + dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) + } // // it("should return the atom with composite key") { dataStores => // dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) From 05cd03aa7fc3bdc6036ddfd544f65e44a879bafe Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 2 Dec 2025 16:41:36 +0000 Subject: [PATCH 06/11] Remove unneeded methods --- .../com/gu/atom/data/DynamoDataStoreV2.scala | 74 +++-------- .../gu/atom/data/DynamoDataStoreV2Spec.scala | 124 +++++++++--------- 2 files changed, 77 insertions(+), 121 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index 095f302..71b4aac 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -1,7 +1,7 @@ package com.gu.atom.data import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, ConditionalCheckFailedException, DeleteItemResponse, ItemResponse} +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, ConditionalCheckFailedException, DeleteItemRequest, DeleteItemResponse, ItemResponse} import software.amazon.awssdk.awscore.exception.AwsServiceException import com.gu.contentatom.thrift.Atom import cats.implicits._ @@ -19,10 +19,15 @@ abstract class DynamoDataStoreV2 (dynamo: DynamoDbClient, tableName: String) extends AtomDataStore { + private val SimpleKeyName = "id" + private object CompositeKey { + val partitionKey = "atomType" + val sortKey = "id" + } lazy val ddb: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(dynamo).build() lazy val tableSchema1 = TableSchema.documentSchemaBuilder() - .addIndexPartitionKey(TableMetadata.primaryIndexName(), "id", AttributeValueType.S) + .addIndexPartitionKey(TableMetadata.primaryIndexName(), SimpleKeyName, AttributeValueType.S) .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) .build() @@ -30,18 +35,12 @@ abstract class DynamoDataStoreV2 lazy val tableSchema2 = TableSchema.documentSchemaBuilder() .addIndexPartitionKey(TableMetadata.primaryIndexName(), CompositeKey.partitionKey, AttributeValueType.S) - .addIndexSortKey("commission-index", CompositeKey.sortKey, AttributeValueType.S) + .addIndexSortKey(TableMetadata.primaryIndexName(), CompositeKey.sortKey, AttributeValueType.S) .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) .build() val table2 = ddb.table(tableName, tableSchema2) - private val SimpleKeyName = "id" - private object CompositeKey { - val partitionKey = "atomType" - val sortKey = "id" - } - import AtomSerializer._ protected def get(key: DynamoCompositeKey): DataStoreResult[Json] = { @@ -66,13 +65,6 @@ abstract class DynamoDataStoreV2 protected def putSimple(json: Json): DataStoreResult[Json] = { put(json, table1) } - - - - protected def putComposite(json: Json): DataStoreResult[Json] = { - ??? - } - /** * Conditional put, ensuring passed revision is higher than the value in dynamo */ @@ -96,31 +88,15 @@ abstract class DynamoDataStoreV2 } } - protected def delete(key: DynamoCompositeKey): DataStoreResult[DeleteItemResponse] = { -// Try { -// key match { -// case DynamoCompositeKey(partitionKey, None) => -// table.deleteItem(SimpleKeyName, partitionKey) -// case DynamoCompositeKey(partitionKey, Some(sortKey)) => -// table.deleteItem(CompositeKey.partitionKey, partitionKey, CompositeKey.sortKey, sortKey) -// } -// } match { -// case Success(outcome) => Right(outcome.getDeleteItemResult) -// case Failure(e) => Left(handleException(e)) -// } - ??? + protected def delete(key: DynamoCompositeKey): DataStoreResult[Unit] = { + Try { + getTableToQuery(key).deleteItem(uniqueKey(key)) + } match { + case Success(_) => Right(()) + case Failure(e) => Left(handleException(e)) + } } - protected def scan: DataStoreResult[List[Json]] = { -// Try { -// table.scan().iterator.asScala.toList -// -// } match { -// case Success(items) => items.traverse(item => parseJson(item.toJSON)) -// case Failure(e) => Left(DynamoError(e.getMessage)) -// } - ??? - } protected def scanSimple: DataStoreResult[List[Json]] = { Try { @@ -130,18 +106,6 @@ abstract class DynamoDataStoreV2 } } -// private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Map[String, AttributeValue] = dynamoCompositeKey match { -// case DynamoCompositeKey(partitionKey, None) => { -// Map(SimpleKeyName -> AttributeValue.builder().s(partitionKey).build()) -// } -// -// case DynamoCompositeKey(partitionKey, Some(sortKey)) =>{ -// Map( -// CompositeKey.partitionKey -> AttributeValue.builder().s(partitionKey).build(), -// CompositeKey.sortKey -> AttributeValue.builder().s(sortKey).build() -// ) -// } -// } private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Key = dynamoCompositeKey match { case DynamoCompositeKey(partitionKey, None) => { Key.builder().partitionValue(partitionKey).build() @@ -163,14 +127,6 @@ abstract class DynamoDataStoreV2 def jsonToAtom(json: Json): DataStoreResult[Atom] = json.as[Atom](backwardsCompatibleAtomDecoder).leftMap(error => DecoderError(error.message)) - def jsonToItem(json: Json): ItemResponse = { -// val item = new Item() -// json.asObject.foreach { obj => -// obj.toMap.map { case (key, value) => item.withJSON(key, value.noSpaces) } -// } -// item - ??? - } private def handleException(e: Throwable) = e match { case serviceError: AwsServiceException => DynamoError(serviceError.awsErrorDetails().errorMessage) diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala index 0006470..b3ce729 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -71,68 +71,68 @@ class DynamoDataStoreV2Spec it("should create the atom with composite key") { dataStores => dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) } -// -// it("should return the atom with composite key") { dataStores => -// dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) -// } -// -// it("should update an atom with composite key") { dataStores => -// val updated = testAtom -// .copy(defaultHtml = "
updated
") -// .bumpRevision -// -// dataStores.compositeKey.updateAtom(updated) should equal(Right(updated)) -// dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(updated)) -// } -// -// it("should delete an atom if it exists in the table") { dataStores => -// dataStores.preview.createAtom(testAtomForDeletion) should equal(Right(testAtomForDeletion)) -// dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal(Right(testAtomForDeletion)) -// } -// -// it("should delete an atom with composite key if it exists in the table") { dataStores => -// val key = DynamoCompositeKey(testAtomForDeletion.atomType.toString, Some(testAtomForDeletion.id)) -// dataStores.compositeKey.createAtom(key, testAtomForDeletion) should equal(Right(testAtomForDeletion)) -// dataStores.compositeKey.deleteAtom(key) should equal(Right(testAtomForDeletion)) -// } -// -// it("should decode the old format from dynamo") { dataStores => -// val json = dataStores.published.parseJson( -// """ -// |{ -// | "defaultHtml" : "
", -// | "data" : { -// | "assets" : [ -// | { -// | "id" : "xyzzy", -// | "version" : 1, -// | "platform" : "Youtube", -// | "assetType" : "Video" -// | }, -// | { -// | "id" : "fizzbuzz", -// | "version" : 2, -// | "platform" : "Youtube", -// | "assetType" : "Video" -// | } -// | ], -// | "activeVersion" : 2, -// | "title" : "Test atom 1", -// | "category" : "News" -// | }, -// | "contentChangeDetails" : { -// | "revision" : 1 -// | }, -// | "id" : "1", -// | "atomType" : "Media", -// | "labels" : [ -// | ] -// |} -// """.stripMargin).toOption.get -// -// val atom = json.as[Atom](JsonSupport.backwardsCompatibleAtomDecoder) -// atom should equal(Right(testAtom)) -// } + + it("should return the atom with composite key") { dataStores => + dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) + } + + it("should update an atom with composite key") { dataStores => + val updated = testAtom + .copy(defaultHtml = "
updated
") + .bumpRevision + + dataStores.compositeKey.updateAtom(updated) should equal(Right(updated)) + dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(updated)) + } + + it("should delete an atom if it exists in the table") { dataStores => + dataStores.preview.createAtom(testAtomForDeletion) should equal(Right(testAtomForDeletion)) + dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal(Right(testAtomForDeletion)) + } + + it("should delete an atom with composite key if it exists in the table") { dataStores => + val key = DynamoCompositeKey(testAtomForDeletion.atomType.toString, Some(testAtomForDeletion.id)) + dataStores.compositeKey.createAtom(key, testAtomForDeletion) should equal(Right(testAtomForDeletion)) + dataStores.compositeKey.deleteAtom(key) should equal(Right(testAtomForDeletion)) + } + + it("should decode the old format from dynamo") { dataStores => + val json = dataStores.published.parseJson( + """ + |{ + | "defaultHtml" : "
", + | "data" : { + | "assets" : [ + | { + | "id" : "xyzzy", + | "version" : 1, + | "platform" : "Youtube", + | "assetType" : "Video" + | }, + | { + | "id" : "fizzbuzz", + | "version" : 2, + | "platform" : "Youtube", + | "assetType" : "Video" + | } + | ], + | "activeVersion" : 2, + | "title" : "Test atom 1", + | "category" : "News" + | }, + | "contentChangeDetails" : { + | "revision" : 1 + | }, + | "id" : "1", + | "atomType" : "Media", + | "labels" : [ + | ] + |} + """.stripMargin).toOption.get + + val atom = json.as[Atom](JsonSupport.backwardsCompatibleAtomDecoder) + atom should equal(Right(testAtom)) + } } val client = LocalDynamoDBV2.client() override def beforeAll() = { From f1539128b952ee989a45e175454147c4894231da Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 2 Dec 2025 17:25:35 +0000 Subject: [PATCH 07/11] Refactor the table query --- .../com/gu/atom/data/DynamoDataStoreV2.scala | 218 +++++++++++------- .../gu/atom/data/DynamoDataStoreV2Spec.scala | 88 +++++-- .../com/gu/atom/data/LocalDynamoDBV2.scala | 62 +++-- 3 files changed, 248 insertions(+), 120 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index 71b4aac..5978d02 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -1,7 +1,15 @@ package com.gu.atom.data import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, ConditionalCheckFailedException, DeleteItemRequest, DeleteItemResponse, ItemResponse} +import software.amazon.awssdk.services.dynamodb.model.{ + AttributeValue, + ConditionalCheckFailedException, + DeleteItemRequest, + DeleteItemResponse, + DescribeTableRequest, + ItemResponse, + KeyType +} import software.amazon.awssdk.awscore.exception.AwsServiceException import com.gu.contentatom.thrift.Atom import cats.implicits._ @@ -10,13 +18,25 @@ import com.gu.atom.util.JsonSupport.backwardsCompatibleAtomDecoder import software.amazon.awssdk.core.exception.SdkException import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest -import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, DynamoDbTable, Expression, Key, TableMetadata, TableSchema} +import software.amazon.awssdk.enhanced.dynamodb.{ + AttributeConverterProvider, + AttributeValueType, + DynamoDbEnhancedClient, + DynamoDbTable, + Expression, + Key, + TableMetadata, + TableSchema +} -import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala, MapHasAsJava} +import scala.jdk.CollectionConverters.{ + CollectionHasAsScala, + IteratorHasAsScala, + MapHasAsJava +} import scala.util.{Failure, Success, Try} -abstract class DynamoDataStoreV2 - (dynamo: DynamoDbClient, tableName: String) +abstract class DynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) extends AtomDataStore { private val SimpleKeyName = "id" @@ -24,63 +44,89 @@ abstract class DynamoDataStoreV2 val partitionKey = "atomType" val sortKey = "id" } - lazy val ddb: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(dynamo).build() - - lazy val tableSchema1 = TableSchema.documentSchemaBuilder() - .addIndexPartitionKey(TableMetadata.primaryIndexName(), SimpleKeyName, AttributeValueType.S) - .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) - .build() - - val table1 = ddb.table(tableName, tableSchema1) - - lazy val tableSchema2 = TableSchema.documentSchemaBuilder() - .addIndexPartitionKey(TableMetadata.primaryIndexName(), CompositeKey.partitionKey, AttributeValueType.S) - .addIndexSortKey(TableMetadata.primaryIndexName(), CompositeKey.sortKey, AttributeValueType.S) - .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) - .build() + val desc = dynamo + .describeTable( + DescribeTableRequest.builder().tableName(tableName).build() + ) + .table() + + val hasSortKey = + desc.keySchema().asScala.exists(_.keyType() == KeyType.RANGE) + + lazy val tableSchema: TableSchema[EnhancedDocument] = { + val builder = TableSchema + .documentSchemaBuilder() + .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) + + if (hasSortKey) { + builder.addIndexPartitionKey( + TableMetadata.primaryIndexName(), + CompositeKey.partitionKey, + AttributeValueType.S + ) + builder.addIndexSortKey( + TableMetadata.primaryIndexName(), + CompositeKey.sortKey, + AttributeValueType.S + ) + } else + builder.addIndexPartitionKey( + TableMetadata.primaryIndexName(), + SimpleKeyName, + AttributeValueType.S + ) + + builder.build() + } + lazy val ddb: DynamoDbEnhancedClient = + DynamoDbEnhancedClient.builder().dynamoDbClient(dynamo).build() - val table2 = ddb.table(tableName, tableSchema2) + val table = ddb.table(tableName, tableSchema) import AtomSerializer._ protected def get(key: DynamoCompositeKey): DataStoreResult[Json] = { Try { - Option(getTableToQuery(key).getItem(uniqueKey(key))) + Option(table.getItem(uniqueKey(key))) } match { case Success(Some(item)) => parseJson(item.toJson) - case Success(None) => Left(IDNotFound) - case Failure(e) => Left(handleException(e)) + case Success(None) => Left(IDNotFound) + case Failure(e) => Left(handleException(e)) } } - protected def put(json: Json, table: DynamoDbTable[EnhancedDocument]): DataStoreResult[Json] = { - Try(table.putItem( - EnhancedDocument.builder().json(json.spaces2).build() - )) match { + protected def put(json: Json): DataStoreResult[Json] = { + Try( + table.putItem( + EnhancedDocument.builder().json(json.spaces2).build() + ) + ) match { case Success(_) => Right(json) case Failure(e) => Left(handleException(e)) } } - protected def putSimple(json: Json): DataStoreResult[Json] = { - put(json, table1) - } - /** - * Conditional put, ensuring passed revision is higher than the value in dynamo + /** Conditional put, ensuring passed revision is higher than the value in + * dynamo */ - protected def putSimple(json: Json, revision: Long): DataStoreResult[Json] = { - val expressionAttrValues = Map[String, AttributeValue](":revision" -> AttributeValue.builder().n(revision.toString).build()) - val expression = Expression.builder().expression("contentChangeDetails.revision < :revision").expressionValues(expressionAttrValues.asJava).build() + protected def put(json: Json, revision: Long): DataStoreResult[Json] = { + val expressionAttrValues = Map[String, AttributeValue]( + ":revision" -> AttributeValue.builder().n(revision.toString).build() + ) + val expression = Expression + .builder() + .expression("contentChangeDetails.revision < :revision") + .expressionValues(expressionAttrValues.asJava) + .build() val doc = EnhancedDocument.fromJson(json.spaces2) val putItemRequest = PutItemEnhancedRequest - .builder(classOf[EnhancedDocument]) - .item(doc) - .conditionExpression(expression) - .build() + .builder(classOf[EnhancedDocument]) + .item(doc) + .conditionExpression(expression) + .build() Try { - table1.putItem(putItemRequest) - } - match { + table.putItem(putItemRequest) + } match { case Success(item) => Right(json) case Failure(conditionError: ConditionalCheckFailedException) => Left(VersionConflictError(revision)) @@ -90,102 +136,110 @@ abstract class DynamoDataStoreV2 protected def delete(key: DynamoCompositeKey): DataStoreResult[Unit] = { Try { - getTableToQuery(key).deleteItem(uniqueKey(key)) + table.deleteItem(uniqueKey(key)) } match { case Success(_) => Right(()) case Failure(e) => Left(handleException(e)) } } - - protected def scanSimple: DataStoreResult[List[Json]] = { + protected def scan: DataStoreResult[List[Json]] = { Try { - table1.scan().iterator().asScala.toList + table.scan().iterator().asScala.toList } match { - case Success(page) => page.flatMap(p => p.items().asScala.map(i => parseJson(i.toJson))).sequence + case Success(page) => + page + .flatMap(p => p.items().asScala.map(i => parseJson(i.toJson))) + .sequence } } - private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Key = dynamoCompositeKey match { - case DynamoCompositeKey(partitionKey, None) => { - Key.builder().partitionValue(partitionKey).build() - } + private def uniqueKey(dynamoCompositeKey: DynamoCompositeKey): Key = + dynamoCompositeKey match { + case DynamoCompositeKey(partitionKey, None) => + Key.builder().partitionValue(partitionKey).build() - case DynamoCompositeKey(partitionKey, Some(sortKey)) =>{ - Key.builder().partitionValue(partitionKey).addSortValue(sortKey).build() + case DynamoCompositeKey(partitionKey, Some(sortKey)) => + Key.builder().partitionValue(partitionKey).addSortValue(sortKey).build() } - } - - private def getTableToQuery(dynamoCompositeKey: DynamoCompositeKey) = dynamoCompositeKey match { - case DynamoCompositeKey(_, None) => table1 - case DynamoCompositeKey(_, Some(_)) => table2 - } def parseJson(s: String): DataStoreResult[Json] = - parser.parse(s).leftMap(parsingFailure => DynamoError(parsingFailure.getMessage)) + parser + .parse(s) + .leftMap(parsingFailure => DynamoError(parsingFailure.getMessage)) def jsonToAtom(json: Json): DataStoreResult[Atom] = - json.as[Atom](backwardsCompatibleAtomDecoder).leftMap(error => DecoderError(error.message)) - + json + .as[Atom](backwardsCompatibleAtomDecoder) + .leftMap(error => DecoderError(error.message)) private def handleException(e: Throwable) = e match { - case serviceError: AwsServiceException => DynamoError(serviceError.awsErrorDetails().errorMessage) + case serviceError: AwsServiceException => + DynamoError(serviceError.awsErrorDetails().errorMessage) case clientError: SdkException => { ClientError(clientError.getMessage) } case other => ReadError } - def getAtom(id: String): DataStoreResult[Atom] = getAtom(DynamoCompositeKey(id)) + def getAtom(id: String): DataStoreResult[Atom] = getAtom( + DynamoCompositeKey(id) + ) def getAtom(dynamoCompositeKey: DynamoCompositeKey): DataStoreResult[Atom] = { get(dynamoCompositeKey) flatMap jsonToAtom } - def createAtom(atom: Atom): DataStoreResult[Atom] = createAtom(DynamoCompositeKey(atom.id), atom) + def createAtom(atom: Atom): DataStoreResult[Atom] = + createAtom(DynamoCompositeKey(atom.id), atom) - def createAtom(dynamoCompositeKey: DynamoCompositeKey, atom: Atom): DataStoreResult[Atom] = { + def createAtom( + dynamoCompositeKey: DynamoCompositeKey, + atom: Atom + ): DataStoreResult[Atom] = { getAtom(dynamoCompositeKey) match { case Right(_) => Left(IDConflictError) case Left(error) => - val table = getTableToQuery(dynamoCompositeKey) - put(toJson(atom), table).map(_ => atom) + put(toJson(atom)).map(_ => atom) } } - def deleteAtom(id: String): DataStoreResult[Atom] = deleteAtom(DynamoCompositeKey(id)) + def deleteAtom(id: String): DataStoreResult[Atom] = deleteAtom( + DynamoCompositeKey(id) + ) - def deleteAtom(dynamoCompositeKey: DynamoCompositeKey): DataStoreResult[Atom] = + def deleteAtom( + dynamoCompositeKey: DynamoCompositeKey + ): DataStoreResult[Atom] = getAtom(dynamoCompositeKey).flatMap { atom => delete(dynamoCompositeKey).map(_ => atom) } private def findAtoms(tableName: String): DataStoreResult[List[Atom]] = - scanSimple.flatMap(_.traverse(jsonToAtom)) + scan.flatMap(_.traverse(jsonToAtom)) def listAtoms: DataStoreResult[List[Atom]] = findAtoms(tableName) } -class PreviewDynamoDataStoreV2 -(dynamo: DynamoDbClient, tableName: String) - extends DynamoDataStoreV2(dynamo, tableName) - with PreviewDataStore { +class PreviewDynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) + extends DynamoDataStoreV2(dynamo, tableName) + with PreviewDataStore { import AtomSerializer._ def updateAtom(newAtom: Atom) = - putSimple(toJson(newAtom), newAtom.contentChangeDetails.revision).map(_ => newAtom) + put(toJson(newAtom), newAtom.contentChangeDetails.revision).map(_ => + newAtom + ) } -class PublishedDynamoDataStoreV2 -(dynamo: DynamoDbClient, tableName: String) - extends DynamoDataStoreV2(dynamo, tableName) - with PublishedDataStore { +class PublishedDynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) + extends DynamoDataStoreV2(dynamo, tableName) + with PublishedDataStore { import AtomSerializer._ - def updateAtom(newAtom: Atom) = putSimple(toJson(newAtom)).map(_ => newAtom) + def updateAtom(newAtom: Atom) = put(toJson(newAtom)).map(_ => newAtom) } - diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala index b3ce729..ae0e86a 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/DynamoDataStoreV2Spec.scala @@ -6,7 +6,7 @@ import com.gu.contentatom.thrift.Atom import org.scalatest.funspec.FixtureAnyFunSpec import org.scalatest.matchers.should._ import org.scalatest.{BeforeAndAfterAll, OptionValues} -import software.amazon.awssdk.services.dynamodb.model.{KeyType, ScalarAttributeType} +import software.amazon.awssdk.services.dynamodb.model.KeyType class DynamoDataStoreV2Spec extends FixtureAnyFunSpec @@ -19,31 +19,42 @@ class DynamoDataStoreV2Spec val publishedTableName = "published-atom-test-table" val compositeKeyTableName = "composite-key-table" - case class DataStoresV2(preview: PreviewDynamoDataStoreV2, - published: PublishedDynamoDataStoreV2, - compositeKey: PreviewDynamoDataStoreV2 - ) + case class DataStoresV2( + preview: PreviewDynamoDataStoreV2, + published: PublishedDynamoDataStoreV2, + compositeKey: PreviewDynamoDataStoreV2 + ) type FixtureParam = DataStoresV2 def withFixture(test: OneArgTest) = { - val previewDb = new PreviewDynamoDataStoreV2(LocalDynamoDBV2.client(), tableName) - val compositeKeyDb = new PreviewDynamoDataStoreV2(LocalDynamoDBV2.client(), compositeKeyTableName) - val publishedDb = new PublishedDynamoDataStoreV2(LocalDynamoDBV2.client(), publishedTableName) - super.withFixture(test.toNoArgTest(DataStoresV2(previewDb, publishedDb, compositeKeyDb))) + val previewDb = + new PreviewDynamoDataStoreV2(LocalDynamoDBV2.client(), tableName) + val compositeKeyDb = new PreviewDynamoDataStoreV2( + LocalDynamoDBV2.client(), + compositeKeyTableName + ) + val publishedDb = new PublishedDynamoDataStoreV2( + LocalDynamoDBV2.client(), + publishedTableName + ) + super.withFixture( + test.toNoArgTest(DataStoresV2(previewDb, publishedDb, compositeKeyDb)) + ) } describe("DynamoDataStore") { it("should create a new atom") { dataStores => val atomCreated = dataStores.preview.createAtom(testAtom) - println(dataStores.preview.listAtoms.map(as => as.map(a => a.id))) atomCreated should equal(Right(testAtom)) } it("should list all atoms of all types") { dataStores => dataStores.preview.createAtom(testAtoms(1)) dataStores.preview.createAtom(testAtoms(2)) - dataStores.preview.listAtoms.map(_.toList).fold(identity, res => res should contain theSameElementsAs testAtoms) + dataStores.preview.listAtoms + .map(_.toList) + .fold(identity, res => res should contain theSameElementsAs testAtoms) } it("should return the atom") { dataStores => @@ -69,11 +80,16 @@ class DynamoDataStoreV2Spec } it("should create the atom with composite key") { dataStores => - dataStores.compositeKey.createAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), testAtom) should equal(Right(testAtom)) + dataStores.compositeKey.createAtom( + DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)), + testAtom + ) should equal(Right(testAtom)) } it("should return the atom with composite key") { dataStores => - dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(testAtom)) + dataStores.compositeKey.getAtom( + DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)) + ) should equal(Right(testAtom)) } it("should update an atom with composite key") { dataStores => @@ -82,23 +98,38 @@ class DynamoDataStoreV2Spec .bumpRevision dataStores.compositeKey.updateAtom(updated) should equal(Right(updated)) - dataStores.compositeKey.getAtom(DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id))) should equal(Right(updated)) + dataStores.compositeKey.getAtom( + DynamoCompositeKey(testAtom.atomType.toString, Some(testAtom.id)) + ) should equal(Right(updated)) } it("should delete an atom if it exists in the table") { dataStores => - dataStores.preview.createAtom(testAtomForDeletion) should equal(Right(testAtomForDeletion)) - dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal(Right(testAtomForDeletion)) + dataStores.preview.createAtom(testAtomForDeletion) should equal( + Right(testAtomForDeletion) + ) + dataStores.preview.deleteAtom(testAtomForDeletion.id) should equal( + Right(testAtomForDeletion) + ) } - it("should delete an atom with composite key if it exists in the table") { dataStores => - val key = DynamoCompositeKey(testAtomForDeletion.atomType.toString, Some(testAtomForDeletion.id)) - dataStores.compositeKey.createAtom(key, testAtomForDeletion) should equal(Right(testAtomForDeletion)) - dataStores.compositeKey.deleteAtom(key) should equal(Right(testAtomForDeletion)) + it("should delete an atom with composite key if it exists in the table") { + dataStores => + val key = DynamoCompositeKey( + testAtomForDeletion.atomType.toString, + Some(testAtomForDeletion.id) + ) + dataStores.compositeKey.createAtom( + key, + testAtomForDeletion + ) should equal(Right(testAtomForDeletion)) + dataStores.compositeKey.deleteAtom(key) should equal( + Right(testAtomForDeletion) + ) } it("should decode the old format from dynamo") { dataStores => - val json = dataStores.published.parseJson( - """ + val json = dataStores.published + .parseJson(""" |{ | "defaultHtml" : "
", | "data" : { @@ -128,7 +159,9 @@ class DynamoDataStoreV2Spec | "labels" : [ | ] |} - """.stripMargin).toOption.get + """.stripMargin) + .toOption + .get val atom = json.as[Atom](JsonSupport.backwardsCompatibleAtomDecoder) atom should equal(Right(testAtom)) @@ -137,8 +170,13 @@ class DynamoDataStoreV2Spec val client = LocalDynamoDBV2.client() override def beforeAll() = { LocalDynamoDBV2.createTable(client)(tableName)(KeyType.HASH -> "id") - LocalDynamoDBV2.createTable(client)(publishedTableName)(KeyType.HASH -> "id") - LocalDynamoDBV2.createTable(client)(compositeKeyTableName)(KeyType.HASH -> "atomType", KeyType.RANGE -> "id") + LocalDynamoDBV2.createTable(client)(publishedTableName)( + KeyType.HASH -> "id" + ) + LocalDynamoDBV2.createTable(client)(compositeKeyTableName)( + KeyType.HASH -> "atomType", + KeyType.RANGE -> "id" + ) } override def afterAll(): Unit = { diff --git a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala index 359f4eb..77cdb4d 100644 --- a/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala +++ b/atom-publisher-lib/src/test/scala/com/gu/atom/data/LocalDynamoDBV2.scala @@ -1,9 +1,21 @@ package com.gu.atom.data -import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.auth.credentials.{ + AwsBasicCredentials, + StaticCredentialsProvider +} import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{AttributeDefinition, CreateTableRequest, DeleteTableRequest, DeleteTableResponse, KeySchemaElement, KeyType, ProvisionedThroughput, ScalarAttributeType} +import software.amazon.awssdk.services.dynamodb.model.{ + AttributeDefinition, + CreateTableRequest, + DeleteTableRequest, + DeleteTableResponse, + KeySchemaElement, + KeyType, + ProvisionedThroughput, + ScalarAttributeType +} import java.net.URI import scala.jdk.CollectionConverters._ @@ -15,30 +27,54 @@ import scala.jdk.CollectionConverters._ object LocalDynamoDBV2 { def client() = { - DynamoDbClient.builder() - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create("key", "secret") - )) + DynamoDbClient + .builder() + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create("key", "secret") + ) + ) .endpointOverride(URI.create("http://localhost:8000")) .region(Region.EU_WEST_1) .build() } - def createTable(client: DynamoDbClient)(tableName: String)(attributes: (KeyType, String)*) = { - val attrs = attributes.toList.map { case(kt, attrName) => KeySchemaElement.builder().keyType(kt).attributeName(attrName).build()} - val attributeDefinitions = attributes.toList.map { case(at, attrName) => AttributeDefinition.builder().attributeType(ScalarAttributeType.S).attributeName(attrName).build() } + def createTable( + client: DynamoDbClient + )(tableName: String)(attributes: (KeyType, String)*) = { + val attrs = attributes.toList.map { case (kt, attrName) => + KeySchemaElement.builder().keyType(kt).attributeName(attrName).build() + } + val attributeDefinitions = attributes.toList.map { case (at, attrName) => + AttributeDefinition + .builder() + .attributeType(ScalarAttributeType.S) + .attributeName(attrName) + .build() + } - val createTableRequest = CreateTableRequest.builder() + val createTableRequest = CreateTableRequest + .builder() .tableName(tableName) .keySchema(attrs.asJava) .attributeDefinitions(attributeDefinitions.asJava) - .provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(1L).writeCapacityUnits(1L).build()) + .provisionedThroughput( + ProvisionedThroughput + .builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build() + ) .build() client.createTable(createTableRequest) } - def deleteTable(client: DynamoDbClient)(tableName: String): DeleteTableResponse = { - client.deleteTable(DeleteTableRequest.builder().tableName(tableName).build()) + def deleteTable( + client: DynamoDbClient + )(tableName: String): DeleteTableResponse = { + client.deleteTable( + DeleteTableRequest.builder().tableName(tableName).build() + ) } } From a6877dba9029239aaa0811d14bbd60db1a8348f6 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 2 Dec 2025 17:28:24 +0000 Subject: [PATCH 08/11] Delete unused methods --- .../scala/com/gu/atom/data/DynamoDataStoreV2.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index 5978d02..cd8b1c5 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -4,10 +4,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient import software.amazon.awssdk.services.dynamodb.model.{ AttributeValue, ConditionalCheckFailedException, - DeleteItemRequest, - DeleteItemResponse, DescribeTableRequest, - ItemResponse, KeyType } import software.amazon.awssdk.awscore.exception.AwsServiceException @@ -22,7 +19,6 @@ import software.amazon.awssdk.enhanced.dynamodb.{ AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, - DynamoDbTable, Expression, Key, TableMetadata, @@ -179,7 +175,7 @@ abstract class DynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) case clientError: SdkException => { ClientError(clientError.getMessage) } - case other => ReadError + case _ => ReadError } def getAtom(id: String): DataStoreResult[Atom] = getAtom( @@ -200,7 +196,7 @@ abstract class DynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) getAtom(dynamoCompositeKey) match { case Right(_) => Left(IDConflictError) - case Left(error) => + case Left(_) => put(toJson(atom)).map(_ => atom) } } @@ -216,10 +212,8 @@ abstract class DynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) delete(dynamoCompositeKey).map(_ => atom) } - private def findAtoms(tableName: String): DataStoreResult[List[Atom]] = - scan.flatMap(_.traverse(jsonToAtom)) - def listAtoms: DataStoreResult[List[Atom]] = findAtoms(tableName) + def listAtoms: DataStoreResult[List[Atom]] = scan.flatMap(_.traverse(jsonToAtom)) } From 5868a9ebe6d29ed176dfb9001292c0c78a09c174 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 3 Dec 2025 17:12:50 +0000 Subject: [PATCH 09/11] Use awsverion variable --- atom-publisher-lib/build.sbt | 7 +++---- project/BuildVars.scala | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/atom-publisher-lib/build.sbt b/atom-publisher-lib/build.sbt index 44db2d3..691d430 100644 --- a/atom-publisher-lib/build.sbt +++ b/atom-publisher-lib/build.sbt @@ -1,7 +1,6 @@ import BuildVars._ name := "atom-publisher-lib" - // for testing dynamodb access dynamoDBLocalDownloadDir := file(".dynamodb-local") startDynamoDBLocal := startDynamoDBLocal.dependsOn(Test / compile).value @@ -26,7 +25,7 @@ libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % Test, "org.scalatestplus" %% "mockito-4-6" % "3.2.14.0" % Test, "org.scalatest" %% "scalatest" % "3.2.14" % Test, - "software.amazon.awssdk" % "kinesis" % "2.39.4", - "software.amazon.awssdk" % "dynamodb" % "2.39.6", - "software.amazon.awssdk" % "dynamodb-enhanced" % "2.39.6", + "software.amazon.awssdk" % "kinesis" % awsV2Version, + "software.amazon.awssdk" % "dynamodb" % awsV2Version, + "software.amazon.awssdk" % "dynamodb-enhanced" % awsV2Version ) diff --git a/project/BuildVars.scala b/project/BuildVars.scala index e9105cd..d60e6e2 100644 --- a/project/BuildVars.scala +++ b/project/BuildVars.scala @@ -6,5 +6,5 @@ object BuildVars { lazy val scroogeVersion = "22.1.0" lazy val playVersion = "3.0.2" lazy val mockitoVersion = "4.11.0" - lazy val awsV2Version = "2.39.4" + lazy val awsV2Version = "2.39.6" } From 517918ee379f8b5810bff19f5a4f25a563cd0535 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 4 Dec 2025 10:45:05 +0000 Subject: [PATCH 10/11] Update failure case for scan --- .../src/main/scala/com/gu/atom/data/AtomSerializer.scala | 3 --- .../src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala index c28368e..8d75f62 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala @@ -4,9 +4,6 @@ import com.gu.contentatom.thrift.Atom import io.circe.Json import io.circe.syntax._ import com.gu.fezziwig.CirceScroogeMacros.encodeThriftStruct -import io.circe.syntax._ -import com.gu.fezziwig.CirceScroogeMacros.{encodeThriftStruct, encodeThriftUnion} -import com.gu.atom.util.JsonSupport.{backwardsCompatibleAtomDecoder, thriftEnumEncoder} object AtomSerializer { def toJson(newAtom: Atom): Json = newAtom.asJson diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala index cd8b1c5..d3eba18 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/DynamoDataStoreV2.scala @@ -147,6 +147,7 @@ abstract class DynamoDataStoreV2(dynamo: DynamoDbClient, tableName: String) page .flatMap(p => p.items().asScala.map(i => parseJson(i.toJson))) .sequence + case Failure(e) => Left(DynamoError(e.getMessage)) } } From 8df78427b6a6f839df599412de924aa7b19b4a32 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 4 Dec 2025 15:18:04 +0000 Subject: [PATCH 11/11] Import rejig --- .../src/main/scala/com/gu/atom/data/AtomSerializer.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala b/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala index 8d75f62..4b0035f 100644 --- a/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala +++ b/atom-publisher-lib/src/main/scala/com/gu/atom/data/AtomSerializer.scala @@ -3,7 +3,8 @@ package com.gu.atom.data import com.gu.contentatom.thrift.Atom import io.circe.Json import io.circe.syntax._ -import com.gu.fezziwig.CirceScroogeMacros.encodeThriftStruct +import com.gu.fezziwig.CirceScroogeMacros.{encodeThriftStruct, encodeThriftUnion} +import com.gu.atom.util.JsonSupport.{backwardsCompatibleAtomDecoder, thriftEnumEncoder} object AtomSerializer { def toJson(newAtom: Atom): Json = newAtom.asJson