From c20810f4ec5d6beffe6d876d19ad210c8ebc3530 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Wed, 20 Nov 2024 15:05:42 -0800 Subject: [PATCH 1/8] feat: add Lambda to process remove events --- .aws/src/elasticache.ts | 30 +- .aws/src/main.ts | 8 +- .aws/src/removeItemLambda.ts | 80 +++ .circleci/config.yml | 51 +- app/main.py | 18 +- remove_item_lambda/Pipfile | 22 + remove_item_lambda/Pipfile.lock | 590 ++++++++++++++++++ remove_item_lambda/__init__.py | 0 remove_item_lambda/config/__init__.py | 1 + remove_item_lambda/config/index.py | 13 + remove_item_lambda/sqs_handler.py | 23 + remove_item_lambda/tests/__init__.py | 0 remove_item_lambda/tests/fixtures/__init__.py | 0 remove_item_lambda/tests/test_sqs_lambda.py | 0 14 files changed, 801 insertions(+), 35 deletions(-) create mode 100644 .aws/src/removeItemLambda.ts create mode 100644 remove_item_lambda/Pipfile create mode 100644 remove_item_lambda/Pipfile.lock create mode 100644 remove_item_lambda/__init__.py create mode 100644 remove_item_lambda/config/__init__.py create mode 100644 remove_item_lambda/config/index.py create mode 100644 remove_item_lambda/sqs_handler.py create mode 100644 remove_item_lambda/tests/__init__.py create mode 100644 remove_item_lambda/tests/fixtures/__init__.py create mode 100644 remove_item_lambda/tests/test_sqs_lambda.py diff --git a/.aws/src/elasticache.ts b/.aws/src/elasticache.ts index 55353a135..fef8d1f26 100644 --- a/.aws/src/elasticache.ts +++ b/.aws/src/elasticache.ts @@ -7,27 +7,25 @@ import { export class Elasticache extends Construct { public readonly nodeList: string[]; + public readonly clusterArn: string; constructor(scope: Construct, name: string) { super(scope, name); - this.nodeList = Elasticache.createElasticache(scope); + const { nodeList, clusterArn } = Elasticache.createElasticache(scope); + this.nodeList = nodeList; + this.clusterArn = clusterArn; } /** - * Creates the elasticache and returns the node address list + * Creates the Elasticache cluster and returns the node list and cluster ARN. * @param scope * @private */ - private static createElasticache(scope: Construct): string[] { + private static createElasticache(scope: Construct): { nodeList: string[]; clusterArn: string } { const pocketVPC = new PocketVPC(scope, 'pocket-shared-vpc'); const elasticache = new ApplicationMemcache(scope, 'memcached', { - //Usually we would set the security group ids of the service that needs to hit this. - //However we don't have the necessary security group because it gets created in PocketALBApplication - //So instead we set it to null and allow anything within the vpc to access it. - //This is not ideal.. - //Ideally we need to be able to add security groups to the ALB application. allowedIngressSecurityGroupIds: undefined, node: { count: config.cacheNodes, @@ -38,17 +36,13 @@ export class Elasticache extends Construct { prefix: config.prefix, }); - let nodeList: string[] = []; + const nodeList: string[] = []; for (let i = 0; i < config.cacheNodes; i++) { - // ${elasticache.elasticacheClister.cacheNodes(i.toString()).port} has a bug and is not rendering the proper terraform address - // its rendering -1.8881545897087503e+289 for some weird reason... - // For now we just hardcode to 11211 which is the default memcache port. - nodeList.push( - `${ - elasticache.elasticacheCluster.cacheNodes.get(i).address - }:11211` - ); + nodeList.push(`${elasticache.elasticacheCluster.cacheNodes.get(i).address}:11211`); } - return nodeList; + + const clusterArn = elasticache.elasticacheCluster.arn; + + return { nodeList, clusterArn }; } } diff --git a/.aws/src/main.ts b/.aws/src/main.ts index 2bb7bef91..8ce582c16 100644 --- a/.aws/src/main.ts +++ b/.aws/src/main.ts @@ -6,6 +6,7 @@ import {DynamoDB} from "./dynamodb"; import {PocketALBApplication, PocketECSCodePipeline} from "@pocket-tools/terraform-modules"; import {SqsLambda} from "./sqsLambda"; import {Elasticache} from "./elasticache"; +import {RemoveItemLambda} from "./removeItemLambda"; import {RecommendationApiSynthetics} from './monitoring'; import {ArchiveProvider} from '@cdktf/provider-archive/lib/provider'; @@ -40,12 +41,13 @@ class RecommendationAPI extends TerraformStack { const caller = new DataAwsCallerIdentity(this, 'caller'); const dynamodb = new DynamoDB(this, 'dynamodb'); + const elasticache = new Elasticache(this, 'elasticache'); const pocketApp = this.createPocketAlbApplication({ secretsManagerKmsAlias: this.getSecretsManagerKmsAlias(), region, caller, - elasticache: new Elasticache(this, 'elasticache'), + elasticache, dynamodb: dynamodb }); @@ -54,7 +56,11 @@ class RecommendationAPI extends TerraformStack { const synthetic = new RecommendationApiSynthetics(this, 'synthetics'); synthetic.createSyntheticCheck([]); + // Lambda for storing candidate sets for Pocket Explore & Topic pages in DynamoDB. new SqsLambda(this, 'sqs-lambda', dynamodb.candidateSetsTable); + + // Lambda for caching removed items, to avoid sending them to Pocket Home. + new RemoveItemLambda(this, 'remove-item-lambda', elasticache); } diff --git a/.aws/src/removeItemLambda.ts b/.aws/src/removeItemLambda.ts new file mode 100644 index 000000000..5e1337874 --- /dev/null +++ b/.aws/src/removeItemLambda.ts @@ -0,0 +1,80 @@ +import { Construct } from 'constructs'; +import { config } from './config'; +import { PocketSQSWithLambdaTarget, PocketVPC } from '@pocket-tools/terraform-modules'; +import { LAMBDA_RUNTIMES } from '@pocket-tools/terraform-modules'; +import { CloudwatchEventRule } from '@cdktf/provider-aws/lib/cloudwatch-event-rule'; +import { CloudwatchEventTarget } from '@cdktf/provider-aws/lib/cloudwatch-event-target'; +import { DataAwsSsmParameter } from '@cdktf/provider-aws/lib/data-aws-ssm-parameter'; +import { Elasticache } from './elasticache'; + +export class RemoveItemLambda extends Construct { + constructor(scope: Construct, name: string, elasticache: Elasticache) { + super(scope, name); + + const vpc = new PocketVPC(this, 'pocket-shared-vpc'); + + const { sentryDsn, gitSha } = this.getEnvVariableValues(); + + const sqsWithLambda = new PocketSQSWithLambdaTarget(this, 'remove-sqs-lambda', { + name: `${config.prefix}-Sqs-RemoveEventHandler`, + batchSize: 1, + sqsQueue: { + maxReceiveCount: 3, // 2 retries + visibilityTimeoutSeconds: 300, + }, + lambda: { + runtime: 'python3.9' as LAMBDA_RUNTIMES, + handler: 'remove_handler.lambda_handler', + timeout: 120, + executionPolicyStatements: [ + { + effect: 'Allow', + actions: ['elasticache:ModifyCacheCluster', 'elasticache:DescribeCacheClusters'], + resources: [elasticache.clusterArn], + }, + ], + environment: { + SENTRY_DSN: sentryDsn, + GIT_SHA: gitSha, + ENVIRONMENT: config.environment === 'Prod' ? 'production' : 'development', + ELASTICACHE_SERVERS: elasticache.nodeList.join(','), // Pass node list to Lambda + }, + vpcConfig: { + securityGroupIds: vpc.internalSecurityGroups.ids, + subnetIds: vpc.privateSubnetIds, + }, + codeDeploy: { + region: 'us-east-1', + accountId: '410318598490', + }, + }, + }); + + const eventRule = new CloudwatchEventRule(this, 'remove-event-rule', { + name: `${config.prefix}-RemoveEventRule`, + description: 'Event rule for REMOVE_ITEM events to SQS', + eventPattern: JSON.stringify({ + source: ['curation-migration-datasync'], + detailType: ['REMOVE_ITEM'], + }), + eventBusName: 'default', + }); + + new CloudwatchEventTarget(this, 'remove-event-target', { + rule: eventRule.name, + arn: sqsWithLambda.sqsQueueResource.arn, + }); + } + + private getEnvVariableValues() { + const sentryDsn = new DataAwsSsmParameter(this, 'sentry-dsn', { + name: `/${config.name}/${config.environment}/SENTRY_DSN`, + }); + + const serviceHash = new DataAwsSsmParameter(this, 'service-hash', { + name: `${config.circleCIPrefix}/SERVICE_HASH`, + }); + + return { sentryDsn: sentryDsn.value, gitSha: serviceHash.value }; + } +} diff --git a/.circleci/config.yml b/.circleci/config.yml index 69653344f..f3d047438 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: path: test-reports lambda: - description: Builds and Optionaly deploys all the associated lambdas + description: Builds and Optionally deploys associated lambdas parameters: env_lower_name: type: string @@ -178,6 +178,10 @@ jobs: deploy: type: boolean default: true + lambda_path: + type: string + description: Path to the lambda directory + default: "aws_lambda/" docker: - image: python:3.9 auth: @@ -194,24 +198,17 @@ jobs: aws-region: << parameters.env_capital_name >>_AWS_DEFAULT_REGION - run: name: Package Lambda - # We bundle our code in a way that lambda can understand and execute - # - pipenv requirements > requirements.txt - output a requirements.txt file from Pipfile - # - pip install -r requirements.txt --no-deps -t package | installs the requirements into a "package" directory - # - change dir into the "package" dir and add all the packages to a zip file as /tmp - # - change dir into the "aws_lambda" dir and remove all files and folder we want to exclude from the build - # - change dir into the root of the application and add the "aws_lambda" dir to the zip file excluding the Pipfile and Pipfile.lock - # - copy the zip file to /tmp/build.zip and store as a CI artifact for quick inspection of the build command: | apt-get update && apt-get install zip pip install pipenv==2022.8.15 - cd aws_lambda + cd <> pipenv requirements > requirements.txt pip install -r requirements.txt --no-deps -t package cd package mkdir -p /tmp zip -r9 "/tmp/$CIRCLE_SHA1.zip" . -x \*__pycache__\* \.git\* cd .. && rm -rf package __pycache__ requirements.txt && cd .. - zip -gr "/tmp/$CIRCLE_SHA1.zip" aws_lambda -x aws_lambda/Pipfile* + zip -gr "/tmp/$CIRCLE_SHA1.zip" <> -x <>/Pipfile* cp "/tmp/$CIRCLE_SHA1.zip" /tmp/build.zip - run: name: Upload Package @@ -221,9 +218,9 @@ jobs: aws-access-key-id: << parameters.env_capital_name >>_AWS_ACCESS_KEY aws-secret-access-key: << parameters.env_capital_name >>_AWS_SECRET_ACCESS_KEY aws-region: << parameters.env_capital_name >>_AWS_DEFAULT_REGION - codedeploy-application-name: RecommendationAPI-<< parameters.env_capital_name >>-Sqs-Translation-Lambda - codedeploy-deployment-group-name: RecommendationAPI-<< parameters.env_capital_name >>-Sqs-Translation-Lambda - function-name: RecommendationAPI-<< parameters.env_capital_name >>-Sqs-Translation-Function + codedeploy-application-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_path >>-Lambda + codedeploy-deployment-group-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_path >>-Lambda + function-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_path >>-Function - store_artifacts: path: /tmp/build.zip @@ -247,9 +244,21 @@ workflows: - lambda: <<: *only_dev context: pocket - name: deploy_lambdas_dev + name: deploy_lambda_dev_aws + env_lower_name: dev + env_capital_name: Dev + lambda_path: "aws_lambda/" + deploy: true + requires: + - setup-deploy-params-dev + + - lambda: + <<: *only_dev + context: pocket + name: deploy_lambda_dev_remove_item env_lower_name: dev env_capital_name: Dev + lambda_path: "remove_item_lambda/" deploy: true requires: - setup-deploy-params-dev @@ -258,9 +267,21 @@ workflows: - lambda: <<: *only_main context: pocket - name: deploy_lambdas_prod + name: deploy_lambda_prod_aws + env_lower_name: prod + env_capital_name: Prod + lambda_path: "aws_lambda/" + deploy: true + requires: + - setup-deploy-params-prod + + - lambda: + <<: *only_main + context: pocket + name: deploy_lambda_prod_remove_item env_lower_name: prod env_capital_name: Prod + lambda_path: "remove_item_lambda/" deploy: true requires: - setup-deploy-params-prod diff --git a/app/main.py b/app/main.py index 863d659a9..33e1d7f6a 100644 --- a/app/main.py +++ b/app/main.py @@ -2,7 +2,7 @@ import sentry_sdk import uvicorn -from fastapi import FastAPI, Response, status +from fastapi import FastAPI, Request, Response, status from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor @@ -27,6 +27,7 @@ from app.singletons import DiContainer logging.getLogger().setLevel(log_level) +logger = logging.getLogger(__name__) sentry_sdk.init( dsn=sentry_config['dsn'], @@ -104,6 +105,21 @@ async def load_slate_configs(): async def startup_event(): DiContainer.init() +@app.post("/remove") +async def remove_event(request: Request, response: Response): + """ + Handles remove events triggered by EventBridge. + Logs the event details for debugging or processing. + """ + try: + body = await request.json() + logger.info("Received remove event: %s", body) + return {"status": "success", "message": "Event logged successfully."} + except Exception as e: + logger.error("Error processing remove event: %s", str(e)) + response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR + return {"status": "error", "message": "Failed to process event."} + if __name__ == "__main__": # This runs uvicorn in a local development environment. diff --git a/remove_item_lambda/Pipfile b/remove_item_lambda/Pipfile new file mode 100644 index 000000000..5d81b8c02 --- /dev/null +++ b/remove_item_lambda/Pipfile @@ -0,0 +1,22 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pytest = "*" +pytest-cov = "*" +coverage = "*" +requests = "*" +moto = "*" +mypy_boto3_dynamodb = "*" +pytest-mock = "*" + +[packages] +boto3 = "*" +# We define the essential stubs +boto3-stubs = {extras = ["essential"], version = "*"} +sentry-sdk = "*" + +[requires] +python_version = "3.9" diff --git a/remove_item_lambda/Pipfile.lock b/remove_item_lambda/Pipfile.lock new file mode 100644 index 000000000..e7746c726 --- /dev/null +++ b/remove_item_lambda/Pipfile.lock @@ -0,0 +1,590 @@ +{ + "_meta": { + "hash": { + "sha256": "6678a46d76124be7ef91a989fda4981c7c0d6ef4e3e56948dd0e00387c21d2fa" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:27824d3767c5213f0006f71e552b912bc4659241b77ca8b0be0b813cf0518a9e", + "sha256:5b585a279478bd6df4b07db7d6150f413ba6add1f38e68aaa533d3337efd0b22" + ], + "index": "pypi", + "version": "==1.18.52" + }, + "boto3-stubs": { + "extras": [ + "essential" + ], + "hashes": [ + "sha256:60f54146d506e71c8a3bf52d4bda7e38f9a387c34bc20cb8a03abe8ceea293c6", + "sha256:a30138e087903a4a52e322d4236ddda0c296a3d2e53173b1551a30e6d85a9ba9" + ], + "index": "pypi", + "version": "==1.18.52" + }, + "botocore": { + "hashes": [ + "sha256:04c071aec4f1981b38e3be760838b976337fd6ebd95a31ceeca9f9e4b4733c1f", + "sha256:e8797c0933c660e130cf2f51667f5003950d7e592f4c3944e8f04f201493d17a" + ], + "markers": "python_version >= '3.6'", + "version": "==1.21.52" + }, + "botocore-stubs": { + "hashes": [ + "sha256:9e97224b8b3db0c12027a0f9ffa5e4ea2bc959a1b4c344fc1e96f7dac81b5784", + "sha256:ff8b9121f24034e3cc31381c233674a695a69b76bdad4eac1a3debabaadf8bca" + ], + "markers": "python_version >= '3.6'", + "version": "==1.21.52" + }, + "certifi": { + "hashes": [ + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + ], + "version": "==2021.5.30" + }, + "jmespath": { + "hashes": [ + "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", + "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.0" + }, + "mypy-boto3-cloudformation": { + "hashes": [ + "sha256:42487340337d99bf335bd1339fa5d01978581804fb97e0246e0fd33c7fa97d87", + "sha256:924e9e6fd13159b59b1bd72346a28a298430de36d6e1efb2ff2ac74bdf8d6cbe" + ], + "version": "==1.18.52" + }, + "mypy-boto3-dynamodb": { + "hashes": [ + "sha256:859f3cdfc1c1cce286ca5b79a21f963844915f372e439125d6f3b568116e4f7c", + "sha256:865cb780439f4b22df2244acfe5d1f5d709b55b7ae8951ba3dee9c947c7819b2" + ], + "version": "==1.18.52" + }, + "mypy-boto3-ec2": { + "hashes": [ + "sha256:d048bf7946ff62c9c0e768f8424da67aaf1fa3e1f8f05b64fc0bf17647c6b5ce", + "sha256:e83cd9886a9b487e146baae7b2777625818893ccd08e94b37a407d491c97cdee" + ], + "version": "==1.18.52" + }, + "mypy-boto3-lambda": { + "hashes": [ + "sha256:36c3f2814720118d1e84fb7271b82388bae55e9727acea19e638879b6a19965f", + "sha256:6046600202d1fb35850413025baf313ad03cfd0879ff8630327ea8689fb339dc" + ], + "version": "==1.18.52" + }, + "mypy-boto3-rds": { + "hashes": [ + "sha256:530dfb7e03dc2e61894656ba4f4c174285b7bc40af1748efedda5338e850987c", + "sha256:d265ce323ea83a01ed3033aa7eccc50edf05d5927b4c3ffffc596686b2d2f18a" + ], + "version": "==1.18.52" + }, + "mypy-boto3-s3": { + "hashes": [ + "sha256:4393485658a19ffad38adefcad89197f762ab6c6f03bbfcf1ad8a356a988c576", + "sha256:d213b8ae7ca7d4c9b808ac2977e9b25cd02cfc47a4298b215748654185d87bdc" + ], + "version": "==1.18.52" + }, + "mypy-boto3-sqs": { + "hashes": [ + "sha256:23b0f9009e21e21b44718b27a3649075a10af785c6e2d5c83cb14b1503cc4633", + "sha256:65e67465d1f0a4c39204da1c446d898ca9c2011b1bd449c9e95c02a6ef51bd2f" + ], + "version": "==1.18.52" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "s3transfer": { + "hashes": [ + "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", + "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" + ], + "markers": "python_version >= '3.6'", + "version": "==0.5.0" + }, + "sentry-sdk": { + "hashes": [ + "sha256:b9844751e40710e84a457c5bc29b21c383ccb2b63d76eeaad72f7f1c808c8828", + "sha256:c091cc7115ff25fe3a0e410dbecd7a996f81a3f6137d2272daef32d6c3cfa6dc" + ], + "index": "pypi", + "version": "==1.4.3" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece", + "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.7" + } + }, + "develop": { + "attrs": { + "hashes": [ + "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1", + "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.2.0" + }, + "boto3": { + "hashes": [ + "sha256:27824d3767c5213f0006f71e552b912bc4659241b77ca8b0be0b813cf0518a9e", + "sha256:5b585a279478bd6df4b07db7d6150f413ba6add1f38e68aaa533d3337efd0b22" + ], + "index": "pypi", + "version": "==1.18.52" + }, + "botocore": { + "hashes": [ + "sha256:04c071aec4f1981b38e3be760838b976337fd6ebd95a31ceeca9f9e4b4733c1f", + "sha256:e8797c0933c660e130cf2f51667f5003950d7e592f4c3944e8f04f201493d17a" + ], + "markers": "python_version >= '3.6'", + "version": "==1.21.52" + }, + "certifi": { + "hashes": [ + "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", + "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + ], + "version": "==2021.5.30" + }, + "cffi": { + "hashes": [ + "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d", + "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771", + "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872", + "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c", + "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc", + "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762", + "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202", + "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5", + "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548", + "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a", + "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f", + "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20", + "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218", + "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c", + "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e", + "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56", + "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224", + "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a", + "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2", + "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a", + "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819", + "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346", + "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b", + "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e", + "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534", + "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb", + "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0", + "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156", + "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd", + "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87", + "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc", + "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195", + "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33", + "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f", + "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d", + "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd", + "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728", + "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7", + "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca", + "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99", + "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf", + "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e", + "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c", + "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5", + "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69" + ], + "version": "==1.14.6" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6", + "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f" + ], + "markers": "python_version >= '3'", + "version": "==2.0.6" + }, + "coverage": { + "hashes": [ + "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c", + "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6", + "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45", + "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a", + "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03", + "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529", + "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a", + "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a", + "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2", + "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6", + "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759", + "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53", + "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a", + "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4", + "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff", + "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502", + "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793", + "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb", + "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905", + "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821", + "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b", + "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81", + "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0", + "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b", + "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3", + "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184", + "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701", + "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a", + "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82", + "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638", + "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5", + "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083", + "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6", + "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90", + "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465", + "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a", + "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3", + "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e", + "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066", + "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf", + "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b", + "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae", + "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669", + "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873", + "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b", + "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6", + "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb", + "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160", + "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c", + "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079", + "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d", + "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6" + ], + "index": "pypi", + "version": "==5.5" + }, + "cryptography": { + "hashes": [ + "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6", + "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6", + "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c", + "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999", + "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e", + "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992", + "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d", + "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588", + "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa", + "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d", + "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd", + "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d", + "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953", + "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2", + "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8", + "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6", + "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9", + "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6", + "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad", + "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76" + ], + "markers": "python_version >= '3.6'", + "version": "==35.0.0" + }, + "idna": { + "hashes": [ + "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", + "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" + ], + "markers": "python_version >= '3'", + "version": "==3.2" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "jinja2": { + "hashes": [ + "sha256:1f06f2da51e7b56b8f238affdd6b4e2c61e39598a378cc49345bc1bd42a978a4", + "sha256:703f484b47a6af502e743c9122595cc812b0271f661722403114f71a79d0f5a4" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "jmespath": { + "hashes": [ + "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", + "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.0" + }, + "markupsafe": { + "hashes": [ + "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", + "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", + "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", + "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", + "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", + "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", + "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", + "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", + "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", + "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", + "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", + "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", + "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", + "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", + "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", + "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", + "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", + "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", + "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", + "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", + "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", + "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", + "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", + "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", + "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", + "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", + "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", + "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", + "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", + "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", + "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", + "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", + "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", + "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", + "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", + "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", + "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", + "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", + "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", + "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", + "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", + "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", + "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", + "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", + "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", + "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", + "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", + "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", + "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", + "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", + "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", + "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", + "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", + "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "more-itertools": { + "hashes": [ + "sha256:1debcabeb1df793814859d64a81ad7cb10504c24349368ccf214c664c474f41f", + "sha256:56ddac45541718ba332db05f464bebfb0768110111affd27f66e0051f276fa43" + ], + "markers": "python_version >= '3.5'", + "version": "==8.10.0" + }, + "moto": { + "hashes": [ + "sha256:418a47e65fd1a0001068d833b3e9924ce881f2e93e8d45b8cbc8bfa26bfeb7e0", + "sha256:ee745c018f3b279a06bf156c844c82a202ccd61e7cfb10075eca59fb1d490068" + ], + "index": "pypi", + "version": "==2.2.8" + }, + "mypy-boto3-dynamodb": { + "hashes": [ + "sha256:859f3cdfc1c1cce286ca5b79a21f963844915f372e439125d6f3b568116e4f7c", + "sha256:865cb780439f4b22df2244acfe5d1f5d709b55b7ae8951ba3dee9c947c7819b2" + ], + "version": "==1.18.52" + }, + "packaging": { + "hashes": [ + "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", + "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14" + ], + "markers": "python_version >= '3.6'", + "version": "==21.0" + }, + "pluggy": { + "hashes": [ + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.0" + }, + "py": { + "hashes": [ + "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3", + "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.10.0" + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" + }, + "pytest": { + "hashes": [ + "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89", + "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134" + ], + "index": "pypi", + "version": "==6.2.5" + }, + "pytest-cov": { + "hashes": [ + "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a", + "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7" + ], + "index": "pypi", + "version": "==2.12.1" + }, + "pytest-mock": { + "hashes": [ + "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3", + "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62" + ], + "index": "pypi", + "version": "==3.6.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", + "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" + ], + "version": "==2021.1" + }, + "requests": { + "hashes": [ + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + ], + "index": "pypi", + "version": "==2.26.0" + }, + "responses": { + "hashes": [ + "sha256:57bab4e9d4d65f31ea5caf9de62095032c4d81f591a8fac2f5858f7777b8567b", + "sha256:93f774a762ee0e27c0d9d7e06227aeda9ff9f5f69392f72bb6c6b73f8763563e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.14.0" + }, + "s3transfer": { + "hashes": [ + "sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c", + "sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803" + ], + "markers": "python_version >= '3.6'", + "version": "==0.5.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "urllib3": { + "hashes": [ + "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece", + "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.7" + }, + "werkzeug": { + "hashes": [ + "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42", + "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "xmltodict": { + "hashes": [ + "sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21", + "sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051" + ], + "version": "==0.12.0" + } + } +} diff --git a/remove_item_lambda/__init__.py b/remove_item_lambda/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/remove_item_lambda/config/__init__.py b/remove_item_lambda/config/__init__.py new file mode 100644 index 000000000..e7df9868d --- /dev/null +++ b/remove_item_lambda/config/__init__.py @@ -0,0 +1 @@ +__all__ = ['index'] diff --git a/remove_item_lambda/config/index.py b/remove_item_lambda/config/index.py new file mode 100644 index 000000000..d43ca457b --- /dev/null +++ b/remove_item_lambda/config/index.py @@ -0,0 +1,13 @@ +import os + +dynamodb = { + 'endpoint_url': os.getenv('AWS_DYNAMODB_ENDPOINT_URL', None), + 'recommendation_api_candidate_sets_table': + os.getenv('RECOMMENDATION_API_CANDIDATE_SETS_TABLE', 'recommendation_api_candidate_sets'), +} + +sentry = { + 'dsn': os.getenv('SENTRY_DSN', 'https://examplePublicKey@o0.ingest.sentry.io/0'), + 'release': os.getenv('GIT_SHA', '1234'), + 'environment': os.getenv('ENVIRONMENT', 'local') +} diff --git a/remove_item_lambda/sqs_handler.py b/remove_item_lambda/sqs_handler.py new file mode 100644 index 000000000..7c14ee02a --- /dev/null +++ b/remove_item_lambda/sqs_handler.py @@ -0,0 +1,23 @@ +import json +import logging +from typing import Dict, Any + +# Set up logging +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +def lambda_handler(event: Dict[str, Any], context=None): + """ + Handles SQS messages triggered by REMOVE_ITEM EventBridge events. + :param event: Event with one or multiple 'Records', where each record's 'body' contains the event payload. + :param context: Context provided by AWS Lambda, unused in this function. + """ + records = event['Records'] + + for record in records: + event_body = json.loads(record['body']) + logger.info("Processing REMOVE_ITEM event: %s", json.dumps(event_body, indent=2)) + + return { + 'message': f'Processed {len(records)} records.' + } diff --git a/remove_item_lambda/tests/__init__.py b/remove_item_lambda/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/remove_item_lambda/tests/fixtures/__init__.py b/remove_item_lambda/tests/fixtures/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/remove_item_lambda/tests/test_sqs_lambda.py b/remove_item_lambda/tests/test_sqs_lambda.py new file mode 100644 index 000000000..e69de29bb From 99eb13b469f70fb9a7aa5beb3e899734d2ec3a1d Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Wed, 20 Nov 2024 16:12:24 -0800 Subject: [PATCH 2/8] fix: ci Lambda names --- .circleci/config.yml | 68 ++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3d047438..9e18cb42b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,18 +170,23 @@ jobs: parameters: env_lower_name: type: string - description: The lower case env name + description: The lower case environment name (e.g., dev or prod) env_capital_name: - default: Env Name - description: The env capital name type: string - deploy: - type: boolean - default: true + description: The capitalized environment name (e.g., Dev or Prod) + lambda_lower_name: + type: string + description: Lowercase part of the Lambda name (e.g., sqs-removeeventhandler or sqs-translation) + lambda_capital_name: + type: string + description: Capitalized part of the Lambda name (e.g., SqsRemoveeventhandler or SqsTranslation) lambda_path: type: string - description: Path to the lambda directory + description: Path to the Lambda source code (e.g., aws_lambda/ or remove_item_lambda/) default: "aws_lambda/" + deploy: + type: boolean + default: true docker: - image: python:3.9 auth: @@ -211,16 +216,17 @@ jobs: zip -gr "/tmp/$CIRCLE_SHA1.zip" <> -x <>/Pipfile* cp "/tmp/$CIRCLE_SHA1.zip" /tmp/build.zip - run: - name: Upload Package - command: aws s3 cp "/tmp/$CIRCLE_SHA1.zip" s3://pocket-recommendationapi-<< parameters.env_lower_name >>-sqs-translation/ + name: Upload Package to S3 + command: | + aws s3 cp "/tmp/$CIRCLE_SHA1.zip" s3://pocket-recommendationapi-<< parameters.env_lower_name >>-<< parameters.lambda_lower_name >>/ - pocket/deploy_lambda: - s3-bucket: pocket-recommendationapi-<< parameters.env_lower_name >>-sqs-translation + s3-bucket: pocket-recommendationapi-<< parameters.env_lower_name >>-<< parameters.lambda_lower_name >> aws-access-key-id: << parameters.env_capital_name >>_AWS_ACCESS_KEY aws-secret-access-key: << parameters.env_capital_name >>_AWS_SECRET_ACCESS_KEY aws-region: << parameters.env_capital_name >>_AWS_DEFAULT_REGION - codedeploy-application-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_path >>-Lambda - codedeploy-deployment-group-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_path >>-Lambda - function-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_path >>-Function + codedeploy-application-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_capital_name >>-Lambda + codedeploy-deployment-group-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_capital_name >>-Lambda + function-name: RecommendationAPI-<< parameters.env_capital_name >>-<< parameters.lambda_capital_name >>-Function - store_artifacts: path: /tmp/build.zip @@ -240,51 +246,51 @@ workflows: - build - # Build & Deploy Development Lambdas + # Deploy Dev Lambdas - lambda: <<: *only_dev context: pocket - name: deploy_lambda_dev_aws + name: deploy_lambda_dev_remove_item env_lower_name: dev env_capital_name: Dev - lambda_path: "aws_lambda/" + lambda_lower_name: sqs-removeeventhandler + lambda_capital_name: Sqs-RemoveEventHandler + lambda_path: "remove_item_lambda/" deploy: true - requires: - - setup-deploy-params-dev - lambda: <<: *only_dev context: pocket - name: deploy_lambda_dev_remove_item + name: deploy_lambda_dev_translation env_lower_name: dev env_capital_name: Dev - lambda_path: "remove_item_lambda/" + lambda_lower_name: sqs-translation + lambda_capital_name: Sqs-Translation + lambda_path: "aws_lambda/" deploy: true - requires: - - setup-deploy-params-dev - # Build & Deploy Production Lambdas + # Deploy Prod Lambdas - lambda: <<: *only_main context: pocket - name: deploy_lambda_prod_aws + name: deploy_lambda_prod_remove_item env_lower_name: prod env_capital_name: Prod - lambda_path: "aws_lambda/" + lambda_lower_name: sqs-removeeventhandler + lambda_capital_name: Sqs-RemoveEventHandler + lambda_path: "remove_item_lambda/" deploy: true - requires: - - setup-deploy-params-prod - lambda: <<: *only_main context: pocket - name: deploy_lambda_prod_remove_item + name: deploy_lambda_prod_translation env_lower_name: prod env_capital_name: Prod - lambda_path: "remove_item_lambda/" + lambda_lower_name: sqs-translation + lambda_capital_name: Sqs-Translation + lambda_path: "aws_lambda/" deploy: true - requires: - - setup-deploy-params-prod # Try building the ECS docker image on each branch - pocket/docker_build: From 25d7d99b15a8b969d4f60ee40187ffd7639c4cb4 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Thu, 21 Nov 2024 10:00:43 -0800 Subject: [PATCH 3/8] fix: Lambda handler name --- .aws/src/removeItemLambda.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.aws/src/removeItemLambda.ts b/.aws/src/removeItemLambda.ts index 5e1337874..ab645a1d7 100644 --- a/.aws/src/removeItemLambda.ts +++ b/.aws/src/removeItemLambda.ts @@ -24,7 +24,7 @@ export class RemoveItemLambda extends Construct { }, lambda: { runtime: 'python3.9' as LAMBDA_RUNTIMES, - handler: 'remove_handler.lambda_handler', + handler: 'remove_item_lambda.sqs_handler.handler', timeout: 120, executionPolicyStatements: [ { From 72c5764bfb09785d06d36d875a2a31252f0c8990 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Thu, 21 Nov 2024 11:11:41 -0800 Subject: [PATCH 4/8] fix: Lambda handler name (2) --- remove_item_lambda/sqs_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remove_item_lambda/sqs_handler.py b/remove_item_lambda/sqs_handler.py index 7c14ee02a..d0b191393 100644 --- a/remove_item_lambda/sqs_handler.py +++ b/remove_item_lambda/sqs_handler.py @@ -6,7 +6,7 @@ logger = logging.getLogger() logger.setLevel(logging.INFO) -def lambda_handler(event: Dict[str, Any], context=None): +def handler(event: Dict[str, Any], context=None): """ Handles SQS messages triggered by REMOVE_ITEM EventBridge events. :param event: Event with one or multiple 'Records', where each record's 'body' contains the event payload. From 0e88bf7e482f1d30939d28cb1aa9ab2f9bae0326 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Thu, 21 Nov 2024 12:18:15 -0800 Subject: [PATCH 5/8] fix: event rule --- .aws/src/removeItemLambda.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.aws/src/removeItemLambda.ts b/.aws/src/removeItemLambda.ts index ab645a1d7..96c17d0c4 100644 --- a/.aws/src/removeItemLambda.ts +++ b/.aws/src/removeItemLambda.ts @@ -55,7 +55,7 @@ export class RemoveItemLambda extends Construct { description: 'Event rule for REMOVE_ITEM events to SQS', eventPattern: JSON.stringify({ source: ['curation-migration-datasync'], - detailType: ['REMOVE_ITEM'], + "detail-type": ['REMOVE_ITEM'], }), eventBusName: 'default', }); From 4ccb313a2d2fade4d97605e3ef9141ce16a05d2f Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Fri, 22 Nov 2024 15:38:59 -0800 Subject: [PATCH 6/8] fix: detail-type --- .aws/src/removeItemLambda.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.aws/src/removeItemLambda.ts b/.aws/src/removeItemLambda.ts index 96c17d0c4..4a2e4bb3a 100644 --- a/.aws/src/removeItemLambda.ts +++ b/.aws/src/removeItemLambda.ts @@ -55,7 +55,7 @@ export class RemoveItemLambda extends Construct { description: 'Event rule for REMOVE_ITEM events to SQS', eventPattern: JSON.stringify({ source: ['curation-migration-datasync'], - "detail-type": ['REMOVE_ITEM'], + 'detail-type': ['remove-approved-item'], }), eventBusName: 'default', }); From f8cee0808c530346e46139b2b8a74d6f8d965c80 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Fri, 22 Nov 2024 16:17:18 -0800 Subject: [PATCH 7/8] fix: add sqs policy --- .aws/src/removeItemLambda.ts | 41 ++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/.aws/src/removeItemLambda.ts b/.aws/src/removeItemLambda.ts index 4a2e4bb3a..c3749405d 100644 --- a/.aws/src/removeItemLambda.ts +++ b/.aws/src/removeItemLambda.ts @@ -1,11 +1,15 @@ -import { Construct } from 'constructs'; -import { config } from './config'; -import { PocketSQSWithLambdaTarget, PocketVPC } from '@pocket-tools/terraform-modules'; -import { LAMBDA_RUNTIMES } from '@pocket-tools/terraform-modules'; -import { CloudwatchEventRule } from '@cdktf/provider-aws/lib/cloudwatch-event-rule'; -import { CloudwatchEventTarget } from '@cdktf/provider-aws/lib/cloudwatch-event-target'; -import { DataAwsSsmParameter } from '@cdktf/provider-aws/lib/data-aws-ssm-parameter'; -import { Elasticache } from './elasticache'; +import {Construct} from 'constructs'; +import {config} from './config'; +import { + LAMBDA_RUNTIMES, + PocketSQSWithLambdaTarget, + PocketVPC +} from '@pocket-tools/terraform-modules'; +import {CloudwatchEventRule} from '@cdktf/provider-aws/lib/cloudwatch-event-rule'; +import {CloudwatchEventTarget} from '@cdktf/provider-aws/lib/cloudwatch-event-target'; +import {DataAwsSsmParameter} from '@cdktf/provider-aws/lib/data-aws-ssm-parameter'; +import {Elasticache} from './elasticache'; +import {SqsQueuePolicy} from '@cdktf/provider-aws/lib/sqs-queue-policy'; export class RemoveItemLambda extends Construct { constructor(scope: Construct, name: string, elasticache: Elasticache) { @@ -15,7 +19,7 @@ export class RemoveItemLambda extends Construct { const { sentryDsn, gitSha } = this.getEnvVariableValues(); - const sqsWithLambda = new PocketSQSWithLambdaTarget(this, 'remove-sqs-lambda', { + const sqsWithLambda = new PocketSQSWithLambdaTarget(this, 'remove-sqs-lambda', { name: `${config.prefix}-Sqs-RemoveEventHandler`, batchSize: 1, sqsQueue: { @@ -53,6 +57,8 @@ export class RemoveItemLambda extends Construct { const eventRule = new CloudwatchEventRule(this, 'remove-event-rule', { name: `${config.prefix}-RemoveEventRule`, description: 'Event rule for REMOVE_ITEM events to SQS', + // source and detail-type are defined in: + // https://github.com/Pocket/content-monorepo/blob/main/servers/curated-corpus-api/src/config/index.ts eventPattern: JSON.stringify({ source: ['curation-migration-datasync'], 'detail-type': ['remove-approved-item'], @@ -64,6 +70,23 @@ export class RemoveItemLambda extends Construct { rule: eventRule.name, arn: sqsWithLambda.sqsQueueResource.arn, }); + + // Allow the EventRule to invoke the SQS Queue + new SqsQueuePolicy(this, 'sqs-queue-policy', { + queueUrl: sqsWithLambda.sqsQueueResource.url, + policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { Service: 'events.amazonaws.com' }, + Action: 'sqs:SendMessage', + Resource: sqsWithLambda.sqsQueueResource.arn, + Condition: { ArnEquals: { 'aws:SourceArn': eventRule.arn } }, + }, + ], + }), + }); } private getEnvVariableValues() { From 2404a795dac9763ae99e2fc0bf8534bfbe6162ed Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Thu, 19 Dec 2024 10:27:13 +0100 Subject: [PATCH 8/8] revert removal of comments and debug code --- .aws/src/elasticache.ts | 16 ++++++++++++++-- .circleci/config.yml | 11 +++++++++++ app/main.py | 18 +----------------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/.aws/src/elasticache.ts b/.aws/src/elasticache.ts index fef8d1f26..e715f5e49 100644 --- a/.aws/src/elasticache.ts +++ b/.aws/src/elasticache.ts @@ -26,6 +26,11 @@ export class Elasticache extends Construct { const pocketVPC = new PocketVPC(scope, 'pocket-shared-vpc'); const elasticache = new ApplicationMemcache(scope, 'memcached', { + //Usually we would set the security group ids of the service that needs to hit this. + //However we don't have the necessary security group because it gets created in PocketALBApplication + //So instead we set it to null and allow anything within the vpc to access it. + //This is not ideal.. + //Ideally we need to be able to add security groups to the ALB application. allowedIngressSecurityGroupIds: undefined, node: { count: config.cacheNodes, @@ -36,9 +41,16 @@ export class Elasticache extends Construct { prefix: config.prefix, }); - const nodeList: string[] = []; + let nodeList: string[] = []; for (let i = 0; i < config.cacheNodes; i++) { - nodeList.push(`${elasticache.elasticacheCluster.cacheNodes.get(i).address}:11211`); + // ${elasticache.elasticacheClister.cacheNodes(i.toString()).port} has a bug and is not rendering the proper terraform address + // its rendering -1.8881545897087503e+289 for some weird reason... + // For now we just hardcode to 11211 which is the default memcache port. + nodeList.push( + `${ + elasticache.elasticacheCluster.cacheNodes.get(i).address + }:11211` + ); } const clusterArn = elasticache.elasticacheCluster.arn; diff --git a/.circleci/config.yml b/.circleci/config.yml index 9e18cb42b..8f4873655 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -203,6 +203,13 @@ jobs: aws-region: << parameters.env_capital_name >>_AWS_DEFAULT_REGION - run: name: Package Lambda + # We bundle our code in a way that lambda can understand and execute + # - pipenv requirements > requirements.txt - output a requirements.txt file from Pipfile + # - pip install -r requirements.txt --no-deps -t package | installs the requirements into a "package" directory + # - change dir into the "package" dir and add all the packages to a zip file as /tmp + # - change dir into the "aws_lambda" dir and remove all files and folder we want to exclude from the build + # - change dir into the root of the application and add the "aws_lambda" dir to the zip file excluding the Pipfile and Pipfile.lock + # - copy the zip file to /tmp/build.zip and store as a CI artifact for quick inspection of the build command: | apt-get update && apt-get install zip pip install pipenv==2022.8.15 @@ -280,6 +287,8 @@ workflows: lambda_capital_name: Sqs-RemoveEventHandler lambda_path: "remove_item_lambda/" deploy: true + requires: + - setup-deploy-params-prod - lambda: <<: *only_main @@ -291,6 +300,8 @@ workflows: lambda_capital_name: Sqs-Translation lambda_path: "aws_lambda/" deploy: true + requires: + - setup-deploy-params-prod # Try building the ECS docker image on each branch - pocket/docker_build: diff --git a/app/main.py b/app/main.py index 33e1d7f6a..863d659a9 100644 --- a/app/main.py +++ b/app/main.py @@ -2,7 +2,7 @@ import sentry_sdk import uvicorn -from fastapi import FastAPI, Request, Response, status +from fastapi import FastAPI, Response, status from opentelemetry import trace from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor @@ -27,7 +27,6 @@ from app.singletons import DiContainer logging.getLogger().setLevel(log_level) -logger = logging.getLogger(__name__) sentry_sdk.init( dsn=sentry_config['dsn'], @@ -105,21 +104,6 @@ async def load_slate_configs(): async def startup_event(): DiContainer.init() -@app.post("/remove") -async def remove_event(request: Request, response: Response): - """ - Handles remove events triggered by EventBridge. - Logs the event details for debugging or processing. - """ - try: - body = await request.json() - logger.info("Received remove event: %s", body) - return {"status": "success", "message": "Event logged successfully."} - except Exception as e: - logger.error("Error processing remove event: %s", str(e)) - response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR - return {"status": "error", "message": "Failed to process event."} - if __name__ == "__main__": # This runs uvicorn in a local development environment.