From 63840a5df295d397e700fc5e996014a9e502824e Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 2 Dec 2025 19:33:05 -0900 Subject: [PATCH 1/4] add log bucket, configure access logging, and s3 inventory. Update betolink permisions to access logs --- its-live-data/bucket-cf.yml | 204 ++++++++++++++++++++++++ its-live-data/its-live-iam-users-cf.yml | 9 +- 2 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 its-live-data/bucket-cf.yml diff --git a/its-live-data/bucket-cf.yml b/its-live-data/bucket-cf.yml new file mode 100644 index 0000000..56b91b0 --- /dev/null +++ b/its-live-data/bucket-cf.yml @@ -0,0 +1,204 @@ +# customized version of the docs/pds-bucket-cf.yml template for creating the its-live-data bucket. +# Customization includes a log bucket and a logging configuration and S3 inventory configuration on the dataset bucket. +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: This template creates the AWS infrastructure to publish a public data set on S3. It creates a publicly-accessible S3 bucket for the dataset, enables CloudWatch Metrics for the dataset bucket, and creates a public SQS and Lambda subscribable SNS Topic. Additionally, it includes a log bucket, logging configuration, and an inventory configuration. + +Parameters: + DataSetName: + AllowedPattern: '[a-z0-9\.\-]*' + ConstraintDescription: may only contain lowercase letters, numbers, and ., or - characters + Description: "The name of the dataset's S3 bucket. This will be used to create the dataset S3 bucket." + MaxLength: '250' + MinLength: '1' + Type: String + +Resources: + SNSTopic: + Properties: + TopicName: !Join [ "", [ !Join [ "", !Split [ ".", !Ref DataSetName ] ], "-object_created" ] ] + Type: AWS::SNS::Topic + + SNSTopicPolicy: + Properties: + Topics: + - !Ref SNSTopic + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: allowS3BucketToPublish + Effect: Allow + Action: + - sns:Publish + Resource: !Ref SNSTopic + Principal: + Service: s3.amazonaws.com + Condition: + ArnLike: + aws:SourceArn: !Sub arn:aws:s3:::${DataSetName} + StringEquals: + aws:SourceAccount: !Ref AWS::AccountId + - Sid: allowOnlySQSandLambdaSubscription + Effect: Allow + Action: + - sns:Subscribe + Resource: !Ref SNSTopic + Principal: + AWS: "*" + Condition: + StringEquals: + SNS:Protocol: + - sqs + - lambda + Type: AWS::SNS::TopicPolicy + + LogBucket: + Type: AWS::S3::Bucket + Properties: + PublicAccessBlockConfiguration: + BlockPublicAcls: True + IgnorePublicAcls: True + BlockPublicPolicy: True + RestrictPublicBuckets: True + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + BucketKeyEnabled: true + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + LifecycleConfiguration: + Rules: + - Id: ExpireAll + Status: Enabled + ExpirationInDays: 30 + - Id: ExpireVelocityInventory + Prefix: 'inventory/velocity_image_pair/' + Status: Enabled + ExpirationInDays: 3 + LogBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref LogBucket + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - logging.s3.amazonaws.com + - s3.amazonaws.com + Action: s3:PutObject + Resource: !Sub "${LogBucket.Arn}/*" + Condition: + ArnLike: + "aws:SourceArn": !GetAtt DataSetBucket.Arn + StringEquals: + "aws:SourceAccount": !Ref AWS::AccountId + + DataSetBucket: + DeletionPolicy: Retain + UpdateReplacePolicy: Delete + DependsOn: + - SNSTopicPolicy + Properties: + BucketName: !Ref DataSetName + LoggingConfiguration: + DestinationBucketName: !Ref LogBucket + TargetObjectKeyFormat: + PartitionedPrefix: + PartitionDateSource: EventTime + MetricsConfigurations: + - Id: EntireBucket + InventoryConfigurations: + - Id: VelocityGranuleInventory + Destination: + BucketAccountId: !Ref AWS::AccountId + BucketArn: !GetAtt LogBucket.Arn + Format: Parquet + Prefix: inventory/velocity_image_pair + Enabled: true + Prefix: velocity_image_pair/ + IncludedObjectVersions: Current + OptionalFields: + - Size + - LastModifiedDate + - StorageClass + - IntelligentTieringAccessTier + ScheduleFrequency: Daily + - Id: BucketInventory + Destination: + BucketAccountId: !Ref AWS::AccountId + BucketArn: !GetAtt LogBucket.Arn + Format: Parquet + Prefix: inventory/all + Enabled: true + IncludedObjectVersions: Current + OptionalFields: + - Size + - LastModifiedDate + - StorageClass + - IntelligentTieringAccessTier + ScheduleFrequency: Weekly + LifecycleConfiguration: + Rules: + - Id: IntelligentTieringRule + Status: Enabled + Transitions: + - TransitionInDays: '0' + StorageClass: INTELLIGENT_TIERING + - Id: AbortIncompleteMultipartUploadRule + Status: Enabled + AbortIncompleteMultipartUpload: + DaysAfterInitiation: 7 + NotificationConfiguration: + TopicConfigurations: + - Event: "s3:ObjectCreated:*" + Topic: !Ref SNSTopic + PublicAccessBlockConfiguration: + BlockPublicPolicy: false + RestrictPublicBuckets: false + CorsConfiguration: + CorsRules: + - AllowedHeaders: + - "*" + AllowedMethods: + - HEAD + - GET + AllowedOrigins: + - "*" + ExposedHeaders: + - ETag + - x-amz-meta-custom-header + MaxAge: 3000 + Type: AWS::S3::Bucket + + DataSetBucketPolicy: + Properties: + Bucket: !Ref DataSetBucket + PolicyDocument: + Statement: + - Action: + - s3:List* + - s3:Get* + Effect: Allow + Principal: "*" + Resource: + - !Sub arn:aws:s3:::${DataSetBucket}/* + - !Sub arn:aws:s3:::${DataSetBucket} + Condition: + Bool: + aws:SecureTransport: true + Type: AWS::S3::BucketPolicy + +Outputs: + DataBucket: + Description: "S3 data bucket name" + Value: !Sub ${DataSetBucket} + LogBucket: + Description: "S3 log bucket name" + Value: !Sub ${LogBucket} + SNSTopic: + Description: "SQS and Lambda subscribable SNS Topic" + Value: !Ref SNSTopic diff --git a/its-live-data/its-live-iam-users-cf.yml b/its-live-data/its-live-iam-users-cf.yml index ce84c0a..4d6b2ad 100644 --- a/its-live-data/its-live-iam-users-cf.yml +++ b/its-live-data/its-live-iam-users-cf.yml @@ -17,4 +17,11 @@ Resources: - Effect: Allow Action: "s3:*" Resource: - - "arn:aws:s3:::its-live-data/cloud-experiments/*" + - "arn:aws:s3:::its-live-data/cloud-experiments/*" + - Effect: Allow + Action: + - s3:List* + - s3:Get* + Resource: + - "arn:aws:s3:::pds-buckets-its-live-logbucket-70tr3aw5f2op/*" + - "arn:aws:s3:::pds-buckets-its-live-logbucket-70tr3aw5f2op" From 17275b80ed73d9d52122a0c25c5bcfedd8762973 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Wed, 3 Dec 2025 10:34:00 -0900 Subject: [PATCH 2/4] add access to the its-live service user in the JPL AWS ITS_LIVE account --- its-live-data/bucket-cf.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/its-live-data/bucket-cf.yml b/its-live-data/bucket-cf.yml index 56b91b0..5f64110 100644 --- a/its-live-data/bucket-cf.yml +++ b/its-live-data/bucket-cf.yml @@ -180,16 +180,28 @@ Resources: PolicyDocument: Statement: - Action: - - s3:List* - - s3:Get* + - s3:List* + - s3:Get* Effect: Allow Principal: "*" Resource: - - !Sub arn:aws:s3:::${DataSetBucket}/* - - !Sub arn:aws:s3:::${DataSetBucket} + - !Sub arn:aws:s3:::${DataSetBucket}/* + - !Sub arn:aws:s3:::${DataSetBucket} Condition: Bool: aws:SecureTransport: true + # used by Masha (JPL) to generate composites, cogs, etc + - Effect: Allow + Action: + - s3:PutObject + - s3:PutObjectTagging + - s3:DeleteObject + - s3:DeleteObjectTagging + Resource: + - !Sub arn:aws:s3:::${DataSetBucket}/* + Principal: + AWS: arn:aws:iam::986442313181:user/srv-its-live + Type: AWS::S3::BucketPolicy Outputs: From 950cd90aa242741c1d621fcf570854b349a2d8f9 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 5 Dec 2025 11:33:36 -0900 Subject: [PATCH 3/4] update its-live-data S3 permission for betolink --- its-live-data/its-live-iam-users-cf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/its-live-data/its-live-iam-users-cf.yml b/its-live-data/its-live-iam-users-cf.yml index 4d6b2ad..653cc39 100644 --- a/its-live-data/its-live-iam-users-cf.yml +++ b/its-live-data/its-live-iam-users-cf.yml @@ -17,7 +17,7 @@ Resources: - Effect: Allow Action: "s3:*" Resource: - - "arn:aws:s3:::its-live-data/cloud-experiments/*" + - "arn:aws:s3:::its-live-data/test-space/*" - Effect: Allow Action: - s3:List* From b4ce6be2b2997cdbc2deb186f697de79eb17eeae Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 5 Dec 2025 11:35:54 -0900 Subject: [PATCH 4/4] remove bad copy-pasta --- its-live-data/its-live-iam-users-cf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/its-live-data/its-live-iam-users-cf.yml b/its-live-data/its-live-iam-users-cf.yml index 653cc39..f520e62 100644 --- a/its-live-data/its-live-iam-users-cf.yml +++ b/its-live-data/its-live-iam-users-cf.yml @@ -1,4 +1,4 @@ -# aws cloudformation deploy --profile opendata-its-live \  SIG(126) ↵  15:53:59 +# aws cloudformation deploy --profile opendata-its-live \ # --stack-name its-live-iam-users \ # --template-file its-live-data/its-live-iam-users-cf.yml \ # --capabilities CAPABILITY_NAMED_IAM