diff --git a/common.yml b/common.yml index 4268ec2b5..636f6c381 100644 --- a/common.yml +++ b/common.yml @@ -343,17 +343,31 @@ functions: content_type: application/octet-stream "upload release packages to s3": + - command: ec2.assume_role + params: + role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools" - command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools + env: + NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} script: | ${_set_shell_env} go run release/release.go upload-release "upload release json feed to s3": + - command: ec2.assume_role + params: + role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools" - command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools + env: + NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} script: | ${_set_shell_env} go run release/release.go upload-json @@ -369,9 +383,16 @@ functions: permissions: public-read "generate full JSON feed": + - command: ec2.assume_role + params: + role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools" - command: shell.exec params: working_dir: src/github.com/mongodb/mongo-tools + env: + NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} script: | ${_set_shell_env} go run release/release.go generate-full-json diff --git a/release/aws/aws.go b/release/aws/aws.go index 6aeac460c..ef9c01808 100644 --- a/release/aws/aws.go +++ b/release/aws/aws.go @@ -14,6 +14,7 @@ import ( "regexp" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -24,6 +25,7 @@ var awsClient *AWS type AWS struct { session *session.Session + acl string } func initializeClient() error { @@ -40,10 +42,28 @@ func initializeClient() error { awsClient = &AWS{ session: s, + acl: "public-read", } return nil } +func NewClientFromCredentials(id, secret, token, acl string) (*AWS, error) { + s, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Credentials: credentials.NewStaticCredentials(id, secret, token), + }) + if err != nil { + return nil, fmt.Errorf("failed to create AWS session: %w", err) + } + + c := &AWS{ + session: s, + acl: acl, + } + + return c, nil +} + // GetClient returns the global AWS client. // It initializes the AWS client if it hasn't already been initialized. func GetClient() (*AWS, error) { @@ -75,7 +95,7 @@ func (a *AWS) UploadBytes(bucket, objPath, filename string, reader io.Reader) er _, err := uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(bucket), Key: aws.String(key), - ACL: aws.String("public-read"), + ACL: aws.String(a.acl), Body: reader, }) if err != nil { diff --git a/release/release.go b/release/release.go index ad0747d3a..6d03c00a6 100644 --- a/release/release.go +++ b/release/release.go @@ -953,13 +953,25 @@ func generateFullReleaseJSON(v version.Version) { return } - awsClient, err := aws.GetClient() - check(err, "get aws client") + awsClientOld, err := aws.GetClient() + check(err, "get aws client old") + + newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID") + newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY") + newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN") + + awsClientNew, err := aws.NewClientFromCredentials( + newAccessKeyID, + newSecretAccessKey, + newSessionToken, + "private", + ) + check(err, "new aws client from credentials") - feed, err := awsClient.GenerateFullReleaseFeedFromObjects() + feed, err := awsClientOld.GenerateFullReleaseFeedFromObjects() check(err, "generate full release feed from s3 objects") - uploadFeedFile("full.json", feed, awsClient) + uploadFeedFile("full.json", feed, awsClientOld, awsClientNew) } func uploadReleaseJSON(v version.Version) { @@ -996,8 +1008,20 @@ func uploadReleaseJSON(v version.Version) { log.Fatalf("found %d sign tasks, but expected %d", len(signTasks), pfCount) } - awsClient, err := aws.GetClient() - check(err, "get aws client") + awsClientOld, err := aws.GetClient() + check(err, "get aws client old") + + newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID") + newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY") + newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN") + + awsClientNew, err := aws.NewClientFromCredentials( + newAccessKeyID, + newSecretAccessKey, + newSessionToken, + "private", + ) + check(err, "new aws client from credentials") // Accumulate all downloaded artifacts from sign tasks for JSON feed. var dls []*download.ToolsDownload @@ -1061,8 +1085,18 @@ func uploadReleaseJSON(v version.Version) { } // Download the current full.json - buff, err := awsClient.DownloadFile("downloads.mongodb.org", "tools/db/full.json") - check(err, "download full.json") + const addr = "https://downloads.mongodb.org/tools/db/full.json" + res, err := http.Get(addr) + check(err, "http get full.json") + + defer res.Body.Close() + + buff, err := io.ReadAll(res.Body) + check(err, "read full.json body") + + if res.StatusCode != http.StatusOK { + panic(fmt.Errorf("get full.json status %d: %s", res.StatusCode, string(buff))) + } var fullFeed download.JSONFeed @@ -1074,7 +1108,7 @@ func uploadReleaseJSON(v version.Version) { fullFeed.Versions, &download.ToolsVersion{Version: v.String(), Downloads: dls}, ) - uploadFeedFile("full.json", &fullFeed, awsClient) + uploadFeedFile("full.json", &fullFeed, awsClientOld, awsClientNew) // Upload only the most recent version to release.json var feed download.JSONFeed @@ -1083,10 +1117,10 @@ func uploadReleaseJSON(v version.Version) { &download.ToolsVersion{Version: v.String(), Downloads: dls}, ) - uploadFeedFile("release.json", &feed, awsClient) + uploadFeedFile("release.json", &feed, awsClientOld, awsClientNew) } -func uploadFeedFile(filename string, feed *download.JSONFeed, awsClient *aws.AWS) { +func uploadFeedFile(filename string, feed *download.JSONFeed, awsClientOld, awsClientNew *aws.AWS) { var feedBuffer bytes.Buffer jsonEncoder := json.NewEncoder(&feedBuffer) @@ -1094,12 +1128,31 @@ func uploadFeedFile(filename string, feed *download.JSONFeed, awsClient *aws.AWS err := jsonEncoder.Encode(*feed) check(err, "encode json feed") + feedBytes := feedBuffer.Bytes() + log.Printf( "uploading download feed to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n", filename, ) - err = awsClient.UploadBytes("downloads.mongodb.org", "/tools/db", filename, &feedBuffer) - check(err, "upload json feed") + err = awsClientOld.UploadBytes( + "downloads.mongodb.org", + "/tools/db", + filename, + bytes.NewReader(feedBytes), + ) + check(err, "upload json feed old") + + log.Printf( + "uploading download feed to s3://cdn-origin-db-tools/tools/db/%s\n", + filename, + ) + err = awsClientNew.UploadBytes( + "cdn-origin-db-tools", + "/tools/db", + filename, + bytes.NewReader(feedBytes), + ) + check(err, "upload json feed new") } func uploadRelease(v version.Version) { @@ -1129,8 +1182,20 @@ func uploadRelease(v version.Version) { log.Fatalf("found %d sign tasks, but expected one", len(signTasks)) } - awsClient, err := aws.GetClient() - check(err, "get aws client") + awsClientOld, err := aws.GetClient() + check(err, "get old aws client") + + newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID") + newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY") + newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN") + + awsClientNew, err := aws.NewClientFromCredentials( + newAccessKeyID, + newSecretAccessKey, + newSessionToken, + "private", + ) + check(err, "new aws client from credentials") for _, task := range signTasks { log.Printf("\ngetting artifacts for %s\n", task.Variant) @@ -1182,14 +1247,33 @@ func uploadRelease(v version.Version) { " uploading to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n", stableFile, ) - err = awsClient.UploadFile("downloads.mongodb.org", "/tools/db", stableFile) + err = awsClientOld.UploadFile("downloads.mongodb.org", "/tools/db", stableFile) check(err, "uploading %q file to S3", stableFile) log.Printf( " uploading to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n", latestStableFile, ) - err = awsClient.UploadFile("downloads.mongodb.org", "/tools/db", latestStableFile) + + err = awsClientOld.UploadFile( + "downloads.mongodb.org", + "/tools/db", + latestStableFile, + ) check(err, "uploading %q file to S3", latestStableFile) + + log.Printf( + " uploading to s3://cdn-origin-db-tools/tools/db/%s\n", + stableFile, + ) + err = awsClientNew.UploadFile("cdn-origin-db-tools", "/tools/db", stableFile) + check(err, "uploading %q file to S3 new", stableFile) + log.Printf( + " uploading to s3://cdn-origin-db-tools/tools/db/%s\n", + latestStableFile, + ) + + err = awsClientNew.UploadFile("cdn-origin-db-tools", "/tools/db", latestStableFile) + check(err, "uploading %q file to S3 new", latestStableFile) } } }