From 637acec9e6cf040f6a629ade83e7db3ba4935fe5 Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Thu, 6 Nov 2025 10:02:38 +1100 Subject: [PATCH 1/2] Initial work on dropbox subcommand --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 5884b2f..8bc5e16 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,29 @@ This will store the encrypted file at `keys/production/oauth.pem.asc`. cat oauth.pen.asc | bay kms decrypt > oauth.pem ``` +## Dropbox + +Commands for pushing and pulling objects to the bay dropbox bucket. Projects can push and pull any files underneath their [project-name] directory in the dropbox bucket. + +### Usage + +Pull files from dropbox bucket. + +``` +bay dropbox pull \ + --remote-path project-name/file.zip \ + --local-path file.zip +``` + +Push files to dropbox bucket. + +``` +bay dropbox push \ + --local-path file.zip \ + --remote-path project-name/file.zip +``` + + ## Elastic Cloud Commands for querying and interacting with the Elastic Cloud API. From d7565136bedcd4eb7dd2ec999e157a16bae5adf8 Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Fri, 14 Nov 2025 16:18:43 +1100 Subject: [PATCH 2/2] Initial work on filestore pull command --- README.md | 21 +++----- cmd/filestore/pull.go | 96 ++++++++++++++++++++++++++++++++++++ go.mod | 32 +++++++----- go.sum | 38 ++++++++++++++ internal/helpers/aws.go | 15 ++++++ internal/helpers/checksum.go | 48 ++++++++++++++++++ main.go | 39 +++++++++++++++ 7 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 cmd/filestore/pull.go create mode 100644 internal/helpers/checksum.go diff --git a/README.md b/README.md index 8bc5e16..21d2987 100644 --- a/README.md +++ b/README.md @@ -21,29 +21,20 @@ This will store the encrypted file at `keys/production/oauth.pem.asc`. cat oauth.pen.asc | bay kms decrypt > oauth.pem ``` -## Dropbox +## s3 -Commands for pushing and pulling objects to the bay dropbox bucket. Projects can push and pull any files underneath their [project-name] directory in the dropbox bucket. +Commands for pushing and pulling objects to the bay s3 buckets. ### Usage -Pull files from dropbox bucket. +Pull file from bay-db-image bucket. ``` -bay dropbox pull \ - --remote-path project-name/file.zip \ - --local-path file.zip +bay filestore pull \ + --remote-path s3://bay-db-image/project-name/production.sql \ + --local-path project-name-production.sql ``` -Push files to dropbox bucket. - -``` -bay dropbox push \ - --local-path file.zip \ - --remote-path project-name/file.zip -``` - - ## Elastic Cloud Commands for querying and interacting with the Elastic Cloud API. diff --git a/cmd/filestore/pull.go b/cmd/filestore/pull.go new file mode 100644 index 0000000..275a9a1 --- /dev/null +++ b/cmd/filestore/pull.go @@ -0,0 +1,96 @@ +package filestore + +import ( + "context" + "fmt" + "net/url" + "os" + + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3_types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/urfave/cli/v3" + + "github.com/dpc-sdp/bay-cli/internal/helpers" +) + +const ( + S3_OBJECT_METADATA_KEY_CHECKSUM = "sha256" +) + +func Pull(ctx context.Context, c *cli.Command) error { + client := helpers.AwsS3Client() + verbose := c.Bool("verbose") + ignoreChecksum := c.Bool("ignore-checksum") + localPath := c.String("local-path") + remotePath := c.String("remote-path") + parsedURL, err := url.Parse(remotePath) + if err != nil { + return fmt.Errorf("Error parsing remote-path flag: %v\n", err) + } + bucket := parsedURL.Host + key := parsedURL.Path[1:] + + if verbose { + fmt.Fprintf(c.ErrWriter, "Pulling s3 object %s to local path %s\n", remotePath, localPath) + } + + file, err := openOrCreateFile(localPath) + if err != nil { + return err + } + defer file.Close() + + if !ignoreChecksum { + remoteHash, algo, err := getS3ObjectChecksum(ctx, client, bucket, key) + if err == nil { + localHash, _ := helpers.GenerateFileChecksum(file, algo) + if remoteHash == localHash { + fmt.Fprintf(c.ErrWriter, "File %s matches %s - skipping. You can bypass this check with the --ignore-checksum flag.\n", localPath, remotePath) + return nil + } + } + } + + // Create a downloader with the S3 client + downloader := manager.NewDownloader(client) + + // Download the file + _, err = downloader.Download(context.TODO(), file, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + + return err +} + +func getS3ObjectChecksum(ctx context.Context, client *s3.Client, bucket, key string) (string, s3_types.ChecksumAlgorithm, error) { + resp, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + return "", "", err + } + + if val, ok := resp.Metadata[S3_OBJECT_METADATA_KEY_CHECKSUM]; ok { + return val, s3_types.ChecksumAlgorithmSha256, nil + } + + return "", "", fmt.Errorf("No checksum found for object %s in bucket %s", key, bucket) +} + +func openOrCreateFile(path string) (*os.File, error) { + _, err := os.Stat(path) + if os.IsNotExist(err) { + file, err := os.Create(path) + if err != nil { + return file, err + } + } else if err != nil { + return nil, err + } + + // File exists, open it + return os.OpenFile(path, os.O_RDWR, 0) +} diff --git a/go.mod b/go.mod index cfa3050..fe8beb7 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,10 @@ toolchain go1.24.5 require ( github.com/alexeyco/simpletable v1.0.0 - github.com/aws/aws-sdk-go-v2/config v1.29.18 + github.com/aws/aws-sdk-go-v2/config v1.31.17 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 github.com/aws/aws-sdk-go-v2/service/kms v1.41.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 github.com/elastic/go-elasticsearch/v9 v9.0.0 github.com/go-git/go-git/v5 v5.16.2 github.com/manifoldco/promptui v0.9.0 @@ -22,18 +24,22 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect - github.com/aws/aws-sdk-go-v2 v1.36.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.71 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.34.1 // indirect - github.com/aws/smithy-go v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2 v1.39.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 // indirect + github.com/aws/smithy-go v1.23.2 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect diff --git a/go.sum b/go.sum index 94b3a8f..16a2f0b 100644 --- a/go.sum +++ b/go.sum @@ -15,54 +15,92 @@ github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBIL github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= github.com/aws/aws-sdk-go-v2 v1.36.6 h1:zJqGjVbRdTPojeCGWn5IR5pbJwSQSBh5RWFTQcEQGdU= github.com/aws/aws-sdk-go-v2 v1.36.6/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= +github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk= +github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8= github.com/aws/aws-sdk-go-v2/config v1.29.18 h1:x4T1GRPnqKV8HMJOMtNktbpQMl3bIsfx8KbqmveUO2I= github.com/aws/aws-sdk-go-v2/config v1.29.18/go.mod h1:bvz8oXugIsH8K7HLhBv06vDqnFv3NsGDt2Znpk7zmOU= +github.com/aws/aws-sdk-go-v2/config v1.31.17 h1:QFl8lL6RgakNK86vusim14P2k8BFSxjvUkcWLDjgz9Y= +github.com/aws/aws-sdk-go-v2/config v1.31.17/go.mod h1:V8P7ILjp/Uef/aX8TjGk6OHZN6IKPM5YW6S78QnRD5c= github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0= github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= github.com/aws/aws-sdk-go-v2/credentials v1.17.71 h1:r2w4mQWnrTMJjOyIsZtGp3R3XGY3nqHn8C26C2lQWgA= github.com/aws/aws-sdk-go-v2/credentials v1.17.71/go.mod h1:E7VF3acIup4GB5ckzbKFrCK0vTvEQxOxgdq4U3vcMCY= +github.com/aws/aws-sdk-go-v2/credentials v1.18.21 h1:56HGpsgnmD+2/KpG0ikvvR8+3v3COCwaF4r+oWwOeNA= +github.com/aws/aws-sdk-go-v2/credentials v1.18.21/go.mod h1:3YELwedmQbw7cXNaII2Wywd+YY58AmLPwX4LzARgmmA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.33 h1:D9ixiWSG4lyUBL2DDNK924Px9V/NBVpML90MHqyTADY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.33/go.mod h1:caS/m4DI+cij2paz3rtProRBI4s/+TCiWoaWZuQ9010= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37 h1:osMWfm/sC/L4tvEdQ65Gri5ZZDCUpuYJZbTTDrsn4I0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37/go.mod h1:ZV2/1fbjOPr4G4v38G3Ww5TBT4+hmsK45s/rxu1fGy0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37 h1:v+X21AvTb2wZ+ycg1gx+orkB/9U6L7AOp93R7qYxsxM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37/go.mod h1:G0uM1kyssELxmJ2VZEfG0q2npObR3BAkF3c1VsfVnfs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18 h1:vvbXsA2TVO80/KT7ZqCbx934dt6PY+vQ8hZpUZ/cpYg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18/go.mod h1:m2JJHledjBGNMsLOF1g9gbAxprzq3KjC8e4lxtn+eWg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8= github.com/aws/aws-sdk-go-v2/service/kms v1.41.1 h1:dkaX98cOXw4EgqpDXPqrVVLjsPR9T24wA2TcjrQiank= github.com/aws/aws-sdk-go-v2/service/kms v1.41.1/go.mod h1:Pqd9k4TuespkireN206cK2QBsaBTL6X+VPAez5Qcijk= github.com/aws/aws-sdk-go-v2/service/kms v1.41.3 h1:P0mjq/4mqTRA8SlS/4jL946RBW287kkKI/fazTTDJ3E= github.com/aws/aws-sdk-go-v2/service/kms v1.41.3/go.mod h1:79gw7fH6dqzJz3a5qwDnQv5GDPs8b6eJIb9hJ+/c/YU= github.com/aws/aws-sdk-go-v2/service/kms v1.41.4 h1:VDzxyStHJ5CKFaj40ti8hLuv+sMARPKTe0jnLZh6Bj4= github.com/aws/aws-sdk-go-v2/service/kms v1.41.4/go.mod h1:79gw7fH6dqzJz3a5qwDnQv5GDPs8b6eJIb9hJ+/c/YU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw= github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= github.com/aws/aws-sdk-go-v2/service/sso v1.25.6 h1:rGtWqkQbPk7Bkwuv3NzpE/scwwL9sC1Ul3tn9x83DUI= github.com/aws/aws-sdk-go-v2/service/sso v1.25.6/go.mod h1:u4ku9OLv4TO4bCPdxf4fA1upaMaJmP9ZijGk3AAOC6Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.1 h1:0JPwLz1J+5lEOfy/g0SURC9cxhbQ1lIMHMa+AHZSzz0= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.1/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4 h1:OV/pxyXh+eMA0TExHEC4jyWdumLxNbzz1P0zJoezkJc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.4/go.mod h1:8Mm5VGYwtm+r305FfPSuc+aFkrypeylGYhFim6XEPoc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5 h1:OWs0/j2UYR5LOGi88sD5/lhN6TDLG6SfA7CqsQO9zF0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.5/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo= github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0= github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w= github.com/aws/aws-sdk-go-v2/service/sts v1.34.1 h1:aUrLQwJfZtwv3/ZNG2xRtEen+NqI3iesuacjP51Mv1s= github.com/aws/aws-sdk-go-v2/service/sts v1.34.1/go.mod h1:3wFBZKoWnX3r+Sm7in79i54fBmNfwhdNdQuscCw7QIk= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.1 h1:mLlUgHn02ue8whiR4BmxxGJLR2gwU6s6ZzJ5wDamBUs= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.1/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk= github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= +github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= diff --git a/internal/helpers/aws.go b/internal/helpers/aws.go index d81bd11..7c07ce1 100644 --- a/internal/helpers/aws.go +++ b/internal/helpers/aws.go @@ -3,8 +3,10 @@ package helpers import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/s3" ) const ( @@ -28,3 +30,16 @@ func AwsKmsClient() *kms.Client { func BuildKmsAlias(project, key string) string { return fmt.Sprintf("alias/%s-%s", project, key) } + +// Return a S3 client with credentials loaded. +func AwsS3Client() *s3.Client { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if cfg.Region == "" { + cfg.Region = AwsDefaultRegion + } + if err != nil { + panic("configuration error, " + err.Error()) + } + + return s3.NewFromConfig(cfg) +} diff --git a/internal/helpers/checksum.go b/internal/helpers/checksum.go new file mode 100644 index 0000000..c99f444 --- /dev/null +++ b/internal/helpers/checksum.go @@ -0,0 +1,48 @@ +package helpers + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "fmt" + s3_types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "hash" + "hash/crc32" + "io" + "os" +) + +// GenerateFileChecksum computes a file checksum using the specified algorithm +// Supported algorithms are SHA256, SHA1, CRC32, and CRC32C +// Returns the checksum as a hex string and any error encountered +func GenerateFileChecksum(file *os.File, algorithm s3_types.ChecksumAlgorithm) (string, error) { + defer file.Seek(0, 0) + var h hash.Hash + + switch algorithm { + case s3_types.ChecksumAlgorithmSha256: + h = sha256.New() + case s3_types.ChecksumAlgorithmSha1: + h = sha1.New() + case s3_types.ChecksumAlgorithmCrc32: + h = crc32.NewIEEE() + case s3_types.ChecksumAlgorithmCrc32c: + h = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + default: + return "", fmt.Errorf("unsupported algorithm: %s", algorithm) + } + + if _, err := io.Copy(h, file); err != nil { + return "", fmt.Errorf("failed to read file: %w", err) + } + + // For CRC32 and CRC32C, we convert the uint32 to hex + if algorithm == s3_types.ChecksumAlgorithmCrc32 || algorithm == s3_types.ChecksumAlgorithmCrc32c { + checksum := h.Sum(nil) + return hex.EncodeToString(checksum), nil + } + + // For other hash algorithms + checksum := h.Sum(nil) + return hex.EncodeToString(checksum), nil +} diff --git a/main.go b/main.go index 72928dd..9e5158e 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( deployment "github.com/dpc-sdp/bay-cli/cmd/deployment" elastic_cloud "github.com/dpc-sdp/bay-cli/cmd/elastic-cloud" + filestore "github.com/dpc-sdp/bay-cli/cmd/filestore" project_map "github.com/dpc-sdp/bay-cli/cmd/project-map" ) @@ -55,6 +56,44 @@ func main() { }, }, }, + { + Name: "filestore", + Usage: "interact with bay s3 storage service", + Commands: []*cli.Command{ + { + Name: "pull", + Usage: "pull object from s3", + UsageText: "bay filestore pull --remote-path s3://bucket/path/to/file.txt --local-path /path/to/file.txt", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "remote-path", + Usage: "remote path", + Required: true, + }, + &cli.StringFlag{ + Name: "local-path", + Usage: "local path", + Required: true, + }, + &cli.BoolFlag{ + Name: "ignore-checksum", + Usage: "ignore checksum when pulling file from s3", + }, + &cli.BoolFlag{ + Name: "verbose", + Usage: "verbose output", + }, + }, + Action: filestore.Pull, + }, + { + Name: "decrypt", + Usage: "decrypt a file", + UsageText: "cat file.pem.asc | bay kms decrypt > file.pem", + Action: kms.Decrypt, + }, + }, + }, { Name: "project-map", Usage: "commands to show relationships between projects",