diff --git a/README.md b/README.md index 3b98119..5f87d1b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Effectively, `gobuildcache` leverages S3OZ as a distributed build cache for concurrent `go build` or `go test` processes regardless of whether they're running on a single machine or distributed across a fleet of CI VMs. This dramatically improves CI performance for large Go repositories because each CI process will behave as if running with an almost completely pre-populated build cache, even if the CI process was started on a completely ephemeral VM that has never compiled code or executed tests for the repository before. -`gobuildcache` is highly sensitive to the latency of the remote storage backend, so it works best when running on self-hosted runners in AWS targeting an S3 Express One Zone bucket in the same region as the self-hosted runners. That said, it doesn't have to be used that way. For example, if you're using Github's hosted runners or self-hosted runners outside of AWS, you can use a different storage solution like Tigris. See `examples/github_actions_tigris.yml` for an example of using `gobuildcache` with Tigris. +`gobuildcache` is highly sensitive to the latency of the remote storage backend, so it works best when running on self-hosted runners in AWS targeting an S3 Express One Zone bucket in the same region (and ideally same availability zone) as the self-hosted runners. That said, it doesn't have to be used that way. For example, if you're using Github's hosted runners or self-hosted runners outside of AWS, you can use a different storage solution like Tigris or Google Cloud Storage (GCS). For GCP users, enabling GCS Anywhere Cache can provide performance similar to S3OZ for read-heavy workloads. See `examples/github_actions_tigris.yml` for an example of using `gobuildcache` with Tigris. # Quick Start @@ -41,7 +41,9 @@ go test ./... By default, `gobuildcache` uses an on-disk cache stored in the OS default temporary directory. This is useful for testing and experimentation with `gobuildcache`, but provides no benefits over the Go compiler's built-in cache, which also stores cached data locally on disk. -For "production" use-cases in CI, you'll want to configure `gobuildcache` to use S3 Express One Zone, or a similarly low latency distributed backend. +For "production" use-cases in CI, you'll want to configure `gobuildcache` to use S3 Express One Zone, Google Cloud Storage, or a similarly low latency distributed backend. + +### Using S3 ```bash export BACKEND_TYPE=s3 @@ -61,7 +63,87 @@ go build ./... go test ./... ``` -Your credentials must have the following permissions: +### Using Google Cloud Storage (GCS) + +```bash +export BACKEND_TYPE=gcs +export GCS_BUCKET=$BUCKET_NAME +``` + +GCS authentication uses Application Default Credentials. You can provide credentials in one of the following ways: + +1. **Service Account JSON file** (recommended for CI): +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json +export GOCACHEPROG=gobuildcache +export BACKEND_TYPE=gcs +export GCS_BUCKET=$BUCKET_NAME +go build ./... +go test ./... +``` + +2. **Metadata service** (when running on GCP): +```bash +# No credentials file needed - uses metadata service automatically +export GOCACHEPROG=gobuildcache +export BACKEND_TYPE=gcs +export GCS_BUCKET=$BUCKET_NAME +go build ./... +go test ./... +``` + +3. **gcloud CLI credentials** (for local development): +```bash +gcloud auth application-default login +export GOCACHEPROG=gobuildcache +export BACKEND_TYPE=gcs +export GCS_BUCKET=$BUCKET_NAME +go build ./... +go test ./... +``` + +#### GCS Anywhere Cache (Recommended for Performance) + +For improved performance, especially in read-heavy workloads, consider enabling [GCS Anywhere Cache](https://cloud.google.com/storage/docs/anywhere-cache). Anywhere Cache provides an SSD-backed zonal read cache that can significantly reduce latency for frequently accessed cache objects. + +**Benefits:** +- **Lower read latency**: Cached reads from the same zone can achieve single-digit millisecond latency, comparable to S3OZ for repeated access +- **Reduced costs**: Lower data transfer costs, especially for multi-region buckets, and reduced retrieval fees +- **Better performance**: Especially beneficial when multiple CI jobs access the same cached artifacts +- **Automatic scaling**: Cache capacity and bandwidth scale automatically based on usage + +**Requirements:** +- Bucket must be in a [supported region/zone](https://cloud.google.com/storage/docs/anywhere-cache#availability) +- CI runners should be in the same zone as the cache for optimal performance +- Anywhere Cache is most effective for read-heavy workloads with high cache hit ratios + +**Setup:** +1. Verify your bucket region/zone supports Anywhere Cache +2. Enable Anywhere Cache on your GCS bucket +3. Configure the cache in the same zone as your CI runners for best performance +4. Set admission policy to "First miss" for faster warm-up (caches on first access) +5. Configure TTL based on your needs (1 hour to 7 days, default 24 hours) + +```bash +# Enable Anywhere Cache using gcloud CLI +# Replace ZONE_NAME with the zone where your CI runners are located +gcloud storage buckets update gs://YOUR_BUCKET_NAME \ + --enable-anywhere-cache \ + --anywhere-cache-zone=ZONE_NAME \ + --anywhere-cache-admission-policy=FIRST_MISS \ + --anywhere-cache-ttl=7d +``` + +**Note:** +- Anywhere Cache only accelerates reads. Writes still go directly to the bucket, but since `gobuildcache` performs writes asynchronously, this typically doesn't impact build performance. +- First-time access to an object will still hit the bucket (cache miss), but subsequent reads will be served from the cache. +- For best results, ensure your CI runners and cache are in the same zone. + +For more details, including availability by region, see the [GCS Anywhere Cache documentation](https://cloud.google.com/storage/docs/anywhere-cache). + +#### AWS Credentials Permissions + +Your AWS credentials must have the following permissions: ```json { @@ -95,15 +177,36 @@ Your credentials must have the following permissions: } ``` +#### GCS Credentials Permissions + +Your GCS service account must have the following IAM roles or permissions: + +- `storage.objects.create` - to upload cache objects +- `storage.objects.get` - to download cache objects +- `storage.objects.delete` - to delete cache objects (for clearing) +- `storage.objects.list` - to list objects (for clearing) + +The simplest way is to grant the `Storage Object Admin` role to your service account: + +```bash +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member="serviceAccount:SERVICE_ACCOUNT_EMAIL" \ + --role="roles/storage.objectAdmin" +``` + +Or for more granular control, create a custom role with only the required permissions. + ## Github Actions Example See the `examples` directory for examples of how to use `gobuildcache` in a Github Actions workflow. -## S3 Lifecycle Policy +## Lifecycle Policies -It's recommended to configure a lifecycle policy on your S3 bucket to automatically expire old cache entries and control storage costs. Build cache data is typically only useful for a limited time (e.g., a few days to a week), after which it's likely stale. +It's recommended to configure a lifecycle policy on your storage bucket to automatically expire old cache entries and control storage costs. Build cache data is typically only useful for a limited time (e.g., a few days to a week), after which it's likely stale. -Here's a sample lifecycle policy that expires objects after 7 days and aborts incomplete multipart uploads after 24 hours: +### S3 Lifecycle Policy + +Here's a sample S3 lifecycle policy that expires objects after 7 days and aborts incomplete multipart uploads after 24 hours: ```json { @@ -125,6 +228,28 @@ Here's a sample lifecycle policy that expires objects after 7 days and aborts in } ``` +### GCS Lifecycle Policy + +For GCS, you can configure a lifecycle policy using `gsutil` or the GCP Console. Here's an example using `gsutil` that expires objects after 7 days: + +```bash +gsutil lifecycle set - <|2. reads/writes| LFS[Local Filesystem Cache] GBC -->|3. GET/PUT| Backend{Backend Type} Backend --> S3OZ[S3 Express One Zone] + Backend --> GCS[Google Cloud Storage] ``` ## Processing `GET` commands @@ -275,4 +403,29 @@ Yes, but the latency of regular S3 is 10-20x higher than S3OZ, which undermines ## Do I have to use `gobuildcache` with self-hosted runners in AWS and S3OZ? -No, you can use `gobuildcache` any way you want as long as the `gobuildcache` binary can reach the remote storage backend. For example, you could run it on your laptop and use regular S3, R2, or Tigris as the remote object storage solution. However, `gobuildcache` works best when the latency of remote backend operations (`GET` and `PUT`) is low, so for best performance we recommend using self-hosted CI running in AWS and targeting a S3OZ bucket in the same region as your CI runners. \ No newline at end of file +No, you can use `gobuildcache` any way you want as long as the `gobuildcache` binary can reach the remote storage backend. For example, you could run it on your laptop and use regular S3, R2, Tigris, or Google Cloud Storage as the remote object storage solution. However, `gobuildcache` works best when the latency of remote backend operations (`GET` and `PUT`) is low, so for best performance we recommend: + +- **AWS**: Self-hosted CI running in AWS targeting a S3OZ bucket in the same region (and ideally same availability zone) as your CI runners +- **GCP**: Self-hosted CI running in GCP targeting a GCS Regional Standard bucket in the same region as your CI runners. For even better performance, consider enabling [GCS Anywhere Cache](https://cloud.google.com/storage/docs/anywhere-cache) to get zonal read caching. + +## Can I use Google Cloud Storage instead of S3? + +Yes! `gobuildcache` supports Google Cloud Storage (GCS) as a backend. GCS is a good alternative to S3, especially if you're already using GCP infrastructure. + +**Performance Considerations:** + +- **Standard GCS**: While GCS doesn't have an exact equivalent to S3 Express One Zone's single-AZ storage, using GCS Regional Standard buckets in the same region as your compute provides good performance. + +- **GCS with Anywhere Cache** (Recommended): For read-heavy workloads like build caches, enabling [GCS Anywhere Cache](https://cloud.google.com/storage/docs/anywhere-cache) can significantly improve performance: + - **Read latency**: Cached reads from the same zone can achieve single-digit millisecond latency, comparable to S3OZ for repeated access + - **Cost savings**: Reduced data transfer costs and lower read operation costs + - **Best for**: Workloads where the same cache objects are accessed multiple times (common in CI where multiple jobs may access the same artifacts) + + Anywhere Cache is particularly effective when: + - Your CI runners are in the same zone as the cache + - You have high cache hit ratios (same objects accessed repeatedly) + - Your bucket is in a [supported region/zone](https://cloud.google.com/storage/docs/anywhere-cache#availability) + +- **Write latency**: GCS write latency may be higher than S3OZ, but since `gobuildcache` performs writes asynchronously, this typically doesn't impact build performance significantly. + +**Recommendation**: If you're using GCP and want performance closer to S3OZ, use GCS Regional Standard buckets with Anywhere Cache enabled in the same zone as your CI runners. This provides excellent read performance while maintaining better durability than single-AZ storage. \ No newline at end of file diff --git a/go.mod b/go.mod index 95ef1f1..a270a52 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,21 @@ module github.com/richardartoul/gobuildcache go 1.25 require ( + cloud.google.com/go/storage v1.40.0 github.com/DataDog/sketches-go v1.4.6 github.com/aws/aws-sdk-go-v2 v1.32.7 github.com/aws/aws-sdk-go-v2/config v1.28.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 github.com/gofrs/flock v0.13.0 github.com/pierrec/lz4/v4 v4.1.23 + google.golang.org/api v0.170.0 ) require ( + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/compute v1.24.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect @@ -27,6 +33,32 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect github.com/aws/smithy-go v1.22.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.37.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index a0706ea..0833c80 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,15 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= +cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/sketches-go v1.4.6 h1:acd5fb+QdUzGrosfNLwrIhqyrbMORpvBy7mE+vHlT3I= github.com/DataDog/sketches-go v1.4.6/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= @@ -36,25 +48,192 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgpp github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48= +google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integrationtests/integration_gcs_test.go b/integrationtests/integration_gcs_test.go new file mode 100644 index 0000000..642d76f --- /dev/null +++ b/integrationtests/integration_gcs_test.go @@ -0,0 +1,159 @@ +package integrationtests + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestCacheIntegrationGCS(t *testing.T) { + if testing.Short() { + t.Skip("Skipping GCS integration test in short mode") + } + + // Get GCS bucket from environment - required for GCS tests + gcsBucket := os.Getenv("TEST_GCS_BUCKET") + if gcsBucket == "" { + t.Fatal("TEST_GCS_BUCKET environment variable not set") + } + + // Verify GCP credentials are available + // GCS client uses Application Default Credentials (GOOGLE_APPLICATION_CREDENTIALS + // or metadata service), so we just check if the env var is set or if we're + // running in GCP (which would use metadata service) + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + // Check if we're in a GCP environment (metadata service available) + // For now, we'll just warn - the actual connection will fail if credentials are missing + t.Log("Warning: GOOGLE_APPLICATION_CREDENTIALS not set. Will attempt to use metadata service if running in GCP.") + } + + currentDir, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + // Go up one directory since we're in integrationtests/ + workspaceDir := filepath.Join(currentDir, "..") + + var ( + buildDir = filepath.Join(workspaceDir, "builds") + binaryPath = filepath.Join(buildDir, "gobuildcache") + testsDir = filepath.Join(workspaceDir, "faketests") + // Use a unique bucket prefix to avoid conflicts with concurrent tests + bucketPrefix = fmt.Sprintf("test-cache-%d", time.Now().Unix()) + ) + + t.Logf("Using GCS bucket: %s with prefix: %s", gcsBucket, bucketPrefix) + + t.Log("Step 1: Compiling the binary...") + if err := os.MkdirAll(buildDir, 0755); err != nil { + t.Fatalf("Failed to create build directory: %v", err) + } + + buildCmd := exec.Command("go", "build", "-o", binaryPath, ".") + buildCmd.Dir = workspaceDir + buildOutput, err := buildCmd.CombinedOutput() + if err != nil { + t.Fatalf("Failed to compile binary: %v\nOutput: %s", err, buildOutput) + } + t.Log("✓ Binary compiled successfully") + + // Use current environment for all commands + baseEnv := os.Environ() + gcsEnv := baseEnv + + t.Log("Step 2: Clearing the GCS cache...") + clearCmd := exec.Command(binaryPath, "clear", + "-debug", + "-backend=gcs", + "-gcs-bucket="+gcsBucket, + "-gcs-prefix="+bucketPrefix+"/") + clearCmd.Dir = workspaceDir + clearCmd.Env = gcsEnv + clearOutput, err := clearCmd.CombinedOutput() + if err != nil { + t.Fatalf("Failed to clear GCS cache: %v\nOutput: %s", err, clearOutput) + } + t.Logf("✓ GCS cache cleared successfully: %s", strings.TrimSpace(string(clearOutput))) + + // Note: We don't start a separate server. Go's GOCACHEPROG will start + // the cache server automatically when needed, using the environment + // variables we set (BACKEND_TYPE, GCS_BUCKET, GCS_PREFIX). + + t.Log("Step 3: Running tests with GCS cache (first run)...") + firstRunCmd := exec.Command("go", "test", "-v", testsDir) + firstRunCmd.Dir = workspaceDir + // Set environment to use GCS backend when Go starts the cache program + firstRunCmd.Env = append(baseEnv, + "GOCACHEPROG="+binaryPath, + "BACKEND_TYPE=gcs", + "DEBUG=true", + "GCS_BUCKET="+gcsBucket, + "GCS_PREFIX="+bucketPrefix+"/") + + var firstRunOutput bytes.Buffer + firstRunCmd.Stdout = &firstRunOutput + firstRunCmd.Stderr = &firstRunOutput + + if err := firstRunCmd.Run(); err != nil { + t.Fatalf("Tests failed on first run: %v\nOutput:\n%s", err, firstRunOutput.String()) + } + + t.Logf("First run output:\n%s", firstRunOutput.String()) + t.Log("✓ Tests passed on first run") + + if strings.Contains(firstRunOutput.String(), "(cached)") { + t.Fatal("First run should not be cached, but found '(cached)' in output") + } + t.Log("✓ First run was not cached (as expected)") + + t.Log("Step 4: Running tests again to verify GCS caching...") + secondRunCmd := exec.Command("go", "test", "-v", testsDir) + secondRunCmd.Dir = workspaceDir + // Set environment to use GCS backend when Go starts the cache program + secondRunCmd.Env = append(baseEnv, + "GOCACHEPROG="+binaryPath, + "BACKEND_TYPE=gcs", + "DEBUG=true", + "GCS_BUCKET="+gcsBucket, + "GCS_PREFIX="+bucketPrefix+"/") + + var secondRunOutput bytes.Buffer + secondRunCmd.Stdout = &secondRunOutput + secondRunCmd.Stderr = &secondRunOutput + + if err := secondRunCmd.Run(); err != nil { + t.Fatalf("Tests failed on second run: %v\nOutput:\n%s", err, secondRunOutput.String()) + } + + t.Logf("Second run output:\n%s", secondRunOutput.String()) + t.Log("✓ Tests passed on second run") + + // Verify that results were cached + if strings.Contains(secondRunOutput.String(), "(cached)") { + t.Log("✓ Tests results were served from GCS cache!") + } else { + t.Fatalf("Tests did not use cached results from GCS. Expected to see '(cached)' in the output.\nOutput:\n%s", secondRunOutput.String()) + } + + // Final cleanup - clear the test data from GCS + t.Log("Step 5: Cleaning up GCS test data...") + finalClearCmd := exec.Command(binaryPath, "clear", + "-debug", + "-backend=gcs", + "-gcs-bucket="+gcsBucket, + "-gcs-prefix="+bucketPrefix+"/") + finalClearCmd.Dir = workspaceDir + finalClearCmd.Env = gcsEnv + if output, err := finalClearCmd.CombinedOutput(); err != nil { + t.Logf("Warning: Failed to clean up GCS test data: %v\nOutput: %s", err, output) + } else { + t.Log("✓ GCS test data cleaned up") + } + + t.Log("=== All GCS integration tests passed! ===") +} diff --git a/main.go b/main.go index e5f3fc3..a6978f1 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,8 @@ var ( cacheDir string s3Bucket string s3Prefix string + gcsBucket string + gcsPrefix string errorRate float64 compression bool asyncBackend bool @@ -68,18 +70,22 @@ func runServerCommand() { cacheDirDefault = getEnv("CACHE_DIR", filepath.Join(os.TempDir(), "gobuildcache", "cache")) s3BucketDefault = getEnv("S3_BUCKET", "") s3PrefixDefault = getEnv("S3_PREFIX", "gobuildcache/") + gcsBucketDefault = getEnv("GCS_BUCKET", "") + gcsPrefixDefault = getEnv("GCS_PREFIX", "gobuildcache/") errorRateDefault = getEnvFloat("ERROR_RATE", 0.0) compressionDefault = getEnvBool("COMPRESSION", true) asyncBackendDefault = getEnvBool("ASYNC_BACKEND", true) ) serverFlags.BoolVar(&debug, "debug", debugDefault, "Enable debug logging to stderr (env: DEBUG)") serverFlags.BoolVar(&printStats, "stats", printStatsDefault, "Print cache statistics on exit (env: PRINT_STATS)") - serverFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3 (env: BACKEND_TYPE)") + serverFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3, gcs (env: BACKEND_TYPE)") serverFlags.StringVar(&lockingType, "lock-type", lockTypeDefault, "Locking type: memory (in-memory), fslock (filesystem) (env: LOCK_TYPE)") serverFlags.StringVar(&lockDir, "lock-dir", lockDirDefault, "Lock directory for fslock (env: LOCK_DIR)") serverFlags.StringVar(&cacheDir, "cache-dir", cacheDirDefault, "Local cache directory (env: CACHE_DIR)") serverFlags.StringVar(&s3Bucket, "s3-bucket", s3BucketDefault, "S3 bucket name (required for s3 backend) (env: S3_BUCKET)") serverFlags.StringVar(&s3Prefix, "s3-prefix", s3PrefixDefault, "S3 key prefix (optional) (env: S3_PREFIX)") + serverFlags.StringVar(&gcsBucket, "gcs-bucket", gcsBucketDefault, "GCS bucket name (required for gcs backend) (env: GCS_BUCKET)") + serverFlags.StringVar(&gcsPrefix, "gcs-prefix", gcsPrefixDefault, "GCS object prefix (optional) (env: GCS_PREFIX)") serverFlags.Float64Var(&errorRate, "error-rate", errorRateDefault, "Error injection rate (0.0-1.0) for testing error handling (env: ERROR_RATE)") serverFlags.BoolVar(&compression, "compression", compressionDefault, "Enable LZ4 compression for backend storage (env: COMPRESSION)") serverFlags.BoolVar(&asyncBackend, "async-backend", asyncBackendDefault, "Enable async backend writer for non-blocking PUT operations (env: ASYNC_BACKEND)") @@ -92,12 +98,14 @@ func runServerCommand() { fmt.Fprintf(os.Stderr, "\nEnvironment Variables:\n") fmt.Fprintf(os.Stderr, " DEBUG Enable debug logging (true/false)\n") fmt.Fprintf(os.Stderr, " PRINT_STATS Print cache statistics on exit (true/false)\n") - fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3)\n") + fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3, gcs)\n") fmt.Fprintf(os.Stderr, " LOCK_TYPE Deduplication type (memory, fslock)\n") fmt.Fprintf(os.Stderr, " LOCK_DIR Lock directory for fslock\n") fmt.Fprintf(os.Stderr, " CACHE_DIR Local cache directory\n") fmt.Fprintf(os.Stderr, " S3_BUCKET S3 bucket name\n") fmt.Fprintf(os.Stderr, " S3_PREFIX S3 key prefix\n") + fmt.Fprintf(os.Stderr, " GCS_BUCKET GCS bucket name\n") + fmt.Fprintf(os.Stderr, " GCS_PREFIX GCS object prefix\n") fmt.Fprintf(os.Stderr, " COMPRESSION Enable LZ4 compression (true/false)\n") fmt.Fprintf(os.Stderr, " ASYNC_BACKEND Enable async backend writer (true/false)\n") fmt.Fprintf(os.Stderr, "\nNote: Command-line flags take precedence over environment variables.\n") @@ -106,8 +114,11 @@ func runServerCommand() { fmt.Fprintf(os.Stderr, " %s -cache-dir=/var/cache/go\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Run with S3 backend using flags:\n") fmt.Fprintf(os.Stderr, " %s -backend=s3 -s3-bucket=my-cache-bucket\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " # Run with GCS backend using flags:\n") + fmt.Fprintf(os.Stderr, " %s -backend=gcs -gcs-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Run with environment variables:\n") fmt.Fprintf(os.Stderr, " BACKEND_TYPE=s3 S3_BUCKET=my-cache-bucket %s\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " BACKEND_TYPE=gcs GCS_BUCKET=my-cache-bucket %s\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Mix environment variables and flags (flags override env):\n") fmt.Fprintf(os.Stderr, " BACKEND_TYPE=s3 %s -s3-bucket=my-cache-bucket -debug\n", os.Args[0]) } @@ -125,12 +136,16 @@ func runClearCommand() { cacheDirDefault = getEnv("CACHE_DIR", filepath.Join(os.TempDir(), "gobuildcache", "cache")) s3BucketDefault = getEnv("S3_BUCKET", "") s3PrefixDefault = getEnv("S3_PREFIX", "") + gcsBucketDefault = getEnv("GCS_BUCKET", "") + gcsPrefixDefault = getEnv("GCS_PREFIX", "") ) clearFlags.BoolVar(&debug, "debug", debugDefault, "Enable debug logging to stderr (env: DEBUG)") - clearFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3 (env: BACKEND_TYPE)") + clearFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3, gcs (env: BACKEND_TYPE)") clearFlags.StringVar(&cacheDir, "cache-dir", cacheDirDefault, "Local cache directory (env: CACHE_DIR)") clearFlags.StringVar(&s3Bucket, "s3-bucket", s3BucketDefault, "S3 bucket name (required for s3 backend) (env: S3_BUCKET)") clearFlags.StringVar(&s3Prefix, "s3-prefix", s3PrefixDefault, "S3 key prefix (optional) (env: S3_PREFIX)") + clearFlags.StringVar(&gcsBucket, "gcs-bucket", gcsBucketDefault, "GCS bucket name (required for gcs backend) (env: GCS_BUCKET)") + clearFlags.StringVar(&gcsPrefix, "gcs-prefix", gcsPrefixDefault, "GCS object prefix (optional) (env: GCS_PREFIX)") clearFlags.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s clear [flags]\n\n", os.Args[0]) @@ -140,10 +155,12 @@ func runClearCommand() { fmt.Fprintf(os.Stderr, "\nEnvironment Variables:\n") fmt.Fprintf(os.Stderr, " DEBUG Enable debug logging (true/false)\n") fmt.Fprintf(os.Stderr, " PRINT_STATS Print cache statistics on exit (true/false)\n") - fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3)\n") + fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3, gcs)\n") fmt.Fprintf(os.Stderr, " CACHE_DIR Local cache directory\n") fmt.Fprintf(os.Stderr, " S3_BUCKET S3 bucket name\n") fmt.Fprintf(os.Stderr, " S3_PREFIX S3 key prefix\n") + fmt.Fprintf(os.Stderr, " GCS_BUCKET GCS bucket name\n") + fmt.Fprintf(os.Stderr, " GCS_PREFIX GCS object prefix\n") fmt.Fprintf(os.Stderr, " S3_TMP_DIR Local temp directory for S3 backend\n") fmt.Fprintf(os.Stderr, "\nNote: Command-line flags take precedence over environment variables.\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") @@ -151,6 +168,8 @@ func runClearCommand() { fmt.Fprintf(os.Stderr, " %s clear -cache-dir=/var/cache/go\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear S3 cache using flags:\n") fmt.Fprintf(os.Stderr, " %s clear -backend=s3 -s3-bucket=my-cache-bucket\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " # Clear GCS cache using flags:\n") + fmt.Fprintf(os.Stderr, " %s clear -backend=gcs -gcs-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear using environment variables:\n") fmt.Fprintf(os.Stderr, " BACKEND_TYPE=s3 S3_BUCKET=my-cache-bucket %s clear\n", os.Args[0]) } @@ -206,11 +225,15 @@ func runClearRemoteCommand() { backendDefault = getEnv("BACKEND_TYPE", getEnv("BACKEND", "disk")) s3BucketDefault = getEnv("S3_BUCKET", "") s3PrefixDefault = getEnv("S3_PREFIX", "") + gcsBucketDefault = getEnv("GCS_BUCKET", "") + gcsPrefixDefault = getEnv("GCS_PREFIX", "") ) clearRemoteFlags.BoolVar(&debug, "debug", debugDefault, "Enable debug logging to stderr (env: DEBUG)") - clearRemoteFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk, s3 (env: BACKEND_TYPE)") + clearRemoteFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk, s3, gcs (env: BACKEND_TYPE)") clearRemoteFlags.StringVar(&s3Bucket, "s3-bucket", s3BucketDefault, "S3 bucket name (required for s3 backend) (env: S3_BUCKET)") clearRemoteFlags.StringVar(&s3Prefix, "s3-prefix", s3PrefixDefault, "S3 key prefix (optional) (env: S3_PREFIX)") + clearRemoteFlags.StringVar(&gcsBucket, "gcs-bucket", gcsBucketDefault, "GCS bucket name (required for gcs backend) (env: GCS_BUCKET)") + clearRemoteFlags.StringVar(&gcsPrefix, "gcs-prefix", gcsPrefixDefault, "GCS object prefix (optional) (env: GCS_PREFIX)") clearRemoteFlags.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s clear-remote [flags]\n\n", os.Args[0]) @@ -219,13 +242,17 @@ func runClearRemoteCommand() { clearRemoteFlags.PrintDefaults() fmt.Fprintf(os.Stderr, "\nEnvironment Variables:\n") fmt.Fprintf(os.Stderr, " DEBUG Enable debug logging (true/false)\n") - fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3)\n") + fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3, gcs)\n") fmt.Fprintf(os.Stderr, " S3_BUCKET S3 bucket name\n") fmt.Fprintf(os.Stderr, " S3_PREFIX S3 key prefix\n") + fmt.Fprintf(os.Stderr, " GCS_BUCKET GCS bucket name\n") + fmt.Fprintf(os.Stderr, " GCS_PREFIX GCS object prefix\n") fmt.Fprintf(os.Stderr, "\nNote: Command-line flags take precedence over environment variables.\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, " # Clear S3 cache using flags:\n") fmt.Fprintf(os.Stderr, " %s clear-remote -backend=s3 -s3-bucket=my-cache-bucket\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " # Clear GCS cache using flags:\n") + fmt.Fprintf(os.Stderr, " %s clear-remote -backend=gcs -gcs-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear S3 cache with prefix:\n") fmt.Fprintf(os.Stderr, " %s clear-remote -backend=s3 -s3-bucket=my-cache-bucket -s3-prefix=myproject/\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear using environment variables:\n") @@ -350,8 +377,15 @@ func createBackend() (backends.Backend, error) { backend, err = backends.NewS3(s3Bucket, s3Prefix) + case "gcs": + if gcsBucket == "" { + return nil, fmt.Errorf("GCS bucket is required for GCS backend (set via -gcs-bucket flag or GCS_BUCKET env var)") + } + + backend, err = backends.NewGCS(gcsBucket, gcsPrefix) + default: - return nil, fmt.Errorf("unknown backend type: %s (supported: disk, s3)", backendType) + return nil, fmt.Errorf("unknown backend type: %s (supported: disk, s3, gcs)", backendType) } if err != nil { diff --git a/pkg/backends/gcs.go b/pkg/backends/gcs.go new file mode 100644 index 0000000..96e387e --- /dev/null +++ b/pkg/backends/gcs.go @@ -0,0 +1,214 @@ +package backends + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "strconv" + "time" + + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" +) + +// GCS implements Backend using Google Cloud Storage. +// This backend only handles GCS operations; local disk caching is handled by server.go. +type GCS struct { + client *storage.Client + bucket *storage.BucketHandle + prefix string + ctx context.Context +} + +// NewGCS creates a new GCS-based cache backend. +// bucket is the GCS bucket name where cache files will be stored. +// prefix is an optional prefix for all GCS object names (e.g., "cache/" or ""). +func NewGCS(bucket, prefix string) (*GCS, error) { + ctx := context.Background() + + // Create GCS client using Application Default Credentials + // This will use GOOGLE_APPLICATION_CREDENTIALS env var or metadata service + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create GCS client: %w", err) + } + + bucketHandle := client.Bucket(bucket) + + backend := &GCS{ + client: client, + bucket: bucketHandle, + prefix: prefix, + ctx: ctx, + } + + // Test bucket access by checking if bucket exists + _, err = bucketHandle.Attrs(ctx) + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to access GCS bucket %s: %w", bucket, err) + } + + return backend, nil +} + +// Put stores an object in GCS. +func (g *GCS) Put(actionID, outputID []byte, body io.Reader, bodySize int64) error { + key := g.actionIDToKey(actionID) + obj := g.bucket.Object(key) + + // Create a writer for the object + writer := obj.NewWriter(g.ctx) + defer writer.Close() + + // Set metadata + writer.Metadata = map[string]string{ + "outputid": hex.EncodeToString(outputID), + "size": strconv.FormatInt(bodySize, 10), + "time": strconv.FormatInt(time.Now().Unix(), 10), + } + + // Copy the body to the writer + if bodySize > 0 && body != nil { + written, err := io.CopyN(writer, body, bodySize) + if err != nil && err != io.EOF { + return fmt.Errorf("failed to write body to GCS: %w", err) + } + if written != bodySize { + return fmt.Errorf("size mismatch: expected %d, wrote %d", bodySize, written) + } + } + + // Close the writer to finalize the upload + if err := writer.Close(); err != nil { + return fmt.Errorf("failed to close GCS writer: %w", err) + } + + return nil +} + +// Get retrieves an object from GCS. +// Returns the object data as an io.ReadCloser that must be closed by the caller. +func (g *GCS) Get(actionID []byte) ([]byte, io.ReadCloser, int64, *time.Time, bool, error) { + key := g.actionIDToKey(actionID) + obj := g.bucket.Object(key) + + // Get object attributes first to check if it exists and get metadata + attrs, err := obj.Attrs(g.ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, nil, 0, nil, true, nil + } + return nil, nil, 0, nil, true, fmt.Errorf("failed to get GCS object attrs: %w", err) + } + + // Parse metadata + outputIDHex := attrs.Metadata["outputid"] + sizeStr := attrs.Metadata["size"] + timeStr := attrs.Metadata["time"] + + outputID, err := hex.DecodeString(outputIDHex) + if err != nil { + return nil, nil, 0, nil, true, nil + } + + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + // Fallback to actual object size if metadata is missing + size = attrs.Size + } + + var putTime *time.Time + if timeStr != "" { + putTimeUnix, err := strconv.ParseInt(timeStr, 10, 64) + if err == nil { + t := time.Unix(putTimeUnix, 0) + putTime = &t + } + } + // Fallback to object creation time if metadata is missing + if putTime == nil { + putTime = &attrs.Created + } + + // Get a reader for the object + reader, err := obj.NewReader(g.ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, nil, 0, nil, true, nil + } + return nil, nil, 0, nil, true, fmt.Errorf("failed to get GCS object reader: %w", err) + } + + // Return the GCS object body as a ReadCloser + // The caller is responsible for closing it + return outputID, reader, size, putTime, false, nil +} + +// Close performs cleanup operations. +func (g *GCS) Close() error { + if g.client != nil { + return g.client.Close() + } + return nil +} + +// Clear removes all entries from the cache in GCS. +func (g *GCS) Clear() error { + // List all objects with the prefix + query := &storage.Query{ + Prefix: g.prefix, + } + + it := g.bucket.Objects(g.ctx, query) + + // Collect objects to delete (GCS allows up to 100 objects per batch delete) + var objectsToDelete []string + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("failed to list GCS objects: %w", err) + } + objectsToDelete = append(objectsToDelete, attrs.Name) + } + + if len(objectsToDelete) == 0 { + return nil + } + + // Delete objects in batches (GCS allows up to 100 objects per batch) + batchSize := 100 + for i := 0; i < len(objectsToDelete); i += batchSize { + end := i + batchSize + if end > len(objectsToDelete) { + end = len(objectsToDelete) + } + batch := objectsToDelete[i:end] + + // Delete each object in the batch + for _, objName := range batch { + obj := g.bucket.Object(objName) + if err := obj.Delete(g.ctx); err != nil { + // Continue deleting other objects even if one fails + // Log error but don't fail the entire operation + _ = err + } + } + } + + return nil +} + +// actionIDToKey converts an actionID to a GCS object name. +func (g *GCS) actionIDToKey(actionID []byte) string { + hexID := hex.EncodeToString(actionID) + if g.prefix != "" { + return g.prefix + hexID + } + return hexID +}