From 34e167e5218d1389cc0da573a6b88c1eddcd7c42 Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Mon, 22 Dec 2025 12:23:05 +0100 Subject: [PATCH] feat(backup_service): apply SourcePathsToExclude regexp on server side --- go.mod | 4 ++-- go.sum | 4 ++-- internal/backup_operations/make_backup.go | 23 ++++++++++--------- internal/connectors/client/connector.go | 1 + internal/types/settings.go | 27 ++++++++++++----------- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 15ea8804..855aacf6 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,9 @@ require ( github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/jonboulle/clockwork v0.5.0 github.com/prometheus/client_golang v1.20.4 + github.com/prometheus/client_model v0.6.1 github.com/stretchr/testify v1.10.0 - github.com/ydb-platform/ydb-go-genproto v0.0.0-20250911135631-b3beddd517d9 + github.com/ydb-platform/ydb-go-genproto v0.0.0-20251222105147-0bf751469a4a github.com/ydb-platform/ydb-go-sdk-prometheus/v2 v2.1.2 github.com/ydb-platform/ydb-go-sdk/v3 v3.108.5 go.uber.org/automaxprocs v1.5.3 @@ -32,7 +33,6 @@ require ( github.com/klauspost/compress v1.17.10 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect go.uber.org/multierr v1.10.0 // indirect diff --git a/go.sum b/go.sum index 3ef2fae7..4b568c9e 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20250911135631-b3beddd517d9 h1:SKqSRP6/ocY2Z4twOqKEKxpmawVTHTvQiom7hrU6jt0= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20250911135631-b3beddd517d9/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20251222105147-0bf751469a4a h1:nRqONRrMFulP2bTWM2RRnPM1VDhWuBZg4ULXkG4xXdk= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20251222105147-0bf751469a4a/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk-prometheus/v2 v2.1.2 h1:/kDHhXMNGjsqy+SZ3Zn7gZ2ziZekUJLnPXqwy6vyAX8= github.com/ydb-platform/ydb-go-sdk-prometheus/v2 v2.1.2/go.mod h1:fGsyzk5v4hqteuWNw8liz3iu3xQSRe+zBEdVzyGQy9s= github.com/ydb-platform/ydb-go-sdk/v3 v3.108.5 h1:h6API3jJKooqBa5MNhBnilscwOMf2xjn+gFhoH56FHk= diff --git a/internal/backup_operations/make_backup.go b/internal/backup_operations/make_backup.go index d3992cf9..0ebae3f2 100644 --- a/internal/backup_operations/make_backup.go +++ b/internal/backup_operations/make_backup.go @@ -379,17 +379,18 @@ func MakeBackup( } s3Settings := types.ExportSettings{ - Endpoint: s3.Endpoint, - Region: s3.Region, - Bucket: s3.Bucket, - AccessKey: accessKey, - SecretKey: secretKey, - Description: "ydbcp backup", // TODO: the description shoud be better - NumberOfRetries: 10, // TODO: get it from configuration - RootPath: req.RootPath, - SourcePaths: pathsForExport, - DestinationPrefix: destinationPrefix, - S3ForcePathStyle: s3.S3ForcePathStyle, + Endpoint: s3.Endpoint, + Region: s3.Region, + Bucket: s3.Bucket, + AccessKey: accessKey, + SecretKey: secretKey, + Description: "ydbcp backup", // TODO: the description shoud be better + NumberOfRetries: 10, // TODO: get it from configuration + RootPath: req.RootPath, + SourcePaths: pathsForExport, + SourcePathsToExclude: req.SourcePathsToExclude, + DestinationPrefix: destinationPrefix, + S3ForcePathStyle: s3.S3ForcePathStyle, } if req.EncryptionSettings != nil { diff --git a/internal/connectors/client/connector.go b/internal/connectors/client/connector.go index 8513c09f..23a57de0 100644 --- a/internal/connectors/client/connector.go +++ b/internal/connectors/client/connector.go @@ -270,6 +270,7 @@ func (d *ClientYdbConnector) ExportToS3( if featureFlags.EnableNewPathsFormat { exportRequest.Settings.SourcePath = path.Join(clientDb.Name(), s3Settings.RootPath) exportRequest.Settings.DestinationPrefix = s3Settings.DestinationPrefix + exportRequest.Settings.ExcludeRegexps = s3Settings.SourcePathsToExclude } if featureFlags.EnableBackupsEncryption && len(s3Settings.EncryptionKey) > 0 { diff --git a/internal/types/settings.go b/internal/types/settings.go index 4c290f44..973d7f2c 100644 --- a/internal/types/settings.go +++ b/internal/types/settings.go @@ -11,19 +11,20 @@ func MakeYdbConnectionString(params YdbConnectionParams) string { } type ExportSettings struct { - Endpoint string - Region string - Bucket string - AccessKey string - SecretKey string - Description string - NumberOfRetries uint32 - RootPath string - SourcePaths []string - DestinationPrefix string - S3ForcePathStyle bool - EncryptionAlgorithm string - EncryptionKey []byte + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + Description string + NumberOfRetries uint32 + RootPath string + SourcePaths []string + SourcePathsToExclude []string + DestinationPrefix string + S3ForcePathStyle bool + EncryptionAlgorithm string + EncryptionKey []byte } type ImportSettings struct {