From 9ee36d9d649811317f70a4aaf4e582439b2928df Mon Sep 17 00:00:00 2001 From: Laurent Doguin Date: Fri, 22 Aug 2025 14:58:21 +0200 Subject: [PATCH 1/4] Add CI script example for bucket,scope and collection cloning --- examples/scripts/ci_scripts.nu | 227 +++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 examples/scripts/ci_scripts.nu diff --git a/examples/scripts/ci_scripts.nu b/examples/scripts/ci_scripts.nu new file mode 100644 index 00000000..4bb66baa --- /dev/null +++ b/examples/scripts/ci_scripts.nu @@ -0,0 +1,227 @@ +# This files contains a collection of scripts useful for CI tasks, like cloning buckets, scopes or collections + +# Clones an entire bucket, scopes and collections +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# SRC_COLLECTION: Source Collection +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +# DEST_COLLECTION: Destination Collection +def bucket-clone [ + bucket?: string, # Name of the source bucket + destbucket?: string, # Name of the destination bucket + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster + --with-indexes # copy all indexes in the bucket +] { + run_with_default { |p| + copy-bucket-definition $p.src_bucket $p.dest_bucket --source $p.src --destination $p.dest + let scopes = scopes --clusters $p.src --bucket $p.src_bucket + for scope in $scopes { + scope-clone $p.src_bucket $scope.scope $p.dest_bucket $scope.scope --source $p.src --destination $p.dest + } + if ( $with_indexes ) { + let indexes = query indexes --definitions --disable-context --clusters $p.src | where bucket == $p.src_bucket + for index in $indexes { + print $"Recreating index ($index.name) on cluster ($p.dest) with: " + print $index.definition + query $index.definition --disable-context --clusters $p.dest + } + } + } --bucket $bucket --destbucket $destbucket --source $source --destination $destination +} + +# Clones an entire Scope and its collections +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# SRC_COLLECTION: Source Collection +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +# DEST_COLLECTION: Destination Collection +def scope-clone [ + bucket?: string, # Name of the source bucket + scope?: string, # Name of the source scope + destbucket?: string, # Name of the destination bucket + destscope?: string, # Name of the destination scope + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster +] { + run_with_default { |p| + if ( scopes --clusters $p.dest --bucket $p.dest_bucket | where scope == $p.dest_scope | is-empty ) { + print $"Create scope ($p.dest)_($p.dest_bucket)_($p.dest_scope)" + scopes create --clusters $p.dest --bucket $p.dest_bucket $p.dest_scope + } + let collections = collections --clusters $p.src --bucket $p.src_bucket --scope $p.src_scope + for col in $collections { + collection-clone $p.src_bucket $p.src_scope $col.collection $p.dest_bucket $p.dest_scope $col.collection --source $p.src --destination $p.dest + } + } --bucket $bucket --destbucket $destbucket --scope $scope --destscope $destscope --source $source --destination $destination +} + +# Clones a collection +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# SRC_COLLECTION: Source Collection +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +# DEST_COLLECTION: Destination Collection +def collection-clone [ + bucket?: string, # Name of the source bucket + scope?: string, # Name of the source scope + collection?: string, # Name of the source collection + destbucket?: string, # Name of the destination bucket + destscope?: string, # Name of the destination scope + destcollection?: string, # Name of the destination collection + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster +] { + run_with_default { |p| + if ( collections --clusters $p.dest --bucket $p.dest_bucket --scope $p.dest_scope | where collection == $p.dest_collection | is-empty ) { + print $"Create collection ($p.dest)_($p.dest_bucket)_($p.dest_scope)_($p.dest_collection)" + collections create --clusters $p.dest --bucket $p.dest_bucket --scope $p.dest_scope $p.dest_collection + } + let filename = $"temp_($p.src_bucket)_($p.src_scope)_($p.src_collection).json" + let query = "SELECT meta().id as meta_id, meta().expiration as expiration, c.* FROM `" + $p.src_bucket + "`." + $p.src_scope + "." + $p.src_collection + " c" + query --disable-context --clusters $p.src $query | save -f $filename + print $"Import collection content from ($p.src)_($p.src_bucket)_($p.src_scope)_($p.src_collection) to ($p.dest)_($p.dest_bucket)_($p.dest_scope)_($p.dest_collection)" + print (doc import --bucket $p.dest_bucket --scope $p.dest_scope --collection $p.dest_collection --clusters $p.dest --id-column meta_id $filename) + + } --bucket $bucket --destbucket $destbucket --scope $scope --destscope $destscope --collection $collection --destcollection $destcollection --source $source --destination $destination +} + +# Create another bucket based on source bucket configuration. +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +def copy-bucket-definition [ + bucket?: string, # Name of the source bucket + destbucket?: string, # Name of the destination bucket + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster +] { + run_with_default { |p| + let clonable = buckets get --clusters $p.src $p.src_bucket | get 0 + print $"Create Bucket ($p.dest)_($p.dest_bucket) with ($clonable.ram_quota) quota, type ($clonable.type), ($clonable.replicas) replicas, ($clonable.min_durability_level) durability, ($clonable.max_expiry) expiry" + if ( $clonable.flush_enabled) { + $clonable | buckets create $p.dest_bucket ( $in.ram_quota / 1MB | into int ) --clusters $p.dest --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry --flush + } else { + $clonable | buckets create $p.dest_bucket ( $in.ram_quota / 1MB | into int ) --clusters $p.dest --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry + } + } --bucket $bucket --destbucket $destbucket --source $source --destination $destination +} + + +# Run the given closure with an object containing all needed +# parameters. +# +# Null parameters are replaced by env variable if given. It +# defaults to current cb-env if nothing is available. +def run_with_default [ + operation: closure, + --bucket: string, + --scope: string, + --collection: string, + --destbucket: string, + --destscope: string, + --destcollection: string, + --source: string, + --destination: string + ] { + let src_bucket = if ($bucket != null) { + $bucket + } else if ( $env.SRC_BUCKET? != null ) { + $env.SRC_BUCKET + } else { + cb-env | get bucket + } + let src_scope = if ($scope != null) { + $scope + } else if ( $env.SRC_SCOPE? != null ) { + $env.SRC_SCOPE + } else { + cb-env | get scope + } + let src_collection = if ($collection != null) { + $collection + } else if ( $env.SRC_COLLECTION? != null ) { + $env.SRC_COLLECTION + } else { + cb-env | get collection + } + + let dest_bucket = if ($destbucket != null) { + $destbucket + } else if ( $env.DEST_BUCKET? != null ) { + $env.DEST_BUCKET + } else { + cb-env | get bucket + } + let dest_scope = if ($destscope != null) { + $destscope + } else if ( $env.DEST_SCOPE? != null ) { + $env.DEST_SCOPE + } else { + cb-env | get scope + } + let dest_collection = if ($destcollection != null) { + $destcollection + } else if ( $env.DEST_COLLECTION? != null ) { + $env.DEST_COLLECTION + } else { + cb-env | get collection + } + + let src_cluster = if ($source != null) { + $source + } else if ( $env.SRC_CLUSTER? != null ) { + $env.SRC_CLUSTER + } else { + cb-env | get cluster + } + let dest_cluster = if ($destination != null) { + $destination + } else if ( $env.DEST_CLUSTER? != null ) { + $env.DEST_CLUSTER + } else { + cb-env | get cluster + } + + let params = { + src : $src_cluster, + src_bucket: $src_bucket, + src_scope: $src_scope, + src_collection: $src_collection, + dest: $dest_cluster, + dest_bucket: $dest_bucket, + dest_scope: $dest_scope, + dest_collection: $dest_collection, + } + do $operation $params +} From 810513c1b544791fe67ea60f51b8783785841f72 Mon Sep 17 00:00:00 2001 From: Laurent Doguin Date: Fri, 22 Aug 2025 18:07:38 +0200 Subject: [PATCH 2/4] refactorto add import/export on cluster structure --- examples/scripts/ci_scripts.nu | 96 ++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/examples/scripts/ci_scripts.nu b/examples/scripts/ci_scripts.nu index 4bb66baa..2c7539b3 100644 --- a/examples/scripts/ci_scripts.nu +++ b/examples/scripts/ci_scripts.nu @@ -29,11 +29,7 @@ def bucket-clone [ } if ( $with_indexes ) { let indexes = query indexes --definitions --disable-context --clusters $p.src | where bucket == $p.src_bucket - for index in $indexes { - print $"Recreating index ($index.name) on cluster ($p.dest) with: " - print $index.definition - query $index.definition --disable-context --clusters $p.dest - } + $indexes | create-indexes $p.dest } } --bucket $bucket --destbucket $destbucket --source $source --destination $destination } @@ -128,12 +124,7 @@ def copy-bucket-definition [ ] { run_with_default { |p| let clonable = buckets get --clusters $p.src $p.src_bucket | get 0 - print $"Create Bucket ($p.dest)_($p.dest_bucket) with ($clonable.ram_quota) quota, type ($clonable.type), ($clonable.replicas) replicas, ($clonable.min_durability_level) durability, ($clonable.max_expiry) expiry" - if ( $clonable.flush_enabled) { - $clonable | buckets create $p.dest_bucket ( $in.ram_quota / 1MB | into int ) --clusters $p.dest --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry --flush - } else { - $clonable | buckets create $p.dest_bucket ( $in.ram_quota / 1MB | into int ) --clusters $p.dest --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry - } + $clonable | _create-bucket-definition $p.dest } --bucket $bucket --destbucket $destbucket --source $source --destination $destination } @@ -225,3 +216,86 @@ def run_with_default [ } do $operation $params } + +# Exports all buckets, scopes and collections structure +# for the given cluster +def export-cluster-struct [ + source: string # The cluster to export +] { + mut export = [] + + let buckets = buckets --clusters $source + + for bucket in $buckets { + mut scope_structs = [] + + let scopes = scopes --clusters $source --bucket $bucket.name + + for scope in $scopes { + let collections = (collections --clusters $source --bucket $bucket.name --scope $scope.scope | reject -i cluster) + + # push scope + its collections into scope_structs + $scope_structs ++= [{ + scope: $scope.scope, + collections: $collections + }] + } + + # push bucket + its scopes into export + let buc = ( $bucket | merge {scopes: $scope_structs } ) + $export ++= [ $buc ] + } + + let indexes = query indexes --definitions --disable-context --clusters $source + let output = { + buckets: $export, + indexes: $indexes + } + return $output +} + + +# Import all buckets, scopes and collections structure +# in the given cluster +def import-cluster-struct [ + destination: string # The cluster to export +] { + let structure = $in + let buckets = $structure.buckets + for bucket in $buckets { + $bucket | _create-bucket-definition $destination + for scope in ($bucket.scopes | where not ( $it.scope | str starts-with "_" ) ) { + print $"Create scope ($destination)_($bucket.name)_($scope.scope)" + scopes create --clusters $destination --bucket $bucket.name $scope.scope + for col in $scope.collections { + print $"Create collection ($destination)_($bucket.name)_($scope.scope)_($col.collection)" + collections create --clusters $destination --bucket $bucket.name --scope $scope.scope $col.collection + } + } + } + let indexes = $structure.indexes + $indexes | _create-indexes $destination +} + +def _create-indexes [ + destination: string # the cluster where to create indexes +] { + let indexes = $in + for index in $indexes { + print $"Recreating index ($index.name) on cluster ($destination) with: " + print $index.definition + query $index.definition --disable-context --clusters $destination + } +} + +def _create-bucket-definition [ + destination: string # the cluster where to create indexes +] { + let bucket = $in + print $"Create Bucket ($destination)_($bucket.name) with ($bucket.ram_quota / 1024 / 1024 ) quota, type ($bucket.type), ($bucket.replicas) replicas, ($bucket.min_durability_level) durability, ($bucket.max_expiry) expiry" + if ( $bucket.flush_enabled) { + $bucket | buckets create $in.name ( $in.ram_quota / 1024 / 1024 | into int ) --clusters $destination --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry --flush + } else { + $bucket | buckets create $in.name ( $in.ram_quota / 1024 / 1024 | into int ) --clusters $destination --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry + } +} From 389a9d84a7a9dbe398d3312721a4ac35c342d944 Mon Sep 17 00:00:00 2001 From: Laurent Doguin Date: Mon, 25 Aug 2025 12:30:43 +0200 Subject: [PATCH 3/4] Refactor ci example to add cluster cloning --- examples/scripts/ci_scripts.nu | 48 +++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/examples/scripts/ci_scripts.nu b/examples/scripts/ci_scripts.nu index 2c7539b3..213072c6 100644 --- a/examples/scripts/ci_scripts.nu +++ b/examples/scripts/ci_scripts.nu @@ -1,4 +1,41 @@ # This files contains a collection of scripts useful for CI tasks, like cloning buckets, scopes or collections +# Most support the following env variable, and will default to cb-env if null. +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# SRC_COLLECTION: Source Collection +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +# DEST_COLLECTION: Destination Collection + + +# Clones all buckets, scopes and collections +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# DEST_CLUSTER: Destination cluster identifier +def cluster-clone [ + source?: string, # Identifier of the source Cluster + destination?: string # Identifier of the destination Cluster + --with-indexes # copy all indexes in the bucket +] { + + run_with_default { |p| + let buckets = buckets --clusters $p.src + for bucket in $buckets { + bucket-clone $bucket.name $bucket.name $p.src $p.dest + } + if ( $with_indexes ) { + let indexes = query indexes --definitions --disable-context --clusters $p.src + $indexes | create-indexes $p.dest + } + } --source $source --destination $destination +} + # Clones an entire bucket, scopes and collections # @@ -8,12 +45,8 @@ # # SRC_CLUSTER: Source cluster identifier # SRC_BUCKET: Source Bucket -# SRC_SCOPE: Source Scope -# SRC_COLLECTION: Source Collection # DEST_CLUSTER: Destination cluster identifier # DEST_BUCKET: Destination Bucket -# DEST_SCOPE: Destination Scope -# DEST_COLLECTION: Destination Collection def bucket-clone [ bucket?: string, # Name of the source bucket destbucket?: string, # Name of the destination bucket @@ -43,11 +76,9 @@ def bucket-clone [ # SRC_CLUSTER: Source cluster identifier # SRC_BUCKET: Source Bucket # SRC_SCOPE: Source Scope -# SRC_COLLECTION: Source Collection # DEST_CLUSTER: Destination cluster identifier # DEST_BUCKET: Destination Bucket # DEST_SCOPE: Destination Scope -# DEST_COLLECTION: Destination Collection def scope-clone [ bucket?: string, # Name of the source bucket scope?: string, # Name of the source scope @@ -258,7 +289,7 @@ def export-cluster-struct [ # Import all buckets, scopes and collections structure # in the given cluster def import-cluster-struct [ - destination: string # The cluster to export + destination: string # The cluster to import ] { let structure = $in let buckets = $structure.buckets @@ -289,8 +320,7 @@ def _create-indexes [ } def _create-bucket-definition [ - destination: string # the cluster where to create indexes -] { + destination: string # the cluster where to create the bucket let bucket = $in print $"Create Bucket ($destination)_($bucket.name) with ($bucket.ram_quota / 1024 / 1024 ) quota, type ($bucket.type), ($bucket.replicas) replicas, ($bucket.min_durability_level) durability, ($bucket.max_expiry) expiry" if ( $bucket.flush_enabled) { From 964ed8616eae4b63bb7f6c7b25a7dbc9fedec2bc Mon Sep 17 00:00:00 2001 From: Laurent Doguin Date: Mon, 25 Aug 2025 12:42:43 +0200 Subject: [PATCH 4/4] fix params typos --- examples/scripts/ci_scripts.nu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/scripts/ci_scripts.nu b/examples/scripts/ci_scripts.nu index 213072c6..4bd8f3de 100644 --- a/examples/scripts/ci_scripts.nu +++ b/examples/scripts/ci_scripts.nu @@ -27,7 +27,7 @@ def cluster-clone [ run_with_default { |p| let buckets = buckets --clusters $p.src for bucket in $buckets { - bucket-clone $bucket.name $bucket.name $p.src $p.dest + bucket-clone $bucket.name $bucket.name --source $p.src --destination $p.dest } if ( $with_indexes ) { let indexes = query indexes --definitions --disable-context --clusters $p.src @@ -321,6 +321,7 @@ def _create-indexes [ def _create-bucket-definition [ destination: string # the cluster where to create the bucket +] { let bucket = $in print $"Create Bucket ($destination)_($bucket.name) with ($bucket.ram_quota / 1024 / 1024 ) quota, type ($bucket.type), ($bucket.replicas) replicas, ($bucket.min_durability_level) durability, ($bucket.max_expiry) expiry" if ( $bucket.flush_enabled) {