diff --git a/examples/scripts/ci_scripts.nu b/examples/scripts/ci_scripts.nu new file mode 100644 index 00000000..4bd8f3de --- /dev/null +++ b/examples/scripts/ci_scripts.nu @@ -0,0 +1,332 @@ +# This files contains a collection of scripts useful for CI tasks, like cloning buckets, scopes or collections +# Most support the following env variable, and will default to cb-env if null. +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# SRC_COLLECTION: Source Collection +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +# DEST_COLLECTION: Destination Collection + + +# Clones all buckets, scopes and collections +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# DEST_CLUSTER: Destination cluster identifier +def cluster-clone [ + source?: string, # Identifier of the source Cluster + destination?: string # Identifier of the destination Cluster + --with-indexes # copy all indexes in the bucket +] { + + run_with_default { |p| + let buckets = buckets --clusters $p.src + for bucket in $buckets { + bucket-clone $bucket.name $bucket.name --source $p.src --destination $p.dest + } + if ( $with_indexes ) { + let indexes = query indexes --definitions --disable-context --clusters $p.src + $indexes | create-indexes $p.dest + } + } --source $source --destination $destination +} + + +# Clones an entire bucket, scopes and collections +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +def bucket-clone [ + bucket?: string, # Name of the source bucket + destbucket?: string, # Name of the destination bucket + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster + --with-indexes # copy all indexes in the bucket +] { + run_with_default { |p| + copy-bucket-definition $p.src_bucket $p.dest_bucket --source $p.src --destination $p.dest + let scopes = scopes --clusters $p.src --bucket $p.src_bucket + for scope in $scopes { + scope-clone $p.src_bucket $scope.scope $p.dest_bucket $scope.scope --source $p.src --destination $p.dest + } + if ( $with_indexes ) { + let indexes = query indexes --definitions --disable-context --clusters $p.src | where bucket == $p.src_bucket + $indexes | create-indexes $p.dest + } + } --bucket $bucket --destbucket $destbucket --source $source --destination $destination +} + +# Clones an entire Scope and its collections +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +def scope-clone [ + bucket?: string, # Name of the source bucket + scope?: string, # Name of the source scope + destbucket?: string, # Name of the destination bucket + destscope?: string, # Name of the destination scope + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster +] { + run_with_default { |p| + if ( scopes --clusters $p.dest --bucket $p.dest_bucket | where scope == $p.dest_scope | is-empty ) { + print $"Create scope ($p.dest)_($p.dest_bucket)_($p.dest_scope)" + scopes create --clusters $p.dest --bucket $p.dest_bucket $p.dest_scope + } + let collections = collections --clusters $p.src --bucket $p.src_bucket --scope $p.src_scope + for col in $collections { + collection-clone $p.src_bucket $p.src_scope $col.collection $p.dest_bucket $p.dest_scope $col.collection --source $p.src --destination $p.dest + } + } --bucket $bucket --destbucket $destbucket --scope $scope --destscope $destscope --source $source --destination $destination +} + +# Clones a collection +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# SRC_SCOPE: Source Scope +# SRC_COLLECTION: Source Collection +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +# DEST_SCOPE: Destination Scope +# DEST_COLLECTION: Destination Collection +def collection-clone [ + bucket?: string, # Name of the source bucket + scope?: string, # Name of the source scope + collection?: string, # Name of the source collection + destbucket?: string, # Name of the destination bucket + destscope?: string, # Name of the destination scope + destcollection?: string, # Name of the destination collection + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster +] { + run_with_default { |p| + if ( collections --clusters $p.dest --bucket $p.dest_bucket --scope $p.dest_scope | where collection == $p.dest_collection | is-empty ) { + print $"Create collection ($p.dest)_($p.dest_bucket)_($p.dest_scope)_($p.dest_collection)" + collections create --clusters $p.dest --bucket $p.dest_bucket --scope $p.dest_scope $p.dest_collection + } + let filename = $"temp_($p.src_bucket)_($p.src_scope)_($p.src_collection).json" + let query = "SELECT meta().id as meta_id, meta().expiration as expiration, c.* FROM `" + $p.src_bucket + "`." + $p.src_scope + "." + $p.src_collection + " c" + query --disable-context --clusters $p.src $query | save -f $filename + print $"Import collection content from ($p.src)_($p.src_bucket)_($p.src_scope)_($p.src_collection) to ($p.dest)_($p.dest_bucket)_($p.dest_scope)_($p.dest_collection)" + print (doc import --bucket $p.dest_bucket --scope $p.dest_scope --collection $p.dest_collection --clusters $p.dest --id-column meta_id $filename) + + } --bucket $bucket --destbucket $destbucket --scope $scope --destscope $destscope --collection $collection --destcollection $destcollection --source $source --destination $destination +} + +# Create another bucket based on source bucket configuration. +# +# All parameters can be null, Env Variables can be used. If +# the parameter is null and no env variables are set, param +# will default to current cb-env. +# +# SRC_CLUSTER: Source cluster identifier +# SRC_BUCKET: Source Bucket +# DEST_CLUSTER: Destination cluster identifier +# DEST_BUCKET: Destination Bucket +def copy-bucket-definition [ + bucket?: string, # Name of the source bucket + destbucket?: string, # Name of the destination bucket + --source: string, # Identifier of the source Cluster + --destination: string # Identifier of the destination Cluster +] { + run_with_default { |p| + let clonable = buckets get --clusters $p.src $p.src_bucket | get 0 + $clonable | _create-bucket-definition $p.dest + } --bucket $bucket --destbucket $destbucket --source $source --destination $destination +} + + +# Run the given closure with an object containing all needed +# parameters. +# +# Null parameters are replaced by env variable if given. It +# defaults to current cb-env if nothing is available. +def run_with_default [ + operation: closure, + --bucket: string, + --scope: string, + --collection: string, + --destbucket: string, + --destscope: string, + --destcollection: string, + --source: string, + --destination: string + ] { + let src_bucket = if ($bucket != null) { + $bucket + } else if ( $env.SRC_BUCKET? != null ) { + $env.SRC_BUCKET + } else { + cb-env | get bucket + } + let src_scope = if ($scope != null) { + $scope + } else if ( $env.SRC_SCOPE? != null ) { + $env.SRC_SCOPE + } else { + cb-env | get scope + } + let src_collection = if ($collection != null) { + $collection + } else if ( $env.SRC_COLLECTION? != null ) { + $env.SRC_COLLECTION + } else { + cb-env | get collection + } + + let dest_bucket = if ($destbucket != null) { + $destbucket + } else if ( $env.DEST_BUCKET? != null ) { + $env.DEST_BUCKET + } else { + cb-env | get bucket + } + let dest_scope = if ($destscope != null) { + $destscope + } else if ( $env.DEST_SCOPE? != null ) { + $env.DEST_SCOPE + } else { + cb-env | get scope + } + let dest_collection = if ($destcollection != null) { + $destcollection + } else if ( $env.DEST_COLLECTION? != null ) { + $env.DEST_COLLECTION + } else { + cb-env | get collection + } + + let src_cluster = if ($source != null) { + $source + } else if ( $env.SRC_CLUSTER? != null ) { + $env.SRC_CLUSTER + } else { + cb-env | get cluster + } + let dest_cluster = if ($destination != null) { + $destination + } else if ( $env.DEST_CLUSTER? != null ) { + $env.DEST_CLUSTER + } else { + cb-env | get cluster + } + + let params = { + src : $src_cluster, + src_bucket: $src_bucket, + src_scope: $src_scope, + src_collection: $src_collection, + dest: $dest_cluster, + dest_bucket: $dest_bucket, + dest_scope: $dest_scope, + dest_collection: $dest_collection, + } + do $operation $params +} + +# Exports all buckets, scopes and collections structure +# for the given cluster +def export-cluster-struct [ + source: string # The cluster to export +] { + mut export = [] + + let buckets = buckets --clusters $source + + for bucket in $buckets { + mut scope_structs = [] + + let scopes = scopes --clusters $source --bucket $bucket.name + + for scope in $scopes { + let collections = (collections --clusters $source --bucket $bucket.name --scope $scope.scope | reject -i cluster) + + # push scope + its collections into scope_structs + $scope_structs ++= [{ + scope: $scope.scope, + collections: $collections + }] + } + + # push bucket + its scopes into export + let buc = ( $bucket | merge {scopes: $scope_structs } ) + $export ++= [ $buc ] + } + + let indexes = query indexes --definitions --disable-context --clusters $source + let output = { + buckets: $export, + indexes: $indexes + } + return $output +} + + +# Import all buckets, scopes and collections structure +# in the given cluster +def import-cluster-struct [ + destination: string # The cluster to import +] { + let structure = $in + let buckets = $structure.buckets + for bucket in $buckets { + $bucket | _create-bucket-definition $destination + for scope in ($bucket.scopes | where not ( $it.scope | str starts-with "_" ) ) { + print $"Create scope ($destination)_($bucket.name)_($scope.scope)" + scopes create --clusters $destination --bucket $bucket.name $scope.scope + for col in $scope.collections { + print $"Create collection ($destination)_($bucket.name)_($scope.scope)_($col.collection)" + collections create --clusters $destination --bucket $bucket.name --scope $scope.scope $col.collection + } + } + } + let indexes = $structure.indexes + $indexes | _create-indexes $destination +} + +def _create-indexes [ + destination: string # the cluster where to create indexes +] { + let indexes = $in + for index in $indexes { + print $"Recreating index ($index.name) on cluster ($destination) with: " + print $index.definition + query $index.definition --disable-context --clusters $destination + } +} + +def _create-bucket-definition [ + destination: string # the cluster where to create the bucket +] { + let bucket = $in + print $"Create Bucket ($destination)_($bucket.name) with ($bucket.ram_quota / 1024 / 1024 ) quota, type ($bucket.type), ($bucket.replicas) replicas, ($bucket.min_durability_level) durability, ($bucket.max_expiry) expiry" + if ( $bucket.flush_enabled) { + $bucket | buckets create $in.name ( $in.ram_quota / 1024 / 1024 | into int ) --clusters $destination --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry --flush + } else { + $bucket | buckets create $in.name ( $in.ram_quota / 1024 / 1024 | into int ) --clusters $destination --type $in.type --replicas $in.replicas --durability $in.min_durability_level --expiry $in.max_expiry + } +}