From 3e450017e93dcc6123c6ba0e2d000d31c644f8c2 Mon Sep 17 00:00:00 2001 From: Jakub Cierlik Date: Fri, 2 Jul 2021 07:20:03 +0200 Subject: [PATCH 1/4] postgres version upgrade - dev --- README.md | 29 +++++++++++++++++++++++++++ infra/dev/vm/aws-blueprint.yaml | 17 ++++++++++++++++ infra/dev/vm/scripts/upgrade-psql.sh | 30 ++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 infra/dev/vm/scripts/upgrade-psql.sh diff --git a/README.md b/README.md index ca7d3c1..5b59675 100644 --- a/README.md +++ b/README.md @@ -205,3 +205,32 @@ Retrieving capabilities for deployment app_prod... Description: URL of S3 bucket Value: https://wtgjexngbucket.s3.us-west-1.amazonaws.com ``` + +## PostgreSQL version upgrade on AWS + +To perform the PostgreSQL version upgrade on `dev` environment run `execute_operation` on: +- `aws-dev-small-vm` deployment when using "development-small" type +- `aws-dev-large-vm-db` deployment when using "development-large" type + +Workflow parameters: +``` +{ + "operation":"cloudify.interfaces.lifecycle.upgrade", + "operation_kwargs":{ + "process": { + "env": { + "POSTGRES_NEW_VERSION": "10" + } + } + }, + "allow_kwargs_override":true, + "run_by_dependency_order":false, + "type_names":[], + "node_ids":[ + "upgrade_psql" + ], + "node_instance_ids":[] +} +``` +`POSTGRES_NEW_VERSION` is the desired major version of PostgreSQL server. +Should be provided as a string and have one of the following values: `10`, `11`, `12` or `13`. diff --git a/infra/dev/vm/aws-blueprint.yaml b/infra/dev/vm/aws-blueprint.yaml index ec86977..c15dfd8 100644 --- a/infra/dev/vm/aws-blueprint.yaml +++ b/infra/dev/vm/aws-blueprint.yaml @@ -3,6 +3,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - https://cloudify.co/spec/cloudify/5.2.0/types.yaml - plugin:cloudify-aws-plugin?version= >=2.5.6 + - plugin:cloudify-fabric-plugin?version= >=2.0.7 dsl_definitions: aws_client: &aws_client @@ -49,6 +50,7 @@ node_templates: target: ip - type: cloudify.relationships.depends_on target: nic + ip: type: cloudify.nodes.aws.ec2.ElasticIP properties: @@ -61,6 +63,7 @@ node_templates: relationships: - type: cloudify.relationships.depends_on target: nic + nic: type: cloudify.nodes.aws.ec2.Interface properties: @@ -77,6 +80,20 @@ node_templates: - target: vpc_deployment type: cloudify.relationships.depends_on + upgrade_psql: + type: cloudify.nodes.Root + interfaces: + cloudify.interfaces.lifecycle: + upgrade: + implementation: fabric.fabric_plugin.tasks.run_script + inputs: + script_path: scripts/upgrade-psql.sh + fabric_env: + host: { get_attribute: [ ip, aws_resource_id ] } + user: centos + connect_kwargs: + pkey: { get_secret: private_key_content } + capabilities: vm_ip: value: { get_attribute: [ ip, aws_resource_id ] } diff --git a/infra/dev/vm/scripts/upgrade-psql.sh b/infra/dev/vm/scripts/upgrade-psql.sh new file mode 100644 index 0000000..65244e3 --- /dev/null +++ b/infra/dev/vm/scripts/upgrade-psql.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +# Creating postgres backup +sudo su postgres -c "cd && pg_dumpall > /tmp/pg9backup" + +# Installing new postgresql version +sudo yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm +sudo yum install -y postgresql$POSTGRES_NEW_VERSION-server + +# Migrating the database +sudo systemctl stop postgresql.service && sudo systemctl stop postgresql-$POSTGRES_NEW_VERSION.service +sudo su postgres -c "cd && /usr/pgsql-$POSTGRES_NEW_VERSION/bin/initdb -D /var/lib/pgsql/$POSTGRES_NEW_VERSION/data/" + +sudo mv /usr/bin/pg_ctl{,-orig} +echo '#!/bin/bash' | sudo tee /usr/bin/pg_ctl +echo '"$0"-orig "${@/unix_socket_directory/unix_socket_directories}"' | sudo tee -a /usr/bin/pg_ctl +sudo chmod +x /usr/bin/pg_ctl + +sudo su postgres -c "cd && /usr/pgsql-$POSTGRES_NEW_VERSION/bin/pg_upgrade --old-datadir /var/lib/pgsql/data/ --new-datadir /var/lib/pgsql/$POSTGRES_NEW_VERSION/data/ --old-bindir /usr/bin/ --new-bindir /usr/pgsql-$POSTGRES_NEW_VERSION/bin/" + +sudo mv -f /usr/bin/pg_ctl{-orig,} + +sudo cp /var/lib/pgsql/data/pg_hba.conf /var/lib/pgsql/$POSTGRES_NEW_VERSION/data/ +sudo cp /var/lib/pgsql/data/postgresql.conf /var/lib/pgsql/$POSTGRES_NEW_VERSION/data/ + +sudo systemctl start postgresql-$POSTGRES_NEW_VERSION.service +sudo systemctl enable postgresql-$POSTGRES_NEW_VERSION + +# Restoring information from the old database +sudo su postgres -c "cd && psql -d postgres -f /tmp/pg9backup" From 1c1db2fb1a20deaede3787cadb4de04a1c44285d Mon Sep 17 00:00:00 2001 From: Jakub Cierlik Date: Fri, 2 Jul 2021 09:44:34 +0200 Subject: [PATCH 2/4] aws rds creation fix --- README.md | 3 ++- infra/prod/rds_psql/aws-blueprint.yaml | 3 ++- secret.json | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5b59675..01faa3c 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ The blueprint requires the following inputs: 8. Create a "production" deployment of the `app` blueprint, and install it on AWS: ```bash - cfy deployments create app_prod -b app -i env_type=production -i cloud_type=AWS + cfy deployments create app_prod -b app -i env_type=production -i cloud_type=aws cfy executions start install -d app_prod ``` @@ -234,3 +234,4 @@ Workflow parameters: ``` `POSTGRES_NEW_VERSION` is the desired major version of PostgreSQL server. Should be provided as a string and have one of the following values: `10`, `11`, `12` or `13`. +Check with the `sudo systemctl -a | grep postgres` command if the desired version of PostgreSQL server is running. diff --git a/infra/prod/rds_psql/aws-blueprint.yaml b/infra/prod/rds_psql/aws-blueprint.yaml index 5542061..6fd5bb2 100644 --- a/infra/prod/rds_psql/aws-blueprint.yaml +++ b/infra/prod/rds_psql/aws-blueprint.yaml @@ -66,10 +66,11 @@ node_templates: MyDB: Type: "AWS::RDS::DBInstance" Properties: - DBName: { concat: [ { get_input: resource_prefix }, 'rdspsql' ] } + DBName: { concat: [ 'rdspsql', { get_input: resource_prefix } ] } AllocatedStorage: 5 DBInstanceClass: db.t2.small Engine: postgres + EngineVersion: "9.6.21" MasterUsername: { get_input: master_username } MasterUserPassword: { get_attribute: [ password_generator, master_password ] } VPCSecurityGroups: diff --git a/secret.json b/secret.json index a49a078..2887a68 100644 --- a/secret.json +++ b/secret.json @@ -89,7 +89,7 @@ "blueprint": "aws_prod_network", "deployment": "aws-prod-network", "inputs": { - "resource_prefix": "aws-production", + "resource_prefix": "awsproduction", "aws_region_name": "us-west-1", "availability_zones": [ "us-west-1a", @@ -104,7 +104,7 @@ "inputs": { "network_deployment_id": "aws-prod-network", "aws_region_name": "us-west-1", - "resource_prefix": "aws-production" + "resource_prefix": "awsproduction" } }, "db": { @@ -113,7 +113,7 @@ "inputs": { "network_deployment_id": "aws-prod-network", "aws_region_name": "us-west-1", - "resource_prefix": "aws-production", + "resource_prefix": "awsproduction", "master_username": "psqladmin" } }, @@ -123,7 +123,7 @@ "inputs": { "bucket_name": "eaasprodbucket", "bucket_region": "us-west-1", - "resource_prefix": "aws-production" + "resource_prefix": "awsproduction" } } } From 687bdcea9e407004fc33fb4d68fcb7e1de69a33b Mon Sep 17 00:00:00 2001 From: Jakub Cierlik Date: Mon, 12 Jul 2021 09:46:04 +0200 Subject: [PATCH 3/4] aws rds scale and postgres version upgrade --- README.md | 49 +++++++++++++++++++ infra/prod/rds_psql/aws-blueprint.yaml | 28 ++++++++++- infra/prod/rds_psql/scripts/scale-up-rds.sh | 12 +++++ .../prod/rds_psql/scripts/upgrade-rds-psql.sh | 13 +++++ 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 infra/prod/rds_psql/scripts/scale-up-rds.sh create mode 100644 infra/prod/rds_psql/scripts/upgrade-rds-psql.sh diff --git a/README.md b/README.md index 01faa3c..e7cca4d 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ Retrieving capabilities for deployment app_prod... ## PostgreSQL version upgrade on AWS +### Development environments To perform the PostgreSQL version upgrade on `dev` environment run `execute_operation` on: - `aws-dev-small-vm` deployment when using "development-small" type - `aws-dev-large-vm-db` deployment when using "development-large" type @@ -235,3 +236,51 @@ Workflow parameters: `POSTGRES_NEW_VERSION` is the desired major version of PostgreSQL server. Should be provided as a string and have one of the following values: `10`, `11`, `12` or `13`. Check with the `sudo systemctl -a | grep postgres` command if the desired version of PostgreSQL server is running. + +### Production environment +To perform the PostgreSQL version upgrade on `prod` environment run `execute operation` on `aws-prod-database` deployment. +Workflow parameters: +``` +{ + "operation":"cloudify.interfaces.lifecycle.upgrade", + "operation_kwargs":{ + "new_postgres_version": "12.7" + }, + "allow_kwargs_override":true, + "run_by_dependency_order":false, + "type_names":[], + "node_ids":[ + "upgrade_rds_psql_version" + ], + "node_instance_ids":[] +} +``` +`new_postgres_version` parameter is the desired major version of PostgreSQL server to be run by AWS RDS. +The workflow automatically creates a snapshot of the RDS (backs-up database) and performs the upgrade +of PostgreSQL engine version if such an upgrade if possible. +For information regarding AWS' requirements of PostgreSQL version upgrade, please refer to the +[AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_UpgradeDBInstance.PostgreSQL.html). + +## PostgreSQL database manual scale on AWS + +### Production environment +To perform the PostgreSQL database on RDS scale-up on `prod` environment run `execute operation` on `aws-prod-database` deployment. +``` +{ + "operation":"cloudify.interfaces.lifecycle.upgrade", + "operation_kwargs":{ + "new_instance_class": "db.t2.large" + }, + "allow_kwargs_override":true, + "run_by_dependency_order":false, + "type_names":[], + "node_ids":[ + "scale_up_rds" + ], + "node_instance_ids":[] +} +``` +By default, the RDS is created and provisioned using DB instance class `db.t2.small`. +`new_instance_class` parameter is the new desired DB instance class of the RDS instance. +For information regarding allowed RDS instance classes, please refer to the +[AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.DBInstanceClass.html). diff --git a/infra/prod/rds_psql/aws-blueprint.yaml b/infra/prod/rds_psql/aws-blueprint.yaml index 6fd5bb2..9f6a9cf 100644 --- a/infra/prod/rds_psql/aws-blueprint.yaml +++ b/infra/prod/rds_psql/aws-blueprint.yaml @@ -70,7 +70,7 @@ node_templates: AllocatedStorage: 5 DBInstanceClass: db.t2.small Engine: postgres - EngineVersion: "9.6.21" + EngineVersion: "12.6" MasterUsername: { get_input: master_username } MasterUserPassword: { get_attribute: [ password_generator, master_password ] } VPCSecurityGroups: @@ -88,6 +88,32 @@ node_templates: - target: password_generator type: cloudify.relationships.depends_on + upgrade_rds_psql_version: + type: cloudify.nodes.Root + interfaces: + cloudify.interfaces.lifecycle: + upgrade: + executor: central_deployment_agent + implementation: scripts/upgrade-rds-psql.sh + inputs: + aws_access_key_id: { get_secret: aws_access_key_id } + aws_secret_access_key: { get_secret: aws_secret_access_key } + aws_region: { get_input: aws_region_name } + db_instance_identifier: { get_attribute: [ database_stack, state, 0, PhysicalResourceId ] } + + scale_up_rds: + type: cloudify.nodes.Root + interfaces: + cloudify.interfaces.lifecycle: + upgrade: + executor: central_deployment_agent + implementation: scripts/scale-up-rds.sh + inputs: + aws_access_key_id: { get_secret: aws_access_key_id } + aws_secret_access_key: { get_secret: aws_secret_access_key } + aws_region: { get_input: aws_region_name } + db_instance_identifier: { get_attribute: [ database_stack, state, 0, PhysicalResourceId ] } + capabilities: host: value: { get_attribute: [ database_stack, outputs_items, IpAddress ] } diff --git a/infra/prod/rds_psql/scripts/scale-up-rds.sh b/infra/prod/rds_psql/scripts/scale-up-rds.sh new file mode 100644 index 0000000..47654b8 --- /dev/null +++ b/infra/prod/rds_psql/scripts/scale-up-rds.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +sudo yum install awscli -y + +aws configure set aws_access_key_id ${aws_access_key_id} +aws configure set aws_secret_access_key ${aws_secret_access_key} +aws configure set default.region ${aws_region} + +aws rds modify-db-instance \ + --db-instance-identifier ${db_instance_identifier} \ + --db-instance-class "${new_instance_class}" \ + --apply-immediately diff --git a/infra/prod/rds_psql/scripts/upgrade-rds-psql.sh b/infra/prod/rds_psql/scripts/upgrade-rds-psql.sh new file mode 100644 index 0000000..dbfd4cf --- /dev/null +++ b/infra/prod/rds_psql/scripts/upgrade-rds-psql.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e + +sudo yum install awscli -y + +aws configure set aws_access_key_id ${aws_access_key_id} +aws configure set aws_secret_access_key ${aws_secret_access_key} +aws configure set default.region ${aws_region} + +aws rds modify-db-instance \ + --db-instance-identifier ${db_instance_identifier} \ + --engine-version "${new_postgres_version}" \ + --allow-major-version-upgrade \ + --apply-immediately From b221f901905454cc1f79e290d341a00d1afb84b7 Mon Sep 17 00:00:00 2001 From: Jakub Cierlik Date: Thu, 15 Jul 2021 13:15:00 +0200 Subject: [PATCH 4/4] manual scale - dev environments --- README.md | 27 ++++++++++++++++++++++++ infra/dev/multi_node/aws-blueprint.yaml | 4 ++++ infra/dev/single_node/aws-blueprint.yaml | 3 +++ infra/dev/vm/aws-blueprint.yaml | 25 ++++++++++++++++++++++ secret.json | 4 ++++ 5 files changed, 63 insertions(+) diff --git a/README.md b/README.md index e7cca4d..e3930eb 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,7 @@ Workflow parameters: } ``` `new_postgres_version` parameter is the desired major version of PostgreSQL server to be run by AWS RDS. +The solution uses AWS CLI which is installed on the Cloudify Manager VM and connects to user's AWS account using AWS access key and AWS secret access key stored in the secrets. The workflow automatically creates a snapshot of the RDS (backs-up database) and performs the upgrade of PostgreSQL engine version if such an upgrade if possible. For information regarding AWS' requirements of PostgreSQL version upgrade, please refer to the @@ -263,6 +264,31 @@ For information regarding AWS' requirements of PostgreSQL version upgrade, pleas ## PostgreSQL database manual scale on AWS +### Development environments +To perform the PostgreSQL database VM scale-up on `dev` environment run `Cloudify custom workflow` `scale vm` on: +- `aws-dev-small-vm` deployment when using "development-small" type +- `aws-dev-large-vm-db` deployment when using "development-large" type + +The workflow takes the desired new instance type name from the `eaas_params` secret according to used environment type: +``` +{ + "aws": { + "dev-small": { + "network": { + "vm_scale_instance_name": "t2.large" + } + }, + "dev-large": { + "network": { + "vm_scale_instance_name": "t2.large" + } + } + } +} +``` +The virtual machine is stopped, modified and reconfigured to different instance type (scale-up) and then immediately started again. +To change the desired instance type after scale-up update the `eaas_params` secret in the Cloudify Manager's secrets store. + ### Production environment To perform the PostgreSQL database on RDS scale-up on `prod` environment run `execute operation` on `aws-prod-database` deployment. ``` @@ -282,5 +308,6 @@ To perform the PostgreSQL database on RDS scale-up on `prod` environment run `ex ``` By default, the RDS is created and provisioned using DB instance class `db.t2.small`. `new_instance_class` parameter is the new desired DB instance class of the RDS instance. +The solution uses AWS CLI which is installed on the Cloudify Manager VM and connects to user's AWS account using AWS access key and AWS secret access key stored in the secrets. For information regarding allowed RDS instance classes, please refer to the [AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.DBInstanceClass.html). diff --git a/infra/dev/multi_node/aws-blueprint.yaml b/infra/dev/multi_node/aws-blueprint.yaml index c80bd81..f0a1b03 100644 --- a/infra/dev/multi_node/aws-blueprint.yaml +++ b/infra/dev/multi_node/aws-blueprint.yaml @@ -11,6 +11,8 @@ dsl_definitions: region_name: { get_input: aws_region_name } inputs: + env_type: + type: string resource_prefix: type: string aws_region_name: @@ -21,6 +23,7 @@ inputs: type: string instance_type: type: string + node_templates: vpc_deployment: @@ -101,6 +104,7 @@ node_templates: aws_region_name: { get_input: aws_region_name } ami_id: { get_input: ami_id } instance_type: { get_input: instance_type } + env_type: { get_input: env_type } relationships: - target: vpc_deployment type: cloudify.relationships.depends_on diff --git a/infra/dev/single_node/aws-blueprint.yaml b/infra/dev/single_node/aws-blueprint.yaml index b856e69..18f8660 100644 --- a/infra/dev/single_node/aws-blueprint.yaml +++ b/infra/dev/single_node/aws-blueprint.yaml @@ -11,6 +11,8 @@ dsl_definitions: region_name: { get_input: aws_region_name } inputs: + env_type: + type: string resource_prefix: type: string aws_region_name: @@ -101,6 +103,7 @@ node_templates: aws_region_name: { get_input: aws_region_name } ami_id: { get_input: ami_id } instance_type: { get_input: instance_type } + env_type: { get_input: env_type } relationships: - target: vpc_deployment type: cloudify.relationships.depends_on diff --git a/infra/dev/vm/aws-blueprint.yaml b/infra/dev/vm/aws-blueprint.yaml index c15dfd8..051440b 100644 --- a/infra/dev/vm/aws-blueprint.yaml +++ b/infra/dev/vm/aws-blueprint.yaml @@ -4,6 +4,7 @@ imports: - https://cloudify.co/spec/cloudify/5.2.0/types.yaml - plugin:cloudify-aws-plugin?version= >=2.5.6 - plugin:cloudify-fabric-plugin?version= >=2.0.7 + - plugin:cloudify-utilities-plugin dsl_definitions: aws_client: &aws_client @@ -22,6 +23,8 @@ inputs: type: string instance_type: type: string + env_type: + type: string node_templates: vpc_deployment: @@ -45,6 +48,15 @@ node_templates: Tags: - Key: Name Value: { concat: [ { get_input: resource_prefix }, '-vm' ] } + interfaces: + cloudify.interfaces.lifecycle: + scale: + implementation: aws.cloudify_aws.ec2.resources.instances.modify_instance_attribute + inputs: + resource_config: + InstanceId: { get_attribute: [ SELF, aws_resource_id ] } + Attribute: instanceType + Value: { get_secret: [ eaas_params, aws, { get_input: env_type }, network, vm_scale_instance_name ] } relationships: - type: cloudify.relationships.depends_on target: ip @@ -94,6 +106,19 @@ node_templates: connect_kwargs: pkey: { get_secret: private_key_content } +workflows: + scale_vm: + mapping: cloudify_custom_workflow.cloudify_custom_workflow.tasks.customwf + parameters: + nodes_to_runon: + default: + - vm + operations_to_execute: + default: + - cloudify.interfaces.lifecycle.stop + - cloudify.interfaces.lifecycle.scale + - cloudify.interfaces.lifecycle.start + capabilities: vm_ip: value: { get_attribute: [ ip, aws_resource_id ] } diff --git a/secret.json b/secret.json index 2887a68..26327f5 100644 --- a/secret.json +++ b/secret.json @@ -4,7 +4,9 @@ "network": { "blueprint": "aws_single_node", "deployment": "aws-dev-small-network", + "vm_scale_instance_name": "t2.large", "inputs": { + "env_type": "dev-small", "resource_prefix": "aws-dev-small", "aws_region_name": "us-west-1", "availability_zones": [ @@ -46,7 +48,9 @@ "network": { "blueprint": "aws_multi_node", "deployment": "aws-dev-large-network", + "vm_scale_instance_name": "t2.large", "inputs": { + "env_type": "dev-large", "resource_prefix": "aws-dev-large", "aws_region_name": "us-west-1", "availability_zones": [