From 3bd924375180b05bfc0cdc0e8594b1cb83b4499a Mon Sep 17 00:00:00 2001 From: Aaron Feledy Date: Wed, 8 May 2024 22:22:40 -0400 Subject: [PATCH 1/6] Add test cases for sql import/export scripts --- .github/workflows/pr-core-tests.yml | 1 + CHANGELOG.md | 1 + examples/sql-helpers/.gitignore | 1 + examples/sql-helpers/.lando.sqlhelpers.yml | 54 +++++++++++ examples/sql-helpers/README.md | 101 +++++++++++++++++++++ examples/sql-helpers/testdata1.sql | 6 ++ examples/sql-helpers/testdata2.sql | 6 ++ 7 files changed, 170 insertions(+) create mode 100644 examples/sql-helpers/.gitignore create mode 100644 examples/sql-helpers/.lando.sqlhelpers.yml create mode 100644 examples/sql-helpers/README.md create mode 100644 examples/sql-helpers/testdata1.sql create mode 100644 examples/sql-helpers/testdata2.sql diff --git a/.github/workflows/pr-core-tests.yml b/.github/workflows/pr-core-tests.yml index 675344605..cd18a7bcd 100644 --- a/.github/workflows/pr-core-tests.yml +++ b/.github/workflows/pr-core-tests.yml @@ -31,6 +31,7 @@ jobs: - examples/orchestrator - examples/plugin-commands - examples/services + - examples/sql-helpers - examples/tooling node-version: - "18" diff --git a/CHANGELOG.md b/CHANGELOG.md index 593f92858..f9c45deec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Unreleased * Updated `sql-export.sh` to use `mariadb-dump` command (if available). [#148](https://github.com/lando/core/pull/148) +* Added `leia` tests for `sql-import.sh` and `sql-export.sh` ## v3.21.0-beta.18 - [April 29, 2024](https://github.com/lando/core/releases/tag/v3.21.0-beta.18) diff --git a/examples/sql-helpers/.gitignore b/examples/sql-helpers/.gitignore new file mode 100644 index 000000000..39d63ac45 --- /dev/null +++ b/examples/sql-helpers/.gitignore @@ -0,0 +1 @@ +sqlhelpers diff --git a/examples/sql-helpers/.lando.sqlhelpers.yml b/examples/sql-helpers/.lando.sqlhelpers.yml new file mode 100644 index 000000000..0a40d7eed --- /dev/null +++ b/examples/sql-helpers/.lando.sqlhelpers.yml @@ -0,0 +1,54 @@ +name: lando-sqlhelpers +services: + mariadb: + api: 3 + type: lando + services: + image: bitnami/mariadb:10.4 + command: /opt/bitnami/scripts/mariadb/entrypoint.sh /opt/bitnami/scripts/mariadb/run.sh + environment: + ALLOW_EMPTY_PASSWORD: yes + MARIADB_DATABASE: lando_test + MYSQL_DATABASE: lando_test + MARIADB_USER: test + MARIADB_PASSWORD: test + + mysql57: + api: 3 + type: lando + services: + image: bitnami/mysql:5.7 + command: /opt/bitnami/scripts/mysql/entrypoint.sh /opt/bitnami/scripts/mysql/run.sh + environment: + ALLOW_EMPTY_PASSWORD: yes + MYSQL_AUTHENTICATION_PLUGIN: mysql_native_password + MYSQL_DATABASE: lando_test + MYSQL_PASSWORD: test + MYSQL_USER: test + + mysql80: + api: 3 + type: lando + services: + image: bitnami/mysql:8.0 + command: /opt/bitnami/scripts/mysql/entrypoint.sh /opt/bitnami/scripts/mysql/run.sh + environment: + ALLOW_EMPTY_PASSWORD: yes + MYSQL_AUTHENTICATION_PLUGIN: caching_sha2_password + MYSQL_DATABASE: lando_test + MYSQL_PASSWORD: test + MYSQL_USER: test + + postgres16: + api: 3 + type: lando + services: + image: bitnami/postgresql:16 + command: /opt/bitnami/scripts/postgresql/entrypoint.sh /opt/bitnami/scripts/postgresql/run.sh + environment: + ALLOW_EMPTY_PASSWORD: yes + POSTGRESQL_DATABASE: lando_test + POSTGRES_DB: lando_test + +plugins: + "@lando/core": "../../.." diff --git a/examples/sql-helpers/README.md b/examples/sql-helpers/README.md new file mode 100644 index 000000000..8be680250 --- /dev/null +++ b/examples/sql-helpers/README.md @@ -0,0 +1,101 @@ +SQL Import/Export Example +============ + +This example exists primarily to test the `sql-import.sh` and `sql-export.sh` helper scripts. + +Start up tests +-------------- + +```bash +# Should init and start a lando app +rm -rf sqlhelpers && mkdir -p sqlhelpers +cp -rf .lando.sqlhelpers.yml sqlhelpers/.lando.yml +cp -rf testdata1.sql sqlhelpers/testdata1.sql +cp -rf testdata2.sql sqlhelpers/testdata2.sql +cd sqlhelpers && lando start +``` + +Verification commands +--------------------- + +Run the following commands to verify things work as expected + +```bash +# Should import test data into mariadb +cd sqlhelpers +lando ssh -s mariadb -c "/helpers/sql-import.sh testdata1.sql" +lando ssh -s mariadb -c "mysql -utest -ptest lando_test -e 'select * from lando_test'" | grep "lando_original" + +# Should import test data into mysql57 +cd sqlhelpers +lando ssh -s mysql57 -c "/helpers/sql-import.sh testdata1.sql" +lando ssh -s mysql57 -c "mysql -utest -ptest lando_test -e 'select * from lando_test'" | grep "lando_original" + +# Should import test data into mysql80 +cd sqlhelpers +lando ssh -s mysql80 -c "/helpers/sql-import.sh testdata1.sql" +lando ssh -s mysql80 -c "mysql -utest -ptest lando_test -e 'select * from lando_test'" | grep "lando_original" + +# Should import test data into postgres16 +cd sqlhelpers +lando ssh -s postgres16 -c "/helpers/sql-import.sh testdata1.sql" +lando ssh -s postgres16 -c "psql -U postgres -d lando_test -c 'select * from lando_test'" | grep "lando_original" + +# Should export gzipped files from mariadb +cd sqlhelpers +lando ssh -s mariadb -c "/helpers/sql-export.sh mariadb_dump.sql" -u root +gzip -d mariadb_dump.sql.gz + +# Should export gzipped files from mysql57 +cd sqlhelpers +lando ssh -s mysql57 -c "/helpers/sql-export.sh mysql57_dump.sql" -u root +gzip -d mysql57_dump.sql.gz + +# Should export gzipped files from mysql80 +cd sqlhelpers +lando ssh -s mysql80 -c "/helpers/sql-export.sh mysql80_dump.sql" -u root +gzip -d mysql80_dump.sql.gz + +# Should export gzipped files from postgres16 +cd sqlhelpers +lando ssh -s postgres16 -c "/helpers/sql-export.sh postgres16_dump.sql" -u root +gzip -d postgres16_dump.sql.gz + +# Should have the correct data in all exported files +cd sqlhelpers +grep "lando_original" mariadb_dump.sql +grep "lando_original" mysql57_dump.sql +grep "lando_original" mysql80_dump.sql +grep "lando_original" postgres16_dump.sql + +# Should be able to replace data with testdata2 in mariadb +cd sqlhelpers +lando ssh -s mariadb -c "/helpers/sql-import.sh testdata2.sql" +lando ssh -s mariadb -c "mysql -utest -ptest lando_test -e 'select * from lando_test'" | grep -v "lando_original" | grep "lando_updated" + +# Should be able to replace data with testdata2 in mysql57 +cd sqlhelpers +lando ssh -s mysql57 -c "/helpers/sql-import.sh testdata2.sql" +lando ssh -s mysql57 -c "mysql -utest -ptest lando_test -e 'select * from lando_test'" | grep -v "lando_original" | grep "lando_updated" + +# Should be able to replace data with testdata2 in mysql80 +cd sqlhelpers +lando ssh -s mysql80 -c "/helpers/sql-import.sh testdata2.sql" +lando ssh -s mysql80 -c "mysql -utest -ptest lando_test -e 'select * from lando_test'" | grep -v "lando_original" | grep "lando_updated" + +# Should be able to replace data with testdata2 in postgres16 +cd sqlhelpers +lando ssh -s postgres16 -c "/helpers/sql-import.sh testdata2.sql" +lando ssh -s postgres16 -c "psql -U postgres -d lando_test -c 'select * from lando_test'" | grep -v "lando_original" | grep "lando_updated" +``` + +Destroy tests +------------- + +```bash +# Should destroy sqlhelpers successfully +cd sqlhelpers && lando destroy -y + +# Should poweroff +lando poweroff +``` diff --git a/examples/sql-helpers/testdata1.sql b/examples/sql-helpers/testdata1.sql new file mode 100644 index 000000000..deaef96d4 --- /dev/null +++ b/examples/sql-helpers/testdata1.sql @@ -0,0 +1,6 @@ +CREATE TABLE lando_test ( + id SERIAL PRIMARY KEY, + column1 VARCHAR(255) DEFAULT NULL, + column2 VARCHAR(255) DEFAULT NULL +); +INSERT INTO lando_test VALUES (1, 'lando_original', 'lando_original'); diff --git a/examples/sql-helpers/testdata2.sql b/examples/sql-helpers/testdata2.sql new file mode 100644 index 000000000..36d757c4d --- /dev/null +++ b/examples/sql-helpers/testdata2.sql @@ -0,0 +1,6 @@ +CREATE TABLE lando_test ( + id SERIAL PRIMARY KEY, + column1 VARCHAR(255) DEFAULT NULL, + column2 VARCHAR(255) DEFAULT NULL +); +INSERT INTO lando_test VALUES (1, 'lando_updated', 'lando_updated'); From 12a5ece9586c06cd94b54cc197a9ff50eb23e2d9 Mon Sep 17 00:00:00 2001 From: Aaron Feledy Date: Thu, 9 May 2024 12:03:27 -0400 Subject: [PATCH 2/6] healthchecks because scripts are faster than me --- examples/sql-helpers/.lando.sqlhelpers.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/sql-helpers/.lando.sqlhelpers.yml b/examples/sql-helpers/.lando.sqlhelpers.yml index 0a40d7eed..42acb1eac 100644 --- a/examples/sql-helpers/.lando.sqlhelpers.yml +++ b/examples/sql-helpers/.lando.sqlhelpers.yml @@ -3,6 +3,7 @@ services: mariadb: api: 3 type: lando + healthcheck: mysqladmin ping -h mariadb -u test -ptest services: image: bitnami/mariadb:10.4 command: /opt/bitnami/scripts/mariadb/entrypoint.sh /opt/bitnami/scripts/mariadb/run.sh @@ -16,6 +17,7 @@ services: mysql57: api: 3 type: lando + healthcheck: mysqladmin ping -h mysql57 -u test -ptest services: image: bitnami/mysql:5.7 command: /opt/bitnami/scripts/mysql/entrypoint.sh /opt/bitnami/scripts/mysql/run.sh @@ -29,6 +31,7 @@ services: mysql80: api: 3 type: lando + healthcheck: mysqladmin ping -h mysql80 -u test -ptest services: image: bitnami/mysql:8.0 command: /opt/bitnami/scripts/mysql/entrypoint.sh /opt/bitnami/scripts/mysql/run.sh @@ -42,6 +45,7 @@ services: postgres16: api: 3 type: lando + healthcheck: pg_isready -h postgres16 -U postgres services: image: bitnami/postgresql:16 command: /opt/bitnami/scripts/postgresql/entrypoint.sh /opt/bitnami/scripts/postgresql/run.sh From ed5329712936db615e084e12fa271054a023618e Mon Sep 17 00:00:00 2001 From: Chris Burgess Date: Sat, 22 Jul 2023 07:18:00 +1200 Subject: [PATCH 3/6] Use DROP/CREATE strategy for MySQL DB reset --- scripts/sql-import.sh | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/scripts/sql-import.sh b/scripts/sql-import.sh index 688f4a18d..cd5d52109 100755 --- a/scripts/sql-import.sh +++ b/scripts/sql-import.sh @@ -108,21 +108,13 @@ if [ "$WIPE" == "true" ]; then lando_green "\t\tCreating database ...\n\n" psql postgresql://$USER@$HOST:$PORT/postgres -c "create database $DATABASE" else - # Build the SQL prefix + # Drop and recreate database SQLSTART="mysql -h $HOST -P $PORT -u $USER ${LANDO_EXTRA_DB_IMPORT_ARGS} $DATABASE" + lando_yellow "\t\tDropping database ...\n\n" + $SQLSTART -e "DROP DATABASE IF EXISTS ${DATABASE}" - # Gather and destroy tables - TABLES=$($SQLSTART -e 'SHOW TABLES' | awk '{ print $1}' | grep -v '^Tables' || true) - - # PURGE IT ALL! Drop views and tables as needed - for t in $TABLES; do - echo "Dropping $t from $DATABASE database..." - $SQLSTART <<-EOF - SET FOREIGN_KEY_CHECKS=0; - DROP VIEW IF EXISTS \`$t\`; - DROP TABLE IF EXISTS \`$t\`; -EOF - done + lando_green "\t\tCreating database ...\n\n" + $SQLSTART -e "CREATE DATABASE ${DATABASE}" fi fi From 6ddc1d19f440badd394791ecc30b4e96eb0a737c Mon Sep 17 00:00:00 2001 From: Aaron Feledy Date: Thu, 9 May 2024 13:33:26 -0400 Subject: [PATCH 4/6] Should work whether or not the database exists --- scripts/sql-import.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/sql-import.sh b/scripts/sql-import.sh index cd5d52109..aa831d8fc 100755 --- a/scripts/sql-import.sh +++ b/scripts/sql-import.sh @@ -103,17 +103,20 @@ if [ "$WIPE" == "true" ]; then if [[ ${POSTGRES_DB} != '' ]]; then # Drop and recreate database lando_yellow "\t\tDropping database ...\n\n" - psql postgresql://$USER@$HOST:$PORT/postgres -c "drop database $DATABASE" + psql postgresql://$USER@$HOST:$PORT/postgres -c "DROP DATABASE IF EXISTS $DATABASE" lando_green "\t\tCreating database ...\n\n" - psql postgresql://$USER@$HOST:$PORT/postgres -c "create database $DATABASE" + psql postgresql://$USER@$HOST:$PORT/postgres -c "CREATE DATABASE $DATABASE" else - # Drop and recreate database - SQLSTART="mysql -h $HOST -P $PORT -u $USER ${LANDO_EXTRA_DB_IMPORT_ARGS} $DATABASE" + # Connection string + SQLSTART="mysql -h $HOST -P $PORT -u $USER ${LANDO_EXTRA_DB_IMPORT_ARGS}" + lando_yellow "\t\tDropping database ...\n\n" + # Drop the database $SQLSTART -e "DROP DATABASE IF EXISTS ${DATABASE}" lando_green "\t\tCreating database ...\n\n" + # Create the database $SQLSTART -e "CREATE DATABASE ${DATABASE}" fi fi From 9e8a7f317251eb85770b015da063ba8e939dab9d Mon Sep 17 00:00:00 2001 From: Aaron Feledy Date: Thu, 9 May 2024 13:54:07 -0400 Subject: [PATCH 5/6] Turn down that racket! --- scripts/sql-import.sh | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/scripts/sql-import.sh b/scripts/sql-import.sh index aa831d8fc..6db2c2b7f 100755 --- a/scripts/sql-import.sh +++ b/scripts/sql-import.sh @@ -95,28 +95,20 @@ echo "Preparing to import $FILE into database '$DATABASE' on service '$SERVICE' # Wipe the database if set if [ "$WIPE" == "true" ]; then - echo "" - echo "Emptying $DATABASE... " + lando_pink "\nEmptying $DATABASE... " lando_yellow "NOTE: See the --no-wipe flag to avoid this step!" # DO db specific wiping if [[ ${POSTGRES_DB} != '' ]]; then # Drop and recreate database - lando_yellow "\t\tDropping database ...\n\n" - psql postgresql://$USER@$HOST:$PORT/postgres -c "DROP DATABASE IF EXISTS $DATABASE" - - lando_green "\t\tCreating database ...\n\n" - psql postgresql://$USER@$HOST:$PORT/postgres -c "CREATE DATABASE $DATABASE" + psql postgresql://$USER@$HOST:$PORT/postgres -c "DROP DATABASE IF EXISTS $DATABASE" --quiet --echo-errors + psql postgresql://$USER@$HOST:$PORT/postgres -c "CREATE DATABASE $DATABASE" --quiet --echo-errors else # Connection string SQLSTART="mysql -h $HOST -P $PORT -u $USER ${LANDO_EXTRA_DB_IMPORT_ARGS}" - lando_yellow "\t\tDropping database ...\n\n" - # Drop the database + # Drop and recreate database $SQLSTART -e "DROP DATABASE IF EXISTS ${DATABASE}" - - lando_green "\t\tCreating database ...\n\n" - # Create the database $SQLSTART -e "CREATE DATABASE ${DATABASE}" fi fi @@ -146,7 +138,7 @@ fi # Build DB specific import command if [[ ${POSTGRES_DB} != '' ]]; then - CMD="$CMD | psql postgresql://$USER@$HOST:$PORT/$DATABASE" + CMD="$CMD | psql --quiet --echo-errors postgresql://$USER@$HOST:$PORT/$DATABASE" else CMD="$CMD | mysql -h $HOST -P $PORT -u $USER ${LANDO_EXTRA_DB_IMPORT_ARGS} $DATABASE" fi From 71f096a305d673e1c404accb7836db6235b48fa9 Mon Sep 17 00:00:00 2001 From: Aaron Feledy Date: Thu, 9 May 2024 14:18:00 -0400 Subject: [PATCH 6/6] loggin --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9c45deec..8fb135d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ -## Unreleased +## {{ UNRELEASED_VERSION }} - [{{ UNRELEASED_DATE }}]({{ UNRELEASED_LINK }}) -* Updated `sql-export.sh` to use `mariadb-dump` command (if available). [#148](https://github.com/lando/core/pull/148) * Added `leia` tests for `sql-import.sh` and `sql-export.sh` +* Improved `sql-import.sh` to drop and recreate `mysql` and `mariadb` tables before importing + +* Updated `sql-export.sh` to use `mariadb-dump` command (if available). [#148](https://github.com/lando/core/pull/148) ## v3.21.0-beta.18 - [April 29, 2024](https://github.com/lando/core/releases/tag/v3.21.0-beta.18)