From b1ed11b14642434e060812c2e2edd623428276ba Mon Sep 17 00:00:00 2001 From: Jazmin Smith Date: Thu, 4 Mar 2021 15:36:11 -0500 Subject: [PATCH 1/8] Google CloudSQL SQL Server/v1 --- .../mssql/google-cloud/v1/google-cloud-v1.md | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 _database-integrations/mssql/google-cloud/v1/google-cloud-v1.md diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md new file mode 100644 index 000000000..677c6df7a --- /dev/null +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -0,0 +1,174 @@ +--- +# -------------------------- # +# USING THIS TEMPLATE # +# -------------------------- # + +## NEED HELP USING THIS TEMPLATE? SEE: +## https://docs-about-stitch-docs.netlify.com/reference/integration-templates/databases/ +## FOR INSTRUCTIONS & REFERENCE INFO + + +# -------------------------- # +# Page & Formatting # +# -------------------------- # + +title: Google Cloud SQL Server +keywords: google-cloudsql-microsoft-sql-server, database integration, etl google-cloudsql-microsoft-sql-server, google-cloudsql-microsoft-sql-server etl +permalink: /integrations/databases/google-cloudsql-microsoft-sql-server +summary: "Connect and replicate data from your Google Cloud SQL Server database using Stitch's Google Cloud SQL Server integration." + +show-in-menus: true +key: "google-cloudsql-microsoft-sql-server-integration" + +# -------------------------- # +# Integration Details # +# -------------------------- # + +name: "google-cloudsql-microsoft-sql-server" +display_name: "Google Cloud SQL Server" + +singer: true +tap-name: "Google Cloud SQL Server" +repo-url: "https://github.com/singer-io/tap-mssql" + +this-version: "1" + +hosting-type: "google-cloudsql" ## amazon, microsoft, google, etc. + +driver: "7.2.1.jre8" + + +# -------------------------- # +# Feature Summary # +# -------------------------- # + +feature-summary: | + Stitch's {{ integration.display_name }} integration replicates data using the {{ integration.driver | flatify | strip }}. [TODO] + + +# -------------------------- # +# Stitch Details # +# -------------------------- # + +certified: true + +frequency: "30 minutes" +tier: "Standard" +port: 1433 +db-type: "mssql" + +## Stitch features + +api-type: "platform.mssql" +versions: "2012 through 2017" +ssh: true +ssl: true + +## General replication features + +anchor-scheduling: true +cron-scheduling: true + +extraction-logs: true +loading-reports: true + +table-selection: true +column-selection: true +select-all: false +select-all-reason: | + {{ integration.display_name }} integrations don't currently support a default Replication Method, which is required to use the Select All feature. The default Replication Method setting is only available for integrations that support Log-based Incremental Replication. + +table-level-reset: true + +## Replication methods + +define-replication-methods: true + +log-based-replication-minimum-version: "n/a" +log-based-replication-master-instance: false +log-based-replication-master-instance-reason: "Google CloudSQL doesn't currently support logical replication." +log-based-replication-master-instance-doc-link: "https://groups.google.com/forum/#!topic/google-cloud-sql-discuss/md_7Rq3LgB0" +log-based-replication-read-replica: false + +## Other Replication Methods + +key-based-incremental-replication: true +full-table-replication: true + +view-replication: true + + +# -------------------------- # +# Setup Requirements # +# -------------------------- # + +requirements-list: + - item: "**Permissions in Google Cloud that allow you to modify the database's connection settings.** This is required to whitelist Stitch's IP addresses." + + - item: "**A database running {{ integration.display_name }} version {{ page.versions }}.** {{ integration.display_name }} 2012 is the miminum version that Stitch supports for this type of integration." + + +# -------------------------- # +# Setup Instructions # +# -------------------------- # + +setup-steps: + - title: "Verify your Stitch account's data pipeline region" + anchor: "verify-stitch-account-region" + content: | + {% include shared/whitelisting-ips/locate-region-ip-addresses.html first-step=true %} + + - title: "Configure database connection settings" + anchor: "connect-settings" + content: | + {% include integrations/templates/configure-connection-settings.html %} + + - title: "Create a Stitch database user" + anchor: "create-a-database-user" + content: | + In this step, you'll create a dedicated database user for Stitch. This will ensure Stitch is visible in any logs or audits, and allow you to maintain your privilege hierarchy. + + {% include integrations/templates/create-database-user-tabs.html %} + + - title: "Connect Stitch" + anchor: "connect-stitch" + content: | + In this step, you'll complete the setup by entering the database's connection details and defining replication settings in Stitch. + + {% for substep in step.substeps %} + - [Step 4.{{ forloop.index }}: {{ substep.title | flatify }}](#{{ substep.anchor }}) + {% endfor %} + + substeps: + - title: "Define the database connection details" + anchor: "define-connection-details" + content: | + {% include shared/database-connection-settings.html type="general" %} + + - title: "Define the SSH connection details" + anchor: "ssh-connection-details" + content: | + {% include shared/database-connection-settings.html type="ssh" %} + + - title: "Define the SSL connection details" + anchor: "ssl-connection-details" + content: | + {% include shared/database-connection-settings.html type="ssl" %} + + - title: "Create a replication schedule" + anchor: "create-replication-schedule" + content: | + {% include integrations/shared-setup/replication-frequency.html %} + + - title: "Save the integration" + anchor: "save-integration" + content: | + {% include shared/database-connection-settings.html type="finish-up" %} + + - title: "Set objects to replicate" + anchor: "setting-data-to-replicate" + content: | + {% include integrations/shared-setup/data-selection/object-selection.html %} +--- +{% assign integration = page %} +{% include misc/data-files.html %} \ No newline at end of file From d77dab1edc0efd33f70b1af352652ad34aecc118 Mon Sep 17 00:00:00 2001 From: Jazmin Smith Date: Thu, 4 Mar 2021 16:35:59 -0500 Subject: [PATCH 2/8] add googleto data file --- _data/ui/database-integration-settings/mssql.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/_data/ui/database-integration-settings/mssql.yml b/_data/ui/database-integration-settings/mssql.yml index c6a68d7d6..16dcf442e 100644 --- a/_data/ui/database-integration-settings/mssql.yml +++ b/_data/ui/database-integration-settings/mssql.yml @@ -94,3 +94,16 @@ sql-server-rds: - *password - *database - *schema-name + + +# --------------------------------------- # +# GOOGLE CLOUD SQL SEVER # +# --------------------------------------- # + +google-cloudsql-microsoft-sql-server: + - *integration-name + - *port + - *username + - *password + - *database + - *schema-name From f159691059a89a7f37d95d1bd40a9f899daad08a Mon Sep 17 00:00:00 2001 From: Jazmin Smith Date: Mon, 22 Mar 2021 11:31:34 -0400 Subject: [PATCH 3/8] corrections notes: 1. When I comment out line 46, I get a YAML error. 2. I remember when we looked at the Google CloudSQL SQL Server features we found out that they didin't support log based replication because they used CDC. When I went back to to their docs to confirm, it looks like they don't support that either now, so now I'm not sure what kind of replication they do. --- .../database-setup/user-privileges/mssql.yml | 9 +++++++ .../mssql/google-cloud/v1/google-cloud-v1.md | 26 ++++++------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/_data/taps/extraction/database-setup/user-privileges/mssql.yml b/_data/taps/extraction/database-setup/user-privileges/mssql.yml index 0fe6122b8..5165f1144 100644 --- a/_data/taps/extraction/database-setup/user-privileges/mssql.yml +++ b/_data/taps/extraction/database-setup/user-privileges/mssql.yml @@ -57,6 +57,15 @@ microsoft-azure: reason: *view-change-tracking-reason doc-link: *view-change-tracking-doc +# ------------------------------------ # +# GOOGLE CLOUDSQL PRIVILEGES # +# ------------------------------------ # + +google-cloudsql-microsoft-sql-server: + - name: *select-name + reason: *select-reason + doc-link: *select-doc + # -------------------------- # # SQL COMMANDS # # -------------------------- # diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md index 677c6df7a..14f29b1e8 100644 --- a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -12,10 +12,10 @@ # Page & Formatting # # -------------------------- # -title: Google Cloud SQL Server +title: Google Cloudsql for SQL Server keywords: google-cloudsql-microsoft-sql-server, database integration, etl google-cloudsql-microsoft-sql-server, google-cloudsql-microsoft-sql-server etl permalink: /integrations/databases/google-cloudsql-microsoft-sql-server -summary: "Connect and replicate data from your Google Cloud SQL Server database using Stitch's Google Cloud SQL Server integration." +summary: "Connect and replicate data from your Google Cloud SQL Server database using Stitch's Microsoft SQL Server integration." show-in-menus: true key: "google-cloudsql-microsoft-sql-server-integration" @@ -26,6 +26,7 @@ key: "google-cloudsql-microsoft-sql-server-integration" name: "google-cloudsql-microsoft-sql-server" display_name: "Google Cloud SQL Server" +setup-name: "mssql" singer: true tap-name: "Google Cloud SQL Server" @@ -42,8 +43,8 @@ driver: "7.2.1.jre8" # Feature Summary # # -------------------------- # -feature-summary: | - Stitch's {{ integration.display_name }} integration replicates data using the {{ integration.driver | flatify | strip }}. [TODO] +#feature-summary: | +Stitch's {{ integration.display_name }} integration replicates data using the {{ integration.driver | flatify | strip }}. [TODO] # -------------------------- # @@ -61,8 +62,8 @@ db-type: "mssql" api-type: "platform.mssql" versions: "2012 through 2017" -ssh: true -ssl: true +ssh: false +ssl: false ## General replication features @@ -86,8 +87,7 @@ define-replication-methods: true log-based-replication-minimum-version: "n/a" log-based-replication-master-instance: false -log-based-replication-master-instance-reason: "Google CloudSQL doesn't currently support logical replication." -log-based-replication-master-instance-doc-link: "https://groups.google.com/forum/#!topic/google-cloud-sql-discuss/md_7Rq3LgB0" +#log-based-replication-master-instance-reason: "Google CloudSQL doesn't currently support Log-based Incremental Replication for SQL Server instances." log-based-replication-read-replica: false ## Other Replication Methods @@ -145,16 +145,6 @@ setup-steps: content: | {% include shared/database-connection-settings.html type="general" %} - - title: "Define the SSH connection details" - anchor: "ssh-connection-details" - content: | - {% include shared/database-connection-settings.html type="ssh" %} - - - title: "Define the SSL connection details" - anchor: "ssl-connection-details" - content: | - {% include shared/database-connection-settings.html type="ssl" %} - - title: "Create a replication schedule" anchor: "create-replication-schedule" content: | From 3c85884dce9da11080dab22cf49f031b78b223c7 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Mon, 29 Mar 2021 10:23:12 -0400 Subject: [PATCH 4/8] Match display names to other CloudSQL flavors --- .../mssql/google-cloud/v1/google-cloud-v1.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md index 14f29b1e8..b34904eea 100644 --- a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -12,10 +12,10 @@ # Page & Formatting # # -------------------------- # -title: Google Cloudsql for SQL Server +title: "Google CloudSQL SQL Server (v1)" keywords: google-cloudsql-microsoft-sql-server, database integration, etl google-cloudsql-microsoft-sql-server, google-cloudsql-microsoft-sql-server etl permalink: /integrations/databases/google-cloudsql-microsoft-sql-server -summary: "Connect and replicate data from your Google Cloud SQL Server database using Stitch's Microsoft SQL Server integration." +summary: "Connect and replicate data from your Google CloudSQL SQL Server database using Stitch's Microsoft SQL Server integration." show-in-menus: true key: "google-cloudsql-microsoft-sql-server-integration" @@ -25,11 +25,11 @@ key: "google-cloudsql-microsoft-sql-server-integration" # -------------------------- # name: "google-cloudsql-microsoft-sql-server" -display_name: "Google Cloud SQL Server" +display_name: "Google CloudSQL SQL Server" setup-name: "mssql" singer: true -tap-name: "Google Cloud SQL Server" +tap-name: "Microsoft SQL Server" repo-url: "https://github.com/singer-io/tap-mssql" this-version: "1" @@ -161,4 +161,4 @@ setup-steps: {% include integrations/shared-setup/data-selection/object-selection.html %} --- {% assign integration = page %} -{% include misc/data-files.html %} \ No newline at end of file +{% include misc/data-files.html %} From b43824adbf68dc35c0cb63041b9927ccadc0cea5 Mon Sep 17 00:00:00 2001 From: Jazmin Smith Date: Fri, 2 Apr 2021 08:53:19 -0400 Subject: [PATCH 5/8] Update google-cloud-v1.md --- .../mssql/google-cloud/v1/google-cloud-v1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md index b34904eea..b7e96fd11 100644 --- a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -25,7 +25,7 @@ key: "google-cloudsql-microsoft-sql-server-integration" # -------------------------- # name: "google-cloudsql-microsoft-sql-server" -display_name: "Google CloudSQL SQL Server" +display_name: "Microsoft SQL Server" setup-name: "mssql" singer: true @@ -43,8 +43,8 @@ driver: "7.2.1.jre8" # Feature Summary # # -------------------------- # -#feature-summary: | -Stitch's {{ integration.display_name }} integration replicates data using the {{ integration.driver | flatify | strip }}. [TODO] +##feature-summary: | +##Stitch's {{ integration.display_name }} integration replicates data using the {{ integration.driver | flatify | strip }}. [TODO] # -------------------------- # From da263841629e1f3a2c16fbf14c14d315438c9460 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Wed, 7 Apr 2021 10:07:35 -0400 Subject: [PATCH 6/8] Fix setup-name --- .../mssql/google-cloud/v1/google-cloud-v1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md index b7e96fd11..be98f3787 100644 --- a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -25,8 +25,8 @@ key: "google-cloudsql-microsoft-sql-server-integration" # -------------------------- # name: "google-cloudsql-microsoft-sql-server" -display_name: "Microsoft SQL Server" -setup-name: "mssql" +display_name: "CloudSQL SQL Server" +setup-name: "Microsoft SQL Server" singer: true tap-name: "Microsoft SQL Server" From 0d8c3f19ad04b13a260cb4320e92eb9225fe8c75 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Wed, 7 Apr 2021 10:09:43 -0400 Subject: [PATCH 7/8] Update google-cloud-v1.md --- _database-integrations/mssql/google-cloud/v1/google-cloud-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md index be98f3787..9cfcaed4d 100644 --- a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -87,7 +87,7 @@ define-replication-methods: true log-based-replication-minimum-version: "n/a" log-based-replication-master-instance: false -#log-based-replication-master-instance-reason: "Google CloudSQL doesn't currently support Log-based Incremental Replication for SQL Server instances." +log-based-replication-master-instance-reason: "Google CloudSQL doesn't currently support Change Tracking for CloudSQL SQL Server." log-based-replication-read-replica: false ## Other Replication Methods From c5c003f9ab159723026504be019278910ee0ab79 Mon Sep 17 00:00:00 2001 From: Erin Cochran Date: Wed, 7 Apr 2021 10:23:04 -0400 Subject: [PATCH 8/8] Fix display name --- _database-integrations/mssql/google-cloud/v1/google-cloud-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md index 9cfcaed4d..3d0d4b85e 100644 --- a/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md +++ b/_database-integrations/mssql/google-cloud/v1/google-cloud-v1.md @@ -25,7 +25,7 @@ key: "google-cloudsql-microsoft-sql-server-integration" # -------------------------- # name: "google-cloudsql-microsoft-sql-server" -display_name: "CloudSQL SQL Server" +display_name: "Google CloudSQL SQL Server" setup-name: "Microsoft SQL Server" singer: true