feat: Create SQL Backup and export module (#296)

* Create SQL Backup and export module

* Findings from codereview by @bharathkkb

* Add tests for CloudSQL backup module

* Add wait time to allow API activation

* Increase API Wait time to 7.5 minutes

* Reduce outputs for examples and add region as output for backup module

* Use default docker image for waiting

* Update gcloud go package and use gcloud.Runf

Author: gleichda

build/int.cloudbuild.yaml

@@ -21,15 +21,23 @@ steps:
       - 'TF_VAR_org_id=$_ORG_ID'
       - 'TF_VAR_folder_id=$_FOLDER_ID'
       - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
+
+  # Sometimes API activation can be bit flaky therefor ensure that at least 3 mins (180s) are passed between
+  # prepare_environment and the first terraform apply (init-all runs parallel to reduce waste of time)
+  - id: wait for api activation
+    waitFor:
+      - prepare
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['sleep', '$_API_ACTIVATION_SECONDS_WAIT']
+
   - id: init-all
     waitFor:
       - prepare
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run all --stage init --verbose']
 
   - id: apply mssql-ha-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestMsSqlHaModule --stage apply --verbose']
   - id: verify mssql-ha-local
@@ -44,8 +52,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestMsSqlHaModule --stage teardown --verbose']
 
   - id: apply mssql-public-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestMsSqlPublicModule --stage apply --verbose']
   - id: verify mssql-public-local
@@ -60,8 +67,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestMsSqlPublicModule --stage teardown --verbose']
 
   - id: apply mysql-ha-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestMySqlHaModule --stage apply --verbose']
   - id: verify mysql-ha-local
@@ -76,8 +82,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestMySqlHaModule --stage teardown --verbose']
 
   - id: apply mysql-private-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestMySqlPrivateModule --stage apply --verbose']
   - id: verify mysql-private-local
@@ -92,8 +97,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestMySqlPrivateModule --stage teardown --verbose']
 
   - id: apply mysql-public-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestMySqlPublicModule --stage apply --verbose']
   - id: verify mysql-public-local
@@ -108,8 +112,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestMySqlPublicModule --stage teardown --verbose']
 
   - id: apply postgresql-ha-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestPostgreSqlHaModule --stage apply --verbose']
   - id: verify postgresql-ha-local
@@ -124,8 +127,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestPostgreSqlHaModule --stage teardown --verbose']
 
   - id: apply postgresql-public-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestPostgreSqlPublicModule --stage apply --verbose']
   - id: verify postgresql-public-local
@@ -140,8 +142,7 @@ steps:
     args: ['/bin/bash', '-c', 'cft test run TestPostgreSqlPublicModule --stage teardown --verbose']
 
   - id: apply postgresql-public-iam-local
-    waitFor:
-      - init-all
+    waitFor: ["init-all", "wait for api activation"]
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestPostgreSqlPublicIamModule --stage apply --verbose']
   - id: verify postgresql-public-iam-local
@@ -155,9 +156,40 @@ steps:
     name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
     args: ['/bin/bash', '-c', 'cft test run TestPostgreSqlPublicIamModule --stage teardown --verbose']
 
+  - id: apply mysql-backup-local
+    waitFor: ["init-all", "wait for api activation"]
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['/bin/bash', '-c', 'cft test run TestMySqlBackupModuleCreateServiceAccount --stage apply --verbose']
+  - id: verify mysql-backup-local
+    waitFor:
+      - apply mysql-backup-local
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['/bin/bash', '-c', 'cft test run TestMySqlBackupModuleCreateServiceAccount --stage verify --verbose']
+  - id: teardown mysql-backup-local
+    waitFor:
+      - verify mysql-backup-local
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['/bin/bash', '-c', 'cft test run TestMySqlBackupModuleCreateServiceAccount --stage teardown --verbose']
+
+  - id: apply postgresql-backup-local
+    waitFor: ["init-all", "wait for api activation"]
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['/bin/bash', '-c', 'cft test run TestPostgresqlBackupModuleProvidedServiceAccount --stage apply --verbose']
+  - id: verify postgresql-backup-local
+    waitFor:
+      - apply postgresql-backup-local
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['/bin/bash', '-c', 'cft test run TestPostgresqlBackupModuleProvidedServiceAccount --stage verify --verbose']
+  - id: teardown postgresql-backup-local
+    waitFor:
+      - verify postgresql-backup-local
+    name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+    args: ['/bin/bash', '-c', 'cft test run TestPostgresqlBackupModuleProvidedServiceAccount --stage teardown --verbose']
+
 tags:
   - 'ci'
   - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
   _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.3'
+  _API_ACTIVATION_SECONDS_WAIT: '450'

examples/mysql-backup-create-service-account/README.md

@@ -0,0 +1,43 @@
+# Cloud SQL Database Backup Example
+
+This example shows how to create:
+
+- a MySQL CloudSQL Instance
+- A GCS Bucket for storing the Backup
+- The Workflows for exports (external backups) and (internal) backups
+
+## Run Terraform
+
+Create resources with terraform:
+
+```bash
+terraform init
+terraform plan
+terraform apply
+```
+
+To remove all resources created by terraform:
+
+```bash
+terraform destroy
+```
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_authorized_networks"></a> [authorized\_networks](#input\_authorized\_networks) | List of mapped public networks authorized to access to the instances. Default - short range of GCP health-checkers IPs | `list(map(string))` | <pre>[<br>  {<br>    "name": "sample-gcp-health-checkers-range",<br>    "value": "130.211.0.0/28"<br>  }<br>]</pre> | no |
+| <a name="input_db_name"></a> [db\_name](#input\_db\_name) | The name of the SQL Database instance | `string` | `"example-mysql-public"` | no |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The ID of the project in which resources will be provisioned. | `string` | n/a | yes |
+| <a name="input_region"></a> [region](#input\_region) | The region of the Cloud SQL resources | `string` | `"us-central1"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_backup_workflow_name"></a> [backup\_workflow\_name](#output\_backup\_workflow\_name) | The name for internal backup workflow |
+| <a name="output_export_workflow_name"></a> [export\_workflow\_name](#output\_export\_workflow\_name) | The name for export workflow |
+| <a name="output_instance_name"></a> [instance\_name](#output\_instance\_name) | The name of the SQL instance |
+| <a name="output_project_id"></a> [project\_id](#output\_project\_id) | The project ID used |
+| <a name="output_service_account"></a> [service\_account](#output\_service\_account) | The service account email running the scheduler and workflow |
+| <a name="output_workflow_location"></a> [workflow\_location](#output\_workflow\_location) | The location where the workflows run |

examples/mysql-backup-create-service-account/main.tf

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "mysql" {
+  source               = "../../modules/mysql"
+  name                 = "example-mysql-public"
+  database_version     = "MYSQL_8_0"
+  random_instance_name = true
+  project_id           = var.project_id
+  zone                 = "us-central1-a"
+  region               = "us-central1"
+  deletion_protection  = false
+
+  ip_configuration = {
+    ipv4_enabled        = true
+    private_network     = null
+    require_ssl         = true
+    allocated_ip_range  = null
+    authorized_networks = []
+  }
+}
+
+resource "google_storage_bucket" "backup" {
+  name     = "${module.mysql.instance_name}-backup"
+  location = "us-central1"
+  # TODO: don't use force_destroy for production this is just required for testing
+  force_destroy = true
+  project       = var.project_id
+}
+
+module "backup" {
+  source                = "../../modules/backup"
+  region                = "us-central1"
+  project_id            = var.project_id
+  sql_instance          = module.mysql.instance_name
+  export_databases      = []
+  export_uri            = google_storage_bucket.backup.url
+  backup_retention_time = 1
+  backup_schedule       = "5 * * * *"
+  export_schedule       = "10 * * * *"
+}

examples/mysql-backup-create-service-account/outputs.tf

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "backup_workflow_name" {
+  value       = module.backup.backup_workflow_name
+  description = "The name for internal backup workflow"
+}
+
+output "export_workflow_name" {
+  value       = module.backup.export_workflow_name
+  description = "The name for export workflow"
+}
+
+output "project_id" {
+  value       = var.project_id
+  description = "The project ID used"
+}
+
+output "service_account" {
+  value       = module.backup.service_account
+  description = "The service account email running the scheduler and workflow"
+}
+
+output "workflow_location" {
+  value       = module.backup.region
+  description = "The location where the workflows run"
+}
+
+output "instance_name" {
+  value       = module.mysql.instance_name
+  description = "The name of the SQL instance"
+}

examples/mysql-backup-create-service-account/variables.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The ID of the project in which resources will be provisioned."
+  type        = string
+}

examples/mysql-backup-create-service-account/versions.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_version = ">= 0.13"
+}
+

examples/postgresql-backup-provided-service-account/README.md

@@ -0,0 +1,43 @@
+# Cloud SQL Database Backup Example
+
+This example shows how to create:
+
+- a PostgreSQL Cloud SQL Instance
+- A GCS Bucket for storing the Backup
+- The Workflows for exports (external backups) and (internal) backups
+
+## Run Terraform
+
+Create resources with terraform:
+
+```bash
+terraform init
+terraform plan
+terraform apply
+```
+
+To remove all resources created by terraform:
+
+```bash
+terraform destroy
+```
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_authorized_networks"></a> [authorized\_networks](#input\_authorized\_networks) | List of mapped public networks authorized to access to the instances. Default - short range of GCP health-checkers IPs | `list(map(string))` | <pre>[<br>  {<br>    "name": "sample-gcp-health-checkers-range",<br>    "value": "130.211.0.0/28"<br>  }<br>]</pre> | no |
+| <a name="input_db_name"></a> [db\_name](#input\_db\_name) | The name of the SQL Database instance | `string` | `"example-mysql-public"` | no |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The ID of the project in which resources will be provisioned. | `string` | n/a | yes |
+| <a name="input_region"></a> [region](#input\_region) | The region of the Cloud SQL resources | `string` | `"us-central1"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_backup_workflow_name"></a> [backup\_workflow\_name](#output\_backup\_workflow\_name) | The name for internal backup workflow |
+| <a name="output_export_workflow_name"></a> [export\_workflow\_name](#output\_export\_workflow\_name) | The name for export workflow |
+| <a name="output_instance_name"></a> [instance\_name](#output\_instance\_name) | The name of the SQL instance |
+| <a name="output_project_id"></a> [project\_id](#output\_project\_id) | The project ID used |
+| <a name="output_service_account"></a> [service\_account](#output\_service\_account) | The service account email running the scheduler and workflow |
+| <a name="output_workflow_location"></a> [workflow\_location](#output\_workflow\_location) | The location where the workflows run |