From 12eac152c4942c9a8d40b8a761914a147ea70e95 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Fri, 18 Feb 2022 20:01:42 -0500 Subject: [PATCH] GCP Terraform (#4303) * gcp wip * more edits on services, secrets manager, readme * updated readme with required variables --- tools/terraform/gcp/artifact_registry.tf | 7 ++ tools/terraform/gcp/cloud_run.tf | 128 +++++++++++++++++++++++ tools/terraform/gcp/loadbalancer.tf | 55 ++++++++++ tools/terraform/gcp/main.tf | 22 ++++ tools/terraform/gcp/mysql.tf | 47 +++++++++ tools/terraform/gcp/readme.md | 54 ++++++++++ tools/terraform/gcp/redis.tf | 9 ++ tools/terraform/gcp/services.tf | 10 ++ tools/terraform/gcp/variables.tf | 72 +++++++++++++ tools/terraform/gcp/vpc.tf | 45 ++++++++ 10 files changed, 449 insertions(+) create mode 100644 tools/terraform/gcp/artifact_registry.tf create mode 100644 tools/terraform/gcp/cloud_run.tf create mode 100644 tools/terraform/gcp/loadbalancer.tf create mode 100644 tools/terraform/gcp/main.tf create mode 100644 tools/terraform/gcp/mysql.tf create mode 100644 tools/terraform/gcp/readme.md create mode 100644 tools/terraform/gcp/redis.tf create mode 100644 tools/terraform/gcp/services.tf create mode 100644 tools/terraform/gcp/variables.tf create mode 100644 tools/terraform/gcp/vpc.tf diff --git a/tools/terraform/gcp/artifact_registry.tf b/tools/terraform/gcp/artifact_registry.tf new file mode 100644 index 0000000000..7e87e4c4dd --- /dev/null +++ b/tools/terraform/gcp/artifact_registry.tf @@ -0,0 +1,7 @@ +resource "google_artifact_registry_repository" "my-repo" { + provider = google-beta + location = var.region + repository_id = "${var.prefix}-repository" + description = "repository to hold fleet container images for cloud run" + format = "DOCKER" +} \ No newline at end of file diff --git a/tools/terraform/gcp/cloud_run.tf b/tools/terraform/gcp/cloud_run.tf new file mode 100644 index 0000000000..cdda0abbb2 --- /dev/null +++ b/tools/terraform/gcp/cloud_run.tf @@ -0,0 +1,128 @@ +resource "google_compute_region_network_endpoint_group" "neg" { + name = "${var.prefix}-neg" + region = var.region + network_endpoint_type = "SERVERLESS" + cloud_run { + service = google_cloud_run_service.default.name + } +} + +data "google_iam_policy" "noauth" { + binding { + role = "roles/run.invoker" + members = [ + "allUsers", + ] + } +} + +resource "google_cloud_run_service_iam_policy" "noauth" { + location = google_cloud_run_service.default.location + project = google_cloud_run_service.default.project + service = google_cloud_run_service.default.name + + policy_data = data.google_iam_policy.noauth.policy_data +} + +resource "random_pet" "suffix" { + length = 1 +} + +resource "google_secret_manager_secret" "secret" { + secret_id = "fleet-db-password-${random_pet.suffix.id}" + replication { + automatic = true + } +} + +resource "google_secret_manager_secret_version" "secret-version-data" { + secret = google_secret_manager_secret.secret.name + secret_data = module.fleet-mysql.generated_user_password +} + +data "google_compute_default_service_account" "default" {} + +resource "google_secret_manager_secret_iam_member" "secret-access" { + secret_id = google_secret_manager_secret.secret.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${data.google_compute_default_service_account.default.email}" + depends_on = [google_secret_manager_secret.secret] +} + +resource "google_cloud_run_service" "default" { + name = "${var.prefix}-backend" + location = var.region + metadata { + annotations = { + "run.googleapis.com/ingress" = "internal-and-cloud-load-balancing" + "run.googleapis.com/ingress-status" = "internal-and-cloud-load-balancing" + } + } + + + template { + spec { + containers { + resources { + limits = { + cpu = var.fleet_cpu + memory = var.fleet_memory + } + } + image = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.my-repo.name}/${var.image}" + ports { + name = "http1" + container_port = 8080 + } + env { + name = "FLEET_MYSQL_USERNAME" + value = var.db_user + } + env { + name = "FLEET_MYSQL_DATABASE" + value = var.db_name + } + env { + name = "FLEET_SERVER_TLS" + value = false + } + env { + name = "FLEET_MYSQL_ADDRESS" + value = module.fleet-mysql.private_ip_address + } + env { + name = "FLEET_REDIS_ADDRESS" + value = "${google_redis_instance.cache.host}:${google_redis_instance.cache.port}" + } + env { + name = "FLEET_MYSQL_PASSWORD" + value_from { + secret_key_ref { + name = google_secret_manager_secret.secret.secret_id + key = "latest" + } + } + } + command = ["/bin/sh"] + args = [ + "-c", + "fleet prepare --no-prompt=true db; exec fleet serve" + ] + } + } + + metadata { + annotations = { + "autoscaling.knative.dev/minScale" = "1" + "autoscaling.knative.dev/maxScale" = "1000" + "run.googleapis.com/cloudsql-instances" = module.fleet-mysql.instance_connection_name + "run.googleapis.com/vpc-access-connector" = tolist(module.serverless-connector.connector_ids)[0] + "run.googleapis.com/vpc-access-egress" = "all-traffic" + "run.googleapis.com/client-name" = "terraform" + "run.googleapis.com/cpu-throttling" = "false" + + } + } + } + autogenerate_revision_name = true +} diff --git a/tools/terraform/gcp/loadbalancer.tf b/tools/terraform/gcp/loadbalancer.tf new file mode 100644 index 0000000000..0ed4ef1931 --- /dev/null +++ b/tools/terraform/gcp/loadbalancer.tf @@ -0,0 +1,55 @@ +resource "google_dns_managed_zone" "default" { + dns_name = var.dns_name + name = "${var.prefix}-zone" +} + +resource "google_dns_record_set" "default" { + managed_zone = google_dns_managed_zone.default.name + name = var.dns_name + type = "A" + ttl = "300" + rrdatas = [module.lb-http.external_ip] + depends_on = [module.lb-http] +} + +module "lb-http" { + source = "GoogleCloudPlatform/lb-http/google//modules/serverless_negs" + version = "~> 6.2.0" + + project = var.project_id + name = "${var.prefix}-load-balancer" + + managed_ssl_certificate_domains = [trim(var.dns_name, ".")] + ssl = true + https_redirect = true + + backends = { + default = { + # List your serverless NEGs, VMs, or buckets as backends + groups = [ + { + group = google_compute_region_network_endpoint_group.neg.id + } + ] + custom_request_headers = null + custom_response_headers = null + + enable_cdn = false + + log_config = { + enable = true + sample_rate = 1.0 + } + + iap_config = { + enable = false + oauth2_client_id = null + oauth2_client_secret = null + } + + description = null + custom_request_headers = null + security_policy = null + } + } +} \ No newline at end of file diff --git a/tools/terraform/gcp/main.tf b/tools/terraform/gcp/main.tf new file mode 100644 index 0000000000..9788263789 --- /dev/null +++ b/tools/terraform/gcp/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "4.9.0" + } + google-beta = { + source = "hashicorp/google-beta" + } + } +} + +provider "google" { + # Configuration options + project = var.project_id + region = var.region +} + +provider "google-beta" { + project = var.project_id + region = var.region +} \ No newline at end of file diff --git a/tools/terraform/gcp/mysql.tf b/tools/terraform/gcp/mysql.tf new file mode 100644 index 0000000000..7d31a4c06e --- /dev/null +++ b/tools/terraform/gcp/mysql.tf @@ -0,0 +1,47 @@ +resource "random_password" "fleet-db-user-pw" { + length = 12 +} + +module "fleet-mysql" { + source = "GoogleCloudPlatform/sql-db/google//modules/mysql" + version = "9.0.0" + name = "${var.prefix}-mysql" + random_instance_name = true + project_id = var.project_id + + deletion_protection = false + + additional_users = [ + { + name = var.db_user + password = random_password.fleet-db-user-pw.result + host = "% (any host)" + type = "BUILT_IN" + } + ] + + ip_configuration = { + ipv4_enabled = false + # We never set authorized networks, we need all connections via the + # public IP to be mediated by Cloud SQL. + authorized_networks = [] + require_ssl = false + private_network = module.vpc.network_self_link + } + + database_version = var.db_version + region = var.region + zone = var.db_zone + tier = var.db_tier + additional_databases = [ + { + name = var.db_name + charset = "utf8mb4" + collation = "utf8mb4_general_ci" + } + ] + + + // Optional: used to enforce ordering in the creation of resources. + module_depends_on = [module.private-service-access.peering_completed] +} \ No newline at end of file diff --git a/tools/terraform/gcp/readme.md b/tools/terraform/gcp/readme.md new file mode 100644 index 0000000000..105c59be29 --- /dev/null +++ b/tools/terraform/gcp/readme.md @@ -0,0 +1,54 @@ +## Fleet on GCP + +Required Variables: +```terraform +project_id = "" +prefix = "fleet" +dns_name = "" // eg. myfleet.fleetdm.com. +``` + +### Overview + +#### Fleet server +The fleet webserver is running as [Google Cloud Run](https://cloud.google.com/run) containers, this is very similar to how the existing terraform for AWS runs fleet as Fargate compute. +_NOTE: Cloud Run has [limitations](https://cloud.google.com/run/docs/deploying#images) on what container images it will run_. In our deployment we create +and Artifact Registry and deploy the public fleet container image into Artifact Registry. + +#### MySQL +We are running MySQL using [Google Cloud SQL](https://cloud.google.com/sql/docs/mysql/introduction) only reachable via [CloudSQLProxy](https://cloud.google.com/sql/docs/mysql/connect-admin-proxy) and from Cloud Run +using [Serverless VPC Access Connector](https://cloud.google.com/sql/docs/mysql/connect-run#private-ip). + +#### Redis +We are running Redis using [Google Cloud Memorystore (Redis engine)](https://cloud.google.com/memorystore). This can run in cluster mode, but by default we +are running in standalone mode. + +### Pushing the Fleet image into Google Artifact registry + +More details can be found [here](https://cloud.google.com/artifact-registry/docs/docker/pushing-and-pulling). + +Login with gcloud helper: + +```shell +gcloud auth configure-docker \ + us-central1-docker.pkg.dev +``` + +Pull latest image: + +`docker pull ` for example `docker pull fleetdm/fleet:v4.10.0` + +Tag it: + +``` +docker tag fleetdm/fleet:v10.0.0 us-central1-docker.pkg.dev//fleet-repository/fleet:v10.0.0 +``` + +Push to Google Artifact registry: + +`docker push us-central1-docker.pkg.dev//fleet-repository/fleet:v4.9.1` + +### GCP Managed Certificates + +In this example we are using [GCP Managed Certificates](https://cloud.google.com/load-balancing/docs/ssl-certificates/google-managed-certs) to handle TLS and TLS termination at the LoadBalancer. +In order for the certificate to be properly issued, you'll need to update your domain registrar with the nameserver values generated +by the new Zone created in GCP DNS. \ No newline at end of file diff --git a/tools/terraform/gcp/redis.tf b/tools/terraform/gcp/redis.tf new file mode 100644 index 0000000000..2e94fafa75 --- /dev/null +++ b/tools/terraform/gcp/redis.tf @@ -0,0 +1,9 @@ +resource "google_redis_instance" "cache" { + name = "${var.prefix}-redis" + tier = "STANDARD_HA" + memory_size_gb = var.redis_mem + authorized_network = module.vpc.network_name + connect_mode = "PRIVATE_SERVICE_ACCESS" + display_name = "${var.prefix}-redis" + depends_on = [module.private-service-access.peering_completed] +} \ No newline at end of file diff --git a/tools/terraform/gcp/services.tf b/tools/terraform/gcp/services.tf new file mode 100644 index 0000000000..92e942977f --- /dev/null +++ b/tools/terraform/gcp/services.tf @@ -0,0 +1,10 @@ +resource "google_project_service" "vpcaccess-api" { + project = var.project_id # Replace this with your project ID in quotes + service = "vpcaccess.googleapis.com" +} + +resource "google_project_service" "secretmanager" { + provider = google-beta + project = var.project_id + service = "secretmanager.googleapis.com" +} \ No newline at end of file diff --git a/tools/terraform/gcp/variables.tf b/tools/terraform/gcp/variables.tf new file mode 100644 index 0000000000..752747eff1 --- /dev/null +++ b/tools/terraform/gcp/variables.tf @@ -0,0 +1,72 @@ +variable "region" { + description = "gcp region" + default = "us-central1" +} + +variable "db_zone" { + default = "us-central1-c" +} + +variable "db_user" { + default = "default" +} + +variable "db_name" { + default = "fleet" +} + +variable "db_tier" { + default = "db-n1-standard-1" +} + +variable "db_version" { + default = "MYSQL_5_7" +} + +variable "fleet_cpu" { + default = "1000m" +} + +variable "fleet_memory" { + default = "1024Mi" +} + +variable "dns_zone" { + default = "" +} + +variable "dns_name" { + default = "" +} + +variable "serverless_connector_min_instances" { + default = 2 +} +variable "serverless_connector_max_instances" { + default = 3 +} + +variable "serverless_connector_instance_type" { + default = "f1-micro" +} + +variable "vpc_subnet" { + default = "10.10.10.0/28" +} + +variable "project_id" { + description = "gcp project id" +} + +variable "prefix" { + default = "fleet-" + description = "prefix resources with this string" +} + +variable "redis_mem" { + default = 1 +} + +variable "image" { + default = "fleet:v4.10.0" +} \ No newline at end of file diff --git a/tools/terraform/gcp/vpc.tf b/tools/terraform/gcp/vpc.tf new file mode 100644 index 0000000000..25dfb4ab86 --- /dev/null +++ b/tools/terraform/gcp/vpc.tf @@ -0,0 +1,45 @@ +locals { + subnet_name = "${var.prefix}-serverless-subnet" + vpc_connector_name = "${var.prefix}-serverless-con" +} + +module "vpc" { + source = "terraform-google-modules/network/google" + version = "~> 4.1.0" + project_id = var.project_id + network_name = "${var.prefix}network" + mtu = 1460 + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = var.vpc_subnet + subnet_region = var.region + } + ] +} + +module "serverless-connector" { + source = "terraform-google-modules/network/google//modules/vpc-serverless-connector-beta" + project_id = var.project_id + vpc_connectors = [{ + name = local.vpc_connector_name + region = var.region + subnet_name = module.vpc.subnets["${var.region}/${local.subnet_name}"].name + machine_type = var.serverless_connector_instance_type + min_instances = var.serverless_connector_min_instances + max_instances = var.serverless_connector_max_instances + } + ] + depends_on = [ + google_project_service.vpcaccess-api + ] +} + +module "private-service-access" { + source = "GoogleCloudPlatform/sql-db/google//modules/private_service_access" + version = "9.0.0" + project_id = var.project_id + vpc_network = module.vpc.network_name + depends_on = [module.vpc] +} \ No newline at end of file