From 6c84209b7373fa94ff9f5a7e57a1d8ed9326384b Mon Sep 17 00:00:00 2001
From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com>
Date: Thu, 4 Jan 2024 14:33:35 -0600
Subject: [PATCH] Update migrations to scale down services before migrating
(#15908)
---
terraform/addons/migrations/README.md | 7 +++--
terraform/addons/migrations/main.tf | 2 +-
terraform/addons/migrations/migrate.sh | 35 ++++++++++++++++++---
terraform/addons/migrations/variables.tf | 15 +++++++++
terraform/byo-vpc/byo-db/byo-ecs/README.md | 3 +-
terraform/byo-vpc/byo-db/byo-ecs/outputs.tf | 4 +++
6 files changed, 58 insertions(+), 8 deletions(-)
diff --git a/terraform/addons/migrations/README.md b/terraform/addons/migrations/README.md
index d7b752e020..7a83426c5a 100644
--- a/terraform/addons/migrations/README.md
+++ b/terraform/addons/migrations/README.md
@@ -13,8 +13,8 @@ No requirements.
| Name | Version |
|------|---------|
-| [aws](#provider\_aws) | 4.49.0 |
-| [null](#provider\_null) | 3.2.1 |
+| [aws](#provider\_aws) | 5.31.0 |
+| [null](#provider\_null) | 3.2.2 |
## Modules
@@ -31,7 +31,10 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
+| [desired\_count](#input\_desired\_count) | n/a | `number` | n/a | yes |
| [ecs\_cluster](#input\_ecs\_cluster) | n/a | `string` | n/a | yes |
+| [ecs\_service](#input\_ecs\_service) | n/a | `string` | n/a | yes |
+| [min\_capacity](#input\_min\_capacity) | n/a | `number` | n/a | yes |
| [security\_groups](#input\_security\_groups) | n/a | `list(string)` | n/a | yes |
| [subnets](#input\_subnets) | n/a | `list(string)` | n/a | yes |
| [task\_definition](#input\_task\_definition) | n/a | `string` | n/a | yes |
diff --git a/terraform/addons/migrations/main.tf b/terraform/addons/migrations/main.tf
index 4fd37c8b38..0c610c8aee 100644
--- a/terraform/addons/migrations/main.tf
+++ b/terraform/addons/migrations/main.tf
@@ -5,6 +5,6 @@ resource "null_resource" "main" {
task_definition_revision = var.task_definition_revision
}
provisioner "local-exec" {
- command = "/bin/bash ${path.module}/migrate.sh REGION=${data.aws_region.current.name} ECS_CLUSTER=${var.ecs_cluster} TASK_DEFINITION=${var.task_definition} TASK_DEFINITION_REVISION=${var.task_definition_revision} SUBNETS=${jsonencode(var.subnets)} SECURITY_GROUPS=${jsonencode(var.security_groups)}"
+ command = "/bin/bash ${path.module}/migrate.sh REGION=${data.aws_region.current.name} ECS_CLUSTER=${var.ecs_cluster} TASK_DEFINITION=${var.task_definition} TASK_DEFINITION_REVISION=${var.task_definition_revision} SUBNETS=${jsonencode(var.subnets)} SECURITY_GROUPS=${jsonencode(var.security_groups)} ECS_SERVICE=${var.ecs_service} MIN_CAPACITY=${var.min_capacity} DESIRED_COUNT=${var.desired_count}"
}
}
diff --git a/terraform/addons/migrations/migrate.sh b/terraform/addons/migrations/migrate.sh
index b4ba15e83e..5ea48a64a7 100644
--- a/terraform/addons/migrations/migrate.sh
+++ b/terraform/addons/migrations/migrate.sh
@@ -1,6 +1,29 @@
#!/bin/bash
set -e
+function scale_services(){
+ UP_DOWN="${1:?}"
+ # Set the minimum capacity and desired count in the cluster to 0 to scale down or to the original size to scale back to normal.
+
+ # This is a bit hacky, but the update-service has to happen first when scaling up and second when scaling down.
+ # Assume scaling down unless "up".
+ CAPACITY=0
+ if [ "${UP_DOWN:?}" = "up" ]; then
+ aws ecs update-service --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --service "${ECS_SERVICE:?}" --desired-count "${DESIRED_COUNT:?}"
+ CAPACITY="${MIN_CAPACITY:?}"
+ fi
+ aws application-autoscaling register-scalable-target --region "${REGION:?}" --service-namespace ecs --resource-id "service/${ECS_CLUSTER:?}/${ECS_SERVICE:?}" --scalable-dimension "ecs:service:DesiredCount" --min-capacity "${CAPACITY:?}"
+ # We are scaling down, make it 0
+ if [ "${UP_DOWN:?}" != "up" ]; then
+ aws ecs update-service --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --service "${ECS_SERVICE:?}" --desired-count 0
+ fi
+ # The first task defintion might never get stable because it never had initial migrations so don't wait before continuing
+ if [ "${TASK_DEFINITION_REVISION}" != "1" ]; then
+ # Wait for scale-down to succeed
+ aws ecs wait services-stable --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --service "${ECS_SERVICE:?}"
+ fi
+}
+
for ARGUMENT in "$@"
do
KEY=$(echo $ARGUMENT | cut -f1 -d=)
@@ -11,12 +34,16 @@ do
export "$KEY"="$VALUE"
done
+scale_services down
+
# Call aws ecs run-task
-TASK_ARN="$(aws ecs run-task --region "${REGION}" --cluster "${ECS_CLUSTER}" --task-definition "${TASK_DEFINITION}":"${TASK_DEFINITION_REVISION}" --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets="${SUBNETS}",securityGroups="${SECURITY_GROUPS}"}" --query 'tasks[].taskArn' --overrides '{"containerOverrides": [{"name": "fleet", "command": ["fleet", "prepare", "db"]}]}' --output text | rev | cut -d'/' -f1 | rev)"
+TASK_ARN="$(aws ecs run-task --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --task-definition "${TASK_DEFINITION:?}":"${TASK_DEFINITION_REVISION:?}" --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets="${SUBNETS:?}",securityGroups="${SECURITY_GROUPS:?}"}" --query 'tasks[].taskArn' --overrides '{"containerOverrides": [{"name": "fleet", "command": ["fleet", "prepare", "db"]}]}' --output text | rev | cut -d'/' -f1 | rev)"
# Wait for completion
-aws ecs wait tasks-stopped --cluster="${ECS_CLUSTER}" --tasks="${TASK_ARN}"
+aws ecs wait tasks-stopped --region "${REGION:?}" --cluster="${ECS_CLUSTER:?}" --tasks="${TASK_ARN:?}"
+
+scale_services up
# Exit with task's exit code
-TASK_EXIT_CODE=$(aws ecs describe-tasks --cluster $ECS_CLUSTER --tasks $TASK_ARN --query "tasks[0].containers[?name=='fleet'].exitCode" --output text)
-exit $TASK_EXIT_CODE
+TASK_EXIT_CODE=$(aws ecs describe-tasks --region "${REGION:?}" --cluster ${ECS_CLUSTER:?} --tasks ${TASK_ARN:?} --query "tasks[0].containers[?name=='fleet'].exitCode" --output text)
+exit "${TASK_EXIT_CODE}"
diff --git a/terraform/addons/migrations/variables.tf b/terraform/addons/migrations/variables.tf
index ae28c22aa5..48c62de886 100644
--- a/terraform/addons/migrations/variables.tf
+++ b/terraform/addons/migrations/variables.tf
@@ -3,6 +3,21 @@ variable "ecs_cluster" {
nullable = false
}
+variable "ecs_service" {
+ type = string
+ nullable = false
+}
+
+variable "min_capacity" {
+ type = number
+ nullable = false
+}
+
+variable "desired_count" {
+ type = number
+ nullable = false
+}
+
variable "task_definition" {
type = string
nullable = false
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/README.md b/terraform/byo-vpc/byo-db/byo-ecs/README.md
index b9475b0946..70f2c4c8a6 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/README.md
+++ b/terraform/byo-vpc/byo-db/byo-ecs/README.md
@@ -42,7 +42,7 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| [ecs\_cluster](#input\_ecs\_cluster) | The name of the ECS cluster to use | `string` | n/a | yes |
-| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.31.1")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
})
redis = object({
address = string
use_tls = optional(bool, true)
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = object({
arn = string
})
networking = object({
subnets = list(string)
security_groups = optional(list(string), null)
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
+| [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | object({
mem = optional(number, 4096)
cpu = optional(number, 512)
image = optional(string, "fleetdm/fleet:v4.42.0")
family = optional(string, "fleet")
sidecars = optional(list(any), [])
depends_on = optional(list(any), [])
mount_points = optional(list(any), [])
volumes = optional(list(any), [])
extra_environment_variables = optional(map(string), {})
extra_iam_policies = optional(list(string), [])
extra_execution_iam_policies = optional(list(string), [])
extra_secrets = optional(map(string), {})
security_groups = optional(list(string), null)
security_group_name = optional(string, "fleet")
iam_role_arn = optional(string, null)
service = optional(object({
name = optional(string, "fleet")
}), {
name = "fleet"
})
database = object({
password_secret_arn = string
user = string
database = string
address = string
rr_address = optional(string, null)
})
redis = object({
address = string
use_tls = optional(bool, true)
})
awslogs = optional(object({
name = optional(string, null)
region = optional(string, null)
create = optional(bool, true)
prefix = optional(string, "fleet")
retention = optional(number, 5)
}), {
name = null
region = null
prefix = "fleet"
retention = 5
})
loadbalancer = object({
arn = string
})
extra_load_balancers = optional(list(any), [])
networking = object({
subnets = list(string)
security_groups = optional(list(string), null)
})
autoscaling = optional(object({
max_capacity = optional(number, 5)
min_capacity = optional(number, 1)
memory_tracking_target_value = optional(number, 80)
cpu_tracking_target_value = optional(number, 80)
}), {
max_capacity = 5
min_capacity = 1
memory_tracking_target_value = 80
cpu_tracking_target_value = 80
})
iam = optional(object({
role = optional(object({
name = optional(string, "fleet-role")
policy_name = optional(string, "fleet-iam-policy")
}), {
name = "fleet-role"
policy_name = "fleet-iam-policy"
})
execution = optional(object({
name = optional(string, "fleet-execution-role")
policy_name = optional(string, "fleet-execution-role")
}), {
name = "fleet-execution-role"
policy_name = "fleet-iam-policy-execution"
})
}), {
name = "fleetdm-execution-role"
})
}) | {
"autoscaling": {
"cpu_tracking_target_value": 80,
"max_capacity": 5,
"memory_tracking_target_value": 80,
"min_capacity": 1
},
"awslogs": {
"create": true,
"name": null,
"prefix": "fleet",
"region": null,
"retention": 5
},
"cpu": 256,
"database": {
"address": null,
"database": null,
"password_secret_arn": null,
"rr_address": null,
"user": null
},
"depends_on": [],
"extra_environment_variables": {},
"extra_execution_iam_policies": [],
"extra_iam_policies": [],
"extra_load_balacners": [],
"extra_secrets": {},
"family": "fleet",
"iam": {
"execution": {
"name": "fleet-execution-role",
"policy_name": "fleet-iam-policy-execution"
},
"role": {
"name": "fleet-role",
"policy_name": "fleet-iam-policy"
}
},
"iam_role_arn": null,
"image": "fleetdm/fleet:v4.31.1",
"loadbalancer": {
"arn": null
},
"mem": 512,
"mount_points": [],
"networking": {
"security_groups": null,
"subnets": null
},
"redis": {
"address": null,
"use_tls": true
},
"security_group_name": "fleet",
"security_groups": null,
"service": {
"name": "fleet"
},
"sidecars": [],
"volumes": []
} | no |
| [migration\_config](#input\_migration\_config) | The configuration object for Fleet's migration task. | object({
mem = number
cpu = number
}) | {
"cpu": 1024,
"mem": 2048
} | no |
| [vpc\_id](#input\_vpc\_id) | n/a | `string` | `null` | no |
@@ -50,6 +50,7 @@ No modules.
| Name | Description |
|------|-------------|
+| [appautoscaling\_target](#output\_appautoscaling\_target) | n/a |
| [execution\_iam\_role\_arn](#output\_execution\_iam\_role\_arn) | n/a |
| [iam\_role\_arn](#output\_iam\_role\_arn) | n/a |
| [logging\_config](#output\_logging\_config) | n/a |
diff --git a/terraform/byo-vpc/byo-db/byo-ecs/outputs.tf b/terraform/byo-vpc/byo-db/byo-ecs/outputs.tf
index 946a04208f..f0f3de397e 100644
--- a/terraform/byo-vpc/byo-db/byo-ecs/outputs.tf
+++ b/terraform/byo-vpc/byo-db/byo-ecs/outputs.tf
@@ -2,6 +2,10 @@ output "service" {
value = aws_ecs_service.fleet
}
+output "appautoscaling_target" {
+ value = aws_appautoscaling_target.ecs_target
+}
+
output "task_definition" {
value = aws_ecs_task_definition.backend
}