mirror of
https://github.com/fleetdm/fleet
synced 2026-05-24 09:28:54 +00:00
Update migrations to scale down services before migrating (#15908)
This commit is contained in:
Robert Fairburn
GitHub
parent
87196f43c3
commit
6c84209b73
6 changed files with 58 additions and 8 deletions
|
|
@ -13,8 +13,8 @@ No requirements.
|
|||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | 4.49.0 |
|
||||
| <a name="provider_null"></a> [null](#provider\_null) | 3.2.1 |
|
||||
| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.31.0 |
|
||||
| <a name="provider_null"></a> [null](#provider\_null) | 3.2.2 |
|
||||
|
||||
## Modules
|
||||
|
||||
|
|
@ -31,7 +31,10 @@ No modules.
|
|||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|---------|:--------:|
|
||||
| <a name="input_desired_count"></a> [desired\_count](#input\_desired\_count) | n/a | `number` | n/a | yes |
|
||||
| <a name="input_ecs_cluster"></a> [ecs\_cluster](#input\_ecs\_cluster) | n/a | `string` | n/a | yes |
|
||||
| <a name="input_ecs_service"></a> [ecs\_service](#input\_ecs\_service) | n/a | `string` | n/a | yes |
|
||||
| <a name="input_min_capacity"></a> [min\_capacity](#input\_min\_capacity) | n/a | `number` | n/a | yes |
|
||||
| <a name="input_security_groups"></a> [security\_groups](#input\_security\_groups) | n/a | `list(string)` | n/a | yes |
|
||||
| <a name="input_subnets"></a> [subnets](#input\_subnets) | n/a | `list(string)` | n/a | yes |
|
||||
| <a name="input_task_definition"></a> [task\_definition](#input\_task\_definition) | n/a | `string` | n/a | yes |
|
||||
|
|
|
|||
|
|
@ -5,6 +5,6 @@ resource "null_resource" "main" {
|
|||
task_definition_revision = var.task_definition_revision
|
||||
}
|
||||
provisioner "local-exec" {
|
||||
command = "/bin/bash ${path.module}/migrate.sh REGION=${data.aws_region.current.name} ECS_CLUSTER=${var.ecs_cluster} TASK_DEFINITION=${var.task_definition} TASK_DEFINITION_REVISION=${var.task_definition_revision} SUBNETS=${jsonencode(var.subnets)} SECURITY_GROUPS=${jsonencode(var.security_groups)}"
|
||||
command = "/bin/bash ${path.module}/migrate.sh REGION=${data.aws_region.current.name} ECS_CLUSTER=${var.ecs_cluster} TASK_DEFINITION=${var.task_definition} TASK_DEFINITION_REVISION=${var.task_definition_revision} SUBNETS=${jsonencode(var.subnets)} SECURITY_GROUPS=${jsonencode(var.security_groups)} ECS_SERVICE=${var.ecs_service} MIN_CAPACITY=${var.min_capacity} DESIRED_COUNT=${var.desired_count}"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,29 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
function scale_services(){
|
||||
UP_DOWN="${1:?}"
|
||||
# Set the minimum capacity and desired count in the cluster to 0 to scale down or to the original size to scale back to normal.
|
||||
|
||||
# This is a bit hacky, but the update-service has to happen first when scaling up and second when scaling down.
|
||||
# Assume scaling down unless "up".
|
||||
CAPACITY=0
|
||||
if [ "${UP_DOWN:?}" = "up" ]; then
|
||||
aws ecs update-service --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --service "${ECS_SERVICE:?}" --desired-count "${DESIRED_COUNT:?}"
|
||||
CAPACITY="${MIN_CAPACITY:?}"
|
||||
fi
|
||||
aws application-autoscaling register-scalable-target --region "${REGION:?}" --service-namespace ecs --resource-id "service/${ECS_CLUSTER:?}/${ECS_SERVICE:?}" --scalable-dimension "ecs:service:DesiredCount" --min-capacity "${CAPACITY:?}"
|
||||
# We are scaling down, make it 0
|
||||
if [ "${UP_DOWN:?}" != "up" ]; then
|
||||
aws ecs update-service --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --service "${ECS_SERVICE:?}" --desired-count 0
|
||||
fi
|
||||
# The first task defintion might never get stable because it never had initial migrations so don't wait before continuing
|
||||
if [ "${TASK_DEFINITION_REVISION}" != "1" ]; then
|
||||
# Wait for scale-down to succeed
|
||||
aws ecs wait services-stable --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --service "${ECS_SERVICE:?}"
|
||||
fi
|
||||
}
|
||||
|
||||
for ARGUMENT in "$@"
|
||||
do
|
||||
KEY=$(echo $ARGUMENT | cut -f1 -d=)
|
||||
|
|
@ -11,12 +34,16 @@ do
|
|||
export "$KEY"="$VALUE"
|
||||
done
|
||||
|
||||
scale_services down
|
||||
|
||||
# Call aws ecs run-task
|
||||
TASK_ARN="$(aws ecs run-task --region "${REGION}" --cluster "${ECS_CLUSTER}" --task-definition "${TASK_DEFINITION}":"${TASK_DEFINITION_REVISION}" --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets="${SUBNETS}",securityGroups="${SECURITY_GROUPS}"}" --query 'tasks[].taskArn' --overrides '{"containerOverrides": [{"name": "fleet", "command": ["fleet", "prepare", "db"]}]}' --output text | rev | cut -d'/' -f1 | rev)"
|
||||
TASK_ARN="$(aws ecs run-task --region "${REGION:?}" --cluster "${ECS_CLUSTER:?}" --task-definition "${TASK_DEFINITION:?}":"${TASK_DEFINITION_REVISION:?}" --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets="${SUBNETS:?}",securityGroups="${SECURITY_GROUPS:?}"}" --query 'tasks[].taskArn' --overrides '{"containerOverrides": [{"name": "fleet", "command": ["fleet", "prepare", "db"]}]}' --output text | rev | cut -d'/' -f1 | rev)"
|
||||
|
||||
# Wait for completion
|
||||
aws ecs wait tasks-stopped --cluster="${ECS_CLUSTER}" --tasks="${TASK_ARN}"
|
||||
aws ecs wait tasks-stopped --region "${REGION:?}" --cluster="${ECS_CLUSTER:?}" --tasks="${TASK_ARN:?}"
|
||||
|
||||
scale_services up
|
||||
|
||||
# Exit with task's exit code
|
||||
TASK_EXIT_CODE=$(aws ecs describe-tasks --cluster $ECS_CLUSTER --tasks $TASK_ARN --query "tasks[0].containers[?name=='fleet'].exitCode" --output text)
|
||||
exit $TASK_EXIT_CODE
|
||||
TASK_EXIT_CODE=$(aws ecs describe-tasks --region "${REGION:?}" --cluster ${ECS_CLUSTER:?} --tasks ${TASK_ARN:?} --query "tasks[0].containers[?name=='fleet'].exitCode" --output text)
|
||||
exit "${TASK_EXIT_CODE}"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,21 @@ variable "ecs_cluster" {
|
|||
nullable = false
|
||||
}
|
||||
|
||||
variable "ecs_service" {
|
||||
type = string
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "min_capacity" {
|
||||
type = number
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "desired_count" {
|
||||
type = number
|
||||
nullable = false
|
||||
}
|
||||
|
||||
variable "task_definition" {
|
||||
type = string
|
||||
nullable = false
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ No modules.
|
|||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|---------|:--------:|
|
||||
| <a name="input_ecs_cluster"></a> [ecs\_cluster](#input\_ecs\_cluster) | The name of the ECS cluster to use | `string` | n/a | yes |
|
||||
| <a name="input_fleet_config"></a> [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | <pre>object({<br> mem = optional(number, 4096)<br> cpu = optional(number, 512)<br> image = optional(string, "fleetdm/fleet:v4.31.1")<br> family = optional(string, "fleet")<br> sidecars = optional(list(any), [])<br> depends_on = optional(list(any), [])<br> mount_points = optional(list(any), [])<br> volumes = optional(list(any), [])<br> extra_environment_variables = optional(map(string), {})<br> extra_iam_policies = optional(list(string), [])<br> extra_execution_iam_policies = optional(list(string), [])<br> extra_secrets = optional(map(string), {})<br> security_groups = optional(list(string), null)<br> security_group_name = optional(string, "fleet")<br> iam_role_arn = optional(string, null)<br> service = optional(object({<br> name = optional(string, "fleet")<br> }), {<br> name = "fleet"<br> })<br> database = object({<br> password_secret_arn = string<br> user = string<br> database = string<br> address = string<br> rr_address = optional(string, null)<br> })<br> redis = object({<br> address = string<br> use_tls = optional(bool, true)<br> })<br> awslogs = optional(object({<br> name = optional(string, null)<br> region = optional(string, null)<br> create = optional(bool, true)<br> prefix = optional(string, "fleet")<br> retention = optional(number, 5)<br> }), {<br> name = null<br> region = null<br> prefix = "fleet"<br> retention = 5<br> })<br> loadbalancer = object({<br> arn = string<br> })<br> networking = object({<br> subnets = list(string)<br> security_groups = optional(list(string), null)<br> })<br> autoscaling = optional(object({<br> max_capacity = optional(number, 5)<br> min_capacity = optional(number, 1)<br> memory_tracking_target_value = optional(number, 80)<br> cpu_tracking_target_value = optional(number, 80)<br> }), {<br> max_capacity = 5<br> min_capacity = 1<br> memory_tracking_target_value = 80<br> cpu_tracking_target_value = 80<br> })<br> iam = optional(object({<br> role = optional(object({<br> name = optional(string, "fleet-role")<br> policy_name = optional(string, "fleet-iam-policy")<br> }), {<br> name = "fleet-role"<br> policy_name = "fleet-iam-policy"<br> })<br> execution = optional(object({<br> name = optional(string, "fleet-execution-role")<br> policy_name = optional(string, "fleet-execution-role")<br> }), {<br> name = "fleet-execution-role"<br> policy_name = "fleet-iam-policy-execution"<br> })<br> }), {<br> name = "fleetdm-execution-role"<br> })<br> })</pre> | <pre>{<br> "autoscaling": {<br> "cpu_tracking_target_value": 80,<br> "max_capacity": 5,<br> "memory_tracking_target_value": 80,<br> "min_capacity": 1<br> },<br> "awslogs": {<br> "create": true,<br> "name": null,<br> "prefix": "fleet",<br> "region": null,<br> "retention": 5<br> },<br> "cpu": 256,<br> "database": {<br> "address": null,<br> "database": null,<br> "password_secret_arn": null,<br> "rr_address": null,<br> "user": null<br> },<br> "depends_on": [],<br> "extra_environment_variables": {},<br> "extra_execution_iam_policies": [],<br> "extra_iam_policies": [],<br> "extra_secrets": {},<br> "family": "fleet",<br> "iam": {<br> "execution": {<br> "name": "fleet-execution-role",<br> "policy_name": "fleet-iam-policy-execution"<br> },<br> "role": {<br> "name": "fleet-role",<br> "policy_name": "fleet-iam-policy"<br> }<br> },<br> "iam_role_arn": null,<br> "image": "fleetdm/fleet:v4.31.1",<br> "loadbalancer": {<br> "arn": null<br> },<br> "mem": 512,<br> "mount_points": [],<br> "networking": {<br> "security_groups": null,<br> "subnets": null<br> },<br> "redis": {<br> "address": null,<br> "use_tls": true<br> },<br> "security_group_name": "fleet",<br> "security_groups": null,<br> "service": {<br> "name": "fleet"<br> },<br> "sidecars": [],<br> "volumes": []<br>}</pre> | no |
|
||||
| <a name="input_fleet_config"></a> [fleet\_config](#input\_fleet\_config) | The configuration object for Fleet itself. Fields that default to null will have their respective resources created if not specified. | <pre>object({<br> mem = optional(number, 4096)<br> cpu = optional(number, 512)<br> image = optional(string, "fleetdm/fleet:v4.42.0")<br> family = optional(string, "fleet")<br> sidecars = optional(list(any), [])<br> depends_on = optional(list(any), [])<br> mount_points = optional(list(any), [])<br> volumes = optional(list(any), [])<br> extra_environment_variables = optional(map(string), {})<br> extra_iam_policies = optional(list(string), [])<br> extra_execution_iam_policies = optional(list(string), [])<br> extra_secrets = optional(map(string), {})<br> security_groups = optional(list(string), null)<br> security_group_name = optional(string, "fleet")<br> iam_role_arn = optional(string, null)<br> service = optional(object({<br> name = optional(string, "fleet")<br> }), {<br> name = "fleet"<br> })<br> database = object({<br> password_secret_arn = string<br> user = string<br> database = string<br> address = string<br> rr_address = optional(string, null)<br> })<br> redis = object({<br> address = string<br> use_tls = optional(bool, true)<br> })<br> awslogs = optional(object({<br> name = optional(string, null)<br> region = optional(string, null)<br> create = optional(bool, true)<br> prefix = optional(string, "fleet")<br> retention = optional(number, 5)<br> }), {<br> name = null<br> region = null<br> prefix = "fleet"<br> retention = 5<br> })<br> loadbalancer = object({<br> arn = string<br> })<br> extra_load_balancers = optional(list(any), [])<br> networking = object({<br> subnets = list(string)<br> security_groups = optional(list(string), null)<br> })<br> autoscaling = optional(object({<br> max_capacity = optional(number, 5)<br> min_capacity = optional(number, 1)<br> memory_tracking_target_value = optional(number, 80)<br> cpu_tracking_target_value = optional(number, 80)<br> }), {<br> max_capacity = 5<br> min_capacity = 1<br> memory_tracking_target_value = 80<br> cpu_tracking_target_value = 80<br> })<br> iam = optional(object({<br> role = optional(object({<br> name = optional(string, "fleet-role")<br> policy_name = optional(string, "fleet-iam-policy")<br> }), {<br> name = "fleet-role"<br> policy_name = "fleet-iam-policy"<br> })<br> execution = optional(object({<br> name = optional(string, "fleet-execution-role")<br> policy_name = optional(string, "fleet-execution-role")<br> }), {<br> name = "fleet-execution-role"<br> policy_name = "fleet-iam-policy-execution"<br> })<br> }), {<br> name = "fleetdm-execution-role"<br> })<br> })</pre> | <pre>{<br> "autoscaling": {<br> "cpu_tracking_target_value": 80,<br> "max_capacity": 5,<br> "memory_tracking_target_value": 80,<br> "min_capacity": 1<br> },<br> "awslogs": {<br> "create": true,<br> "name": null,<br> "prefix": "fleet",<br> "region": null,<br> "retention": 5<br> },<br> "cpu": 256,<br> "database": {<br> "address": null,<br> "database": null,<br> "password_secret_arn": null,<br> "rr_address": null,<br> "user": null<br> },<br> "depends_on": [],<br> "extra_environment_variables": {},<br> "extra_execution_iam_policies": [],<br> "extra_iam_policies": [],<br> "extra_load_balacners": [],<br> "extra_secrets": {},<br> "family": "fleet",<br> "iam": {<br> "execution": {<br> "name": "fleet-execution-role",<br> "policy_name": "fleet-iam-policy-execution"<br> },<br> "role": {<br> "name": "fleet-role",<br> "policy_name": "fleet-iam-policy"<br> }<br> },<br> "iam_role_arn": null,<br> "image": "fleetdm/fleet:v4.31.1",<br> "loadbalancer": {<br> "arn": null<br> },<br> "mem": 512,<br> "mount_points": [],<br> "networking": {<br> "security_groups": null,<br> "subnets": null<br> },<br> "redis": {<br> "address": null,<br> "use_tls": true<br> },<br> "security_group_name": "fleet",<br> "security_groups": null,<br> "service": {<br> "name": "fleet"<br> },<br> "sidecars": [],<br> "volumes": []<br>}</pre> | no |
|
||||
| <a name="input_migration_config"></a> [migration\_config](#input\_migration\_config) | The configuration object for Fleet's migration task. | <pre>object({<br> mem = number<br> cpu = number<br> })</pre> | <pre>{<br> "cpu": 1024,<br> "mem": 2048<br>}</pre> | no |
|
||||
| <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | n/a | `string` | `null` | no |
|
||||
|
||||
|
|
@ -50,6 +50,7 @@ No modules.
|
|||
|
||||
| Name | Description |
|
||||
|------|-------------|
|
||||
| <a name="output_appautoscaling_target"></a> [appautoscaling\_target](#output\_appautoscaling\_target) | n/a |
|
||||
| <a name="output_execution_iam_role_arn"></a> [execution\_iam\_role\_arn](#output\_execution\_iam\_role\_arn) | n/a |
|
||||
| <a name="output_iam_role_arn"></a> [iam\_role\_arn](#output\_iam\_role\_arn) | n/a |
|
||||
| <a name="output_logging_config"></a> [logging\_config](#output\_logging\_config) | n/a |
|
||||
|
|
|
|||
|
|
@ -2,6 +2,10 @@ output "service" {
|
|||
value = aws_ecs_service.fleet
|
||||
}
|
||||
|
||||
output "appautoscaling_target" {
|
||||
value = aws_appautoscaling_target.ecs_target
|
||||
}
|
||||
|
||||
output "task_definition" {
|
||||
value = aws_ecs_task_definition.backend
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue