From e1ccf76cec9ea447aeedf4ffc0140fc73f91a39e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 12 Nov 2026 07:18:00 +0000 Subject: [PATCH] ecs_cluster: roll the hosts when their launch template changes The ECS hosts never got a new AMI or launch template settings: the group used the template's "$Latest", so a new template version didn't change the group and nothing started an instance refresh. A refresh started by hand skipped every host running tasks, since managed termination protection protects them from scale-in and the refresh ignored protected hosts by default. The dev hosts still run their April 2026 AMI. Point the group at the template's latest_version, so a new template starts the refresh, and refresh the protected hosts too, launching each new host before the old one goes (min 100%, max 200% healthy) and skipping hosts already on the template. managed_draining, the default for these capacity providers, is set explicitly: ECS moves a host's tasks and drains them from the load balancer before it terminates. The first apply changes the group's template version and so replaces every host, picking up the current ECS AMI and the template's metadata options. --- tf/modules/ecs_cluster/main.tf | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tf/modules/ecs_cluster/main.tf b/tf/modules/ecs_cluster/main.tf index 48b6ff02..3a6676fc 100644 --- a/tf/modules/ecs_cluster/main.tf +++ b/tf/modules/ecs_cluster/main.tf @@ -207,15 +207,29 @@ resource "aws_autoscaling_group" "container_host" { # defined below. Note that this is an ECS cluster, so # cluster capacity is directed by task load demands + # The template's version, not "$Latest": a new template (e.g. a new ECS AMI, + # which the template picks up on every apply) then changes the group, and + # that starts the instance refresh below. launch_template { id = aws_launch_template.container_host.id - version = "$Latest" + version = aws_launch_template.container_host.latest_version } + # Replace the hosts one at a time when the template changes: start a new + # host, let ECS drain the old one's tasks onto it (managed_draining below), + # then terminate the old host. instance_refresh { strategy = "Rolling" preferences { - min_healthy_percentage = 50 + # Launch the new host before terminating the old one, so capacity never + # drops below what's running + min_healthy_percentage = 100 + max_healthy_percentage = 200 + # Hosts running tasks are protected from scale-in (managed termination + # protection); without this the refresh skips them, i.e. all of them + scale_in_protected_instances = "Refresh" + # Leave hosts already on the new template alone + skip_matching = true } triggers = ["tag"] @@ -236,7 +250,10 @@ resource "aws_ecs_capacity_provider" "capacity_provider" { auto_scaling_group_provider { auto_scaling_group_arn = aws_autoscaling_group.container_host.arn managed_termination_protection = "ENABLED" - # managed_draining = "ENABLED" + # Before a host terminates (e.g. in an instance refresh), ECS starts its + # tasks on other hosts and drains them from the load balancer. The + # default for capacity providers created since 2024; set to say so. + managed_draining = "ENABLED" managed_scaling { maximum_scaling_step_size = 1000