ECS Fargate task stopped unexpectedly: diagnosing container exits

# Check stopped task reason
aws ecs describe-tasks \
  --cluster my-cluster \
  --tasks $(aws ecs list-tasks --cluster my-cluster \
    --service-name my-service --desired-status STOPPED \
    --query 'taskArns[0]' --output text) \
  --query 'tasks[0].{reason:stoppedReason,status:lastStatus,stopCode:stopCode}'

# Common outputs:
# "Essential container in task exited"
# "Task stopped by: OutOfMemoryError: Container killed due to memory usage"
# "CannotPullContainerError: pull image manifest has been retried 5 time(s)"

# Check service events for patterns
aws ecs describe-services \
  --cluster my-cluster \
  --services my-service \
  --query 'services[0].events[:10].[createdAt,message]' \
  --output table

# Check actual memory usage before the kill
aws logs filter-log-events \
  --log-group-name /ecs/my-service \
  --filter-pattern "memory" \
  --start-time $(date -d '1 hour ago' +%s000) \
  --query 'events[*].message'

# Check Container Insights for peak memory (if enabled)
aws cloudwatch get-metric-statistics \
  --namespace ECS/ContainerInsights \
  --metric-name MemoryUtilized \
  --dimensions Name=ServiceName,Value=my-service \
               Name=ClusterName,Value=my-cluster \
  --start-time $(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%S) \
  --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
  --period 60 --statistics Maximum \
  --query 'Datapoints | sort_by(@, &Timestamp) | [-5:]'

# Update task definition with higher memory
# Rule of thumb: set hard limit to 1.5x observed peak
aws ecs register-task-definition \
  --cli-input-json file://task-definition.json
# Then update the service to use the new task definition revision
aws ecs update-service \
  --cluster my-cluster \
  --service my-service \
  --task-definition my-service:NEW_REVISION

# Get the exit code of the stopped container
aws ecs describe-tasks \
  --cluster my-cluster \
  --tasks <TASK_ARN> \
  --query 'tasks[0].containers[*].{name:name,exitCode:exitCode,reason:reason}'

# Check CloudWatch logs for crash reason
aws logs tail /ecs/my-service --since 30m --format short | tail -50

# If health check is killing the container, review the health check config:
# - Ensure the health check endpoint returns 200 within the timeout
# - Increase startPeriod for slow-starting applications (default: 0s)
# - Common fix: set startPeriod to 60-120s for Java/Spring apps

# Check environment variables are properly set
aws ecs describe-task-definition \
  --task-definition my-service \
  --query 'taskDefinition.containerDefinitions[0].environment'

# Verify the image exists in ECR
aws ecr describe-images \
  --repository-name my-service \
  --image-ids imageTag=latest \
  --query 'imageDetails[0].imagePushedAt'

# Check task execution role has ECR permissions
aws iam simulate-principal-policy \
  --policy-source-arn arn:aws:iam::123456789012:role/ecsTaskExecutionRole \
  --action-names ecr:GetDownloadUrlForLayer ecr:BatchGetImage \
  --query 'EvaluationResults[*].{Action:EvalActionName,Decision:EvalDecision}'

# If using private subnets, verify VPC endpoints or NAT Gateway exist
aws ec2 describe-vpc-endpoints \
  --filters "Name=service-name,Values=com.amazonaws.eu-west-1.ecr.dkr" \
  --query 'VpcEndpoints[*].{VpcId:VpcId,State:State}'

# 1. Check tasks are running and stable
aws ecs describe-services \
  --cluster my-cluster \
  --services my-service \
  --query 'services[0].{running:runningCount,desired:desiredCount,pending:pendingCount}'
# Expected: running == desired, pending == 0

# 2. Verify no recent task stops
aws ecs list-tasks --cluster my-cluster \
  --service-name my-service --desired-status STOPPED \
  --query 'taskArns | length(@)'
# Expected: 0 or only old tasks

# 3. Check service events for stability (no recent restarts)
aws ecs describe-services \
  --cluster my-cluster \
  --services my-service \
  --query 'services[0].events[:5].message'
# Expected: "has reached a steady state" message

# 4. Verify health check passing
aws elbv2 describe-target-health \
  --target-group-arn <TARGET_GROUP_ARN> \
  --query 'TargetHealthDescriptions[*].{Target:Target.Id,Health:TargetHealth.State}'
# Expected: all targets "healthy"