aws ce get-cost-and-usage \
  --time-period Start=$(date -d '-30 days' +%Y-%m-%d),End=$(date +%Y-%m-%d) \
  --granularity MONTHLY \
  --metrics BlendedCost \
  --group-by Type=DIMENSION,Key=SERVICE \
  | jq '.ResultsByTime[0].Groups
        | sort_by(.Metrics.BlendedCost.Amount | tonumber)
        | reverse | .[:10]'

# Unattached EBS volumes (pure waste)
aws ec2 describe-volumes \
  --filters Name=status,Values=available \
  --query 'Volumes[*].{ID:VolumeId,Size:Size,Type:VolumeType}' \
  --output table

# Idle Elastic IPs (charged when unattached)
aws ec2 describe-addresses \
  --query 'Addresses[?AssociationId==null].{IP:PublicIp,AllocId:AllocationId}' \
  --output table

resource "aws_autoscaling_group" "workers" {
  desired_capacity = 4
  max_size         = 10
  min_size         = 2

  mixed_instances_policy {
    instances_distribution {
      on_demand_base_capacity                  = 1  # 1 guaranteed instance
      on_demand_percentage_above_base_capacity = 0  # rest are spot
      spot_allocation_strategy                 = "capacity-optimized"
      spot_max_price                           = "" # on-demand price cap
    }
    launch_template {
      launch_template_specification {
        launch_template_id = aws_launch_template.worker.id
        version            = "$Latest"
      }
      override { instance_type = "m6i.xlarge"; weighted_capacity = "1" }
      override { instance_type = "m6a.xlarge"; weighted_capacity = "1" }
      override { instance_type = "m5.xlarge";  weighted_capacity = "1" }
      override { instance_type = "m5a.xlarge"; weighted_capacity = "1" }
    }
  }
}

import requests, signal, sys, threading, time

METADATA_URL = "http://169.254.169.254/latest/meta-data/spot/instance-action"

def check_spot_interruption() -> dict | None:
    try:
        resp = requests.get(METADATA_URL, timeout=1)
        if resp.status_code == 200:
            return resp.json()
    except requests.exceptions.RequestException:
        pass
    return None

def graceful_shutdown(checkpoint_fn):
    def handler(signum, frame):
        checkpoint_fn()
        sys.exit(0)
    signal.signal(signal.SIGTERM, handler)

    def poll():
        while True:
            if check_spot_interruption():
                checkpoint_fn()
                sys.exit(0)
            time.sleep(5)
    threading.Thread(target=poll, daemon=True).start()

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-server
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-server
  minReplicas: 2
  maxReplicas: 20
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
        - type: Percent
          value: 25
          periodSeconds: 60
    scaleUp:
      stabilizationWindowSeconds: 0
      policies:
        - type: Percent
          value: 100
          periodSeconds: 60
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 65

# Scale down at 20:00 weekdays
aws autoscaling update-auto-scaling-group \
  --auto-scaling-group-name dev-workers \
  --min-size 0 --max-size 0 --desired-capacity 0

# Scale up at 08:00 weekdays
aws autoscaling update-auto-scaling-group \
  --auto-scaling-group-name dev-workers \
  --min-size 1 --max-size 4 --desired-capacity 2

resource "aws_budgets_budget" "monthly" {
  name         = "monthly-total"
  budget_type  = "COST"
  limit_amount = "5000"
  limit_unit   = "USD"
  time_unit    = "MONTHLY"

  dynamic "notification" {
    for_each = [50, 80, 100, 120]
    content {
      comparison_operator       = "GREATER_THAN"
      threshold                 = notification.value
      threshold_type            = "PERCENTAGE"
      notification_type         = notification.value <= 100 ? "FORECASTED" : "ACTUAL"
      subscriber_email_addresses = ["eng-leads@company.com"]
      subscriber_sns_topic_arns  = [aws_sns_topic.budget_alerts.arn]
    }
  }
}

resource "aws_budgets_budget" "gpu" {
  name         = "gpu-spend"
  budget_type  = "COST"
  limit_amount = "2000"
  limit_unit   = "USD"
  time_unit    = "MONTHLY"

  cost_filter {
    name   = "Service"
    values = ["Amazon Elastic Compute Cloud - Compute"]
  }

  notification {
    comparison_operator       = "GREATER_THAN"
    threshold                 = 80
    threshold_type            = "PERCENTAGE"
    notification_type         = "ACTUAL"
    subscriber_email_addresses = ["ml-team@company.com"]
  }
}