Amazon CloudWatch Essentials

# List available metrics for a namespace
aws cloudwatch list-metrics \
    --namespace AWS/EC2

# List metrics for a specific instance
aws cloudwatch list-metrics \
    --namespace AWS/EC2 \
    --dimensions Name=InstanceId,Value=i-xxxxx

# Get CPU statistics (last 24 hours, 5 min periods)
aws cloudwatch get-metric-statistics \
    --namespace AWS/EC2 \
    --metric-name CPUUtilization \
    --dimensions Name=InstanceId,Value=i-xxxxx \
    --start-time $(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%S) \
    --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
    --period 300 \
    --statistics Average,Maximum \
    --output table

# Get multiple metrics with get-metric-data (more efficient)
cat > metric-queries.json <<EOF
[
  {
    "Id": "cpu",
    "MetricStat": {
      "Metric": {
        "Namespace": "AWS/EC2",
        "MetricName": "CPUUtilization",
        "Dimensions": [{"Name": "InstanceId", "Value": "i-xxxxx"}]
      },
      "Period": 300,
      "Stat": "Average"
    }
  },
  {
    "Id": "network",
    "MetricStat": {
      "Metric": {
        "Namespace": "AWS/EC2",
        "MetricName": "NetworkIn",
        "Dimensions": [{"Name": "InstanceId", "Value": "i-xxxxx"}]
      },
      "Period": 300,
      "Stat": "Sum"
    }
  }
]
EOF

aws cloudwatch get-metric-data \
    --metric-data-queries file://metric-queries.json \
    --start-time $(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%S) \
    --end-time $(date -u +%Y-%m-%dT%H:%M:%S)

# Get ALB metrics (request count, latency, errors)
aws cloudwatch get-metric-statistics \
    --namespace AWS/ApplicationELB \
    --metric-name TargetResponseTime \
    --dimensions Name=LoadBalancer,Value=app/myapp-lb/xxxxx \
    --start-time $(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%S) \
    --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
    --period 60 \
    --statistics Average,Maximum

# Get Route53 health check status
aws cloudwatch get-metric-statistics \
    --namespace AWS/Route53 \
    --metric-name HealthCheckStatus \
    --dimensions Name=HealthCheckId,Value=abc123-healthcheck \
    --start-time $(date -u -d '12 hours ago' +%Y-%m-%dT%H:%M:%S) \
    --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
    --period 60 \
    --statistics Minimum

# Publish simple custom metric
aws cloudwatch put-metric-data \
    --namespace CustomApp/Business \
    --metric-name OrdersProcessed \
    --value 142 \
    --timestamp $(date -u +%Y-%m-%dT%H:%M:%S)

# Publish metric with dimensions
aws cloudwatch put-metric-data \
    --namespace CustomApp/API \
    --metric-name ResponseTime \
    --value 234 \
    --unit Milliseconds \
    --dimensions Environment=Production,Region=sa-east-1

# Publish multiple metrics (batch)
aws cloudwatch put-metric-data \
    --namespace CustomApp/Database \
    --metric-data \
        MetricName=ActiveConnections,Value=45,Unit=Count \
        MetricName=QueryTime,Value=123,Unit=Milliseconds \
        MetricName=CacheHitRate,Value=0.89,Unit=Percent

# Publish with high resolution (1 second)
aws cloudwatch put-metric-data \
    --namespace CustomApp/HighRes \
    --metric-name Latency \
    --value 78 \
    --unit Milliseconds \
    --storage-resolution 1

# Simple alarm: CPU over 80%
aws cloudwatch put-metric-alarm \
    --alarm-name high-cpu-i-xxxxx \
    --alarm-description "CPU above 80% for 10 minutes" \
    --namespace AWS/EC2 \
    --metric-name CPUUtilization \
    --dimensions Name=InstanceId,Value=i-xxxxx \
    --statistic Average \
    --period 300 \
    --evaluation-periods 2 \
    --threshold 80 \
    --comparison-operator GreaterThanThreshold \
    --treat-missing-data notBreaching

# Alarm with SNS notification
aws cloudwatch put-metric-alarm \
    --alarm-name high-cpu-with-notification \
    --alarm-description "Alert ops team when CPU high" \
    --namespace AWS/EC2 \
    --metric-name CPUUtilization \
    --dimensions Name=InstanceId,Value=i-xxxxx \
    --statistic Average \
    --period 300 \
    --evaluation-periods 2 \
    --threshold 80 \
    --comparison-operator GreaterThanThreshold \
    --alarm-actions arn:aws:sns:sa-east-1:123456789012:ops-alerts \
    --ok-actions arn:aws:sns:sa-east-1:123456789012:ops-alerts

# Alarm for ALB (high latency)
aws cloudwatch put-metric-alarm \
    --alarm-name alb-high-latency \
    --alarm-description "Target response time over 500ms" \
    --namespace AWS/ApplicationELB \
    --metric-name TargetResponseTime \
    --dimensions Name=LoadBalancer,Value=app/myapp-lb/xxxxx \
    --statistic Average \
    --period 60 \
    --evaluation-periods 3 \
    --threshold 0.5 \
    --comparison-operator GreaterThanThreshold \
    --alarm-actions arn:aws:sns:sa-east-1:123456789012:critical-alerts

# Alarm for Route53 health check
aws cloudwatch put-metric-alarm \
    --alarm-name route53-primary-unhealthy \
    --alarm-description "Primary region health check failed - failover activated" \
    --namespace AWS/Route53 \
    --metric-name HealthCheckStatus \
    --dimensions Name=HealthCheckId,Value=abc123-healthcheck \
    --statistic Minimum \
    --period 60 \
    --evaluation-periods 2 \
    --threshold 1 \
    --comparison-operator LessThanThreshold \
    --alarm-actions arn:aws:sns:sa-east-1:123456789012:failover-alerts

# Alarm for error rate (5XX over 100 requests)
aws cloudwatch put-metric-alarm \
    --alarm-name alb-high-error-rate \
    --alarm-description "Too many 5XX errors from targets" \
    --namespace AWS/ApplicationELB \
    --metric-name HTTPCode_Target_5XX_Count \
    --dimensions Name=LoadBalancer,Value=app/myapp-lb/xxxxx \
    --statistic Sum \
    --period 300 \
    --evaluation-periods 1 \
    --threshold 100 \
    --comparison-operator GreaterThanThreshold \
    --alarm-actions arn:aws:sns:sa-east-1:123456789012:critical-alerts

# Alarm with "M of N" datapoints (more flexible)
aws cloudwatch put-metric-alarm \
    --alarm-name cpu-high-3-of-5 \
    --alarm-description "CPU high in 3 out of 5 datapoints" \
    --namespace AWS/EC2 \
    --metric-name CPUUtilization \
    --dimensions Name=InstanceId,Value=i-xxxxx \
    --statistic Average \
    --period 60 \
    --evaluation-periods 5 \
    --datapoints-to-alarm 3 \
    --threshold 80 \
    --comparison-operator GreaterThanThreshold

# Composite Alarm (combines multiple alarms)
aws cloudwatch put-composite-alarm \
    --alarm-name app-unhealthy \
    --alarm-description "App is unhealthy if high CPU AND high error rate" \
    --alarm-rule "ALARM(high-cpu-i-xxxxx) AND ALARM(alb-high-error-rate)" \
    --alarm-actions arn:aws:sns:sa-east-1:123456789012:critical-alerts

# List all alarms
aws cloudwatch describe-alarms

# View alarms in ALARM state
aws cloudwatch describe-alarms \
    --state-value ALARM

# View specific alarm details
aws cloudwatch describe-alarms \
    --alarm-names high-cpu-i-xxxxx

# View alarm history (state changes)
aws cloudwatch describe-alarm-history \
    --alarm-name high-cpu-i-xxxxx \
    --max-records 10

# Disable alarm temporarily
aws cloudwatch disable-alarm-actions \
    --alarm-names high-cpu-i-xxxxx

# Re-enable alarm
aws cloudwatch enable-alarm-actions \
    --alarm-names high-cpu-i-xxxxx

# Change state manually (testing)
aws cloudwatch set-alarm-state \
    --alarm-name high-cpu-i-xxxxx \
    --state-value ALARM \
    --state-reason "Testing alarm notifications"

# Delete alarms
aws cloudwatch delete-alarms \
    --alarm-names high-cpu-i-xxxxx alb-high-latency

# Create log group
aws logs create-log-group \
    --log-group-name /aws/ec2/myapp

# Configure retention (7 days)
aws logs put-retention-policy \
    --log-group-name /aws/ec2/myapp \
    --retention-in-days 7

# Create log stream
aws logs create-log-stream \
    --log-group-name /aws/ec2/myapp \
    --log-stream-name i-xxxxx-app.log

# Send log events (from application)
cat > log-events.json <<EOF
[
  {
    "timestamp": $(date +%s)000,
    "message": "Application started successfully"
  },
  {
    "timestamp": $(date +%s)000,
    "message": "Connected to database"
  }
]
EOF

aws logs put-log-events \
    --log-group-name /aws/ec2/myapp \
    --log-stream-name i-xxxxx-app.log \
    --log-events file://log-events.json

# Tag log group (for cost allocation)
aws logs tag-log-group \
    --log-group-name /aws/ec2/myapp \
    --tags Environment=Production,Application=MyApp

# List log groups
aws logs describe-log-groups

# List log streams in a group
aws logs describe-log-streams \
    --log-group-name /aws/ec2/myapp \
    --order-by LastEventTime \
    --descending \
    --max-items 10

# Tail logs (real-time)
aws logs tail /aws/ec2/myapp --follow

# Filter logs by pattern (last 24 hours)
aws logs filter-log-events \
    --log-group-name /aws/ec2/myapp \
    --start-time $(date -u -d '24 hours ago' +%s)000 \
    --filter-pattern "ERROR"

# Filter with structured pattern
aws logs filter-log-events \
    --log-group-name /aws/ec2/myapp \
    --filter-pattern '[timestamp, level=ERROR, msg]' \
    --max-items 50

# Search in multiple log groups
aws logs filter-log-events \
    --log-group-name-prefix /aws/ec2/ \
    --filter-pattern "database connection failed"

# Export logs to S3 (for later analysis)
aws logs create-export-task \
    --log-group-name /aws/ec2/myapp \
    --from $(date -u -d '7 days ago' +%s)000 \
    --to $(date -u +%s)000 \
    --destination myapp-logs-bucket \
    --destination-prefix logs/2025/11/

# Query: Top 10 most frequent errors
aws logs start-query \
    --log-group-name /aws/ec2/myapp \
    --start-time $(date -u -d '24 hours ago' +%s) \
    --end-time $(date -u +%s) \
    --query-string '
        fields @timestamp, @message
        | filter @message like /ERROR/
        | stats count() by @message
        | sort count desc
        | limit 10
    '

# Get query results
aws logs get-query-results --query-id xxxxx-yyyy-zzzz

# Query: Average latency by endpoint
aws logs start-query \
    --log-group-name /aws/lambda/my-api \
    --start-time $(date -u -d '1 hour ago' +%s) \
    --end-time $(date -u +%s) \
    --query-string '
        fields @timestamp, endpoint, duration
        | stats avg(duration) as avg_latency, max(duration) as max_latency by endpoint
        | sort avg_latency desc
    '

# Query: Slow requests (over 500ms)
aws logs start-query \
    --log-group-name /aws/ec2/myapp \
    --start-time $(date -u -d '1 hour ago' +%s) \
    --end-time $(date -u +%s) \
    --query-string '
        fields @timestamp, @message, latency
        | filter latency > 500
        | sort latency desc
        | limit 100
    '

# Create metric filter to count errors
aws logs put-metric-filter \
    --log-group-name /aws/ec2/myapp \
    --filter-name ErrorCount \
    --filter-pattern "[timestamp, level=ERROR, msg]" \
    --metric-transformations \
        metricName=ApplicationErrors,\
        metricNamespace=CustomApp/Logs,\
        metricValue=1,\
        defaultValue=0

# Metric filter to extract latency from logs
aws logs put-metric-filter \
    --log-group-name /aws/ec2/myapp \
    --filter-name ResponseTime \
    --filter-pattern "[timestamp, level, msg, latency]" \
    --metric-transformations \
        metricName=ResponseLatency,\
        metricNamespace=CustomApp/Logs,\
        metricValue='$latency',\
        unit=Milliseconds

# List metric filters
aws logs describe-metric-filters \
    --log-group-name /aws/ec2/myapp

# Delete metric filter
aws logs delete-metric-filter \
    --log-group-name /aws/ec2/myapp \
    --filter-name ErrorCount

# Create SNS topic for notifications
aws sns create-topic --name ec2-state-changes

# Subscribe email
aws sns subscribe \
    --topic-arn arn:aws:sns:sa-east-1:123456789012:ec2-state-changes \
    --protocol email \
    --notification-endpoint ops@example.com

# Event pattern: EC2 instance terminated
cat > event-pattern-terminated.json <<EOF
{
  "source": ["aws.ec2"],
  "detail-type": ["EC2 Instance State-change Notification"],
  "detail": {
    "state": ["terminated"]
  }
}
EOF

# Create rule
aws events put-rule \
    --name notify-instance-terminated \
    --description "Notify when EC2 instance is terminated" \
    --event-pattern file://event-pattern-terminated.json

# Add SNS as target
aws events put-targets \
    --rule notify-instance-terminated \
    --targets "Id"="1","Arn"="arn:aws:sns:sa-east-1:123456789012:ec2-state-changes"

# Event pattern: Auto Scaling activities
cat > event-pattern-asg.json <<EOF
{
  "source": ["aws.autoscaling"],
  "detail-type": ["EC2 Instance Launch Successful", "EC2 Instance Terminate Successful"],
  "detail": {
    "AutoScalingGroupName": ["myapp-asg"]
  }
}
EOF

aws events put-rule \
    --name asg-scaling-events \
    --event-pattern file://event-pattern-asg.json

# Scheduled rule (cron - daily at 2 AM UTC)
aws events put-rule \
    --name daily-cleanup \
    --schedule-expression "cron(0 2 * * ? *)" \
    --description "Run cleanup Lambda daily at 2 AM UTC"

# Scheduled rule (rate - every 5 minutes)
aws events put-rule \
    --name health-check-poller \
    --schedule-expression "rate(5 minutes)" \
    --description "Poll external health check every 5 minutes"

# Add Lambda as target
aws events put-targets \
    --rule daily-cleanup \
    --targets "Id"="1","Arn"="arn:aws:lambda:sa-east-1:123456789012:function:cleanup-function"

# Give EventBridge permission to invoke Lambda
aws lambda add-permission \
    --function-name cleanup-function \
    --statement-id AllowEventBridgeInvoke \
    --action lambda:InvokeFunction \
    --principal events.amazonaws.com \
    --source-arn arn:aws:events:sa-east-1:123456789012:rule/daily-cleanup

# List rules
aws events list-rules

# View rule details
aws events describe-rule --name notify-instance-terminated

# List targets for a rule
aws events list-targets-by-rule --rule notify-instance-terminated

# Disable rule temporarily
aws events disable-rule --name daily-cleanup

# Re-enable rule
aws events enable-rule --name daily-cleanup

# Remove targets first
aws events remove-targets \
    --rule daily-cleanup \
    --ids "1"

# Then delete rule
aws events delete-rule --name daily-cleanup

# Create dashboard
cat > dashboard-body.json <<EOF
{
  "widgets": [
    {
      "type": "metric",
      "properties": {
        "metrics": [
          ["AWS/EC2", "CPUUtilization", {"stat": "Average"}]
        ],
        "period": 300,
        "stat": "Average",
        "region": "sa-east-1",
        "title": "EC2 CPU Utilization"
      }
    },
    {
      "type": "metric",
      "properties": {
        "metrics": [
          ["AWS/ApplicationELB", "RequestCount", {"stat": "Sum"}]
        ],
        "period": 60,
        "stat": "Sum",
        "region": "sa-east-1",
        "title": "ALB Request Count"
      }
    }
  ]
}
EOF

aws cloudwatch put-dashboard \
    --dashboard-name MyApp-Production \
    --dashboard-body file://dashboard-body.json

# List dashboards
aws cloudwatch list-dashboards

# View dashboard
aws cloudwatch get-dashboard --dashboard-name MyApp-Production

# Delete dashboard
aws cloudwatch delete-dashboards --dashboard-names MyApp-Production

# Install CloudWatch Agent on EC2 (Amazon Linux 2)
sudo yum install amazon-cloudwatch-agent -y

# Create configuration
cat > /opt/aws/amazon-cloudwatch-agent/etc/config.json <<EOF
{
  "agent": {
    "metrics_collection_interval": 60,
    "run_as_user": "cwagent"
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/app.log",
            "log_group_name": "/aws/ec2/myapp",
            "log_stream_name": "{instance_id}-app.log",
            "timezone": "UTC"
          },
          {
            "file_path": "/var/log/nginx/error.log",
            "log_group_name": "/aws/ec2/nginx",
            "log_stream_name": "{instance_id}-error.log"
          }
        ]
      }
    }
  },
  "metrics": {
    "namespace": "CustomApp/System",
    "metrics_collected": {
      "mem": {
        "measurement": [
          {
            "name": "mem_used_percent",
            "rename": "MemoryUtilization",
            "unit": "Percent"
          }
        ],
        "metrics_collection_interval": 60
      },
      "disk": {
        "measurement": [
          {
            "name": "used_percent",
            "rename": "DiskUtilization",
            "unit": "Percent"
          }
        ],
        "metrics_collection_interval": 60,
        "resources": ["/"]
      }
    }
  }
}
EOF

# Start agent
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
    -a fetch-config \
    -m ec2 \
    -s \
    -c file:/opt/aws/amazon-cloudwatch-agent/etc/config.json

# Verify status
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
    -a query \
    -m ec2 \
    -s

# Stop agent
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
    -a stop \
    -m ec2 \
    -s

Infrastructure:
- 10 EC2 instances (5 per region)
- 2 ALB (1 per region)
- 2 RDS instances
- Auto-Scaling Groups
- Route53 health checks

CloudWatch Setup:
- AWS service metrics (EC2, ALB, RDS, ASG): FREE
- 5 custom business metrics (orders/min, revenue, etc.): 5 x $0.30 = $1.50/month
- 20 standard alarms (CPU, latency, errors, etc.): 20 x $0.10 = $2.00/month
- 2 composite alarms (app-healthy): 2 x $0.50 = $1.00/month
- 1 dashboard (production overview): $3.00/month

Logs:
- 50 GB/month ingestion: (50 - 5 free) x $0.50 = $22.50/month
- Average 200 GB stored (30 day retention): (200 - 5 free) x $0.03 = $5.85/month
- Logs Insights queries: ~10 GB/month scanned: 10 x $0.005 = $0.05/month

EventBridge:
- ~5M events/month (EC2, ASG, custom): FREE (under 14M)

TOTAL MONTHLY: $35.90/month

Custom metrics:         $1.50  (4%)
Alarms:                 $3.00  (8%)
Dashboard:              $3.00  (8%)
Logs ingestion:        $22.50 (63%) - Largest cost
Logs storage:           $5.85 (16%)
Logs queries:           $0.05 (under 1%)
EventBridge:            $0.00  (0%)

# EXPENSIVE: 90 day retention for ALL logs
# 50 GB/month x 90 days = 4500 GB stored x $0.03 = $135/month

# ECONOMICAL: Differentiated retention by type
# Debug logs: 7 days
aws logs put-retention-policy \
    --log-group-name /aws/ec2/myapp-debug \
    --retention-in-days 7

# Application logs: 30 days
aws logs put-retention-policy \
    --log-group-name /aws/ec2/myapp \
    --retention-in-days 30

# Error logs: 90 days
aws logs put-retention-policy \
    --log-group-name /aws/ec2/myapp-errors \
    --retention-in-days 90

# Audit logs: 1 year (compliance)
aws logs put-retention-policy \
    --log-group-name /aws/audit \
    --retention-in-days 365

# Savings: ~60% in storage costs

import random

def log_request(status_code, latency):
    # Always log errors
    if status_code >= 400:
        logger.error(f"Error {status_code}, latency {latency}ms")
        return

    # Sample 10% of successful requests
    if random.random() < 0.1:
        logger.info(f"Success {status_code}, latency {latency}ms")

# Log reduction: ~90% less data
# Maintain visibility of problems (100% errors)
# Still see trends (10% sample is statistically significant)

# EXPENSIVE: Publish custom metric from code
# Code: cloudwatch.put_metric_data(...)
# Cost: $0.30/month per custom metric

# ECONOMICAL: Metric filter on existing logs
aws logs put-metric-filter \
    --log-group-name /aws/ec2/myapp \
    --filter-name ErrorCount \
    --filter-pattern "[timestamp, level=ERROR, msg]" \
    --metric-transformations \
        metricName=ApplicationErrors,\
        metricNamespace=CustomApp/Logs,\
        metricValue=1

# Cost: $0 (you're already paying for log ingestion)
# Same result: error metric

# EXPENSIVE: Dashboard per resource
# Dashboard EC2 Instance 1: $3/month
# Dashboard EC2 Instance 2: $3/month
# ... x 10 instances = $30/month

# ECONOMICAL: Aggregated dashboard
# 1 Dashboard "Production Overview" with all instances: $3/month
# Savings: $27/month