[Network] 서비스 안정성을 지키는 메트릭, 로그, 알림 구축 전략

http_requests_total{method="GET", status="200"} 1234567
http_request_duration_seconds{quantile="0.99"} 0.342

2026-04-26T14:32:01Z ERROR user_id=4892 action=checkout
  message="payment gateway timeout after 30s"
  order_id=8847291 amount=49800

Request → API Gateway (5ms)
  → Auth Service (8ms)
  → Product Service (187ms) ← 여기가 느림
      → DB Query: SELECT * FROM products (175ms) ← 병목

CloudWatch
├── Metrics — AWS 서비스 + 커스텀 메트릭
├── Logs — 로그 수집, 저장, 검색 (Logs Insights)
├── Alarms — 임계값 기반 알림 (SNS → Slack/PagerDuty)
└── Dashboards — 메트릭 시각화

# 실습 디렉토리 준비
mkdir -p /tmp/networking/part6/exam_30 && cd /tmp/networking/part6/exam_30

# CloudWatch Agent 설치
wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
sudo dpkg -i amazon-cloudwatch-agent.deb

# 설정 파일 생성
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-config-wizard

# 메모리/디스크 메트릭은 기본 수집 안됨 — agent 필요
# 설정 예시 (/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json)
{
  "metrics": {
    "namespace": "MyApp/System",
    "metrics_collected": {
      "mem": { "measurement": ["mem_used_percent"] },
      "disk": { "measurement": ["disk_used_percent"],
                "resources": ["/", "/data"] }
    }
  },
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [{
          "file_path": "/var/log/myapp/app.log",
          "log_group_name": "/myapp/application",
          "log_stream_name": "{instance_id}"
        }]
      }
    }
  }
}

# 최근 1시간 HTTP 5xx 에러가 가장 많은 엔드포인트
fields @timestamp, path, status, duration
| filter status >= 500
| stats count(*) as errors, avg(duration) as avg_ms by path
| sort errors desc
| limit 10

# 특정 사용자의 최근 요청 흐름
fields @timestamp, user_id, action, status
| filter user_id = "4892"
| sort @timestamp desc
| limit 50

# 에러율이 급증한 시간대
fields @timestamp, status
| filter status >= 500
| stats count(*) as errors by bin(5m)
| sort @timestamp asc

# P99 응답시간 1초 초과 시 알림
aws cloudwatch put-metric-alarm \
  --alarm-name "API-HighLatency" \
  --metric-name "TargetResponseTime" \
  --namespace "AWS/ApplicationELB" \
  --statistic "p99" \
  --period 60 \
  --threshold 1.0 \
  --comparison-operator "GreaterThanThreshold" \
  --evaluation-periods 3 \
  --alarm-actions "arn:aws:sns:ap-northeast-2:123456:alerts"

애플리케이션 서버들
├── /metrics 엔드포인트 노출 (Prometheus 형식)
│
Prometheus Server (15초마다 pull)
├── 시계열 데이터 저장 (TSDB)
├── Alert Rules 평가
└── Alertmanager → Slack/PagerDuty
│
Grafana
└── Prometheus 쿼리 → 대시보드 시각화

# docker-compose.yml
version: '3'
services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin123

# prometheus.yml
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'myapp'
    static_configs:
      - targets: ['myapp:8080']  # /metrics 엔드포인트

# Node Exporter 설치 (서버 메트릭 수집)
docker run -d \
  --name node-exporter \
  --pid="host" \
  -v "/:/host:ro,rslave" \
  prom/node-exporter \
  --path.rootfs=/host

# 초당 요청 수 (5분 평균)
rate(http_requests_total[5m])

# 에러율 (%)
rate(http_requests_total{status=~"5.."}[5m])
/ rate(http_requests_total[5m]) * 100

# P99 응답시간
histogram_quantile(0.99,
  rate(http_request_duration_seconds_bucket[5m])
)

# 서비스별 초당 에러 수 비교
sum by (service) (
  rate(http_requests_total{status=~"5.."}[5m])
)

# 메모리 사용률 (Node Exporter)
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# alert.rules.yml
groups:
  - name: api_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          rate(http_requests_total{status=~"5.."}[5m])
          / rate(http_requests_total[5m]) > 0.05
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "에러율 5% 초과: {{ $labels.service }}"
          description: "현재 에러율: {{ $value | humanizePercentage }}"