마이크로서비스 모니터링 가이드 2026: 관측성 중심 운영

# otel-collector.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  prometheus:
    config:
      scrape_configs:
        - job_name: 'microservices'
          static_configs:
            - targets: ['app:8080', 'db:3306']

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024
  resource:
    attributes:
      - key: environment
        value: production
        action: upsert

exporters:
  jaeger:
    endpoint: jaeger:14250
    tls:
      insecure: true
  prometheus:
    endpoint: "0.0.0.0:8889"
  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, resource]
      exporters: [jaeger]
    metrics:
      receivers: [otlp, prometheus]
      processors: [batch, resource]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [batch, resource]
      exporters: [loki]

// 애플리케이션 계측
import { NodeSDK } from '@opentelemetry/sdk-node';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'user-service',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
    [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: 'production',
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      '@opentelemetry/instrumentation-http': {
        requestHook: (span, request) => {
          span.setAttributes({
            'http.request.size': request.headers['content-length'],
            'user.id': request.headers['x-user-id'],
          });
        },
      },
    }),
  ],
});

sdk.start();

// 커스텀 메트릭
import { metrics } from '@opentelemetry/api';
const meter = metrics.getMeter('user-service', '1.0.0');

const requestCounter = meter.createCounter('http_requests_total', {
  description: 'Total number of HTTP requests',
});

const responseTimeHistogram = meter.createHistogram('http_request_duration_ms', {
  description: 'HTTP request duration in milliseconds',
});

export const metricsMiddleware = (req: Request, res: Response, next: NextFunction) => {
  const startTime = Date.now();

  requestCounter.add(1, {
    method: req.method,
    route: req.route?.path || 'unknown',
  });

  res.on('finish', () => {
    const duration = Date.now() - startTime;
    responseTimeHistogram.record(duration, {
      method: req.method,
      status_code: res.statusCode.toString(),
    });
  });

  next();
};

# 지능형 트레이스 분석
from opentelemetry import trace
from opentelemetry.sdk.trace.export import SpanExporter
import asyncio
from typing import List, Dict

class IntelligentTraceAnalyzer:
    def __init__(self):
        self.anomaly_detector = AnomalyDetector()
        self.pattern_analyzer = PatternAnalyzer()
        self.correlation_engine = CorrelationEngine()

    async def analyze_trace(self, trace_id: str) -> TraceAnalysis:
        # 전체 트레이스 수집
        spans = await self.collect_trace_spans(trace_id)

        # 성능 분석
        performance_analysis = await self.analyze_performance(spans)

        # 에러 상관관계 분석
        error_correlation = await self.analyze_error_correlation(spans)

        # 비정상 패턴 감지
        anomalies = await self.detect_anomalies(spans)

        return TraceAnalysis(
            trace_id=trace_id,
            performance=performance_analysis,
            errors=error_correlation,
            anomalies=anomalies,
            recommendations=await self.generate_recommendations(spans)
        )

    async def real_time_trace_monitoring(self):
        """실시간 트레이스 모니터링"""
        while True:
            recent_traces = await self.get_recent_traces()

            for trace in recent_traces:
                analysis = await self.analyze_trace(trace.trace_id)

                if analysis.has_critical_issues():
                    await self.send_alert(analysis)

                # 패턴 학습
                await self.pattern_analyzer.learn_from_trace(trace)

            await asyncio.sleep(10)  # 10초마다 분석

class ServiceDependencyTracker:
    def __init__(self):
        self.dependency_graph = DependencyGraph()
        self.health_checker = ServiceHealthChecker()

    async def build_dependency_map(self) -> DependencyMap:
        """서비스 의존성 맵 구축"""
        traces = await self.collect_recent_traces(hours=24)

        for trace in traces:
            # 서비스 간 호출 관계 추출
            dependencies = self.extract_service_calls(trace)
            self.dependency_graph.add_dependencies(dependencies)

        # 의존성 맵 생성
        return self.dependency_graph.to_map()

    async def analyze_cascade_failures(self, failed_service: str) -> CascadeAnalysis:
        """연쇄 장애 분석"""
        dependent_services = self.dependency_graph.get_dependents(failed_service)

        impact_analysis = []
        for service in dependent_services:
            health = await self.health_checker.check_service(service)
            impact = self.calculate_impact_score(service, failed_service)

            impact_analysis.append({
                'service': service,
                'health': health,
                'impact_score': impact,
                'recovery_time': await self.estimate_recovery_time(service)
            })

        return CascadeAnalysis(
            root_cause=failed_service,
            affected_services=impact_analysis,
            total_impact_score=sum(ia['impact_score'] for ia in impact_analysis)
        )

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"
  - "recording_rules.yml"

scrape_configs:
  - job_name: 'microservices'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)

  - job_name: 'istio-mesh'
    kubernetes_sd_configs:
      - role: endpoints
        namespaces:
          names:
          - istio-system
    relabel_configs:
      - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: istio-telemetry;prometheus

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

# alert_rules.yml
groups:
  - name: microservice.rules
    rules:
      - alert: HighLatency
        expr: histogram_quantile(0.95, http_request_duration_seconds_bucket) > 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High latency detected"
          description: "95th percentile latency is {{ $value }}s"

      - alert: ErrorRateHigh
        expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }}"

      - alert: ServiceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Service is down"
          description: "{{ $labels.instance }} has been down for more than 1 minute"

// 고급 메트릭 수집
class AdvancedMetricsCollector {
    private prometheus: PrometheusRegistry;
    private customMetrics: Map<string, Metric>;

    constructor() {
        this.prometheus = new PrometheusRegistry();
        this.setupBusinessMetrics();
        this.setupSLIMetrics();
    }

    private setupBusinessMetrics() {
        // 비즈니스 메트릭
        this.customMetrics.set('user_registration', new Counter({
            name: 'user_registrations_total',
            help: 'Total number of user registrations',
            labelNames: ['source', 'plan_type']
        }));

        this.customMetrics.set('order_value', new Histogram({
            name: 'order_value_dollars',
            help: 'Order value in dollars',
            buckets: [10, 50, 100, 500, 1000, 5000]
        }));

        this.customMetrics.set('active_users', new Gauge({
            name: 'active_users_current',
            help: 'Currently active users',
            labelNames: ['region']
        }));
    }

    private setupSLIMetrics() {
        // SLI 메트릭
        this.customMetrics.set('availability', new Gauge({
            name: 'service_availability_ratio',
            help: 'Service availability ratio'
        }));

        this.customMetrics.set('latency_sli', new Histogram({
            name: 'request_latency_sli_seconds',
            help: 'Request latency SLI',
            buckets: [0.1, 0.5, 1.0, 2.5, 5.0, 10.0]
        }));

        this.customMetrics.set('error_budget', new Gauge({
            name: 'error_budget_remaining',
            help: 'Remaining error budget percentage'
        }));
    }

    async collectBusinessMetrics(timeWindow: string = '5m') {
        // 사용자 활동 메트릭
        const activeUsers = await this.getActiveUsers();
        this.customMetrics.get('active_users')?.set(activeUsers);

        // 수익 메트릭
        const revenueMetrics = await this.getRevenueMetrics(timeWindow);
        this.updateRevenueMetrics(revenueMetrics);

        // 사용자 경험 메트릭
        const uxMetrics = await this.getUXMetrics();
        this.updateUXMetrics(uxMetrics);
    }
}

// 구조화된 로깅
package logging

import (
    "context"
    "go.uber.org/zap"
    "go.opentelemetry.io/otel/trace"
)

type StructuredLogger struct {
    logger *zap.Logger
}

func NewStructuredLogger() *StructuredLogger {
    config := zap.NewProductionConfig()
    config.EncoderConfig.TimeKey = "@timestamp"
    config.EncoderConfig.MessageKey = "message"

    logger, _ := config.Build()

    return &StructuredLogger{logger: logger}
}

func (l *StructuredLogger) InfoWithTrace(ctx context.Context, msg string, fields ...zap.Field) {
    span := trace.SpanFromContext(ctx)
    spanContext := span.SpanContext()

    if spanContext.IsValid() {
        fields = append(fields,
            zap.String("trace_id", spanContext.TraceID().String()),
            zap.String("span_id", spanContext.SpanID().String()),
        )
    }

    l.logger.Info(msg, fields...)
}

func (l *StructuredLogger) ErrorWithTrace(ctx context.Context, msg string, err error, fields ...zap.Field) {
    span := trace.SpanFromContext(ctx)
    spanContext := span.SpanContext()

    fields = append(fields,
        zap.Error(err),
        zap.String("error_type", fmt.Sprintf("%T", err)),
    )

    if spanContext.IsValid() {
        fields = append(fields,
            zap.String("trace_id", spanContext.TraceID().String()),
            zap.String("span_id", spanContext.SpanID().String()),
        )

        // 스패너에 에러 정보 추가
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
    }

    l.logger.Error(msg, fields...)
}

// 비즈니스 이벤트 로깅
func (l *StructuredLogger) LogBusinessEvent(ctx context.Context, event BusinessEvent) {
    fields := []zap.Field{
        zap.String("event_type", event.Type),
        zap.String("entity_id", event.EntityID),
        zap.String("user_id", event.UserID),
        zap.Time("event_time", event.Timestamp),
        zap.Any("metadata", event.Metadata),
    }

    l.InfoWithTrace(ctx, "Business event occurred", fields...)
}

# ELK 스택 자동화
from elasticsearch import Elasticsearch
from kibana_api import KibanaAPI
import asyncio

class LogAnalysisAutomation:
    def __init__(self):
        self.es = Elasticsearch(['http://elasticsearch:9200'])
        self.kibana = KibanaAPI('http://kibana:5601')
        self.pattern_detector = LogPatternDetector()

    async def setup_log_analysis_pipeline(self):
        # Elasticsearch 인덱스 템플릿 설정
        await self.setup_index_templates()

        # Kibana 대시보드 자동 생성
        await self.create_default_dashboards()

        # 알림 규칙 설정
        await self.setup_alerting_rules()

    async def analyze_error_patterns(self, time_range: str = '1h'):
        """에러 패턴 자동 분석"""
        query = {
            "query": {
                "bool": {
                    "must": [
                        {"term": {"level": "ERROR"}},
                        {"range": {"@timestamp": {"gte": f"now-{time_range}"}}}
                    ]
                }
            },
            "aggs": {
                "error_messages": {
                    "terms": {"field": "message.keyword", "size": 10}
                },
                "services": {
                    "terms": {"field": "service.name", "size": 20}
                }
            }
        }

        result = await self.es.search(index="logs-*", body=query)

        # 패턴 분석
        patterns = await self.pattern_detector.detect_patterns(result)

        # 자동 대응 제안
        recommendations = await self.generate_recommendations(patterns)

        return {
            'patterns': patterns,
            'recommendations': recommendations,
            'affected_services': result['aggregations']['services']['buckets']
        }

class IntelligentLogAggregation:
    def __init__(self):
        self.ml_model = LogClassificationModel()
        self.anomaly_detector = LogAnomalyDetector()

    async def smart_log_aggregation(self, logs: List[LogEntry]) -> AggregatedLogs:
        """지능형 로그 집계"""
        aggregated = AggregatedLogs()

        for log in logs:
            # 로그 분류
            category = await self.ml_model.classify(log)

            # 이상 탐지
            is_anomaly = await self.anomaly_detector.detect(log)

            # 심각도 평가
            severity = await self.assess_severity(log, category)

            aggregated.add_log(log, category, severity, is_anomaly)

        return aggregated

    async def generate_log_insights(self, aggregated_logs: AggregatedLogs) -> LogInsights:
        """로그 인사이트 생성"""
        return LogInsights(
            error_trends=await self.analyze_error_trends(aggregated_logs),
            performance_insights=await self.analyze_performance_patterns(aggregated_logs),
            security_alerts=await self.detect_security_issues(aggregated_logs),
            capacity_predictions=await self.predict_capacity_needs(aggregated_logs)
        )

class SLOManager {
    private sloRepository: SLORepository;
    private metricsCollector: MetricsCollector;
    private alertManager: AlertManager;

    async defineSLO(sloDefinition: SLODefinition): Promise<SLO> {
        // SLO 검증
        await this.validateSLO(sloDefinition);

        // SLI 메트릭 설정
        const sliMetrics = await this.setupSLIMetrics(sloDefinition);

        // 에러 버젯 계산
        const errorBudget = this.calculateErrorBudget(sloDefinition);

        const slo = new SLO({
            ...sloDefinition,
            sliMetrics,
            errorBudget,
            createdAt: new Date(),
        });

        await this.sloRepository.save(slo);

        // 자동 모니터링 설정
        await this.setupSLOMonitoring(slo);

        return slo;
    }

    async monitorSLOCompliance(): Promise<void> {
        const activeSLOs = await this.sloRepository.findActive();

        for (const slo of activeSLOs) {
            const compliance = await this.calculateCompliance(slo);

            // SLO 위반 확인
            if (compliance.isViolated) {
                await this.handleSLOViolation(slo, compliance);
            }

            // 에러 버젯 소진 확인
            if (compliance.errorBudgetDepletion > 0.9) {
                await this.alertManager.sendAlert({
                    type: 'error_budget_exhaustion',
                    slo: slo.id,
                    remainingBudget: compliance.remainingErrorBudget,
                });
            }

            // SLO 메트릭 업데이트
            await this.updateSLOMetrics(slo, compliance);
        }
    }

    private async calculateCompliance(slo: SLO): Promise<SLOCompliance> {
        const timeWindow = slo.timeWindow;
        const now = new Date();
        const startTime = new Date(now.getTime() - timeWindow * 1000);

        // SLI 값 계산
        const sliValue = await this.calculateSLI(slo, startTime, now);

        // 목표 대비 성과 계산
        const targetAchievement = sliValue / slo.target;

        // 에러 버젯 소진률 계산
        const errorBudgetDepletion = this.calculateErrorBudgetDepletion(slo, sliValue);

        return {
            sliValue,
            target: slo.target,
            isViolated: sliValue < slo.target,
            targetAchievement,
            errorBudgetDepletion,
            remainingErrorBudget: 1 - errorBudgetDepletion,
        };
    }
}

# HPA with custom metrics
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: microservice-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: microservice
  minReplicas: 3
  maxReplicas: 100
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1000"
  - type: External
    external:
      metric:
        name: queue_depth
        selector:
          matchLabels:
            queue: "user-processing"
      target:
        type: Value
        value: "10"
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 10
        periodSeconds: 60
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
      - type: Percent
        value: 50
        periodSeconds: 30
      - type: Pods
        value: 2
        periodSeconds: 60
      selectPolicy: Max