OpenTelemetry — 분산 시스템의 추적, 메트릭, 로그를 통합하는 표준

[서비스 A] → [서비스 B] → [서비스 C]
    ↓              ↓              ↓
    └──── 텔레메트리 데이터 ────────┘
                   ↓
           [OTel Collector]
           ↓       ↓       ↓
       [Jaeger] [Prometheus] [Loki]
                   ↓
              [Grafana]

// build.gradle
implementation 'org.springframework.boot:spring-boot-starter-actuator'

// 트레이싱
implementation 'io.micrometer:micrometer-tracing-bridge-otel'
implementation 'io.opentelemetry:opentelemetry-exporter-otlp'

// 메트릭
implementation 'io.micrometer:micrometer-registry-otlp'

# application.yml
management:
  tracing:
    sampling:
      probability: 1.0  # 모든 요청 추적 (운영에서는 0.1 등 낮게)
  otlp:
    tracing:
      endpoint: http://otel-collector:4318/v1/traces
    metrics:
      export:
        endpoint: http://otel-collector:4318/v1/metrics
        step: 30s

java -javaagent:opentelemetry-javaagent.jar \
  -Dotel.service.name=order-service \
  -Dotel.exporter.otlp.endpoint=http://otel-collector:4317 \
  -jar myapp.jar

Trace: 하나의 요청에 대한 전체 흐름
├── Span A: API Gateway (150ms)
│   ├── Span B: Order Service (100ms)
│   │   ├── Span C: DB Query (30ms)
│   │   └── Span D: Payment Service (50ms)
│   │       └── Span E: PG 연동 (40ms)
│   └── Span F: Notification (20ms)

@Service
@RequiredArgsConstructor
public class OrderService {
    private final ObservationRegistry observationRegistry;

    public Order processOrder(OrderRequest request) {
        // Observation API로 Span 생성
        return Observation.createNotStarted("order.process",
                observationRegistry)
            .lowCardinalityKeyValue("order.type",
                request.getType().name())
            .observe(() -> {
                // 이 블록이 Span으로 기록됨
                Order order = createOrder(request);
                validateInventory(order);
                processPayment(order);
                return order;
            });
    }
}

@Service
@RequiredArgsConstructor
public class PaymentService {
    private final Tracer tracer;  // Micrometer Tracer

    public PaymentResult charge(PaymentRequest request) {
        Span currentSpan = tracer.currentSpan();
        if (currentSpan != null) {
            // Span에 태그 추가
            currentSpan.tag("payment.gateway", request.getGateway());
            currentSpan.tag("payment.amount",
                String.valueOf(request.getAmount()));

            // Span 이벤트 추가
            currentSpan.event("결제 시작");
        }

        PaymentResult result = gateway.charge(request);

        if (currentSpan != null) {
            currentSpan.tag("payment.status", result.getStatus());
            currentSpan.event("결제 완료");
        }

        return result;
    }
}

Service A                          Service B
[Span A]                          [Span B]
  → HTTP 요청 →
  Header: traceparent=00-trace123-span456-01
                                   ← trace123과 연결된 Span B 생성

@Bean
public RestClient restClient(RestClient.Builder builder) {
    return builder
        .baseUrl("http://payment-service:8080")
        .build();
    // Micrometer가 자동으로 tracing 헤더 추가
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    send_batch_size: 1024
    timeout: 5s
  memory_limiter:
    check_interval: 1s
    limit_mib: 512

exporters:
  # 트레이스 → Jaeger/Tempo
  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true

  # 메트릭 → Prometheus
  prometheus:
    endpoint: 0.0.0.0:8889

  # 로그 → Loki
  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, memory_limiter]
      exporters: [otlp/tempo]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [batch]
      exporters: [loki]

services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"   # gRPC
      - "4318:4318"   # HTTP
      - "8889:8889"   # Prometheus 메트릭

  tempo:
    image: grafana/tempo:latest
    ports:
      - "3200:3200"

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true

# application.yml
logging:
  pattern:
    console: "%d{HH:mm:ss.SSS} [%thread] [traceId=%X{traceId}] %-5level %logger{36} - %msg%n"

10:23:45.123 [http-nio-8080-1] [traceId=abc123] INFO OrderService - 주문 생성 시작
10:23:45.234 [http-nio-8080-1] [traceId=abc123] INFO PaymentService - 결제 처리
10:23:45.345 [http-nio-8080-1] [traceId=abc123] INFO OrderService - 주문 완료

management:
  tracing:
    sampling:
      probability: 0.1  # 10%만 샘플링

# 또는 커스텀 샘플러
# 에러가 발생한 요청은 항상 추적, 나머지는 10%

# 서비스 식별 정보
management:
  opentelemetry:
    resource-attributes:
      service.name: order-service
      service.version: 1.2.0
      deployment.environment: production