OpenTelemetry Collector & eBPF trong Production: Tracing, Metrics & Prometheus Pipelines Không Cần Code

# otel-collector-agent-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector-agent
  namespace: observability
  labels:
    app: otel-collector-agent
spec:
  selector:
    matchLabels:
      app: otel-collector-agent
  template:
    metadata:
      labels:
        app: otel-collector-agent
    spec:
      serviceAccountName: otel-collector-agent
      hostNetwork: true # Required for eBPF agents to capture all network traffic
      dnsPolicy: ClusterFirstWithHostNet # Required with hostNetwork
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.90.1 # Use contrib for more receivers/processors
          command: ["/otelcol", "--config=/conf/otel-collector-config.yaml"]
          securityContext:
            privileged: true # Required for eBPF agent capabilities
            runAsUser: 0
            runAsGroup: 0
          volumeMounts:
            - name: otel-collector-config
              mountPath: /conf
            - name: varlog
              mountPath: /var/log # For host log collection
              readOnly: true
            - name: varlibdockercontainers
              mountPath: /var/lib/docker/containers # For Docker container logs
              readOnly: true
            - name: sysfs
              mountPath: /sys # For eBPF kernel access
              readOnly: true
            - name: procfs
              mountPath: /proc # For eBPF process info
              readOnly: true
          env:
            - name: KUBERNETES_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          ports:
            - name: otlp-grpc
              containerPort: 4317
              hostPort: 4317 # Expose OTLP gRPC on host
            - name: otlp-http
              containerPort: 4318
              hostPort: 4318 # Expose OTLP HTTP on host
            - name: prometheus
              containerPort: 8888 # Collector's own metrics
            - name: health-check
              containerPort: 13133
          livenessProbe:
            httpGet:
              path: /health
              port: 13133
            initialDelaySeconds: 5
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /health
              port: 13133
            initialDelaySeconds: 5
            periodSeconds: 10
          resources:
            limits:
              cpu: 500m
              memory: 512Mi
            requests:
              cpu: 100m
              memory: 256Mi
      volumes:
        - name: otel-collector-config
          configMap:
            name: otel-collector-agent-config
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: sysfs
          hostPath:
            path: /sys
        - name: procfs
          hostPath:
            path: /proc
---
# otel-collector-agent-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-collector-agent
  namespace: observability
---
# otel-collector-agent-clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector-agent
rules:
  - apiGroups: [""]
    resources: ["nodes", "nodes/proxy", "pods", "services", "endpoints"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["apps"]
    resources: ["replicasets"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["extensions"]
    resources: ["daemonsets", "deployments", "replicasets"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "patch"]
  - apiGroups: ["policy"]
    resources: ["podsecuritypolicies"]
    verbs: ["use"]
    resourceNames:
      - otel-collector-agent # If using PSPs
---
# otel-collector-agent-clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector-agent
subjects:
  - kind: ServiceAccount
    name: otel-collector-agent
    namespace: observability
roleRef:
  kind: ClusterRole
  name: otel-collector-agent
  apiGroup: rbac.authorization.k8s.io

# otel-collector-agent-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-agent-config
  namespace: observability
data:
  otel-collector-config.yaml: |
    receivers:
      # OTLP receiver for applications and eBPF agents exporting directly
      otlp:
        protocols:
          grpc:
          http:

      # Host metrics receiver
      hostmetrics:
        collection_interval: 10s
        scrapers:
          cpu:
          memory:
          disk:
          filesystem:
          network:
          load:
          paging:
          processes:

      # Filelog receiver for host logs (e.g., systemd, kernel logs)
      # Requires hostPath mount for /var/log
      filelog:
        include:
          - /var/log/*.log
          - /var/log/*/*.log
        start_at: beginning
        poll_interval: 1s
        operators:
          - type: json_parser
            output: json
          - type: regex_parser
            regex: '^(?P<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)\s(?P<level>\w+)\s(?P<message>.*)$'
            output: json
            timestamp:
              parse_from: time
              layout: '%Y-%m-%dT%H:%M:%S.%LZ'
            severity:
              parse_from: level
              preset: syslog
          - type: move
            from: json.message
            to: body
          - type: remove
            field: json

      # Kubernetes Container Logs (Docker/CRI-O)
      # Requires hostPath mount for /var/lib/docker/containers
      # This is a basic example; for production, consider a dedicated log agent or more robust K8s integration
      filelog/container:
        include:
          - /var/lib/docker/containers/*/*-json.log
        start_at: beginning
        poll_interval: 1s
        operators:
          - type: json_parser
            output: json
          - type: move
            from: json.log
            to: body
          - type: move
            from: json.stream
            to: attributes.log.stream
          - type: move
            from: json.time
            to: attributes.log.time
          - type: add
            field: attributes.log.file.path
            value: "{{ .file.path }}"
          - type: add
            field: attributes.container.id
            value: "{{ .file.name }}" # Extract container ID from filename
          - type: add
            field: attributes.k8s.node.name
            value: "${KUBERNETES_NODE_NAME}" # Injected via env var
          - type: regex_parser
            regex: '^/var/lib/docker/containers/(?P<container_id>[a-f0-9]{64})/(?P<container_id_short>[a-f0-9]{12})-json.log$'
            parse_from: attributes.log.file.path
            output: attributes
          - type: remove
            field: json

    processors:
      # Batching for efficiency
      batch:
        send_batch_size: 1024
        timeout: 5s

      # Resource detection for Kubernetes metadata
      resourcedetection:
        detectors: ["system", "env", "kubernetes"]
        timeout: 2s
        kubernetes:
          pod_association:
            - from: "ip"
            - from: "cgroup"
          exclude:
            pods:
              - name: "otel-collector-agent" # Exclude self-instrumentation

      # Memory limiter to prevent OOMs
      memory_limiter:
        check_interval: 1s
        limit_mib: 256
        spike_limit_mib: 64

      # Tail-based sampling (example, typically done at gateway)
      # For agent, head-based is more common if sampling is needed here
      # This example is for demonstration, usually agents forward all data.
      # tail_sampling:
      #   decision_wait: 10s
      #   num_traces: 100000
      #   expected_new_traces_per_sec: 100
      #   policies:
      #     [
      #       {
      #         name: "error-policy",
      #         type: "status_code",
      #         status_code: { status_codes: ["ERROR"] }
      #       },
      #       {
      #         name: "latency-policy",
      #         type: "latency",
      #         latency: { threshold_ms: 500 }
      #       }
      #     ]

    exporters:
      # Export to a central OpenTelemetry Collector Gateway
      otlp:
        endpoint: "otel-collector-gateway.observability.svc.cluster.local:4317" # Internal K8s service
        tls:
          insecure: true # Use mTLS in production

      # Optional: Prometheus exporter for collector's own metrics
      prometheus:
        endpoint: "0.0.0.0:8888"

    service:
      telemetry:
        metrics:
          address: 0.0.0.0:8888
      pipelines:
        traces:
          receivers: [otlp]
          processors: [resourcedetection, batch, memory_limiter] # Add tail_sampling if needed
          exporters: [otlp]
        metrics:
          receivers: [otlp, hostmetrics]
          processors: [resourcedetection, batch, memory_limiter]
          exporters: [otlp]
        logs:
          receivers: [otlp, filelog, filelog/container]
          processors: [resourcedetection, batch, memory_limiter]
          exporters: [otlp]

# otel-collector-gateway-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector-gateway
  namespace: observability
  labels:
    app: otel-collector-gateway
spec:
  replicas: 2 # Scale as needed
  selector:
    matchLabels:
      app: otel-collector-gateway
  template:
    metadata:
      labels:
        app: otel-collector-gateway
    spec:
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.90.1
          command: ["/otelcol", "--config=/conf/otel-collector-config.yaml"]
          volumeMounts:
            - name: otel-collector-config
              mountPath: /conf
          ports:
            - name: otlp-grpc
              containerPort: 4317
            - name: otlp-http
              containerPort: 4318
            - name: prometheus
              containerPort: 8888
            - name: health-check
              containerPort: 13133
          livenessProbe:
            httpGet:
              path: /health
              port: 13133
            initialDelaySeconds: 5
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /health
              port: 13133
            initialDelaySeconds: 5
            periodSeconds: 10
          resources:
            limits:
              cpu: 1000m
              memory: 2Gi
            requests:
              cpu: 200m
              memory: 512Mi
      volumes:
        - name: otel-collector-config
          configMap:
            name: otel-collector-gateway-config
---
# otel-collector-gateway-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: otel-collector-gateway
  namespace: observability
spec:
  selector:
    app: otel-collector-gateway
  ports:
    - name: otlp-grpc
      protocol: TCP
      port: 4317
      targetPort: 4317
    - name: otlp-http
      protocol: TCP
      port: 4318
      targetPort: 4318
    - name: prometheus
      protocol: TCP
      port: 8888
      targetPort: 8888

# otel-collector-gateway-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-gateway-config
  namespace: observability
data:
  otel-collector-config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
          http:

    processors:
      batch:
        send_batch_size: 1024
        timeout: 5s

      memory_limiter:
        check_interval: 1s
        limit_mib: 1500
        spike_limit_mib: 500

      # Tail-based sampling for traces
      # This is crucial for reducing trace volume while retaining important traces.
      tail_sampling:
        decision_wait: 10s # How long to wait for all spans of a trace
        num_traces: 100000 # Max number of traces to keep in memory for sampling
        expected_new_traces_per_sec: 1000 # Estimate of new traces per second
        policies:
          - name: "always-sample"
            type: "always_on" # Always sample 100% of traces (for dev/low traffic)
          - name: "error-policy"
            type: "status_code"
            status_code: { status_codes: ["ERROR", "UNSET"] } # Sample traces with errors
          - name: "latency-policy"
            type: "latency"
            latency: { threshold_ms: 500 } # Sample traces exceeding 500ms
          - name: "probabilistic-policy"
            type: "probabilistic"
            probabilistic: { sampling_percentage: 10 } # Sample 10% of remaining traces

      # Attributes processor for general attribute manipulation
      attributes:
        actions:
          - key: "service.namespace"
            action: "insert"
            value: "default" # Default namespace if not present
          - key: "k8s.cluster.name"
            action: "insert"
            value: "my-prod-cluster" # Add cluster name
          - key: "host.ip"
            action: "delete" # Remove sensitive host IP if not needed downstream

      # Resource processor for adding/modifying resource attributes
      resource:
        attributes:
          - key: "cloud.provider"
            value: "aws"
            action: "insert"
          - key: "cloud.region"
            value: "us-east-1"
            action: "insert"

    exporters:
      # Export traces to Jaeger
      jaeger:
        endpoint: "jaeger-collector.observability.svc.cluster.local:14250" # gRPC
        tls:
          insecure: true

      # Export metrics to Prometheus remote write (e.g., Mimir, Thanos)
      prometheusremotewrite:
        endpoint: "http://prometheus-mimir-gateway.observability.svc.cluster.local:9009/api/v1/push"
        headers:
          X-Scope-OrgID: "locionic"
        # auth:
        #   oauth2:
        #     client_id: "..."
        #     client_secret: "..."
        #     token_url: "..."

      # Export logs to Loki
      loki:
        endpoint: "http://loki.observability.svc.cluster.local:3100/loki/api/v1/push"
        tls:
          insecure: true
        # auth:
        #   basic:
        #     username: "..."
        #     password: "..."
        labels:
          attributes:
            - host.name
            - k8s.namespace.name
            - k8s.pod.name
            - service.name
          resource:
            - k8s.node.name
            - k8s.cluster.name

      # Optional: Prometheus exporter for collector's own metrics
      prometheus:
        endpoint: "0.0.0.0:8888"

    service:
      telemetry:
        metrics:
          address: 0.0.0.0:8888
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch, tail_sampling, attributes, resource]
          exporters: [jaeger]
        metrics:
          receivers: [otlp]
          processors: [memory_limiter, batch, attributes, resource]
          exporters: [prometheusremotewrite]
        logs:
          receivers: [otlp]
          processors: [memory_limiter, batch, attributes, resource]
          exporters: [loki]

# Example Prometheus scrape config for OTel Collector Gateway
# Add this to your Prometheus server's scrape_configs
- job_name: 'otel-collector-gateway'
  kubernetes_sd_configs:
    - role: endpoints
      namespaces:
        names: ['observability']
  relabel_configs:
    - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
      action: keep
      regex: otel-collector-gateway;prometheus
    - source_labels: [__meta_kubernetes_pod_name]
      target_label: kubernetes_pod_name
      action: replace
    - source_labels: [__meta_kubernetes_namespace]
      target_label: kubernetes_namespace
      action: replace