Grafana Alloy 指标详解

1

alloy_build_info{branch="HEAD",goarch="amd64",goos="linux",goversion="go1.24.4",revision="30277f7",tags="netgo,builtinassets,promtail_journal_enabled",version="v1.9.2"} 1

1

alloy_component_controller_evaluating{controller_id="",controller_path="/"} 0

1

alloy_component_controller_running_components{controller_id="",controller_path="/",health_type="healthy"} 3

1

alloy_component_dependencies_wait_seconds_bucket{controller_id="",controller_path="/",le="0.005"} 2

1

alloy_component_evaluation_queue_size{controller_id="",controller_path="/"} 1

1

alloy_component_evaluation_seconds_bucket{controller_id="",controller_path="/",le="0.005"} 2

1

alloy_config_hash{cluster_name="",sha256="d9c2272a88ea08d0ee0e95f352077329357cad4408c066378360bf27808cb2ac"} 1

1

alloy_config_last_load_success_timestamp_seconds 1.7526530693099432e+09

1

alloy_config_last_load_successful 1

1

alloy_config_load_failures_total 0

1

alloy_labelstore_global_ids_count 0

1

alloy_labelstore_last_stale_check_timestamp 1.752717869e+09

1
2

alloy_resources_machine_rx_bytes_total 3.87777615344e+11
alloy_resources_machine_tx_bytes_total 1.06780165711e+11

1
2
3
4

alloy_resources_process_cpu_seconds_total 35217.79
alloy_resources_process_resident_memory_bytes 2.31657472e+08
alloy_resources_process_virtual_memory_bytes 3.355598848e+09
alloy_resources_process_start_time_seconds 1.752653069e+09

1
2
3

go_gc_duration_seconds{quantile="0.5"} 0.000149737
go_gc_gogc_percent 100
go_gc_gomemlimit_bytes 9.223372036854776e+18

1
2
3

go_memstats_alloc_bytes 5.0047336e+07
go_memstats_heap_objects 233785
go_memstats_next_gc_bytes 5.3414458e+07

1
2
3

go_goroutines 96
go_threads 24
go_sched_gomaxprocs_threads 16

1
2

otelcol_exporter_sent_log_records_total{component_id="otelcol.exporter.otlphttp.default"} 477974
otelcol_exporter_send_failed_log_records_total{component_id="otelcol.exporter.otlphttp.default"} 0

1
2

otelcol_exporter_queue_capacity{data_type="logs"} 1000
otelcol_exporter_queue_size{data_type="logs"} 0

1
2
3

otelcol_processor_batch_batch_send_size_bucket{le="2000"} 13
otelcol_processor_batch_batch_send_size_sum 477974
otelcol_processor_batch_batch_send_size_count 87

1
2
3

http_client_duration_milliseconds_bucket{le="250"} 87
http_client_request_size_bytes_total 2.09639717e+08
http_client_response_size_bytes_total 0

1
2

loki_logql_querystats_duplicates_total 0
loki_logql_querystats_ingester_sent_lines_total 0

1
2
3

loki_querier_index_cache_gets_total 0
loki_querier_index_cache_hits_total 0
loki_querier_index_cache_puts_total 0

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106

groups:
- name: alloy-core-alerts
  rules:
  # 配置加载失败
  - alert: AlloyConfigLoadFailure
    expr: alloy_config_last_load_successful == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: "Alloy 配置加载失败"
      description: "Alloy 实例 {{ $labels.instance }} 配置加载失败，请检查配置文件语法"
      
  # 组件不健康
  - alert: AlloyUnhealthyComponents
    expr: alloy_component_controller_running_components{health_type="unhealthy"} > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "发现不健康的 Alloy 组件"
      description: "Alloy 实例 {{ $labels.instance }} 有 {{ $value }} 个不健康组件"
      
  # 内存使用过高
  - alert: AlloyHighMemoryUsage
    expr: alloy_resources_process_resident_memory_bytes > 500 * 1024 * 1024
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Alloy 内存使用过高"
      description: "Alloy 实例 {{ $labels.instance }} 内存使用 {{ $value | humanize1024 }}B，超过 500MB"

- name: alloy-otel-alerts
  rules:
  # OpenTelemetry 日志发送失败
  - alert: OTelLogSendFailure
    expr: rate(otelcol_exporter_send_failed_log_records_total[5m]) > 0
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "OpenTelemetry 日志发送失败"
      description: "组件 {{ $labels.component_id }} 日志发送失败率: {{ $value | humanizePercentage }}"
      
  # OpenTelemetry 队列积压
  - alert: OTelQueueBacklog
    expr: otelcol_exporter_queue_size / otelcol_exporter_queue_capacity > 0.8
    for: 3m
    labels:
      severity: warning
    annotations:
      summary: "OpenTelemetry 导出器队列积压"
      description: "组件 {{ $labels.component_id }} 队列使用率 {{ $value | humanizePercentage }}，接近容量上限"
      
  # HTTP 请求延迟过高
  - alert: HTTPClientHighLatency
    expr: histogram_quantile(0.95, rate(http_client_duration_milliseconds_bucket[5m])) > 1000
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "HTTP 客户端延迟过高"
      description: "组件 {{ $labels.component_id }} HTTP 请求 P95 延迟 {{ $value }}ms，超过 1000ms"
      
  # 批处理效率低下
  - alert: BatchProcessorInefficient
    expr: rate(otelcol_processor_batch_timeout_trigger_send_total[5m]) / rate(otelcol_processor_batch_batch_send_size_count[5m]) > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "批处理器效率低下"
      description: "组件 {{ $labels.component_id }} 超时触发批次发送比例 {{ $value | humanizePercentage }}，建议调整批次大小或超时时间"

- name: alloy-system-alerts
  rules:
  # Goroutine 泄漏
  - alert: AlloyGoroutineLeak
    expr: go_goroutines > 200
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: "Alloy Goroutine 数量异常"
      description: "Alloy 实例 {{ $labels.instance }} Goroutine 数量 {{ $value }}，可能存在泄漏"
      
  # GC 时间过长
  - alert: AlloyHighGCTime
    expr: rate(go_gc_duration_seconds_sum[5m]) / rate(go_gc_duration_seconds_count[5m]) > 0.01
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Alloy GC 时间过长"
      description: "Alloy 实例 {{ $labels.instance }} 平均 GC 时间 {{ $value | humanizeDuration }}，影响性能"
      
  # 网络流量异常
  - alert: AlloyNetworkTrafficAnomaly
    expr: rate(alloy_resources_machine_rx_bytes_total[5m]) > 100 * 1024 * 1024
    for: 5m
    labels:
      severity: info
    annotations:
      summary: "Alloy 网络接收流量异常"
      description: "Alloy 实例 {{ $labels.instance }} 网络接收速率 {{ $value | humanize1024 }}B/s，请关注"

1
2
3

# 分析批处理触发机制比例
rate(otelcol_processor_batch_timeout_trigger_send_total[5m]) / 
rate(otelcol_processor_batch_batch_send_size_count[5m])

1
2
3
4
5
6
7

# HTTP 请求延迟分析
histogram_quantile(0.95, 
  rate(http_client_duration_milliseconds_bucket[5m])
)

# 请求大小趋势
rate(http_client_request_size_bytes_total[5m])

1
2
3
4
5
6
7
8
9

# 内存使用趋势预测
predict_linear(
  alloy_resources_process_resident_memory_bytes[1h], 
  3600 * 24
)

# GC 效率分析
rate(go_gc_duration_seconds_sum[5m]) / 
rate(go_gc_duration_seconds_count[5m])

1
2
3
4
5

# 队列使用率监控
otelcol_exporter_queue_size / otelcol_exporter_queue_capacity

# 队列积压预警
otelcol_exporter_queue_size > 800

1
2
3
4
5

# Kubernetes 事件处理速率
rate(prometheus_sd_kubernetes_events_total[5m])

# API 调用失败率
rate(prometheus_sd_kubernetes_failures_total[5m])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# 1. 检查整体健康状态
alloy_config_last_load_successful
alloy_component_controller_running_components{health_type="healthy"}

# 2. 分析处理链路
rate(otelcol_exporter_sent_log_records_total[5m])
rate(otelcol_exporter_send_failed_log_records_total[5m])

# 3. 检查资源状态
go_goroutines
alloy_resources_process_resident_memory_bytes

# 4. 分析网络状况
rate(http_client_duration_milliseconds_sum[5m]) / 
rate(http_client_duration_milliseconds_count[5m])