Monitoring

let meter_provider = init_metrics(
    String::from("scheduler"),
    instance_id,
    deployment_env.clone(),
)
.expect("failed to initialize metrics");

/// Global OTel meter for the kafka consumer
pub static CONSUMER_METER: LazyLock<Meter> =
    LazyLock::new(|| opentelemetry::global::meter("boom-consumer-meter"));

/// Global OTel meter for the kafka producer
pub static PRODUCER_METER: LazyLock<Meter> =
    LazyLock::new(|| opentelemetry::global::meter("boom-producer-meter"));

/// Global OTel meter for the scheduler
pub static SCHEDULER_METER: LazyLock<Meter> =
    LazyLock::new(|| opentelemetry::global::meter("boom-scheduler-meter"));

let exporter = opentelemetry_otlp::MetricExporter::builder()
    .with_temporality(Temporality::Cumulative)
    .with_tonic()
    .with_endpoint("https://localhost:4317/v1/metrics")
    .build()?;

let meter_provider = SdkMeterProvider::builder()
    .with_resource(resource)
    .with_periodic_exporter(exporter)  // Exports every 60 seconds
    .build();

groups:
  - name: boom
    interval: 30s
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(alert_worker_alert_processed_total{status="error"}[5m]))
          /
          sum(rate(alert_worker_alert_processed_total[5m]))
          > 0.05
        for: 5m
        annotations:
          summary: "Alert worker error rate above 5%"
          
      - alert: NoAlertsProcessed
        expr: |
          rate(kafka_consumer_alert_processed_total[5m]) == 0
        for: 10m
        annotations:
          summary: "No alerts consumed in 10 minutes"
          
      - alert: FilterWorkerStalled
        expr: |
          filter_worker_active > 0
          and
          rate(filter_worker_batch_processed_total[5m]) == 0
        for: 5m
        annotations:
          summary: "Filter workers stalled with active alerts"

if let Err(error) = meter_provider.shutdown() {
    log_error!(WARN, error, "failed to shut down the meter provider");
}