Monitoring Health

automaton-cli status

Name: MyAutomaton
Address: 0x742d35Cc6634C0532925a3b844Bc9e7595f0bEb1
State: running
Credits: $12.45
USDC Balance: $50.00
Survival Tier: normal
Uptime: 3h 24m
Last Heartbeat: 2 minutes ago
Turns: 147

export type AgentState =
  | "setup"         // Initial configuration
  | "waking"        // Starting up
  | "running"       // Normal operation
  | "sleeping"      // Idle, waiting for input
  | "low_compute"   // Low credits, reduced activity
  | "critical"      // Zero credits, survival mode
  | "dead";         // Non-operational

┌──────┐
│ setup│
└───┬──┘
    ▼
┌───────┐
│waking │
└───┬───┘
    ▼
┌────────┐    ┌──────────┐
│running │◄──►│ sleeping │
└───┬────┘    └──────────┘
    │
    ▼ (credits low)
┌─────────────┐
│ low_compute │
└──────┬──────┘
       │
       ▼ (credits exhausted)
   ┌──────────┐
   │ critical │
   └─────┬────┘
         │
         ▼ (negative balance)
     ┌──────┐
     │ dead │
     └──────┘

{
  "timestamp": "2026-03-03T10:15:30.123Z",
  "level": "info",
  "module": "heartbeat.scheduler",
  "message": "Heartbeat task completed",
  "context": {
    "taskName": "check_inbox",
    "durationMs": 234
  }
}

# All logs
automaton-cli logs

# Filter by level
automaton-cli logs --level error

# Follow live
automaton-cli logs --follow

# Search content
automaton-cli logs --grep "credit"

# Last 100 lines
automaton-cli logs --tail 100

import { getMetrics } from "./observability/metrics.js";

const metrics = getMetrics();

// Increment counter
metrics.increment("turns_total", { state: "running" });

// Set gauge
metrics.gauge("balance_cents", creditsCents);

// Record histogram value
metrics.histogram("turn_duration_ms", durationMs);

// Query metrics
const turnCount = metrics.getCounter("turns_total");
const balance = metrics.getGauge("balance_cents");

# Current metrics snapshot
automaton-cli metrics

# Specific metric
automaton-cli metrics --name balance_cents

# Export to JSON
automaton-cli metrics --format json > metrics.json

{
  name: "balance_below_reserve",
  severity: "critical",
  message: "Balance is below minimum reserve (1000 cents)",
  cooldownMs: 5 * 60 * 1000,
  condition: (metrics) => {
    const balance = metrics.gauges.get("balance_cents") ?? Infinity;
    return balance < 1000;
  },
}

{
  name: "heartbeat_high_failure_rate",
  severity: "warning",
  message: "Heartbeat task failure rate exceeds 20%",
  cooldownMs: 15 * 60 * 1000,
  condition: (metrics) => {
    const failures = metrics.counters.get("heartbeat_task_failures_total") ?? 0;
    const successes = metrics.counters.get("heartbeat_task_successes_total") ?? 0;
    const total = failures + successes;
    if (total === 0) return false;
    return failures / total > 0.2;
  },
}

{
  name: "context_near_capacity",
  severity: "warning",
  message: "Context token usage above 90% of budget",
  cooldownMs: 10 * 60 * 1000,
  condition: (metrics) => {
    const tokens = metrics.gauges.get("context_tokens_total") ?? 0;
    return tokens > 90_000; // 100k default budget
  },
}

{
  name: "zero_turns_last_hour",
  severity: "critical",
  message: "No successful turns in the last hour",
  cooldownMs: 60 * 60 * 1000,
  condition: (metrics) => {
    const turnsLastHour = metrics.gauges.get("turns_last_hour") ?? -1;
    if (turnsLastHour >= 0) return turnsLastHour === 0;
    return false;
  },
}

# Active alerts
automaton-cli alerts

# Alert history
automaton-cli alerts --history

entries:
  - name: check_balance
    schedule: "0 */5 * * * *"  # Every 5 minutes
    task: check_balance
    enabled: true

  - name: check_inbox
    schedule: "0 */2 * * * *"  # Every 2 minutes
    task: check_inbox
    enabled: true

  - name: self_reflect
    schedule: "0 0 */6 * * *"  # Every 6 hours
    task: self_reflect
    enabled: true

automaton-cli heartbeat status

Task               Schedule        Last Run         Next Run         Status
check_balance      */5 * * * *     2 minutes ago    3 minutes        OK
check_inbox        */2 * * * *     1 minute ago     1 minute         OK
self_reflect       0 */6 * * *     4 hours ago      2 hours          OK

export interface AgentHealthStatus {
  address: string;
  name: string;
  status: string;
  healthy: boolean;
  lastHeartbeat: string | null;
  currentTaskId: string | null;
  creditBalance: number | null;
  errorRate: number;
  issues: string[];
}

# View child health
automaton-cli children health

# Trigger auto-heal
automaton-cli children heal

SELECT 
  AVG(cost_cents) as avg_cost,
  AVG(json_extract(token_usage, '$.totalTokens')) as avg_tokens,
  COUNT(*) as turn_count
FROM turns
WHERE timestamp > datetime('now', '-1 day');

SELECT 
  model,
  SUM(cost_cents) as total_cost,
  SUM(input_tokens) as total_input,
  SUM(output_tokens) as total_output,
  AVG(latency_ms) as avg_latency
FROM inference_costs
GROUP BY model;

SELECT 
  name,
  COUNT(*) as call_count,
  AVG(duration_ms) as avg_duration,
  SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) as error_count
FROM tool_calls
GROUP BY name
ORDER BY call_count DESC;

# Database size
ls -lh ~/.automaton/state.db

# Integrity check
sqlite3 ~/.automaton/state.db "PRAGMA integrity_check;"

# Table sizes
sqlite3 ~/.automaton/state.db "SELECT name, SUM(pgsize) as size FROM dbstat GROUP BY name;"

# Compact database
sqlite3 ~/.automaton/state.db "VACUUM;"

# Analyze query planner
sqlite3 ~/.automaton/state.db "ANALYZE;"

automaton-cli sandbox stats

# Process memory
ps aux | grep automaton

# Database memory
du -sh ~/.automaton/

automaton-cli export --output state-dump.json

automaton-cli health-report