Files
3x-ui-agent/internal/web/service/probe/history_test.go
T
Rogue 3e052c42b8 feat(probe): history store + whole-machine traffic ledger in probe.db (item 7 B2)
New internal/web/service/probe package keeps probe history in its own
probe.db beside the panel DB (single connection, WAL), so history
writes never contend with the panel write lock.

- Minute rows per hub semantics: a held-reading rhythm books traffic
  against each reading own arrival; minute buckets merge (means,
  rounded int means, peaks) and the stored net rate derives from the
  kernel counters between row marks, falling back to agent rates at
  epoch breaks.
- probe_traffic ledger: accumulate is a line port of hub traffic.rs
  (baseline alignment, same-boot shrink books 0, epoch change books 0,
  unseen-boot exception books the whole counters-since-boot measured
  against the master last-seen stamp, per-direction period resets,
  saturating adds); reads gate stale day/month stamps to zero.
- probe_meta rows are ensured for every reporting agent (B3 edits
  them in place); probe_ping_* tables are created now, written by B4.
- Hourly roll_up behind a watermark (hour-aligned, late grace,
  pre-retention hours skipped unfoldered) and watermark-guarded
  chunked prune (minute tier 7 days or the retention window when
  shorter; hourly tier to retention_days, default 7, cap 365).
- Wiring: heartbeat files probe samples after the snapshot update
  (last-seen captured before touch); housekeeping hangs off the
  existing robfig/cron in web.startTask (@hourly + one pass a minute
  after startup, mirroring the hub boot pass); Server.stop books held
  readings. Every failure path logs and degrades - a broken probe.db
  never fails a heartbeat or the panel.
2026-10-22 19:43:59 +00:00

176 lines
6.5 KiB
Go

package probe
import (
"testing"
"time"
)
type metricRow struct {
ts int64
cpu, cpuMax float64
mem, swap, disk int64
netRx, netTx, netRxMax, netTxMax int64
tcp, udp, procs int64
}
func readMetricRows(t *testing.T, st *Store, agent int64) []metricRow {
t.Helper()
rows, err := st.db.Query(`
SELECT ts, cpu, mem_used, swap_used, disk_used, net_rx, net_tx,
tcp, udp, procs, net_rx_max, net_tx_max, cpu_max
FROM probe_metric WHERE agent_id = ? ORDER BY ts`, agent)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var out []metricRow
for rows.Next() {
var r metricRow
if err := rows.Scan(&r.ts, &r.cpu, &r.mem, &r.swap, &r.disk, &r.netRx, &r.netTx,
&r.tcp, &r.udp, &r.procs, &r.netRxMax, &r.netTxMax, &r.cpuMax); err != nil {
t.Fatal(err)
}
out = append(out, r)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return out
}
func rec(t *testing.T, st *Store, agent int64, smp Sample, at time.Time) {
t.Helper()
if err := st.Record(agent, smp, at, 0); err != nil {
t.Fatalf("record: %v", err)
}
}
// Samples landing in the same minute bucket merge into one row when
// the next minute's sample flushes it: mean fields average (ints
// rounded), peaks max, and the stored net rate derives from the
// kernel counters between the row marks, not from the agent's rates.
func TestMinuteBucketMergeAndDerivedRates(t *testing.T) {
st := newTestStore(t)
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
mk := func(cpu float32, rxTot uint64, rate uint64, mem uint64) Sample {
return Sample{BootID: "k/fp", CPU: cpu, MemUsed: mem, NetRxTotal: rxTot, NetTxTotal: rxTot * 2,
NetRx: rate, NetTx: rate * 2, TCP: 3, UDP: 2, Procs: 10}
}
rec(t, st, 1, mk(10, 60_000, 1000, 100), base) // minute M, first sample
rec(t, st, 1, mk(20, 90_000, 1000, 200), base.Add(30*time.Second)) // minute M again
rec(t, st, 1, mk(30, 150_000, 1000, 300), base.Add(time.Minute)) // minute M+1: flushes
rows := readMetricRows(t, st, 1)
if len(rows) != 1 {
t.Fatalf("want 1 row, got %d", len(rows))
}
r := rows[0]
if r.ts != base.Add(time.Minute).Unix() {
t.Fatalf("row stamped %d, want flushing minute %d", r.ts, base.Add(time.Minute).Unix())
}
if r.cpu != 20 { // mean of 10,20,30 (flushing sample included)
t.Fatalf("cpu mean=%v, want 20", r.cpu)
}
if r.mem != 200 {
t.Fatalf("mem mean=%d, want 200", r.mem)
}
if r.cpuMax != 30 {
t.Fatalf("cpu_max=%v, want 30", r.cpuMax)
}
if r.netRx != 1500 { // (150000-60000) / 60s between marks
t.Fatalf("net_rx=%d, want 1500 (counter-derived)", r.netRx)
}
if r.netRxMax != 1000 {
t.Fatalf("net_rx_max=%d, want agent peak 1000", r.netRxMax)
}
// Next minute: single-sample row; rate from the new mark.
rec(t, st, 1, mk(40, 210_000, 700, 400), base.Add(2*time.Minute))
rows = readMetricRows(t, st, 1)
if len(rows) != 2 {
t.Fatalf("want 2 rows, got %d", len(rows))
}
r = rows[1]
if r.cpu != 40 || r.netRx != 1000 || r.netRxMax != 700 {
t.Fatalf("second row wrong: %+v", r)
}
}
// At an epoch break the counter mark no longer applies, so the row
// keeps the agent's own reported rates.
func TestMinuteRowEpochBreakKeepsAgentRates(t *testing.T) {
st := newTestStore(t)
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
rec(t, st, 1, Sample{BootID: "k1/fp", NetRxTotal: 1000, NetTxTotal: 1000, NetRx: 111, NetTx: 222}, base)
rec(t, st, 1, Sample{BootID: "k2/fp", NetRxTotal: 500, NetTxTotal: 500, NetRx: 333, NetTx: 444}, base.Add(time.Minute))
rows := readMetricRows(t, st, 1)
if len(rows) != 1 {
t.Fatalf("want 1 row, got %d", len(rows))
}
if rows[0].netRx != 333 || rows[0].netTx != 444 {
t.Fatalf("epoch break must keep agent rates: %+v", rows[0])
}
}
// Samples without a boot_id still file metric rows (hub files rows
// from any well-formed report) but never touch the traffic ledger.
func TestRecordWithoutBootFilesRowOnly(t *testing.T) {
st := newTestStore(t)
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
rec(t, st, 1, Sample{CPU: 5, MemUsed: 42}, base)
rec(t, st, 1, Sample{CPU: 7, MemUsed: 43}, base.Add(time.Minute))
if rows := readMetricRows(t, st, 1); len(rows) != 1 {
t.Fatalf("want 1 row, got %d", len(rows))
}
if _, ok, err := st.TrafficFor(1); err != nil || ok {
t.Fatalf("no traffic row expected: ok=%v err=%v", ok, err)
}
}
// The held-reading rhythm: readings inside a minute are not booked
// one by one; the minute change books the held one (dated by its own
// arrival), and BookHeld books the tail — so totals equal the full
// counter climb exactly once.
func TestHeldReadingRhythmAndBookHeld(t *testing.T) {
st := newTestStore(t)
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
mk := func(rx uint64) Sample { return Sample{BootID: "k/fp", NetRxTotal: rx, NetTxTotal: rx} }
rec(t, st, 1, mk(1000), base) // booked at once (first), delta 0
rec(t, st, 1, mk(1300), base.Add(20*time.Second)) // held
rec(t, st, 1, mk(1600), base.Add(40*time.Second)) // held (replaces)
rec(t, st, 1, mk(1900), base.Add(time.Minute)) // new minute: books held@40s (delta 600)
tr, ok, _ := st.TrafficFor(1)
if !ok || tr.TotalRx != 600 {
t.Fatalf("after minute change total=%d ok=%v, want 600", tr.TotalRx, ok)
}
rec(t, st, 1, mk(2200), base.Add(2*time.Minute)) // books held@60s (delta 300)
tr, _, _ = st.TrafficFor(1)
if tr.TotalRx != 900 {
t.Fatalf("total=%d, want 900", tr.TotalRx)
}
st.BookHeld() // books the @120s reading (delta 300)
tr, _, _ = st.TrafficFor(1)
if tr.TotalRx != 1200 {
t.Fatalf("after BookHeld total=%d, want 1200 (=2200-1000)", tr.TotalRx)
}
}
// A break (epoch change) books both sides at once: the held reading
// closes its span, the new one aligns the new baseline with delta 0.
func TestRecordBreakBooksBothSides(t *testing.T) {
st := newTestStore(t)
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
mk := func(boot string, rx uint64) Sample { return Sample{BootID: boot, NetRxTotal: rx, NetTxTotal: rx} }
rec(t, st, 1, mk("k1/fp", 1000), base)
rec(t, st, 1, mk("k1/fp", 1400), base.Add(20*time.Second)) // held, unbooked
rec(t, st, 1, mk("k2/fp", 50), base.Add(40*time.Second)) // break: books held (400) + new (0)
tr, _, _ := st.TrafficFor(1)
if tr.TotalRx != 400 {
t.Fatalf("break booking wrong: total=%d, want 400", tr.TotalRx)
}
rec(t, st, 1, mk("k2/fp", 150), base.Add(80*time.Second)) // new minute books held@40s (0), holds this
st.BookHeld()
tr, _, _ = st.TrafficFor(1)
if tr.TotalRx != 500 {
t.Fatalf("total=%d, want 500", tr.TotalRx)
}
}