New internal/web/service/probe package keeps probe history in its own probe.db beside the panel DB (single connection, WAL), so history writes never contend with the panel write lock. - Minute rows per hub semantics: a held-reading rhythm books traffic against each reading own arrival; minute buckets merge (means, rounded int means, peaks) and the stored net rate derives from the kernel counters between row marks, falling back to agent rates at epoch breaks. - probe_traffic ledger: accumulate is a line port of hub traffic.rs (baseline alignment, same-boot shrink books 0, epoch change books 0, unseen-boot exception books the whole counters-since-boot measured against the master last-seen stamp, per-direction period resets, saturating adds); reads gate stale day/month stamps to zero. - probe_meta rows are ensured for every reporting agent (B3 edits them in place); probe_ping_* tables are created now, written by B4. - Hourly roll_up behind a watermark (hour-aligned, late grace, pre-retention hours skipped unfoldered) and watermark-guarded chunked prune (minute tier 7 days or the retention window when shorter; hourly tier to retention_days, default 7, cap 365). - Wiring: heartbeat files probe samples after the snapshot update (last-seen captured before touch); housekeeping hangs off the existing robfig/cron in web.startTask (@hourly + one pass a minute after startup, mirroring the hub boot pass); Server.stop books held readings. Every failure path logs and degrades - a broken probe.db never fails a heartbeat or the panel.
176 lines
6.5 KiB
Go
176 lines
6.5 KiB
Go
package probe
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type metricRow struct {
|
|
ts int64
|
|
cpu, cpuMax float64
|
|
mem, swap, disk int64
|
|
netRx, netTx, netRxMax, netTxMax int64
|
|
tcp, udp, procs int64
|
|
}
|
|
|
|
func readMetricRows(t *testing.T, st *Store, agent int64) []metricRow {
|
|
t.Helper()
|
|
rows, err := st.db.Query(`
|
|
SELECT ts, cpu, mem_used, swap_used, disk_used, net_rx, net_tx,
|
|
tcp, udp, procs, net_rx_max, net_tx_max, cpu_max
|
|
FROM probe_metric WHERE agent_id = ? ORDER BY ts`, agent)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
var out []metricRow
|
|
for rows.Next() {
|
|
var r metricRow
|
|
if err := rows.Scan(&r.ts, &r.cpu, &r.mem, &r.swap, &r.disk, &r.netRx, &r.netTx,
|
|
&r.tcp, &r.udp, &r.procs, &r.netRxMax, &r.netTxMax, &r.cpuMax); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func rec(t *testing.T, st *Store, agent int64, smp Sample, at time.Time) {
|
|
t.Helper()
|
|
if err := st.Record(agent, smp, at, 0); err != nil {
|
|
t.Fatalf("record: %v", err)
|
|
}
|
|
}
|
|
|
|
// Samples landing in the same minute bucket merge into one row when
|
|
// the next minute's sample flushes it: mean fields average (ints
|
|
// rounded), peaks max, and the stored net rate derives from the
|
|
// kernel counters between the row marks, not from the agent's rates.
|
|
func TestMinuteBucketMergeAndDerivedRates(t *testing.T) {
|
|
st := newTestStore(t)
|
|
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
|
|
mk := func(cpu float32, rxTot uint64, rate uint64, mem uint64) Sample {
|
|
return Sample{BootID: "k/fp", CPU: cpu, MemUsed: mem, NetRxTotal: rxTot, NetTxTotal: rxTot * 2,
|
|
NetRx: rate, NetTx: rate * 2, TCP: 3, UDP: 2, Procs: 10}
|
|
}
|
|
rec(t, st, 1, mk(10, 60_000, 1000, 100), base) // minute M, first sample
|
|
rec(t, st, 1, mk(20, 90_000, 1000, 200), base.Add(30*time.Second)) // minute M again
|
|
rec(t, st, 1, mk(30, 150_000, 1000, 300), base.Add(time.Minute)) // minute M+1: flushes
|
|
rows := readMetricRows(t, st, 1)
|
|
if len(rows) != 1 {
|
|
t.Fatalf("want 1 row, got %d", len(rows))
|
|
}
|
|
r := rows[0]
|
|
if r.ts != base.Add(time.Minute).Unix() {
|
|
t.Fatalf("row stamped %d, want flushing minute %d", r.ts, base.Add(time.Minute).Unix())
|
|
}
|
|
if r.cpu != 20 { // mean of 10,20,30 (flushing sample included)
|
|
t.Fatalf("cpu mean=%v, want 20", r.cpu)
|
|
}
|
|
if r.mem != 200 {
|
|
t.Fatalf("mem mean=%d, want 200", r.mem)
|
|
}
|
|
if r.cpuMax != 30 {
|
|
t.Fatalf("cpu_max=%v, want 30", r.cpuMax)
|
|
}
|
|
if r.netRx != 1500 { // (150000-60000) / 60s between marks
|
|
t.Fatalf("net_rx=%d, want 1500 (counter-derived)", r.netRx)
|
|
}
|
|
if r.netRxMax != 1000 {
|
|
t.Fatalf("net_rx_max=%d, want agent peak 1000", r.netRxMax)
|
|
}
|
|
// Next minute: single-sample row; rate from the new mark.
|
|
rec(t, st, 1, mk(40, 210_000, 700, 400), base.Add(2*time.Minute))
|
|
rows = readMetricRows(t, st, 1)
|
|
if len(rows) != 2 {
|
|
t.Fatalf("want 2 rows, got %d", len(rows))
|
|
}
|
|
r = rows[1]
|
|
if r.cpu != 40 || r.netRx != 1000 || r.netRxMax != 700 {
|
|
t.Fatalf("second row wrong: %+v", r)
|
|
}
|
|
}
|
|
|
|
// At an epoch break the counter mark no longer applies, so the row
|
|
// keeps the agent's own reported rates.
|
|
func TestMinuteRowEpochBreakKeepsAgentRates(t *testing.T) {
|
|
st := newTestStore(t)
|
|
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
|
|
rec(t, st, 1, Sample{BootID: "k1/fp", NetRxTotal: 1000, NetTxTotal: 1000, NetRx: 111, NetTx: 222}, base)
|
|
rec(t, st, 1, Sample{BootID: "k2/fp", NetRxTotal: 500, NetTxTotal: 500, NetRx: 333, NetTx: 444}, base.Add(time.Minute))
|
|
rows := readMetricRows(t, st, 1)
|
|
if len(rows) != 1 {
|
|
t.Fatalf("want 1 row, got %d", len(rows))
|
|
}
|
|
if rows[0].netRx != 333 || rows[0].netTx != 444 {
|
|
t.Fatalf("epoch break must keep agent rates: %+v", rows[0])
|
|
}
|
|
}
|
|
|
|
// Samples without a boot_id still file metric rows (hub files rows
|
|
// from any well-formed report) but never touch the traffic ledger.
|
|
func TestRecordWithoutBootFilesRowOnly(t *testing.T) {
|
|
st := newTestStore(t)
|
|
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
|
|
rec(t, st, 1, Sample{CPU: 5, MemUsed: 42}, base)
|
|
rec(t, st, 1, Sample{CPU: 7, MemUsed: 43}, base.Add(time.Minute))
|
|
if rows := readMetricRows(t, st, 1); len(rows) != 1 {
|
|
t.Fatalf("want 1 row, got %d", len(rows))
|
|
}
|
|
if _, ok, err := st.TrafficFor(1); err != nil || ok {
|
|
t.Fatalf("no traffic row expected: ok=%v err=%v", ok, err)
|
|
}
|
|
}
|
|
|
|
// The held-reading rhythm: readings inside a minute are not booked
|
|
// one by one; the minute change books the held one (dated by its own
|
|
// arrival), and BookHeld books the tail — so totals equal the full
|
|
// counter climb exactly once.
|
|
func TestHeldReadingRhythmAndBookHeld(t *testing.T) {
|
|
st := newTestStore(t)
|
|
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
|
|
mk := func(rx uint64) Sample { return Sample{BootID: "k/fp", NetRxTotal: rx, NetTxTotal: rx} }
|
|
rec(t, st, 1, mk(1000), base) // booked at once (first), delta 0
|
|
rec(t, st, 1, mk(1300), base.Add(20*time.Second)) // held
|
|
rec(t, st, 1, mk(1600), base.Add(40*time.Second)) // held (replaces)
|
|
rec(t, st, 1, mk(1900), base.Add(time.Minute)) // new minute: books held@40s (delta 600)
|
|
tr, ok, _ := st.TrafficFor(1)
|
|
if !ok || tr.TotalRx != 600 {
|
|
t.Fatalf("after minute change total=%d ok=%v, want 600", tr.TotalRx, ok)
|
|
}
|
|
rec(t, st, 1, mk(2200), base.Add(2*time.Minute)) // books held@60s (delta 300)
|
|
tr, _, _ = st.TrafficFor(1)
|
|
if tr.TotalRx != 900 {
|
|
t.Fatalf("total=%d, want 900", tr.TotalRx)
|
|
}
|
|
st.BookHeld() // books the @120s reading (delta 300)
|
|
tr, _, _ = st.TrafficFor(1)
|
|
if tr.TotalRx != 1200 {
|
|
t.Fatalf("after BookHeld total=%d, want 1200 (=2200-1000)", tr.TotalRx)
|
|
}
|
|
}
|
|
|
|
// A break (epoch change) books both sides at once: the held reading
|
|
// closes its span, the new one aligns the new baseline with delta 0.
|
|
func TestRecordBreakBooksBothSides(t *testing.T) {
|
|
st := newTestStore(t)
|
|
base := time.Now().Add(-10 * time.Minute).Truncate(time.Minute)
|
|
mk := func(boot string, rx uint64) Sample { return Sample{BootID: boot, NetRxTotal: rx, NetTxTotal: rx} }
|
|
rec(t, st, 1, mk("k1/fp", 1000), base)
|
|
rec(t, st, 1, mk("k1/fp", 1400), base.Add(20*time.Second)) // held, unbooked
|
|
rec(t, st, 1, mk("k2/fp", 50), base.Add(40*time.Second)) // break: books held (400) + new (0)
|
|
tr, _, _ := st.TrafficFor(1)
|
|
if tr.TotalRx != 400 {
|
|
t.Fatalf("break booking wrong: total=%d, want 400", tr.TotalRx)
|
|
}
|
|
rec(t, st, 1, mk("k2/fp", 150), base.Add(80*time.Second)) // new minute books held@40s (0), holds this
|
|
st.BookHeld()
|
|
tr, _, _ = st.TrafficFor(1)
|
|
if tr.TotalRx != 500 {
|
|
t.Fatalf("total=%d, want 500", tr.TotalRx)
|
|
}
|
|
}
|