Rogue
3e052c42b8
New internal/web/service/probe package keeps probe history in its own probe.db beside the panel DB (single connection, WAL), so history writes never contend with the panel write lock. - Minute rows per hub semantics: a held-reading rhythm books traffic against each reading own arrival; minute buckets merge (means, rounded int means, peaks) and the stored net rate derives from the kernel counters between row marks, falling back to agent rates at epoch breaks. - probe_traffic ledger: accumulate is a line port of hub traffic.rs (baseline alignment, same-boot shrink books 0, epoch change books 0, unseen-boot exception books the whole counters-since-boot measured against the master last-seen stamp, per-direction period resets, saturating adds); reads gate stale day/month stamps to zero. - probe_meta rows are ensured for every reporting agent (B3 edits them in place); probe_ping_* tables are created now, written by B4. - Hourly roll_up behind a watermark (hour-aligned, late grace, pre-retention hours skipped unfoldered) and watermark-guarded chunked prune (minute tier 7 days or the retention window when shorter; hourly tier to retention_days, default 7, cap 365). - Wiring: heartbeat files probe samples after the snapshot update (last-seen captured before touch); housekeeping hangs off the existing robfig/cron in web.startTask (@hourly + one pass a minute after startup, mirroring the hub boot pass); Server.stop books held readings. Every failure path logs and degrades - a broken probe.db never fails a heartbeat or the panel.
200 lines
7.1 KiB
Go
200 lines
7.1 KiB
Go
package probe
|
|
|
|
import (
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func strconvFormat(n int64) string { return strconv.FormatInt(n, 10) }
|
|
|
|
func seedMetric(t *testing.T, st *Store, agent, ts int64, cpu, cpuMax float64, mem, netRx, netRxMax, tcpC int64) {
|
|
t.Helper()
|
|
_, err := st.db.Exec(`
|
|
INSERT INTO probe_metric (agent_id, ts, cpu, mem_used, swap_used, disk_used,
|
|
net_rx, net_tx, tcp, udp, procs, net_rx_max, net_tx_max, cpu_max)
|
|
VALUES (?, ?, ?, ?, 0, 0, ?, 0, ?, 0, 0, ?, 0, ?)`,
|
|
agent, ts, cpu, mem, netRx, tcpC, netRxMax, cpuMax)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func seedPing(t *testing.T, st *Store, agent, ts, task, latency int64) {
|
|
t.Helper()
|
|
if _, err := st.db.Exec(`INSERT INTO probe_ping_record (agent_id, ts, task_id, latency) VALUES (?, ?, ?, ?)`,
|
|
agent, ts, task, latency); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// One closed hour folds into one hourly row per agent: means over the
|
|
// minute rows (byte columns truncated, counts rounded), peaks maxed.
|
|
func TestRollUpFoldsHour(t *testing.T) {
|
|
st := newTestStore(t)
|
|
now := time.Now().Unix()
|
|
hourA := hourFloor(now) - 3*3600 // closed and past the late grace
|
|
seedMetric(t, st, 1, hourA+60, 10, 15, 100, 100, 500, 1)
|
|
seedMetric(t, st, 1, hourA+120, 20, 25, 200, 100, 500, 2)
|
|
seedMetric(t, st, 1, hourA+180, 30, 35, 300, 100, 500, 4)
|
|
seedMetric(t, st, 2, hourA+60, 99, 99, 999, 9, 9, 9) // agent 2, same hour
|
|
// The most recent hour must not fold yet (late grace).
|
|
recent := hourFloor(now) - 3600
|
|
seedMetric(t, st, 1, recent+60, 77, 77, 777, 7, 7, 7)
|
|
|
|
folded, err := st.RollUp(now, 7)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if folded == 0 {
|
|
t.Fatal("expected at least one folded hour")
|
|
}
|
|
var minutes, mem, netRx, netRxMax, tcpC int64
|
|
var cpu, cpuMax float64
|
|
err = st.db.QueryRow(`
|
|
SELECT minutes, cpu, cpu_max, mem_used, net_rx, net_rx_max, tcp
|
|
FROM probe_metric_hour WHERE agent_id = 1 AND ts = ?`, hourA).
|
|
Scan(&minutes, &cpu, &cpuMax, &mem, &netRx, &netRxMax, &tcpC)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if minutes != 3 || cpu != 20 || cpuMax != 35 || mem != 200 || netRx != 100 || netRxMax != 500 || tcpC != 2 {
|
|
t.Fatalf("folded row wrong: minutes=%d cpu=%v cpuMax=%v mem=%d netRx=%d netRxMax=%d tcp=%d",
|
|
minutes, cpu, cpuMax, mem, netRx, netRxMax, tcpC)
|
|
}
|
|
var recentCount int
|
|
if err := st.db.QueryRow(`SELECT COUNT(*) FROM probe_metric_hour WHERE ts = ?`, recent).Scan(&recentCount); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recentCount != 0 {
|
|
t.Fatal("recent hour must still be unfolded")
|
|
}
|
|
wm, ok, err := st.watermark()
|
|
if err != nil || !ok {
|
|
t.Fatalf("watermark: ok=%v err=%v", ok, err)
|
|
}
|
|
if wm < hourA+3600 || wm > recent+3600 {
|
|
t.Fatalf("watermark %d must sit past hour A (%d) and no later than past the recent hour (%d)", wm, hourA, recent)
|
|
}
|
|
// Folding again is a rewrite, not a duplicate: same values.
|
|
if _, err := st.RollUp(now, 7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var count int
|
|
if err := st.db.QueryRow(`SELECT COUNT(*) FROM probe_metric_hour WHERE agent_id = 1 AND ts = ?`, hourA).Scan(&count); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count != 1 {
|
|
t.Fatalf("hour rows for A = %d, want 1", count)
|
|
}
|
|
}
|
|
|
|
// Ping hours: answered/lost counts, the answered median, and lo/hi;
|
|
// an all-lost task keeps NULL latency fields.
|
|
func TestRollUpPingTallies(t *testing.T) {
|
|
st := newTestStore(t)
|
|
now := time.Now().Unix()
|
|
hourA := hourFloor(now) - 3*3600
|
|
seedPing(t, st, 1, hourA+10, 7, 10)
|
|
seedPing(t, st, 1, hourA+20, 7, 20)
|
|
seedPing(t, st, 1, hourA+30, 7, -1)
|
|
seedPing(t, st, 1, hourA+40, 7, 40)
|
|
seedPing(t, st, 1, hourA+10, 8, -1)
|
|
seedPing(t, st, 1, hourA+20, 8, -1)
|
|
if _, err := st.RollUp(now, 7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var answered, lost, latency, lo, hi int64
|
|
err := st.db.QueryRow(`
|
|
SELECT answered, lost, latency, lo, hi FROM probe_ping_hour
|
|
WHERE agent_id = 1 AND ts = ? AND task_id = 7`, hourA).Scan(&answered, &lost, &latency, &lo, &hi)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if answered != 3 || lost != 1 || latency != 20 || lo != 10 || hi != 40 {
|
|
t.Fatalf("tally wrong: answered=%d lost=%d latency=%d lo=%d hi=%d", answered, lost, latency, lo, hi)
|
|
}
|
|
var latNull bool
|
|
err = st.db.QueryRow(`
|
|
SELECT latency IS NULL FROM probe_ping_hour
|
|
WHERE agent_id = 1 AND ts = ? AND task_id = 8`, hourA).Scan(&latNull)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !latNull {
|
|
t.Fatal("all-lost task must keep NULL latency")
|
|
}
|
|
}
|
|
|
|
// Prune respects the watermark: minute rows past the detail window
|
|
// survive until their hour is folded; hourly rows past the retention
|
|
// window go; the traffic ledger is never pruned.
|
|
func TestPruneBoundaries(t *testing.T) {
|
|
st := newTestStore(t)
|
|
now := time.Now()
|
|
nowU := now.Unix()
|
|
oldMinute := hourFloor(nowU) - 8*86400 // past the 7-day detail window
|
|
recentMinute := hourFloor(nowU) - 3600
|
|
seedMetric(t, st, 1, oldMinute+60, 1, 1, 1, 1, 1, 1)
|
|
seedMetric(t, st, 1, recentMinute+60, 1, 1, 1, 1, 1, 1)
|
|
// Watermark only covers up to oldMinute: the old row is NOT yet
|
|
// deletable even though it is past the window.
|
|
if _, err := st.db.Exec(`INSERT OR REPLACE INTO probe_setting (key, value) VALUES (?, ?)`,
|
|
rolledKey, strconvFormat(oldMinute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := st.Prune(7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var n int
|
|
if err := st.db.QueryRow(`SELECT COUNT(*) FROM probe_metric WHERE agent_id = 1`).Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 2 {
|
|
t.Fatalf("unfolded old row must survive prune, rows=%d", n)
|
|
}
|
|
// Roll forward past the old row's hour, making the watermark cover
|
|
// it; prune must now drop it (the recent row stays).
|
|
if _, err := st.db.Exec(`UPDATE probe_setting SET value = ? WHERE key = ?`,
|
|
strconvFormat(oldMinute+3600), rolledKey); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := st.Prune(7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.db.QueryRow(`SELECT COUNT(*) FROM probe_metric WHERE agent_id = 1`).Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("folded old row must be pruned, rows=%d", n)
|
|
}
|
|
// Hourly rows past retention go; within stay.
|
|
if _, err := st.db.Exec(`INSERT INTO probe_metric_hour (agent_id, ts, minutes, cpu, cpu_max, mem_used, swap_used, disk_used, net_rx, net_tx, net_rx_max, net_tx_max, tcp, udp, procs) VALUES (1, ?, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`,
|
|
hourFloor(nowU)-8*86400); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := st.db.Exec(`INSERT INTO probe_metric_hour (agent_id, ts, minutes, cpu, cpu_max, mem_used, swap_used, disk_used, net_rx, net_tx, net_rx_max, net_tx_max, tcp, udp, procs) VALUES (1, ?, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`,
|
|
hourFloor(nowU)-6*86400); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := st.Prune(7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := st.db.QueryRow(`SELECT COUNT(*) FROM probe_metric_hour WHERE agent_id = 1`).Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("hour rows=%d, want 1 (only the in-window one)", n)
|
|
}
|
|
// Traffic bookkeeping is never pruned.
|
|
book(t, st, 1, "k/fp", 100, 100, now.Add(-30*24*time.Hour), nil, 0)
|
|
book(t, st, 1, "k/fp", 300, 300, now.Add(-30*24*time.Hour).Add(time.Minute), nil, 0)
|
|
if _, err := st.Prune(7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tr, ok, _ := st.TrafficFor(1)
|
|
if !ok || tr.TotalRx != 200 {
|
|
t.Fatalf("traffic must survive prune: %+v ok=%v", tr, ok)
|
|
}
|
|
}
|