Muse
0d1607c358
Baseline note: TestMetricHistoryMinuteFold fails identically on the pristine B3 tip in the same wall-clock window (minute-parity sensitive seed), passes in others — pre-existing, untouched path.
412 lines
15 KiB
Go
412 lines
15 KiB
Go
package probe
|
||
|
||
import (
|
||
"errors"
|
||
"strconv"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
func saveTask(t *testing.T, st *Store, task PingTask) int64 {
|
||
t.Helper()
|
||
id, err := st.SavePingTask(task, nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return id
|
||
}
|
||
|
||
// Create assigns an id from the high-water mark, lists back with its
|
||
// assignments, and a deleted id is never reissued (hub next_id).
|
||
func TestPingTaskCRUDAndIDReuse(t *testing.T) {
|
||
st := newTestStore(t)
|
||
id1 := saveTask(t, st, PingTask{Name: "CF", Target: "1.1.1.1:443", Interval: 60, Region: "全球", AgentIDs: []int64{7}})
|
||
id2 := saveTask(t, st, PingTask{Name: "G", Target: "8.8.8.8:53", Interval: 30})
|
||
if id1 == 0 || id2 == 0 || id1 == id2 {
|
||
t.Fatalf("ids %d %d", id1, id2)
|
||
}
|
||
tasks, err := st.PingTasks()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(tasks) != 2 || tasks[0].ID != id1 || tasks[1].ID != id2 {
|
||
t.Fatalf("list %+v", tasks)
|
||
}
|
||
if len(tasks[0].AgentIDs) != 1 || tasks[0].AgentIDs[0] != 7 || tasks[0].Region != "全球" {
|
||
t.Fatalf("task0 %+v", tasks[0])
|
||
}
|
||
if tasks[1].Sort <= tasks[0].Sort {
|
||
t.Fatalf("sort not appended: %d %d", tasks[0].Sort, tasks[1].Sort)
|
||
}
|
||
if err := st.DeletePingTask(id1); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
id3 := saveTask(t, st, PingTask{Name: "Q", Target: "9.9.9.9:443", Interval: 60})
|
||
if id3 == id1 || id3 == id2 {
|
||
t.Fatalf("id reissued: %d", id3)
|
||
}
|
||
// Updating a task another session deleted reports it (hub).
|
||
if _, err := st.SavePingTask(PingTask{ID: id1, Name: "x", Target: "1.1.1.1:443", Interval: 60}, nil); !errors.Is(err, ErrPingTaskGone) {
|
||
t.Fatalf("gone update: %v", err)
|
||
}
|
||
}
|
||
|
||
// Base-diff saving: an assignment made after the editor's base snapshot
|
||
// survives the save; base members removed from the new set go away.
|
||
func TestPingTaskSaveBaseDiff(t *testing.T) {
|
||
st := newTestStore(t)
|
||
id := saveTask(t, st, PingTask{Name: "A", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{1, 2}})
|
||
// Another session assigns agent 3 meanwhile.
|
||
other := saveTaskPingAssign(t, st, id, []int64{1, 2, 3})
|
||
_ = other
|
||
base := []int64{1, 2}
|
||
if _, err := st.SavePingTask(PingTask{ID: id, Name: "A", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{2}}, &base); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tasks, _ := st.PingTasks()
|
||
got := map[int64]bool{}
|
||
for _, a := range tasks[0].AgentIDs {
|
||
got[a] = true
|
||
}
|
||
if got[1] || !got[2] || !got[3] {
|
||
t.Fatalf("base-diff result %v", tasks[0].AgentIDs)
|
||
}
|
||
// Without base the assignment is replaced wholesale.
|
||
if _, err := st.SavePingTask(PingTask{ID: id, Name: "A", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{9}}, nil); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tasks, _ = st.PingTasks()
|
||
if len(tasks[0].AgentIDs) != 1 || tasks[0].AgentIDs[0] != 9 {
|
||
t.Fatalf("wholesale replace %v", tasks[0].AgentIDs)
|
||
}
|
||
}
|
||
|
||
func saveTaskPingAssign(t *testing.T, st *Store, taskID int64, agents []int64) error {
|
||
t.Helper()
|
||
_, err := st.db.Exec(`DELETE FROM probe_ping_node WHERE task_id = ?`, taskID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, a := range agents {
|
||
if _, err := st.db.Exec(`INSERT INTO probe_ping_node (task_id, agent_id) VALUES (?, ?)`, taskID, a); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Crowding: the 65th task on one agent refuses the whole save, and so
|
||
// does a 65th auto_join task (a fresh agent would be over its cap the
|
||
// moment it appears).
|
||
func TestPingTaskCrowding(t *testing.T) {
|
||
st := newTestStore(t)
|
||
for i := 0; i < MaxProbesPerNode; i++ {
|
||
saveTask(t, st, PingTask{Name: "t", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{5}})
|
||
}
|
||
_, err := st.SavePingTask(PingTask{Name: "one too many", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{5}}, nil)
|
||
var crowded *PingCrowdedError
|
||
if !errors.As(err, &crowded) || crowded.AgentID != 5 {
|
||
t.Fatalf("crowded: %v", err)
|
||
}
|
||
if tasks, _ := st.PingTasks(); len(tasks) != MaxProbesPerNode {
|
||
t.Fatalf("refused save leaked: %d tasks", len(tasks))
|
||
}
|
||
// The same count on a different agent is fine.
|
||
saveTask(t, st, PingTask{Name: "ok", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{6}})
|
||
|
||
st2 := newTestStore(t)
|
||
for i := 0; i < MaxProbesPerNode; i++ {
|
||
saveTask(t, st2, PingTask{Name: "t", Target: "1.1.1.1:443", Interval: 60, AutoJoin: true})
|
||
}
|
||
_, err = st2.SavePingTask(PingTask{Name: "join too many", Target: "1.1.1.1:443", Interval: 60, AutoJoin: true}, nil)
|
||
var aj *PingAutoJoinError
|
||
if !errors.As(err, &aj) {
|
||
t.Fatalf("autoJoin crowded: %v", err)
|
||
}
|
||
}
|
||
|
||
// SetNodePingTasks applies only the base difference and refuses when
|
||
// THIS agent would overflow; another agent's overflow is not its fault.
|
||
func TestSetNodePingTasks(t *testing.T) {
|
||
st := newTestStore(t)
|
||
a := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60})
|
||
b := saveTask(t, st, PingTask{Name: "b", Target: "1.1.1.1:443", Interval: 60})
|
||
// Assigned meanwhile by the task-side editor: survives.
|
||
saveTaskPingAssign(t, st, b, []int64{3})
|
||
if err := st.SetNodePingTasks(3, []int64{a}, []int64{}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if refs, _ := st.PingTasksFor(3); len(refs) != 2 {
|
||
t.Fatalf("node tasks %v", refs)
|
||
}
|
||
// A task deleted since the base makes the edit refuse.
|
||
if err := st.SetNodePingTasks(3, []int64{a, 9999}, []int64{a}); err == nil {
|
||
t.Fatal("deleted task accepted")
|
||
}
|
||
}
|
||
|
||
// Reorder: duplicates and stale lists refuse; a full list renumbers
|
||
// sort and PingTasks reflects it.
|
||
func TestReorderPingTasks(t *testing.T) {
|
||
st := newTestStore(t)
|
||
a := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60})
|
||
b := saveTask(t, st, PingTask{Name: "b", Target: "1.1.1.1:443", Interval: 60})
|
||
if err := st.ReorderPingTasks([]int64{a, a}); err == nil {
|
||
t.Fatal("duplicate accepted")
|
||
}
|
||
if err := st.ReorderPingTasks([]int64{a}); err == nil {
|
||
t.Fatal("short list accepted")
|
||
}
|
||
if err := st.ReorderPingTasks([]int64{b, a}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
tasks, _ := st.PingTasks()
|
||
if tasks[0].ID != b || tasks[1].ID != a || tasks[0].Sort != 0 || tasks[1].Sort != 1 {
|
||
t.Fatalf("after reorder %+v", tasks)
|
||
}
|
||
}
|
||
|
||
// PingTasksFor is id-ordered (the agent's cap keeps the first N) and
|
||
// PingTaskNames covers only assigned tasks.
|
||
func TestPingTasksForAndNames(t *testing.T) {
|
||
st := newTestStore(t)
|
||
a := saveTask(t, st, PingTask{Name: "first", Target: "1.1.1.1:443", Interval: 45, AgentIDs: []int64{2}})
|
||
b := saveTask(t, st, PingTask{Name: "second", Target: "8.8.8.8:53", Interval: 90, AgentIDs: []int64{2}})
|
||
saveTask(t, st, PingTask{Name: "elsewhere", Target: "9.9.9.9:443", Interval: 60, AgentIDs: []int64{3}})
|
||
refs, err := st.PingTasksFor(2)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(refs) != 2 || refs[0].ID != a || refs[1].ID != b || refs[0].Target != "1.1.1.1:443" || refs[0].Interval != 45 {
|
||
t.Fatalf("refs %+v", refs)
|
||
}
|
||
names, err := st.PingTaskNames(2)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(names) != 2 || names[strconv.FormatInt(a, 10)] != "first" {
|
||
t.Fatalf("names %v", names)
|
||
}
|
||
}
|
||
|
||
// InsertPings files only under live assignments: a result for an
|
||
// unassigned or deleted task is dropped silently; a second sample at
|
||
// the same stamp replaces the first (one session, one stamp).
|
||
func TestInsertPingsAssignmentGate(t *testing.T) {
|
||
st := newTestStore(t)
|
||
id := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{4}})
|
||
ts := time.Now().Unix()
|
||
err := st.InsertPings(4, []PingResult{
|
||
{TaskID: id, LatencyMs: 12},
|
||
{TaskID: id + 100, LatencyMs: 5}, // never assigned
|
||
}, ts)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// From an agent the task is not assigned to: dropped too.
|
||
if err := st.InsertPings(9, []PingResult{{TaskID: id, LatencyMs: 99}}, ts); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var count int64
|
||
var latency int64
|
||
st.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(latency), -999) FROM probe_ping_record`).Scan(&count, &latency)
|
||
if count != 1 || latency != 12 {
|
||
t.Fatalf("rows %d latency %d", count, latency)
|
||
}
|
||
if err := st.InsertPings(4, []PingResult{{TaskID: id, LatencyMs: 20}}, ts); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(latency), -999) FROM probe_ping_record`).Scan(&count, &latency)
|
||
if count != 1 || latency != 20 {
|
||
t.Fatalf("after replace rows %d latency %d", count, latency)
|
||
}
|
||
}
|
||
|
||
// PingHistory over minute rows: per-bucket medians, band only when the
|
||
// bucket varied, bucket loss rounded up, window loss unrounded and
|
||
// only for tasks that lost something.
|
||
func TestPingHistoryMinuteRows(t *testing.T) {
|
||
st := newTestStore(t)
|
||
a := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{1}})
|
||
b := saveTask(t, st, PingTask{Name: "b", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{1}})
|
||
now := time.Now().Unix()
|
||
base := (now/60)*60 - 600 // ten minutes back, bucket aligned
|
||
span := Span{Since: base, Step: 60, Hourly: false}
|
||
// Task A: one bucket with answers 10, 20, 40 and one miss.
|
||
for i, lat := range []int64{10, 20, 40, -1} {
|
||
seedPing(t, st, 1, base+int64(i)*7, a, lat)
|
||
}
|
||
// Task B: one bucket, all misses.
|
||
seedPing(t, st, 1, base+5, b, -1)
|
||
seedPing(t, st, 1, base+15, b, -1)
|
||
|
||
rows, loss, err := st.PingHistory(1, span)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(rows) != 2 {
|
||
t.Fatalf("rows %+v", rows)
|
||
}
|
||
ra, rb := rows[0], rows[1]
|
||
if ra.TaskID != a || rb.TaskID != b {
|
||
t.Fatalf("order %+v", rows)
|
||
}
|
||
if ra.Latency == nil || *ra.Latency != 20 {
|
||
t.Fatalf("median %v", ra.Latency)
|
||
}
|
||
if ra.Band == nil || (*ra.Band)[0] != 10 || (*ra.Band)[1] != 40 {
|
||
t.Fatalf("band %v", ra.Band)
|
||
}
|
||
if ra.Loss == nil || *ra.Loss != 25 { // 1 of 4, exact
|
||
t.Fatalf("bucket loss %v", ra.Loss)
|
||
}
|
||
if rb.Latency != nil || rb.Band != nil {
|
||
t.Fatalf("all-miss row %+v", rb)
|
||
}
|
||
if rb.Loss == nil || *rb.Loss != 100 {
|
||
t.Fatalf("all-miss loss %v", rb.Loss)
|
||
}
|
||
if loss[strconv.FormatInt(a, 10)] != 25.0 {
|
||
t.Fatalf("window loss a %v", loss)
|
||
}
|
||
if loss[strconv.FormatInt(b, 10)] != 100.0 {
|
||
t.Fatalf("window loss b %v", loss)
|
||
}
|
||
// An agent without the assignment sees none of it.
|
||
rows2, loss2, err := st.PingHistory(2, span)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(rows2) != 0 || len(loss2) != 0 {
|
||
t.Fatalf("unassigned view %v %v", rows2, loss2)
|
||
}
|
||
}
|
||
|
||
// PingHistory over an hourly span merges folded hours (weighted by
|
||
// their answer counts) with the minute rows after the watermark.
|
||
func TestPingHistoryHourlyMerge(t *testing.T) {
|
||
st := newTestStore(t)
|
||
a := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{1}})
|
||
now := time.Now().Unix()
|
||
hourStart := hourFloor(now) - 3*3600
|
||
span := Span{Since: hourStart, Step: 3600, Hourly: true}
|
||
// Folded hour: 3 answers (median 30, range 10..50), 1 lost.
|
||
if _, err := st.db.Exec(`INSERT INTO probe_ping_hour (agent_id, ts, task_id, answered, lost, latency, lo, hi)
|
||
VALUES (1, ?, ?, 3, 1, 30, 10, 50)`, hourStart, a); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// Watermark at the hour boundary: minute rows at/after it count.
|
||
if _, err := st.db.Exec(`INSERT INTO probe_setting (key, value) VALUES (?, ?)`, rolledKey, strconv.FormatInt(hourStart+3600, 10)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
minuteTs := hourStart + 3600 + 120 // inside the NEXT hour bucket
|
||
seedPing(t, st, 1, minuteTs, a, 100)
|
||
// A stale minute row below the watermark must not double count.
|
||
seedPing(t, st, 1, hourStart+60, a, 999)
|
||
|
||
rows, loss, err := st.PingHistory(1, span)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(rows) != 2 {
|
||
t.Fatalf("rows %+v", rows)
|
||
}
|
||
if rows[0].Ts != hourStart || rows[0].Latency == nil || *rows[0].Latency != 30 {
|
||
t.Fatalf("hour row %+v", rows[0])
|
||
}
|
||
if rows[0].Band == nil || (*rows[0].Band)[0] != 10 || (*rows[0].Band)[1] != 50 {
|
||
t.Fatalf("hour band %+v", rows[0].Band)
|
||
}
|
||
if rows[0].Loss == nil || *rows[0].Loss != 25 {
|
||
t.Fatalf("hour loss %v", rows[0].Loss)
|
||
}
|
||
if rows[1].Ts != hourStart+3600 || rows[1].Latency == nil || *rows[1].Latency != 100 {
|
||
t.Fatalf("minute row %+v", rows[1])
|
||
}
|
||
// Window loss counts the hour tally plus the minute row: 1 lost of 5.
|
||
if loss[strconv.FormatInt(a, 10)] != 20.0 {
|
||
t.Fatalf("window loss %v", loss)
|
||
}
|
||
}
|
||
|
||
// Weighted median: with answers weighted by counts, the middle ranks
|
||
// decide — two middle values average (hub Tally::median).
|
||
func TestPingWeightedMedianAveragesMiddleTwo(t *testing.T) {
|
||
st := newTestStore(t)
|
||
a := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{1}})
|
||
now := time.Now().Unix()
|
||
base := (now / 3600) * 3600
|
||
span := Span{Since: base, Step: 3600, Hourly: true}
|
||
// Two folded hours inside one bucket: medians 10 (×1) and 21 (×1)
|
||
// → total 2, ranks 1 and 2 → (10+21)/2 = 15 (integer).
|
||
for i, m := range []int64{10, 21} {
|
||
if _, err := st.db.Exec(`INSERT INTO probe_ping_hour (agent_id, ts, task_id, answered, lost, latency, lo, hi)
|
||
VALUES (1, ?, ?, 1, 0, ?, ?, ?)`, base+int64(i)*60, a, m, m, m); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
if _, err := st.db.Exec(`INSERT INTO probe_setting (key, value) VALUES (?, ?)`, rolledKey, strconv.FormatInt(base+3600, 10)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rows, _, err := st.PingHistory(1, span)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(rows) != 1 || rows[0].Latency == nil || *rows[0].Latency != 15 {
|
||
t.Fatalf("weighted median %+v", rows)
|
||
}
|
||
}
|
||
|
||
// Auto-join: an agent's first recorded sample assigns it every
|
||
// auto_join task; agents already known are NOT retroactively joined
|
||
// when a task is flagged later (hub semantics).
|
||
func TestAutoJoinOnFirstRecord(t *testing.T) {
|
||
st := newTestStore(t)
|
||
auto := saveTask(t, st, PingTask{Name: "auto", Target: "1.1.1.1:443", Interval: 60, AutoJoin: true})
|
||
saveTask(t, st, PingTask{Name: "manual", Target: "1.1.1.1:443", Interval: 60})
|
||
|
||
// Existing agent 1 records before any task existed... (tasks were
|
||
// created first here, so use a pre-existing agent row instead:)
|
||
st.Record(1, Sample{BootID: "b1"}, time.Now(), 0)
|
||
if refs, _ := st.PingTasksFor(1); len(refs) != 1 || refs[0].ID != auto {
|
||
t.Fatalf("first record join %v", refs)
|
||
}
|
||
// Agent 2's first record joins too.
|
||
st.Record(2, Sample{BootID: "b2"}, time.Now(), 0)
|
||
if refs, _ := st.PingTasksFor(2); len(refs) != 1 {
|
||
t.Fatalf("agent2 join %v", refs)
|
||
}
|
||
// A task flagged auto_join afterwards does not reach agent 1/2.
|
||
late := saveTask(t, st, PingTask{Name: "late", Target: "1.1.1.1:443", Interval: 60, AutoJoin: true})
|
||
st.Record(1, Sample{BootID: "b1"}, time.Now(), time.Now().UnixMilli())
|
||
if refs, _ := st.PingTasksFor(1); len(refs) != 1 {
|
||
t.Fatalf("retroactive join happened: %v", refs)
|
||
}
|
||
// But a brand-new agent 3 picks up both.
|
||
st.Record(3, Sample{BootID: "b3"}, time.Now(), 0)
|
||
if refs, _ := st.PingTasksFor(3); len(refs) != 2 || refs[1].ID != late {
|
||
t.Fatalf("agent3 join %v", refs)
|
||
}
|
||
}
|
||
|
||
// DeletePingTask clears records, hours and assignments with the task.
|
||
func TestDeletePingTaskClearsAll(t *testing.T) {
|
||
st := newTestStore(t)
|
||
id := saveTask(t, st, PingTask{Name: "a", Target: "1.1.1.1:443", Interval: 60, AgentIDs: []int64{1}})
|
||
seedPing(t, st, 1, time.Now().Unix(), id, 5)
|
||
if _, err := st.db.Exec(`INSERT INTO probe_ping_hour (agent_id, ts, task_id, answered, lost, latency, lo, hi)
|
||
VALUES (1, 0, ?, 1, 0, 5, 5, 5)`, id); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := st.DeletePingTask(id); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, table := range []string{"probe_ping_record", "probe_ping_hour", "probe_ping_node", "probe_ping_task"} {
|
||
var n int64
|
||
st.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n)
|
||
if n != 0 {
|
||
t.Fatalf("%s still has %d rows", table, n)
|
||
}
|
||
}
|
||
}
|