Muse
e03ceaa41f
- probe package: probe_ping_task/probe_ping_node tables; task CRUD with hub save semantics (id high-water, base-diff assignment, 64-per-agent crowding refusal, auto_join cap); filing with assignment-gated insert; PingHistory weighted-median chart read over minute+hour tiers; auto-join on first Record (ensureRows). - controller: /panel/api/agent/probe-tasks* management endpoints (node id space, hub validation chain verbatim); series=ping now serves real rows from probe.PingHistory; heartbeat ingests probeResults (missing latency skipped, batch capped); config response carries probe_tasks (always present, empty = clear). - agent: per-task probe loops (immediate first probe, ticker, 900ms resolve/dial budgets, <=3 addresses, resolve overrun = no sample), result buffer drained on heartbeat and requeued on failure; task set applied before the config-hash early return.
740 lines
22 KiB
Go
740 lines
22 KiB
Go
package probe
|
|
|
|
import (
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
)
|
|
|
|
// Item 7 B4: probe (latency) tasks — the operator-maintained target
|
|
// set, its per-agent assignment, the filing of agent-reported results
|
|
// and the chart read. A port of hub db.rs's ping section and api.rs's
|
|
// validation chain (ITEM7-SCOPE §5.7): no preset targets exist, the
|
|
// operator keeps the list (grouped by the free-form region label).
|
|
//
|
|
// The one structural difference from the hub: hub nodes live in the
|
|
// same database, so its foreign keys police assignments. Here the
|
|
// agents live in the panel database and probe.db cannot reference
|
|
// them; the controller validates agent existence against the panel
|
|
// database before saving, and filing still re-checks the assignment
|
|
// inside the insert statement, exactly like hub insert_pings.
|
|
|
|
// MaxProbesPerNode bounds how many probe tasks one agent may be
|
|
// assigned; the agent enforces the same cap as a backstop (hub
|
|
// MAX_PROBES_PER_NODE). The master refuses the write that would
|
|
// exceed it, because a silent truncation would draw empty charts.
|
|
const MaxProbesPerNode = 64
|
|
|
|
// MinProbeInterval is the fastest probe cadence, in seconds; the
|
|
// editing surface refuses anything outside [MinProbeInterval, 3600]
|
|
// rather than clamping it (hub save_ping_task).
|
|
const MinProbeInterval = 5
|
|
|
|
// pingTaskIDIssuedKey is the probe_setting high-water mark of issued
|
|
// ping task ids (hub "{table}_id_issued").
|
|
const pingTaskIDIssuedKey = "ping_task_id_issued"
|
|
|
|
// PingTask is one probe target as the panel keeps it. AgentIDs is the
|
|
// assignment side (probe_ping_node), filled by reads.
|
|
type PingTask struct {
|
|
ID int64
|
|
Name string
|
|
Target string
|
|
Interval int64
|
|
AutoJoin bool
|
|
Sort int64
|
|
Region string
|
|
AgentIDs []int64
|
|
}
|
|
|
|
// PingTaskRef is the wire form of one task as pushed to an agent:
|
|
// id, target and interval, nothing else (hub ping_tasks_for).
|
|
type PingTaskRef struct {
|
|
ID int64 `json:"id"`
|
|
Target string `json:"target"`
|
|
Interval int64 `json:"interval"`
|
|
}
|
|
|
|
// PingResult is one probe sample an agent reports: the task it ran
|
|
// and the handshake latency in milliseconds (negative = the target
|
|
// did not answer; a resolution overrun produces no sample at all and
|
|
// never reaches this type).
|
|
type PingResult struct {
|
|
TaskID int64
|
|
LatencyMs int64
|
|
}
|
|
|
|
// ErrPingTaskGone reports an update against a task another session
|
|
// deleted first (hub save_ping_task's refusal).
|
|
var ErrPingTaskGone = errors.New("监控不存在，可能已被删除")
|
|
|
|
// PingCrowdedError reports a save that would leave one agent over
|
|
// MaxProbesPerNode; the controller renders the hub message with the
|
|
// agent's display name.
|
|
type PingCrowdedError struct{ AgentID int64 }
|
|
|
|
func (e *PingCrowdedError) Error() string {
|
|
return fmt.Sprintf("agent %d would exceed %d probe tasks", e.AgentID, MaxProbesPerNode)
|
|
}
|
|
|
|
// PingAutoJoinError reports a save under which the next agent to
|
|
// appear would auto-join more tasks than an agent can run.
|
|
type PingAutoJoinError struct{ Count int64 }
|
|
|
|
func (e *PingAutoJoinError) Error() string {
|
|
return fmt.Sprintf("new nodes would auto-join %d probe tasks", e.Count)
|
|
}
|
|
|
|
const pingTaskColumns = `id, name, target, interval, auto_join, sort, region`
|
|
|
|
func scanPingTask(sc interface{ Scan(...any) error }) (PingTask, error) {
|
|
var t PingTask
|
|
var autoJoin int64
|
|
if err := sc.Scan(&t.ID, &t.Name, &t.Target, &t.Interval, &autoJoin, &t.Sort, &t.Region); err != nil {
|
|
return PingTask{}, err
|
|
}
|
|
t.AutoJoin = autoJoin != 0
|
|
return t, nil
|
|
}
|
|
|
|
// PingTasks lists every probe task in panel order (sort, then id),
|
|
// each with its assigned agent ids.
|
|
func (s *Store) PingTasks() ([]PingTask, error) {
|
|
rows, err := s.db.Query(`SELECT ` + pingTaskColumns + ` FROM probe_ping_task ORDER BY sort, id`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var tasks []PingTask
|
|
for rows.Next() {
|
|
t, err := scanPingTask(rows)
|
|
if err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
tasks = append(tasks, t)
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
if tasks == nil {
|
|
tasks = []PingTask{}
|
|
}
|
|
assignRows, err := s.db.Query(`SELECT task_id, agent_id FROM probe_ping_node ORDER BY agent_id`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer assignRows.Close()
|
|
byTask := make(map[int64][]int64)
|
|
for assignRows.Next() {
|
|
var taskID, agentID int64
|
|
if err := assignRows.Scan(&taskID, &agentID); err != nil {
|
|
return nil, err
|
|
}
|
|
byTask[taskID] = append(byTask[taskID], agentID)
|
|
}
|
|
if err := assignRows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range tasks {
|
|
if ids := byTask[tasks[i].ID]; ids != nil {
|
|
tasks[i].AgentIDs = ids
|
|
} else {
|
|
tasks[i].AgentIDs = []int64{}
|
|
}
|
|
}
|
|
return tasks, nil
|
|
}
|
|
|
|
// nextPingTaskID issues the next task id from the high-water mark,
|
|
// falling back to the table's own maximum for databases that predate
|
|
// the mark (hub next_id/issued). Runs inside the save transaction.
|
|
func nextPingTaskID(tx *sql.Tx) (int64, error) {
|
|
var issued, maxID int64
|
|
var raw sql.NullString
|
|
if err := tx.QueryRow(`SELECT value FROM probe_setting WHERE key = ?`, pingTaskIDIssuedKey).Scan(&raw); err != nil && err != sql.ErrNoRows {
|
|
return 0, err
|
|
} else if raw.Valid {
|
|
if n, perr := strconv.ParseInt(raw.String, 10, 64); perr == nil {
|
|
issued = n
|
|
}
|
|
}
|
|
if err := tx.QueryRow(`SELECT COALESCE(MAX(id), 0) FROM probe_ping_task`).Scan(&maxID); err != nil {
|
|
return 0, err
|
|
}
|
|
next := issued
|
|
if maxID > next {
|
|
next = maxID
|
|
}
|
|
next++
|
|
if _, err := tx.Exec(`INSERT INTO probe_setting (key, value) VALUES (?, ?)
|
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
pingTaskIDIssuedKey, fmt.Sprint(next)); err != nil {
|
|
return 0, err
|
|
}
|
|
return next, nil
|
|
}
|
|
|
|
// SavePingTask creates (ID == 0) or replaces one probe task and its
|
|
// assignments, in one transaction (hub save_ping_task). base, when
|
|
// non-nil on an update, is the assignment set the editor started
|
|
// from: only the difference from it is applied, so an assignment
|
|
// made while the editor was open survives. Without base the
|
|
// assignment is replaced wholesale. The caller validates field
|
|
// values and agent existence beforehand (hub api layer).
|
|
func (s *Store) SavePingTask(t PingTask, base *[]int64) (int64, error) {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
id := t.ID
|
|
if id > 0 {
|
|
res, err := tx.Exec(`UPDATE probe_ping_task SET name=?2, target=?3, interval=?4, auto_join=?5, region=?6 WHERE id=?1`,
|
|
id, t.Name, t.Target, t.Interval, boolToInt(t.AutoJoin), t.Region)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if n, err := res.RowsAffected(); err != nil {
|
|
return 0, err
|
|
} else if n == 0 {
|
|
// Deleted from another session; without this the save
|
|
// would report success against nothing.
|
|
return 0, ErrPingTaskGone
|
|
}
|
|
} else {
|
|
next, err := nextPingTaskID(tx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO probe_ping_task (id, name, target, interval, auto_join, sort, region)
|
|
VALUES (?1, ?2, ?3, ?4, ?5, (SELECT COALESCE(MAX(sort), -1) + 1 FROM probe_ping_task), ?6)`,
|
|
next, t.Name, t.Target, t.Interval, boolToInt(t.AutoJoin), t.Region); err != nil {
|
|
return 0, err
|
|
}
|
|
id = next
|
|
}
|
|
|
|
var added []int64
|
|
if base != nil && t.ID > 0 {
|
|
baseSet := int64Set(*base)
|
|
newSet := int64Set(t.AgentIDs)
|
|
for _, agentID := range *base {
|
|
if !newSet[agentID] {
|
|
if _, err := tx.Exec(`DELETE FROM probe_ping_node WHERE task_id=?1 AND agent_id=?2`, id, agentID); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
}
|
|
for _, agentID := range t.AgentIDs {
|
|
if !baseSet[agentID] {
|
|
added = append(added, agentID)
|
|
}
|
|
}
|
|
} else {
|
|
if _, err := tx.Exec(`DELETE FROM probe_ping_node WHERE task_id=?1`, id); err != nil {
|
|
return 0, err
|
|
}
|
|
added = t.AgentIDs
|
|
}
|
|
for _, agentID := range added {
|
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO probe_ping_node (task_id, agent_id) VALUES (?1, ?2)`, id, agentID); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
|
|
if err := refuseCrowded(tx, nil); err != nil {
|
|
return 0, err
|
|
}
|
|
var joining int64
|
|
if err := tx.QueryRow(`SELECT COUNT(*) FROM probe_ping_task WHERE auto_join`).Scan(&joining); err != nil {
|
|
return 0, err
|
|
}
|
|
if joining > MaxProbesPerNode {
|
|
return 0, &PingAutoJoinError{Count: joining}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return 0, err
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// refuseCrowded fails the transaction when some agent is assigned
|
|
// more tasks than an agent will run — queried from the table after
|
|
// the rows are in, inside the write's own transaction (hub
|
|
// refuse_crowded). onlyAgent, when set, scopes the check to that
|
|
// agent: another agent already over the limit is not this edit's
|
|
// problem to refuse.
|
|
func refuseCrowded(tx *sql.Tx, onlyAgent *int64) error {
|
|
var scope any
|
|
if onlyAgent != nil {
|
|
scope = *onlyAgent
|
|
}
|
|
var agentID sql.NullInt64
|
|
err := tx.QueryRow(`SELECT agent_id FROM probe_ping_node
|
|
WHERE ?1 IS NULL OR agent_id = ?1
|
|
GROUP BY agent_id HAVING COUNT(*) > ?2 LIMIT 1`, scope, MaxProbesPerNode).Scan(&agentID)
|
|
if err == sql.ErrNoRows {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return &PingCrowdedError{AgentID: agentID.Int64}
|
|
}
|
|
|
|
// SetNodePingTasks edits one agent's assignments from the agent's
|
|
// side (hub set_node_ping_tasks): only the difference from base is
|
|
// applied, inside one transaction. The caller checks the agent exists.
|
|
func (s *Store) SetNodePingTasks(agentID int64, tasks, base []int64) error {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
taskSet := int64Set(tasks)
|
|
baseSet := int64Set(base)
|
|
for _, taskID := range base {
|
|
if !taskSet[taskID] {
|
|
if _, err := tx.Exec(`DELETE FROM probe_ping_node WHERE task_id=?1 AND agent_id=?2`, taskID, agentID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
for _, taskID := range tasks {
|
|
if !baseSet[taskID] {
|
|
// A task deleted since the editor opened fails its foreign
|
|
// key on the hub; probe.db has no cross-database key, so
|
|
// existence is tested in the statement instead — naming
|
|
// the probe is all the panel needs.
|
|
res, err := tx.Exec(`INSERT OR IGNORE INTO probe_ping_node (task_id, agent_id)
|
|
SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM probe_ping_task WHERE id = ?1)`, taskID, agentID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, err := res.RowsAffected(); err != nil {
|
|
return err
|
|
} else if n == 0 {
|
|
return errors.New("有监控已被删除，刷新后重试")
|
|
}
|
|
}
|
|
}
|
|
if err := refuseCrowded(tx, &agentID); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// DeletePingTask removes a task and everything filed under it
|
|
// (hub delete_ping_task): the record table is cleared explicitly,
|
|
// the node's assignments and the hourly tier go with it.
|
|
func (s *Store) DeletePingTask(id int64) error {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
for _, stmt := range []string{
|
|
`DELETE FROM probe_ping_record WHERE task_id = ?`,
|
|
`DELETE FROM probe_ping_hour WHERE task_id = ?`,
|
|
`DELETE FROM probe_ping_node WHERE task_id = ?`,
|
|
`DELETE FROM probe_ping_task WHERE id = ?`,
|
|
} {
|
|
if _, err := tx.Exec(stmt, id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// ReorderPingTasks rewrites the panel order (hub reorder): the id
|
|
// list must name every task exactly once, checked against the table
|
|
// inside the transaction, so a list edited elsewhere cannot sort
|
|
// against a stale view.
|
|
func (s *Store) ReorderPingTasks(ids []int64) error {
|
|
seen := make(map[int64]bool, len(ids))
|
|
for _, id := range ids {
|
|
if seen[id] {
|
|
return errors.New("排序里有重复的条目")
|
|
}
|
|
seen[id] = true
|
|
}
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var count int64
|
|
if err := tx.QueryRow(`SELECT COUNT(*) FROM probe_ping_task`).Scan(&count); err != nil {
|
|
return err
|
|
}
|
|
if count != int64(len(ids)) {
|
|
return errors.New("列表已在别处改动，刷新后再排序")
|
|
}
|
|
for i, id := range ids {
|
|
res, err := tx.Exec(`UPDATE probe_ping_task SET sort=?2 WHERE id=?1`, id, int64(i))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, err := res.RowsAffected(); err != nil {
|
|
return err
|
|
} else if n != 1 {
|
|
return errors.New("列表已在别处改动，刷新后再排序")
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// PingTasksFor returns the task list pushed to one agent: id order,
|
|
// because the agent's own cap keeps the first MaxProbesPerNode and an
|
|
// unordered list at that boundary could yield a different subset on
|
|
// each push (hub ping_tasks_for).
|
|
func (s *Store) PingTasksFor(agentID int64) ([]PingTaskRef, error) {
|
|
rows, err := s.db.Query(`SELECT t.id, t.target, t.interval FROM probe_ping_task t
|
|
JOIN probe_ping_node n ON n.task_id = t.id WHERE n.agent_id = ?1 ORDER BY t.id`, agentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := []PingTaskRef{}
|
|
for rows.Next() {
|
|
var r PingTaskRef
|
|
if err := rows.Scan(&r.ID, &r.Target, &r.Interval); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// PingTaskNames returns the chart labels of one agent's probes
|
|
// (id → name), restricted to the tasks assigned to it (hub
|
|
// ping_task_names): names are operator text and the rest of the
|
|
// table belongs to agents this caller may not be able to see.
|
|
func (s *Store) PingTaskNames(agentID int64) (map[string]string, error) {
|
|
rows, err := s.db.Query(`SELECT id, name FROM probe_ping_task
|
|
WHERE id IN (SELECT task_id FROM probe_ping_node WHERE agent_id = ?1)`, agentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := map[string]string{}
|
|
for rows.Next() {
|
|
var id int64
|
|
var name string
|
|
if err := rows.Scan(&id, &name); err != nil {
|
|
return nil, err
|
|
}
|
|
out[fmt.Sprint(id)] = name
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// InsertPings files one agent's reported results at the arrival
|
|
// stamp ts, one transaction for the batch (hub insert_pings). A
|
|
// result only lands under a task the agent is assigned: the test is
|
|
// inside the statement, the only place it is atomic with the write.
|
|
// Results for anything else — a probe deleted while its results were
|
|
// in flight, a task id the reporter invented — are dropped, not
|
|
// errors, since the agent can do nothing useful with the distinction.
|
|
func (s *Store) InsertPings(agentID int64, results []PingResult, ts int64) error {
|
|
if len(results) == 0 {
|
|
return nil
|
|
}
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
stmt, err := tx.Prepare(`INSERT OR REPLACE INTO probe_ping_record (agent_id, ts, task_id, latency)
|
|
SELECT ?1, ?2, ?3, ?4
|
|
WHERE EXISTS (SELECT 1 FROM probe_ping_node WHERE task_id = ?3 AND agent_id = ?1)`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer stmt.Close()
|
|
for _, r := range results {
|
|
if _, err := stmt.Exec(agentID, ts, r.TaskID, r.LatencyMs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// PingRow is one thinned latency point of the contract chart (hub
|
|
// ping_records rows): the bucket's median answer, present keys only
|
|
// — band appears when the bucket actually varied, loss when it lost
|
|
// anything (absence reads as 0, the themes' documented semantics).
|
|
type PingRow struct {
|
|
TaskID int64 `json:"task_id"`
|
|
Ts int64 `json:"ts"`
|
|
Latency *int64 `json:"latency"`
|
|
Band *[2]int64 `json:"band,omitempty"`
|
|
Loss *int64 `json:"loss,omitempty"`
|
|
}
|
|
|
|
type pingWeightedMedian struct {
|
|
value int64
|
|
weight int64
|
|
}
|
|
|
|
// pingReadTally is one task's samples within one bucket of the chart
|
|
// read (hub Tally): hourly rows enter as their hour's median weighted
|
|
// by their answer count, single results as themselves.
|
|
type pingReadTally struct {
|
|
medians []pingWeightedMedian
|
|
lost int64
|
|
lo, hi *int64
|
|
}
|
|
|
|
func (t *pingReadTally) add(answered, lost int64, median, lo, hi *int64) {
|
|
if median != nil {
|
|
t.medians = append(t.medians, pingWeightedMedian{value: *median, weight: answered})
|
|
}
|
|
t.lost += lost
|
|
if lo != nil && (t.lo == nil || *lo < *t.lo) {
|
|
v := *lo
|
|
t.lo = &v
|
|
}
|
|
if hi != nil && (t.hi == nil || *hi > *t.hi) {
|
|
v := *hi
|
|
t.hi = &v
|
|
}
|
|
}
|
|
|
|
func (t *pingReadTally) answered() int64 {
|
|
var total int64
|
|
for _, m := range t.medians {
|
|
total += m.weight
|
|
}
|
|
return total
|
|
}
|
|
|
|
// median is the weighted median of the answers: the middle answer's
|
|
// value, or the average of the middle two (hub Tally::median).
|
|
func (t *pingReadTally) median() *int64 {
|
|
if len(t.medians) == 0 {
|
|
return nil
|
|
}
|
|
sorted := append([]pingWeightedMedian(nil), t.medians...)
|
|
sort.Slice(sorted, func(i, j int) bool {
|
|
if sorted[i].value != sorted[j].value {
|
|
return sorted[i].value < sorted[j].value
|
|
}
|
|
return sorted[i].weight < sorted[j].weight
|
|
})
|
|
total := t.answered()
|
|
at := func(rank int64) (int64, bool) {
|
|
var seen int64
|
|
for _, m := range sorted {
|
|
seen += m.weight
|
|
if seen >= rank {
|
|
return m.value, true
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
lo, ok1 := at((total + 1) / 2)
|
|
hi, ok2 := at(total/2 + 1)
|
|
if !ok1 || !ok2 {
|
|
return nil
|
|
}
|
|
v := (lo + hi) / 2
|
|
return &v
|
|
}
|
|
|
|
// PingHistory answers one agent's latency window (hub ping_records):
|
|
// one row per task per step-sized bucket holding the bucket median,
|
|
// its range and its loss, plus the per-task loss over the whole
|
|
// window (unrounded — the caller renders it). Only tasks still
|
|
// assigned to the agent contribute. Hourly windows merge the folded
|
|
// hourly tier (up to the watermark) with the minute rows after it,
|
|
// one read snapshot throughout.
|
|
func (s *Store) PingHistory(agentID int64, sp Span) ([]PingRow, map[string]float64, error) {
|
|
tx, err := s.db.Begin()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
out := []PingRow{}
|
|
totals := make(map[int64][2]int64) // task → [lost, samples]
|
|
type openTally struct {
|
|
task int64
|
|
tally *pingReadTally
|
|
}
|
|
var open []openTally
|
|
var bucket int64
|
|
flush := func() {
|
|
for _, o := range open {
|
|
row := PingRow{TaskID: o.task, Ts: bucket * sp.Step, Latency: o.tally.median()}
|
|
if o.tally.lo != nil && o.tally.hi != nil && *o.tally.hi > *o.tally.lo {
|
|
row.Band = &[2]int64{*o.tally.lo, *o.tally.hi}
|
|
}
|
|
if o.tally.lost > 0 {
|
|
total := o.tally.answered() + o.tally.lost
|
|
loss := (100*o.tally.lost + total - 1) / total
|
|
row.Loss = &loss
|
|
}
|
|
out = append(out, row)
|
|
}
|
|
open = open[:0]
|
|
}
|
|
feed := func(b, task int64, answered, lost int64, median, lo, hi *int64) {
|
|
if b != bucket {
|
|
flush()
|
|
bucket = b
|
|
}
|
|
tot := totals[task]
|
|
tot[0] += lost
|
|
tot[1] += answered + lost
|
|
totals[task] = tot
|
|
for i := range open {
|
|
if open[i].task == task {
|
|
open[i].tally.add(answered, lost, median, lo, hi)
|
|
return
|
|
}
|
|
}
|
|
tally := &pingReadTally{}
|
|
tally.add(answered, lost, median, lo, hi)
|
|
open = append(open, openTally{task, tally})
|
|
}
|
|
feedRow := func(b, task, latency int64) {
|
|
if latency >= 0 {
|
|
feed(b, task, 1, 0, &latency, &latency, &latency)
|
|
} else {
|
|
feed(b, task, 0, 1, nil, nil, nil)
|
|
}
|
|
}
|
|
|
|
minutesFrom := sp.Since
|
|
if sp.Hourly {
|
|
rolled := int64(-1) << 62 // hub: unwrap_or(i64::MIN)
|
|
var raw sql.NullString
|
|
if qerr := tx.QueryRow(`SELECT value FROM probe_setting WHERE key = ?`, rolledKey).Scan(&raw); qerr != nil && qerr != sql.ErrNoRows {
|
|
return nil, nil, qerr
|
|
} else if raw.Valid {
|
|
if n, perr := parseInt64(raw.String); perr == nil {
|
|
rolled = n
|
|
}
|
|
}
|
|
var newest sql.NullInt64
|
|
if err := tx.QueryRow(`SELECT MAX(ts) FROM probe_ping_record WHERE agent_id=?1`, agentID).Scan(&newest); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if newest.Valid && newest.Int64-DetailDays*86400 > minutesFrom {
|
|
minutesFrom = newest.Int64 - DetailDays*86400
|
|
}
|
|
if rolled > minutesFrom {
|
|
minutesFrom = rolled
|
|
}
|
|
hours, err := tx.Query(`SELECT ts/?3, task_id, answered, lost, latency, lo, hi FROM probe_ping_hour
|
|
WHERE agent_id=?1 AND ts>=?2 AND ts<?4
|
|
AND task_id IN (SELECT task_id FROM probe_ping_node WHERE agent_id=?1)
|
|
ORDER BY ts`, agentID, sp.Since, sp.Step, rolled)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for hours.Next() {
|
|
var b, task, answered, lost int64
|
|
var latency, lo, hi sql.NullInt64
|
|
if err := hours.Scan(&b, &task, &answered, &lost, &latency, &lo, &hi); err != nil {
|
|
hours.Close()
|
|
return nil, nil, err
|
|
}
|
|
feed(b, task, answered, lost, nullIntPtr(latency), nullIntPtr(lo), nullIntPtr(hi))
|
|
}
|
|
if err := hours.Err(); err != nil {
|
|
hours.Close()
|
|
return nil, nil, err
|
|
}
|
|
hours.Close()
|
|
}
|
|
|
|
rows, err := tx.Query(`SELECT ts/?3, task_id, latency FROM probe_ping_record
|
|
WHERE agent_id=?1 AND ts>=?2
|
|
AND task_id IN (SELECT task_id FROM probe_ping_node WHERE agent_id=?1)
|
|
ORDER BY ts`, agentID, minutesFrom, sp.Step)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for rows.Next() {
|
|
var b, task, latency int64
|
|
if err := rows.Scan(&b, &task, &latency); err != nil {
|
|
rows.Close()
|
|
return nil, nil, err
|
|
}
|
|
feedRow(b, task, latency)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return nil, nil, err
|
|
}
|
|
rows.Close()
|
|
flush()
|
|
|
|
// Panel order decides each task's series order (hub: themes take
|
|
// colours and legend from first appearance across the window,
|
|
// which per-bucket feeding would leave to chance).
|
|
rank := make(map[int64]int)
|
|
rankRows, err := tx.Query(`SELECT id FROM probe_ping_task ORDER BY sort, id`)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for rankRows.Next() {
|
|
var id int64
|
|
if err := rankRows.Scan(&id); err != nil {
|
|
rankRows.Close()
|
|
return nil, nil, err
|
|
}
|
|
rank[id] = len(rank)
|
|
}
|
|
rankRows.Close()
|
|
if err := rankRows.Err(); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
const unranked = int(^uint(0) >> 1)
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
ri, ok := rank[out[i].TaskID]
|
|
if !ok {
|
|
ri = unranked
|
|
}
|
|
rj, ok := rank[out[j].TaskID]
|
|
if !ok {
|
|
rj = unranked
|
|
}
|
|
return ri < rj
|
|
})
|
|
|
|
loss := map[string]float64{}
|
|
for task, tot := range totals {
|
|
if tot[0] > 0 && tot[1] > 0 {
|
|
loss[fmt.Sprint(task)] = 100.0 * float64(tot[0]) / float64(tot[1])
|
|
}
|
|
}
|
|
return out, loss, nil
|
|
}
|
|
|
|
func int64Set(ids []int64) map[int64]bool {
|
|
out := make(map[int64]bool, len(ids))
|
|
for _, id := range ids {
|
|
out[id] = true
|
|
}
|
|
return out
|
|
}
|
|
|
|
func nullIntPtr(v sql.NullInt64) *int64 {
|
|
if !v.Valid {
|
|
return nil
|
|
}
|
|
return &v.Int64
|
|
}
|
|
|
|
func parseInt64(s string) (int64, error) {
|
|
return strconv.ParseInt(s, 10, 64)
|
|
}
|