feat(probe): history store + whole-machine traffic ledger in probe.db (item 7 B2)

New internal/web/service/probe package keeps probe history in its own
probe.db beside the panel DB (single connection, WAL), so history
writes never contend with the panel write lock.

- Minute rows per hub semantics: a held-reading rhythm books traffic
  against each reading own arrival; minute buckets merge (means,
  rounded int means, peaks) and the stored net rate derives from the
  kernel counters between row marks, falling back to agent rates at
  epoch breaks.
- probe_traffic ledger: accumulate is a line port of hub traffic.rs
  (baseline alignment, same-boot shrink books 0, epoch change books 0,
  unseen-boot exception books the whole counters-since-boot measured
  against the master last-seen stamp, per-direction period resets,
  saturating adds); reads gate stale day/month stamps to zero.
- probe_meta rows are ensured for every reporting agent (B3 edits
  them in place); probe_ping_* tables are created now, written by B4.
- Hourly roll_up behind a watermark (hour-aligned, late grace,
  pre-retention hours skipped unfoldered) and watermark-guarded
  chunked prune (minute tier 7 days or the retention window when
  shorter; hourly tier to retention_days, default 7, cap 365).
- Wiring: heartbeat files probe samples after the snapshot update
  (last-seen captured before touch); housekeeping hangs off the
  existing robfig/cron in web.startTask (@hourly + one pass a minute
  after startup, mirroring the hub boot pass); Server.stop books held
  readings. Every failure path logs and degrades - a broken probe.db
  never fails a heartbeat or the panel.
This commit is contained in:
Rogue
2026-10-22 19:43:59 +00:00
parent 0a973bc95c
commit 3e052c42b8
11 changed files with 2046 additions and 1 deletions
+61
View File
@@ -17,6 +17,7 @@ import (
"github.com/mhsanaei/3x-ui/v3/internal/logger"
"github.com/mhsanaei/3x-ui/v3/internal/selfupdate"
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
"github.com/mhsanaei/3x-ui/v3/internal/web/service/probe"
"github.com/mhsanaei/3x-ui/v3/internal/xray"
"github.com/gin-gonic/gin"
@@ -276,6 +277,7 @@ type heartbeatProbeFacts struct {
type heartbeatProbeMetrics struct {
BootID *string `json:"bootId"`
Iface *string `json:"iface"`
Uptime *uint64 `json:"uptime"`
Load *[3]float32 `json:"load"`
MemTotal *uint64 `json:"memTotal"`
MemUsed *uint64 `json:"memUsed"`
@@ -393,6 +395,55 @@ func (a *AgentController) updateAgentProbe(id uint, f *heartbeatProbeFacts, m *h
_ = database.GetDB().Model(&model.AgentNode{}).Where("id = ?", id).Updates(updates).Error
}
// probeSample flattens a heartbeat metrics block into the probe
// package's Sample. Absent fields become zero values — the agent sends
// a full block whenever it sends one, and the probe store treats the
// block's presence (not per-field presence) as the signal, mirroring
// the hub's report frames.
func probeSample(m *heartbeatProbeMetrics) probe.Sample {
var s probe.Sample
if m.BootID != nil {
s.BootID = *m.BootID
}
if m.Uptime != nil {
s.Uptime = *m.Uptime
}
if m.CPU != nil {
s.CPU = *m.CPU
}
if m.MemUsed != nil {
s.MemUsed = *m.MemUsed
}
if m.SwapUsed != nil {
s.SwapUsed = *m.SwapUsed
}
if m.DiskUsed != nil {
s.DiskUsed = *m.DiskUsed
}
if m.NetRxTotal != nil {
s.NetRxTotal = *m.NetRxTotal
}
if m.NetTxTotal != nil {
s.NetTxTotal = *m.NetTxTotal
}
if m.NetRx != nil {
s.NetRx = *m.NetRx
}
if m.NetTx != nil {
s.NetTx = *m.NetTx
}
if m.TCP != nil {
s.TCP = *m.TCP
}
if m.UDP != nil {
s.UDP = *m.UDP
}
if m.Procs != nil {
s.Procs = *m.Procs
}
return s
}
// heartbeatUpdateStatus is the agent's report about a one-click update
// intent (item 20), piggybacked on the heartbeat like telemetry.
type heartbeatUpdateStatus struct {
@@ -476,11 +527,21 @@ func (a *AgentController) heartbeat(c *gin.Context) {
ProbeMetrics *heartbeatProbeMetrics `json:"probeMetrics"`
}
_ = c.ShouldBindJSON(&req)
// The last-seen stamp as it stood before this heartbeat: the probe
// ledger's unseen-boot rule measures a reboot against it, so it
// must be captured before touchAgent overwrites it.
prevLastSeen := agent.LastSeen
a.touchAgent(uint(agent.Id), "")
reportedVersion := agent.AgentVersion
if req.ProbeFacts != nil || req.ProbeMetrics != nil {
a.updateAgentProbe(uint(agent.Id), req.ProbeFacts, req.ProbeMetrics)
}
if req.ProbeMetrics != nil {
// Item 7 B2: file the sample into probe.db history + the
// whole-machine traffic ledger. Best-effort by design — the
// probe package logs and swallows its own failures.
probe.Record(int64(agent.Id), probeSample(req.ProbeMetrics), prevLastSeen)
}
if req.Telemetry != nil {
a.updateAgentTelemetry(uint(agent.Id), req.Telemetry)
if req.Telemetry.AgentVersion != nil && *req.Telemetry.AgentVersion != "" {