feat(probe): history store + whole-machine traffic ledger in probe.db (item 7 B2)
New internal/web/service/probe package keeps probe history in its own probe.db beside the panel DB (single connection, WAL), so history writes never contend with the panel write lock. - Minute rows per hub semantics: a held-reading rhythm books traffic against each reading own arrival; minute buckets merge (means, rounded int means, peaks) and the stored net rate derives from the kernel counters between row marks, falling back to agent rates at epoch breaks. - probe_traffic ledger: accumulate is a line port of hub traffic.rs (baseline alignment, same-boot shrink books 0, epoch change books 0, unseen-boot exception books the whole counters-since-boot measured against the master last-seen stamp, per-direction period resets, saturating adds); reads gate stale day/month stamps to zero. - probe_meta rows are ensured for every reporting agent (B3 edits them in place); probe_ping_* tables are created now, written by B4. - Hourly roll_up behind a watermark (hour-aligned, late grace, pre-retention hours skipped unfoldered) and watermark-guarded chunked prune (minute tier 7 days or the retention window when shorter; hourly tier to retention_days, default 7, cap 365). - Wiring: heartbeat files probe samples after the snapshot update (last-seen captured before touch); housekeeping hangs off the existing robfig/cron in web.startTask (@hourly + one pass a minute after startup, mirroring the hub boot pass); Server.stop books held readings. Every failure path logs and degrades - a broken probe.db never fails a heartbeat or the panel.
This commit is contained in:
Rogue
@@ -0,0 +1,231 @@
|
||||
// Package probe is the master-side history store of item 7 (probe merge):
|
||||
// it persists the per-minute metric rows and the whole-machine traffic
|
||||
// ledger reported by agents (B1) into a dedicated SQLite file, probe.db,
|
||||
// beside the panel database. The model, the booking rules and the
|
||||
// roll-up/prune discipline are a port of monitor-probe's hub (src/db.rs,
|
||||
// src/agent_ws.rs); see ITEM7-SCOPE.md §3.2/§3.3.
|
||||
//
|
||||
// probe.db is deliberately a separate file from x-ui.db: history writes,
|
||||
// roll-ups and prunes then never contend for the panel database's write
|
||||
// lock (ITEM7-SCOPE §7 risk 4). Everything here is best-effort by design:
|
||||
// a broken or missing probe.db must never fail a heartbeat or a panel
|
||||
// request, so the facade entry points log and swallow errors.
|
||||
package probe
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/config"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/logger"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// DetailDays is how long minute rows are kept before they may be pruned;
|
||||
// it only ever bounds the minute tier — the hourly tier and the traffic
|
||||
// ledger live by the retention window itself. Mirrors hub DETAIL_DAYS.
|
||||
const DetailDays = 7
|
||||
|
||||
// lateSeconds is the grace a just-closed hour gets before roll_up folds
|
||||
// it, so rows landing slightly late still make it into their hour.
|
||||
// Mirrors hub LATE (3600 + 60).
|
||||
const lateSeconds = 3600 + 60
|
||||
|
||||
// rolledKey is the probe_setting key holding the roll-up watermark: the
|
||||
// start (unix seconds) of the next hour not yet folded into the hourly
|
||||
// tier. Mirrors hub ROLLED ("history_rolled").
|
||||
const rolledKey = "history_rolled"
|
||||
|
||||
const schema = `
|
||||
CREATE TABLE IF NOT EXISTS probe_setting (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Per-node probe metadata (ITEM7-SCOPE §5.2). B2 creates the table and
|
||||
-- reads traffic_reset_day; the editing surface lands in B3.
|
||||
CREATE TABLE IF NOT EXISTS probe_meta (
|
||||
agent_id INTEGER PRIMARY KEY,
|
||||
price REAL NOT NULL DEFAULT 0,
|
||||
currency TEXT NOT NULL DEFAULT 'USD',
|
||||
billing_cycle TEXT NOT NULL DEFAULT 'monthly',
|
||||
expires_at TEXT,
|
||||
traffic_limit INTEGER NOT NULL DEFAULT 0,
|
||||
traffic_mode TEXT NOT NULL DEFAULT 'sum',
|
||||
traffic_reset_day INTEGER NOT NULL DEFAULT 1,
|
||||
public INTEGER NOT NULL DEFAULT 1,
|
||||
sort INTEGER NOT NULL DEFAULT 0,
|
||||
group_name TEXT NOT NULL DEFAULT '',
|
||||
public_remark TEXT NOT NULL DEFAULT '',
|
||||
notify INTEGER NOT NULL DEFAULT 0,
|
||||
country_pin TEXT NOT NULL DEFAULT '',
|
||||
ipv4_pin TEXT NOT NULL DEFAULT '',
|
||||
ipv6_pin TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- Monotonic byte counters that survive agent reboots and master
|
||||
-- restarts; the whole-machine ledger (hub "traffic"). One row per agent.
|
||||
CREATE TABLE IF NOT EXISTS probe_traffic (
|
||||
agent_id INTEGER PRIMARY KEY,
|
||||
boot_id TEXT NOT NULL DEFAULT '',
|
||||
last_rx INTEGER NOT NULL DEFAULT 0,
|
||||
last_tx INTEGER NOT NULL DEFAULT 0,
|
||||
total_rx INTEGER NOT NULL DEFAULT 0,
|
||||
total_tx INTEGER NOT NULL DEFAULT 0,
|
||||
month_rx INTEGER NOT NULL DEFAULT 0,
|
||||
month_tx INTEGER NOT NULL DEFAULT 0,
|
||||
month_start TEXT NOT NULL DEFAULT '',
|
||||
day_rx INTEGER NOT NULL DEFAULT 0,
|
||||
day_tx INTEGER NOT NULL DEFAULT 0,
|
||||
day_start TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
-- One row per agent per minute (hub "metric"). net_rx/net_tx are bytes
|
||||
-- per second derived from the kernel counters, the *_max columns are the
|
||||
-- minute's peaks of the agent-reported figures.
|
||||
CREATE TABLE IF NOT EXISTS probe_metric (
|
||||
agent_id INTEGER NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
cpu REAL NOT NULL,
|
||||
mem_used INTEGER NOT NULL,
|
||||
swap_used INTEGER NOT NULL,
|
||||
disk_used INTEGER NOT NULL,
|
||||
net_rx INTEGER NOT NULL,
|
||||
net_tx INTEGER NOT NULL,
|
||||
tcp INTEGER NOT NULL,
|
||||
udp INTEGER NOT NULL,
|
||||
procs INTEGER NOT NULL,
|
||||
net_rx_max INTEGER NOT NULL DEFAULT 0,
|
||||
net_tx_max INTEGER NOT NULL DEFAULT 0,
|
||||
cpu_max REAL NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (agent_id, ts)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- The hourly tier (hub "metric_hour"): every metric column plus the
|
||||
-- count of minute rows folded in, which weights later merges.
|
||||
CREATE TABLE IF NOT EXISTS probe_metric_hour (
|
||||
agent_id INTEGER NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
minutes INTEGER NOT NULL,
|
||||
cpu REAL NOT NULL,
|
||||
cpu_max REAL NOT NULL,
|
||||
mem_used INTEGER NOT NULL,
|
||||
swap_used INTEGER NOT NULL,
|
||||
disk_used INTEGER NOT NULL,
|
||||
net_rx INTEGER NOT NULL,
|
||||
net_tx INTEGER NOT NULL,
|
||||
net_rx_max INTEGER NOT NULL,
|
||||
net_tx_max INTEGER NOT NULL,
|
||||
tcp INTEGER NOT NULL,
|
||||
udp INTEGER NOT NULL,
|
||||
procs INTEGER NOT NULL,
|
||||
PRIMARY KEY (agent_id, ts)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
-- Probe results (hub "ping_record" / "ping_hour"). B2 creates the
|
||||
-- tables so roll_up/prune cover them uniformly; B4 writes the rows.
|
||||
-- latency < 0 is a lost probe; the hourly latency is the median of the
|
||||
-- hour's answers and NULL when none arrived.
|
||||
CREATE TABLE IF NOT EXISTS probe_ping_record (
|
||||
agent_id INTEGER NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
task_id INTEGER NOT NULL,
|
||||
latency INTEGER NOT NULL,
|
||||
PRIMARY KEY (agent_id, ts, task_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS probe_ping_hour (
|
||||
agent_id INTEGER NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
task_id INTEGER NOT NULL,
|
||||
answered INTEGER NOT NULL,
|
||||
lost INTEGER NOT NULL,
|
||||
latency INTEGER,
|
||||
lo INTEGER,
|
||||
hi INTEGER,
|
||||
PRIMARY KEY (agent_id, ts, task_id)
|
||||
) WITHOUT ROWID;
|
||||
`
|
||||
|
||||
// Store is one open probe.db. The zero value is not usable; call Open.
|
||||
// A Store is safe for concurrent use: the single connection serializes
|
||||
// writes, and the ingest state is mutex-guarded.
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
|
||||
mu sync.Mutex // guards nodes
|
||||
nodes map[int64]*nodeState
|
||||
}
|
||||
|
||||
// Open creates (if needed) and initializes the probe database at path.
|
||||
func Open(path string) (*Store, error) {
|
||||
if dir := filepath.Dir(path); dir != "" {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
dsn := path + "?_journal_mode=WAL&_busy_timeout=10000&_synchronous=NORMAL&_txlock=immediate"
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// One connection: writes serialize on it (hub's single write
|
||||
// connection), and WAL still lets the read side (B3) proceed.
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.Exec(schema); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("probe schema: %w", err)
|
||||
}
|
||||
return &Store{db: db, nodes: make(map[int64]*nodeState)}, nil
|
||||
}
|
||||
|
||||
// Close releases the database handle.
|
||||
func (s *Store) Close() error {
|
||||
if s == nil || s.db == nil {
|
||||
return nil
|
||||
}
|
||||
return s.db.Close()
|
||||
}
|
||||
|
||||
// DB exposes the raw handle for the read side (B3) and tests.
|
||||
func (s *Store) DB() *sql.DB { return s.db }
|
||||
|
||||
// --- default store (lazy, failure-tolerant) ---
|
||||
|
||||
var def struct {
|
||||
mu sync.Mutex
|
||||
store *Store
|
||||
retryAt time.Time
|
||||
}
|
||||
|
||||
// DefaultPath is where probe.db lives: beside the panel database.
|
||||
func DefaultPath() string {
|
||||
return filepath.Join(config.GetDBFolderPath(), "probe.db")
|
||||
}
|
||||
|
||||
// Default returns the process-wide store, opening it on first use. A
|
||||
// failed open is retried at most once a minute and reported as nil, so
|
||||
// callers degrade to "no history" instead of failing.
|
||||
func Default() *Store {
|
||||
def.mu.Lock()
|
||||
defer def.mu.Unlock()
|
||||
if def.store != nil {
|
||||
return def.store
|
||||
}
|
||||
if time.Now().Before(def.retryAt) {
|
||||
return nil
|
||||
}
|
||||
st, err := Open(DefaultPath())
|
||||
if err != nil {
|
||||
logger.Warningf("probe: open %s failed (history disabled until retry): %v", DefaultPath(), err)
|
||||
def.retryAt = time.Now().Add(time.Minute)
|
||||
return nil
|
||||
}
|
||||
def.store = st
|
||||
return st
|
||||
}
|
||||
Reference in New Issue
Block a user