B6 backend: probe notification engine — hub-faithful sweep (offline/recovery with flap discipline, traffic steps, expiry digest + auto-renewal, login), Telegram via the panel bot + webhook channel, notify settings/test/coverage API, login hook

2026-10-22 23:50:07 +00:00
parent f646212df2
commit 031b422365
14 changed files with 2353 additions and 0 deletions
@@ -206,6 +206,10 @@ func (a *APIController) initRouter(g *gin.RouterGroup) {
 	// group itself (/panel/api/probe/*).
 	NewProbeThemesController(api)

+	// Item 7 B6: notification settings + test send + cf-probe
+	// retirement coverage, same group (/panel/api/probe/notify*).
+	NewProbeNotifyController(api)
+
 	// Hosts API — per-inbound override endpoints for subscription links
 	hosts := api.Group("/hosts")
 	a.hostController = NewHostController(hosts)
@@ -9,6 +9,7 @@ import (
 	"github.com/mhsanaei/3x-ui/v3/internal/web/middleware"
 	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
 	"github.com/mhsanaei/3x-ui/v3/internal/web/service/panel"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service/probenotify"
 	"github.com/mhsanaei/3x-ui/v3/internal/web/service/tgbot"
 	"github.com/mhsanaei/3x-ui/v3/internal/web/session"

@@ -124,6 +125,10 @@ func (a *IndexController) login(c *gin.Context) {
 		return
 	}

+	// Item 7 B6: the probe notification system's sign-in alert
+	// (queued, non-blocking; notify_login gates it inside).
+	probenotify.SignedIn("密码登录", remoteIP)
+
 	logger.Infof("%s logged in successfully", safeUser)
 	jsonMsg(c, I18nWeb(c, "pages.login.toasts.successLogin"), nil)
 }
@@ -0,0 +1,68 @@
+package controller
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/web/entity"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service/probenotify"
+
+	"github.com/gin-gonic/gin"
+)
+
+// Item 7 B6: the notification settings surface — the notify_* block
+// (offline grace, traffic step, expiry window, login alerts, the
+// webhook channel and the two message templates), a test send, and
+// the cf-probe retirement coverage the operator checks before
+// standing the old probe fleet down. Like the rest of the probe
+// surfaces these endpoints answer in the panel's Msg envelope and
+// take effect immediately; validation refusals are operator-facing
+// text, shown verbatim.
+
+type ProbeNotifyController struct{}
+
+func NewProbeNotifyController(g *gin.RouterGroup) *ProbeNotifyController {
+	pc := &ProbeNotifyController{}
+	g.GET("/probe/notify", pc.get)
+	g.PUT("/probe/notify", pc.put)
+	g.POST("/probe/notify/test", pc.test)
+	return pc
+}
+
+type probeNotifyPayload struct {
+	Settings probenotify.SettingsView   `json:"settings"`
+	Coverage []service.ProbeCoverageRow `json:"coverage"`
+}
+
+func (pc *ProbeNotifyController) get(c *gin.Context) {
+	jsonObj(c, probeNotifyPayload{
+		Settings: probenotify.GetSettings(),
+		Coverage: service.ProbeAgentCoverage(),
+	}, nil)
+}
+
+func (pc *ProbeNotifyController) put(c *gin.Context) {
+	var body map[string]json.RawMessage
+	if err := c.ShouldBindJSON(&body); err != nil {
+		c.JSON(http.StatusOK, entity.Msg{Success: false, Msg: "请求格式不对"})
+		return
+	}
+	if err := probenotify.SaveSettings(body); err != nil {
+		// SaveSettings validates everything before writing anything;
+		// its errors are the operator's answer (a range, a malformed
+		// header block), not machine faults.
+		c.JSON(http.StatusOK, entity.Msg{Success: false, Msg: err.Error()})
+		return
+	}
+	jsonMsg(c, "", nil)
+}
+
+func (pc *ProbeNotifyController) test(c *gin.Context) {
+	sent, err := probenotify.Test()
+	if err != nil {
+		c.JSON(http.StatusOK, entity.Msg{Success: false, Msg: err.Error()})
+		return
+	}
+	jsonObj(c, gin.H{"sent": sent}, nil)
+}
@@ -0,0 +1,38 @@
+package job
+
+import (
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service/probenotify"
+)
+
+// ProbeNotifyJob is the notification sweep (item 7 B6): every 30s it
+// examines agent connectivity and billing-period traffic and queues
+// the offline / recovery / traffic alerts. The engine swallows its
+// own failures into log lines — a broken notification pass must
+// never disturb the panel.
+type ProbeNotifyJob struct{}
+
+// NewProbeNotifyJob creates a probe notification sweep job instance.
+func NewProbeNotifyJob() *ProbeNotifyJob {
+	return new(ProbeNotifyJob)
+}
+
+// Run executes one sweep pass.
+func (j *ProbeNotifyJob) Run() {
+	probenotify.Sweep()
+}
+
+// ProbeNotifyHousekeepingJob is the hourly half (item 7 B6): roll
+// online nodes' expiry dates forward, then send the day's expiry
+// digest over the dates as they now stand.
+type ProbeNotifyHousekeepingJob struct{}
+
+// NewProbeNotifyHousekeepingJob creates a probe notification
+// housekeeping job instance.
+func NewProbeNotifyHousekeepingJob() *ProbeNotifyHousekeepingJob {
+	return new(ProbeNotifyHousekeepingJob)
+}
+
+// Run executes one housekeeping pass.
+func (j *ProbeNotifyHousekeepingJob) Run() {
+	probenotify.Housekeeping()
+}
@@ -2,6 +2,7 @@ package probe

 import (
 	"database/sql"
+	"strconv"
 	"strings"
 )

@@ -149,5 +150,12 @@ func (s *Store) DeleteAgentData(agentID int64) error {
 			return err
 		}
 	}
+	// The B6 offline mark is keyed per agent in probe_setting; the
+	// traffic steps are one shared JSON value the sweep prunes to
+	// live agents on its next write, so only the mark is struck here.
+	if _, err := tx.Exec(`DELETE FROM probe_setting WHERE key = ?`,
+		"notify_down:"+strconv.FormatInt(agentID, 10)); err != nil {
+		return err
+	}
 	return tx.Commit()
 }
@@ -0,0 +1,60 @@
+package probe
+
+import (
+	"database/sql"
+	"errors"
+)
+
+// Item 7 B6: the notification engine's durable state lives in
+// probe_setting, next to the roll-up watermark — the offline marks
+// (one key per agent, "notify_down:<id>", the hub's down_since column
+// in KV form so DeleteAgentData can strike exactly one agent's mark),
+// the traffic steps already announced ("traffic_alerted", hub's key
+// of the same name) and the expiry digest's last-sent date
+// ("notify_expiry_sent"). The store stays the only writer of
+// probe.db; the notify package reads and writes through here.
+
+// StateGet reads one probe_setting value from the default store.
+// ok is false when the key was never written; err is non-nil when no
+// store is available, which callers treat as "skip this section"
+// (the hub skips the traffic steps on a failed read rather than
+// recording crossings it did not announce).
+func StateGet(key string) (string, bool, error) {
+	st := Default()
+	if st == nil {
+		return "", false, errors.New("probe store unavailable")
+	}
+	return st.getState(key)
+}
+
+// StateSet writes one probe_setting value on the default store. An
+// empty value deletes the key, so "cleared" and "never set" are the
+// same state, as with the hub's down_since = 0.
+func StateSet(key, value string) error {
+	st := Default()
+	if st == nil {
+		return errors.New("probe store unavailable")
+	}
+	return st.setState(key, value)
+}
+
+func (s *Store) getState(key string) (string, bool, error) {
+	var v string
+	err := s.db.QueryRow(`SELECT value FROM probe_setting WHERE key = ?`, key).Scan(&v)
+	if err == sql.ErrNoRows {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+
+func (s *Store) setState(key, value string) error {
+	if value == "" {
+		_, err := s.db.Exec(`DELETE FROM probe_setting WHERE key = ?`, key)
+		return err
+	}
+	_, err := s.db.Exec(`INSERT OR REPLACE INTO probe_setting (key, value) VALUES (?, ?)`, key, value)
+	return err
+}
@@ -0,0 +1,57 @@
+package service
+
+import (
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+// ProbeCoverageRow is one agent's cf-probe retirement readiness
+// (item 7 B6): whether its heartbeats carry the probe block the
+// public page is fed from. The retirement checklist in the settings
+// tab renders from these rows — the operator verifies every node is
+// covered by the in-panel probe before standing the old cf-probe
+// fleet down, node by node.
+type ProbeCoverageRow struct {
+	Name     string `json:"name"`
+	Online   bool   `json:"online"`
+	HasProbe bool   `json:"hasProbe"`
+	ProbeAt  int64  `json:"probeAt"` // unix ms of the last probe-carrying heartbeat, 0 = never
+}
+
+// ProbeAgentCoverage lists every agent with its probe coverage. The
+// display name follows the public view's rule (the bound node's name
+// first, the agent's own name otherwise); online is the public
+// view's 150s freshness window.
+func ProbeAgentCoverage() []ProbeCoverageRow {
+	db := database.GetDB()
+	var agents []model.AgentNode
+	if err := db.Find(&agents).Error; err != nil {
+		return []ProbeCoverageRow{}
+	}
+	nodeNames := map[int]string{}
+	var nodes []model.Node
+	if err := db.Find(&nodes).Error; err == nil {
+		for _, n := range nodes {
+			nodeNames[n.Id] = n.Name
+		}
+	}
+	nowMs := time.Now().UnixMilli()
+	out := make([]ProbeCoverageRow, 0, len(agents))
+	for _, a := range agents {
+		name := a.Name
+		if a.NodeID != nil {
+			if bound, ok := nodeNames[*a.NodeID]; ok && bound != "" {
+				name = bound
+			}
+		}
+		out = append(out, ProbeCoverageRow{
+			Name:     name,
+			Online:   a.LastSeen > 0 && nowMs-a.LastSeen <= 150_000,
+			HasProbe: a.ProbeAt > 0,
+			ProbeAt:  a.ProbeAt,
+		})
+	}
+	return out
+}
@@ -0,0 +1,324 @@
+package probenotify
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/logger"
+)
+
+// Delivery: a queue of 64 drained by one worker, each channel retried
+// independently (3 attempts, 10s apart, retryable failures only).
+// At most once: an alert that exhausts its attempts is dropped, and
+// the state that produced it (the offline mark, the traffic step,
+// the digest date) already records it as sent, so it is not raised
+// again (hub deliver()).
+const (
+	queueSize   = 64
+	attempts    = 3
+	retryPause  = 10 * time.Second
+	postTimeout = 15 * time.Second
+)
+
+var (
+	inbox      = make(chan Note, queueSize)
+	workerOnce sync.Once
+)
+
+// Send queues an alert without waiting, so that sign-in and the sweep
+// never stall behind a slow channel. Beyond the queue the alert is
+// dropped with a warning rather than held in memory without bound.
+func Send(note Note) {
+	note.Time = time.Now().Unix()
+	workerOnce.Do(func() { go deliver() })
+	select {
+	case inbox <- note:
+	default:
+		logger.Warningf("probenotify: queue full, dropped %q", note.Title)
+	}
+}
+
+func deliver() {
+	for note := range inbox {
+		for _, ch := range channelsFn() {
+			for attempt := 1; attempt <= attempts; attempt++ {
+				f := ch.post(note)
+				if f == nil {
+					break
+				}
+				if !f.retry || attempt == attempts {
+					logger.Warningf("probenotify: %s alert %q not delivered: %s", ch.name(), note.Title, f.reason)
+					break
+				}
+				logger.Debugf("probenotify: %s alert %q, attempt %d: %s", ch.name(), note.Title, attempt, f.reason)
+				time.Sleep(retryPause)
+			}
+		}
+	}
+}
+
+// failure is why a delivery failed, and whether another attempt could
+// change that. A refused credential or a malformed request fails
+// identically every time, and retrying it would hold the queue.
+// reason is for the log; shown is what the panel says, composed from
+// the status rather than quoted from the far end's answer. Neither
+// ever carries the request URL: Telegram's carries the bot token and
+// a webhook's is often the credential itself.
+type failure struct {
+	retry  bool
+	reason string
+	shown  string
+}
+
+// channel is one configured delivery target, read from the settings
+// at send time so a panel edit applies to the next alert.
+type channel interface {
+	name() string
+	post(note Note) *failure
+}
+
+var channelsFn = resolveChannels
+
+func resolveChannels() []channel {
+	var out []channel
+	if tg, ok := buildTelegramChannel(); ok {
+		out = append(out, tg)
+	}
+	if v, ok := settingFn("notify_webhook_url"); ok && v != "" {
+		headers, _ := settingFn("notify_webhook_headers")
+		out = append(out, webhookChannel{
+			url:     v,
+			headers: headers,
+			body:    templateSetting("notify_webhook_body", defaultBody),
+		})
+	}
+	return out
+}
+
+// TelegramConfigured reports whether the panel bot settings amount
+// to a usable Telegram channel (token + at least one numeric chat).
+func TelegramConfigured() bool {
+	_, ok := buildTelegramChannel()
+	return ok
+}
+
+// WebhookConfigured reports whether a webhook URL is set.
+func WebhookConfigured() bool {
+	v, ok := settingFn("notify_webhook_url")
+	return ok && v != ""
+}
+
+// buildTelegramChannel builds the Telegram channel from the PANEL bot's
+// settings (tgBotToken / tgBotChatId / tgBotProxy / tgBotAPIServer):
+// the notify system deliberately keeps no Telegram credentials of its
+// own (ITEM7-SCOPE §5.6). The chat setting is the bot's admin list —
+// comma-separated numeric ids; the alert goes to each.
+func buildTelegramChannel() (telegramChannel, bool) {
+	token, _ := settingFn("tgBotToken")
+	rawChats, _ := settingFn("tgBotChatId")
+	var chats []int64
+	for _, part := range strings.Split(rawChats, ",") {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		if id, err := strconv.ParseInt(part, 10, 64); err == nil {
+			chats = append(chats, id)
+		}
+	}
+	if token == "" || len(chats) == 0 {
+		return telegramChannel{}, false
+	}
+	apiBase, _ := settingFn("tgBotAPIServer")
+	proxy, _ := settingFn("tgBotProxy")
+	return telegramChannel{
+		token:   token,
+		chats:   chats,
+		text:    templateSetting("notify_telegram_text", defaultText),
+		apiBase: strings.TrimSuffix(apiBase, "/"),
+		proxy:   proxy,
+	}, true
+}
+
+type telegramChannel struct {
+	token   string
+	chats   []int64
+	text    string
+	apiBase string
+	proxy   string
+}
+
+func (telegramChannel) name() string { return "Telegram" }
+
+func (ch telegramChannel) post(note Note) *failure {
+	base := ch.apiBase
+	if base == "" {
+		base = "https://api.telegram.org"
+	}
+	// Plain text: under a parse mode Telegram rejects a message
+	// whose node name happens to contain markup.
+	text := render(ch.text, note, siteName(), false)
+	for _, chat := range ch.chats {
+		body, _ := json.Marshal(map[string]any{"chat_id": chat, "text": text})
+		req, err := http.NewRequest(http.MethodPost, base+"/bot"+ch.token+"/sendMessage", bytes.NewReader(body))
+		if err != nil {
+			return &failure{retry: false, reason: "请求构造失败", shown: "请求构造失败"}
+		}
+		req.Header.Set("Content-Type", "application/json")
+		if f := doPost(httpClient(ch.proxy), req, true); f != nil {
+			return f
+		}
+	}
+	return nil
+}
+
+type webhookChannel struct {
+	url     string
+	headers string
+	body    string
+}
+
+func (webhookChannel) name() string { return "Webhook" }
+
+func (ch webhookChannel) post(note Note) *failure {
+	headers, perr := parseHeaders(ch.headers)
+	if perr != "" {
+		return &failure{retry: false, reason: perr, shown: perr}
+	}
+	req, err := http.NewRequest(http.MethodPost, ch.url,
+		strings.NewReader(render(ch.body, note, siteName(), true)))
+	if err != nil {
+		return &failure{retry: false, reason: "请求构造失败", shown: "请求构造失败"}
+	}
+	// One map, assigned not appended: a configured Content-Type takes
+	// the default's place instead of riding alongside it (hub).
+	req.Header.Set("Content-Type", "application/json")
+	for _, h := range headers {
+		req.Header.Set(h[0], h[1])
+	}
+	return doPost(httpClient(""), req, false)
+}
+
+// httpClient is the alerts' client: 15s timeout and no redirect
+// following (a redirected POST loses its body or leaks credential
+// headers to the next host — hub builds its client the same way).
+// Telegram posts may ride the panel bot's configured proxy.
+func httpClient(proxy string) *http.Client {
+	transport := &http.Transport{}
+	if proxy != "" {
+		if u, err := url.Parse(proxy); err == nil && u.Host != "" {
+			transport.Proxy = http.ProxyURL(u)
+		}
+	}
+	return &http.Client{
+		Timeout:   postTimeout,
+		Transport: transport,
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+}
+
+func doPost(client *http.Client, req *http.Request, telegram bool) *failure {
+	resp, err := client.Do(req)
+	if err != nil {
+		// The error itself is never quoted: it embeds the URL.
+		shown := "连不上对方服务器"
+		var netErr interface{ Timeout() bool }
+		if errors.As(err, &netErr) && netErr.Timeout() {
+			shown = "请求超时"
+		}
+		return &failure{retry: true, reason: shown, shown: shown}
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return nil
+	}
+	// The first chunk carries the reason (Telegram's description,
+	// Discord's message) without reading an error page of arbitrary
+	// length.
+	head, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+	code := resp.StatusCode
+	var why string
+	switch {
+	case code >= 300 && code <= 399:
+		why = "地址发生了跳转，请填写跳转后的地址"
+	case telegram && (code == 401 || code == 404):
+		why = "Bot Token 不对"
+	case telegram && code == 400:
+		why = "Chat ID 不对，或者 bot 还没有加入这个会话"
+	case telegram && code == 403:
+		why = "bot 被这个会话移除或屏蔽了"
+	case code == 401 || code == 403:
+		why = "对方拒绝了鉴权，检查 URL 或请求头里的凭据"
+	case code == 404:
+		why = "地址不存在，检查 URL"
+	case code == 429:
+		why = "发送太频繁，被对方限流"
+	case code >= 500:
+		why = "对方服务器出错"
+	default:
+		why = "对方拒收了这条消息，检查请求体格式"
+	}
+	reason := strings.TrimSpace(string(head))
+	if len(reason) > 300 {
+		reason = reason[:300]
+	}
+	return &failure{
+		retry:  code >= 500 || code == http.StatusTooManyRequests,
+		reason: fmt.Sprintf("HTTP %d: %s", code, reason),
+		shown:  fmt.Sprintf("%s（HTTP %d）", why, code),
+	}
+}
+
+// Test sends one alert through every configured channel, past the
+// queue and its retries, so each channel's own error reaches the
+// panel (hub POST /api/notify/test). No channel configured is an
+// error of its own: "还没有配置通知渠道".
+func Test() (sent []string, err error) {
+	channels := channelsFn()
+	if len(channels) == 0 {
+		return nil, errors.New("还没有配置通知渠道")
+	}
+	note := Note{
+		Event:   "test",
+		Title:   "✅ 测试通知",
+		Message: "收到这条说明通知渠道可用",
+		Time:    time.Now().Unix(),
+	}
+	var failed []string
+	for _, ch := range channels {
+		if f := ch.post(note); f != nil {
+			logger.Warningf("probenotify: %s test alert not delivered: %s", ch.name(), f.reason)
+			failed = append(failed, fmt.Sprintf("%s：%s", ch.name(), f.shown))
+		} else {
+			sent = append(sent, ch.name())
+		}
+	}
+	if len(failed) > 0 {
+		return sent, errors.New(strings.Join(failed, "；"))
+	}
+	return sent, nil
+}
+
+// SignedIn raises the sign-in alert (hub signed_in): on by default,
+// the notify_login setting turns it off.
+func SignedIn(how, ip string) {
+	if !loginEnabled() {
+		return
+	}
+	Send(Note{
+		Event:   "login",
+		Title:   "🔑 面板登录",
+		Message: fmt.Sprintf("%s · 来自 %s", how, ip),
+	})
+}
@@ -0,0 +1,167 @@
+package probenotify
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestWebhookPostRendersAndClassifies(t *testing.T) {
+	newFakeEnv(t) // post() reads the site name through the settings seam
+	var mu sync.Mutex
+	var gotBody []byte
+	var gotHeader string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		gotBody, _ = io.ReadAll(r.Body)
+		gotHeader = r.Header.Get("X-Api-Key")
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer srv.Close()
+
+	ch := webhookChannel{
+		url:     srv.URL,
+		headers: "X-Api-Key: s3cret",
+		body:    `{"text":"{{title}}: {{message}}","node":"{{node}}"}`,
+	}
+	note := Note{Event: "offline", Node: "tokyo", Title: "🔴 tokyo 离线", Message: "最后上报 \"x\"", Time: t0.Unix()}
+	if f := ch.post(note); f != nil {
+		t.Fatalf("post failed: %+v", f)
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if gotHeader != "s3cret" {
+		t.Fatalf("custom header not sent: %q", gotHeader)
+	}
+	var parsed map[string]string
+	if err := json.Unmarshal(gotBody, &parsed); err != nil {
+		t.Fatalf("body not JSON: %v (%s)", err, gotBody)
+	}
+	if parsed["node"] != "tokyo" || !strings.Contains(parsed["text"], "🔴 tokyo 离线") {
+		t.Fatalf("placeholders not substituted: %v", parsed)
+	}
+
+	// Failure classification: 5xx retries, 4xx does not, a redirect
+	// is reported rather than followed.
+	codes := []struct {
+		code  int
+		retry bool
+	}{
+		{500, true}, {429, true}, {400, false}, {401, false}, {404, false},
+	}
+	for _, c := range codes {
+		s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(c.code)
+		}))
+		f := webhookChannel{url: s.URL, body: defaultBody}.post(note)
+		s.Close()
+		if f == nil || f.retry != c.retry {
+			t.Errorf("HTTP %d: failure = %+v, want retry=%v", c.code, f, c.retry)
+		}
+	}
+	redir := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Location", "/elsewhere")
+		w.WriteHeader(http.StatusFound)
+	}))
+	defer redir.Close()
+	f := webhookChannel{url: redir.URL, body: defaultBody}.post(note)
+	if f == nil || !strings.Contains(f.shown, "跳转") {
+		t.Fatalf("redirect must be reported, not followed: %+v", f)
+	}
+	if strings.Contains(f.reason, redir.URL) {
+		t.Fatalf("the webhook URL (a credential) must not leak into the reason: %q", f.reason)
+	}
+}
+
+func TestTelegramPostShapeAndSecrecy(t *testing.T) {
+	newFakeEnv(t)
+	var mu sync.Mutex
+	var paths []string
+	var bodies []map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		paths = append(paths, r.URL.Path)
+		var body map[string]any
+		data, _ := io.ReadAll(r.Body)
+		_ = json.Unmarshal(data, &body)
+		bodies = append(bodies, body)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	ch := telegramChannel{
+		token:   "123:SECRET",
+		chats:   []int64{42, -100},
+		text:    "{{title}}\n{{message}}",
+		apiBase: srv.URL,
+	}
+	note := Note{Event: "login", Title: "🔑 面板登录", Message: "密码登录 · 来自 203.0.113.9", Time: t0.Unix()}
+	if f := ch.post(note); f != nil {
+		t.Fatalf("post failed: %+v", f)
+	}
+	mu.Lock()
+	if len(paths) != 2 || paths[0] != "/bot123:SECRET/sendMessage" {
+		t.Fatalf("want one sendMessage per chat, got %v", paths)
+	}
+	if bodies[0]["chat_id"] != float64(42) || bodies[1]["chat_id"] != float64(-100) {
+		t.Fatalf("chat ids wrong: %v", bodies)
+	}
+	if bodies[0]["text"] != "🔑 面板登录\n密码登录 · 来自 203.0.113.9" {
+		t.Fatalf("text not rendered: %v", bodies[0]["text"])
+	}
+	mu.Unlock()
+
+	// A 401 answers "Bot Token 不对" and the reason never carries
+	// the token.
+	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusUnauthorized)
+	}))
+	defer bad.Close()
+	ch.apiBase = bad.URL
+	f := ch.post(note)
+	if f == nil || !strings.Contains(f.shown, "Bot Token 不对") {
+		t.Fatalf("401 must read as a bad token: %+v", f)
+	}
+	if strings.Contains(f.reason, "SECRET") || strings.Contains(f.shown, "SECRET") {
+		t.Fatalf("the token must not leak: reason %q shown %q", f.reason, f.shown)
+	}
+}
+
+func TestTestSendChannels(t *testing.T) {
+	e := newFakeEnv(t)
+	if _, err := Test(); err == nil || err.Error() != "还没有配置通知渠道" {
+		t.Fatalf("no channels must be its own error, got %v", err)
+	}
+	ok := &fakeChannel{label: "Webhook"}
+	e.channels = []channel{ok}
+	sent, err := Test()
+	if err != nil || len(sent) != 1 || sent[0] != "Webhook" || len(ok.notes) != 1 || ok.notes[0].Event != "test" {
+		t.Fatalf("test send = %v, %v, notes %v", sent, err, ok.notes)
+	}
+	e.channels = []channel{&fakeChannel{label: "Telegram", fail: &failure{shown: "Bot Token 不对（HTTP 401）", reason: "HTTP 401"}}}
+	if _, err := Test(); err == nil || !strings.Contains(err.Error(), "Telegram：Bot Token 不对") {
+		t.Fatalf("a failing channel must surface its own error, got %v", err)
+	}
+}
+
+func TestTelegramConfiguredParsing(t *testing.T) {
+	e := newFakeEnv(t)
+	if TelegramConfigured() {
+		t.Fatal("nothing set must be unconfigured")
+	}
+	e.settings["tgBotToken"] = "123:abc"
+	e.settings["tgBotChatId"] = "not-a-number"
+	if TelegramConfigured() {
+		t.Fatal("a non-numeric chat list is not a channel")
+	}
+	e.settings["tgBotChatId"] = "42, -100123 , junk"
+	if !TelegramConfigured() {
+		t.Fatal("numeric ids among the list configure the channel")
+	}
+}
@@ -0,0 +1,372 @@
+// Package probenotify is the item 7 notification engine: the monitor
+// hub's notify.rs, re-homed on the panel's own stores (ITEM7-SCOPE
+// §3.5/§5.6/B6). Events: a node going offline and returning, a billing
+// period's traffic crossing the configured step and the full
+// allowance, expiry dates approaching (a daily digest) or rolled
+// forward for a node still online past them, and a sign-in to the
+// panel.
+//
+// Channels: Telegram through the panel's existing bot (the
+// tgBotToken/tgBotChatId settings — the hub's separate
+// notify_telegram_token/chat are deliberately NOT ported, one bot
+// configuration serves both) and a new webhook channel (JSON body
+// template, custom headers, no redirect following). Delivery is
+// queued and retried per channel; nothing here may stall or fail the
+// panel flow that raised the event — a dead channel degrades to log
+// lines only.
+//
+// Durable state (offline marks, traffic steps already announced, the
+// digest's last-sent date) lives in probe.db's probe_setting through
+// the probe package, so a master restart neither replays an offline
+// alert nor loses the recovery pairing.
+package probenotify
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+// Note is one alert. Node lists the node names it concerns,
+// comma-separated, and is empty for a sign-in.
+type Note struct {
+	Event   string
+	Node    string
+	Title   string
+	Message string
+	// Unix seconds, stamped when queued: delivery can trail the
+	// event by minutes while a channel is retried.
+	Time int64
+}
+
+// Numeric settings as (key, min, max, default, the panel's name for
+// it) — the hub's NUMBERS table, unchanged.
+var numbersTable = []struct {
+	key, label    string
+	min, max, def int64
+}{
+	// Minutes a node may stay away before it is reported. Capped at
+	// the flap grace (30 min): a longer grace would outwait a
+	// flapping node too.
+	{"notify_grace", "离线宽限期", 1, flapGrace / 60, 3},
+	// Percent of the allowance that raises the first traffic alert;
+	// 0 disables traffic alerts.
+	{"notify_traffic", "流量提醒", 0, 100, 80},
+	// Days ahead an expiry is listed; 0 disables both expiry
+	// reminders and renewal notices.
+	{"notify_expiry", "到期提醒", 0, 365, 7},
+}
+
+const (
+	// How often connectivity and traffic are examined (the sweep
+	// job's cadence); an offline alert trails the grace period by at
+	// most this.
+	sweepInterval = 30 * time.Second
+
+	// A node that returned from an absence longer than the grace
+	// period within flapWindow of going away again is flapping, and
+	// that absence is reported only once it lasts flapGrace.
+	flapWindow = 3600 // seconds
+	flapGrace  = 1800 // seconds
+
+	// Nodes listed in one alert before the rest are only counted.
+	listedMax = 20
+
+	// onlineWindow is how fresh an agent's last heartbeat must be
+	// for the sweep to call it online — the B3 public view's rule
+	// (LastSeen ≤150s against the 60s heartbeat).
+	onlineWindow = 150 // seconds
+
+	defaultBody = `{"event":"{{event}}","node":"{{node}}","title":"{{title}}","message":"{{message}}"}`
+	defaultText = "{{title}}\n{{message}}"
+)
+
+// setting reads one main-store settings value; ok=false when unset.
+// The notify keys live in the same KV as every other panel setting
+// (hub names kept), read fresh at each use so a panel edit applies
+// to the next alert.
+var settingFn = func(key string) (string, bool) {
+	var row model.Setting
+	if err := database.GetDB().Where("key = ?", key).First(&row).Error; err != nil {
+		return "", false
+	}
+	return row.Value, true
+}
+
+// setSetting writes one main-store settings value (empty deletes the
+// row, hub's "empty clears" shape).
+var setSettingFn = func(key, value string) error {
+	db := database.GetDB()
+	if value == "" {
+		return db.Where("key = ?", key).Delete(&model.Setting{}).Error
+	}
+	var row model.Setting
+	err := db.Where("key = ?", key).First(&row).Error
+	if database.IsNotFound(err) {
+		return db.Create(&model.Setting{Key: key, Value: value}).Error
+	}
+	if err != nil {
+		return err
+	}
+	row.Value = value
+	return db.Save(&row).Error
+}
+
+// number reads a numeric notify setting the hub way: parse, range
+// check, default on anything else.
+func number(key string) int64 {
+	for _, n := range numbersTable {
+		if n.key == key {
+			if v, ok := settingFn(key); ok {
+				if parsed, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64); err == nil && parsed >= n.min && parsed <= n.max {
+					return parsed
+				}
+			}
+			return n.def
+		}
+	}
+	return 0
+}
+
+// templateSetting is a template setting, where empty means default.
+func templateSetting(key, def string) string {
+	if v, ok := settingFn(key); ok && v != "" {
+		return v
+	}
+	return def
+}
+
+// loginEnabled reports the notify_login switch (default on, hub).
+func loginEnabled() bool {
+	v, _ := settingFn("notify_login")
+	return v != "off"
+}
+
+// siteName is the {{site}} placeholder's value: the public page's
+// site_name setting, "Monitor" by default (hub site_name()).
+func siteName() string {
+	if v, ok := settingFn("site_name"); ok && v != "" {
+		return v
+	}
+	return "Monitor"
+}
+
+// SettingError reports why a notification setting value cannot be
+// stored, or "" when it can — the hub's setting_error for the notify_*
+// keys, minus the Telegram token/chat pair (the panel bot owns those
+// credentials; see the package comment) and with the same texts.
+func SettingError(key, value string) string {
+	for _, n := range numbersTable {
+		if n.key == key {
+			parsed, err := strconv.ParseInt(value, 10, 64)
+			if err != nil || parsed < n.min || parsed > n.max {
+				return fmt.Sprintf("%s要填 %d 到 %d 之间的整数", n.label, n.min, n.max)
+			}
+			return ""
+		}
+	}
+	switch key {
+	case "notify_login":
+		if value != "on" && value != "off" {
+			return "登录提醒只能是 on 或 off"
+		}
+	case "notify_webhook_headers":
+		_, err := parseHeaders(value)
+		return err
+	case "notify_telegram_text", "notify_webhook_url", "notify_webhook_body":
+		// Empty clears the channel field, or restores a template's
+		// default.
+		if value == "" {
+			return ""
+		}
+		switch key {
+		case "notify_telegram_text":
+			// Any text goes; placeholders substitute at send time.
+		case "notify_webhook_url":
+			if !validWebhookURL(value) {
+				return "Webhook URL 必须以 http:// 或 https:// 开头"
+			}
+		case "notify_webhook_body":
+			var v any
+			if err := json.Unmarshal([]byte(render(value, sampleNote(), "s\"i\\te", true)), &v); err != nil {
+				return "Webhook 请求体填入内容后必须是合法的 JSON，占位符要放在引号里"
+			}
+		}
+	default:
+		return fmt.Sprintf("没有这个设置项：%s", key)
+	}
+	return ""
+}
+
+func validWebhookURL(value string) bool {
+	return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://")
+}
+
+// sampleNote carries values that exercise every escape the body
+// template must survive (hub sample()).
+func sampleNote() Note {
+	return Note{
+		Event:   "test",
+		Node:    "a\"b\\c",
+		Title:   "{{message}}",
+		Message: "line\nline",
+	}
+}
+
+// parseHeaders parses "Name: value", one per line. Values are never
+// echoed in an error: they are typically the credential.
+func parseHeaders(text string) ([][2]string, string) {
+	var out [][2]string
+	for _, line := range strings.Split(text, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		name, value, found := strings.Cut(line, ":")
+		if !found {
+			return nil, "Webhook 请求头每行要写成「名称: 值」"
+		}
+		name = strings.TrimSpace(name)
+		value = strings.TrimSpace(value)
+		if !validHeaderName(name) {
+			return nil, fmt.Sprintf("Webhook 请求头名称 %q 不合法", name)
+		}
+		if strings.ContainsAny(value, "\r\n") {
+			return nil, fmt.Sprintf("Webhook 请求头 %s 的值不合法", name)
+		}
+		out = append(out, [2]string{name, value})
+	}
+	return out, ""
+}
+
+// validHeaderName is RFC 7230 token.
+func validHeaderName(name string) bool {
+	if name == "" {
+		return false
+	}
+	for _, r := range name {
+		if r <= ' ' || r >= 0x7f || strings.ContainsRune("()<>@,;:\\\"/[]?={}", r) {
+			return false
+		}
+	}
+	return true
+}
+
+// render substitutes the placeholders in a single pass, JSON-escaping
+// each value when jsonEsc is set so it is valid inside a string
+// literal. A second pass would also substitute placeholders contained
+// in the inserted values, such as a node name (hub render()).
+func render(tmpl string, note Note, site string, jsonEsc bool) string {
+	fields := []struct{ key, value string }{
+		{"{{event}}", note.Event},
+		{"{{node}}", note.Node},
+		{"{{title}}", note.Title},
+		{"{{message}}", note.Message},
+		{"{{site}}", site},
+		{"{{time}}", clock(note.Time)},
+	}
+	var out strings.Builder
+	out.Grow(len(tmpl))
+	rest := tmpl
+	for {
+		at := strings.Index(rest, "{{")
+		if at < 0 {
+			out.WriteString(rest)
+			return out.String()
+		}
+		out.WriteString(rest[:at])
+		rest = rest[at:]
+		matched := false
+		for _, f := range fields {
+			if strings.HasPrefix(rest, f.key) {
+				if jsonEsc {
+					quoted, _ := json.Marshal(f.value)
+					out.Write(quoted[1 : len(quoted)-1])
+				} else {
+					out.WriteString(f.value)
+				}
+				rest = rest[len(f.key):]
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			out.WriteString("{{")
+			rest = rest[2:]
+		}
+	}
+}
+
+// batch builds one alert for any number of nodes: a list in the
+// message, the names in Note.Node. ok is false when there is nothing
+// to say (hub batch()).
+func batch(event, mark, what string, items []batchItem) (Note, bool) {
+	if len(items) == 0 {
+		return Note{}, false
+	}
+	var title, message string
+	if len(items) == 1 {
+		title = fmt.Sprintf("%s %s %s", mark, items[0].name, what)
+		message = items[0].detail
+	} else {
+		lines := make([]string, 0, listedMax+1)
+		for i, it := range items {
+			if i >= listedMax {
+				break
+			}
+			lines = append(lines, fmt.Sprintf("%s · %s", it.name, it.detail))
+		}
+		if len(items) > listedMax {
+			lines = append(lines, fmt.Sprintf("……另外 %d 台", len(items)-listedMax))
+		}
+		title = fmt.Sprintf("%s %d 台节点%s", mark, len(items), what)
+		message = strings.Join(lines, "\n")
+	}
+	names := make([]string, len(items))
+	for i, it := range items {
+		names[i] = it.name
+	}
+	return Note{Event: event, Node: strings.Join(names, ", "), Title: title, Message: message}, true
+}
+
+type batchItem struct {
+	name, detail string
+}
+
+// span renders a duration the hub way ("3 分钟" / "2 小时 5 分钟" /
+// "1 天 2 小时").
+func span(seconds int64) string {
+	if seconds < 0 {
+		seconds = 0
+	}
+	minutes := seconds / 60
+	d, h, m := minutes/1440, minutes/60%24, minutes%60
+	switch {
+	case d == 0 && h == 0:
+		return fmt.Sprintf("%d 分钟", m)
+	case d == 0:
+		return fmt.Sprintf("%d 小时 %d 分钟", h, m)
+	default:
+		return fmt.Sprintf("%d 天 %d 小时", d, h)
+	}
+}
+
+// clock formats a unix stamp in local time with its offset: a master
+// in a container commonly runs on UTC while its operator reads local
+// time (hub clock()).
+func clock(ts int64) string {
+	if ts == 0 {
+		return ""
+	}
+	return time.Unix(ts, 0).Format("01-02 15:04 -07:00")
+}
+
+// gib renders bytes as GiB, labelled GB as in the panel (hub gib()).
+func gib(bytes int64) string {
+	return fmt.Sprintf("%.2f GB", float64(bytes)/float64(int64(1)<<30))
+}
@@ -0,0 +1,562 @@
+package probenotify
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service/probe"
+)
+
+// fakeEnv replaces every store seam of the package for one test.
+type fakeEnv struct {
+	settings map[string]string
+	state    map[string]string
+	stateErr map[string]bool
+	nodes    []nodeInfo
+	channels []channel
+	saved    []probe.Meta
+}
+
+func newFakeEnv(t *testing.T) *fakeEnv {
+	t.Helper()
+	e := &fakeEnv{
+		settings: map[string]string{},
+		state:    map[string]string{},
+		stateErr: map[string]bool{},
+	}
+	oldSetting, oldSetSetting := settingFn, setSettingFn
+	oldChannels, oldLoad := channelsFn, loadNodesFn
+	oldGet, oldSet, oldSave := stateGetFn, stateSetFn, saveMetaFn
+	settingFn = func(key string) (string, bool) {
+		v, ok := e.settings[key]
+		return v, ok
+	}
+	setSettingFn = func(key, value string) error {
+		if value == "" {
+			delete(e.settings, key)
+		} else {
+			e.settings[key] = value
+		}
+		return nil
+	}
+	channelsFn = func() []channel { return e.channels }
+	loadNodesFn = func() ([]nodeInfo, error) { return e.nodes, nil }
+	stateGetFn = func(key string) (string, bool, error) {
+		if e.stateErr[key] {
+			return "", false, errors.New("store down")
+		}
+		v, ok := e.state[key]
+		return v, ok, nil
+	}
+	stateSetFn = func(key, value string) error {
+		if e.stateErr[key] {
+			return errors.New("store down")
+		}
+		if value == "" {
+			delete(e.state, key)
+		} else {
+			e.state[key] = value
+		}
+		return nil
+	}
+	saveMetaFn = func(m probe.Meta) error {
+		e.saved = append(e.saved, m)
+		return nil
+	}
+	t.Cleanup(func() {
+		settingFn, setSettingFn = oldSetting, oldSetSetting
+		channelsFn, loadNodesFn = oldChannels, oldLoad
+		stateGetFn, stateSetFn, saveMetaFn = oldGet, oldSet, oldSave
+	})
+	return e
+}
+
+type fakeChannel struct {
+	label string
+	notes []Note
+	fail  *failure
+}
+
+func (f *fakeChannel) name() string { return f.label }
+func (f *fakeChannel) post(n Note) *failure {
+	if f.fail != nil {
+		return f.fail
+	}
+	f.notes = append(f.notes, n)
+	return nil
+}
+
+var t0 = time.Date(2026, 10, 23, 12, 0, 0, 0, time.Local)
+
+func offlineNode(id int64, name string, lastSeen time.Time, notify bool) nodeInfo {
+	return nodeInfo{
+		id:          id,
+		name:        name,
+		lastSeenSec: lastSeen.Unix(),
+		meta:        probe.Meta{AgentID: id, Notify: notify, TrafficResetDay: 1},
+	}
+}
+
+func TestSweepOfflineAfterGraceOnce(t *testing.T) {
+	e := newFakeEnv(t)
+	e.settings["notify_grace"] = "1" // 60s
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	e.nodes = []nodeInfo{offlineNode(7, "tokyo", t0.Add(-300*time.Second), true)}
+	w := newWatch()
+
+	if notes := runSweep(t0, w); len(notes) != 0 {
+		t.Fatalf("first sighting must only start the absence clock, got %v", notes)
+	}
+	if notes := runSweep(t0.Add(30*time.Second), w); len(notes) != 0 {
+		t.Fatalf("before grace, got %v", notes)
+	}
+	notes := runSweep(t0.Add(61*time.Second), w)
+	if len(notes) != 1 || notes[0].Event != "offline" || !strings.Contains(notes[0].Title, "tokyo") {
+		t.Fatalf("after grace want one offline note, got %v", notes)
+	}
+	if got := e.state["notify_down:7"]; got != fmt.Sprint(t0.Add(-300*time.Second).Unix()) {
+		t.Fatalf("offline mark = %q, want last-seen stamp", got)
+	}
+	if notes := runSweep(t0.Add(10*time.Minute), w); len(notes) != 0 {
+		t.Fatalf("announced mark must mute repeats, got %v", notes)
+	}
+}
+
+func TestSweepRecoveryOnceAndRearms(t *testing.T) {
+	e := newFakeEnv(t)
+	e.settings["notify_grace"] = "1"
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	n := offlineNode(7, "tokyo", t0.Add(-300*time.Second), true)
+	e.nodes = []nodeInfo{n}
+	w := newWatch()
+	runSweep(t0, w)
+	runSweep(t0.Add(61*time.Second), w) // offline announced
+
+	back := t0.Add(5 * time.Minute)
+	n.lastSeenSec = back.Unix()
+	e.nodes = []nodeInfo{n}
+	notes := runSweep(back, w)
+	if len(notes) != 1 || notes[0].Event != "online" || !strings.Contains(notes[0].Message, "离线") {
+		t.Fatalf("want one recovery note, got %v", notes)
+	}
+	if _, ok := e.state["notify_down:7"]; ok {
+		t.Fatal("recovery must clear the offline mark")
+	}
+	if notes := runSweep(back.Add(time.Minute), w); len(notes) != 0 {
+		t.Fatalf("recovery must not repeat, got %v", notes)
+	}
+	// Down again: a fresh absence past grace announces again.
+	n.lastSeenSec = back.Unix()
+	e.nodes = []nodeInfo{n}
+	later := back.Add(2 * time.Hour)
+	if notes := runSweep(later, w); len(notes) != 0 {
+		t.Fatalf("first sighting of the new absence, got %v", notes)
+	}
+	notes = runSweep(later.Add(61*time.Second), w)
+	if len(notes) != 1 || notes[0].Event != "offline" {
+		t.Fatalf("second outage must announce again, got %v", notes)
+	}
+}
+
+func TestSweepFlapUsesFlapGrace(t *testing.T) {
+	e := newFakeEnv(t)
+	e.settings["notify_grace"] = "1"
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	n := offlineNode(7, "tokyo", t0.Add(-300*time.Second), true)
+	e.nodes = []nodeInfo{n}
+	w := newWatch()
+	runSweep(t0, w)
+	runSweep(t0.Add(61*time.Second), w) // announced
+	back := t0.Add(5 * time.Minute)
+	n.lastSeenSec = back.Unix()
+	e.nodes = []nodeInfo{n}
+	runSweep(back, w) // recovered; returned[] = back
+
+	// Away again ten minutes after returning: inside the flap
+	// window, so the wait is the 30-minute flap grace, not 60s.
+	away := back.Add(10 * time.Minute)
+	n.lastSeenSec = back.Unix()
+	e.nodes = []nodeInfo{n}
+	runSweep(away, w) // first sighting of this absence
+	if notes := runSweep(away.Add(61*time.Second), w); len(notes) != 0 {
+		t.Fatalf("flapping absence must wait the flap grace, got %v", notes)
+	}
+	notes := runSweep(away.Add(flapGrace*time.Second), w)
+	if len(notes) != 1 || notes[0].Event != "offline" {
+		t.Fatalf("after flap grace want offline note, got %v", notes)
+	}
+}
+
+func TestSweepSkipsNeverReportedAndOptedOut(t *testing.T) {
+	e := newFakeEnv(t)
+	e.settings["notify_grace"] = "1"
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	e.nodes = []nodeInfo{
+		{id: 1, name: "ghost", lastSeenSec: 0, meta: probe.Meta{AgentID: 1, Notify: true}},
+		offlineNode(2, "quiet", t0.Add(-time.Hour), false),
+	}
+	w := newWatch()
+	runSweep(t0, w)
+	if notes := runSweep(t0.Add(2*time.Minute), w); len(notes) != 0 {
+		t.Fatalf("never-reported and opted-out nodes stay silent, got %v", notes)
+	}
+	for key := range e.state {
+		if strings.HasPrefix(key, "notify_down:") {
+			t.Fatalf("no offline marks expected, got %v", e.state)
+		}
+	}
+}
+
+func TestSweepUnarmedRecordsNothing(t *testing.T) {
+	e := newFakeEnv(t)
+	e.settings["notify_grace"] = "1"
+	n := offlineNode(7, "tokyo", t0.Add(-300*time.Second), true)
+	e.nodes = []nodeInfo{n}
+	w := newWatch()
+	runSweep(t0, w)
+	if notes := runSweep(t0.Add(61*time.Second), w); len(notes) != 0 {
+		t.Fatalf("no channels, no notes, got %v", notes)
+	}
+	if _, ok := e.state["notify_down:7"]; ok {
+		t.Fatal("unarmed sweep must not record the announcement")
+	}
+	// Configure a channel afterwards: the already-down node is
+	// reported on the next pass (its absence clock is old by now).
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	notes := runSweep(t0.Add(2*time.Minute), w)
+	if len(notes) != 1 || notes[0].Event != "offline" {
+		t.Fatalf("late-configured channel must still hear about the outage, got %v", notes)
+	}
+}
+
+func TestSweepRestartKeepsPairing(t *testing.T) {
+	e := newFakeEnv(t)
+	e.settings["notify_grace"] = "1"
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	down := offlineNode(7, "tokyo", t0.Add(-300*time.Second), true)
+	e.nodes = []nodeInfo{down}
+	// The mark from before the "restart": announced already.
+	e.state["notify_down:7"] = fmt.Sprint(down.lastSeenSec)
+	w := newWatch() // fresh in-memory watch, as after a restart
+	runSweep(t0, w)
+	if notes := runSweep(t0.Add(2*time.Minute), w); len(notes) != 0 {
+		t.Fatalf("restart must not replay the offline alert, got %v", notes)
+	}
+	back := t0.Add(3 * time.Minute)
+	down.lastSeenSec = back.Unix()
+	e.nodes = []nodeInfo{down}
+	notes := runSweep(back, w)
+	if len(notes) != 1 || notes[0].Event != "online" {
+		t.Fatalf("the recovery pairing must survive the restart, got %v", notes)
+	}
+}
+
+func trafficNode(id int64, name string, used, limit int64) nodeInfo {
+	return nodeInfo{
+		id:          id,
+		name:        name,
+		lastSeenSec: t0.Unix(),
+		meta: probe.Meta{
+			AgentID: id, TrafficLimit: limit, TrafficMode: "sum",
+			TrafficResetDay: 1, Notify: true,
+		},
+		traffic:    probe.Traffic{MonthRx: used, MonthStart: "2026-10-01"},
+		hasTraffic: true,
+	}
+}
+
+func TestTrafficStepsPrimeThenAnnounce(t *testing.T) {
+	e := newFakeEnv(t)
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	e.nodes = []nodeInfo{trafficNode(7, "tokyo", 850, 1000)}
+	w := newWatch()
+
+	// Unprimed (no stored state): the crossing is recorded silently.
+	if notes := runSweep(t0, w); len(notes) != 0 {
+		t.Fatalf("first pass must prime, not announce, got %v", notes)
+	}
+	stored := map[int64]alertStep{}
+	if err := json.Unmarshal([]byte(e.state["traffic_alerted"]), &stored); err != nil {
+		t.Fatalf("state not JSON: %v", err)
+	}
+	if stored[7].Step != 80 || stored[7].Period != "2026-10-01" {
+		t.Fatalf("primed step = %+v", stored[7])
+	}
+	// Same step again: told already.
+	if notes := runSweep(t0.Add(time.Minute), w); len(notes) != 0 {
+		t.Fatalf("same step must not repeat, got %v", notes)
+	}
+	// Cross to 100%: the second step announces.
+	e.nodes = []nodeInfo{trafficNode(7, "tokyo", 1000, 1000)}
+	notes := runSweep(t0.Add(2*time.Minute), w)
+	if len(notes) != 1 || notes[0].Event != "traffic" || !strings.Contains(notes[0].Message, "100%") {
+		t.Fatalf("want one 100%% traffic note, got %v", notes)
+	}
+	if notes := runSweep(t0.Add(3*time.Minute), w); len(notes) != 0 {
+		t.Fatalf("100%% step must not repeat within the period, got %v", notes)
+	}
+	// Next period: the same usage announces again.
+	e.nodes = []nodeInfo{trafficNode(7, "tokyo", 900, 1000)}
+	nov := time.Date(2026, 11, 15, 12, 0, 0, 0, time.Local)
+	if notes := runSweep(nov, newWatch()); len(notes) != 1 || notes[0].Event != "traffic" {
+		t.Fatalf("a new period re-arms the steps, got %v", notes)
+	}
+}
+
+func TestTrafficCorrectionRearms(t *testing.T) {
+	e := newFakeEnv(t)
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	e.state["traffic_alerted"] = `{"7":["2026-10-01",80]}`
+	e.nodes = []nodeInfo{trafficNode(7, "tokyo", 500, 1000)} // corrected down
+	w := newWatch()
+	if notes := runSweep(t0, w); len(notes) != 0 {
+		t.Fatalf("a correction announces nothing, got %v", notes)
+	}
+	stored := map[int64]alertStep{}
+	_ = json.Unmarshal([]byte(e.state["traffic_alerted"]), &stored)
+	if stored[7].Step != 0 {
+		t.Fatalf("the lower step must be stored, got %+v", stored[7])
+	}
+	e.nodes = []nodeInfo{trafficNode(7, "tokyo", 850, 1000)}
+	if notes := runSweep(t0.Add(time.Minute), w); len(notes) != 1 || notes[0].Event != "traffic" {
+		t.Fatalf("crossing again after a correction must announce, got %v", notes)
+	}
+}
+
+func TestTrafficReadErrorSkipsSection(t *testing.T) {
+	e := newFakeEnv(t)
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	e.stateErr["traffic_alerted"] = true
+	e.nodes = []nodeInfo{trafficNode(7, "tokyo", 1000, 1000)}
+	if notes := runSweep(t0, newWatch()); len(notes) != 0 {
+		t.Fatalf("a failed state read must skip the section, got %v", notes)
+	}
+}
+
+func renewNode(id int64, name string, expires string, cycle string, online bool) nodeInfo {
+	exp := expires
+	ls := t0.Add(-time.Hour).Unix()
+	if online {
+		ls = t0.Unix()
+	}
+	return nodeInfo{
+		id: id, name: name, lastSeenSec: ls,
+		meta: probe.Meta{AgentID: id, ExpiresAt: &exp, BillingCycle: cycle, TrafficResetDay: 1},
+	}
+}
+
+func TestHousekeepingRenewsOnlineNode(t *testing.T) {
+	e := newFakeEnv(t)
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	past := t0.AddDate(0, 0, -40).Format("2006-01-02")
+	e.nodes = []nodeInfo{renewNode(7, "tokyo", past, "monthly", true)}
+	notes := runHousekeeping(t0)
+	if len(e.saved) != 1 {
+		t.Fatalf("want one rolled meta, saved %v", e.saved)
+	}
+	want := t0.AddDate(0, 0, -40).AddDate(0, 2, 0).Format("2006-01-02")
+	if *e.saved[0].ExpiresAt != want {
+		t.Fatalf("rolled to %s, want %s (two whole cycles)", *e.saved[0].ExpiresAt, want)
+	}
+	if len(notes) != 1 || notes[0].Event != "renew" {
+		t.Fatalf("want one renewal note, got %v", notes)
+	}
+	// Not past any more: no second roll, no second note.
+	e.nodes = []nodeInfo{renewNode(7, "tokyo", *e.saved[0].ExpiresAt, "monthly", true)}
+	e.saved = nil
+	if notes := runHousekeeping(t0); len(notes) != 0 || len(e.saved) != 0 {
+		t.Fatalf("a future date must not roll, notes %v saved %v", notes, e.saved)
+	}
+}
+
+func TestHousekeepingRenewalSkipsAndGate(t *testing.T) {
+	e := newFakeEnv(t)
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	past := t0.AddDate(0, 0, -10).Format("2006-01-02")
+	e.nodes = []nodeInfo{
+		renewNode(1, "once-node", past, "once", true),     // one-off plan: no cycle
+		renewNode(2, "away-node", past, "monthly", false), // offline: no roll
+		renewNode(3, "live-node", past, "monthly", true),
+	}
+	e.settings["notify_expiry"] = "0" // renewals still roll, silently
+	notes := runHousekeeping(t0)
+	if len(e.saved) != 1 || e.saved[0].AgentID != 3 {
+		t.Fatalf("only the online monthly node rolls, saved %v", e.saved)
+	}
+	if len(notes) != 0 {
+		t.Fatalf("notify_expiry=0 must mute the renewal note, got %v", notes)
+	}
+}
+
+func TestExpiryDigestOncePerDay(t *testing.T) {
+	e := newFakeEnv(t)
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	soon := t0.AddDate(0, 0, 3).Format("2006-01-02")
+	e.nodes = []nodeInfo{renewNode(7, "tokyo", soon, "monthly", true)}
+
+	morning := time.Date(2026, 10, 23, 8, 0, 0, 0, time.Local)
+	if notes := runHousekeeping(morning); len(notes) != 0 {
+		t.Fatalf("before 09:00 no digest, got %v", notes)
+	}
+	if _, ok := e.state["notify_expiry_sent"]; ok {
+		t.Fatal("an ungated hour must not consume the day")
+	}
+	notes := runHousekeeping(t0) // 12:00
+	found := false
+	for _, n := range notes {
+		if n.Event == "expiry" && strings.Contains(n.Message, soon) {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("want the expiry digest, got %v", notes)
+	}
+	if notes := runHousekeeping(t0.Add(time.Hour)); len(notes) != 0 {
+		t.Fatalf("the digest is once a day, got %v", notes)
+	}
+}
+
+func TestExpiryDigestNeedsChannel(t *testing.T) {
+	e := newFakeEnv(t)
+	soon := t0.AddDate(0, 0, 3).Format("2006-01-02")
+	e.nodes = []nodeInfo{renewNode(7, "tokyo", soon, "monthly", true)}
+	if notes := runHousekeeping(t0); len(notes) != 0 {
+		t.Fatalf("no channel, no digest, got %v", notes)
+	}
+	if _, ok := e.state["notify_expiry_sent"]; ok {
+		t.Fatal("the day must stay unconsumed so a channel configured later still gets the digest")
+	}
+	e.channels = []channel{&fakeChannel{label: "Webhook"}}
+	if notes := runHousekeeping(t0); len(notes) != 1 || notes[0].Event != "expiry" {
+		t.Fatalf("want the digest once a channel exists, got %v", notes)
+	}
+}
+
+func TestCycleMonthsAndClamp(t *testing.T) {
+	for cycle, want := range map[string]int64{
+		"monthly": 1, "quarterly": 3, "semiannual": 6, "yearly": 12,
+		"biennial": 24, "triennial": 36, "5m": 5, "1200m": 1200,
+	} {
+		if got, ok := cycleMonths(cycle); !ok || got != want {
+			t.Errorf("cycleMonths(%q) = %d,%v want %d", cycle, got, ok, want)
+		}
+	}
+	for _, bad := range []string{"once", "", "0m", "1201m", "weekly", "m"} {
+		if _, ok := cycleMonths(bad); ok {
+			t.Errorf("cycleMonths(%q) must not renew", bad)
+		}
+	}
+	jan31 := time.Date(2024, 1, 31, 0, 0, 0, 0, time.Local)
+	if got := addMonthsClamped(jan31, 1); got.Format("2006-01-02") != "2024-02-29" {
+		t.Errorf("Jan 31 + 1 month = %v, want 2024-02-29 (clamped)", got)
+	}
+}
+
+func TestSettingError(t *testing.T) {
+	cases := []struct {
+		key, value string
+		wantErr    bool
+	}{
+		{"notify_grace", "3", false}, {"notify_grace", "0", true},
+		{"notify_grace", "31", true}, {"notify_grace", "abc", true},
+		{"notify_traffic", "0", false}, {"notify_traffic", "100", false},
+		{"notify_traffic", "101", true},
+		{"notify_expiry", "0", false}, {"notify_expiry", "365", false},
+		{"notify_expiry", "366", true},
+		{"notify_login", "on", false}, {"notify_login", "off", false},
+		{"notify_login", "yes", true},
+		{"notify_webhook_url", "", false},
+		{"notify_webhook_url", "https://example.com/hook", false},
+		{"notify_webhook_url", "http://127.0.0.1:9/x", false},
+		{"notify_webhook_url", "ftp://example.com", true},
+		{"notify_webhook_headers", "", false},
+		{"notify_webhook_headers", "X-Api-Key: secret", false},
+		{"notify_webhook_headers", "Broken Line", true},
+		{"notify_webhook_headers", "Bad Name: v", true},
+		{"notify_webhook_body", "", false},
+		{"notify_webhook_body", `{"text":"{{title}}"}`, false},
+		{"notify_webhook_body", `{"text":{{title}}}`, true},
+		{"notify_webhook_body", `not json`, true},
+		{"notify_telegram_text", "", false},
+		{"notify_telegram_text", "anything {{title}}", false},
+		{"notify_unknown", "1", true},
+	}
+	for _, c := range cases {
+		if got := SettingError(c.key, c.value); (got != "") != c.wantErr {
+			t.Errorf("SettingError(%q, %q) = %q, wantErr %v", c.key, c.value, got, c.wantErr)
+		}
+	}
+}
+
+func TestRenderSinglePassAndEscapes(t *testing.T) {
+	note := Note{Event: "offline", Node: "a\"b", Title: "{{message}}", Message: "line\nline", Time: t0.Unix()}
+	// Single pass: the title's literal {{message}} must survive.
+	if got := render("{{title}}", note, "site", false); got != "{{message}}" {
+		t.Errorf("render single pass = %q", got)
+	}
+	got := render(`{"t":"{{title}}","m":"{{message}}","n":"{{node}}"}`, note, "site", true)
+	var parsed map[string]string
+	if err := json.Unmarshal([]byte(got), &parsed); err != nil {
+		t.Fatalf("rendered body not JSON: %v (%s)", err, got)
+	}
+	if parsed["m"] != "line\nline" || parsed["n"] != "a\"b" {
+		t.Errorf("escapes broken: %v", parsed)
+	}
+	if got := render("x{{bogus}}y", note, "site", false); got != "x{{bogus}}y" {
+		t.Errorf("unknown placeholder must pass through, got %q", got)
+	}
+}
+
+func TestBatchShapes(t *testing.T) {
+	if _, ok := batch("offline", "🔴", "离线", nil); ok {
+		t.Error("empty batch must be no note")
+	}
+	one, _ := batch("offline", "🔴", "离线", []batchItem{{"a", "最后上报 10-23 11:00 +08:00"}})
+	if one.Title != "🔴 a 离线" || one.Node != "a" {
+		t.Errorf("single batch = %+v", one)
+	}
+	items := make([]batchItem, 25)
+	for i := range items {
+		items[i] = batchItem{fmt.Sprint("n", i), "d"}
+	}
+	many, _ := batch("offline", "🔴", "离线", items)
+	if !strings.Contains(many.Title, "25 台节点离线") || !strings.Contains(many.Message, "……另外 5 台") {
+		t.Errorf("many batch = %q / %q", many.Title, many.Message)
+	}
+}
+
+func TestSaveSettingsValidationAndSecrets(t *testing.T) {
+	e := newFakeEnv(t)
+	// All-or-nothing: one bad field refuses the whole write.
+	body := map[string]json.RawMessage{
+		"notifyGrace":   json.RawMessage("5"),
+		"notifyTraffic": json.RawMessage("101"),
+	}
+	if err := SaveSettings(body); err == nil {
+		t.Fatal("out-of-range traffic must refuse")
+	}
+	if _, ok := e.settings["notify_grace"]; ok {
+		t.Fatal("a refused save must write nothing")
+	}
+	// Secrets: null leaves, string sets, "" clears.
+	e.settings["notify_webhook_url"] = "https://old.example/hook"
+	if err := SaveSettings(map[string]json.RawMessage{"notifyWebhookUrl": json.RawMessage("null")}); err != nil {
+		t.Fatal(err)
+	}
+	if e.settings["notify_webhook_url"] != "https://old.example/hook" {
+		t.Fatal("null must leave the secret")
+	}
+	if err := SaveSettings(map[string]json.RawMessage{"notifyWebhookUrl": json.RawMessage(`""`)}); err != nil {
+		t.Fatal(err)
+	}
+	if _, ok := e.settings["notify_webhook_url"]; ok {
+		t.Fatal("empty string must clear the secret")
+	}
+	if err := SaveSettings(map[string]json.RawMessage{"bogus": json.RawMessage("1")}); err == nil {
+		t.Fatal("unknown keys are refused")
+	}
+}
@@ -0,0 +1,125 @@
+package probenotify
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+)
+
+// SettingsView is the notify block of the public-page settings tab:
+// every value with defaults filled in (so the panel can echo it back
+// unchanged), the two webhook secrets reported only as set or unset
+// (a webhook URL is commonly the credential itself, and headers
+// typically carry one), and the live channel status.
+type SettingsView struct {
+	NotifyGrace             int64  `json:"notifyGrace"`
+	NotifyTraffic           int64  `json:"notifyTraffic"`
+	NotifyExpiry            int64  `json:"notifyExpiry"`
+	NotifyLogin             bool   `json:"notifyLogin"`
+	NotifyTelegramText      string `json:"notifyTelegramText"`
+	NotifyWebhookURLSet     bool   `json:"notifyWebhookUrlSet"`
+	NotifyWebhookHeadersSet bool   `json:"notifyWebhookHeadersSet"`
+	NotifyWebhookBody       string `json:"notifyWebhookBody"`
+	TelegramConfigured      bool   `json:"telegramConfigured"`
+	WebhookConfigured       bool   `json:"webhookConfigured"`
+}
+
+// GetSettings assembles the view from the settings store.
+func GetSettings() SettingsView {
+	_, urlSet := settingFn("notify_webhook_url")
+	if v, ok := settingFn("notify_webhook_url"); ok && v == "" {
+		urlSet = false
+	}
+	_, headersSet := settingFn("notify_webhook_headers")
+	if v, ok := settingFn("notify_webhook_headers"); ok && v == "" {
+		headersSet = false
+	}
+	return SettingsView{
+		NotifyGrace:             number("notify_grace"),
+		NotifyTraffic:           number("notify_traffic"),
+		NotifyExpiry:            number("notify_expiry"),
+		NotifyLogin:             loginEnabled(),
+		NotifyTelegramText:      templateSetting("notify_telegram_text", defaultText),
+		NotifyWebhookURLSet:     urlSet,
+		NotifyWebhookHeadersSet: headersSet,
+		NotifyWebhookBody:       templateSetting("notify_webhook_body", defaultBody),
+		TelegramConfigured:      TelegramConfigured(),
+		WebhookConfigured:       WebhookConfigured(),
+	}
+}
+
+// SaveSettings validates every provided field first (all-or-nothing,
+// as the hub's save_settings), then writes. The wire keys are the
+// view's camelCase names; the two secrets are tri-state — absent or
+// JSON null leaves the stored value, a string replaces it, "" clears
+// it — because the view never echoes them back to be re-sent.
+func SaveSettings(body map[string]json.RawMessage) error {
+	type write struct{ key, value string }
+	var writes []write
+	for key, raw := range body {
+		switch key {
+		case "notifyGrace", "notifyTraffic", "notifyExpiry":
+			hubKey := map[string]string{
+				"notifyGrace":   "notify_grace",
+				"notifyTraffic": "notify_traffic",
+				"notifyExpiry":  "notify_expiry",
+			}[key]
+			var n int64
+			if err := json.Unmarshal(raw, &n); err != nil {
+				return errors.New("请求格式不对")
+			}
+			if msg := SettingError(hubKey, strconv.FormatInt(n, 10)); msg != "" {
+				return errors.New(msg)
+			}
+			writes = append(writes, write{hubKey, strconv.FormatInt(n, 10)})
+		case "notifyLogin":
+			var b bool
+			if err := json.Unmarshal(raw, &b); err != nil {
+				return errors.New("请求格式不对")
+			}
+			value := "off"
+			if b {
+				value = "on"
+			}
+			writes = append(writes, write{"notify_login", value})
+		case "notifyTelegramText", "notifyWebhookBody":
+			hubKey := map[string]string{
+				"notifyTelegramText": "notify_telegram_text",
+				"notifyWebhookBody":  "notify_webhook_body",
+			}[key]
+			var s string
+			if err := json.Unmarshal(raw, &s); err != nil {
+				return errors.New("请求格式不对")
+			}
+			if msg := SettingError(hubKey, s); msg != "" {
+				return errors.New(msg)
+			}
+			writes = append(writes, write{hubKey, s})
+		case "notifyWebhookUrl", "notifyWebhookHeaders":
+			hubKey := map[string]string{
+				"notifyWebhookUrl":     "notify_webhook_url",
+				"notifyWebhookHeaders": "notify_webhook_headers",
+			}[key]
+			if string(raw) == "null" {
+				continue
+			}
+			var s string
+			if err := json.Unmarshal(raw, &s); err != nil {
+				return errors.New("请求格式不对")
+			}
+			if msg := SettingError(hubKey, s); msg != "" {
+				return errors.New(msg)
+			}
+			writes = append(writes, write{hubKey, s})
+		default:
+			return fmt.Errorf("没有这个设置项：%s", key)
+		}
+	}
+	for _, w := range writes {
+		if err := setSettingFn(w.key, w.value); err != nil {
+			return err
+		}
+	}
+	return nil
+}
@@ -0,0 +1,550 @@
+package probenotify
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+	"github.com/mhsanaei/3x-ui/v3/internal/logger"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service/probe"
+)
+
+// nodeInfo is one agent as the sweep sees it: identity and freshness
+// from the panel store, operations metadata and the traffic ledger
+// from probe.db.
+type nodeInfo struct {
+	id          int64
+	name        string
+	lastSeenSec int64 // 0 = never reported
+	meta        probe.Meta
+	traffic     probe.Traffic
+	hasTraffic  bool
+}
+
+// Seams the production wiring fills with the real stores; tests
+// substitute fakes. Everything the sweep records goes through these.
+var (
+	loadNodesFn = loadNodes
+	stateGetFn  = probe.StateGet
+	stateSetFn  = probe.StateSet
+	saveMetaFn  = probe.SaveMeta
+)
+
+func loadNodes() ([]nodeInfo, error) {
+	db := database.GetDB()
+	var agents []model.AgentNode
+	if err := db.Find(&agents).Error; err != nil {
+		return nil, err
+	}
+	nodeNames := map[int]string{}
+	var nodes []model.Node
+	if err := db.Find(&nodes).Error; err == nil {
+		for _, n := range nodes {
+			nodeNames[n.Id] = n.Name
+		}
+	}
+	metas, _ := probe.MetaSnapshot()
+	traffics, _ := probe.TrafficSnapshot()
+	out := make([]nodeInfo, 0, len(agents))
+	for _, a := range agents {
+		name := a.Name
+		if a.NodeID != nil {
+			if bound, ok := nodeNames[*a.NodeID]; ok && bound != "" {
+				name = bound
+			}
+		}
+		meta, ok := metas[int64(a.Id)]
+		if !ok {
+			meta = probe.DefaultMeta(int64(a.Id))
+		}
+		traffic, hasTraffic := traffics[int64(a.Id)]
+		out = append(out, nodeInfo{
+			id:          int64(a.Id),
+			name:        name,
+			lastSeenSec: a.LastSeen / 1000,
+			meta:        meta,
+			traffic:     traffic,
+			hasTraffic:  hasTraffic,
+		})
+	}
+	return out, nil
+}
+
+func (n nodeInfo) online(nowSec int64) bool {
+	return n.lastSeenSec > 0 && nowSec-n.lastSeenSec <= onlineWindow
+}
+
+// watch is the sweep state that need not outlive the process (hub
+// Watch): when a sweep first found each node absent, and when each
+// node last returned from a long absence. The durable half of the
+// offline bookkeeping — the announced mark — is in probe.db.
+type watch struct {
+	mu       sync.Mutex
+	absent   map[int64]int64
+	returned map[int64]int64
+}
+
+func newWatch() *watch {
+	return &watch{absent: map[int64]int64{}, returned: map[int64]int64{}}
+}
+
+var defaultWatch = newWatch()
+
+func downKey(agentID int64) string { return "notify_down:" + strconv.FormatInt(agentID, 10) }
+
+// Sweep runs one connectivity + traffic pass and queues what it
+// raises. Called by the 30s job.
+func Sweep() {
+	for _, note := range runSweep(time.Now(), defaultWatch) {
+		Send(note)
+	}
+}
+
+// runSweep is one pass over every node (hub sweep()). Offline and
+// returning nodes are each gathered into one alert per pass, so an
+// outage on the master's side — which drops every agent within about
+// 30s — produces one or two alerts rather than one per node. A return
+// is announced only after an offline alert, whether or not the node
+// still has alerts enabled. Nothing is recorded as announced while
+// no channel is configured, so configuring one afterwards still
+// reports nodes already down and traffic already past a step.
+func runSweep(now time.Time, w *watch) []Note {
+	nodes, err := loadNodesFn()
+	if err != nil {
+		logger.Warningf("probenotify: sweep load nodes: %v", err)
+		return nil
+	}
+	nowSec := now.Unix()
+	ids := make(map[int64]bool, len(nodes))
+	for _, n := range nodes {
+		ids[n.id] = true
+	}
+
+	w.mu.Lock()
+	for id := range w.absent {
+		if !ids[id] {
+			delete(w.absent, id)
+		}
+	}
+	for id := range w.returned {
+		if !ids[id] {
+			delete(w.returned, id)
+		}
+	}
+	w.mu.Unlock()
+
+	grace := number("notify_grace") * 60
+	armed := len(channelsFn()) > 0
+
+	// The announced offline marks. A failed read skips the whole
+	// connectivity section: taken for "not announced", it would
+	// re-raise alerts the state says already went out.
+	marks := make(map[int64]int64, len(nodes))
+	marksOK := true
+	for _, n := range nodes {
+		v, ok, err := stateGetFn(downKey(n.id))
+		if err != nil {
+			logger.Warningf("probenotify: read offline mark %d: %v", n.id, err)
+			marksOK = false
+			break
+		}
+		if ok {
+			if ts, perr := strconv.ParseInt(v, 10, 64); perr == nil {
+				marks[n.id] = ts
+			}
+		}
+	}
+
+	var notes []Note
+	if marksOK {
+		var down, up []batchItem
+		aborted := false
+		for _, n := range nodes {
+			if n.online(nowSec) {
+				w.mu.Lock()
+				if since, was := w.absent[n.id]; was {
+					delete(w.absent, n.id)
+					if nowSec-since >= grace {
+						w.returned[n.id] = nowSec
+					}
+				}
+				w.mu.Unlock()
+				if ds := marks[n.id]; ds > 0 {
+					if err := stateSetFn(downKey(n.id), ""); err != nil {
+						logger.Warningf("probenotify: clear offline mark %d: %v", n.id, err)
+						aborted = true
+						break
+					}
+					delete(marks, n.id)
+					up = append(up, batchItem{n.name, "离线 " + span(nowSec-ds)})
+				}
+				continue
+			}
+			// A node that has never reported cannot be announced,
+			// and tracking its absence would count its first
+			// connection as a return from an outage.
+			if n.lastSeenSec == 0 {
+				continue
+			}
+			w.mu.Lock()
+			since, seen := w.absent[n.id]
+			if !seen {
+				since = nowSec
+				w.absent[n.id] = nowSec
+			}
+			flapping := false
+			if back, ok := w.returned[n.id]; ok && since-back < flapWindow {
+				flapping = true
+			}
+			w.mu.Unlock()
+			wait := grace
+			if flapping {
+				wait = flapGrace
+			}
+			if armed && n.meta.Notify && marks[n.id] == 0 && nowSec-since >= wait {
+				if err := stateSetFn(downKey(n.id), strconv.FormatInt(n.lastSeenSec, 10)); err != nil {
+					logger.Warningf("probenotify: set offline mark %d: %v", n.id, err)
+					aborted = true
+					break
+				}
+				marks[n.id] = n.lastSeenSec
+				down = append(down, batchItem{n.name, "最后上报 " + clock(n.lastSeenSec)})
+			}
+		}
+		if !aborted {
+			if note, ok := batch("offline", "🔴", "离线", down); ok {
+				notes = append(notes, note)
+			}
+			if note, ok := batch("online", "🟢", "恢复在线", up); ok {
+				notes = append(notes, note)
+			}
+		}
+	}
+
+	notes = append(notes, trafficNotes(now, nodes, ids, armed)...)
+	return notes
+}
+
+// alertStep is one node's announced traffic step in its billing
+// period, persisted as JSON under "traffic_alerted" so a step
+// crossed while the master was stopped is announced on its return
+// and one announced before is not repeated (hub TRAFFIC_ALERTED).
+// The wire shape matches the hub's: {"<id>": ["<period>", <step>]}.
+type alertStep struct {
+	Period string
+	Step   int64
+}
+
+func (s alertStep) MarshalJSON() ([]byte, error) {
+	return json.Marshal([]any{s.Period, s.Step})
+}
+
+func (s *alertStep) UnmarshalJSON(data []byte) error {
+	var pair []json.RawMessage
+	if err := json.Unmarshal(data, &pair); err != nil {
+		return err
+	}
+	if len(pair) != 2 {
+		return fmt.Errorf("traffic step: want [period, step]")
+	}
+	if err := json.Unmarshal(pair[0], &s.Period); err != nil {
+		return err
+	}
+	return json.Unmarshal(pair[1], &s.Step)
+}
+
+func reached(used, limit, percent int64) bool {
+	// i128 in the hub: 100 × a limit near the int64 ceiling would
+	// overflow, so compare in big integers (this runs per node per
+	// sweep, dozens at most).
+	u := new(big.Int).Mul(big.NewInt(used), big.NewInt(100))
+	l := new(big.Int).Mul(big.NewInt(limit), big.NewInt(percent))
+	return u.Cmp(l) >= 0
+}
+
+// trafficNotes is the sweep's traffic section. Read and written as
+// one state value every pass, so a restored database brings its own
+// steps. Unset/unparseable state means "not primed": the steps
+// already reached are recorded rather than announced, so an upgrade
+// does not re-announce every crossing (hub).
+func trafficNotes(now time.Time, nodes []nodeInfo, ids map[int64]bool, armed bool) []Note {
+	percent := number("notify_traffic")
+	if percent <= 0 || !armed {
+		return nil
+	}
+	stored, hasStored, err := stateGetFn("traffic_alerted")
+	if err != nil {
+		// A failed read skips the section: taken for "nothing
+		// stored", it would record this sweep's crossings without
+		// announcing them.
+		logger.Warningf("probenotify: reading the traffic steps announced failed: %v", err)
+		return nil
+	}
+	alerted := map[int64]alertStep{}
+	primed := false
+	if hasStored {
+		var parsed map[int64]alertStep
+		if jerr := json.Unmarshal([]byte(stored), &parsed); jerr == nil {
+			alerted = parsed
+			primed = true
+		}
+	}
+	for id := range alerted {
+		if !ids[id] {
+			delete(alerted, id)
+		}
+	}
+	var metered []batchItem
+	for _, n := range nodes {
+		if n.meta.TrafficLimit <= 0 || !n.hasTraffic {
+			continue
+		}
+		used := n.traffic.MonthUsed(n.meta.TrafficMode)
+		// The hub's ledger read reports the CURRENT period start as
+		// month_start (its gate zeroes the counters of a stale
+		// period); our gate keeps the stored stamp, so the current
+		// period is computed here and the gated counters already
+		// agree with it.
+		period := probe.PeriodStart(dateOnly(now), n.meta.TrafficResetDay).Format("2006-01-02")
+		step := int64(0)
+		switch {
+		case reached(used, n.meta.TrafficLimit, 100):
+			step = 100
+		case reached(used, n.meta.TrafficLimit, percent):
+			step = percent
+		}
+		// A step below the one recorded is a manual correction;
+		// storing it lets the same crossing be announced again.
+		last, existed := alerted[n.id]
+		alerted[n.id] = alertStep{Period: period, Step: step}
+		told := existed && last.Period == period && last.Step >= step
+		if step == 0 || told || !primed {
+			continue
+		}
+		mode := switchModeName(n.meta.TrafficMode)
+		share := new(big.Int).Mul(big.NewInt(used), big.NewInt(100))
+		share.Div(share, big.NewInt(n.meta.TrafficLimit))
+		metered = append(metered, batchItem{n.name, fmt.Sprintf(
+			"已用 %s%%，%s / %s（%s），本期自 %s 起",
+			share.String(), gib(used), gib(n.meta.TrafficLimit), mode, period)})
+	}
+	// Announced only once recorded: unrecorded, the next sweep would
+	// announce the same step again, every 30 seconds while writes
+	// fail.
+	state, merr := json.Marshal(alerted)
+	if merr != nil {
+		logger.Warningf("probenotify: encoding the traffic steps failed: %v", merr)
+		return nil
+	}
+	if !hasStored || stored != string(state) {
+		if err := stateSetFn("traffic_alerted", string(state)); err != nil {
+			logger.Warningf("probenotify: saving the traffic steps announced failed: %v", err)
+			return nil
+		}
+	}
+	if note, ok := batch("traffic", "⚠️", "流量提醒", metered); ok {
+		return []Note{note}
+	}
+	return nil
+}
+
+func switchModeName(mode string) string {
+	switch mode {
+	case "up":
+		return "仅上行"
+	case "down":
+		return "仅下行"
+	case "max":
+		return "取较大值"
+	default:
+		return "上下行相加"
+	}
+}
+
+// Housekeeping runs the hourly half: roll online nodes' expiry dates
+// forward first, then the daily expiry digest over the dates as they
+// now stand (hub housekeeping order).
+func Housekeeping() {
+	for _, note := range runHousekeeping(time.Now()) {
+		Send(note)
+	}
+}
+
+func runHousekeeping(now time.Time) []Note {
+	nodes, err := loadNodesFn()
+	if err != nil {
+		logger.Warningf("probenotify: housekeeping load nodes: %v", err)
+		return nil
+	}
+	nowSec := now.Unix()
+	today := dateOnly(now)
+	var notes []Note
+
+	// Auto-renewal: an online node past its expiry date is still
+	// being paid for in practice — roll the date whole cycles into
+	// the future and say so. Dates are the operator's local calendar
+	// days (hub renew_online_nodes).
+	var rolled []batchItem
+	for _, n := range nodes {
+		if !n.online(nowSec) || n.meta.ExpiresAt == nil {
+			continue
+		}
+		expires, ok := parseDate(*n.meta.ExpiresAt)
+		if !ok {
+			continue
+		}
+		months, ok := cycleMonths(n.meta.BillingCycle)
+		if !ok {
+			continue
+		}
+		next := expires
+		for next.Before(today) {
+			next = addMonthsClamped(next, months)
+		}
+		if next.Equal(expires) {
+			continue
+		}
+		meta := n.meta
+		nextStr := next.Format("2006-01-02")
+		meta.ExpiresAt = &nextStr
+		if err := saveMetaFn(meta); err != nil {
+			logger.Warningf("probenotify: roll expiry of %d: %v", n.id, err)
+			continue
+		}
+		logger.Infof("probenotify: node %s is still up past %s, expiry rolled to %s", n.name, *n.meta.ExpiresAt, nextStr)
+		rolled = append(rolled, batchItem{n.name, fmt.Sprintf("%s → %s", expires.Format("2006-01-02"), nextStr)})
+	}
+	if number("notify_expiry") > 0 {
+		if note, ok := batch("renew", "🔁", "已自动续期", rolled); ok {
+			notes = append(notes, note)
+		}
+	}
+
+	if note, ok := expiryDigest(now, today, nodes); ok {
+		notes = append(notes, note)
+	}
+	return notes
+}
+
+// expiryDigest is the once-a-day list of nodes expiring within the
+// configured window, sent from 09:00 local. The date is stored once
+// sent, so a restart does not repeat it — and not while no channel
+// is configured, so one configured later that day still receives it
+// (hub expiry_digest).
+func expiryDigest(now, today time.Time, nodes []nodeInfo) (Note, bool) {
+	days := number("notify_expiry")
+	todayStr := today.Format("2006-01-02")
+	if days == 0 || now.Hour() < 9 {
+		return Note{}, false
+	}
+	if v, ok, err := stateGetFn("notify_expiry_sent"); err != nil {
+		logger.Warningf("probenotify: read expiry digest date: %v", err)
+		return Note{}, false
+	} else if ok && v == todayStr {
+		return Note{}, false
+	}
+	if len(channelsFn()) == 0 {
+		return Note{}, false
+	}
+	type soonItem struct {
+		date time.Time
+		name string
+	}
+	var soon []soonItem
+	for _, n := range nodes {
+		if n.meta.ExpiresAt == nil {
+			continue
+		}
+		date, ok := parseDate(*n.meta.ExpiresAt)
+		if !ok {
+			continue
+		}
+		diff := int(date.Sub(today).Hours() / 24)
+		if diff >= 0 && diff <= int(days) {
+			soon = append(soon, soonItem{date, n.name})
+		}
+	}
+	sort.Slice(soon, func(i, j int) bool {
+		if !soon[i].date.Equal(soon[j].date) {
+			return soon[i].date.Before(soon[j].date)
+		}
+		return soon[i].name < soon[j].name
+	})
+	if err := stateSetFn("notify_expiry_sent", todayStr); err != nil {
+		logger.Warningf("probenotify: save expiry digest date: %v", err)
+		return Note{}, false
+	}
+	items := make([]batchItem, 0, len(soon))
+	for _, s := range soon {
+		diff := int(s.date.Sub(today).Hours() / 24)
+		left := "今天到期"
+		if diff > 0 {
+			left = fmt.Sprintf("还剩 %d 天", diff)
+		}
+		items = append(items, batchItem{s.name, fmt.Sprintf("%s %s", s.date.Format("2006-01-02"), left)})
+	}
+	return batch("expiry", "⏳", "即将到期", items)
+}
+
+// cycleMonths resolves a stored billing cycle to its length in
+// months (hub cycle_months): the named cycles, or "{n}m" with
+// 1 ≤ n ≤ 1200. "once" and anything unknown do not renew.
+func cycleMonths(cycle string) (int64, bool) {
+	switch cycle {
+	case "monthly":
+		return 1, true
+	case "quarterly":
+		return 3, true
+	case "semiannual":
+		return 6, true
+	case "yearly":
+		return 12, true
+	case "biennial":
+		return 24, true
+	case "triennial":
+		return 36, true
+	}
+	if strings.HasSuffix(cycle, "m") {
+		if n, err := strconv.ParseInt(strings.TrimSuffix(cycle, "m"), 10, 64); err == nil && n >= 1 && n <= 1200 {
+			return n, true
+		}
+	}
+	return 0, false
+}
+
+// addMonthsClamped adds whole months, clamping the day to the target
+// month's last day (Rust checked_add_months semantics; Go's AddDate
+// would overflow Jan 31 + 1 month into March).
+func addMonthsClamped(t time.Time, months int64) time.Time {
+	y, m, d := t.Date()
+	total := int64(int(m)-1) + months
+	ny, nm := y+int(total/12), time.Month(total%12+1)
+	if last := daysIn(ny, nm); d > last {
+		d = last
+	}
+	return time.Date(ny, nm, d, 0, 0, 0, 0, t.Location())
+}
+
+func daysIn(year int, month time.Month) int {
+	return time.Date(year, month+1, 0, 0, 0, 0, 0, time.UTC).Day()
+}
+
+func dateOnly(t time.Time) time.Time {
+	y, m, d := t.Date()
+	return time.Date(y, m, d, 0, 0, 0, 0, t.Location())
+}
+
+func parseDate(s string) (time.Time, bool) {
+	t, err := time.ParseInLocation("2006-01-02", s, time.Local)
+	if err != nil {
+		return time.Time{}, false
+	}
+	return t, true
+}
@@ -365,6 +365,19 @@ func (s *Server) startTask(restartXray bool, loc *time.Location) {
 		job.NewProbeHistoryJob().Run()
 	}()

+	// Probe notifications (item 7 B6): the connectivity/traffic sweep
+	// every 30s (the hub's cadence — an offline alert trails its
+	// grace period by at most one sweep), and the expiry
+	// housekeeping hourly on the hour so a date rolls within seconds
+	// of midnight, plus the same one-minute startup kick as above
+	// (the hub runs its housekeeping immediately at boot).
+	_, _ = s.cron.AddJob("@every 30s", job.NewProbeNotifyJob())
+	_, _ = s.cron.AddJob("@hourly", job.NewProbeNotifyHousekeepingJob())
+	go func() {
+		time.Sleep(time.Minute)
+		job.NewProbeNotifyHousekeepingJob().Run()
+	}()
+
 	// check client ips from log file every day
 	_, _ = s.cron.AddJob("@daily", job.NewClearLogsJob())
 	_, _ = s.cron.AddJob(cadenceXrayLogPrune, job.NewPruneXrayLogsJob())