feat: heartbeat reconciliation — panel re-dispatches resources a node lost

Agents (Go + Node) now report the resource IDs they actually hold in
each heartbeat. When an expected resource is missing locally (e.g. the
node's /etc/nexusgate state was wiped and only the agent restarted, so
no re-enrollment happens), the panel re-queues applies for just the
affected nodes via reconcileFromHeartbeat: gated on no queued/running
jobs for the server (in-flight jobs are never failed by this path) and
throttled to one trigger per server per 120s, audited as
system:heartbeat. reconcileServer gains optional onlyNodeIds /
failInFlight parameters; the enrollment call is unchanged. Heartbeats
without a resources array (old agents) behave exactly as before.
This commit is contained in:
aaaamoooo
2026-10-20 07:46:33 +00:00
parent f2b7d8e181
commit abb1a4344b
5 changed files with 342 additions and 46 deletions
+22 -13
View File
@@ -201,13 +201,9 @@ type engineReport struct {
SingBoxStatus string `json:"singBoxStatus"`
}
// engineHealth mirrors agent.js engineHealth(). Callers must hold stateMu.
// A readResources error propagates (JS throws, so the heartbeat is skipped).
func engineHealthLocked() (engineReport, error) {
resources, err := readResources()
if err != nil {
return engineReport{}, err
}
// engineHealth mirrors agent.js engineHealth(). Callers must hold stateMu
// and pass the resources they read under the same lock.
func engineHealthLocked(resources []map[string]interface{}) engineReport {
installed := fileExists(cfg.SingBoxBin) && fileExists(cfg.SingBoxStatsBin)
service := serviceHealth("nexusgate-sing-box")
result := engineReport{
@@ -236,20 +232,33 @@ func engineHealthLocked() (engineReport, error) {
result.Detail = "Hysteria 2 UDP 未监听:" + strings.Join(parts, ", ")
}
}
return result, nil
return result
}
func heartbeat() error {
stateMu.Lock()
engine, err := engineHealthLocked()
stateMu.Unlock()
// A readResources error skips the heartbeat entirely (JS throws).
resources, err := readResources()
if err != nil {
stateMu.Unlock()
return err
}
engine := engineHealthLocked(resources)
stateMu.Unlock()
// The panel reconciles against this list: any expected resource the node
// no longer holds (e.g. local state was wiped) is re-dispatched
// automatically. An empty list is meaningful, so keep it non-nil.
resourceIDs := []string{}
for _, res := range resources {
if s, ok := res["id"].(string); ok && s != "" {
resourceIDs = append(resourceIDs, s)
}
}
body := map[string]interface{}{
"version": version,
"system": systemInfo(),
"engine": engine,
"version": version,
"system": systemInfo(),
"engine": engine,
"resources": resourceIDs,
}
if _, err := requestJSON("/api/agent/heartbeat", body); err != nil {
return err