feat: heartbeat reconciliation — panel re-dispatches resources a node lost
Agents (Go + Node) now report the resource IDs they actually hold in each heartbeat. When an expected resource is missing locally (e.g. the node's /etc/nexusgate state was wiped and only the agent restarted, so no re-enrollment happens), the panel re-queues applies for just the affected nodes via reconcileFromHeartbeat: gated on no queued/running jobs for the server (in-flight jobs are never failed by this path) and throttled to one trigger per server per 120s, audited as system:heartbeat. reconcileServer gains optional onlyNodeIds / failInFlight parameters; the enrollment call is unchanged. Heartbeats without a resources array (old agents) behave exactly as before.
This commit is contained in:
+22
-13
@@ -201,13 +201,9 @@ type engineReport struct {
|
||||
SingBoxStatus string `json:"singBoxStatus"`
|
||||
}
|
||||
|
||||
// engineHealth mirrors agent.js engineHealth(). Callers must hold stateMu.
|
||||
// A readResources error propagates (JS throws, so the heartbeat is skipped).
|
||||
func engineHealthLocked() (engineReport, error) {
|
||||
resources, err := readResources()
|
||||
if err != nil {
|
||||
return engineReport{}, err
|
||||
}
|
||||
// engineHealth mirrors agent.js engineHealth(). Callers must hold stateMu
|
||||
// and pass the resources they read under the same lock.
|
||||
func engineHealthLocked(resources []map[string]interface{}) engineReport {
|
||||
installed := fileExists(cfg.SingBoxBin) && fileExists(cfg.SingBoxStatsBin)
|
||||
service := serviceHealth("nexusgate-sing-box")
|
||||
result := engineReport{
|
||||
@@ -236,20 +232,33 @@ func engineHealthLocked() (engineReport, error) {
|
||||
result.Detail = "Hysteria 2 UDP 未监听:" + strings.Join(parts, ", ")
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
return result
|
||||
}
|
||||
|
||||
func heartbeat() error {
|
||||
stateMu.Lock()
|
||||
engine, err := engineHealthLocked()
|
||||
stateMu.Unlock()
|
||||
// A readResources error skips the heartbeat entirely (JS throws).
|
||||
resources, err := readResources()
|
||||
if err != nil {
|
||||
stateMu.Unlock()
|
||||
return err
|
||||
}
|
||||
engine := engineHealthLocked(resources)
|
||||
stateMu.Unlock()
|
||||
// The panel reconciles against this list: any expected resource the node
|
||||
// no longer holds (e.g. local state was wiped) is re-dispatched
|
||||
// automatically. An empty list is meaningful, so keep it non-nil.
|
||||
resourceIDs := []string{}
|
||||
for _, res := range resources {
|
||||
if s, ok := res["id"].(string); ok && s != "" {
|
||||
resourceIDs = append(resourceIDs, s)
|
||||
}
|
||||
}
|
||||
body := map[string]interface{}{
|
||||
"version": version,
|
||||
"system": systemInfo(),
|
||||
"engine": engine,
|
||||
"version": version,
|
||||
"system": systemInfo(),
|
||||
"engine": engine,
|
||||
"resources": resourceIDs,
|
||||
}
|
||||
if _, err := requestJSON("/api/agent/heartbeat", body); err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user