Muse
66bf6a88eb
After an agent restart the tracked sing-box handle is lost while the old core survives reparented holding the inbound ports, so every apply died on bind, rollback died the same way, and the agent retried forever. startSingboxLocked now scans /proc for processes whose argv carries run -c <exact live config path> (excluding the tracked pid) and stops them SIGTERM-then-SIGKILL before starting, covering the apply, rollback, and unchanged-but-not-running paths. The exact-path match never touches cores on other config files; Linux only.
214 lines
6.6 KiB
Go
214 lines
6.6 KiB
Go
package agent
|
|
|
|
import (
|
|
"os"
|
|
"os/exec"
|
|
"runtime"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Orphan-core recovery (next-version fix batch item 1): after an
|
|
// agent restart the tracked sing-box handle is lost while the old
|
|
// core survives reparented holding the inbound ports. The reconcile
|
|
// in startSingboxLocked must stop exactly those orphans — processes
|
|
// whose argv carries `run -c <live config path>` — before starting
|
|
// a new core, on the apply path, the rollback path, and the
|
|
// unchanged-config-but-not-running path alike.
|
|
//
|
|
// The dummy below is the test binary re-exec'd: with the marker env
|
|
// it sleeps until signalled, and its argv carries a crafted `run -c`
|
|
// triple, which is exactly what the reconcile matches on. Without
|
|
// the marker it is a no-op so ordinary test runs pass through it.
|
|
const orphanDummyEnv = "XUI_ORPHAN_DUMMY"
|
|
|
|
func TestOrphanDummyProcess(t *testing.T) {
|
|
if os.Getenv(orphanDummyEnv) != "1" {
|
|
return
|
|
}
|
|
time.Sleep(3600 * time.Second)
|
|
}
|
|
|
|
func spawnOrphanDummy(t *testing.T, cfgPath string) (*exec.Cmd, int) {
|
|
t.Helper()
|
|
if runtime.GOOS != "linux" {
|
|
t.Skip("orphan reconcile is Linux-only")
|
|
}
|
|
cmd := exec.Command(os.Args[0], "-test.run=^TestOrphanDummyProcess$", "--", "run", "-c", cfgPath)
|
|
cmd.Env = append(os.Environ(), orphanDummyEnv+"=1")
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatalf("start dummy: %v", err)
|
|
}
|
|
pid := cmd.Process.Pid
|
|
t.Cleanup(func() {
|
|
if cmd.ProcessState == nil || !cmd.ProcessState.Exited() {
|
|
_ = cmd.Process.Kill()
|
|
}
|
|
_ = cmd.Wait()
|
|
})
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if foreignPidAlive(pid) && singboxCmdlineMatchesConfig(pid, cfgPath) {
|
|
return cmd, pid
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatalf("dummy pid %d never showed the crafted argv", pid)
|
|
return nil, 0
|
|
}
|
|
|
|
func waitDummyExit(t *testing.T, cmd *exec.Cmd) bool {
|
|
t.Helper()
|
|
done := make(chan error, 1)
|
|
go func() { done <- cmd.Wait() }()
|
|
select {
|
|
case <-done:
|
|
return true
|
|
case <-time.After(10 * time.Second):
|
|
return false
|
|
}
|
|
}
|
|
|
|
func TestReconcileSingboxOrphansStopsMatchingProcess(t *testing.T) {
|
|
cfgPath := "/tmp/orphan-test-live-config.json"
|
|
cmd, pid := spawnOrphanDummy(t, cfgPath)
|
|
if n := reconcileSingboxOrphans(cfgPath, 0); n != 1 {
|
|
t.Fatalf("reaped = %d, want 1", n)
|
|
}
|
|
if !waitDummyExit(t, cmd) {
|
|
t.Fatalf("dummy pid %d still alive after reconcile", pid)
|
|
}
|
|
if foreignPidAlive(pid) {
|
|
t.Fatalf("dummy pid %d still reported alive after reconcile", pid)
|
|
}
|
|
}
|
|
|
|
func TestReconcileSingboxOrphansLeavesOtherConfigAlone(t *testing.T) {
|
|
_, pid := spawnOrphanDummy(t, "/tmp/orphan-test-other-config.json")
|
|
if n := reconcileSingboxOrphans("/tmp/orphan-test-live-config.json", 0); n != 0 {
|
|
t.Fatalf("reaped = %d, want 0", n)
|
|
}
|
|
if !foreignPidAlive(pid) {
|
|
t.Fatalf("dummy on a different config path was killed, want untouched")
|
|
}
|
|
}
|
|
|
|
func TestReconcileSingboxOrphansSkipsTrackedPid(t *testing.T) {
|
|
cmd, pid := spawnOrphanDummy(t, "/tmp/orphan-test-live-config.json")
|
|
if n := reconcileSingboxOrphans("/tmp/orphan-test-live-config.json", pid); n != 0 {
|
|
t.Fatalf("reaped = %d, want 0 for the tracked pid", n)
|
|
}
|
|
if !foreignPidAlive(pid) {
|
|
t.Fatalf("tracked pid %d was killed, want untouched", pid)
|
|
}
|
|
_ = cmd.Process.Kill()
|
|
}
|
|
|
|
// dropSingboxHandle simulates the agent-restart half of the incident:
|
|
// the tracked handle is gone, but the previously started core keeps
|
|
// running. The caller owns killing the leaked fake core afterwards.
|
|
func dropSingboxHandle(t *testing.T, a *Agent) int {
|
|
t.Helper()
|
|
a.sbMu.Lock()
|
|
pid := a.sbProc.pid()
|
|
a.sbProc = &singboxProcess{}
|
|
a.sbMu.Unlock()
|
|
if pid <= 0 {
|
|
t.Fatal("no tracked core to orphan")
|
|
}
|
|
t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGTERM) })
|
|
return pid
|
|
}
|
|
|
|
func TestApplySingboxRecoversFromStaleHandleAndOrphan(t *testing.T) {
|
|
setupFakeSingbox(t)
|
|
a := newApplyTestAgent()
|
|
t.Cleanup(a.stopSingbox)
|
|
|
|
if err := a.applySingboxConfig([]byte(sbCfgA)); err != nil {
|
|
t.Fatalf("apply A: %v", err)
|
|
}
|
|
pidBefore := dropSingboxHandle(t, a)
|
|
cmd, orphanPid := spawnOrphanDummy(t, singboxConfigPath())
|
|
|
|
if err := a.applySingboxConfig([]byte(sbCfgB)); err != nil {
|
|
t.Fatalf("apply B with stale handle + orphan: %v", err)
|
|
}
|
|
if !waitDummyExit(t, cmd) {
|
|
t.Fatalf("orphan pid %d survived the apply", orphanPid)
|
|
}
|
|
a.sbMu.Lock()
|
|
running := a.sbProc.isRunning()
|
|
pidAfter := a.sbProc.pid()
|
|
a.sbMu.Unlock()
|
|
if !running || pidAfter <= 0 || pidAfter == pidBefore {
|
|
t.Fatalf("no new tracked core after recovery: running=%v pid %d -> %d", running, pidBefore, pidAfter)
|
|
}
|
|
if got := appliedConfig(t); got != sbCfgB {
|
|
t.Fatalf("applied config = %q, want cfgB", got)
|
|
}
|
|
}
|
|
|
|
func TestApplySingboxUnchangedRecoversFromStaleHandleAndOrphan(t *testing.T) {
|
|
setupFakeSingbox(t)
|
|
a := newApplyTestAgent()
|
|
t.Cleanup(a.stopSingbox)
|
|
|
|
if err := a.applySingboxConfig([]byte(sbCfgA)); err != nil {
|
|
t.Fatalf("apply A: %v", err)
|
|
}
|
|
dropSingboxHandle(t, a)
|
|
cmd, orphanPid := spawnOrphanDummy(t, singboxConfigPath())
|
|
|
|
// Unchanged config + tracked handle says not-running: this used
|
|
// to fall into the start-fail/rollback-fail loop in production
|
|
// (the orphan held the ports); the reconcile inside the start
|
|
// must clear it before the new core binds.
|
|
if err := a.applySingboxConfig([]byte(sbCfgA)); err != nil {
|
|
t.Fatalf("re-apply A with stale handle + orphan: %v", err)
|
|
}
|
|
if !waitDummyExit(t, cmd) {
|
|
t.Fatalf("orphan pid %d survived the unchanged re-apply", orphanPid)
|
|
}
|
|
a.sbMu.Lock()
|
|
running := a.sbProc.isRunning()
|
|
a.sbMu.Unlock()
|
|
if !running {
|
|
t.Fatal("no tracked core running after unchanged re-apply recovery")
|
|
}
|
|
}
|
|
|
|
func TestApplySingboxRollbackRecoversFromStaleHandleAndOrphan(t *testing.T) {
|
|
setupFakeSingbox(t)
|
|
a := newApplyTestAgent()
|
|
t.Cleanup(a.stopSingbox)
|
|
|
|
if err := a.applySingboxConfig([]byte(sbCfgA)); err != nil {
|
|
t.Fatalf("apply A: %v", err)
|
|
}
|
|
dropSingboxHandle(t, a)
|
|
cmd, orphanPid := spawnOrphanDummy(t, singboxConfigPath())
|
|
|
|
// A config that dies at startup: the new core fails, so the
|
|
// apply rolls back — and the rollback's start must clear the
|
|
// orphan too, or it dies on bind the same way and the agent
|
|
// loops. The apply itself still reports failure either way.
|
|
bad := `{"log":{"level":"info"},"note":"RUNTIMEFAIL","inbounds":[]}`
|
|
if err := a.applySingboxConfig([]byte(bad)); err == nil {
|
|
t.Fatal("apply of a config that dies at startup must fail")
|
|
}
|
|
if !waitDummyExit(t, cmd) {
|
|
t.Fatalf("orphan pid %d survived the apply/rollback sequence", orphanPid)
|
|
}
|
|
a.sbMu.Lock()
|
|
running := a.sbProc.isRunning()
|
|
a.sbMu.Unlock()
|
|
if !running {
|
|
t.Fatal("rollback must leave a running core")
|
|
}
|
|
if got := appliedConfig(t); got != sbCfgA {
|
|
t.Fatalf("active config after rollback = %q, want cfgA", got)
|
|
}
|
|
}
|