l-ui/internal/agent/singbox.go

package agent

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/mhsanaei/3x-ui/v3/internal/config"
	"github.com/mhsanaei/3x-ui/v3/internal/logger"
)

// ITEM6 B3→B5: the agent's sing-box runtime. The master delivers a
// complete sing-box configuration on the config poll
// (singbox_config + singbox_config_hash, B2). Since the B5
// switchover that payload is the node's only runtime config: the
// agent treats its arrival as the switch signal, retires the legacy
// xray core (see switchApplySingbox in agent.go) and runs sing-box
// alone. A legacy master serves no sing-box keys, and the agent
// then keeps the xray path exactly as before — the dual capability
// is what makes the agents-first rollout order safe.
//
// Activation borrows NexusGate's discipline (agent-go activateSingBox):
// candidate file -> `sing-box check` -> flush the running core's
// counters -> swap with a .previous copy kept -> start -> on failure
// restore .previous and start that. A config whose bytes are unchanged
// and whose process is alive is a no-op.

// errSingboxNotInstalled marks "the sing-box binary is not on this
// node yet" — an expected state during the rollout, logged without
// alarming the operator every poll (apply is only attempted when the
// delivered hash changes).
var errSingboxNotInstalled = errors.New("sing-box binary not installed on this node")

// SingboxVersionRecord is the install-time version record file. The
// self-built sing-box binaries carry no version stamp (`sing-box
// version` prints "unknown"), so install-agent.sh records the release
// version next to the binary for telemetry, NexusGate-style.
const singboxVersionRecord = "sing-box.version"

// singboxBinaryName mirrors the xray naming: sing-box-linux-<arch>.
func singboxBinaryName() string {
	arch := runtime.GOARCH
	if arch == "arm" {
		arch = "arm32"
	}
	return fmt.Sprintf("sing-box-%s-%s", runtime.GOOS, arch)
}

// singboxBinaryPath resolves the managed sing-box binary: the
// arch-specific name install-agent.sh installs, falling back to a
// plain "sing-box" in the same bin folder. "" when neither exists.
func singboxBinaryPath() string {
	bin := config.GetBinFolderPath()
	if p := filepath.Join(bin, singboxBinaryName()); fileExists(p) {
		return p
	}
	if p := filepath.Join(bin, "sing-box"); fileExists(p) {
		return p
	}
	return ""
}

// singboxConfigPath is the live configuration file the managed process
// runs with; candidate/previous siblings implement the swap discipline.
func singboxConfigPath() string {
	return filepath.Join(config.GetBinFolderPath(), "config-singbox.json")
}

func singboxCandidatePath() string { return singboxConfigPath() + ".candidate" }
func singboxPreviousPath() string  { return singboxConfigPath() + ".previous" }

// singboxLogPath is where the managed process's stdout/stderr land.
// The online tracker (singbox_online.go) tails this file for the
// authenticated-connection lines sing-box logs per user.
func singboxLogPath() string {
	return filepath.Join(config.GetBinFolderPath(), "singbox.log")
}

// singboxLogMaxBytes bounds the managed core's log on disk (ITEM6
// B5 rotation policy). The online tracker only ever tails the file,
// so beyond a few MB it is pure weight on small node volumes — but
// the log is also the online record, so the cap is generous rather
// than tight. Rotation keeps exactly one previous file
// (singbox.log.1), bounding the pair at ~2× the cap.
const singboxLogMaxBytes = 8 << 20 // 8 MiB

// rotateSingboxLogIfOversize copy-truncates the sing-box log once it
// passes the cap: the content is copied to singbox.log.1, then the
// live file is truncated in place. Copy-truncate (rather than
// rename) because the running core holds the file open — its fd
// stays valid and keeps appending; the online tracker already
// treats a size drop as a cursor reset, so the tail survives the
// rotation. Called on the sing-box traffic cycle; failures only log
// — log hygiene must never disturb the runtime.
func rotateSingboxLogIfOversize(path string) {
	fi, err := os.Stat(path)
	if err != nil || fi.Size() <= singboxLogMaxBytes {
		return
	}
	src, err := os.Open(path)
	if err != nil {
		logger.Warningf("sing-box log rotation: open %s: %v", path, err)
		return
	}
	dst, err := os.OpenFile(path+".1", os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o640)
	if err != nil {
		_ = src.Close()
		logger.Warningf("sing-box log rotation: create %s.1: %v", path, err)
		return
	}
	_, copyErr := io.Copy(dst, src)
	closeErr := dst.Close()
	_ = src.Close()
	if copyErr != nil || closeErr != nil {
		logger.Warningf("sing-box log rotation: copy %s: %v / %v", path, copyErr, closeErr)
		return
	}
	if err := os.Truncate(path, 0); err != nil {
		logger.Warningf("sing-box log rotation: truncate %s: %v", path, err)
		return
	}
	logger.Infof("sing-box log rotated at %d bytes (previous kept as %s.1)", fi.Size(), path)
}

// singboxProcess manages one sing-box child process: lifecycle only,
// no configuration semantics (mirrors the xray process discipline:
// done channel as the race-free liveness signal, SIGTERM then SIGKILL).
type singboxProcess struct {
	mu              sync.Mutex
	cmd             *exec.Cmd
	done            chan struct{}
	logFile         *os.File
	exitErr         error
	startedAt       time.Time
	intentionalStop bool
}

func (p *singboxProcess) isRunning() bool {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.cmd == nil || p.cmd.Process == nil || p.done == nil {
		return false
	}
	select {
	case <-p.done:
		return false
	default:
		return true
	}
}

// pid returns the running process's pid, 0 when not running.
func (p *singboxProcess) pid() int {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.cmd == nil || p.cmd.Process == nil {
		return 0
	}
	select {
	case <-p.done:
		return 0
	default:
		return p.cmd.Process.Pid
	}
}

// epoch identifies this process instance for the stats tracker:
// "sing-box:<pid>:<agent-side start time>". The agent starts the
// process itself, so its recorded start time distinguishes instances
// even when the pid is recycled (NexusGate reads the kernel start
// tick for the same reason). "" while no process is running.
func (p *singboxProcess) epoch() string {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.cmd == nil || p.cmd.Process == nil || p.done == nil {
		return ""
	}
	select {
	case <-p.done:
		return ""
	default:
	}
	return fmt.Sprintf("sing-box:%d:%d", p.cmd.Process.Pid, p.startedAt.UnixNano())
}

// singboxCoreEnv is the environment the sing-box core process runs
// with (V2.0.1 ⑧): a 32 MiB Go memory limit and GOGC=50, so the
// core's heap is collected back instead of only growing — the
// untuned core on HW-SG idles around 30 MB RSS where the tuned
// NexusGate nodes sit under 10 MB. Every core launch goes through
// singboxProcess.start, so the first start and every re-apply /
// restart inherit this. Only the child is tuned: the agent's own
// runtime settings are untouched, and values the agent itself was
// started with are replaced, not duplicated (a duplicate key's
// effective value would depend on the reader).
func singboxCoreEnv() []string {
	env := os.Environ()
	out := make([]string, 0, len(env)+2)
	for _, kv := range env {
		if strings.HasPrefix(kv, "GOMEMLIMIT=") || strings.HasPrefix(kv, "GOGC=") {
			continue
		}
		out = append(out, kv)
	}
	return append(out, "GOMEMLIMIT=32MiB", "GOGC=50")
}

// start launches sing-box with the given config file, appending its
// output to the log file the online tracker tails.
func (p *singboxProcess) start(binPath, configPath, logPath string) error {
	p.mu.Lock()
	defer p.mu.Unlock()
	if p.cmd != nil && p.done != nil {
		select {
		case <-p.done:
		default:
			return errors.New("sing-box is already running")
		}
	}
	if p.logFile != nil {
		_ = p.logFile.Close()
		p.logFile = nil
	}
	logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
	if err != nil {
		return fmt.Errorf("open sing-box log: %w", err)
	}
	cmd := exec.Command(binPath, "run", "-c", configPath)
	cmd.Env = singboxCoreEnv()
	cmd.Stdout = logFile
	cmd.Stderr = logFile
	if err := cmd.Start(); err != nil {
		_ = logFile.Close()
		return fmt.Errorf("start sing-box: %w", err)
	}
	p.cmd = cmd
	p.logFile = logFile
	p.done = make(chan struct{})
	p.exitErr = nil
	p.startedAt = time.Now()
	p.intentionalStop = false
	done := p.done
	go func() {
		err := cmd.Wait()
		p.mu.Lock()
		if err != nil && !p.intentionalStop {
			p.exitErr = err
		}
		p.mu.Unlock()
		close(done)
	}()
	return nil
}

// stop terminates the process (SIGTERM, then SIGKILL after a grace
// period) and waits for it to be reaped.
func (p *singboxProcess) stop() {
	p.mu.Lock()
	cmd, done := p.cmd, p.done
	if cmd == nil || cmd.Process == nil {
		p.mu.Unlock()
		return
	}
	p.intentionalStop = true
	proc := cmd.Process
	logFile := p.logFile
	p.mu.Unlock()

	if done != nil {
		select {
		case <-done:
			p.finishStop(logFile)
			return
		default:
		}
	}
	if runtime.GOOS == "windows" {
		_ = proc.Kill()
	} else if err := proc.Signal(syscall.SIGTERM); err != nil {
		_ = proc.Kill()
	}
	if done != nil {
		select {
		case <-done:
		case <-time.After(singboxStopGrace):
			_ = proc.Kill()
			select {
			case <-done:
			case <-time.After(singboxStopForce):
			}
		}
	}
	p.finishStop(logFile)
}

func (p *singboxProcess) finishStop(logFile *os.File) {
	p.mu.Lock()
	p.cmd = nil
	p.done = nil
	if p.logFile != nil {
		p.logFile = nil
	}
	p.mu.Unlock()
	if logFile != nil {
		_ = logFile.Close()
	}
}

// exitedErr reports the error the process died with, if it has exited
// unexpectedly.
func (p *singboxProcess) exitedErr() error {
	p.mu.Lock()
	defer p.mu.Unlock()
	return p.exitErr
}

var (
	singboxStopGrace = 5 * time.Second
	singboxStopForce = 2 * time.Second
	// singboxStartSettle is the post-start window in which an
	// immediate exit means the new config cannot run (the check gate
	// catches shape errors earlier; this catches runtime failures
	// like a port already in use).
	singboxStartSettle  = 300 * time.Millisecond
	singboxCheckTimeout = 30 * time.Second
)

// checkSingboxConfig runs `sing-box check` against a candidate file.
func checkSingboxConfig(binPath, candidate string) error {
	ctx, cancel := context.WithTimeout(context.Background(), singboxCheckTimeout)
	defer cancel()
	out, err := exec.CommandContext(ctx, binPath, "check", "-c", candidate).CombinedOutput()
	if err != nil {
		return fmt.Errorf("sing-box check failed: %w: %s", err, strings.TrimSpace(string(out)))
	}
	return nil
}

// singboxStatsListen extracts experimental.v2ray_api.listen from a
// sing-box config, the address the stats client must query. "" when
// the config carries no v2ray_api block (no stats to read).
func singboxStatsListen(raw json.RawMessage) string {
	var cfg struct {
		Experimental struct {
			V2RayAPI struct {
				Listen string `json:"listen"`
			} `json:"v2ray_api"`
		} `json:"experimental"`
	}
	if err := json.Unmarshal(raw, &cfg); err != nil {
		return ""
	}
	return cfg.Experimental.V2RayAPI.Listen
}

// applySingboxConfig validates and activates a newly delivered
// sing-box configuration. Errors leave the previously running core
// (if any) completely untouched.
func (a *Agent) applySingboxConfig(raw json.RawMessage) error {
	a.sbMu.Lock()
	defer a.sbMu.Unlock()
	a.ensureSingboxState()
	return a.applySingboxConfigLocked(raw)
}

func (a *Agent) applySingboxConfigLocked(raw json.RawMessage) error {
	binPath := singboxBinaryPath()
	if binPath == "" {
		return errSingboxNotInstalled
	}
	var probe map[string]any
	if err := json.Unmarshal(raw, &probe); err != nil {
		return fmt.Errorf("invalid sing-box config JSON: %w", err)
	}

	configPath := singboxConfigPath()
	candidate := singboxCandidatePath()
	if err := os.WriteFile(candidate, raw, 0o600); err != nil {
		return fmt.Errorf("write sing-box candidate config: %w", err)
	}
	if err := checkSingboxConfig(binPath, candidate); err != nil {
		_ = os.Remove(candidate)
		return err
	}

	current, readErr := os.ReadFile(configPath)
	unchanged := readErr == nil && string(current) == string(raw)
	if unchanged && a.sbProc.isRunning() {
		_ = os.Remove(candidate)
		return nil
	}

	// Flush before change: the running core's counters die with the
	// process, so report them while it still lives. A failed flush
	// aborts the swap — NexusGate's discipline; the epoch guard in the
	// stats tracker is the second line of defence.
	if a.sbProc.isRunning() {
		if err := a.reportSingboxTrafficLocked(); err != nil {
			_ = os.Remove(candidate)
			return fmt.Errorf("flush traffic before sing-box config change: %w", err)
		}
	}

	if !unchanged {
		if readErr == nil {
			if err := os.WriteFile(singboxPreviousPath(), current, 0o600); err != nil {
				_ = os.Remove(candidate)
				return fmt.Errorf("keep previous sing-box config: %w", err)
			}
		}
		if err := os.Rename(candidate, configPath); err != nil {
			_ = os.Remove(candidate)
			return fmt.Errorf("activate sing-box config: %w", err)
		}
	} else {
		_ = os.Remove(candidate)
	}

	// The stats address follows the applied config; a config without
	// a v2ray_api block clears it so the traffic cycle stays quiet.
	a.sbStatsAddr = singboxStatsListen(raw)

	a.sbProc.stop()
	if err := a.startSingboxLocked(binPath, configPath); err != nil {
		// Roll back to the previous config (if one exists) so the node
		// keeps serving; the apply itself reports failure either way.
		if rbErr := a.rollbackSingboxLocked(binPath, configPath); rbErr != nil {
			logger.Warningf("sing-box rollback failed: %v", rbErr)
		}
		return err
	}
	return nil
}

// reconcileSingboxOrphans stops untracked sing-box processes running
// this exact live config file: an agent restart loses the handle
// while the old core survives reparented holding the inbound ports,
// which otherwise makes every apply/rollback die on bind forever.
// The exact-path match is the safety boundary — a core on any other
// config file belongs to another panel/agent/router and is never
// touched. Linux only; the Windows agent line is frozen, so the scan
// is a no-op there. Returns how many orphans were reaped.
func reconcileSingboxOrphans(configPath string, ownPid int) int {
	if runtime.GOOS != "linux" {
		return 0
	}
	self := os.Getpid()
	reaped := 0
	entries, err := os.ReadDir("/proc")
	if err != nil {
		return 0
	}
	for _, e := range entries {
		pid, err := strconv.Atoi(e.Name())
		if err != nil || pid <= 0 || pid == self || pid == ownPid {
			continue
		}
		if !singboxCmdlineMatchesConfig(pid, configPath) {
			continue
		}
		logger.Infof("sing-box orphan on %s (pid %d) from a lost handle; stopping it before start", configPath, pid)
		stopForeignProcess(pid)
		if foreignPidAlive(pid) {
			logger.Warningf("sing-box orphan pid %d survived SIGKILL; the new core may fail to bind", pid)
			continue
		}
		reaped++
	}
	return reaped
}

// singboxCmdlineMatchesConfig reports whether pid's argv contains
// the consecutive triple `run -c <configPath>`, the exact invocation
// the agent itself uses. A substring match would be unsafe: another
// config path merely sharing a prefix must never qualify.
func singboxCmdlineMatchesConfig(pid int, configPath string) bool {
	data, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
	if err != nil {
		return false
	}
	args := strings.Split(string(data), "\x00")
	for len(args) > 0 && args[len(args)-1] == "" {
		args = args[:len(args)-1]
	}
	for i := 0; i+2 < len(args); i++ {
		if args[i] == "run" && args[i+1] == "-c" && args[i+2] == configPath {
			return true
		}
	}
	return false
}

// foreignPidAlive treats a zombie as dead: a reparented orphan's
// parent reaps it eventually, but its ports are already released.
func foreignPidAlive(pid int) bool {
	if st, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil {
		if i := strings.LastIndexByte(string(st), ')'); i >= 0 && i+2 < len(st) && st[i+2] == 'Z' {
			return false
		}
	} else {
		return false
	}
	err := syscall.Kill(pid, 0)
	return err == nil || err == syscall.EPERM
}

// stopForeignProcess mirrors singboxProcess.stop: SIGTERM, then
// SIGKILL after the same grace, then a bounded wait for the kill.
func stopForeignProcess(pid int) {
	proc, _ := os.FindProcess(pid)
	if proc == nil {
		return
	}
	_ = proc.Signal(syscall.SIGTERM)
	if waitForeignPidExit(pid, singboxStopGrace) {
		return
	}
	_ = proc.Kill()
	waitForeignPidExit(pid, singboxStopForce)
}

func waitForeignPidExit(pid int, timeout time.Duration) bool {
	deadline := time.Now().Add(timeout)
	for {
		if !foreignPidAlive(pid) {
			return true
		}
		if time.Now().After(deadline) {
			return false
		}
		time.Sleep(25 * time.Millisecond)
	}
}

// startSingboxLocked starts the process and verifies it survives the
// settle window.
func (a *Agent) startSingboxLocked(binPath, configPath string) error {
	ownPid := 0
	if a.sbProc != nil {
		ownPid = a.sbProc.pid()
	}
	reconcileSingboxOrphans(configPath, ownPid)
	if err := a.sbProc.start(binPath, configPath, singboxLogPath()); err != nil {
		return err
	}
	time.Sleep(singboxStartSettle)
	if !a.sbProc.isRunning() {
		if exitErr := a.sbProc.exitedErr(); exitErr != nil {
			return fmt.Errorf("sing-box exited immediately after start: %w", exitErr)
		}
		return errors.New("sing-box exited immediately after start")
	}
	return nil
}

// rollbackSingboxLocked restores the .previous config and starts it.
func (a *Agent) rollbackSingboxLocked(binPath, configPath string) error {
	previous, err := os.ReadFile(singboxPreviousPath())
	if err != nil {
		// Nothing to roll back to: drop the bad config so a later poll
		// starts from a clean slate.
		_ = os.Remove(configPath)
		return fmt.Errorf("no previous sing-box config to roll back to: %w", err)
	}
	if err := os.WriteFile(configPath, previous, 0o600); err != nil {
		return fmt.Errorf("restore previous sing-box config: %w", err)
	}
	a.sbProc.stop()
	return a.startSingboxLocked(binPath, configPath)
}

// stopSingbox stops the managed sing-box process, if any.
func (a *Agent) stopSingbox() {
	a.sbMu.Lock()
	defer a.sbMu.Unlock()
	if a.sbProc != nil {
		a.sbProc.stop()
	}
}

// parseSingboxVersion extracts the version from `sing-box version`
// output ("sing-box version 1.13.21"). Self-built binaries print
// "unknown"; the caller falls back to the install-time record.
func parseSingboxVersion(out string) (string, bool) {
	line := out
	if i := strings.IndexByte(line, '\n'); i >= 0 {
		line = line[:i]
	}
	fields := strings.Fields(line)
	if len(fields) < 3 || !strings.EqualFold(fields[0], "sing-box") || !strings.EqualFold(fields[1], "version") {
		return "", false
	}
	v := fields[2]
	if v == "" {
		return "", false
	}
	return v, true
}

// cachedSingboxVersion mirrors cachedXrayVersion for the sing-box
// binary: re-read only when the binary file changes; an "unknown"
// stamp falls back to the version record install-agent.sh wrote.
func (a *Agent) cachedSingboxVersion() (string, bool) {
	path := singboxBinaryPath()
	if path == "" {
		return "", false
	}
	fi, err := os.Stat(path)
	if err != nil {
		return "", false
	}
	a.sbVerMu.Lock()
	defer a.sbVerMu.Unlock()
	if a.sbVer != "" && a.sbVerFor == path && fi.ModTime().Equal(a.sbVerMod) {
		return a.sbVer, true
	}
	v := ""
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
	out, err := exec.CommandContext(ctx, path, "version").Output()
	cancel()
	if err == nil {
		if parsed, ok := parseSingboxVersion(string(out)); ok && parsed != "unknown" {
			v = parsed
		}
	}
	if v == "" {
		if data, err := os.ReadFile(filepath.Join(config.GetBinFolderPath(), singboxVersionRecord)); err == nil {
			v = strings.TrimSpace(string(data))
		}
	}
	if v == "" {
		return "", false
	}
	a.sbVer, a.sbVerFor, a.sbVerMod = v, path, fi.ModTime()
	return v, true
}