l-ui/internal/database/singbox_migrate_test.go

package database

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/mhsanaei/3x-ui/v3/internal/database/model"

	"gorm.io/driver/sqlite"
	"gorm.io/gorm"
)

// The migrator runs inside InitDB. These tests seed a legacy-shaped
// database file directly (no InitDB), then drive InitDB over it and
// assert the post-migration state.

func seedLegacyDB(t *testing.T, path string, seed func(gdb *gorm.DB)) {
	t.Helper()
	gdb, err := gorm.Open(sqlite.Open(path), &gorm.Config{})
	if err != nil {
		t.Fatalf("open seed db: %v", err)
	}
	if err := gdb.AutoMigrate(
		&model.Inbound{}, &model.ClientRecord{}, &model.ClientInbound{},
		&model.Setting{}, &model.AgentNode{}, &model.Node{},
		&model.InboundFallback{},
	); err != nil {
		t.Fatalf("seed automigrate: %v", err)
	}
	seed(gdb)
	sqlDB, err := gdb.DB()
	if err != nil {
		t.Fatalf("seed sql db: %v", err)
	}
	if err := sqlDB.Close(); err != nil {
		t.Fatalf("close seed db: %v", err)
	}
}

func openRaw(t *testing.T, path string) *gorm.DB {
	t.Helper()
	gdb, err := gorm.Open(sqlite.Open(path), &gorm.Config{})
	if err != nil {
		t.Fatalf("open raw db: %v", err)
	}
	t.Cleanup(func() {
		if sqlDB, err := gdb.DB(); err == nil {
			_ = sqlDB.Close()
		}
	})
	return gdb
}

func getSettingRaw(t *testing.T, gdb *gorm.DB, key string) (string, bool) {
	t.Helper()
	var s model.Setting
	err := gdb.Where("key = ?", key).First(&s).Error
	if err == gorm.ErrRecordNotFound {
		return "", false
	}
	if err != nil {
		t.Fatalf("get setting %s: %v", key, err)
	}
	return s.Value, true
}

func factoryTemplate(t *testing.T) string {
	t.Helper()
	data, err := os.ReadFile(filepath.Join("..", "web", "service", "config.json"))
	if err != nil {
		t.Fatalf("read factory template: %v", err)
	}
	return string(data)
}

// modifiedTemplate returns the factory template with the stats switches
// flipped to a non-default combination and one extra non-default
// section (routing.domainStrategy), as a stored-template stand-in.
func modifiedTemplate(t *testing.T) string {
	t.Helper()
	var tpl map[string]any
	if err := json.Unmarshal([]byte(factoryTemplate(t)), &tpl); err != nil {
		t.Fatalf("parse factory: %v", err)
	}
	sys := tpl["policy"].(map[string]any)["system"].(map[string]any)
	sys["statsInboundUplink"] = true
	sys["statsInboundDownlink"] = false
	sys["statsOutboundUplink"] = true
	sys["statsOutboundDownlink"] = false
	tpl["routing"].(map[string]any)["domainStrategy"] = "UseIPv4"
	b, err := json.Marshal(tpl)
	if err != nil {
		t.Fatalf("marshal template: %v", err)
	}
	return string(b)
}

const realityStream = `{"network":"tcp","security":"reality","tcpSettings":{"acceptProxyProtocol":false,"header":{"type":"none"}},"realitySettings":{"show":false,"xver":0,"target":"www.microsoft.com:443","serverNames":["www.microsoft.com"],"privateKey":"ISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0A=","minClientVer":"","maxClientVer":"","maxTimediff":0,"shortIds":["77aa10f0"],"mldsa65Seed":"","settings":{"publicKey":"PUBKEYDERIVED0000000000000000000000000000000","fingerprint":"chrome","serverName":"","spiderX":"/"}}}`

func seedAgentEraDB(t *testing.T, path string) {
	t.Helper()
	seedLegacyDB(t, path, func(gdb *gorm.DB) {
		// VLESS + Reality master with a fallback row and an embedded
		// duplicate of the same fallback.
		vless := &model.Inbound{
			UserId: 1, Enable: true, Listen: "203.0.113.10", Port: 20443,
			Protocol: model.VLESS, Tag: "n1-in-20443-tcp", Remark: "vless-main",
			Settings:       `{"clients":[{"email":"alice@example.com","id":"11111111-2222-4333-8444-555555555555","flow":"xtls-rprx-vision","password":"mirror-only-pass"},{"email":"bob@example.com","id":"22222222-3333-4444-8555-666666666666"},{"email":"carol@example.com","id":"33333333-4444-4555-8666-777777777777","password":"carol-pass"}],"decryption":"none","encryption":"none","fallbacks":[{"name":"","alpn":"","path":"","dest":8080,"xver":0}]}`,
			StreamSettings: realityStream,
			Sniffing:       `{"enabled":true,"destOverride":["http","tls"],"metadataOnly":false,"routeOnly":false}`,
			Forwarding:     `{"enabled":true}`,
		}
		if err := gdb.Create(vless).Error; err != nil {
			t.Fatalf("seed vless: %v", err)
		}
		// alice: identity row exists but password is empty (mirror fills
		// it — alice's mirror entry carries a password on purpose).
		// bob: identity only. carol: mirror only (adopted).
		alice := &model.ClientRecord{Email: "alice@example.com", UUID: "11111111-2222-4333-8444-555555555555", Flow: "xtls-rprx-vision", Enable: true}
		bob := &model.ClientRecord{Email: "bob@example.com", UUID: "22222222-3333-4444-8555-666666666666", Enable: true}
		if err := gdb.Create(alice).Error; err != nil {
			t.Fatalf("seed alice: %v", err)
		}
		if err := gdb.Create(bob).Error; err != nil {
			t.Fatalf("seed bob: %v", err)
		}
		for _, c := range []*model.ClientRecord{alice, bob} {
			if err := gdb.Create(&model.ClientInbound{ClientId: c.Id, InboundId: vless.Id, FlowOverride: c.Flow}).Error; err != nil {
				t.Fatalf("seed link: %v", err)
			}
		}
		child := &model.Inbound{
			UserId: 1, Enable: true, Listen: "127.0.0.1", Port: 8080,
			Protocol: model.VLESS, Tag: "n1-in-8080-tcp", Remark: "fallback-child",
			Settings: `{"clients":[]}`, StreamSettings: `{"network":"tcp","security":"none"}`,
		}
		if err := gdb.Create(child).Error; err != nil {
			t.Fatalf("seed child: %v", err)
		}
		if err := gdb.Create(&model.InboundFallback{MasterId: vless.Id, ChildId: child.Id, Dest: "8080"}).Error; err != nil {
			t.Fatalf("seed fallback: %v", err)
		}

		ss := &model.Inbound{
			UserId: 1, Enable: true, Listen: "", Port: 28443,
			Protocol: model.Shadowsocks, Tag: "n1-in-28443-tcpudp", Remark: "ss-main",
			Settings:       `{"method":"2022-blake3-aes-128-gcm","password":"MDEyMzQ1Njc4OWFiY2RlZg==","network":"tcp,udp","ivCheck":false,"clients":[{"email":"ss@example.com","password":"Y2xpZW50LXBhc3MtMDAwMQ=="}]}`,
			StreamSettings: `{"network":"tcp,udp","security":"none"}`,
			Sniffing:       `{"enabled":false,"destOverride":[],"metadataOnly":false,"routeOnly":false}`,
		}
		if err := gdb.Create(ss).Error; err != nil {
			t.Fatalf("seed ss: %v", err)
		}
		ssClient := &model.ClientRecord{Email: "ss@example.com", Password: "Y2xpZW50LXBhc3MtMDAwMQ==", Enable: true}
		if err := gdb.Create(ssClient).Error; err != nil {
			t.Fatalf("seed ss client: %v", err)
		}
		if err := gdb.Create(&model.ClientInbound{ClientId: ssClient.Id, InboundId: ss.Id}).Error; err != nil {
			t.Fatalf("seed ss link: %v", err)
		}

		xhttp := &model.Inbound{
			UserId: 1, Enable: true, Listen: "203.0.113.10", Port: 21443,
			Protocol: model.VLESS, Tag: "n1-in-21443-tcp", Remark: "vless-xhttp",
			Settings:       `{"clients":[{"email":"xh@example.com","id":"44444444-5555-4666-8777-888888888888"}],"decryption":"none"}`,
			StreamSettings: `{"network":"xhttp","security":"tls","xhttpSettings":{"path":"/xh"},"tlsSettings":{"serverName":"xh.example.com","certificates":[{"certificateFile":"/etc/x-ui/cert/fx.pem","keyFile":"/etc/x-ui/cert/fx.key"}]}}`,
		}
		if err := gdb.Create(xhttp).Error; err != nil {
			t.Fatalf("seed xhttp: %v", err)
		}
		xhClient := &model.ClientRecord{Email: "xh@example.com", UUID: "44444444-5555-4666-8777-888888888888", Enable: true}
		if err := gdb.Create(xhClient).Error; err != nil {
			t.Fatalf("seed xh client: %v", err)
		}
		if err := gdb.Create(&model.ClientInbound{ClientId: xhClient.Id, InboundId: xhttp.Id}).Error; err != nil {
			t.Fatalf("seed xh link: %v", err)
		}

		agent := &model.AgentNode{
			Name: "node-a",
			Outbounds: `[{"tag":"direct","protocol":"freedom","settings":{}},` +
				`{"tag":"block","protocol":"blackhole","settings":{}},` +
				`{"tag":"up","protocol":"socks","settings":{"servers":[{"address":"10.0.0.2","port":1080,"users":[{"user":"u1","pass":"p1"}]}]}}]`,
			RoutingRules: `[{"type":"field","outboundTag":"direct","domain":["full:example.com","example.org"]},` +
				`{"type":"field","outboundTag":"block","ip":["geoip:private"]}]`,
		}
		if err := gdb.Create(agent).Error; err != nil {
			t.Fatalf("seed agent: %v", err)
		}
		if err := gdb.Create(&model.Node{Name: "legacy-node-1"}).Error; err != nil {
			t.Fatalf("seed node: %v", err)
		}
		if err := gdb.Create(&model.Setting{Key: "xrayTemplateConfig", Value: modifiedTemplate(t)}).Error; err != nil {
			t.Fatalf("seed template: %v", err)
		}
	})
}

func TestSingboxMigration_AgentEraSample(t *testing.T) {
	FactoryXrayTemplate = factoryTemplate(t)
	dir := t.TempDir()
	path := filepath.Join(dir, "x-ui.db")
	seedAgentEraDB(t, path)

	if err := InitDB(path); err != nil {
		t.Fatalf("InitDB (migration): %v", err)
	}
	t.Cleanup(func() { _ = CloseDB() })
	db := GetDB()

	// Marker + stats: stats migrated BEFORE the template row went away.
	if v, ok := getSettingRaw(t, db, singboxMigrationVersionKey); !ok || v != "1" {
		t.Fatalf("marker = %q, %v; want 1", v, ok)
	}
	statsRaw, ok := getSettingRaw(t, db, trafficStatsSwitchesKey)
	if !ok {
		t.Fatalf("trafficStatsSwitches missing")
	}
	var stats trafficStatsSwitches
	if err := json.Unmarshal([]byte(statsRaw), &stats); err != nil {
		t.Fatalf("parse stats: %v", err)
	}
	wantStats := trafficStatsSwitches{InboundUplink: true, InboundDownlink: false, OutboundUplink: true, OutboundDownlink: false}
	if stats != wantStats {
		t.Fatalf("stats = %+v, want %+v", stats, wantStats)
	}
	if _, ok := getSettingRaw(t, db, xrayTemplateSettingKey); ok {
		t.Fatalf("xrayTemplateConfig row should be discarded after stats migration")
	}

	// VLESS fragment: reality key carried, 3 users (2 identity + carol
	// adopted), fallback registered exactly once despite the duplicate.
	var vless model.Inbound
	if err := db.Where("port = ?", 20443).First(&vless).Error; err != nil {
		t.Fatalf("load vless: %v", err)
	}
	var frag map[string]any
	if err := json.Unmarshal([]byte(vless.SingboxConfig), &frag); err != nil {
		t.Fatalf("vless fragment not JSON: %v (%q)", err, vless.SingboxConfig)
	}
	if frag["type"] != "vless" {
		t.Fatalf("fragment type = %v", frag["type"])
	}
	tlsBlk := frag["tls"].(map[string]any)
	reality := tlsBlk["reality"].(map[string]any)
	if reality["private_key"] != "ISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0A=" {
		t.Fatalf("reality private_key = %v", reality["private_key"])
	}
	users := frag["users"].([]any)
	if len(users) != 3 {
		t.Fatalf("users = %d, want 3 (alice, bob, carol adopted)", len(users))
	}
	var pres struct {
		Unmapped []struct {
			Concept string `json:"concept"`
		} `json:"unmapped"`
		Panel map[string]any `json:"panel"`
	}
	if err := json.Unmarshal([]byte(vless.SingboxPreserved), &pres); err != nil {
		t.Fatalf("preserved not JSON: %v", err)
	}
	fbCount := 0
	for _, u := range pres.Unmapped {
		if strings.HasPrefix(u.Concept, "fallbacks") {
			fbCount++
		}
	}
	if fbCount != 1 {
		t.Fatalf("fallback unmapped entries = %d, want 1 (table row + embedded dup deduped)", fbCount)
	}
	if pres.Panel["realityClientSettings"] == nil {
		t.Fatalf("reality client settings should be panel-preserved")
	}

	// SS fragment.
	var ss model.Inbound
	if err := db.Where("port = ?", 28443).First(&ss).Error; err != nil {
		t.Fatalf("load ss: %v", err)
	}
	var ssFrag map[string]any
	if err := json.Unmarshal([]byte(ss.SingboxConfig), &ssFrag); err != nil {
		t.Fatalf("ss fragment: %v", err)
	}
	if ssFrag["method"] != "2022-blake3-aes-128-gcm" || ssFrag["password"] != "MDEyMzQ1Njc4OWFiY2RlZg==" {
		t.Fatalf("ss fragment wrong: %v", ssFrag)
	}
	if _, hasNetwork := ssFrag["network"]; hasNetwork {
		t.Fatalf("ss tcp,udp should be omitted (sing-box default)")
	}

	// xhttp: blocked, no fragment, preserved explains.
	var xh model.Inbound
	if err := db.Where("port = ?", 21443).First(&xh).Error; err != nil {
		t.Fatalf("load xhttp: %v", err)
	}
	if xh.SingboxConfig != "" {
		t.Fatalf("xhttp fragment should be empty, got %q", xh.SingboxConfig)
	}
	if !strings.Contains(xh.SingboxPreserved, "xhttp") {
		t.Fatalf("xhttp preserved should register the transport: %q", xh.SingboxPreserved)
	}

	// Agent overrides converted; the geo routing rule is reported, the
	// plain rule converted.
	var agent model.AgentNode
	if err := db.First(&agent).Error; err != nil {
		t.Fatalf("load agent: %v", err)
	}
	if !strings.Contains(agent.SingboxOutbounds, `"type":"direct"`) ||
		!strings.Contains(agent.SingboxOutbounds, `"type":"block"`) ||
		!strings.Contains(agent.SingboxOutbounds, `"server":"10.0.0.2"`) {
		t.Fatalf("agent singbox outbounds wrong: %q", agent.SingboxOutbounds)
	}
	if strings.Contains(agent.SingboxRoutingRules, "geoip") {
		t.Fatalf("geo rule must not be converted: %q", agent.SingboxRoutingRules)
	}
	if !strings.Contains(agent.SingboxRoutingRules, `"domain":["example.com"]`) {
		t.Fatalf("converted routing rule wrong: %q", agent.SingboxRoutingRules)
	}

	// Backup + report files exist next to the DB.
	baks, _ := filepath.Glob(path + ".pre-singbox-*.bak")
	if len(baks) != 1 {
		t.Fatalf("backup files = %v, want exactly 1", baks)
	}
	reports, _ := filepath.Glob(path + ".singbox-migration-*.json")
	if len(reports) != 1 {
		t.Fatalf("report files = %v, want exactly 1", reports)
	}
	repData, err := os.ReadFile(reports[0])
	if err != nil {
		t.Fatalf("read report: %v", err)
	}
	var rep singboxMigrationReport
	if err := json.Unmarshal(repData, &rep); err != nil {
		t.Fatalf("parse report: %v", err)
	}
	if rep.InboundsTotal != 4 || rep.StatsSource != "stored-template" {
		t.Fatalf("report wrong: total=%d statsSource=%q", rep.InboundsTotal, rep.StatsSource)
	}
	sawRouting := false
	for _, s := range rep.TemplateNonDefault {
		if s == "routing" {
			sawRouting = true
		}
		if s == "policy" {
			t.Fatalf("policy must not count as non-default once its stats keys are migrated out")
		}
	}
	if !sawRouting {
		t.Fatalf("report templateNonDefault = %v, want routing listed", rep.TemplateNonDefault)
	}
	kinds := map[string]int{}
	for _, d := range rep.Discrepancies {
		kinds[d.Kind]++
	}
	if kinds["mirror-only-client"] != 1 {
		t.Fatalf("discrepancy kinds = %v, want one mirror-only-client (carol)", kinds)
	}
	if len(rep.AgentNodes) != 1 || len(rep.AgentNodes[0].RoutingIssues) != 1 {
		t.Fatalf("agent report wrong: %+v", rep.AgentNodes)
	}
	if rep.ForwardingInbounds != 1 || rep.TraditionalNodes != 1 {
		t.Fatalf("report counts wrong: forwarding=%d nodes=%d", rep.ForwardingInbounds, rep.TraditionalNodes)
	}
}

func snapshotTables(t *testing.T, path string) string {
	t.Helper()
	gdb := openRaw(t, path)
	var sb strings.Builder
	var inbounds []model.Inbound
	if err := gdb.Order("id ASC").Find(&inbounds).Error; err != nil {
		t.Fatalf("snapshot inbounds: %v", err)
	}
	for _, ib := range inbounds {
		b, _ := json.Marshal(ib)
		sb.Write(b)
		sb.WriteByte('\n')
	}
	var settings []model.Setting
	if err := gdb.Order("key ASC").Find(&settings).Error; err != nil {
		t.Fatalf("snapshot settings: %v", err)
	}
	for _, s := range settings {
		sb.WriteString(s.Key + "=" + s.Value + "\n")
	}
	var agents []model.AgentNode
	if err := gdb.Order("id ASC").Find(&agents).Error; err != nil {
		t.Fatalf("snapshot agents: %v", err)
	}
	for _, a := range agents {
		b, _ := json.Marshal(a)
		sb.Write(b)
		sb.WriteByte('\n')
	}
	return sb.String()
}

func TestSingboxMigration_Idempotent(t *testing.T) {
	FactoryXrayTemplate = factoryTemplate(t)
	dir := t.TempDir()
	path := filepath.Join(dir, "x-ui.db")
	seedAgentEraDB(t, path)

	if err := InitDB(path); err != nil {
		t.Fatalf("first InitDB: %v", err)
	}
	if err := CloseDB(); err != nil {
		t.Fatalf("close: %v", err)
	}
	first := snapshotTables(t, path)

	if err := InitDB(path); err != nil {
		t.Fatalf("second InitDB: %v", err)
	}
	if err := CloseDB(); err != nil {
		t.Fatalf("close 2: %v", err)
	}
	second := snapshotTables(t, path)
	if first != second {
		t.Fatalf("second run changed the database\nfirst:\n%s\nsecond:\n%s", first, second)
	}
	baks, _ := filepath.Glob(path + ".pre-singbox-*.bak")
	if len(baks) != 1 {
		t.Fatalf("second run must not take another backup, got %v", baks)
	}
}

func TestSingboxMigration_BadInboundRowRefusesStart(t *testing.T) {
	FactoryXrayTemplate = factoryTemplate(t)
	dir := t.TempDir()
	path := filepath.Join(dir, "x-ui.db")
	seedLegacyDB(t, path, func(gdb *gorm.DB) {
		bad := &model.Inbound{
			UserId: 1, Enable: true, Port: 30443, Protocol: model.VLESS,
			Tag: "bad-1", Settings: `{"clients": [broken`,
			StreamSettings: `{"network":"tcp","security":"none"}`,
		}
		if err := gdb.Create(bad).Error; err != nil {
			t.Fatalf("seed bad: %v", err)
		}
		if err := gdb.Create(&model.Setting{Key: "xrayTemplateConfig", Value: modifiedTemplate(t)}).Error; err != nil {
			t.Fatalf("seed template: %v", err)
		}
	})

	err := InitDB(path)
	if err == nil {
		_ = CloseDB()
		t.Fatalf("InitDB should refuse to start on a malformed inbound row")
	}
	_ = CloseDB()

	// Old database content is untouched: no marker, template intact, no
	// converted fragments.
	gdb := openRaw(t, path)
	if _, ok := getSettingRaw(t, gdb, singboxMigrationVersionKey); ok {
		t.Fatalf("marker must not be written on a failed migration")
	}
	if _, ok := getSettingRaw(t, gdb, xrayTemplateSettingKey); !ok {
		t.Fatalf("template row must survive a failed migration")
	}
	var ib model.Inbound
	if err := gdb.First(&ib).Error; err != nil {
		t.Fatalf("load inbound: %v", err)
	}
	if ib.SingboxConfig != "" || ib.SingboxPreserved != "" {
		t.Fatalf("failed migration must leave no converted state, got config=%q preserved=%q", ib.SingboxConfig, ib.SingboxPreserved)
	}
	// The pre-migration backup was still taken before the attempt.
	baks, _ := filepath.Glob(path + ".pre-singbox-*.bak")
	if len(baks) != 1 {
		t.Fatalf("backup files = %v, want 1", baks)
	}
}

func TestSingboxMigration_BadTemplateRefusesStart(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "x-ui.db")
	seedLegacyDB(t, path, func(gdb *gorm.DB) {
		if err := gdb.Create(&model.Setting{Key: "xrayTemplateConfig", Value: "not-json{"}).Error; err != nil {
			t.Fatalf("seed: %v", err)
		}
	})
	if err := InitDB(path); err == nil {
		_ = CloseDB()
		t.Fatalf("InitDB should refuse to start when the stored template cannot be parsed")
	}
	_ = CloseDB()
	gdb := openRaw(t, path)
	if _, ok := getSettingRaw(t, gdb, singboxMigrationVersionKey); ok {
		t.Fatalf("marker must not be written on a failed migration")
	}
	if v, ok := getSettingRaw(t, gdb, xrayTemplateSettingKey); !ok || v != "not-json{" {
		t.Fatalf("template row must be untouched, got %q", v)
	}
}

func TestSingboxMigration_FreshDatabase(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "x-ui.db")
	if err := InitDB(path); err != nil {
		t.Fatalf("InitDB fresh: %v", err)
	}
	t.Cleanup(func() { _ = CloseDB() })
	db := GetDB()
	if v, ok := getSettingRaw(t, db, singboxMigrationVersionKey); !ok || v != "1" {
		t.Fatalf("fresh db marker = %q, %v", v, ok)
	}
	statsRaw, ok := getSettingRaw(t, db, trafficStatsSwitchesKey)
	if !ok {
		t.Fatalf("fresh db should record the factory-default stats switches")
	}
	var stats trafficStatsSwitches
	if err := json.Unmarshal([]byte(statsRaw), &stats); err != nil {
		t.Fatalf("parse stats: %v", err)
	}
	if stats != defaultTrafficStatsSwitches() {
		t.Fatalf("fresh stats = %+v, want factory defaults", stats)
	}
	// No legacy content: no backup taken.
	baks, _ := filepath.Glob(path + ".pre-singbox-*.bak")
	if len(baks) != 0 {
		t.Fatalf("fresh db must not produce backups, got %v", baks)
	}
}