3x-ui-agent/internal/database/singbox_migrate_backupdb_test.go

package database

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"golang.org/x/crypto/curve25519"
	"gorm.io/driver/sqlite"
	"gorm.io/gorm"
)

// TestSingboxMigration_OldBackupValidation runs the real migration over
// a copy of a production-era backup database and verifies, field by
// field, that nothing the panel depends on moved: identity credentials,
// Reality key material (stored private key → derived pbk equals the
// stored client-side public key), traffic counters and reset cycles,
// and every legacy column byte-for-byte.
//
// Gated on XUI_B1_OLD_BACKUP pointing at the backup .db file so normal
// test runs skip it.
func TestSingboxMigration_OldBackupValidation(t *testing.T) {
	src := os.Getenv("XUI_B1_OLD_BACKUP")
	if src == "" {
		t.Skip("XUI_B1_OLD_BACKUP not set")
	}
	dir := t.TempDir()
	dst := filepath.Join(dir, "backup.db")
	copyFile(t, src, dst)
	for _, suffix := range []string{"-wal", "-shm"} {
		if _, err := os.Stat(src + suffix); err == nil {
			copyFile(t, src+suffix, dst+suffix)
		}
	}

	pre := dumpAllTables(t, dst)
	preIdentity := identityByInbound(t, dst)
	preMirror := mirrorUUIDsByInbound(t, dst)
	preTemplateStats := templateStats(t, dst)
	preSettings := settingsMap(t, dst)

	if err := InitDB(dst); err != nil {
		t.Fatalf("InitDB over backup copy: %v", err)
	}
	if err := CloseDB(); err != nil {
		t.Fatalf("close: %v", err)
	}

	post := dumpAllTables(t, dst)
	postSettings := settingsMap(t, dst)

	// 1. Legacy content identical apart from the sanctioned deltas:
	// the new singbox_* columns on inbounds, the discarded template
	// row, and the two new settings keys. Comparison projects each row
	// onto the columns that existed pre-migration (InitDB's own legacy
	// migrations may add columns), and the three JSON text columns of
	// inbounds are compared semantically (legacy migrations may
	// normalize their formatting without changing meaning).
	assertTableUnchanged(t, pre, post, "clients")
	assertTableUnchanged(t, pre, post, "client_inbounds")
	assertTableUnchanged(t, pre, post, "client_traffics")
	assertInboundsUnchanged(t, pre, post)
	assertSettingsDelta(t, preSettings, postSettings)

	// 2. Converted fragments carry exactly the identity layer.
	gdb := openRaw(t, dst)
	var rows []struct {
		Id               int
		Protocol         string
		SingboxConfig    string
		SingboxPreserved string
		StreamSettings   string
		Settings         string
	}
	if err := gdb.Table("inbounds").
		Select("id, protocol, singbox_config, singbox_preserved, stream_settings, settings").
		Order("id ASC").Scan(&rows).Error; err != nil {
		t.Fatalf("scan converted inbounds: %v", err)
	}
	if len(rows) != 25 {
		t.Fatalf("inbounds = %d, want 25", len(rows))
	}
	blocked := 0
	for _, r := range rows {
		if r.SingboxConfig == "" {
			blocked++
			t.Errorf("inbound %d (%s) unexpectedly blocked: preserved=%s", r.Id, r.Protocol, r.SingboxPreserved)
			continue
		}
		var frag map[string]any
		if err := json.Unmarshal([]byte(r.SingboxConfig), &frag); err != nil {
			t.Fatalf("inbound %d fragment not JSON: %v", r.Id, err)
		}
		want := preIdentity[r.Id]
		switch r.Protocol {
		case "vless":
			got := map[string]bool{}
			for _, u := range fragUsers(frag) {
				got[u["uuid"]] = true
				if u["uuid"] == "" {
					t.Errorf("inbound %d: vless user with empty uuid: %v", r.Id, u)
				}
			}
			// Every identity uuid must be projected; extras are only
			// legitimate as spec §7-1 mirror adoptions.
			for uuid := range want {
				if !got[uuid] {
					t.Errorf("inbound %d: identity uuid %s missing from fragment", r.Id, uuid)
				}
			}
			for uuid := range got {
				if _, inIdentity := want[uuid]; !inIdentity && !preMirror[r.Id][uuid] {
					t.Errorf("inbound %d: fragment uuid %s in neither identity nor mirror", r.Id, uuid)
				}
			}
			checkReality(t, r.Id, r.StreamSettings, frag)
		case "shadowsocks":
			for _, u := range fragUsers(frag) {
				wantPW, ok := want[u["name"]]
				if !ok {
					t.Errorf("inbound %d: ss user %q not in identity", r.Id, u["name"])
				} else if u["password"] != wantPW {
					t.Errorf("inbound %d: ss user %q password mismatch", r.Id, u["name"])
				}
			}
			if len(fragUsers(frag)) != len(want) {
				t.Errorf("inbound %d: ss users %d != identity %d", r.Id, len(fragUsers(frag)), len(want))
			}
		case "mixed":
			for _, u := range fragUsers(frag) {
				if wantPW, ok := want[u["username"]]; ok && u["password"] != wantPW {
					t.Errorf("inbound %d: mixed user %q password mismatch", r.Id, u["username"])
				}
			}
		}
	}
	if blocked != 0 {
		t.Fatalf("blocked inbounds = %d, want 0 (production profile is all direct)", blocked)
	}

	// 3. Stats switches equal the pre-migration template's values.
	if v, ok := getSettingRaw(t, gdb, trafficStatsSwitchesKey); !ok {
		t.Fatalf("trafficStatsSwitches missing after migration")
	} else {
		var got trafficStatsSwitches
		if err := json.Unmarshal([]byte(v), &got); err != nil {
			t.Fatalf("parse stats: %v", err)
		}
		if preTemplateStats != nil && got != *preTemplateStats {
			t.Fatalf("stats = %+v, want template's %+v", got, *preTemplateStats)
		}
	}
	if _, ok := getSettingRaw(t, gdb, xrayTemplateSettingKey); ok {
		t.Fatalf("template row should be discarded")
	}
	if v, ok := getSettingRaw(t, gdb, singboxMigrationVersionKey); !ok || v != "1" {
		t.Fatalf("marker = %q, %v", v, ok)
	}

	// 4. Backup + report artifacts exist.
	baks, _ := filepath.Glob(dst + ".pre-singbox-*.bak")
	if len(baks) != 1 {
		t.Fatalf("backup files = %v, want 1", baks)
	}
	reports, _ := filepath.Glob(dst + ".singbox-migration-*.json")
	if len(reports) != 1 {
		t.Fatalf("report files = %v, want 1", reports)
	}
	repData, _ := os.ReadFile(reports[0])
	var rep singboxMigrationReport
	if err := json.Unmarshal(repData, &rep); err != nil {
		t.Fatalf("parse report: %v", err)
	}
	t.Logf("report: inbounds=%d fragments=%d endpoints=%d clients=%d dispositions=%v discrepancies=%d unmapped=%d notes=%d statsSource=%s templateNonDefault=%v",
		rep.InboundsTotal, rep.FragmentsWritten, rep.EndpointsWritten, rep.ClientsProjected,
		rep.Dispositions, len(rep.Discrepancies), len(rep.Unmapped), len(rep.Notes),
		rep.StatsSource, rep.TemplateNonDefault)
	for _, d := range rep.Discrepancies {
		t.Logf("discrepancy: inbound=%d email=%s kind=%s field=%s", d.InboundID, d.Email, d.Kind, d.Field)
	}
	for _, u := range rep.Unmapped {
		t.Logf("unmapped: inbound=%d concept=%s pending=%s material=%v", u.InboundID, u.Concept, u.Pending, u.Material)
	}
}

// checkReality verifies the fragment's private_key matches the stored
// streamSettings and that its derived public key equals the stored
// client-side publicKey (the pbk subscribers already use).
func checkReality(t *testing.T, inboundID int, streamSettings string, frag map[string]any) {
	t.Helper()
	var ss map[string]any
	if err := json.Unmarshal([]byte(streamSettings), &ss); err != nil {
		t.Fatalf("inbound %d stream: %v", inboundID, err)
	}
	rs, _ := ss["realitySettings"].(map[string]any)
	if rs == nil {
		return // not a reality inbound
	}
	privStored, _ := rs["privateKey"].(string)
	tlsBlk, _ := frag["tls"].(map[string]any)
	reality, _ := tlsBlk["reality"].(map[string]any)
	if reality == nil {
		t.Errorf("inbound %d: reality block missing from fragment", inboundID)
		return
	}
	if reality["private_key"] != privStored {
		t.Errorf("inbound %d: fragment private_key != stored privateKey", inboundID)
	}
	settings, _ := rs["settings"].(map[string]any)
	storedPub, _ := settings["publicKey"].(string)
	if storedPub == "" || privStored == "" {
		return
	}
	privBytes, err := decodeB64(privStored)
	if err != nil || len(privBytes) != 32 {
		t.Errorf("inbound %d: stored privateKey not a 32-byte base64 key", inboundID)
		return
	}
	pubBytes, err := decodeB64(storedPub)
	if err != nil || len(pubBytes) != 32 {
		t.Errorf("inbound %d: stored publicKey not a 32-byte base64 key", inboundID)
		return
	}
	var priv, pub [32]byte
	copy(priv[:], privBytes)
	curve25519.ScalarBaseMult(&pub, &priv)
	if string(pub[:]) != string(pubBytes) {
		t.Errorf("inbound %d: derived pbk does not match stored publicKey", inboundID)
	}
}

// decodeB64 accepts the base64 flavours xray uses for Reality keys
// (raw URL encoding without padding, or padded standard/url).
func decodeB64(s string) ([]byte, error) {
	for _, enc := range []*base64.Encoding{
		base64.RawURLEncoding, base64.URLEncoding,
		base64.StdEncoding, base64.RawStdEncoding,
	} {
		if b, err := enc.DecodeString(s); err == nil {
			return b, nil
		}
	}
	return nil, fmt.Errorf("not base64")
}

func fragUsers(frag map[string]any) []map[string]string {
	out := []map[string]string{}
	if arr, ok := frag["users"].([]any); ok {
		for _, e := range arr {
			m, _ := e.(map[string]any)
			u := map[string]string{}
			for k, v := range m {
				if s, ok := v.(string); ok {
					u[k] = s
				}
			}
			out = append(out, u)
		}
	}
	return out
}

func keys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// identityByInbound computes, from the PRE-migration copy, each
// inbound's identity credentials (email → uuid for vless/vmess,
// email → password elsewhere) using ListForInbound's join semantics.
func identityByInbound(t *testing.T, path string) map[int]map[string]string {
	t.Helper()
	gdb := openRaw(t, path)
	type row struct {
		InboundId int
		Email     string
		UUID      string
		Password  string
		Protocol  string
	}
	var rows []row
	err := gdb.Table("clients").
		Select("client_inbounds.inbound_id AS inbound_id, clients.email AS email, clients.uuid AS uuid, clients.password AS password, '' AS protocol").
		Joins("JOIN client_inbounds ON client_inbounds.client_id = clients.id").
		Scan(&rows).Error
	if err != nil {
		t.Fatalf("identity query: %v", err)
	}
	protoByInbound := map[int]string{}
	var ibs []struct {
		Id       int
		Protocol string
	}
	if err := gdb.Table("inbounds").Select("id, protocol").Scan(&ibs).Error; err != nil {
		t.Fatalf("inbound protocols: %v", err)
	}
	for _, ib := range ibs {
		protoByInbound[ib.Id] = ib.Protocol
	}
	out := map[int]map[string]string{}
	for _, r := range rows {
		if out[r.InboundId] == nil {
			out[r.InboundId] = map[string]string{}
		}
		switch protoByInbound[r.InboundId] {
		case "vless", "vmess":
			out[r.InboundId][r.UUID] = r.Email
		default:
			out[r.InboundId][r.Email] = r.Password
		}
	}
	return out
}

func templateStats(t *testing.T, path string) *trafficStatsSwitches {
	t.Helper()
	gdb := openRaw(t, path)
	v, ok := getSettingRaw(t, gdb, xrayTemplateSettingKey)
	if !ok {
		return nil
	}
	var tpl map[string]any
	if err := json.Unmarshal([]byte(v), &tpl); err != nil {
		t.Fatalf("pre template parse: %v", err)
	}
	stats := defaultTrafficStatsSwitches()
	if policy, ok := tpl["policy"].(map[string]any); ok {
		if sys, ok := policy["system"].(map[string]any); ok {
			if b, ok := sys["statsInboundUplink"].(bool); ok {
				stats.InboundUplink = b
			}
			if b, ok := sys["statsInboundDownlink"].(bool); ok {
				stats.InboundDownlink = b
			}
			if b, ok := sys["statsOutboundUplink"].(bool); ok {
				stats.OutboundUplink = b
			}
			if b, ok := sys["statsOutboundDownlink"].(bool); ok {
				stats.OutboundDownlink = b
			}
		}
	}
	return &stats
}

// dumpAllTables returns every table's rows as stringified key/value
// maps (map-based, so it works against pre-migration schemas lacking
// the new columns).
func dumpAllTables(t *testing.T, path string) map[string][]map[string]string {
	t.Helper()
	gdb, err := gorm.Open(sqlite.Open(path), &gorm.Config{})
	if err != nil {
		t.Fatalf("dump open: %v", err)
	}
	defer func() {
		if sqlDB, err := gdb.DB(); err == nil {
			_ = sqlDB.Close()
		}
	}()
	var tables []string
	if err := gdb.Raw("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name").Scan(&tables).Error; err != nil {
		t.Fatalf("list tables: %v", err)
	}
	out := map[string][]map[string]string{}
	for _, tbl := range tables {
		var rows []map[string]any
		if err := gdb.Table(tbl).Find(&rows).Error; err != nil {
			t.Fatalf("dump %s: %v", tbl, err)
		}
		rendered := make([]map[string]string, 0, len(rows))
		for _, r := range rows {
			m := map[string]string{}
			for k, v := range r {
				m[k] = fmt.Sprintf("%v", v)
			}
			rendered = append(rendered, m)
		}
		out[tbl] = rendered
	}
	return out
}

// rowsByID indexes rows by their id column.
func rowsByID(rows []map[string]string) map[string]map[string]string {
	out := map[string]map[string]string{}
	for _, r := range rows {
		out[r["id"]] = r
	}
	return out
}

// renderRow renders the given columns of a row canonically.
func renderRow(r map[string]string, cols []string) string {
	var sb strings.Builder
	for _, c := range cols {
		fmt.Fprintf(&sb, "%s=%s;", c, r[c])
	}
	return sb.String()
}

// assertTableUnchanged requires the pre-migration rows to survive as
// a multiset with identical values on every column that existed
// pre-migration (post rows may carry additional, newly-added
// columns). Works for composite-key tables too.
func assertTableUnchanged(t *testing.T, pre, post map[string][]map[string]string, table string) {
	t.Helper()
	preRows, ok := pre[table]
	if !ok {
		return // table did not exist pre-migration (e.g. agent_nodes)
	}
	if len(post[table]) != len(preRows) {
		t.Fatalf("table %s row count changed: %d -> %d", table, len(preRows), len(post[table]))
	}
	colSet := map[string]bool{}
	for _, r := range preRows {
		for k := range r {
			colSet[k] = true
		}
	}
	cols := make([]string, 0, len(colSet))
	for k := range colSet {
		cols = append(cols, k)
	}
	sort.Strings(cols)
	project := func(rows []map[string]string) []string {
		out := make([]string, 0, len(rows))
		for _, r := range rows {
			out = append(out, renderRow(r, cols))
		}
		sort.Strings(out)
		return out
	}
	preP, postP := project(preRows), project(post[table])
	for i := range preP {
		if preP[i] != postP[i] {
			t.Fatalf("table %s row changed:\npre:  %s\npost: %s", table, preP[i], postP[i])
		}
	}
}

// assertInboundsUnchanged is assertTableUnchanged for inbounds, with
// the three legacy JSON text columns compared semantically (other
// InitDB migrations may reformat them) and the new singbox_* columns
// excluded.
func assertInboundsUnchanged(t *testing.T, pre, post map[string][]map[string]string) {
	t.Helper()
	preRows := pre["inbounds"]
	postByID := rowsByID(post["inbounds"])
	if len(postByID) != len(preRows) {
		t.Fatalf("inbounds row count changed: %d -> %d", len(preRows), len(postByID))
	}
	jsonCols := map[string]bool{"settings": true, "stream_settings": true, "sniffing": true}
	for _, pr := range preRows {
		qr, ok := postByID[pr["id"]]
		if !ok {
			t.Fatalf("inbound id=%s vanished", pr["id"])
		}
		for k, v := range pr {
			if k == "singbox_config" || k == "singbox_preserved" {
				continue
			}
			if jsonCols[k] {
				if !jsonSemanticallyEqual(v, qr[k]) {
					t.Fatalf("inbound id=%s column %s changed semantically:\npre:  %s\npost: %s", pr["id"], k, v, qr[k])
				}
				continue
			}
			if qr[k] != v {
				t.Fatalf("inbound id=%s column %s changed: %q -> %q", pr["id"], k, v, qr[k])
			}
		}
	}
}

func jsonSemanticallyEqual(a, b string) bool {
	if a == b {
		return true
	}
	var av, bv any
	if err := json.Unmarshal([]byte(a), &av); err != nil {
		return false
	}
	if err := json.Unmarshal([]byte(b), &bv); err != nil {
		return false
	}
	ab, _ := json.Marshal(av)
	bb, _ := json.Marshal(bv)
	return string(ab) == string(bb)
}

func settingsMap(t *testing.T, path string) map[string]string {
	t.Helper()
	gdb := openRaw(t, path)
	var rows []struct {
		Key   string
		Value string
	}
	if err := gdb.Table("settings").Select("key, value").Scan(&rows).Error; err != nil {
		t.Fatalf("settings scan: %v", err)
	}
	out := map[string]string{}
	for _, r := range rows {
		out[r.Key] = r.Value
	}
	return out
}

func mirrorUUIDsByInbound(t *testing.T, path string) map[int]map[string]bool {
	t.Helper()
	gdb := openRaw(t, path)
	var rows []struct {
		Id       int
		Settings string
	}
	if err := gdb.Table("inbounds").Select("id, settings").Scan(&rows).Error; err != nil {
		t.Fatalf("mirror scan: %v", err)
	}
	out := map[int]map[string]bool{}
	for _, r := range rows {
		set := map[string]bool{}
		var wrap struct {
			Clients []struct {
				ID string `json:"id"`
			} `json:"clients"`
		}
		if err := json.Unmarshal([]byte(r.Settings), &wrap); err == nil {
			for _, c := range wrap.Clients {
				if c.ID != "" {
					set[c.ID] = true
				}
			}
		}
		out[r.Id] = set
	}
	return out
}

func assertSettingsDelta(t *testing.T, preM, postM map[string]string) {
	t.Helper()
	for k, v := range preM {
		if k == xrayTemplateSettingKey {
			continue
		}
		if postM[k] != v {
			t.Fatalf("setting %q changed: %q -> %q", k, v, postM[k])
		}
	}
	for k := range postM {
		if _, ok := preM[k]; !ok && k != singboxMigrationVersionKey && k != trafficStatsSwitchesKey {
			t.Fatalf("unexpected new setting %q", k)
		}
	}
	if _, ok := postM[xrayTemplateSettingKey]; ok {
		t.Fatalf("template row should be gone post-migration")
	}
}

func copyFile(t *testing.T, src, dst string) {
	t.Helper()
	data, err := os.ReadFile(src)
	if err != nil {
		t.Fatalf("read %s: %v", src, err)
	}
	if err := os.WriteFile(dst, data, 0o644); err != nil {
		t.Fatalf("write %s: %v", dst, err)
	}
}