NexusGate/panel-go/difftest/lib.mjs

// Shared harness for the NexusGate Node-vs-Go panel differential E2E tests.
//
// Only Node >= 18 standard library is used (fetch / fs / child_process / net).
// This module provides:
//   - panel process management (start/stop, health wait, random free ports)
//   - a small HTTP client that tracks the admin session cookie + CSRF token
//     and can also speak as an agent (Bearer agentKey) or a subscriber
//   - a "learning" normalizer: volatile values (prefixed UUID ids, bare
//     UUIDs, ISO timestamps, tokens, hashes, Reality keys, per-node secrets,
//     randomly allocated ports) are replaced by stable placeholders such as
//     <cus>#2. Each side of a differential run owns one Normalizer instance;
//     both instances are fed the same step sequence and walk objects with
//     sorted keys, so the n-th distinct value of a category maps to the same
//     placeholder on both sides even though the raw values differ.
//   - job drain helpers that simulate a real agent (poll -> complete), where
//     Reality apply jobs report artifacts.realityPublicKey taken from the
//     polled payload.keyMaterial, exactly like the real agent materializing
//     the panel-hosted keypair.

import { spawn } from 'node:child_process';
import fs from 'node:fs';
import net from 'node:net';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

export const DIFFTEST_DIR = path.dirname(fileURLToPath(import.meta.url));
export const REPO_ROOT = path.resolve(DIFFTEST_DIR, '..', '..');
export const NODE_SERVER_JS = path.join(REPO_ROOT, 'server.js');
export const DEFAULT_GO_BIN = path.resolve(DIFFTEST_DIR, '..', 'nexusgate-panel-linux-amd64');

export const ADMIN_USERNAME = 'admin';
export const ADMIN_PASSWORD = 'diff-admin-pass';

// User-Agents used for subscription pulls. `auto` resolves to clash for
// clash-like UAs and to base64 otherwise (see formatSubscription).
export const UA_DEFAULT = 'NexusGate-DiffTest/1.0';
export const UA_CLASH = 'clash-verge/v2.0.0';
export const UA_GENERIC = 'Mozilla/5.0 (DiffTest)';

// All subscription shapes exercised by seed + phase 2, in a fixed order.
// `key` is the stable name used in seed-expect.json and step names.
export const SUB_CASES = [
  { key: 'raw', format: 'raw', ua: UA_DEFAULT },
  { key: 'base64', format: 'base64', ua: UA_DEFAULT },
  { key: 'clash', format: 'clash', ua: UA_DEFAULT },
  { key: 'clash-smart', format: 'clash-smart', ua: UA_DEFAULT },
  { key: 'singbox', format: 'singbox', ua: UA_DEFAULT },
  { key: 'surge', format: 'surge', ua: UA_DEFAULT },
  { key: 'auto-clash', format: 'auto', ua: UA_CLASH },
  { key: 'auto-generic', format: 'auto', ua: UA_GENERIC },
];

// ---------------------------------------------------------------------------
// Process management
// ---------------------------------------------------------------------------

export function freePort() {
  return new Promise((resolve, reject) => {
    const server = net.createServer();
    server.once('error', reject);
    server.listen(0, '127.0.0.1', () => {
      const { port } = server.address();
      server.close(() => resolve(port));
    });
  });
}

const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

/**
 * Start one panel process.
 * kind 'node' runs the Node panel (server.js); kind 'go' execs the Go binary.
 * Both are configured purely through the NG_* environment contract:
 * NG_HOST / NG_PORT / NG_DATA_FILE / NG_ADMIN_PASSWORD / NG_COOKIE_SECURE.
 */
export function startPanel({ kind, bin, dataFile, port }) {
  const env = {
    ...process.env,
    NG_HOST: '127.0.0.1',
    NG_PORT: String(port),
    NG_DATA_FILE: dataFile,
    NG_ADMIN_PASSWORD: ADMIN_PASSWORD,
    NG_COOKIE_SECURE: 'false',
  };
  let child;
  if (kind === 'node') {
    child = spawn(process.execPath, [NODE_SERVER_JS], { cwd: REPO_ROOT, env, stdio: ['ignore', 'pipe', 'pipe'] });
  } else {
    child = spawn(bin, [], { env, stdio: ['ignore', 'pipe', 'pipe'] });
  }
  const proc = {
    kind, child, port, dataFile,
    base: `http://127.0.0.1:${port}`,
    output: '',
    exited: null,
  };
  const capture = (chunk) => { proc.output = (proc.output + chunk.toString()).slice(-8000); };
  child.stdout.on('data', capture);
  child.stderr.on('data', capture);
  proc.exited = new Promise((resolve) => child.once('exit', (code, signal) => resolve({ code, signal })));
  return proc;
}

export async function waitHealth(proc, timeoutMs = 20000) {
  const deadline = Date.now() + timeoutMs;
  let exited = false;
  proc.exited.then(() => { exited = true; });
  while (Date.now() < deadline) {
    if (exited) throw new Error(`panel (${proc.kind}) exited before becoming healthy:\n${proc.output}`);
    try {
      const res = await fetch(`${proc.base}/healthz`);
      if (res.ok) { await res.text(); return; }
    } catch { /* not up yet */ }
    await sleep(100);
  }
  throw new Error(`panel (${proc.kind}) did not become healthy within ${timeoutMs}ms:\n${proc.output}`);
}

export async function stopPanel(proc) {
  if (!proc || !proc.child) return;
  if (proc.child.exitCode !== null || proc.child.signalCode !== null) return;
  proc.child.kill('SIGTERM');
  const done = await Promise.race([proc.exited.then(() => true), sleep(3000).then(() => false)]);
  if (!done) {
    proc.child.kill('SIGKILL');
    await Promise.race([proc.exited, sleep(2000)]);
  }
}

export function makeTempDir(prefix = 'nexusgate-difftest-') {
  return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
}

// ---------------------------------------------------------------------------
// HTTP client (admin session / agent bearer / anonymous)
// ---------------------------------------------------------------------------

export class PanelClient {
  constructor(base) {
    this.base = base;
    this.cookie = null;
    this.csrf = null;
  }

  async request(method, pathname, { body, headers = {}, cookie, csrf } = {}) {
    const finalHeaders = { ...headers };
    const useCookie = cookie !== undefined ? cookie : this.cookie;
    if (useCookie) finalHeaders.cookie = useCookie;
    if (csrf) finalHeaders['x-csrf-token'] = this.csrf;
    let payload;
    if (body !== undefined) {
      finalHeaders['content-type'] = 'application/json';
      payload = JSON.stringify(body);
    }
    const res = await fetch(`${this.base}${pathname}`, { method, headers: finalHeaders, body: payload, redirect: 'manual' });
    const text = await res.text();
    let json = null;
    try { json = JSON.parse(text); } catch { /* HTML / plain text bodies */ }
    return { status: res.status, headers: res.headers, text, json };
  }

  // Admin call. Non-GET requests carry the session CSRF token unless
  // csrf:false is passed (used by the CSRF error-case step).
  async admin(method, pathname, body, { csrf = true } = {}) {
    return this.request(method, pathname, { body, csrf: csrf && method !== 'GET' });
  }

  async login(username = ADMIN_USERNAME, password = ADMIN_PASSWORD) {
    const res = await this.request('POST', '/api/auth/login', { body: { username, password }, cookie: null });
    if (res.status === 200 && res.json) {
      const setCookie = res.headers.get('set-cookie') || '';
      this.cookie = setCookie.split(';')[0];
      this.csrf = res.json.csrf;
    }
    return res;
  }

  async agentPost(pathname, agentKey, body = {}) {
    return this.request('POST', pathname, {
      body, cookie: null, headers: { authorization: `Bearer ${agentKey}` },
    });
  }

  async getSubscription(token, format, ua) {
    const res = await fetch(`${this.base}/s/${token}/${format}`, {
      headers: { 'user-agent': ua }, redirect: 'manual',
    });
    const text = await res.text();
    return {
      status: res.status,
      contentType: res.headers.get('content-type') || '',
      userinfo: res.headers.get('subscription-userinfo') || '',
      retryAfter: res.headers.get('retry-after') || '',
      body: text,
    };
  }
}

// ---------------------------------------------------------------------------
// Agent simulation helpers
// ---------------------------------------------------------------------------

// Artifacts a real agent would report for a completed job. For Reality apply
// jobs (entry leg only) the agent materializes the panel-hosted keypair and
// reports its public key; everything else reports no artifacts.
export function artifactsForJob(job) {
  if (!job || job.action !== 'apply_resource') return {};
  const payload = job.payload || {};
  const kind = payload.kind || (payload.resource && payload.resource.meta && payload.resource.meta.kind);
  if (kind === 'node-exit') return {};
  const keyMaterial = payload.keyMaterial;
  if (!keyMaterial || typeof keyMaterial !== 'object') return {};
  const pair = keyMaterial[`reality-${job.nodeId}`] || Object.values(keyMaterial)[0];
  return pair && pair.publicKey ? { realityPublicKey: pair.publicKey } : {};
}

export async function pollOnce(client, agentKey) {
  const res = await client.agentPost('/api/agent/poll', agentKey, {});
  if (res.status !== 200) throw new Error(`poll failed: ${res.status} ${res.text}`);
  return res.json.job;
}

export async function completeJob(client, agentKey, job, { success = true } = {}) {
  const res = await client.agentPost(`/api/agent/jobs/${job.id}/complete`, agentKey, {
    success, result: { artifacts: artifactsForJob(job) },
  });
  return res;
}

// Poll + complete until the agent's queue is empty. Returns every
// { job, complete } pair in the order the jobs were picked up.
export async function drainAgentJobs(client, agentKey, { maxRounds = 40 } = {}) {
  const done = [];
  for (let round = 0; round < maxRounds; round += 1) {
    const job = await pollOnce(client, agentKey);
    if (!job) return done;
    const complete = await completeJob(client, agentKey, job);
    if (complete.status !== 200) {
      throw new Error(`complete for job ${job.id} failed: ${complete.status} ${complete.text}`);
    }
    done.push({ job, complete: complete.json });
  }
  throw new Error(`agent queue did not drain within ${maxRounds} rounds`);
}

// ---------------------------------------------------------------------------
// Subscription helpers
// ---------------------------------------------------------------------------

// A base64 subscription body decodes to the raw URI list; comparing the
// decoded form is far more robust than comparing the opaque blob.
export function comparisonBody(caseKey, body) {
  if (caseKey !== 'base64' && caseKey !== 'auto-generic') return body;
  const compact = String(body).trim();
  if (!/^[A-Za-z0-9+/=]+$/.test(compact) || compact.length % 4 !== 0) return body;
  try {
    const decoded = Buffer.from(compact, 'base64').toString('utf8');
    return decoded.includes('://') ? decoded : body;
  } catch {
    return body;
  }
}

export async function fetchAllSubscriptions(client, token) {
  const out = {};
  for (const c of SUB_CASES) {
    const res = await client.getSubscription(token, c.format, c.ua);
    out[c.key] = {
      status: res.status,
      contentType: res.contentType,
      userinfo: res.userinfo,
      body: comparisonBody(c.key, res.body),
    };
  }
  return out;
}

// ---------------------------------------------------------------------------
// Learning normalizer
// ---------------------------------------------------------------------------

const ID_PREFIXES = ['usr', 'srv', 'agt', 'enr', 'cus', 'nod', 'ncl', 'res', 'job', 'evt', 'obs', 'acc', 'src', 'lnk'];
const PREFIXED_ID_RE = new RegExp(`\\b(${ID_PREFIXES.join('|')})_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})`, 'g');
const BARE_UUID_RE = /(?<![0-9A-Za-z_-])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9A-Za-z_-])/g;
const ISO_TS_RE = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})/g;
const LOCALE_DATE_RE = /\d{4}\/\d{1,2}\/\d{1,2}\s+\d{1,2}:\d{2}:\d{2}/g;
const SCRYPT_RE = /scrypt:[0-9a-f]+:[0-9a-f]+/g;
const SS_URI_RE = /ss:\/\/[A-Za-z0-9_-]+@/g;

// JSON field name -> placeholder category for high-entropy values that only
// ever appear under a recognizable field name.
const SECRET_FIELDS = {
  subscriptionToken: 'subtoken',
  agentKey: 'agentkey',
  token: 'enrolltoken',
  csrf: 'csrf',
  passwordHash: 'hash',
  tokenHash: 'hash',
  keyHash: 'hash',
  keyFingerprint: 'fingerprint',
  clientKey: 'clientkey',
  privateKey: 'rpriv',
  publicKey: 'rpub',
  realityPublicKey: 'rpub',
  relayPassword: 'secret',
  ssServerKey: 'secret',
  wsPath: 'secret',
  password: 'secret',
  shortId: 'secret',
  short_id: 'secret',
};

// Numeric fields whose values are randomly allocated ports. Only values
// under these field names are learned / replaced, so deterministic numbers
// (portRangeStart, usedBytes, traffic limits, ...) are never rewritten.
const PORT_FIELDS = new Set(['port', 'exitPort', 'listen_port', 'server_port']);

export class Normalizer {
  constructor({ panelOrigin = '' } = {}) {
    this.panelOrigin = panelOrigin;
    this.counters = new Map(); // category -> last n
    this.secrets = new Map(); // raw string -> placeholder (field-learned)
    this.ids = new Map(); // `${prefix}` -> Map(raw -> placeholder)
    this.fragments = new Map(); // derived id fragment -> owning id placeholder
    this.uuids = new Map(); // raw -> placeholder
    this.ports = new Map(); // number -> placeholder
    this.hashes = new Map(); // raw scrypt string -> placeholder
    this.blobs = new Map(); // ss:// userinfo blob -> placeholder
  }

  #assign(map, raw, label) {
    let placeholder = map.get(raw);
    if (!placeholder) {
      const n = (this.counters.get(label) || 0) + 1;
      this.counters.set(label, n);
      placeholder = `<${label}>#${n}`;
      map.set(raw, placeholder);
    }
    return placeholder;
  }

  #idPlaceholder(prefix, raw) {
    if (!this.ids.has(prefix)) this.ids.set(prefix, new Map());
    const placeholder = this.#assign(this.ids.get(prefix), raw, prefix);
    // Derived forms: the panel builds sing-box tags from the node id with
    // non-alphanumerics stripped, truncated to the last 18 chars
    // (tagForNode -> "ng-<fragment>-in"). Register the fragment so those
    // tags normalize through their owning id's placeholder.
    if (prefix === 'nod') {
      const fragment = raw.replace(/[^0-9a-z]/gi, '').slice(-18);
      if (fragment && !this.fragments.has(fragment)) this.fragments.set(fragment, placeholder);
    }
    return placeholder;
  }

  // Feed one raw step result into the learning tables. Object keys are
  // walked in sorted order so the learning order does not depend on either
  // implementation's serialization order; arrays keep their semantic order.
  learn(value, keyName = null) {
    if (Array.isArray(value)) {
      for (const item of value) this.learn(item, keyName);
      return;
    }
    if (value && typeof value === 'object') {
      for (const key of Object.keys(value).sort()) this.learn(value[key], key);
      return;
    }
    if (typeof value === 'number') {
      if (keyName && PORT_FIELDS.has(keyName) && Number.isInteger(value) && value >= 1 && value <= 65535) {
        this.#assign(this.ports, value, 'port');
      }
      return;
    }
    if (typeof value !== 'string') return;
    const category = SECRET_FIELDS[keyName];
    if (category && value.length >= 4) this.#assign(this.secrets, value, category);
    // ws paths appear as transport `path` in job payloads.
    if (keyName === 'path' && /^\/[0-9a-f]{12,}$/.test(value)) this.#assign(this.secrets, value, 'secret');
    for (const match of value.matchAll(PREFIXED_ID_RE)) this.#idPlaceholder(match[1], match[0]);
    for (const match of value.matchAll(SCRYPT_RE)) this.#assign(this.hashes, match[0], 'hash');
    // Bare UUIDs are learned only when the whole value is one (credentials);
    // embedded ones are assigned on the fly during normalizeString, in the
    // same textual order on both sides.
    if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value)) {
      this.#assign(this.uuids, value, 'uuid');
    }
  }

  normalizeString(text) {
    let out = String(text);
    if (this.panelOrigin) out = out.split(this.panelOrigin).join('<panel>');
    // Learned secrets first (longest raw first so a full token wins over its
    // own prefixes), including the 16-char prefix shown on the info page.
    // A base64 secret's prefix truncations (the SS2022 user keys derived
    // from a customer's 32-byte root key) are the same logical secret, so
    // their encodings are replaced with the same placeholder.
    const replacements = [];
    for (const [raw, placeholder] of this.secrets) {
      if (raw.length >= 8) replacements.push([raw, placeholder]);
      if (raw.length > 16) replacements.push([raw.slice(0, 16), placeholder]);
      if (/^[A-Za-z0-9+/]+={0,2}$/.test(raw) && raw.length % 4 === 0) {
        const bytes = Buffer.from(raw, 'base64');
        if (bytes.toString('base64') === raw) {
          for (const n of [16, 24]) {
            if (n < bytes.length) replacements.push([bytes.subarray(0, n).toString('base64'), placeholder]);
          }
        }
      }
    }
    // The Surge formatter mangles '=' to '_' inside passwords, so the
    // mangled variant of every replacement is the same secret too.
    for (const [raw, placeholder] of [...replacements]) {
      const mangled = raw.replace(/=/g, '_');
      if (mangled !== raw) replacements.push([mangled, placeholder]);
    }
    replacements.sort((a, b) => b[0].length - a[0].length);
    for (const [raw, placeholder] of replacements) out = out.split(raw).join(placeholder);
    for (const [raw, placeholder] of this.hashes) out = out.split(raw).join(placeholder);
    out = out.replace(PREFIXED_ID_RE, (whole, prefix) => this.#idPlaceholder(prefix, whole));
    for (const [fragment, placeholder] of this.fragments) out = out.split(fragment).join(placeholder);
    out = out.replace(BARE_UUID_RE, (whole) => this.#assign(this.uuids, whole, 'uuid'));
    out = out.replace(SCRYPT_RE, (whole) => this.#assign(this.hashes, whole, 'hash'));
    out = out.replace(ISO_TS_RE, '<ts>');
    out = out.replace(LOCALE_DATE_RE, '<date>');
    out = out.replace(SS_URI_RE, (whole) => `ss://${this.#assign(this.blobs, whole, 'ssblob')}@`);
    // Learned ports as standalone numbers, longest digit string first.
    const ports = [...this.ports.entries()].sort((a, b) => String(b[0]).length - String(a[0]).length);
    for (const [num, placeholder] of ports) {
      out = out.replace(new RegExp(`(?<![0-9])${num}(?![0-9])`, 'g'), placeholder);
    }
    return out;
  }

  normalize(value, keyName = null) {
    if (Array.isArray(value)) return value.map((item) => this.normalize(item, keyName));
    if (value && typeof value === 'object') {
      // Sorted-key traversal (like learn) so placeholder assignment order is
      // canonical and independent of either side's serialization order.
      // firstDiff compares objects key-insensitively, so reordering is safe.
      const out = {};
      for (const key of Object.keys(value).sort()) out[this.normalizeString(key)] = this.normalize(value[key], key);
      return out;
    }
    if (typeof value === 'number') {
      // Speeds are derived from adjacent usage reports over wall-clock
      // time (dt differs per side by construction), so only their shape
      // is comparable: positive vs zero. See UI-RESKIN-REPORT.md.
      if (keyName === 'speedUpBps' || keyName === 'speedDownBps') return value > 0 ? '<speed+>' : 0;
      if (keyName && PORT_FIELDS.has(keyName) && this.ports.has(value)) return this.ports.get(value);
      return value;
    }
    if (typeof value === 'string') return this.normalizeString(value);
    return value;
  }

  // Full pipeline for one step result: learn, then normalize.
  process(value) {
    this.learn(value);
    return this.normalize(value);
  }

  normalizeText(text) {
    return this.normalizeString(text);
  }
}

// ---------------------------------------------------------------------------
// Deep diff + reporter
// ---------------------------------------------------------------------------

function summarize(value) {
  let text;
  try { text = JSON.stringify(value); } catch { text = String(value); }
  if (text === undefined) text = String(value);
  return text.length > 200 ? `${text.slice(0, 200)}…` : text;
}

// Returns null when equal, otherwise the first difference found.
export function firstDiff(a, b, at = '$') {
  if (Object.is(a, b)) return null;
  if (typeof a !== typeof b || a === null || b === null || typeof a !== 'object') {
    return { path: at, a: summarize(a), b: summarize(b) };
  }
  if (Array.isArray(a) !== Array.isArray(b)) return { path: at, a: summarize(a), b: summarize(b) };
  if (Array.isArray(a)) {
    if (a.length !== b.length) return { path: `${at}.length`, a: a.length, b: b.length };
    for (let i = 0; i < a.length; i += 1) {
      const diff = firstDiff(a[i], b[i], `${at}[${i}]`);
      if (diff) return diff;
    }
    return null;
  }
  const keys = [...new Set([...Object.keys(a), ...Object.keys(b)])].sort();
  for (const key of keys) {
    if (!(key in a)) return { path: `${at}.${key}`, a: '<absent>', b: summarize(b[key]) };
    if (!(key in b)) return { path: `${at}.${key}`, a: summarize(a[key]), b: '<absent>' };
    const diff = firstDiff(a[key], b[key], `${at}.${key}`);
    if (diff) return diff;
  }
  return null;
}

export class Reporter {
  constructor() {
    this.total = 0;
    this.passed = 0;
    this.failed = 0;
    this.skipped = 0;
    this.failures = [];
  }

  pass(name) {
    this.total += 1; this.passed += 1;
    console.log(`PASS ${name}`);
  }

  fail(name, detail) {
    this.total += 1; this.failed += 1;
    this.failures.push({ name, detail });
    console.log(`FAIL ${name} — ${detail}`);
  }

  skip(name, reason) {
    this.total += 1; this.skipped += 1;
    console.log(`SKIP ${name} — ${reason}`);
  }

  // Compare two already-normalized values as one step.
  compare(name, a, b) {
    const diff = firstDiff(a, b);
    if (!diff) { this.pass(name); return true; }
    this.fail(name, `${diff.path}: A=${diff.a} B=${diff.b}`);
    return false;
  }

  summary(label) {
    console.log(`SUMMARY ${label} total=${this.total} pass=${this.passed} fail=${this.failed} skip=${this.skipped}`);
  }
}