NexusGate/test/heartbeat-reconcile.test.js

'use strict';

const test = require('node:test');
const assert = require('node:assert/strict');
const crypto = require('node:crypto');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const net = require('node:net');
const { spawn } = require('node:child_process');
const { Store } = require('../lib/store');
const { Orchestrator } = require('../lib/orchestrator');

const B64URL_43 = /^[A-Za-z0-9_-]{43}$/;

// One server, one customer, one active direct Reality node whose resource the
// panel believes is deployed on the agent.
async function fixture(t) {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nexusgate-hb-'));
  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
  const store = await new Store(path.join(dir, 'data.json')).init();
  const stamp = new Date().toISOString();
  await store.transaction((data) => {
    data.servers.push({ id: 'srv_hb', name: '入口', publicAddress: '203.0.113.10', portRangeStart: 20000, portRangeEnd: 50000, status: 'online', lastSeenAt: stamp });
    data.customers.push({ id: 'cus_hb', name: '客户', status: 'active' });
    data.nodes.push({
      id: 'nod_hb', serverId: 'srv_hb', name: 'Reality 节点', protocol: 'vless-reality-vision',
      topology: 'direct', networkMode: 'ipv4', port: 23456, resourceId: 'res_nod_hb_1',
      status: 'active', entryReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp,
    });
    data.nodeClients.push({
      id: 'ncl_hb', nodeId: 'nod_hb', customerId: 'cus_hb', status: 'active',
      credentials: { clientId: crypto.randomUUID(), relayPassword: 'secret', shortId: 'abcd1234' },
      createdAt: stamp, updatedAt: stamp,
    });
  });
  return { store, orchestrator: new Orchestrator(store) };
}

function applyJobs(store, nodeId) {
  return store.data.jobs.filter((job) => job.action === 'apply_resource' && (!nodeId || job.nodeId === nodeId));
}

test('heartbeat reconcile requeues an apply with keyMaterial for a missing resource', async (t) => {
  const { store, orchestrator } = await fixture(t);
  const count = await orchestrator.reconcileFromHeartbeat('srv_hb', []);
  assert.equal(count, 1);
  const jobs = applyJobs(store, 'nod_hb');
  assert.equal(jobs.length, 1);
  assert.equal(jobs[0].status, 'queued');
  assert.equal(jobs[0].serverId, 'srv_hb');
  assert.equal(jobs[0].payload.kind, 'node');
  const material = jobs[0].payload.keyMaterial;
  assert.deepEqual(Object.keys(material), ['reality-nod_hb']);
  assert.match(material['reality-nod_hb'].privateKey, B64URL_43);
  assert.equal(store.data.keyStore['reality-nod_hb'].privateKey, material['reality-nod_hb'].privateKey);
  assert.equal(store.data.nodes[0].status, 'deploying');
  assert.ok(store.data.activity.some((event) => event.action === 'reconcile_server' && event.actor === 'system:heartbeat'));
});

test('heartbeat reconcile is a no-op when the agent holds every expected resource', async (t) => {
  const { store, orchestrator } = await fixture(t);
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', ['res_nod_hb_1']), 0);
  assert.equal(store.data.jobs.length, 0);
});

test('heartbeat reconcile does not trigger or disturb jobs while a deploy is in flight', async (t) => {
  const { store, orchestrator } = await fixture(t);
  const stamp = new Date().toISOString();
  await store.transaction((data) => {
    data.jobs.push({ id: 'job_busy', serverId: 'srv_hb', nodeId: 'nod_hb', action: 'apply_resource',
      payload: { kind: 'node' }, status: 'queued', attempts: 0, createdAt: stamp, updatedAt: stamp });
  });
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 0);
  assert.equal(store.data.jobs.length, 1);
  // The in-flight job must NOT be failed the way enrollment reconcile fails it.
  assert.equal(store.data.jobs[0].status, 'queued');
  assert.equal(store.data.activity.some((event) => event.action === 'reconcile_server'), false);
});

test('heartbeat reconcile is throttled per server', async (t) => {
  const { store, orchestrator } = await fixture(t);
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 1);
  assert.equal(applyJobs(store, 'nod_hb').length, 1);
  // Clear the in-flight gate so only the throttle can explain a no-op.
  await store.transaction((data) => { data.jobs[0].status = 'completed'; });
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 0);
  assert.equal(applyJobs(store, 'nod_hb').length, 1);
});

test('heartbeat reconcile ignores legacy reports and out-of-scope nodes', async (t) => {
  const { store, orchestrator } = await fixture(t);
  // Old agents: no resources field at all.
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', undefined), 0);
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', 'res_nod_hb_1'), 0);
  const stamp = new Date().toISOString();
  await store.transaction((data) => {
    // Draft node (never deployed), clientless node, and a removing node:
    // none of their resources are "expected" on the agent.
    data.nodes.push(
      { id: 'nod_draft', serverId: 'srv_hb', name: '草稿', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23457, resourceId: 'res_draft', status: 'draft', realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp },
      { id: 'nod_idle', serverId: 'srv_hb', name: '无客户', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23458, resourceId: 'res_idle', status: 'active', entryReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp },
      { id: 'nod_rm', serverId: 'srv_hb', name: '删除中', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23459, resourceId: 'res_rm', status: 'removing', entryReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp },
    );
    data.nodeClients.push(
      { id: 'ncl_draft', nodeId: 'nod_draft', customerId: 'cus_hb', status: 'active', credentials: { clientId: crypto.randomUUID(), relayPassword: 'x', shortId: 'aa' }, createdAt: stamp, updatedAt: stamp },
      { id: 'ncl_rm', nodeId: 'nod_rm', customerId: 'cus_hb', status: 'active', credentials: { clientId: crypto.randomUUID(), relayPassword: 'x', shortId: 'bb' }, createdAt: stamp, updatedAt: stamp },
    );
  });
  // Only the in-scope node (nod_hb) is requeued; the heartbeat holds nothing.
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 1);
  assert.deepEqual(applyJobs(store).map((job) => job.nodeId), ['nod_hb']);
});

test('heartbeat reconcile requeues a forward node whose exit resource is missing', async (t) => {
  const { store, orchestrator } = await fixture(t);
  const stamp = new Date().toISOString();
  await store.transaction((data) => {
    data.servers.push({ id: 'srv_exit', name: '出口', publicAddress: '203.0.113.20', portRangeStart: 20000, portRangeEnd: 50000, status: 'online', lastSeenAt: stamp });
    data.nodes.push({
      id: 'nod_fwd', serverId: 'srv_hb', name: '中转节点', protocol: 'vless-reality-vision', topology: 'forward',
      exitServerId: 'srv_exit', exitProtocol: 'shadowsocks-2022-aes128', exitPort: 23460, exitResourceId: 'res_fwd_exit',
      exitCredentials: { password: 'exit-secret' },
      networkMode: 'ipv4', port: 23461, resourceId: 'res_fwd_in',
      status: 'active', entryReady: true, exitReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp,
    });
    data.nodeClients.push({
      id: 'ncl_fwd', nodeId: 'nod_fwd', customerId: 'cus_hb', status: 'active',
      credentials: { clientId: crypto.randomUUID(), relayPassword: 'secret', shortId: 'ccdd1234' },
      createdAt: stamp, updatedAt: stamp,
    });
  });
  // The exit server's agent holds nothing: the exit resource is re-dispatched.
  assert.equal(await orchestrator.reconcileFromHeartbeat('srv_exit', []), 1);
  const exitJobs = applyJobs(store, 'nod_fwd').filter((job) => job.serverId === 'srv_exit');
  assert.equal(exitJobs.length, 1);
  assert.equal(exitJobs[0].payload.kind, 'node-exit');
  // The entry server's agent holds its resource: nothing is queued for it.
  assert.equal(applyJobs(store, 'nod_fwd').filter((job) => job.serverId === 'srv_hb').length, 0);
});

async function freePort() {
  return new Promise((resolve, reject) => {
    const server = net.createServer();
    server.once('error', reject);
    server.listen(0, '127.0.0.1', () => { const port = server.address().port; server.close(() => resolve(port)); });
  });
}

async function waitFor(url) {
  for (let i = 0; i < 60; i += 1) {
    try { const response = await fetch(url); if (response.ok) return; } catch { /* retry */ }
    await new Promise((resolve) => setTimeout(resolve, 100));
  }
  throw new Error('test server did not start');
}

test('E2E: wiped agent state is healed by a heartbeat reporting no resources', async (t) => {
  const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'nexusgate-hb-e2e-'));
  const port = await freePort();
  const child = spawn(process.execPath, ['server.js'], {
    cwd: path.resolve(__dirname, '..'),
    env: { ...process.env, NG_HOST: '127.0.0.1', NG_PORT: String(port), NG_DATA_FILE: path.join(dir, 'data.json'), NG_ADMIN_PASSWORD: 'test-password', NG_COOKIE_SECURE: 'false' },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  t.after(async () => { child.kill('SIGTERM'); await fs.promises.rm(dir, { recursive: true, force: true }); });
  const base = `http://127.0.0.1:${port}`;
  await waitFor(`${base}/healthz`);

  const login = await fetch(`${base}/api/auth/login`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ username: 'admin', password: 'test-password' }) });
  assert.equal(login.status, 200);
  const session = await login.json();
  const cookie = login.headers.get('set-cookie').split(';')[0];
  const request = async (pathname, method = 'GET', body) => {
    const response = await fetch(`${base}${pathname}`, { method, headers: { cookie, 'x-csrf-token': session.csrf, ...(body ? { 'content-type': 'application/json' } : {}) }, body: body ? JSON.stringify(body) : undefined });
    const json = await response.json();
    assert.ok(response.ok, JSON.stringify(json));
    return json;
  };

  const server = (await request('/api/servers', 'POST', { name: '入口 01', role: 'relay', publicAddress: 'entry.example.com' })).server;
  const customer = (await request('/api/customers', 'POST', { name: '客户 A' })).customer;
  const ticket = await request(`/api/servers/${server.id}/enrollment-token`, 'POST', {});
  const enrolled = await fetch(`${base}/api/agent/enroll`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ token: ticket.token, hostname: 'sim-agent', version: '0.6.8' }) });
  assert.equal(enrolled.status, 201);
  const agentKey = (await enrolled.json()).agentKey;
  const agentHeaders = { authorization: `Bearer ${agentKey}`, 'content-type': 'application/json' };
  const heartbeat = async (body) => {
    const response = await fetch(`${base}/api/agent/heartbeat`, { method: 'POST', headers: agentHeaders, body: JSON.stringify(body) });
    assert.equal(response.status, 200);
  };
  const poll = async () => {
    const response = await fetch(`${base}/api/agent/poll`, { method: 'POST', headers: agentHeaders, body: '{}' });
    assert.equal(response.status, 200);
    return (await response.json()).job;
  };

  await heartbeat({ version: '0.6.8', system: {} });
  const node = (await request('/api/nodes', 'POST', { name: 'Reality 直连', serverId: server.id, protocol: 'vless-reality-vision', topology: 'direct', realityServerName: 'www.amd.com' })).node;
  await request(`/api/nodes/${node.id}/assign`, 'POST', { customerId: customer.id });
  await request(`/api/nodes/${node.id}/deploy`, 'POST', {});

  const keyId = `reality-${node.id}`;
  const first = await poll();
  assert.ok(first, 'initial deploy job was dispatched');
  assert.equal(first.action, 'apply_resource');
  assert.match(first.payload.keyMaterial[keyId].privateKey, B64URL_43);
  const completed = await fetch(`${base}/api/agent/jobs/${first.id}/complete`, { method: 'POST', headers: agentHeaders, body: JSON.stringify({ success: true, result: { artifacts: {} } }) });
  assert.equal(completed.status, 200);
  assert.equal(await poll(), null);

  // The node's local state is wiped and only the agent restarts: its next
  // heartbeat reports no resources, and the panel must re-dispatch on its own.
  await heartbeat({ version: '0.6.8', system: {}, resources: [] });
  const healed = await poll();
  assert.ok(healed, 'heartbeat with empty resources triggered a re-dispatch');
  assert.equal(healed.action, 'apply_resource');
  assert.equal(healed.nodeId, node.id);
  assert.equal(healed.payload.keyMaterial[keyId].privateKey, first.payload.keyMaterial[keyId].privateKey);
  assert.equal(healed.payload.keyMaterial[keyId].publicKey, first.payload.keyMaterial[keyId].publicKey);
});