'use strict'; const test = require('node:test'); const assert = require('node:assert/strict'); const crypto = require('node:crypto'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const net = require('node:net'); const { spawn } = require('node:child_process'); const { Store } = require('../lib/store'); const { Orchestrator } = require('../lib/orchestrator'); const B64URL_43 = /^[A-Za-z0-9_-]{43}$/; // One server, one customer, one active direct Reality node whose resource the // panel believes is deployed on the agent. async function fixture(t) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'nexusgate-hb-')); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); const store = await new Store(path.join(dir, 'data.json')).init(); const stamp = new Date().toISOString(); await store.transaction((data) => { data.servers.push({ id: 'srv_hb', name: '入口', publicAddress: '203.0.113.10', portRangeStart: 20000, portRangeEnd: 50000, status: 'online', lastSeenAt: stamp }); data.customers.push({ id: 'cus_hb', name: '客户', status: 'active' }); data.nodes.push({ id: 'nod_hb', serverId: 'srv_hb', name: 'Reality 节点', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23456, resourceId: 'res_nod_hb_1', status: 'active', entryReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp, }); data.nodeClients.push({ id: 'ncl_hb', nodeId: 'nod_hb', customerId: 'cus_hb', status: 'active', credentials: { clientId: crypto.randomUUID(), relayPassword: 'secret', shortId: 'abcd1234' }, createdAt: stamp, updatedAt: stamp, }); }); return { store, orchestrator: new Orchestrator(store) }; } function applyJobs(store, nodeId) { return store.data.jobs.filter((job) => job.action === 'apply_resource' && (!nodeId || job.nodeId === nodeId)); } test('heartbeat reconcile requeues an apply with keyMaterial for a missing resource', async (t) => { const { store, orchestrator } = await fixture(t); const count = await orchestrator.reconcileFromHeartbeat('srv_hb', []); assert.equal(count, 1); const jobs = applyJobs(store, 'nod_hb'); assert.equal(jobs.length, 1); assert.equal(jobs[0].status, 'queued'); assert.equal(jobs[0].serverId, 'srv_hb'); assert.equal(jobs[0].payload.kind, 'node'); const material = jobs[0].payload.keyMaterial; assert.deepEqual(Object.keys(material), ['reality-nod_hb']); assert.match(material['reality-nod_hb'].privateKey, B64URL_43); assert.equal(store.data.keyStore['reality-nod_hb'].privateKey, material['reality-nod_hb'].privateKey); assert.equal(store.data.nodes[0].status, 'deploying'); assert.ok(store.data.activity.some((event) => event.action === 'reconcile_server' && event.actor === 'system:heartbeat')); }); test('heartbeat reconcile is a no-op when the agent holds every expected resource', async (t) => { const { store, orchestrator } = await fixture(t); assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', ['res_nod_hb_1']), 0); assert.equal(store.data.jobs.length, 0); }); test('heartbeat reconcile does not trigger or disturb jobs while a deploy is in flight', async (t) => { const { store, orchestrator } = await fixture(t); const stamp = new Date().toISOString(); await store.transaction((data) => { data.jobs.push({ id: 'job_busy', serverId: 'srv_hb', nodeId: 'nod_hb', action: 'apply_resource', payload: { kind: 'node' }, status: 'queued', attempts: 0, createdAt: stamp, updatedAt: stamp }); }); assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 0); assert.equal(store.data.jobs.length, 1); // The in-flight job must NOT be failed the way enrollment reconcile fails it. assert.equal(store.data.jobs[0].status, 'queued'); assert.equal(store.data.activity.some((event) => event.action === 'reconcile_server'), false); }); test('heartbeat reconcile is throttled per server', async (t) => { const { store, orchestrator } = await fixture(t); assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 1); assert.equal(applyJobs(store, 'nod_hb').length, 1); // Clear the in-flight gate so only the throttle can explain a no-op. await store.transaction((data) => { data.jobs[0].status = 'completed'; }); assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 0); assert.equal(applyJobs(store, 'nod_hb').length, 1); }); test('heartbeat reconcile ignores legacy reports and out-of-scope nodes', async (t) => { const { store, orchestrator } = await fixture(t); // Old agents: no resources field at all. assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', undefined), 0); assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', 'res_nod_hb_1'), 0); const stamp = new Date().toISOString(); await store.transaction((data) => { // Draft node (never deployed), clientless node, and a removing node: // none of their resources are "expected" on the agent. data.nodes.push( { id: 'nod_draft', serverId: 'srv_hb', name: '草稿', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23457, resourceId: 'res_draft', status: 'draft', realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp }, { id: 'nod_idle', serverId: 'srv_hb', name: '无客户', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23458, resourceId: 'res_idle', status: 'active', entryReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp }, { id: 'nod_rm', serverId: 'srv_hb', name: '删除中', protocol: 'vless-reality-vision', topology: 'direct', networkMode: 'ipv4', port: 23459, resourceId: 'res_rm', status: 'removing', entryReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp }, ); data.nodeClients.push( { id: 'ncl_draft', nodeId: 'nod_draft', customerId: 'cus_hb', status: 'active', credentials: { clientId: crypto.randomUUID(), relayPassword: 'x', shortId: 'aa' }, createdAt: stamp, updatedAt: stamp }, { id: 'ncl_rm', nodeId: 'nod_rm', customerId: 'cus_hb', status: 'active', credentials: { clientId: crypto.randomUUID(), relayPassword: 'x', shortId: 'bb' }, createdAt: stamp, updatedAt: stamp }, ); }); // Only the in-scope node (nod_hb) is requeued; the heartbeat holds nothing. assert.equal(await orchestrator.reconcileFromHeartbeat('srv_hb', []), 1); assert.deepEqual(applyJobs(store).map((job) => job.nodeId), ['nod_hb']); }); test('heartbeat reconcile requeues a forward node whose exit resource is missing', async (t) => { const { store, orchestrator } = await fixture(t); const stamp = new Date().toISOString(); await store.transaction((data) => { data.servers.push({ id: 'srv_exit', name: '出口', publicAddress: '203.0.113.20', portRangeStart: 20000, portRangeEnd: 50000, status: 'online', lastSeenAt: stamp }); data.nodes.push({ id: 'nod_fwd', serverId: 'srv_hb', name: '中转节点', protocol: 'vless-reality-vision', topology: 'forward', exitServerId: 'srv_exit', exitProtocol: 'shadowsocks-2022-aes128', exitPort: 23460, exitResourceId: 'res_fwd_exit', exitCredentials: { password: 'exit-secret' }, networkMode: 'ipv4', port: 23461, resourceId: 'res_fwd_in', status: 'active', entryReady: true, exitReady: true, realityServerName: 'www.tesla.com', createdAt: stamp, updatedAt: stamp, }); data.nodeClients.push({ id: 'ncl_fwd', nodeId: 'nod_fwd', customerId: 'cus_hb', status: 'active', credentials: { clientId: crypto.randomUUID(), relayPassword: 'secret', shortId: 'ccdd1234' }, createdAt: stamp, updatedAt: stamp, }); }); // The exit server's agent holds nothing: the exit resource is re-dispatched. assert.equal(await orchestrator.reconcileFromHeartbeat('srv_exit', []), 1); const exitJobs = applyJobs(store, 'nod_fwd').filter((job) => job.serverId === 'srv_exit'); assert.equal(exitJobs.length, 1); assert.equal(exitJobs[0].payload.kind, 'node-exit'); // The entry server's agent holds its resource: nothing is queued for it. assert.equal(applyJobs(store, 'nod_fwd').filter((job) => job.serverId === 'srv_hb').length, 0); }); async function freePort() { return new Promise((resolve, reject) => { const server = net.createServer(); server.once('error', reject); server.listen(0, '127.0.0.1', () => { const port = server.address().port; server.close(() => resolve(port)); }); }); } async function waitFor(url) { for (let i = 0; i < 60; i += 1) { try { const response = await fetch(url); if (response.ok) return; } catch { /* retry */ } await new Promise((resolve) => setTimeout(resolve, 100)); } throw new Error('test server did not start'); } test('E2E: wiped agent state is healed by a heartbeat reporting no resources', async (t) => { const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'nexusgate-hb-e2e-')); const port = await freePort(); const child = spawn(process.execPath, ['server.js'], { cwd: path.resolve(__dirname, '..'), env: { ...process.env, NG_HOST: '127.0.0.1', NG_PORT: String(port), NG_DATA_FILE: path.join(dir, 'data.json'), NG_ADMIN_PASSWORD: 'test-password', NG_COOKIE_SECURE: 'false' }, stdio: ['ignore', 'pipe', 'pipe'], }); t.after(async () => { child.kill('SIGTERM'); await fs.promises.rm(dir, { recursive: true, force: true }); }); const base = `http://127.0.0.1:${port}`; await waitFor(`${base}/healthz`); const login = await fetch(`${base}/api/auth/login`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ username: 'admin', password: 'test-password' }) }); assert.equal(login.status, 200); const session = await login.json(); const cookie = login.headers.get('set-cookie').split(';')[0]; const request = async (pathname, method = 'GET', body) => { const response = await fetch(`${base}${pathname}`, { method, headers: { cookie, 'x-csrf-token': session.csrf, ...(body ? { 'content-type': 'application/json' } : {}) }, body: body ? JSON.stringify(body) : undefined }); const json = await response.json(); assert.ok(response.ok, JSON.stringify(json)); return json; }; const server = (await request('/api/servers', 'POST', { name: '入口 01', role: 'relay', publicAddress: 'entry.example.com' })).server; const customer = (await request('/api/customers', 'POST', { name: '客户 A' })).customer; const ticket = await request(`/api/servers/${server.id}/enrollment-token`, 'POST', {}); const enrolled = await fetch(`${base}/api/agent/enroll`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ token: ticket.token, hostname: 'sim-agent', version: '0.6.8' }) }); assert.equal(enrolled.status, 201); const agentKey = (await enrolled.json()).agentKey; const agentHeaders = { authorization: `Bearer ${agentKey}`, 'content-type': 'application/json' }; const heartbeat = async (body) => { const response = await fetch(`${base}/api/agent/heartbeat`, { method: 'POST', headers: agentHeaders, body: JSON.stringify(body) }); assert.equal(response.status, 200); }; const poll = async () => { const response = await fetch(`${base}/api/agent/poll`, { method: 'POST', headers: agentHeaders, body: '{}' }); assert.equal(response.status, 200); return (await response.json()).job; }; await heartbeat({ version: '0.6.8', system: {} }); const node = (await request('/api/nodes', 'POST', { name: 'Reality 直连', serverId: server.id, protocol: 'vless-reality-vision', topology: 'direct', realityServerName: 'www.amd.com' })).node; await request(`/api/nodes/${node.id}/assign`, 'POST', { customerId: customer.id }); await request(`/api/nodes/${node.id}/deploy`, 'POST', {}); const keyId = `reality-${node.id}`; const first = await poll(); assert.ok(first, 'initial deploy job was dispatched'); assert.equal(first.action, 'apply_resource'); assert.match(first.payload.keyMaterial[keyId].privateKey, B64URL_43); const completed = await fetch(`${base}/api/agent/jobs/${first.id}/complete`, { method: 'POST', headers: agentHeaders, body: JSON.stringify({ success: true, result: { artifacts: {} } }) }); assert.equal(completed.status, 200); assert.equal(await poll(), null); // The node's local state is wiped and only the agent restarts: its next // heartbeat reports no resources, and the panel must re-dispatch on its own. await heartbeat({ version: '0.6.8', system: {}, resources: [] }); const healed = await poll(); assert.ok(healed, 'heartbeat with empty resources triggered a re-dispatch'); assert.equal(healed.action, 'apply_resource'); assert.equal(healed.nodeId, node.id); assert.equal(healed.payload.keyMaterial[keyId].privateKey, first.payload.keyMaterial[keyId].privateKey); assert.equal(healed.payload.keyMaterial[keyId].publicKey, first.payload.keyMaterial[keyId].publicKey); });