// Real central-management coverage. Requires built Web (embed), Core and CLI // binaries and the privileged `rust` container with the shared /data mount. // Results and process logs are retained in .test-env/central-e2e-* for diagnosis. import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { createServer } from 'node:net'; import { resolve, join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { setTimeout as delay } from 'node:timers/promises'; import { DatabaseSync } from 'node:sqlite'; import { randomUUID } from 'node:crypto'; import { chromium } from 'playwright'; const repo = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const run = String(Date.now()).slice(-7); const output = join(repo, '.test-env', `central-e2e-${run}`); await mkdir(output, { recursive: true }); console.log(`ARTIFACTS=${output}`); const webBin = join(repo, 'target/debug/easytier-web'); const coreBin = join(repo, 'target/debug/easytier-core'); const username = `audit-${run}`; const password = 'e2e-password'; const passwordHash = '$argon2id$v=19$m=4096,t=3,p=1$ZWFzeXRpZXItZTJlLXNhbHQ$FHbDjVElaTXuArgEEQrQrO4AdsnqNOcXGA5PFY8cG/s'; const database = join(output, 'web.db'); const results = []; const processes = []; const namespaces = []; const bridge = `ce${run}`; const subnet = `10.245.${170 + Math.floor(Math.random() * 50)}`; const gatewayIp = `${subnet}.1`; let bridgeCreated = false; let browser, context, page, web; const nodes = []; let network; function docker(args, { allowFailure = false, input, timeout = 20000 } = {}) { const r = spawnSync('docker', ['exec', ...(input === undefined ? [] : ['-i']), 'rust', ...args], { encoding: 'utf8', input, timeout }); if (!allowFailure && r.status !== 0) throw new Error(`docker ${args.join(' ')}: ${r.stderr || r.stdout || r.error}`); return allowFailure ? r : r.stdout.trim(); } function ns(node, args, options) { return docker(['ip', 'netns', 'exec', node.ns, ...args], options); } async function waitFor(check, label, timeout = 45000) { const end = Date.now() + timeout; let last; while (Date.now() < end) { try { const value = await check(); if (value) return value; } catch (error) { last = error.message; } await delay(300); } throw new Error(`Timed out: ${label}${last ? `; last error: ${last}` : ''}`); } async function step(ids, name, action) { const started = new Date().toISOString(); try { const evidence = await action(); results.push({ ids, name, status: 'PASS', started, evidence: evidence ?? 'assertions passed' }); console.log(`PASS ${ids.join(',')} ${name}`); return true; } catch (error) { results.push({ ids, name, status: 'FAIL', started, error: error.stack }); console.error(`FAIL ${ids.join(',')} ${name}: ${error.message}`); if (network && context) await writeFile(join(output, `failure-${results.length}-members.json`), JSON.stringify(await members().catch(() => null), null, 2)); if (network && context) for (const node of nodes.slice(0, 3)) { const config = await runtimeConfig(node).catch(() => null); const evidence = { proxy_cidrs: config?.proxy_cidrs, grants: config?.managed_credentials?.map(c => ({ credential_id: c.credential_id, allowed_proxy_cidrs: c.allowed_proxy_cidrs })), routes: await rpc(node, network.network_id, 'list_route').catch(() => null), }; await writeFile(join(output, `failure-${results.length}-${node.name}-routes.json`), JSON.stringify(evidence, null, 2)); } if (page) await page.screenshot({ path: join(output, `failure-${results.length}.png`), fullPage: true }).catch(() => {}); return false; } finally { await writeFile(join(output, 'results.json'), JSON.stringify(results, null, 2)); } } async function freePort() { return new Promise((resolvePort, reject) => { const server = createServer(); server.once('error', reject); server.listen(0, '0.0.0.0', () => { const port = server.address().port; server.close(() => resolvePort(port)); }); }); } function processLog(child, label, pidFile) { let logs = ''; for (const stream of [child.stdout, child.stderr]) stream.on('data', data => { logs += data; }); const p = { child, label, pidFile, logs: () => logs }; processes.push(p); return p; } async function dockerProcess(label, args) { const pidFile = join(output, `${label}.pid`); return processLog(spawn('docker', ['exec', 'rust', 'sh', '-c', 'echo $$ > "$1"; shift; exec "$@"', 'audit', pidFile, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }), label, pidFile); } async function stop(p) { if (!p || p.stopped) return; if (p.pidFile) { const pid = await readFile(p.pidFile, 'utf8').catch(() => ''); if (pid.trim()) docker(['kill', '-INT', pid.trim()], { allowFailure: true }); } else p.child.kill('SIGINT'); if (p.child.exitCode === null) await Promise.race([new Promise(r => p.child.once('exit', r)), delay(3000)]); if (p.child.exitCode === null && p.pidFile) { const pid = await readFile(p.pidFile, 'utf8').catch(() => ''); if (pid.trim()) docker(['kill', '-KILL', pid.trim()], { allowFailure: true }); } if (p.child.exitCode === null) p.child.kill('SIGTERM'); p.stopped = true; await writeFile(join(output, `${p.label}.log`), p.logs()); } const apiPort = await freePort(); const configPort = await freePort(); const proxyPort = await freePort(); const base = `http://127.0.0.1:${apiPort}`; const peerUrl = `tcp://${gatewayIp}:${proxyPort}`; function startWeb(relay = true) { return processLog(spawn(webBin, ['--db', database, '--api-server-addr', '127.0.0.1', '--api-server-port', String(apiPort), '--config-server-protocol', 'tcp', '--config-server-port', String(configPort), '--gateway-peer-url', peerUrl, ...(relay ? ['--gateway-relay-data'] : []), '--console-log-level', 'error'], { stdio: ['ignore', 'pipe', 'pipe'] }), `web-${processes.length}`); } async function request(method, path, data, expected = 200) { const r = await context.request.fetch(`/api/v1${path}`, { method, ...(data === undefined ? {} : { data }) }); const text = await r.text(); assert.equal(r.status(), expected, `${method} ${path}: ${text}`); return text ? JSON.parse(text) : undefined; } function protoUuid(value) { const hex = value.replaceAll('-', ''); return Object.fromEntries([0, 1, 2, 3].map(i => [`part${i + 1}`, parseInt(hex.slice(i * 8, i * 8 + 8), 16)])); } async function rpc(node, id, method, service = 'api.instance.PeerManageRpcService', payload = {}) { return request('POST', `/machines/${node.id}/proxy-rpc`, { service_name: service, method_name: method, payload: { ...(id ? { instance: { id: protoUuid(id) } } : {}), ...payload } }); } const memberPath = (node, id = network.network_id) => `/networks/${id}/members/${node.id}`; async function runtimeConfig(node, id = network.network_id) { return request('GET', `/machines/${node.id}/networks/config/${id}`); } async function runningInfo(node, id = network.network_id) { return (await rpc(node, id, 'show_node_info')).node_info; } async function members(id = network.network_id) { return (await request('GET', `/networks/${id}/members`)).members; } async function setOverride(node, override, id = network.network_id) { await request('PUT', `${memberPath(node, id)}/config`, { config: override }); } function baseOverride(node) { return { no_tun: false, disable_ipv6: true, multi_thread: false, dev_name: `et${node.name}`, listener_urls: ['tcp://0.0.0.0:11010'], disable_udp_hole_punching: true, disable_tcp_hole_punching: true, disable_upnp: true }; } async function createNetwork(name, options = {}) { return request('POST', '/networks', { settings: { display_name: name, network_name: `${username}-${name}`, networking_method: 'Gateway', virtual_cidr: '10.88.99.0/24', secure_mode: true, ...options } }); } async function updateNetwork(settings, extra = {}) { network = await request('PATCH', `/networks/${network.network_id}`, { settings: { ...network, ...settings }, ...extra }); } async function addNodes(selected, id = network.network_id, extra = {}) { await request('POST', `/networks/${id}/members`, { device_ids: selected.map(n => n.id), ...extra }, 204); await waitFor(async () => { const current = await members(id); return selected.every(n => current.some(m => m.device_id === n.id && m.running)); }, 'members running'); } async function startNode(node) { node.process = await dockerProcess(`core-${node.name}-${processes.length}`, ['ip', 'netns', 'exec', node.ns, coreBin, '--config-server', `${peerUrl}/${username}`, '--machine-id', node.id, '--config-dir', node.config, '--no-listener', '--no-tun', '--hostname', `audit-${node.name}`, '--rpc-portal', '127.0.0.1:15888', '--console-log-level', 'error']); await waitFor(async () => (await request('GET', '/machines')).machines.some(m => m.info?.machine_id && m.online && m.info.hostname === `audit-${node.name}`), `enroll ${node.name}`); } async function login() { await page.goto('/'); await page.locator('#username').fill(username); await page.locator('#password input').fill(password); await page.getByRole('button', { name: 'Login', exact: true }).click(); await page.locator('.console-page').waitFor(); await context.storageState({ path: join(output, 'browser-state.json') }); } function ping(from, to, expected = true) { const r = ns(from, ['ping', '-c', '2', '-W', '1', to], { allowFailure: true, timeout: 5000 }); assert.equal(r.status === 0, expected, `ping ${from.name} -> ${to}: ${r.stdout} ${r.stderr}`); return r.stdout; } async function eventualPing(from, to) { return waitFor(async () => ns(from, ['ping', '-c', '1', '-W', '1', to], { allowFailure: true }).status === 0, `ping ${from.name} -> ${to}`); } async function policy(value) { await request('PUT', `/networks/${network.network_id}/acl-policy`, value); await delay(2000); } function rule(id, sources, destinations, protocols, action = 'allow') { return { id, name: id, enabled: true, action, sources, destinations, protocols: protocols.map(p => ({ stateful: false, ports: [], ...p })) }; } try { docker(['ip', 'link', 'add', bridge, 'type', 'bridge']); bridgeCreated = true; docker(['ip', 'addr', 'add', `${gatewayIp}/24`, 'dev', bridge]); docker(['ip', 'link', 'set', bridge, 'up']); for (const [i, name] of ['a', 'b', 'c', 'wg'].entries()) { const node = { name, ns: `ce-${run}-${name}`, id: randomUUID(), ip: `${subnet}.${10 + i}`, config: join(output, name) }; await mkdir(node.config); docker(['ip', 'netns', 'add', node.ns]); namespaces.push(node.ns); const veth = `v${run}${name}`; docker(['ip', 'link', 'add', veth, 'type', 'veth', 'peer', 'name', 'eth0', 'netns', node.ns]); docker(['ip', 'link', 'set', veth, 'master', bridge]); docker(['ip', 'link', 'set', veth, 'up']); ns(node, ['ip', 'link', 'set', 'lo', 'up']); ns(node, ['ip', 'addr', 'add', `${node.ip}/24`, 'dev', 'eth0']); ns(node, ['ip', 'link', 'set', 'eth0', 'up']); nodes.push(node); } const route = docker(['ip', 'route']); const host = route.match(/default via ([\d.]+)/)?.[1]; assert.ok(host); const proxyScript = join(output, 'proxy.py'); await writeFile(proxyScript, `import socket,socketserver,threading\nclass H(socketserver.BaseRequestHandler):\n def handle(self):\n remote=socket.create_connection(('${host}',${configPort}))\n def copy(a,b):\n try:\n while True:\n data=a.recv(65536)\n if not data: break\n b.sendall(data)\n except OSError: pass\n finally:\n try: b.shutdown(socket.SHUT_WR)\n except OSError: pass\n t=threading.Thread(target=copy,args=(self.request,remote));t.start();copy(remote,self.request);t.join();remote.close()\nclass S(socketserver.ThreadingTCPServer):\n allow_reuse_address=True\n daemon_threads=True\nS(('${gatewayIp}',${proxyPort}),H).serve_forever()\n`); await dockerProcess('config-proxy', ['python3', proxyScript]); web = startWeb(); await waitFor(async () => (await fetch(`${base}/api_meta.js`)).ok, 'Web ready'); const db = new DatabaseSync(database); const user = db.prepare('INSERT INTO users(username,password) VALUES (?,?)').run(username, passwordHash); const group = db.prepare("SELECT id FROM groups WHERE name='users'").get(); db.prepare('INSERT INTO users_groups(user_id,group_id) VALUES (?,?)').run(user.lastInsertRowid, group.id); db.close(); browser = await chromium.launch({ headless: true }); context = await browser.newContext({ baseURL: base }); await context.addInitScript(() => localStorage.setItem('lang', 'en')); page = await context.newPage(); await login(); await writeFile(join(output, 'environment.json'), JSON.stringify({ run, base, apiPort, configPort, peerUrl, bridge, nodes, code: spawnSync('git', ['rev-parse', 'HEAD'], { cwd: repo, encoding: 'utf8' }).stdout.trim() }, null, 2)); const [a, b, c, wg] = nodes; await step(['D01'], 'enroll three real managed Cores', async () => { for (const n of [a, b, c]) await startNode(n); return request('GET', '/machines'); }); await step(['N01', 'M01'], 'secure Gateway with three real members and TUN devices', async () => { network = await createNetwork('gateway'); await addNodes([a, b, c]); for (const n of [a, b, c]) await setOverride(n, baseOverride(n)); const current = await waitFor(async () => { const current = await members(); return current.length === 3 && current.every(m => m.runtime_virtual_ipv4) ? current : false; }, 'member IPv4 addresses'); for (const n of [a, b, c]) n.vip = current.find(m => m.device_id === n.id).runtime_virtual_ipv4.split("/")[0]; await eventualPing(a, b.vip); return { members: current, ping: ping(a, b.vip) }; }); await step(['REG-DIRECT'], 'direct enabled and disabled networks survive central deletion and Web restart', async () => { const directIds = [randomUUID(), randomUUID()]; for (const id of directIds) await request('POST', `/machines/${a.id}/networks`, { save: true, config: { instance_id: id, network_name: id, networking_method: 'Standalone', no_tun: true, disable_ipv6: true, multi_thread: false } }); await request('PUT', `/machines/${a.id}/networks/${directIds[1]}`, { disabled: true }); const extra = await createNetwork('direct-coexist', { networking_method: 'Standalone', virtual_cidr: null }); await addNodes([a], extra.network_id); await request('DELETE', `/networks/${extra.network_id}`, undefined, 204); await delay(6500); await stop(web); web = startWeb(); await waitFor(async () => (await fetch(`${base}/api_meta.js`)).ok, 'Web restart with direct configs'); await login(); await waitFor(async () => (await members()).find(m => m.device_id === a.id)?.online, 'direct device reconnected'); await delay(6500); const db = new DatabaseSync(database); try { for (const [index, id] of directIds.entries()) { const stored = db.prepare('SELECT disabled FROM user_running_network_configs WHERE device_id = ? AND network_instance_id = ?').get(a.id, id); assert.ok(stored, 'direct row retained'); assert.equal(stored.disabled, index); } assert.equal(db.prepare('SELECT COUNT(*) AS n FROM user_running_network_configs WHERE network_instance_id = ?').get(extra.network_id).n, 0); } finally { db.close(); } const state = await request('GET', `/machines/${a.id}/networks`); assert.ok(state.running_inst_ids.some(id => JSON.stringify(id) === JSON.stringify(protoUuid(directIds[0])))); assert.ok(!state.running_inst_ids.some(id => JSON.stringify(id) === JSON.stringify(protoUuid(directIds[1])))); for (const id of directIds) await request('DELETE', `/machines/${a.id}/networks/${id}`); }); await step(['D03'], 'device alias set, clear and length validation', async () => { await request('PUT', `/machines/${a.id}/alias`, { alias: 'Audit Alias' }, 204); assert.ok((await request('GET', '/machines')).machines.some(m => m.alias === 'Audit Alias')); await request('PUT', `/machines/${a.id}/alias`, { alias: 'x'.repeat(65) }, 400); await request('PUT', `/machines/${a.id}/alias`, { alias: '' }, 204); await request('PUT', `/machines/${a.id}/alias`, { alias: 'Persisted Alias' }, 204); }); await step(['M02'], 'member hostname and static address updates and rejections', async () => { await request('PATCH', memberPath(a), { hostname_override: 'member-a', virtual_ipv4: '10.88.99.101' }); await waitFor(async () => (await runningInfo(a)).hostname === 'member-a', 'hostname update'); await request('PATCH', memberPath(b), { virtual_ipv4: '10.88.99.101' }, 400); await request('PATCH', memberPath(b), { virtual_ipv4: '10.99.0.1' }, 400); await request('PATCH', memberPath(a), { hostname_override: '', virtual_ipv4: '' }); await waitFor(async () => (await members()).find(m => m.device_id === a.id).runtime_virtual_ipv4?.split("/")[0] === a.vip, 'address restored'); await eventualPing(a, b.vip); return ping(a, b.vip); }); await step(['M04', 'M05'], 'advanced overrides and protected central identity', async () => { await setOverride(a, { ...baseOverride(a), mtu: 1300, instance_id: randomUUID(), network_name: 'hijack', network_secret: 'hijack', dhcp: true, virtual_ipv4: '1.2.3.4', peer_urls: ['tcp://127.0.0.1:9'] }); await waitFor(async () => (await runtimeConfig(a)).mtu === 1300, 'MTU override'); const actual = await runtimeConfig(a); assert.equal(actual.network_name, network.network_name); assert.equal(actual.instance_id, network.network_id); assert.equal(actual.virtual_ipv4, a.vip); assert.notEqual(actual.network_secret, 'hijack'); await request('DELETE', `${memberPath(a)}/config`, undefined, 204); assert.deepEqual(await request('GET', `${memberPath(a)}/config`), {}); await setOverride(a, baseOverride(a)); return { identity: actual.network_name, mtu: actual.mtu }; }); await step(['M07'], 'concurrent member changes preserve both updates', async () => { await Promise.all([request('PATCH', memberPath(a), { hostname_override: 'concurrent-a' }), request('PATCH', memberPath(b), { hostname_override: 'concurrent-b' })]); const current = await members(); assert.equal(current.find(m => m.device_id === a.id).hostname_override, 'concurrent-a'); assert.equal(current.find(m => m.device_id === b.id).hostname_override, 'concurrent-b'); return current; }); await step(['R01', 'R02'], 'runtime details and device logger round trip', async () => { const routes = await waitFor(async () => { const v = await rpc(a, network.network_id, 'list_route'); return v.routes?.length ? v : false; }, 'route snapshot'); const peers = await waitFor(async () => { const v = await rpc(a, network.network_id, 'list_peer'); return v.peer_infos?.length ? v : false; }, 'peer snapshot'); const info = await waitFor(() => runningInfo(a), 'runtime snapshot after member update'); assert.ok(routes.routes.length); assert.ok(peers.peer_infos.length); assert.ok(info.config.includes(network.network_name)); const service = 'api.logger.LoggerRpcService'; const before = await rpc(a, null, 'get_logger_config', service); await rpc(a, null, 'set_logger_config', service, { level: 2 }); assert.equal((await rpc(a, null, 'get_logger_config', service)).level, 'WARNING'); await rpc(a, null, 'set_logger_config', service, { level: before.level ?? 0 }); return { peerCount: peers.peer_infos.length, routeCount: routes.routes.length }; }); await step(['L02'], 'ACL default deny blocks actual ICMP and default allow restores it', async () => { await policy({ default_action: 'deny', rules: [] }); ping(a, b.vip, false); await policy({ default_action: 'allow', rules: [] }); await eventualPing(a, b.vip); return ping(a, b.vip); }); await step(['L03', 'L06'], 'ACL authenticated member selection and counters', async () => { const current = await members(); const select = n => ({ type: 'member', member_id: current.find(m => m.device_id === n.id).member_id }); await policy({ default_action: 'deny', rules: [rule('icmp-a-b', [select(a)], [select(b)], [{ protocol: 'icmp' }])] }); await eventualPing(a, b.vip); ping(c, b.vip, false); const stats = await rpc(b, network.network_id, 'get_acl_stats', 'api.instance.AclManageRpcService'); assert.ok(JSON.stringify(stats).includes('icmp-a-b')); await policy({ default_action: 'allow', rules: [] }); return stats; }); await step(['L04'], 'TCP and UDP port policy with real socket traffic', async () => { const echo = join(output, 'echo.py'); await writeFile(echo, `import socket,threading def udp(port): s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM);s.bind(("0.0.0.0",port)) while True: d,a=s.recvfrom(1024);s.sendto(d,a) def tcp(port): s=socket.socket();s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1);s.bind(("0.0.0.0",port));s.listen() while True: c,a=s.accept();c.sendall(c.recv(1024));c.close() for port in [18080,18082]: threading.Thread(target=tcp,args=(port,),daemon=True).start() for port in [18081,18082,18083]: threading.Thread(target=udp,args=(port,),daemon=True).start() threading.Event().wait() `); await dockerProcess('echo-b', ['ip', 'netns', 'exec', b.ns, 'python3', echo]); const probe = (proto, port) => ns(a, ['python3', '-c', `import socket;s=socket.socket(socket.AF_INET,socket.${proto === 'tcp' ? 'SOCK_STREAM' : 'SOCK_DGRAM'});s.settimeout(2);s.connect(('${b.vip}',${port}));s.send(b'audit');assert s.recv(1024)==b'audit'`], { allowFailure: true, timeout: 5000 }).status === 0; await policy({ default_action: 'allow', rules: [] }); for (const port of [18080, 18082]) assert.ok(probe('tcp', port)); for (const port of [18081, 18082, 18083]) assert.ok(probe('udp', port)); await policy({ default_action: 'deny', rules: [rule('tcp-only', [{ type: 'all' }], [{ type: 'all' }], [{ protocol: 'tcp', ports: ['18080'], stateful: true }])] }); assert.ok(probe('tcp', 18080)); assert.equal(probe('tcp', 18082), false); assert.equal(probe('udp', 18081), false); await policy({ default_action: 'deny', rules: [rule('udp-range', [{ type: 'all' }], [{ type: 'all' }], [{ protocol: 'udp', ports: ['18081-18082'] }])] }); assert.ok(probe('udp', 18081)); assert.ok(probe('udp', 18082)); assert.equal(probe('udp', 18083), false); assert.equal(probe('tcp', 18080), false); await policy({ default_action: 'allow', rules: [] }); }); await step(['M03', 'L05'], 'proxy CIDR advertisement and real subnet ICMP under ACL', async () => { ns(b, ['ip', 'addr', 'add', '198.18.240.1/24', 'dev', 'lo']); await request('PATCH', memberPath(b), { proxy_cidrs: ['198.18.240.0/24'] }); await waitFor(async () => JSON.stringify(await rpc(a, network.network_id, 'list_route')).includes('198.18.240.0/24'), 'proxy route advertised'); await eventualPing(a, '198.18.240.1'); const current = await members(); const bid = current.find(m => m.device_id === b.id).member_id; await policy({ default_action: 'deny', rules: [rule('proxy-ping', [{ type: 'all' }], [{ type: 'subnet', member_id: bid, cidrs: ['198.18.240.0/24'] }], [{ protocol: 'icmp' }])] }); await eventualPing(a, '198.18.240.1'); ping(a, b.vip, false); await policy({ default_action: 'allow', rules: [] }); return rpc(a, network.network_id, 'list_route'); }); await step(['REG-PROXY'], 'temporary member mapped proxy routes are granted through PATCH and PUT', async () => { ns(c, ['ip', 'addr', 'add', '198.18.241.1/24', 'dev', 'lo']); for (const [method, subnet] of [['PATCH', 242], ['PUT', 243]]) { // Configure each fresh credential while its proxy is offline. Live // grant updates have a separate preexisting Core route-cache race. await stop(c.process); await request('DELETE', memberPath(c), undefined, 204); await request('POST', `/networks/${network.network_id}/members`, { device_ids: [c.id], temporary: true, ttl_seconds: 600 }, 204); const proxy_cidrs = [`198.18.241.0/24->198.18.${subnet}.0/24`]; await setOverride(c, baseOverride(c)); if (method === 'PATCH') await request('PATCH', memberPath(c), { proxy_cidrs }); else await setOverride(c, { ...baseOverride(c), proxy_cidrs }); for (const permanent of [a, b]) await waitFor(async () => (await runtimeConfig(permanent)).managed_credentials?.some(grant => grant.allowed_proxy_cidrs?.includes(`198.18.${subnet}.0/24`)), `${method} mapped CIDR grant`); await startNode(c); await waitFor(async () => JSON.stringify(await rpc(a, network.network_id, 'list_route')).includes(`198.18.${subnet}.0/24`), `${method} mapped proxy route`); await eventualPing(a, `198.18.${subnet}.1`); } await request('DELETE', memberPath(c), undefined, 204); await addNodes([c]); await setOverride(c, baseOverride(c)); }); await step(['REG-SUBNET'], 'changing the subnet reallocates automatic addresses without member edits', async () => { await updateNetwork({ virtual_cidr: '10.89.98.0/24' }); await waitFor(async () => (await members()).every(m => m.runtime_virtual_ipv4?.startsWith('10.89.98.')), 'automatic subnet migration'); const migrated = await members(); assert.ok(migrated.every(m => m.virtual_ipv4 === null && m.allocated_ipv4?.startsWith('10.89.98.'))); await eventualPing(a, migrated.find(m => m.device_id === b.id).allocated_ipv4); await updateNetwork({ virtual_cidr: '10.88.99.0/24' }); await waitFor(async () => (await members()).every(m => m.runtime_virtual_ipv4?.startsWith('10.88.99.')), 'subnet restored'); for (const n of [a, b, c]) n.vip = (await members()).find(m => m.device_id === n.id).runtime_virtual_ipv4.split('/')[0]; }); await step(['N05', 'N06'], 'network identity, secret and secure-mode updates converge', async () => { await updateNetwork({ display_name: 'Renamed audit', network_name: `${username}-renamed` }, { network_secret: 'audit-rotated-secret' }); await waitFor(async () => (await runtimeConfig(b)).network_name === network.network_name && (await runtimeConfig(b)).network_secret === 'audit-rotated-secret', 'identity rotated'); await eventualPing(a, b.vip); await updateNetwork({ secure_mode: false }); await waitFor(async () => !(await runtimeConfig(b)).secure_mode?.enabled, 'secure disabled'); await eventualPing(a, b.vip); await updateNetwork({ secure_mode: true }); await waitFor(async () => (await runtimeConfig(b)).secure_mode?.enabled, 'secure restored'); await eventualPing(a, b.vip); }); await step(['N02', 'N03', 'N04', 'N07'], 'Gateway to Manual, PublicServer, Standalone and back', async () => { await updateNetwork({ networking_method: 'Manual', peer_urls: [`tcp://${a.ip}:11010`] }); await eventualPing(b, a.vip); assert.equal((await runtimeConfig(b)).peer_urls[0], `tcp://${a.ip}:11010`); await updateNetwork({ networking_method: 'PublicServer', public_server_url: `tcp://${a.ip}:11010` }); await eventualPing(b, a.vip); await waitFor(async () => (await runtimeConfig(b)).peer_urls?.includes(`tcp://${a.ip}:11010`), 'public server applied'); await updateNetwork({ networking_method: 'Standalone', peer_urls: [] }); await waitFor(async () => !(await runtimeConfig(b)).peer_urls?.length, 'standalone applied'); assert.deepEqual((await runtimeConfig(b)).peer_urls ?? [], []); await updateNetwork({ networking_method: 'Gateway', peer_urls: ['tcp://invalid:9'] }); await waitFor(async () => (await runtimeConfig(b)).peer_urls.includes(peerUrl), 'Gateway restored'); await eventualPing(a, b.vip); }); await step(['N10'], 'one device in two networks retains the other on deletion', async () => { const second = await createNetwork('second', { networking_method: 'Standalone', virtual_cidr: '10.89.99.0/24' }); await addNodes([a], second.network_id); let state = await request('GET', `/machines/${a.id}/networks`); assert.equal(state.running_inst_ids.length, 2); await request('DELETE', `/networks/${second.network_id}`, undefined, 204); await waitFor(async () => (await request('GET', `/machines/${a.id}/networks`)).running_inst_ids.length === 1, 'secondary instance gone'); return runtimeConfig(a); }); await step(['R03', 'R04'], 'all-DHCP network obtains distinct addresses and retains survivor address', async () => { await updateNetwork({ virtual_cidr: null }); const current = await waitFor(async () => { const m = await members(); return m.every(x => x.runtime_virtual_ipv4) && new Set(m.map(x => x.runtime_virtual_ipv4)).size === 3 && m.every(x => x.runtime_virtual_ipv4.startsWith('10.126.126.')) ? m : false; }, 'distinct DHCP addresses', 60000); const aa = current.find(m => m.device_id === a.id).runtime_virtual_ipv4; const bb = current.find(m => m.device_id === b.id).runtime_virtual_ipv4.split('/')[0]; await eventualPing(a, bb); await stop(b.process); await stop(c.process); await delay(16000); assert.equal((await members()).find(m => m.device_id === a.id).runtime_virtual_ipv4, aa); await startNode(b); await startNode(c); await updateNetwork({ virtual_cidr: '10.88.99.0/24' }); await waitFor(async () => (await members()).every(m => m.runtime_virtual_ipv4?.startsWith('10.88.99.')), 'static addresses restored'); for (const n of [a, b, c]) n.vip = (await members()).find(m => m.device_id === n.id).runtime_virtual_ipv4.split('/')[0]; return current; }); await step(['P01', 'P02', 'D04', 'M06'], 'offline update, Web restart and Core reconnect use latest intent', async () => { await stop(b.process); await request('PATCH', memberPath(b), { hostname_override: 'offline-latest' }); await stop(web); web = startWeb(); await waitFor(async () => (await fetch(`${base}/api_meta.js`)).ok, 'Web restart'); await login(); await startNode(b); await waitFor(async () => (await runningInfo(b)).hostname === 'offline-latest', 'offline desired config replay'); await eventualPing(a, b.vip); assert.ok((await request('GET', '/machines')).machines.some(m => m.alias === 'Persisted Alias')); return { members: await members(), configFiles: docker(['ls', b.config]) }; }); await step(['W01', 'W03'], 'real native WireGuard handshake and ping to managed member', async () => { const privateKey = docker(['wg', 'genkey']); a.portal = { enabled: true, wireguard_listen: '0.0.0.0:15820', wireguard_private_key: privateKey, clients: [{ name: 'audit-wg', virtual_ip: '10.88.99.210/24', groups: [] }] }; await setOverride(a, { ...baseOverride(a), vpn_portal_config: a.portal }); const portal = await waitFor(async () => { const p = await rpc(a, network.network_id, 'get_vpn_portal_info', 'api.instance.VpnPortalRpcService'); return p.vpn_portal_info?.clients?.[0]?.client_config ? p.vpn_portal_info : false; }, 'WireGuard portal'); const config = portal.clients[0].client_config.replace(/^Endpoint\s*=.*$/m, `Endpoint = ${a.ip}:15820`); await writeFile(join(output, 'wg-client.conf'), config); const stripped = config.split('\n').filter(line => !/^\s*(Address|DNS|MTU|Table|PreUp|PostUp|PreDown|PostDown)\s*=/.test(line)).join('\n'); await writeFile(join(output, 'wg-client-stripped.conf'), stripped); ns(wg, ['ip', 'link', 'add', 'wgaudit', 'type', 'wireguard']); ns(wg, ['wg', 'setconf', 'wgaudit', join(output, 'wg-client-stripped.conf')]); ns(wg, ['ip', 'addr', 'add', '10.88.99.210/24', 'dev', 'wgaudit']); ns(wg, ['ip', 'link', 'set', 'wgaudit', 'up']); await eventualPing(wg, b.vip); const handshake = ns(wg, ['wg', 'show', 'wgaudit', 'latest-handshakes']); assert.ok(!handshake.endsWith('\t0')); return { handshake, ping: ping(wg, b.vip) }; }); await step(['W01', 'W05'], 'WireGuard disable preserves config, re-enable and restart recover', async () => { assert.ok(a.portal); await setOverride(a, { ...baseOverride(a), vpn_portal_config: { ...a.portal, enabled: false } }); await delay(3000); ping(wg, b.vip, false); const stored = await request('GET', `${memberPath(a)}/config`); assert.equal(stored.vpn_portal_config.wireguard_private_key, a.portal.wireguard_private_key); assert.equal(stored.vpn_portal_config.clients.length, 1); await setOverride(a, { ...baseOverride(a), vpn_portal_config: a.portal }); await eventualPing(wg, b.vip); await stop(a.process); await startNode(a); await eventualPing(wg, b.vip); return { clientsRetained: stored.vpn_portal_config.clients.length }; }); await step(['W02', 'W05'], 'removed WireGuard client loses access', async () => { assert.ok(a.portal); await eventualPing(wg, b.vip); await setOverride(a, { ...baseOverride(a), vpn_portal_config: { ...a.portal, clients: [] } }); await delay(3000); ping(wg, b.vip, false); }); await step(['R06'], 'Gateway relay-data off blocks forced relay; on restores traffic', async () => { for (const n of [a, b]) await setOverride(n, { ...baseOverride(n), disable_p2p: true }); await eventualPing(a, b.vip); await stop(web); web = startWeb(false); await waitFor(async () => (await fetch(`${base}/api_meta.js`)).ok, 'Web relay off'); await login(); await waitFor(async () => (await members()).filter(m => [a.id, b.id].includes(m.device_id)).every(m => m.running), 'nodes reconnected'); await delay(3000); ping(a, b.vip, false); await stop(web); web = startWeb(true); await waitFor(async () => (await fetch(`${base}/api_meta.js`)).ok, 'Web relay on'); await login(); await eventualPing(a, b.vip); }); await step(['D05', 'D06'], 'device ban rejects reconnect, unban reenrolls without old memberships', async () => { await waitFor(async () => (await members()).some(m => m.device_id === c.id && m.online && m.running), 'member online before ban'); await waitFor(async () => ns(c, ['ip', 'link', 'show', 'dev', 'etc'], { allowFailure: true }).status === 0, 'member TUN before ban'); await request('DELETE', `/machines/${c.id}?block=true`, undefined, 204); assert.notEqual(ns(c, ['ip', 'link', 'show', 'dev', 'etc'], { allowFailure: true }).status, 0, 'deleted device TUN is gone before DELETE returns'); await waitFor(async () => (await request('GET', '/blocked-devices')).blocked.some(d => d.id === c.id && d.attempt_count > 0), 'blocked reconnect attempt', 45000); assert.ok(!(await members()).some(m => m.device_id === c.id)); await request('DELETE', `/blocked-devices/${c.id}`, undefined, 204); await waitFor(async () => (await request('GET', '/machines')).machines.some(m => m.info?.hostname === 'audit-c' && m.online), 'unban reenroll'); assert.ok(!(await members()).some(m => m.device_id === c.id)); await stop(c.process); await request('DELETE', `/machines/${c.id}`, undefined, 204); await startNode(c); }); await step(['N08'], 'delete final network removes online and offline Core configurations', async () => { await stop(b.process); await request('DELETE', `/networks/${network.network_id}`, undefined, 204); await waitFor(async () => (await request('GET', `/machines/${a.id}/networks`)).running_inst_ids.length === 0, 'online instance deletion'); await stop(web); web = startWeb(); await waitFor(async () => (await fetch(`${base}/api_meta.js`)).ok, 'Web restart after delete'); await login(); await startNode(b); await waitFor(async () => (await request('GET', `/machines/${b.id}/networks`)).running_inst_ids.length === 0, 'offline instance deletion'); assert.deepEqual((await request('GET', '/networks')).networks, []); assert.ok(!docker(['ls', b.config]).includes(network.network_id)); }); } catch (error) { results.push({ ids: [], name: 'suite setup or infrastructure', status: 'FAIL', error: error.stack }); console.error(error); } finally { await browser?.close(); for (const p of [...processes].reverse()) await stop(p); for (const name of namespaces) docker(['ip', 'netns', 'delete', name], { allowFailure: true }); if (bridgeCreated) docker(['ip', 'link', 'delete', bridge], { allowFailure: true }); await writeFile(join(output, 'results.json'), JSON.stringify(results, null, 2)); console.log(`RESULTS=${join(output, 'results.json')}`); } process.exitCode = results.some(r => r.status === 'FAIL') ? 1 : 0;