import assert from 'node:assert/strict' import { spawn } from 'node:child_process' import { createServer } from 'node:http' import { mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import test from 'node:test' const script = fileURLToPath(new URL('./event-delivery-fault-proxy.py', import.meta.url)) const targetId = 'e6259b1a-e112-4233-83e1-ae0071db3c52' const otherId = '1b344fb7-d5d9-4324-b2d1-51bd0ab3af55' const samePlanOtherId = '05dab8d1-10fd-4c9e-b242-54f4c6ff76a4' const hash = `sha256:${'a'.repeat(64)}` const secret = 'test-service-identity-secret-do-not-log' const controlToken = 'test-control-secret-with-more-than-24-characters' const event = (id = targetId, planId = '2100000000000000001') => ({ eventId: id, eventType: 'LoadingCompleted', payloadHash: hash, payload: { loadingPlanId: planId, syntheticPrivateField: secret } }) const close = server => new Promise(resolve => server.close(resolve)) async function fixture(selector = ['--loading-plan-id', '2100000000000000001']) { const calls = [] const applied = new Set() let invalidAck = false const consumer = createServer(async (request, response) => { const chunks = [] for await (const chunk of request) chunks.push(chunk) const raw = Buffer.concat(chunks).toString() const envelope = JSON.parse(raw) calls.push({ raw, authorization: request.headers.authorization, eventId: envelope.eventId }) const outcome = applied.has(envelope.eventId) ? 'DUPLICATE' : 'APPLIED' applied.add(envelope.eventId) const body = JSON.stringify({ code: '200', data: { eventId: invalidAck ? otherId : envelope.eventId, payloadHash: envelope.payloadHash, outcome } }) response.writeHead(200, { 'Content-Type': 'application/json', 'X-Consumer-Evidence': 'actual-upstream' }) response.end(body) }) await new Promise(resolve => consumer.listen(0, '127.0.0.1', resolve)) const directory = await mkdtemp(join(tmpdir(), 'event-fault-offline-')) const log = join(directory, 'evidence.jsonl') const child = spawn(process.env.PYTHON || 'python', [script, '--upstream', `http://127.0.0.1:${consumer.address().port}/internal/events`, '--event-type', 'LoadingCompleted', ...selector, '--log', log], { env: { ...process.env, EVENT_FAULT_CONTROL_TOKEN: controlToken }, stdio: ['ignore', 'pipe', 'pipe'] }) let output = ''; let errors = '' child.stdout.on('data', chunk => { output += chunk }) child.stderr.on('data', chunk => { errors += chunk }) const ready = await new Promise((resolve, reject) => { const timer = setTimeout(() => { child.kill(); reject(new Error('Proxy readiness timeout')) }, 8000) const onData = () => { const line = output.split('\n').find(item => item.trim()) if (!line) return try { const value = JSON.parse(line); clearTimeout(timer); child.stdout.off('data', onData); resolve(value) } catch { clearTimeout(timer); reject(new Error('Invalid proxy readiness JSON')) } } child.stdout.on('data', onData) child.once('error', error => { clearTimeout(timer); reject(error) }) child.once('exit', code => { if (!output) { clearTimeout(timer); reject(new Error(`Proxy exit ${code}`)) } }) }) const dataUrl = `http://127.0.0.1:${ready.dataPort}/internal/events` const controlUrl = `http://127.0.0.1:${ready.controlPort}` return { calls, applied, log, invalidAck: value => { invalidAck = value }, send: async envelope => { const sent = JSON.stringify(envelope) const response = await fetch(dataUrl, { method: 'POST', body: sent, headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${secret}` } }) return { status: response.status, raw: await response.text(), headers: response.headers, sent } }, control: (path, body, token = controlToken) => fetch(`${controlUrl}${path}`, { method: body ? 'POST' : 'GET', headers: { 'X-Event-Fault-Control-Token': token, 'Content-Type': 'application/json' }, body: body ? JSON.stringify(body) : undefined, }), output: () => `${output}\n${errors}`, stop: async () => { if (child.exitCode === null) { const exited = new Promise(resolve => child.once('exit', resolve)) child.kill(); await exited } await close(consumer) await rm(directory, { recursive: true }) }, } } test('ACK-loss proxy only targets one event and forwards committed upstream results before dropping ACK', async t => { const proxy = await fixture() t.after(() => proxy.stop()) const unauthorized = await proxy.control('/mode', { mode: 'FAULT' }, 'wrong-control-token') assert.equal(unauthorized.status, 403) let status = await (await proxy.control('/status')).json() assert.equal(status.mode, 'PASS') assert.equal((await proxy.control('/mode', { mode: 'FAULT' })).status, 200) await t.test('real consumer is reached and durable APPLIED then DUPLICATE both lose ACK', async () => { const first = await proxy.send(event()) assert.equal(first.status, 503) assert.equal(JSON.parse(first.raw).errorCode, 'ACK_LOSS_INJECTED') assert.ok(proxy.applied.has(targetId)) const duplicate = await proxy.send(event()) assert.equal(duplicate.status, 503) assert.equal(proxy.calls.length, 2) assert.equal(proxy.calls[0].raw, first.sent) assert.equal(proxy.calls[0].authorization, `Bearer ${secret}`) status = await (await proxy.control('/status')).json() assert.deepEqual(status.records.map(item => item.outcome), ['APPLIED', 'DUPLICATE']) assert.deepEqual(status.records.map(item => item.attempt), [1, 2]) }) await t.test('other plans, other types and another same-plan ID are unaffected', async () => { for (const envelope of [event(otherId, '2100000000000000002'), { ...event(otherId), eventType: 'CarrierScheduleNodeUpserted' }, event(samePlanOtherId)]) { const result = await proxy.send(envelope) assert.equal(result.status, 200) assert.equal(JSON.parse(result.raw).data.eventId, envelope.eventId) assert.equal(result.headers.get('x-consumer-evidence'), 'actual-upstream') } status = await (await proxy.control('/status')).json() assert.equal(status.attempt, 2) assert.equal(status.records.length, 2) }) await t.test('pinned identity rejects hash, hidden payload and malformed-hash changes before upstream', async () => { const before = proxy.calls.length for (const envelope of [{ ...event(), payloadHash: `sha256:${'b'.repeat(64)}` }, { ...event(), payload: { loadingPlanId: '2100000000000000002', changed: secret } }, { ...event(), payloadHash: secret }]) { const result = await proxy.send(envelope) assert.equal(result.status, 409) assert.equal(JSON.parse(result.raw).errorCode, 'TARGET_EVENT_HASH_CONFLICT') } assert.equal(proxy.calls.length, before) }) await t.test('PASS forwards matching duplicate ACK and original upstream response bytes and headers', async () => { assert.equal((await proxy.control('/mode', { mode: 'PASS' })).status, 200) const result = await proxy.send(event()) assert.equal(result.status, 200) assert.equal(result.raw, JSON.stringify({ code: '200', data: { eventId: targetId, payloadHash: hash, outcome: 'DUPLICATE' } })) assert.equal(result.headers.get('x-consumer-evidence'), 'actual-upstream') status = await (await proxy.control('/status')).json() assert.equal(status.records.at(-1).outcome, 'DUPLICATE') assert.equal(status.records.at(-1).errorCode, null) }) await t.test('incorrect upstream ACK is never replaced with plausible success evidence', async () => { proxy.invalidAck(true) const result = await proxy.send(event()) assert.equal(result.status, 502) assert.equal(JSON.parse(result.raw).errorCode, 'UPSTREAM_ACK_INVALID') }) await t.test('state, JSONL evidence and subprocess output never expose private payload or identity secrets', async () => { const records = (await readFile(proxy.log, 'utf8')).trim().split('\n').map(line => JSON.parse(line)) for (const record of records) { assert.deepEqual(Object.keys(record).sort(), ['time', 'eventId', 'payloadHash', 'outcome', 'attempt', 'matched', 'errorCode'].sort()) assert.ok(record.eventId === targetId) } const evidence = `${JSON.stringify(await (await proxy.control('/status')).json())}\n${await readFile(proxy.log, 'utf8')}\n${proxy.output()}` assert.ok(!evidence.includes(secret)) assert.ok(!evidence.includes(controlToken)) assert.ok(!evidence.includes('syntheticPrivateField')) assert.ok(!evidence.includes('authorization')) }) }) test('explicit event-ID selector requires exact event type and forwards unrelated IDs unchanged', async t => { const proxy = await fixture(['--event-id', targetId]) t.after(() => proxy.stop()) await proxy.control('/mode', { mode: 'FAULT' }) const other = await proxy.send(event(otherId)) assert.equal(other.status, 200) const wrongType = await proxy.send({ ...event(), eventType: 'OverseasDeliveryCompleted' }) assert.equal(wrongType.status, 200) const target = await proxy.send(event()) assert.equal(target.status, 503) const status = await (await proxy.control('/status')).json() assert.equal(status.attempt, 1) assert.equal(status.eventId, targetId) })