359 lines
30 KiB
JavaScript
359 lines
30 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import test from 'node:test'
|
|
import { createHash } from 'node:crypto'
|
|
import { spawnSync } from 'node:child_process'
|
|
import { createTestFaultExecutor, createTestUnloadFaultExecutor, createTestDeliveryFaultExecutor, assertTestDeliveryFaultProof, assertTestUnloadFaultProof, renderTestFaultAction } from './test-event-fault-executor.mjs'
|
|
import { withDeliveryEventRecovery } from './admin-test-env-modules/phase5-root-fulfillment.mjs'
|
|
|
|
const scope = { marker: 'TEST-ROOT', customerId: '7', loadingPlanId: '99', containerBookingId: '66' }
|
|
const environment = { CHONGCHONG_ENV: 'test', JENKINS_URL: 'https://jenkins.example.invalid/',
|
|
JENKINS_USER: 'offline', JENKINS_API_TOKEN: 'offline-secret', FBA_CORE_JENKINS_AGENT: 'java-test-agent' }
|
|
function requestedAction(options) {
|
|
const script = options.body.get('script')
|
|
const chunks = [...script.matchAll(/'([A-Za-z0-9+/=]+)'/g)].map(match => match[1]).join('')
|
|
const groovy = Buffer.from(chunks, 'base64').toString('utf8')
|
|
const argument = groovy.match(/'UTF-8'\),'([A-Za-z0-9+/=]+)'\]\)/)?.[1]
|
|
return JSON.parse(Buffer.from(argument, 'base64').toString('utf8'))
|
|
}
|
|
function transport(calls, failActivate = false) {
|
|
let recovered = false
|
|
return async (_url, options) => {
|
|
assert.equal(options.redirect, 'error')
|
|
const { action } = requestedAction(options)
|
|
calls.push(action)
|
|
if (action === 'recover') recovered = true
|
|
const data = action === 'proxy-ready' ? { ready: true } : action === 'health' ? { healthy: true }
|
|
: action === 'status' ? { producer: { publishStatus: recovered ? 'PUBLISHED' : 'DEAD', failedAttempts: 12 } }
|
|
: action === 'proof' ? { ...scope, restored: false } : {}
|
|
const result = action === 'activate' && failActivate ? { ok: false, error: 'DOCKER_409' } : { ok: true, data }
|
|
return { ok: true, text: async () => 'TEST_EVENT_JSON=' + JSON.stringify(result) }
|
|
}
|
|
}
|
|
test('scope and actions are bounded and renderer never embeds caller credentials', () => {
|
|
const rendered = renderTestFaultAction('status', scope, environment.FBA_CORE_JENKINS_AGENT)
|
|
assert.ok(!rendered.includes(environment.JENKINS_API_TOKEN))
|
|
for (const change of [{ ...scope, customerId: '7;DROP' }, { ...scope, marker: '../other' }]) {
|
|
assert.throws(() => renderTestFaultAction('status', change, environment.FBA_CORE_JENKINS_AGENT))
|
|
}
|
|
assert.throws(() => renderTestFaultAction('arbitrary', scope, 'java-test-agent'))
|
|
assert.throws(() => createTestFaultExecutor({ ...environment, CHONGCHONG_ENV: 'prod' }))
|
|
})
|
|
|
|
const unloadScope = { marker: 'TEST-ROOT', customerId: '7', containerBookingId: '66' }
|
|
const originalImageId = 'sha256:' + 'a'.repeat(64), originalFingerprint = 'b'.repeat(64)
|
|
function unloadProof(nanoseconds = false) {
|
|
const payload = { actualEmptyReportedAt: null, containerNo: 'OFFLINE66', sourceContainerBookingId: '66',
|
|
unloadingCompletedAt: nanoseconds ? '2026-10-02T01:00:00.136827515Z' : '2026-10-02T01:00:00Z', unloadingPhotoUploadedAt: '2026-10-02T00:59:00Z' }
|
|
const payloadHash = 'sha256:' + createHash('sha256').update(JSON.stringify(payload)).digest('hex')
|
|
const eventId = '00000000-0000-4000-8000-000000000001', entryId = '123'
|
|
const envelope = { eventId, eventType: 'OverseasContainerUnloaded', eventVersion: 1, tenantId: '0',
|
|
aggregateId: '66', aggregateType: 'CONTAINER_BOOKING', aggregateVersion: 3, payloadHash, payload }
|
|
const row = { entryId, eventId, payloadHash, envelope, eventType: envelope.eventType, customerId: '0',
|
|
aggregateId: '66', aggregateType: 'CONTAINER_BOOKING', eventKey: 'OverseasContainerUnloaded:66', publishStatus: 'PUBLISHED' }
|
|
const container = { id: '66', containerNo: 'OFFLINE66', version: '9', emptyReportSourceVersion: '3',
|
|
unloadingCompletedAt: nanoseconds ? '2026-10-02 01:00:00.137' : '2026-10-02 01:00:00', unloadingPhotoUploadedAt: '2026-10-02 00:59:00',
|
|
estimatedEmptyReportedAt: nanoseconds ? '2026-10-05 01:00:00.137' : '2026-10-05 01:00:00', actualEmptyReportedAt: null }
|
|
return JSON.parse(JSON.stringify({ ...unloadScope, readOnly: true, producer: row,
|
|
beforeRetry: { ...row, publishStatus: 'DEAD', failedAttempts: 12, container }, afterRetry: { ...row, container },
|
|
consumer: { count: 1, status: 'PROCESSED', eventId, payloadHash, eventType: envelope.eventType, aggregateId: '66', payload },
|
|
attempts: Array.from({ length: 13 }, (_, i) => ({ attemptId: `attempt-${i}`, entryId, eventId, serviceName: 'lastleg', outcome: i < 11 ? 'RETRY' : i === 11 ? 'DEAD' : 'DUPLICATE' })),
|
|
proxyRecords: Array.from({ length: 13 }, (_, i) => ({ eventId, payloadHash, attempt: i + 1, matched: true,
|
|
outcome: i ? 'DUPLICATE' : 'APPLIED', errorCode: i < 12 ? 'ACK_LOSS_INJECTED' : null })),
|
|
retryRequestId: '00000000-0000-4000-8000-000000000002', retryAuditCount: 1,
|
|
retryAudit: { requestId: '00000000-0000-4000-8000-000000000002', entryId, eventId, payloadHash, previousFailures: 12 } }))
|
|
}
|
|
function unloadTransport(calls, proof = unloadProof(), finish = {}) {
|
|
let recovered = false
|
|
return async (_url, options) => {
|
|
const config = requestedAction(options), { action } = config
|
|
assert.equal(config.direction, 'UNLOAD')
|
|
assert.deepEqual(config.scope, unloadScope)
|
|
calls.push(action)
|
|
if (action === 'recover') recovered = true
|
|
const data = action === 'prepare' ? { originalImageId, originalFingerprint }
|
|
: action === 'proxy-ready' ? { ready: true } : action === 'health' ? { healthy: true }
|
|
: action === 'status' ? { producer: { publishStatus: recovered ? 'PUBLISHED' : 'DEAD', failedAttempts: 12 } }
|
|
: action === 'proof' ? proof : action === 'finish' ? { restored: true, healthy: true, proxyStopped: true,
|
|
producerName: 'firstleg-test-lastleg', originalImageId, currentImageId: originalImageId,
|
|
originalFingerprint, currentFingerprint: originalFingerprint, ...finish } : {}
|
|
return { ok: true, text: async () => 'TEST_EVENT_JSON=' + JSON.stringify({ ok: true, data }) }
|
|
}
|
|
}
|
|
test('unload factory pins exactly three scope fields and verifies restoration before success', async () => {
|
|
const calls = [], executor = createTestUnloadFaultExecutor(environment, unloadTransport(calls))
|
|
await assert.rejects(executor.prepare({ ...unloadScope, eventType: 'Other' }), /Invalid/)
|
|
await executor.prepare(unloadScope)
|
|
await assert.rejects(executor.prepare({ ...unloadScope, containerBookingId: '67' }), /scope drift/)
|
|
await assert.rejects(executor.readProof({ ...unloadScope, eventType: 'Other' }), /scope drift/)
|
|
const proof = await executor.readProof(unloadScope)
|
|
assert.equal(proof.restored, true)
|
|
assert.equal(proof.restoration.currentImageId, originalImageId)
|
|
await executor.cleanup()
|
|
assert.deepEqual(calls, ['prepare', 'proxy-ready', 'activate', 'health', 'status', 'recover', 'status', 'proof', 'restore', 'health', 'finish'])
|
|
})
|
|
test('unload proof rejects changed original event, fabricated ACK, nonunique audit and changed business facts', () => {
|
|
assertTestUnloadFaultProof(unloadProof(), unloadScope)
|
|
for (const mutate of [
|
|
p => { p.producer.customerId = '7' }, p => { p.producer.eventKey = 'other' },
|
|
p => { p.producer.envelope.tenantId = '7' }, p => { p.producer.envelope.payload.containerNo = 'CHANGED' },
|
|
p => { p.beforeRetry.entryId = '124' }, p => { p.beforeRetry.envelope.aggregateVersion = 4 },
|
|
p => { p.beforeRetry.failedAttempts = 11 }, p => { p.afterRetry.eventId = 'OTHER' },
|
|
p => { p.consumer.count = 2 }, p => { p.consumer.status = 'RECEIVED' }, p => { p.consumer.payload = {} },
|
|
p => { p.attempts.pop() }, p => { p.attempts[0].serviceName = 'firstleg' },
|
|
p => { p.proxyRecords[0].outcome = 'DUPLICATE' }, p => { p.proxyRecords[0].errorCode = null },
|
|
p => { p.proxyRecords[12].attempt = 1 }, p => { p.retryAuditCount = 2 },
|
|
p => { p.retryAudit.requestId = 'OTHER' }, p => { p.afterRetry.container.version = '10' },
|
|
p => { p.beforeRetry.container.estimatedEmptyReportedAt = p.afterRetry.container.estimatedEmptyReportedAt = '2026-10-04 01:00:00' },
|
|
]) {
|
|
const proof = unloadProof(); mutate(proof)
|
|
assert.throws(() => assertTestUnloadFaultProof(proof, unloadScope), /Invalid unload fault proof/)
|
|
}
|
|
})
|
|
|
|
test('unload timestamps follow DATETIME(3) rounding rather than allowing time drift', () => {
|
|
const proof=unloadProof(true)
|
|
assertTestUnloadFaultProof(proof,unloadScope)
|
|
for(const row of [proof.beforeRetry,proof.afterRetry]) {
|
|
row.container.unloadingCompletedAt='2026-10-02 01:00:00.138'
|
|
row.container.estimatedEmptyReportedAt='2026-10-05 01:00:00.138'
|
|
}
|
|
assert.throws(()=>assertTestUnloadFaultProof(proof,unloadScope),/CONTAINER_FACTS/)
|
|
})
|
|
test('invalid unload proof still restores and incomplete restoration never becomes success', async () => {
|
|
const invalid = unloadProof(); invalid.consumer.count = 0
|
|
const calls = [], executor = createTestUnloadFaultExecutor(environment, unloadTransport(calls, invalid))
|
|
await executor.prepare(unloadScope)
|
|
await assert.rejects(executor.readProof(unloadScope), /INBOX/)
|
|
assert.deepEqual(calls.slice(-3), ['restore', 'health', 'finish'])
|
|
for (const change of [{ restored: false }, { healthy: false }, { proxyStopped: false },
|
|
{ currentImageId: 'sha256:' + 'c'.repeat(64) }, { currentFingerprint: 'c'.repeat(64) }]) {
|
|
const failed = createTestUnloadFaultExecutor(environment, unloadTransport([], unloadProof(), change))
|
|
await failed.prepare(unloadScope)
|
|
await assert.rejects(failed.readProof(unloadScope), /restoration evidence mismatch/)
|
|
}
|
|
})
|
|
test('Python reverse SQL routing, scope isolation and full configuration recovery fail closed offline', () => {
|
|
const result = spawnSync('python', ['-c', String.raw`
|
|
import ast, copy, importlib.util, io, pathlib, sys
|
|
from unittest.mock import patch
|
|
sys.dont_write_bytecode = True
|
|
path = pathlib.Path('scripts/test-event-fault-runtime.py')
|
|
ast.parse(path.read_text(encoding='utf-8'))
|
|
spec = importlib.util.spec_from_file_location('fault_runtime', path)
|
|
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
|
scope = {'marker':'TEST-ROOT','customerId':'7','containerBookingId':'66'}
|
|
assert m.target_setting({'direction':'UNLOAD'}) == ('FIRSTLEG_SERVICE_URL','http://firstleg:9202')
|
|
assert m.scope_directory(scope, 'UNLOAD') != m.scope_directory(dict(scope, loadingPlanId='99'))
|
|
try: m.scope_directory(dict(scope, eventType='Other'), 'UNLOAD'); assert False
|
|
except RuntimeError as error: assert str(error) == 'INVALID_SCOPE'
|
|
queries=[]
|
|
with patch.object(m, 'inspect', side_effect=lambda name: name), patch.object(m, 'read_sql', side_effect=lambda name, sql: queries.append((name, sql)) or []):
|
|
assert m.producer(scope, 'UNLOAD') is None
|
|
assert queries[0][0] == 'firstleg-test-lastleg'
|
|
assert "customer_id='0'" in queries[0][1] and "event_key='OverseasContainerUnloaded:66'" in queries[0][1]
|
|
assert "JSON_EXTRACT(payload_json,'$')" in queries[0][1] and "publishStatus',status" in queries[0][1]
|
|
row={'entryId':'123','eventId':'00000000-0000-4000-8000-000000000001','payloadHash':'sha256:'+'a'*64,'publishStatus':'DEAD','failedAttempts':12,'envelope':{'eventType':'OverseasContainerUnloaded'}}
|
|
retry_state={'direction':'UNLOAD','operationsRequestId':'00000000-0000-4000-8000-000000000002'}
|
|
retry_container={'Config':{'Env':['EVENT_OPERATIONS_TOKEN=offline']},'NetworkSettings':{'Networks':{'firstleg-test-network':{'IPAddress':'172.20.0.3'}}}}
|
|
with patch.object(m,'producer',return_value=row),patch.object(m,'control',return_value=row),patch.object(m,'container_facts',return_value={'version':'9'}),patch.object(m,'business_ids',side_effect=AssertionError('reverse must not query inventory or orders')),patch.object(m,'private_write'),patch.object(m,'inspect',return_value=retry_container),patch.object(m.urllib.request,'urlopen',return_value=io.BytesIO(b'{"code":"200"}')) as http:
|
|
assert m.recover(scope,pathlib.Path('/offline'),retry_state)['retrySubmitted']
|
|
assert http.call_args.args[0].full_url == 'http://172.20.0.3:9303/internal/lastleg/event-operations/123/retry'
|
|
assert retry_state['beforeRetry']['container']=={'version':'9'}
|
|
assert retry_state['beforeRetry']['envelope']==row['envelope']
|
|
row['publishStatus']='PUBLISHED'
|
|
queries=[]
|
|
with patch.object(m,'producer',return_value=row),patch.object(m,'control',return_value={'records':[]}),patch.object(m,'container_facts',return_value={'version':'9'}),patch.object(m,'business_ids',side_effect=AssertionError('reverse must not query inventory or orders')),patch.object(m,'inspect',side_effect=lambda name:name),patch.object(m,'read_sql',side_effect=lambda name,sql: queries.append((name,sql)) or []),patch.object(m,'private_write') as save:
|
|
proof=m.proof(scope,retry_state)
|
|
save.assert_called_once_with(m.scope_directory(scope,'UNLOAD')/'proof.json',proof)
|
|
assert proof['retryAuditCount']==0 and proof['retryRequestId']==retry_state['operationsRequestId']
|
|
assert [item[0] for item in queries]==['firstleg-test-lastleg','firstleg-test-lastleg','firstleg-test-business']
|
|
assert "WHERE entry_id='123' AND event_type='OverseasContainerUnloaded'" in queries[1][1]
|
|
assert 'fl_container_event_inbox' in queries[2][1]
|
|
container={'Name':'/firstleg-test-lastleg','Id':'container123456789','Image':'sha256:'+'a'*64,
|
|
'Config':{'Image':'lastleg:test','Env':['FIRSTLEG_SERVICE_URL=http://firstleg:9202','A=1'],'Labels':{'com.docker.compose.project':'firstleg-test'}},
|
|
'HostConfig':{'Binds':['source:/data:rw'],'PortBindings':{'9303/tcp':[{'HostPort':'9303'}]}},
|
|
'Mounts':[{'Type':'bind','Source':'/source','Destination':'/data','RW':True}],
|
|
'NetworkSettings':{'Networks':{'firstleg-test-network':{'IPAddress':'172.20.0.3','Aliases':['lastleg-service','container123']}}},
|
|
'State':{'Health':{'Status':'healthy'}}}
|
|
fingerprint=m.fingerprint(container)
|
|
multi=copy.deepcopy(container)
|
|
multi['NetworkSettings']['Networks']['firstleg-test-network']['MacAddress']='02:42:ac:15:00:04'
|
|
multi['NetworkSettings']['Networks']['second-test-network']={'IPAddress':'172.22.0.7','MacAddress':'02:42:ac:16:00:07','Aliases':['lastleg']}
|
|
legacy=copy.deepcopy(multi); legacy['Config']['MacAddress']='02:42:ac:16:00:07'
|
|
assert m.fingerprint(legacy)==m.fingerprint(multi)
|
|
changed=copy.deepcopy(legacy); changed['Config']['MacAddress']='02:42:ac:99:00:01'
|
|
assert m.fingerprint(changed)!=m.fingerprint(multi)
|
|
changed=copy.deepcopy(legacy); changed['NetworkSettings']['Networks']['second-test-network']['MacAddress']='02:42:ac:16:00:08'
|
|
assert m.fingerprint(changed)!=m.fingerprint(multi)
|
|
state={'direction':'UNLOAD','restored':False,'image':container['Config']['Image'],'originalImageId':container['Image'],
|
|
'originalFingerprint':fingerprint,'originalBody':m.recreation(container,m.env_of(container)),'proxyPid':123,'controlToken':'offline'}
|
|
for mutate in [lambda c:c['HostConfig'].update(Binds=['other:/data:rw']),lambda c:c['NetworkSettings']['Networks']['firstleg-test-network'].update(IPAddress='172.20.0.4'),lambda c:c.update(Image='sha256:'+'b'*64),lambda c:c['State']['Health'].update(Status='unhealthy')]:
|
|
changed=copy.deepcopy(container); mutate(changed)
|
|
with patch.object(m,'inspect',return_value=changed), patch.object(m,'private_write') as write:
|
|
try: m.finish(pathlib.Path('/offline'),copy.deepcopy(state)); assert False
|
|
except RuntimeError as error: assert str(error).startswith('RESTORED_')
|
|
write.assert_not_called()
|
|
with patch.object(m,'inspect',return_value=container),patch.object(m.Path,'exists',return_value=False),patch.object(m,'private_write'):
|
|
restored=m.finish(pathlib.Path('/offline'),copy.deepcopy(state))
|
|
assert restored['proxyStopped'] and restored['currentFingerprint']==fingerprint and restored['originalImageId']==restored['currentImageId']
|
|
with patch.object(m,'inspect',return_value=container),patch.object(m.Path,'exists',return_value=True),patch.object(m.Path,'read_bytes',return_value=b'python3\x00'+str(pathlib.Path('/offline')/'proxy.py').encode()+b'\x00'),patch.object(m.os,'kill'),patch.object(m.time,'sleep'),patch.object(m,'private_write') as write:
|
|
try: m.finish(pathlib.Path('/offline'),copy.deepcopy(state)); assert False
|
|
except RuntimeError as error: assert str(error)=='PROXY_NOT_STOPPED'
|
|
write.assert_not_called()
|
|
`], { encoding: 'utf8', cwd: new URL('..', import.meta.url), timeout: 10000 })
|
|
assert.equal(result.status, 0, result.stderr || result.error?.message)
|
|
})
|
|
test('only original scoped event is recovered and restoration precedes a successful proof', async () => {
|
|
const calls = [], executor = createTestFaultExecutor(environment, transport(calls))
|
|
await executor.prepare(scope)
|
|
await executor.prepare({ ...scope, loadingPlanId: '100' })
|
|
const proof = await executor.readProof({ containerBookingId: '66', loadingPlanId: '99', customerId: '7', marker: 'TEST-ROOT' })
|
|
assert.equal(proof.restored, true)
|
|
await executor.cleanup()
|
|
assert.deepEqual(calls, ['prepare', 'proxy-ready', 'activate', 'health', 'status', 'recover',
|
|
'status', 'proof', 'restore', 'health', 'finish'])
|
|
})
|
|
test('failed activation retains task ownership for explicit restoration, never retries activation', async () => {
|
|
const calls = [], executor = createTestFaultExecutor(environment, transport(calls, true))
|
|
await assert.rejects(executor.prepare(scope), /DOCKER_409/)
|
|
await executor.cleanup()
|
|
assert.equal(calls.filter(name => name === 'activate').length, 1)
|
|
assert.deepEqual(calls.slice(-3), ['restore', 'health', 'finish'])
|
|
})
|
|
|
|
const deliveryScope={marker:'TEST-DELIVERY',customerId:'7',deliveryId:'90',sourceInboundId:'200',sourceOrderId:'100',rootOrderId:'100'}
|
|
function deliveryProof(mode='ROOT_PENDING') {
|
|
const scope={...deliveryScope,...(mode==='CHILD_FIRST'?{sourceOrderId:'101',sourceInboundId:'201'}:{})}
|
|
const eventId='00000000-0000-4000-8000-000000000001', entryId='123'
|
|
const payload={deliveryOrderId:'90',deliveryNo:'DL-OFFLINE',lastlegOrderId:'300',lastlegOrderNo:'LL-OFFLINE',
|
|
sourceInboundId:scope.sourceInboundId,inboundCode:'IN-OFFLINE',sourceOrderId:scope.sourceOrderId,
|
|
sourceOrderNo:'FL-OFFLINE',rootOrderId:'100',fbaWarehouseCode:'ABE4',completedAt:'2026-10-03T01:00:00Z'}
|
|
const payloadHash='sha256:'+createHash('sha256').update(JSON.stringify(payload,Object.keys(payload).sort())).digest('hex')
|
|
const envelope={eventId,eventType:'OverseasDeliveryCompleted',eventVersion:1,tenantId:'7',aggregateType:'INBOUND_ORDER',aggregateId:scope.sourceInboundId,aggregateVersion:5,payloadHash,payload}
|
|
const producer={entryId,eventId,eventType:envelope.eventType,customerId:'7',aggregateId:scope.sourceInboundId,aggregateType:'INBOUND_ORDER',eventKey:`OverseasDeliveryCompleted:DL-OFFLINE:${scope.sourceInboundId}`,envelope,payloadHash,publishStatus:'PUBLISHED'}
|
|
const ownRoot=mode!=='CHILD_FIRST', childComplete=mode!=='ROOT_PENDING', rootComplete=mode==='ALL_COMPLETE'
|
|
const orders=['100','101'].map(id=>({id,rootOrderId:'100',customerId:'7',orderRole:id==='100'?'MASTER':'CHILD',
|
|
orderNo:id===scope.sourceOrderId?'FL-OFFLINE':'OTHER',inboundCode:id===scope.sourceOrderId?'IN-OFFLINE':'OTHER',
|
|
status:(id==='100'?rootComplete:childComplete)?'COMPLETED':'APPROVED',version:3,
|
|
completedAt:(id==='100'?rootComplete:childComplete)?'2026-10-03 01:00:00':null}))
|
|
const consumer={eventId,eventType:envelope.eventType,aggregateId:scope.sourceInboundId,payloadHash,payload,status:'PROCESSED'}
|
|
const business={inbound:[{id:scope.sourceInboundId,customerId:'7',orderId:scope.sourceOrderId,orderNo:'FL-OFFLINE',inboundCode:'IN-OFFLINE',processStatus:'DELIVERED',orderStatusSnapshot:orders.find(o=>o.id===scope.sourceOrderId).status,version:2}],
|
|
inboundHistory:[{id:'41',inboundId:scope.sourceInboundId,action:'COMPLETE_DELIVERY',fromStatus:'LOADED',toStatus:'DELIVERED',occurredAt:'2026-10-03 01:00:00'}],
|
|
inbox:[{...consumer,processedAt:'2026-10-03 01:00:01'}],order:orders,
|
|
orderHistory:[...(ownRoot?[{id:'51',orderId:'100',action:'OVERSEAS_DELIVERED',occurredAt:'2026-10-03 01:00:00'}]:[]),...(childComplete?[{id:'52',orderId:'101',action:'OVERSEAS_DELIVERED',occurredAt:'2026-10-03 01:00:00'}]:[]),...(rootComplete?[{id:'53',orderId:'100',action:'FULFILLMENT_COMPLETED'}]:[])],
|
|
completionEvent:orders.filter(o=>o.status==='COMPLETED').map((o,i)=>({id:String(60+i),orderId:o.id,eventType:'FIRST_LEG_ORDER_FULFILLMENT_COMPLETED',status:'PUBLISHED',payload:{orderId:o.id}}))}
|
|
const old=unloadProof()
|
|
return JSON.parse(JSON.stringify({...scope,readOnly:true,producer,consumer:{...consumer,count:1},
|
|
beforeRetry:{...producer,publishStatus:'DEAD',failedAttempts:12,business},afterRetry:{...producer,business},
|
|
attempts:old.attempts,proxyRecords:old.proxyRecords.map(p=>({...p,payloadHash})),retryAudit:{...old.retryAudit,payloadHash},retryAuditCount:1,retryRequestId:old.retryRequestId}))
|
|
}
|
|
function deliveryTransport(calls, proof=deliveryProof(), finish={}) {
|
|
let recovered=false
|
|
return async(_url,options)=>{
|
|
const {action,direction,scope}=requestedAction(options)
|
|
assert.equal(direction,'DELIVERY');assert.deepEqual(scope,deliveryScope);calls.push(action)
|
|
if(action==='recover')recovered=true
|
|
const data=action==='prepare'?{originalImageId,originalFingerprint}:action==='proxy-ready'?{ready:true}:action==='health'?{healthy:true}
|
|
:action==='status'?{producer:{publishStatus:recovered?'PUBLISHED':'DEAD',failedAttempts:12}}:action==='proof'?proof
|
|
:action==='finish'?{restored:true,healthy:true,proxyStopped:true,producerName:'firstleg-test-lastleg',originalImageId,currentImageId:originalImageId,originalFingerprint,currentFingerprint:originalFingerprint,...finish}:{}
|
|
return {ok:true,text:async()=> 'TEST_EVENT_JSON='+JSON.stringify({ok:true,data})}
|
|
}
|
|
}
|
|
test('DELIVERY has a separate strict scope and restores before accepting original event proof',async()=>{
|
|
const calls=[],executor=createTestDeliveryFaultExecutor(environment,deliveryTransport(calls))
|
|
await assert.rejects(executor.prepare({...deliveryScope,eventType:'OverseasContainerUnloaded'}),/Invalid/)
|
|
await executor.prepare(deliveryScope)
|
|
await assert.rejects(executor.prepare({...deliveryScope,deliveryId:'91'}),/scope drift/)
|
|
const proof=await executor.readProof(deliveryScope)
|
|
assert.equal(proof.restored,true);assert.equal(proof.restoration.currentImageId,originalImageId)
|
|
assert.deepEqual(calls,['prepare','proxy-ready','activate','health','status','recover','status','proof','restore','health','finish'])
|
|
})
|
|
test('DELIVERY proof distinguishes root pending, child first and all completed with immutable original evidence',()=>{
|
|
for(const mode of ['ROOT_PENDING','CHILD_FIRST','ALL_COMPLETE']) {
|
|
const p=deliveryProof(mode),scope={...deliveryScope,sourceOrderId:p.sourceOrderId,sourceInboundId:p.sourceInboundId}
|
|
assertTestDeliveryFaultProof(p,scope)
|
|
}
|
|
for(const mutate of [p=>p.producer.envelope.payload.deliveryId='90',p=>delete p.producer.envelope.payload.deliveryOrderId,
|
|
p=>p.producer.customerId='0',p=>p.producer.eventKey='wrong',p=>p.beforeRetry.entryId='124',
|
|
p=>p.consumer.count=2,p=>p.attempts.pop(),p=>p.proxyRecords[0].outcome='DUPLICATE',
|
|
p=>p.retryAuditCount=2,p=>p.afterRetry.business.inbound[0].version++,p=>p.afterRetry.business.order[0].version++,
|
|
p=>p.beforeRetry.business.inbox[0].status='RECEIVED',p=>p.beforeRetry.business.inboundHistory.push({...p.beforeRetry.business.inboundHistory[0]}),
|
|
p=>{p.beforeRetry.business.order[0].status=p.afterRetry.business.order[0].status='COMPLETED'},
|
|
p=>p.beforeRetry.business.completionEvent.push({orderId:'100',status:'PUBLISHED'})]) {
|
|
const proof=deliveryProof();mutate(proof);assert.throws(()=>assertTestDeliveryFaultProof(proof,deliveryScope),/Invalid delivery/)
|
|
}
|
|
})
|
|
test('DELIVERY evidence failure still restores and restoration drift is a blocking failure',async()=>{
|
|
const p=deliveryProof();p.consumer.count=0
|
|
const calls=[],executor=createTestDeliveryFaultExecutor(environment,deliveryTransport(calls,p))
|
|
await executor.prepare(deliveryScope);await assert.rejects(executor.readProof(deliveryScope),/INBOX/)
|
|
assert.deepEqual(calls.slice(-3),['restore','health','finish'])
|
|
const broken=createTestDeliveryFaultExecutor(environment,deliveryTransport([],deliveryProof(),{proxyStopped:false}))
|
|
await broken.prepare(deliveryScope);await assert.rejects(broken.readProof(deliveryScope),/Delivery restoration/)
|
|
})
|
|
test('F40 hook faults only first signature, captures whole inventory and always cleans failed signing',async()=>{
|
|
const calls=[],restoration={restored:true,healthy:true,proxyStopped:true}
|
|
const inventory={scope:{...deliveryScope,warehouseId:'9',coverage:'CUSTOMER_WAREHOUSE_COMPLETE'},balances:[],positions:[],ledger:[]}
|
|
const parentProof={}
|
|
const context={context:{f40RootFulfillmentProof:parentProof,f40DeliveryEventFaultExecutor:{prepare:async s=>{assert.deepEqual(s,deliveryScope);calls.push('prepare')},
|
|
readProof:async()=>{calls.push('proof');return {...deliveryProof(),restored:true,restoration}},cleanup:async()=>{calls.push('cleanup');return restoration}},
|
|
f40ReadOnlyInventoryEvidence:async s=>{assert.equal(s.warehouseId,'9');calls.push('inventory');return structuredClone(inventory)}},createdResources:{phase3OrderSplitting:{}}}
|
|
assert.equal(await withDeliveryEventRecovery(context,deliveryScope,'9',0,async()=>{calls.push('sign');return 'signed'}),'signed')
|
|
assert.deepEqual(calls,['inventory','prepare','sign','proof','inventory','cleanup'])
|
|
assert.equal(parentProof.deliveryEventRecovery.status,'VERIFIED')
|
|
assert.equal(context.createdResources.phase5RootFulfillment,undefined)
|
|
await withDeliveryEventRecovery(context,deliveryScope,'9',1,async()=>calls.push('second'))
|
|
assert.equal(calls.at(-1),'second')
|
|
context.context.f40RootFulfillmentProof={}
|
|
await assert.rejects(withDeliveryEventRecovery(context,deliveryScope,'9',0,async()=>{throw Error('sign failed')}),/sign failed/)
|
|
assert.equal(calls.at(-1),'cleanup')
|
|
assert.equal(context.context.f40RootFulfillmentProof.deliveryEventRecovery.restored,true)
|
|
})
|
|
test('Python DELIVERY SQL uses actual payload, original Lastleg retry, and Order database without inventory queries',()=>{
|
|
const result=spawnSync('python',['-c',String.raw`
|
|
import importlib.util,sys,ast,pathlib,copy
|
|
from unittest.mock import patch,MagicMock,mock_open
|
|
sys.dont_write_bytecode=True
|
|
p=pathlib.Path('scripts/test-event-fault-runtime.py');ast.parse(p.read_text(encoding='utf-8'))
|
|
spec=importlib.util.spec_from_file_location('runtime',p);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
|
s={'marker':'TEST-DELIVERY','customerId':'7','deliveryId':'90','sourceInboundId':'200','sourceOrderId':'100','rootOrderId':'100'}
|
|
assert m.scope_directory(s,'DELIVERY')!=m.scope_directory({'marker':s['marker'],'customerId':'7','containerBookingId':'200'},'UNLOAD')
|
|
assert m.producer_name({'direction':'DELIVERY'})=='firstleg-test-lastleg'
|
|
assert m.target_setting({'direction':'DELIVERY'})==('FIRSTLEG_SERVICE_URL','http://firstleg:9202')
|
|
queries=[]
|
|
with patch.object(m,'inspect',side_effect=lambda name:{'Name':'/'+name}),patch.object(m,'require_test'),patch.object(m,'read_sql',side_effect=lambda container,sql:queries.append((container['Name'],sql)) or []):
|
|
assert m.producer(s,'DELIVERY') is None
|
|
m.delivery_facts(s,'00000000-0000-4000-8000-000000000001')
|
|
assert [q[0] for q in queries]==['/firstleg-test-lastleg','/firstleg-test-business','/firstleg-test-order']
|
|
assert "$.payload.deliveryOrderId" in queries[0][1] and "customer_id='7'" in queries[0][1]
|
|
assert 'fl_inbound_status_history' in queries[1][1] and 'fl_container_event_inbox' in queries[1][1]
|
|
assert 'ord_order_status_history' in queries[2][1] and 'FIRST_LEG_ORDER_FULFILLMENT_COMPLETED' in queries[2][1]
|
|
assert all('inventory' not in q[1] for q in queries)
|
|
original={'Name':'/firstleg-test-lastleg','Id':'original-container','Image':'sha256:'+'a'*64,
|
|
'Config':{'Image':'lastleg:test','Env':['FIRSTLEG_SERVICE_URL=http://firstleg:9202'],'Labels':{'com.docker.compose.project':'firstleg-test'}},
|
|
'HostConfig':{},'Mounts':[],'NetworkSettings':{'Networks':{'firstleg-test-network':{'IPAddress':'172.20.0.3','Aliases':['lastleg-service']}}},
|
|
'State':{'Running':True,'Health':{'Status':'healthy'}}}
|
|
consumer=copy.deepcopy(original);consumer['Name']='/firstleg-test-business'
|
|
with patch.object(m,'producer',return_value=None),patch.object(m,'inspect',side_effect=lambda name:original if name=='firstleg-test-lastleg' else consumer),patch.object(m.Path,'mkdir'),patch.object(m.Path,'write_bytes'),patch('builtins.open',mock_open()),patch.object(m.socket,'socket') as sock,patch.object(m.subprocess,'Popen') as process,patch.object(m,'private_write'):
|
|
sock.return_value.getsockname.return_value=('127.0.0.1',18001)
|
|
m.prepare(s,pathlib.Path('/offline'),{'direction':'DELIVERY','proxySource':''})
|
|
args=process.call_args.args[0]
|
|
assert args[args.index('--delivery-id')+1]=='90' and args[args.index('--event-type')+1]=='OverseasDeliveryCompleted'
|
|
assert '--container-booking-id' not in args and args[args.index('--upstream')+1].endswith(':9202/internal/firstleg/v1/container-events')
|
|
state={'direction':'DELIVERY','restored':False,'prepared':True,'image':'lastleg:test','originalId':'original-container',
|
|
'originalImageId':original['Image'],'originalFingerprint':m.fingerprint(original),'originalBody':m.recreation(original,m.env_of(original),full=True),'proxyPort':18001}
|
|
restored=copy.deepcopy(original);restored['Id']='restored-new-id'
|
|
with patch.object(m,'control',side_effect=OSError('proxy already stopped')),patch.object(m,'inspect',return_value=restored),patch.object(m,'private_write'),patch.object(m,'replace') as replace,patch.object(m,'docker') as docker:
|
|
assert m.restore(pathlib.Path('/offline'),state)=={'restoreStarted':True}
|
|
assert state['prepared'] is False
|
|
replace.assert_not_called();docker.assert_not_called()
|
|
state['prepared']=True
|
|
restored['Config']['Env']=['FIRSTLEG_SERVICE_URL=http://firstleg:9202','UNEXPECTED=1']
|
|
with patch.object(m,'control'),patch.object(m,'inspect',return_value=restored),patch.object(m,'replace') as replace:
|
|
try: m.restore(pathlib.Path('/offline'),state);assert False
|
|
except RuntimeError as error: assert str(error)=='CONCURRENT_PRODUCER_CHANGE'
|
|
replace.assert_not_called()
|
|
`],{encoding:'utf8',cwd:new URL('..',import.meta.url),timeout:10000})
|
|
assert.equal(result.status,0,result.stderr||result.error?.message)
|
|
})
|