test(fba): verify signed delivery ACK loss and original event recovery
This commit is contained in:
@@ -2,7 +2,8 @@ import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { createTestFaultExecutor, createTestUnloadFaultExecutor, assertTestUnloadFaultProof, renderTestFaultAction } from './test-event-fault-executor.mjs'
|
||||
import { createTestFaultExecutor, createTestUnloadFaultExecutor, createTestDeliveryFaultExecutor, assertTestDeliveryFaultProof, assertTestUnloadFaultProof, renderTestFaultAction } from './test-event-fault-executor.mjs'
|
||||
import { withDeliveryEventRecovery } from './admin-test-env-modules/phase5-root-fulfillment.mjs'
|
||||
|
||||
const scope = { marker: 'TEST-ROOT', customerId: '7', loadingPlanId: '99', containerBookingId: '66' }
|
||||
const environment = { CHONGCHONG_ENV: 'test', JENKINS_URL: 'https://jenkins.example.invalid/',
|
||||
@@ -219,3 +220,137 @@ test('failed activation retains task ownership for explicit restoration, never r
|
||||
assert.equal(calls.filter(name => name === 'activate').length, 1)
|
||||
assert.deepEqual(calls.slice(-3), ['restore', 'health', 'finish'])
|
||||
})
|
||||
|
||||
const deliveryScope={marker:'TEST-DELIVERY',customerId:'7',deliveryId:'90',sourceInboundId:'200',sourceOrderId:'100',rootOrderId:'100'}
|
||||
function deliveryProof(mode='ROOT_PENDING') {
|
||||
const scope={...deliveryScope,...(mode==='CHILD_FIRST'?{sourceOrderId:'101',sourceInboundId:'201'}:{})}
|
||||
const eventId='00000000-0000-4000-8000-000000000001', entryId='123'
|
||||
const payload={deliveryOrderId:'90',deliveryNo:'DL-OFFLINE',lastlegOrderId:'300',lastlegOrderNo:'LL-OFFLINE',
|
||||
sourceInboundId:scope.sourceInboundId,inboundCode:'IN-OFFLINE',sourceOrderId:scope.sourceOrderId,
|
||||
sourceOrderNo:'FL-OFFLINE',rootOrderId:'100',fbaWarehouseCode:'ABE4',completedAt:'2026-10-03T01:00:00Z'}
|
||||
const payloadHash='sha256:'+createHash('sha256').update(JSON.stringify(payload,Object.keys(payload).sort())).digest('hex')
|
||||
const envelope={eventId,eventType:'OverseasDeliveryCompleted',eventVersion:1,tenantId:'7',aggregateType:'INBOUND_ORDER',aggregateId:scope.sourceInboundId,aggregateVersion:5,payloadHash,payload}
|
||||
const producer={entryId,eventId,eventType:envelope.eventType,customerId:'7',aggregateId:scope.sourceInboundId,aggregateType:'INBOUND_ORDER',eventKey:`OverseasDeliveryCompleted:DL-OFFLINE:${scope.sourceInboundId}`,envelope,payloadHash,publishStatus:'PUBLISHED'}
|
||||
const ownRoot=mode!=='CHILD_FIRST', childComplete=mode!=='ROOT_PENDING', rootComplete=mode==='ALL_COMPLETE'
|
||||
const orders=['100','101'].map(id=>({id,rootOrderId:'100',customerId:'7',orderRole:id==='100'?'MASTER':'CHILD',
|
||||
orderNo:id===scope.sourceOrderId?'FL-OFFLINE':'OTHER',inboundCode:id===scope.sourceOrderId?'IN-OFFLINE':'OTHER',
|
||||
status:(id==='100'?rootComplete:childComplete)?'COMPLETED':'APPROVED',version:3,
|
||||
completedAt:(id==='100'?rootComplete:childComplete)?'2026-10-03 01:00:00':null}))
|
||||
const consumer={eventId,eventType:envelope.eventType,aggregateId:scope.sourceInboundId,payloadHash,payload,status:'PROCESSED'}
|
||||
const business={inbound:[{id:scope.sourceInboundId,customerId:'7',orderId:scope.sourceOrderId,orderNo:'FL-OFFLINE',inboundCode:'IN-OFFLINE',processStatus:'DELIVERED',orderStatusSnapshot:orders.find(o=>o.id===scope.sourceOrderId).status,version:2}],
|
||||
inboundHistory:[{id:'41',inboundId:scope.sourceInboundId,action:'COMPLETE_DELIVERY',fromStatus:'LOADED',toStatus:'DELIVERED',occurredAt:'2026-10-03 01:00:00'}],
|
||||
inbox:[{...consumer,processedAt:'2026-10-03 01:00:01'}],order:orders,
|
||||
orderHistory:[...(ownRoot?[{id:'51',orderId:'100',action:'OVERSEAS_DELIVERED',occurredAt:'2026-10-03 01:00:00'}]:[]),...(childComplete?[{id:'52',orderId:'101',action:'OVERSEAS_DELIVERED',occurredAt:'2026-10-03 01:00:00'}]:[]),...(rootComplete?[{id:'53',orderId:'100',action:'FULFILLMENT_COMPLETED'}]:[])],
|
||||
completionEvent:orders.filter(o=>o.status==='COMPLETED').map((o,i)=>({id:String(60+i),orderId:o.id,eventType:'FIRST_LEG_ORDER_FULFILLMENT_COMPLETED',status:'PUBLISHED',payload:{orderId:o.id}}))}
|
||||
const old=unloadProof()
|
||||
return JSON.parse(JSON.stringify({...scope,readOnly:true,producer,consumer:{...consumer,count:1},
|
||||
beforeRetry:{...producer,publishStatus:'DEAD',failedAttempts:12,business},afterRetry:{...producer,business},
|
||||
attempts:old.attempts,proxyRecords:old.proxyRecords.map(p=>({...p,payloadHash})),retryAudit:{...old.retryAudit,payloadHash},retryAuditCount:1,retryRequestId:old.retryRequestId}))
|
||||
}
|
||||
function deliveryTransport(calls, proof=deliveryProof(), finish={}) {
|
||||
let recovered=false
|
||||
return async(_url,options)=>{
|
||||
const {action,direction,scope}=requestedAction(options)
|
||||
assert.equal(direction,'DELIVERY');assert.deepEqual(scope,deliveryScope);calls.push(action)
|
||||
if(action==='recover')recovered=true
|
||||
const data=action==='prepare'?{originalImageId,originalFingerprint}:action==='proxy-ready'?{ready:true}:action==='health'?{healthy:true}
|
||||
:action==='status'?{producer:{publishStatus:recovered?'PUBLISHED':'DEAD',failedAttempts:12}}:action==='proof'?proof
|
||||
:action==='finish'?{restored:true,healthy:true,proxyStopped:true,producerName:'firstleg-test-lastleg',originalImageId,currentImageId:originalImageId,originalFingerprint,currentFingerprint:originalFingerprint,...finish}:{}
|
||||
return {ok:true,text:async()=> 'TEST_EVENT_JSON='+JSON.stringify({ok:true,data})}
|
||||
}
|
||||
}
|
||||
test('DELIVERY has a separate strict scope and restores before accepting original event proof',async()=>{
|
||||
const calls=[],executor=createTestDeliveryFaultExecutor(environment,deliveryTransport(calls))
|
||||
await assert.rejects(executor.prepare({...deliveryScope,eventType:'OverseasContainerUnloaded'}),/Invalid/)
|
||||
await executor.prepare(deliveryScope)
|
||||
await assert.rejects(executor.prepare({...deliveryScope,deliveryId:'91'}),/scope drift/)
|
||||
const proof=await executor.readProof(deliveryScope)
|
||||
assert.equal(proof.restored,true);assert.equal(proof.restoration.currentImageId,originalImageId)
|
||||
assert.deepEqual(calls,['prepare','proxy-ready','activate','health','status','recover','status','proof','restore','health','finish'])
|
||||
})
|
||||
test('DELIVERY proof distinguishes root pending, child first and all completed with immutable original evidence',()=>{
|
||||
for(const mode of ['ROOT_PENDING','CHILD_FIRST','ALL_COMPLETE']) {
|
||||
const p=deliveryProof(mode),scope={...deliveryScope,sourceOrderId:p.sourceOrderId,sourceInboundId:p.sourceInboundId}
|
||||
assertTestDeliveryFaultProof(p,scope)
|
||||
}
|
||||
for(const mutate of [p=>p.producer.envelope.payload.deliveryId='90',p=>delete p.producer.envelope.payload.deliveryOrderId,
|
||||
p=>p.producer.customerId='0',p=>p.producer.eventKey='wrong',p=>p.beforeRetry.entryId='124',
|
||||
p=>p.consumer.count=2,p=>p.attempts.pop(),p=>p.proxyRecords[0].outcome='DUPLICATE',
|
||||
p=>p.retryAuditCount=2,p=>p.afterRetry.business.inbound[0].version++,p=>p.afterRetry.business.order[0].version++,
|
||||
p=>p.beforeRetry.business.inbox[0].status='RECEIVED',p=>p.beforeRetry.business.inboundHistory.push({...p.beforeRetry.business.inboundHistory[0]}),
|
||||
p=>{p.beforeRetry.business.order[0].status=p.afterRetry.business.order[0].status='COMPLETED'},
|
||||
p=>p.beforeRetry.business.completionEvent.push({orderId:'100',status:'PUBLISHED'})]) {
|
||||
const proof=deliveryProof();mutate(proof);assert.throws(()=>assertTestDeliveryFaultProof(proof,deliveryScope),/Invalid delivery/)
|
||||
}
|
||||
})
|
||||
test('DELIVERY evidence failure still restores and restoration drift is a blocking failure',async()=>{
|
||||
const p=deliveryProof();p.consumer.count=0
|
||||
const calls=[],executor=createTestDeliveryFaultExecutor(environment,deliveryTransport(calls,p))
|
||||
await executor.prepare(deliveryScope);await assert.rejects(executor.readProof(deliveryScope),/INBOX/)
|
||||
assert.deepEqual(calls.slice(-3),['restore','health','finish'])
|
||||
const broken=createTestDeliveryFaultExecutor(environment,deliveryTransport([],deliveryProof(),{proxyStopped:false}))
|
||||
await broken.prepare(deliveryScope);await assert.rejects(broken.readProof(deliveryScope),/Delivery restoration/)
|
||||
})
|
||||
test('F40 hook faults only first signature, captures whole inventory and always cleans failed signing',async()=>{
|
||||
const calls=[],restoration={restored:true,healthy:true,proxyStopped:true}
|
||||
const inventory={scope:{...deliveryScope,warehouseId:'9',coverage:'CUSTOMER_WAREHOUSE_COMPLETE'},balances:[],positions:[],ledger:[]}
|
||||
const context={context:{f40DeliveryEventFaultExecutor:{prepare:async s=>{assert.deepEqual(s,deliveryScope);calls.push('prepare')},
|
||||
readProof:async()=>{calls.push('proof');return {...deliveryProof(),restored:true,restoration}},cleanup:async()=>{calls.push('cleanup');return restoration}},
|
||||
f40ReadOnlyInventoryEvidence:async s=>{assert.equal(s.warehouseId,'9');calls.push('inventory');return structuredClone(inventory)}},createdResources:{phase5RootFulfillment:{}}}
|
||||
assert.equal(await withDeliveryEventRecovery(context,deliveryScope,'9',0,async()=>{calls.push('sign');return 'signed'}),'signed')
|
||||
assert.deepEqual(calls,['inventory','prepare','sign','proof','inventory','cleanup'])
|
||||
assert.equal(context.createdResources.phase5RootFulfillment.deliveryEventRecovery.status,'VERIFIED')
|
||||
await withDeliveryEventRecovery(context,deliveryScope,'9',1,async()=>calls.push('second'))
|
||||
assert.equal(calls.at(-1),'second')
|
||||
context.createdResources.phase5RootFulfillment={}
|
||||
await assert.rejects(withDeliveryEventRecovery(context,deliveryScope,'9',0,async()=>{throw Error('sign failed')}),/sign failed/)
|
||||
assert.equal(calls.at(-1),'cleanup')
|
||||
assert.equal(context.createdResources.phase5RootFulfillment.deliveryEventRecovery.restored,true)
|
||||
})
|
||||
test('Python DELIVERY SQL uses actual payload, original Lastleg retry, and Order database without inventory queries',()=>{
|
||||
const result=spawnSync('python',['-c',String.raw`
|
||||
import importlib.util,sys,ast,pathlib,copy
|
||||
from unittest.mock import patch,MagicMock,mock_open
|
||||
sys.dont_write_bytecode=True
|
||||
p=pathlib.Path('scripts/test-event-fault-runtime.py');ast.parse(p.read_text(encoding='utf-8'))
|
||||
spec=importlib.util.spec_from_file_location('runtime',p);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
s={'marker':'TEST-DELIVERY','customerId':'7','deliveryId':'90','sourceInboundId':'200','sourceOrderId':'100','rootOrderId':'100'}
|
||||
assert m.scope_directory(s,'DELIVERY')!=m.scope_directory({'marker':s['marker'],'customerId':'7','containerBookingId':'200'},'UNLOAD')
|
||||
assert m.producer_name({'direction':'DELIVERY'})=='firstleg-test-lastleg'
|
||||
assert m.target_setting({'direction':'DELIVERY'})==('FIRSTLEG_SERVICE_URL','http://firstleg:9202')
|
||||
queries=[]
|
||||
with patch.object(m,'inspect',side_effect=lambda name:{'Name':'/'+name}),patch.object(m,'require_test'),patch.object(m,'read_sql',side_effect=lambda container,sql:queries.append((container['Name'],sql)) or []):
|
||||
assert m.producer(s,'DELIVERY') is None
|
||||
m.delivery_facts(s,'00000000-0000-4000-8000-000000000001')
|
||||
assert [q[0] for q in queries]==['/firstleg-test-lastleg','/firstleg-test-business','/firstleg-test-order']
|
||||
assert "$.payload.deliveryOrderId" in queries[0][1] and "customer_id='7'" in queries[0][1]
|
||||
assert 'fl_inbound_status_history' in queries[1][1] and 'fl_container_event_inbox' in queries[1][1]
|
||||
assert 'ord_order_status_history' in queries[2][1] and 'FIRST_LEG_ORDER_FULFILLMENT_COMPLETED' in queries[2][1]
|
||||
assert all('inventory' not in q[1] for q in queries)
|
||||
original={'Name':'/firstleg-test-lastleg','Id':'original-container','Image':'sha256:'+'a'*64,
|
||||
'Config':{'Image':'lastleg:test','Env':['FIRSTLEG_SERVICE_URL=http://firstleg:9202'],'Labels':{'com.docker.compose.project':'firstleg-test'}},
|
||||
'HostConfig':{},'Mounts':[],'NetworkSettings':{'Networks':{'firstleg-test-network':{'IPAddress':'172.20.0.3','Aliases':['lastleg-service']}}},
|
||||
'State':{'Running':True,'Health':{'Status':'healthy'}}}
|
||||
consumer=copy.deepcopy(original);consumer['Name']='/firstleg-test-business'
|
||||
with patch.object(m,'producer',return_value=None),patch.object(m,'inspect',side_effect=lambda name:original if name=='firstleg-test-lastleg' else consumer),patch.object(m.Path,'mkdir'),patch.object(m.Path,'write_bytes'),patch('builtins.open',mock_open()),patch.object(m.socket,'socket') as sock,patch.object(m.subprocess,'Popen') as process,patch.object(m,'private_write'):
|
||||
sock.return_value.getsockname.return_value=('127.0.0.1',18001)
|
||||
m.prepare(s,pathlib.Path('/offline'),{'direction':'DELIVERY','proxySource':''})
|
||||
args=process.call_args.args[0]
|
||||
assert args[args.index('--delivery-id')+1]=='90' and args[args.index('--event-type')+1]=='OverseasDeliveryCompleted'
|
||||
assert '--container-booking-id' not in args and args[args.index('--upstream')+1].endswith(':9202/internal/firstleg/v1/container-events')
|
||||
state={'direction':'DELIVERY','restored':False,'prepared':True,'image':'lastleg:test','originalId':'original-container',
|
||||
'originalImageId':original['Image'],'originalFingerprint':m.fingerprint(original),'originalBody':m.recreation(original,m.env_of(original),full=True),'proxyPort':18001}
|
||||
restored=copy.deepcopy(original);restored['Id']='restored-new-id'
|
||||
with patch.object(m,'control',side_effect=OSError('proxy already stopped')),patch.object(m,'inspect',return_value=restored),patch.object(m,'private_write'),patch.object(m,'replace') as replace,patch.object(m,'docker') as docker:
|
||||
assert m.restore(pathlib.Path('/offline'),state)=={'restoreStarted':True}
|
||||
assert state['prepared'] is False
|
||||
replace.assert_not_called();docker.assert_not_called()
|
||||
state['prepared']=True
|
||||
restored['Config']['Env']=['FIRSTLEG_SERVICE_URL=http://firstleg:9202','UNEXPECTED=1']
|
||||
with patch.object(m,'control'),patch.object(m,'inspect',return_value=restored),patch.object(m,'replace') as replace:
|
||||
try: m.restore(pathlib.Path('/offline'),state);assert False
|
||||
except RuntimeError as error: assert str(error)=='CONCURRENT_PRODUCER_CHANGE'
|
||||
replace.assert_not_called()
|
||||
`],{encoding:'utf8',cwd:new URL('..',import.meta.url),timeout:10000})
|
||||
assert.equal(result.status,0,result.stderr||result.error?.message)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user