import assert from 'node:assert/strict' import test from 'node:test' import { createHash } from 'node:crypto' import { spawnSync } from 'node:child_process' import { createTestFeeVoidFaultExecutor } from './test-event-fault-executor.mjs' import { assertTestFeeVoidFaultProof } from './test-fee-void-fault-proof.mjs' const scope={marker:'FEE-VOID-OFFLINE',customerId:'7',sourceFeeId:'123',sourceVersion:'1',sourceType:'OVERSEAS_WAREHOUSE_FEE'} const env={CHONGCHONG_ENV:'test',JENKINS_URL:'https://jenkins.example.invalid/',JENKINS_USER:'offline',JENKINS_API_TOKEN:'offline',FBA_CORE_JENKINS_AGENT:'java-test-agent'} const sort=value=>Array.isArray(value)?value.map(sort):value&&typeof value==='object'?Object.fromEntries(Object.keys(value).sort().map(key=>[key,sort(value[key])])):value const text=value=>JSON.stringify(sort(value)), hash=value=>'sha256:'+createHash('sha256').update(value).digest('hex') const clone=value=>JSON.parse(JSON.stringify(value)) const imageId='sha256:'+'a'.repeat(64),fingerprint='b'.repeat(64) const restoration={restored:true,healthy:true,proxyStopped:true,producerName:'firstleg-test-lastleg',originalImageId:imageId,currentImageId:imageId,originalFingerprint:fingerprint,currentFingerprint:fingerprint} function fixture() { const base=Object.fromEntries(Object.entries({action:null,sourceSystem:'LASTLEG',sourceType:scope.sourceType,sourceFeeId:'123',sourceVersion:1,customerId:'7',customerName:'Offline', sourceNo:'WH-OFFLINE',businessType:'WAREHOUSE_SERVICE',businessId:'200',businessNo:'WH-OFFLINE',orderId:null,inboundId:null,inboundCode:null, containerId:null,containerBookingId:null,containerNo:null,warehouseId:'9',feeItemCode:'TEST',feeItemName:'Offline fee',direction:'RECEIVABLE', supplierId:null,supplierName:null,supplierCategory:null,settlementMode:null,paidAt:null,paymentMethod:null,paymentReferenceNo:null,paymentVoucherFileIds:[], businessDate:'2026-10-03',currency:'CNY',totalAmount:7.25,originalCurrency:'CNY',originalAmount:7.25,exchangeRate:1,exchangeRateMonth:'2026-10', chargeableQuantity:null,quantityUnit:null,unitPrice:null,periodStart:null,periodEnd:null,reviewStatus:'APPROVED',publicationStatus:'PUBLISHED',sourceStatus:'ACTIVE',sourceHash:null}).filter(([,value])=>value!==null)) const sourceCanonicalJson=text(base),sourceHash=hash(sourceCanonicalJson) const event=(action,index)=>{ const payload={...base,action,sourceStatus:action==='VOIDED'?'VOIDED':'ACTIVE',sourceHash} const payloadCanonicalJson=text(payload),payloadHash=hash(payloadCanonicalJson) const eventId=`00000000-0000-4000-8000-00000000000${index}` const envelope={eventId,eventType:'BusinessFeeSourceChanged',eventVersion:1,tenantId:'7',aggregateType:scope.sourceType,aggregateId:'123',aggregateVersion:1,payload,payloadHash} return {entryId:String(index),eventId,payloadHash,eventType:envelope.eventType,customerId:'7',aggregateId:'123',aggregateType:scope.sourceType, eventKey:`BusinessFeeSourceChanged:LASTLEG:${scope.sourceType}:123:1:${action}`,envelope,envelopeJson:JSON.stringify(envelope),payloadCanonicalJson,sourceCanonicalJson,publishStatus:'PUBLISHED',failedAttempts:0} } const publication=event('PUBLISHED',1),producer=event('VOIDED',2) const local={id:'500',sourceFeeId:'123',sourceVersion:'1',customerId:'7',sourceType:scope.sourceType,sourceStatus:'ACTIVE',reviewStatus:'APPROVED',publicationStatus:'PUBLISHED',direction:'RECEIVABLE',amount:'7.250000',businessType:'WAREHOUSE_SERVICE',businessId:'200',reviewReason:'approved',updatedBy:'7',updatedAt:'2026-10-03 01:00:00'} const finance={id:'600',sourceSystem:'LASTLEG',sourceFeeId:'123',sourceFeeVersion:'1',customerId:'7',sourceType:scope.sourceType,direction:'RECEIVABLE',currencyCode:'CNY',amount:'7.250000',walletHoldId:null,operationVersion:'0',status:'AVAILABLE',sourceHash,sourceSnapshot:publication.envelope.payload,updatedAt:'2026-10-03 01:00:01'} const inbox=(row,id)=>({id,eventId:row.eventId,sourceSystem:'LASTLEG',eventType:'BusinessFeeSourceChanged',envelope:row.envelope,payloadHash:row.payloadHash,resultResourceId:'600',status:'PROCESSED',errorCode:null,receivedAt:'2026-10-03 01:00:01',processedAt:'2026-10-03 01:00:01'}) const initial={lastleg:[local],finance:{source:[finance],bill:[],settlement:[],cost:[],inbox:[inbox(publication,'700')]}} const dead=clone(initial);dead.lastleg[0].reviewReason='void reason';dead.lastleg[0].updatedAt='2026-10-03 01:01:00' Object.assign(dead.finance.source[0],{status:'VOIDED',operationVersion:'1',updatedAt:'2026-10-03 01:01:01'});dead.finance.inbox=[inbox(producer,'701')] const after=clone(dead);Object.assign(after.lastleg[0],{sourceStatus:'VOIDED',updatedAt:'2026-10-03 01:02:00'}) const prior={ready:true,publication,business:initial,attempts:[{entryId:'1',eventId:publication.eventId,outcome:'APPLIED'}]} const requestId='00000000-0000-4000-8000-000000000003' return clone({...scope,readOnly:true,producer,prerequisite:prior,publicationNow:publication, beforeRetry:{...producer,publishStatus:'DEAD',failedAttempts:12,fee:dead},afterRetry:{...producer,fee:after}, consumer:{...dead.finance.inbox[0],count:1}, attempts:Array.from({length:13},(_,i)=>({attemptId:`a${i}`,entryId:'2',eventId:producer.eventId,serviceName:'lastleg',outcome:i<11?'RETRY':i===11?'DEAD':'DUPLICATE'})), proxyRecords:Array.from({length:13},(_,i)=>({matched:true,attempt:i+1,eventId:producer.eventId,payloadHash:producer.payloadHash,outcome:i===0?'APPLIED':'DUPLICATE',errorCode:i<12?'ACK_LOSS_INJECTED':null})), retryAuditCount:1,retryRequestId:requestId,retryAudit:{requestId,entryId:'2',eventId:producer.eventId,payloadHash:producer.payloadHash,previousFailures:12}}) } function config(options) { const chunks=[...options.body.get('script').matchAll(/'([A-Za-z0-9+/=]+)'/g)].map(m=>m[1]).join('') const groovy=Buffer.from(chunks,'base64').toString('utf8'),encoded=groovy.match(/'UTF-8'\),'([A-Za-z0-9+/=]+)'\]\)/)?.[1] return JSON.parse(Buffer.from(encoded,'base64').toString('utf8')) } function transport(calls,{pending=false,preflightFailure=false,proof=fixture(),finish={}}={}) { let recovered=false,preflightCount=0 return async(_url,options)=>{ const request=config(options);assert.equal(request.direction,'FEE_VOID');assert.deepEqual(request.scope,scope) const action=request.action;calls.push(action) if(action==='recover')recovered=true if(action==='preflight')preflightCount++ const data=action==='preflight'?{ready:!pending||preflightCount>1}:action==='prepare'?{originalImageId:imageId,originalFingerprint:fingerprint,prerequisite:proof.prerequisite} :action==='proxy-ready'?{ready:true}:action==='health'?{healthy:true}:action==='status'?{producer:{publishStatus:recovered?'PUBLISHED':'DEAD',failedAttempts:12}} :action==='proof'?proof:action==='finish'?{...restoration,...finish}:{} const result=preflightFailure&&action==='preflight'?{ok:false,error:'FEE_PUBLICATION_NOT_READY'}:{ok:true,data} return {ok:true,text:async()=> 'TEST_EVENT_JSON='+JSON.stringify(result)} } } test('VOID proof accepts Finance VOIDED at DEAD while Lastleg remains ACTIVE until matched retry ACK',()=>{ const proof=fixture();assertTestFeeVoidFaultProof(proof,scope) assert.equal(proof.beforeRetry.fee.lastleg[0].sourceStatus,'ACTIVE') assert.equal(proof.beforeRetry.fee.finance.source[0].status,'VOIDED') assert.equal(proof.afterRetry.fee.lastleg[0].sourceStatus,'VOIDED') }) test('VOID proof rejects premature local confirmation, extra financial writes, altered source and lost original event evidence',()=>{ const mutations=[p=>p.beforeRetry.fee.lastleg[0].sourceStatus='VOIDED',p=>p.afterRetry.fee.lastleg[0].sourceStatus='ACTIVE', p=>p.afterRetry.fee.lastleg[0].amount='8.250000',p=>p.afterRetry.fee.lastleg[0].sourceVersion='2', p=>p.afterRetry.fee.finance.source[0].operationVersion='2',p=>p.beforeRetry.fee.finance.source[0].status='AVAILABLE', p=>p.prerequisite.publication.publishStatus='PENDING',p=>p.prerequisite.business.finance.source[0].walletHoldId='99', p=>p.afterRetry.fee.finance.bill.push({id:'123'}),p=>p.consumer.resultResourceId='601', p=>p.producer.envelope.payload.sourceFeeId='124',p=>p.beforeRetry.entryId='9',p=>p.retryAuditCount=2, p=>p.proxyRecords[0].outcome='DUPLICATE',p=>p.attempts.pop(),p=>p.producer.sourceCanonicalJson='{}', p=>p.prerequisite.business.lastleg.push({...p.prerequisite.business.lastleg[0],id:'501',sourceVersion:'2'})] for(const mutate of mutations){const proof=fixture();mutate(proof);assert.throws(()=>assertTestFeeVoidFaultProof(proof,scope),/Invalid fee void/)} }) test('preflight waits for original publish ACK before a single prepare; restoration precedes successful proof',async()=>{ const calls=[],executor=createTestFeeVoidFaultExecutor(env,transport(calls,{pending:true})) for(const invalid of [{...scope,sourceVersion:1},{...scope,sourceType:'UNKNOWN'},{...scope,sourceVersion:'2147483648'},{...scope,action:'VOIDED'}]) await assert.rejects(executor.prepare(invalid),/Invalid/) await executor.prepare(scope) const proof=await executor.readProof(scope);assert.equal(proof.restored,true) assert.deepEqual(calls,['preflight','preflight','prepare','proxy-ready','activate','health','status','recover','status','proof','restore','health','finish']) await executor.cleanup();assert.equal(calls.filter(a=>a==='prepare').length,1) }) test('preflight failure never mutates environment and cannot silently reuse an unprepared executor',async()=>{ const calls=[],executor=createTestFeeVoidFaultExecutor(env,transport(calls,{preflightFailure:true})) await assert.rejects(executor.prepare(scope),/FEE_PUBLICATION_NOT_READY/) assert.deepEqual(await executor.cleanup(),{attempted:false});await assert.rejects(executor.prepare(scope),/did not complete/) assert.deepEqual(calls,['preflight']) }) test('proof failure restores; missing original configuration or proxy stop remains a blocker',async()=>{ const proof=fixture();proof.afterRetry.fee.finance.source[0].operationVersion='2' const calls=[],executor=createTestFeeVoidFaultExecutor(env,transport(calls,{proof})) await executor.prepare(scope);await assert.rejects(executor.readProof(scope),/FINANCE_CHANGED/) assert.deepEqual(calls.slice(-3),['restore','health','finish']) for(const finish of [{proxyStopped:false},{currentFingerprint:'c'.repeat(64)}]) { const failed=createTestFeeVoidFaultExecutor(env,transport([],{finish})) await failed.prepare(scope);await assert.rejects(failed.readProof(scope),/Fee void restoration/) } }) test('unknown prepare result is never reissued and cleanup still attempts original restoration',async()=>{ const calls=[],base=transport(calls) const executor=createTestFeeVoidFaultExecutor(env,async(url,options)=>{ if(config(options).action==='prepare'){calls.push('prepare');throw Error('response lost')} return base(url,options) }) await assert.rejects(executor.prepare(scope),/response lost/) await assert.rejects(executor.prepare(scope),/did not complete/) await assert.rejects(executor.cleanup(),/Fee void restoration evidence mismatch/) assert.deepEqual(calls,['preflight','prepare','restore','health','finish']) }) test('NON_NULL source hashes agree across runtime and proof and reject explicit null unsigned fields',()=>{ const proof=fixture(),publication=proof.prerequisite.publication const unsigned=JSON.parse(publication.sourceCanonicalJson) assert.equal(Object.hasOwn(unsigned,'action'),false) assert.equal(Object.hasOwn(unsigned,'sourceHash'),false) assert.equal(Object.hasOwn(unsigned,'orderId'),false) assert.deepEqual(unsigned.paymentVoucherFileIds,[]) const invalid=clone(proof) invalid.producer.sourceCanonicalJson=text({...unsigned,action:null,sourceHash:null}) invalid.producer.envelope.payload.sourceHash=hash(invalid.producer.sourceCanonicalJson) invalid.producer.payloadCanonicalJson=text(invalid.producer.envelope.payload) invalid.producer.payloadHash=hash(invalid.producer.payloadCanonicalJson) invalid.producer.envelope.payloadHash=invalid.producer.payloadHash invalid.producer.envelopeJson=JSON.stringify(invalid.producer.envelope) assert.throws(()=>assertTestFeeVoidFaultProof(invalid,scope),/SOURCE_HASH/) const result=spawnSync(process.env.PYTHON || 'python',['-c',String.raw` import sys,importlib.util,json,copy,hashlib from unittest.mock import patch sys.dont_write_bytecode=True spec=importlib.util.spec_from_file_location('fault','scripts/test-event-fault-runtime.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) data=json.load(sys.stdin);scope=data['scope'];row=data['row'] with patch.object(m,'inspect'),patch.object(m,'read_sql',side_effect=lambda *args:[copy.deepcopy(row)]): result=m.fee_event(scope,'PUBLISHED') assert result['sourceCanonicalJson']==row['sourceCanonicalJson'] assert result['payloadCanonicalJson']==row['payloadCanonicalJson'] payload=json.loads(row['envelopeJson'])['payload'] unsigned={**payload,'action':None,'sourceHash':None,'sourceStatus':'ACTIVE'} payload['sourceHash']='sha256:'+hashlib.sha256(m.fee_canonical(unsigned).encode()).hexdigest() envelope=json.loads(row['envelopeJson']);envelope['payload']=payload row['envelopeJson']=json.dumps(envelope) row['payloadHash']='sha256:'+hashlib.sha256(m.fee_canonical(payload).encode()).hexdigest() try:m.fee_event(scope,'PUBLISHED');assert False except RuntimeError as e:assert str(e)=='FEE_SOURCE_HASH_MISMATCH' row['payloadHash']='sha256:'+'0'*64 try:m.fee_event(scope,'PUBLISHED');assert False except RuntimeError as e:assert str(e)=='FEE_PAYLOAD_HASH_MISMATCH' `],{cwd:new URL('..',import.meta.url),input:JSON.stringify({scope,row:publication}),encoding:'utf8',timeout:10000}) assert.equal(result.status,0,result.stderr||result.error?.message) }) test('Python FEE_VOID routing, exact decimal hashes, preflight gating and restore reentry are offline',()=>{ const result=spawnSync(process.env.PYTHON || 'python',['-c',String.raw` import sys,importlib.util,pathlib,copy,decimal,ast,io from unittest.mock import patch,mock_open sys.dont_write_bytecode=True p=pathlib.Path('scripts/test-event-fault-runtime.py');ast.parse(p.read_text(encoding='utf-8')) spec=importlib.util.spec_from_file_location('fault',p);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) s={'marker':'FEE-VOID-OFFLINE','customerId':'7','sourceFeeId':'123','sourceVersion':'1','sourceType':'OVERSEAS_WAREHOUSE_FEE'} assert m.target_setting({'direction':'FEE_VOID'})==('FINANCE_SERVICE_URL','http://finance:9203') assert m.target_setting({'direction':'DELIVERY'})==('FIRSTLEG_SERVICE_URL','http://firstleg:9202') assert m.producer_name({'direction':'FEE_VOID'})=='firstleg-test-lastleg' assert m.fee_canonical(decimal.Decimal('100.000000'))=='1E+2' assert m.fee_canonical(decimal.Decimal('0.00000010'))=='1E-7' assert m.fee_canonical(decimal.Decimal('12345678901234.123456'))=='12345678901234.123456' assert m.fee_canonical(decimal.Decimal('-0.000'))=='0' queries=[] with patch.object(m,'inspect',side_effect=lambda name:{'Name':'/'+name}),patch.object(m,'require_test'),patch.object(m,'read_sql',side_effect=lambda c,sql:queries.append((c['Name'],sql)) or []): assert m.producer(s,'FEE_VOID') is None m.fee_facts(s,'00000000-0000-4000-8000-000000000002') assert [r[0] for r in queries]==['/firstleg-test-lastleg','/firstleg-test-lastleg','/firstleg-test-finance','/firstleg-test-finance'] assert 'BusinessFeeSourceChanged:LASTLEG:OVERSEAS_WAREHOUSE_FEE:123:1:VOIDED' in queries[0][1] assert 'void_status' not in queries[1][1] and 'source_status' in queries[1][1] assert all(t in queries[2][1] for t in ['fin_fee_source','fin_bill_line','fin_settlement_line','fin_cost','operation_version']) assert 'fin_inbox_event' in queries[3][1] and 'result_resource_id' in queries[3][1] publication={'entryId':'1','eventId':'00000000-0000-4000-8000-000000000001','publishStatus':'PENDING','payloadHash':'sha256:'+'a'*64} facts={'lastleg':[{'sourceVersion':'1','sourceType':s['sourceType'],'customerId':'7','sourceStatus':'ACTIVE','reviewStatus':'APPROVED','publicationStatus':'PUBLISHED','direction':'RECEIVABLE'}], 'finance':{'source':[{'id':'600','sourceFeeVersion':'1','sourceType':s['sourceType'],'customerId':'7','status':'AVAILABLE','walletHoldId':None}], 'bill':[],'settlement':[],'cost':[],'inbox':[{'status':'PROCESSED','payloadHash':publication['payloadHash'],'resultResourceId':'600'}]}} with patch.object(m,'fee_event',side_effect=lambda scope,action:None if action=='VOIDED' else publication),patch.object(m,'fee_facts',return_value=facts),patch.object(m,'inspect'),patch.object(m,'read_sql',return_value=[{'eventId':publication['eventId'],'outcome':'APPLIED'}]): assert m.fee_prerequisite(s,wait=True)['ready'] is False publication['publishStatus']='PUBLISHED';assert m.fee_prerequisite(s,wait=True)['ready'] is True publication['publishStatus']='DEAD' try:m.fee_prerequisite(s,wait=True);assert False except RuntimeError as e:assert str(e)=='FEE_PUBLICATION_NOT_READY' original={'Name':'/firstleg-test-lastleg','Id':'original','Image':'sha256:'+'a'*64,'Config':{'Image':'lastleg:test','Env':['FINANCE_SERVICE_URL=http://finance:9203','FIRSTLEG_SERVICE_URL=http://firstleg:9202'],'Labels':{'com.docker.compose.project':'firstleg-test'}},'HostConfig':{},'Mounts':[], 'NetworkSettings':{'Networks':{'firstleg-test-network':{'IPAddress':'172.20.0.3','Aliases':['lastleg-service']}}},'State':{'Running':True,'Health':{'Status':'healthy'}}} finance=copy.deepcopy(original);finance['Name']='/firstleg-test-finance';finance['NetworkSettings']['Networks']['firstleg-test-network']['IPAddress']='172.20.0.9' with patch.object(m,'fee_prerequisite',return_value={'ready':True}),patch.object(m,'inspect',side_effect=lambda name:finance if name=='firstleg-test-finance' else original),patch.object(m.Path,'mkdir'),patch.object(m.Path,'write_bytes'),patch('builtins.open',mock_open()),patch.object(m.socket,'socket') as sock,patch.object(m.subprocess,'Popen') as process,patch.object(m,'private_write'): sock.return_value.getsockname.return_value=('127.0.0.1',18001) m.prepare(s,pathlib.Path('/offline'),{'direction':'FEE_VOID','proxySource':''}) args=process.call_args.args[0] assert args[args.index('--upstream')+1]=='http://172.20.0.9:9203/internal/finance/v1/business-fee-source-events' assert args[args.index('--fee-source-id')+1]=='123' and args[args.index('--source-version')+1]=='1' assert '--delivery-id' not in args and '--container-booking-id' not in args state={'direction':'FEE_VOID','restored':False,'prepared':True,'originalId':'original','image':'lastleg:test','originalImageId':original['Image'],'originalFingerprint':m.fingerprint(original),'originalBody':m.recreation(original,m.env_of(original),full=True),'proxyPort':18001} with patch.object(m,'control',return_value={}),patch.object(m,'inspect',return_value=original),patch.object(m,'private_write'),patch.object(m,'replace') as replace: m.activate(s,pathlib.Path('/offline'),copy.deepcopy(state)) changed=dict(item.split('=',1) for item in replace.call_args.args[1]['Env']) assert changed['FINANCE_SERVICE_URL']=='http://host.docker.internal:18001' assert changed['FIRSTLEG_SERVICE_URL']=='http://firstleg:9202' and changed['EVENT_MAX_ATTEMPTS']=='12' row={'entryId':'2','eventId':'00000000-0000-4000-8000-000000000002','payloadHash':'sha256:'+'b'*64,'publishStatus':'DEAD','failedAttempts':12} retry_state={**state,'operationsRequestId':'00000000-0000-4000-8000-000000000003'} runtime=copy.deepcopy(original);runtime['Config']['Env'].append('EVENT_OPERATIONS_TOKEN=offline') dead={'lastleg':[{'sourceVersion':'1','sourceStatus':'ACTIVE'}],'finance':{'source':[{'sourceFeeVersion':'1','status':'VOIDED'}]}} with patch.object(m,'producer',return_value=row),patch.object(m,'control',return_value=row),patch.object(m,'fee_facts',return_value=dead),patch.object(m,'private_write'),patch.object(m,'inspect',return_value=runtime),patch.object(m.urllib.request,'urlopen',return_value=io.BytesIO(b'{"code":"200"}')) as http: assert m.recover(s,pathlib.Path('/offline'),retry_state)['retrySubmitted'] assert http.call_args.args[0].full_url=='http://172.20.0.3:9303/internal/lastleg/event-operations/2/retry' assert retry_state['beforeRetry']['fee']['lastleg'][0]['sourceStatus']=='ACTIVE' restored=copy.deepcopy(original);restored['Id']='restored' with patch.object(m,'control',side_effect=OSError('stopped')),patch.object(m,'inspect',return_value=restored),patch.object(m,'private_write'),patch.object(m,'replace') as replace: m.restore(pathlib.Path('/offline'),state);assert not state['prepared'];replace.assert_not_called() state['prepared']=True;restored['Config']['Env'].append('UNEXPECTED=1') with patch.object(m,'control'),patch.object(m,'inspect',return_value=restored): try:m.restore(pathlib.Path('/offline'),state);assert False except RuntimeError as e:assert str(e)=='CONCURRENT_PRODUCER_CHANGE' `],{cwd:new URL('..',import.meta.url),encoding:'utf8',timeout:10000}) assert.equal(result.status,0,result.stderr||result.error?.message) })