scripts/test-fee-void-fault-proof.test.mjs

import assert from 'node:assert/strict'
import test from 'node:test'
import { createHash } from 'node:crypto'
import { spawnSync } from 'node:child_process'
import { createTestFeeVoidFaultExecutor } from './test-event-fault-executor.mjs'
import { assertTestFeeVoidFaultProof } from './test-fee-void-fault-proof.mjs'

const scope={marker:'FEE-VOID-OFFLINE',customerId:'7',sourceFeeId:'123',sourceVersion:'1',sourceType:'OVERSEAS_WAREHOUSE_FEE'}
const env={CHONGCHONG_ENV:'test',JENKINS_URL:'https://jenkins.example.invalid/',JENKINS_USER:'offline',JENKINS_API_TOKEN:'offline',FBA_CORE_JENKINS_AGENT:'java-test-agent'}
const sort=value=>Array.isArray(value)?value.map(sort):value&&typeof value==='object'?Object.fromEntries(Object.keys(value).sort().map(key=>[key,sort(value[key])])):value
const text=value=>JSON.stringify(sort(value)), hash=value=>'sha256:'+createHash('sha256').update(value).digest('hex')
const clone=value=>JSON.parse(JSON.stringify(value))
const imageId='sha256:'+'a'.repeat(64),fingerprint='b'.repeat(64)
const restoration={restored:true,healthy:true,proxyStopped:true,producerName:'firstleg-test-lastleg',originalImageId:imageId,currentImageId:imageId,originalFingerprint:fingerprint,currentFingerprint:fingerprint}
function fixture() {
  const base=Object.fromEntries(Object.entries({action:null,sourceSystem:'LASTLEG',sourceType:scope.sourceType,sourceFeeId:'123',sourceVersion:1,customerId:'7',customerName:'Offline',
    sourceNo:'WH-OFFLINE',businessType:'WAREHOUSE_SERVICE',businessId:'200',businessNo:'WH-OFFLINE',orderId:null,inboundId:null,inboundCode:null,
    containerId:null,containerBookingId:null,containerNo:null,warehouseId:'9',feeItemCode:'TEST',feeItemName:'Offline fee',direction:'RECEIVABLE',
    supplierId:null,supplierName:null,supplierCategory:null,settlementMode:null,paidAt:null,paymentMethod:null,paymentReferenceNo:null,paymentVoucherFileIds:[],
    businessDate:'2026-10-03',currency:'CNY',totalAmount:7.25,originalCurrency:'CNY',originalAmount:7.25,exchangeRate:1,exchangeRateMonth:'2026-10',
    chargeableQuantity:null,quantityUnit:null,unitPrice:null,periodStart:null,periodEnd:null,reviewStatus:'APPROVED',publicationStatus:'PUBLISHED',sourceStatus:'ACTIVE',sourceHash:null}).filter(([,value])=>value!==null))
  const sourceCanonicalJson=text(base),sourceHash=hash(sourceCanonicalJson)
  const event=(action,index)=>{
    const payload={...base,action,sourceStatus:action==='VOIDED'?'VOIDED':'ACTIVE',sourceHash}
    const payloadCanonicalJson=text(payload),payloadHash=hash(payloadCanonicalJson)
    const eventId=`00000000-0000-4000-8000-00000000000${index}`
    const envelope={eventId,eventType:'BusinessFeeSourceChanged',eventVersion:1,tenantId:'7',aggregateType:scope.sourceType,aggregateId:'123',aggregateVersion:1,payload,payloadHash}
    return {entryId:String(index),eventId,payloadHash,eventType:envelope.eventType,customerId:'7',aggregateId:'123',aggregateType:scope.sourceType,
      eventKey:`BusinessFeeSourceChanged:LASTLEG:${scope.sourceType}:123:1:${action}`,envelope,envelopeJson:JSON.stringify(envelope),payloadCanonicalJson,sourceCanonicalJson,publishStatus:'PUBLISHED',failedAttempts:0}
  }
  const publication=event('PUBLISHED',1),producer=event('VOIDED',2)
  const local={id:'500',sourceFeeId:'123',sourceVersion:'1',customerId:'7',sourceType:scope.sourceType,sourceStatus:'ACTIVE',reviewStatus:'APPROVED',publicationStatus:'PUBLISHED',direction:'RECEIVABLE',amount:'7.250000',businessType:'WAREHOUSE_SERVICE',businessId:'200',reviewReason:'approved',updatedBy:'7',updatedAt:'2026-10-03 01:00:00'}
  const finance={id:'600',sourceSystem:'LASTLEG',sourceFeeId:'123',sourceFeeVersion:'1',customerId:'7',sourceType:scope.sourceType,direction:'RECEIVABLE',currencyCode:'CNY',amount:'7.250000',walletHoldId:null,operationVersion:'0',status:'AVAILABLE',sourceHash,sourceSnapshot:publication.envelope.payload,updatedAt:'2026-10-03 01:00:01'}
  const inbox=(row,id)=>({id,eventId:row.eventId,sourceSystem:'LASTLEG',eventType:'BusinessFeeSourceChanged',envelope:row.envelope,payloadHash:row.payloadHash,resultResourceId:'600',status:'PROCESSED',errorCode:null,receivedAt:'2026-10-03 01:00:01',processedAt:'2026-10-03 01:00:01'})
  const initial={lastleg:[local],finance:{source:[finance],bill:[],settlement:[],cost:[],inbox:[inbox(publication,'700')]}}
  const dead=clone(initial);dead.lastleg[0].reviewReason='void reason';dead.lastleg[0].updatedAt='2026-10-03 01:01:00'
  Object.assign(dead.finance.source[0],{status:'VOIDED',operationVersion:'1',updatedAt:'2026-10-03 01:01:01'});dead.finance.inbox=[inbox(producer,'701')]
  const after=clone(dead);Object.assign(after.lastleg[0],{sourceStatus:'VOIDED',updatedAt:'2026-10-03 01:02:00'})
  const prior={ready:true,publication,business:initial,attempts:[{entryId:'1',eventId:publication.eventId,outcome:'APPLIED'}]}
  const requestId='00000000-0000-4000-8000-000000000003'
  return clone({...scope,readOnly:true,producer,prerequisite:prior,publicationNow:publication,
    beforeRetry:{...producer,publishStatus:'DEAD',failedAttempts:12,fee:dead},afterRetry:{...producer,fee:after},
    consumer:{...dead.finance.inbox[0],count:1},
    attempts:Array.from({length:13},(_,i)=>({attemptId:`a${i}`,entryId:'2',eventId:producer.eventId,serviceName:'lastleg',outcome:i<11?'RETRY':i===11?'DEAD':'DUPLICATE'})),
    proxyRecords:Array.from({length:13},(_,i)=>({matched:true,attempt:i+1,eventId:producer.eventId,payloadHash:producer.payloadHash,outcome:i===0?'APPLIED':'DUPLICATE',errorCode:i<12?'ACK_LOSS_INJECTED':null})),
    retryAuditCount:1,retryRequestId:requestId,retryAudit:{requestId,entryId:'2',eventId:producer.eventId,payloadHash:producer.payloadHash,previousFailures:12}})
}
function config(options) {
  const chunks=[...options.body.get('script').matchAll(/'([A-Za-z0-9+/=]+)'/g)].map(m=>m[1]).join('')
  const groovy=Buffer.from(chunks,'base64').toString('utf8'),encoded=groovy.match(/'UTF-8'\),'([A-Za-z0-9+/=]+)'\]\)/)?.[1]
  return JSON.parse(Buffer.from(encoded,'base64').toString('utf8'))
}
function transport(calls,{pending=false,preflightFailure=false,proof=fixture(),finish={}}={}) {
  let recovered=false,preflightCount=0
  return async(_url,options)=>{
    const request=config(options);assert.equal(request.direction,'FEE_VOID');assert.deepEqual(request.scope,scope)
    const action=request.action;calls.push(action)
    if(action==='recover')recovered=true
    if(action==='preflight')preflightCount++
    const data=action==='preflight'?{ready:!pending||preflightCount>1}:action==='prepare'?{originalImageId:imageId,originalFingerprint:fingerprint,prerequisite:proof.prerequisite}
      :action==='proxy-ready'?{ready:true}:action==='health'?{healthy:true}:action==='status'?{producer:{publishStatus:recovered?'PUBLISHED':'DEAD',failedAttempts:12}}
        :action==='proof'?proof:action==='finish'?{...restoration,...finish}:{}
    const result=preflightFailure&&action==='preflight'?{ok:false,error:'FEE_PUBLICATION_NOT_READY'}:{ok:true,data}
    return {ok:true,text:async()=> 'TEST_EVENT_JSON='+JSON.stringify(result)}
  }
}
test('VOID proof accepts Finance VOIDED at DEAD while Lastleg remains ACTIVE until matched retry ACK',()=>{
  const proof=fixture();assertTestFeeVoidFaultProof(proof,scope)
  assert.equal(proof.beforeRetry.fee.lastleg[0].sourceStatus,'ACTIVE')
  assert.equal(proof.beforeRetry.fee.finance.source[0].status,'VOIDED')
  assert.equal(proof.afterRetry.fee.lastleg[0].sourceStatus,'VOIDED')
})
test('VOID proof rejects premature local confirmation, extra financial writes, altered source and lost original event evidence',()=>{
  const mutations=[p=>p.beforeRetry.fee.lastleg[0].sourceStatus='VOIDED',p=>p.afterRetry.fee.lastleg[0].sourceStatus='ACTIVE',
    p=>p.afterRetry.fee.lastleg[0].amount='8.250000',p=>p.afterRetry.fee.lastleg[0].sourceVersion='2',
    p=>p.afterRetry.fee.finance.source[0].operationVersion='2',p=>p.beforeRetry.fee.finance.source[0].status='AVAILABLE',
    p=>p.prerequisite.publication.publishStatus='PENDING',p=>p.prerequisite.business.finance.source[0].walletHoldId='99',
    p=>p.afterRetry.fee.finance.bill.push({id:'123'}),p=>p.consumer.resultResourceId='601',
    p=>p.producer.envelope.payload.sourceFeeId='124',p=>p.beforeRetry.entryId='9',p=>p.retryAuditCount=2,
    p=>p.proxyRecords[0].outcome='DUPLICATE',p=>p.attempts.pop(),p=>p.producer.sourceCanonicalJson='{}',
    p=>p.prerequisite.business.lastleg.push({...p.prerequisite.business.lastleg[0],id:'501',sourceVersion:'2'})]
  for(const mutate of mutations){const proof=fixture();mutate(proof);assert.throws(()=>assertTestFeeVoidFaultProof(proof,scope),/Invalid fee void/)}
})
test('preflight waits for original publish ACK before a single prepare; restoration precedes successful proof',async()=>{
  const calls=[],executor=createTestFeeVoidFaultExecutor(env,transport(calls,{pending:true}))
  for(const invalid of [{...scope,sourceVersion:1},{...scope,sourceType:'UNKNOWN'},{...scope,sourceVersion:'2147483648'},{...scope,action:'VOIDED'}])
    await assert.rejects(executor.prepare(invalid),/Invalid/)
  await executor.prepare(scope)
  const proof=await executor.readProof(scope);assert.equal(proof.restored,true)
  assert.deepEqual(calls,['preflight','preflight','prepare','proxy-ready','activate','health','status','recover','status','proof','restore','health','finish'])
  await executor.cleanup();assert.equal(calls.filter(a=>a==='prepare').length,1)
})
test('preflight failure never mutates environment and cannot silently reuse an unprepared executor',async()=>{
  const calls=[],executor=createTestFeeVoidFaultExecutor(env,transport(calls,{preflightFailure:true}))
  await assert.rejects(executor.prepare(scope),/FEE_PUBLICATION_NOT_READY/)
  assert.deepEqual(await executor.cleanup(),{attempted:false});await assert.rejects(executor.prepare(scope),/did not complete/)
  assert.deepEqual(calls,['preflight'])
})
test('proof failure restores; missing original configuration or proxy stop remains a blocker',async()=>{
  const proof=fixture();proof.afterRetry.fee.finance.source[0].operationVersion='2'
  const calls=[],executor=createTestFeeVoidFaultExecutor(env,transport(calls,{proof}))
  await executor.prepare(scope);await assert.rejects(executor.readProof(scope),/FINANCE_CHANGED/)
  assert.deepEqual(calls.slice(-3),['restore','health','finish'])
  for(const finish of [{proxyStopped:false},{currentFingerprint:'c'.repeat(64)}]) {
    const failed=createTestFeeVoidFaultExecutor(env,transport([],{finish}))
    await failed.prepare(scope);await assert.rejects(failed.readProof(scope),/Fee void restoration/)
  }
})

test('unknown prepare result is never reissued and cleanup still attempts original restoration',async()=>{
  const calls=[],base=transport(calls)
  const executor=createTestFeeVoidFaultExecutor(env,async(url,options)=>{
    if(config(options).action==='prepare'){calls.push('prepare');throw Error('response lost')}
    return base(url,options)
  })
  await assert.rejects(executor.prepare(scope),/response lost/)
  await assert.rejects(executor.prepare(scope),/did not complete/)
  await assert.rejects(executor.cleanup(),/Fee void restoration evidence mismatch/)
  assert.deepEqual(calls,['preflight','prepare','restore','health','finish'])
})

test('NON_NULL source hashes agree across runtime and proof and reject explicit null unsigned fields',()=>{
  const proof=fixture(),publication=proof.prerequisite.publication
  const unsigned=JSON.parse(publication.sourceCanonicalJson)
  assert.equal(Object.hasOwn(unsigned,'action'),false)
  assert.equal(Object.hasOwn(unsigned,'sourceHash'),false)
  assert.equal(Object.hasOwn(unsigned,'orderId'),false)
  assert.deepEqual(unsigned.paymentVoucherFileIds,[])
  const invalid=clone(proof)
  invalid.producer.sourceCanonicalJson=text({...unsigned,action:null,sourceHash:null})
  invalid.producer.envelope.payload.sourceHash=hash(invalid.producer.sourceCanonicalJson)
  invalid.producer.payloadCanonicalJson=text(invalid.producer.envelope.payload)
  invalid.producer.payloadHash=hash(invalid.producer.payloadCanonicalJson)
  invalid.producer.envelope.payloadHash=invalid.producer.payloadHash
  invalid.producer.envelopeJson=JSON.stringify(invalid.producer.envelope)
  assert.throws(()=>assertTestFeeVoidFaultProof(invalid,scope),/SOURCE_HASH/)
  const result=spawnSync(process.env.PYTHON || 'python',['-c',String.raw`
import sys,importlib.util,json,copy,hashlib
from unittest.mock import patch
sys.dont_write_bytecode=True
spec=importlib.util.spec_from_file_location('fault','scripts/test-event-fault-runtime.py');m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
data=json.load(sys.stdin);scope=data['scope'];row=data['row']
with patch.object(m,'inspect'),patch.object(m,'read_sql',side_effect=lambda *args:[copy.deepcopy(row)]):
 result=m.fee_event(scope,'PUBLISHED')
 assert result['sourceCanonicalJson']==row['sourceCanonicalJson']
 assert result['payloadCanonicalJson']==row['payloadCanonicalJson']
 payload=json.loads(row['envelopeJson'])['payload']
 unsigned={**payload,'action':None,'sourceHash':None,'sourceStatus':'ACTIVE'}
 payload['sourceHash']='sha256:'+hashlib.sha256(m.fee_canonical(unsigned).encode()).hexdigest()
 envelope=json.loads(row['envelopeJson']);envelope['payload']=payload
 row['envelopeJson']=json.dumps(envelope)
 row['payloadHash']='sha256:'+hashlib.sha256(m.fee_canonical(payload).encode()).hexdigest()
 try:m.fee_event(scope,'PUBLISHED');assert False
 except RuntimeError as e:assert str(e)=='FEE_SOURCE_HASH_MISMATCH'
 row['payloadHash']='sha256:'+'0'*64
 try:m.fee_event(scope,'PUBLISHED');assert False
 except RuntimeError as e:assert str(e)=='FEE_PAYLOAD_HASH_MISMATCH'
`],{cwd:new URL('..',import.meta.url),input:JSON.stringify({scope,row:publication}),encoding:'utf8',timeout:10000})
  assert.equal(result.status,0,result.stderr||result.error?.message)
})

test('Python FEE_VOID routing, exact decimal hashes, preflight gating and restore reentry are offline',()=>{
  const result=spawnSync(process.env.PYTHON || 'python',['-c',String.raw`
import sys,importlib.util,pathlib,copy,decimal,ast,io
from unittest.mock import patch,mock_open
sys.dont_write_bytecode=True
p=pathlib.Path('scripts/test-event-fault-runtime.py');ast.parse(p.read_text(encoding='utf-8'))
spec=importlib.util.spec_from_file_location('fault',p);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
s={'marker':'FEE-VOID-OFFLINE','customerId':'7','sourceFeeId':'123','sourceVersion':'1','sourceType':'OVERSEAS_WAREHOUSE_FEE'}
assert m.target_setting({'direction':'FEE_VOID'})==('FINANCE_SERVICE_URL','http://finance:9203')
assert m.target_setting({'direction':'DELIVERY'})==('FIRSTLEG_SERVICE_URL','http://firstleg:9202')
assert m.producer_name({'direction':'FEE_VOID'})=='firstleg-test-lastleg'
assert m.fee_canonical(decimal.Decimal('100.000000'))=='1E+2'
assert m.fee_canonical(decimal.Decimal('0.00000010'))=='1E-7'
assert m.fee_canonical(decimal.Decimal('12345678901234.123456'))=='12345678901234.123456'
assert m.fee_canonical(decimal.Decimal('-0.000'))=='0'
queries=[]
with patch.object(m,'inspect',side_effect=lambda name:{'Name':'/'+name}),patch.object(m,'require_test'),patch.object(m,'read_sql',side_effect=lambda c,sql:queries.append((c['Name'],sql)) or []):
 assert m.producer(s,'FEE_VOID') is None
 m.fee_facts(s,'00000000-0000-4000-8000-000000000002')
assert [r[0] for r in queries]==['/firstleg-test-lastleg','/firstleg-test-lastleg','/firstleg-test-finance','/firstleg-test-finance']
assert 'BusinessFeeSourceChanged:LASTLEG:OVERSEAS_WAREHOUSE_FEE:123:1:VOIDED' in queries[0][1]
assert 'void_status' not in queries[1][1] and 'source_status' in queries[1][1]
assert all(t in queries[2][1] for t in ['fin_fee_source','fin_bill_line','fin_settlement_line','fin_cost','operation_version'])
assert 'fin_inbox_event' in queries[3][1] and 'result_resource_id' in queries[3][1]
publication={'entryId':'1','eventId':'00000000-0000-4000-8000-000000000001','publishStatus':'PENDING','payloadHash':'sha256:'+'a'*64}
facts={'lastleg':[{'sourceVersion':'1','sourceType':s['sourceType'],'customerId':'7','sourceStatus':'ACTIVE','reviewStatus':'APPROVED','publicationStatus':'PUBLISHED','direction':'RECEIVABLE'}],
 'finance':{'source':[{'id':'600','sourceFeeVersion':'1','sourceType':s['sourceType'],'customerId':'7','status':'AVAILABLE','walletHoldId':None}],
 'bill':[],'settlement':[],'cost':[],'inbox':[{'status':'PROCESSED','payloadHash':publication['payloadHash'],'resultResourceId':'600'}]}}
with patch.object(m,'fee_event',side_effect=lambda scope,action:None if action=='VOIDED' else publication),patch.object(m,'fee_facts',return_value=facts),patch.object(m,'inspect'),patch.object(m,'read_sql',return_value=[{'eventId':publication['eventId'],'outcome':'APPLIED'}]):
 assert m.fee_prerequisite(s,wait=True)['ready'] is False
 publication['publishStatus']='PUBLISHED';assert m.fee_prerequisite(s,wait=True)['ready'] is True
 publication['publishStatus']='DEAD'
 try:m.fee_prerequisite(s,wait=True);assert False
 except RuntimeError as e:assert str(e)=='FEE_PUBLICATION_NOT_READY'
original={'Name':'/firstleg-test-lastleg','Id':'original','Image':'sha256:'+'a'*64,'Config':{'Image':'lastleg:test','Env':['FINANCE_SERVICE_URL=http://finance:9203','FIRSTLEG_SERVICE_URL=http://firstleg:9202'],'Labels':{'com.docker.compose.project':'firstleg-test'}},'HostConfig':{},'Mounts':[],
 'NetworkSettings':{'Networks':{'firstleg-test-network':{'IPAddress':'172.20.0.3','Aliases':['lastleg-service']}}},'State':{'Running':True,'Health':{'Status':'healthy'}}}
finance=copy.deepcopy(original);finance['Name']='/firstleg-test-finance';finance['NetworkSettings']['Networks']['firstleg-test-network']['IPAddress']='172.20.0.9'
with patch.object(m,'fee_prerequisite',return_value={'ready':True}),patch.object(m,'inspect',side_effect=lambda name:finance if name=='firstleg-test-finance' else original),patch.object(m.Path,'mkdir'),patch.object(m.Path,'write_bytes'),patch('builtins.open',mock_open()),patch.object(m.socket,'socket') as sock,patch.object(m.subprocess,'Popen') as process,patch.object(m,'private_write'):
 sock.return_value.getsockname.return_value=('127.0.0.1',18001)
 m.prepare(s,pathlib.Path('/offline'),{'direction':'FEE_VOID','proxySource':''})
 args=process.call_args.args[0]
 assert args[args.index('--upstream')+1]=='http://172.20.0.9:9203/internal/finance/v1/business-fee-source-events'
 assert args[args.index('--fee-source-id')+1]=='123' and args[args.index('--source-version')+1]=='1'
 assert '--delivery-id' not in args and '--container-booking-id' not in args
state={'direction':'FEE_VOID','restored':False,'prepared':True,'originalId':'original','image':'lastleg:test','originalImageId':original['Image'],'originalFingerprint':m.fingerprint(original),'originalBody':m.recreation(original,m.env_of(original),full=True),'proxyPort':18001}
with patch.object(m,'control',return_value={}),patch.object(m,'inspect',return_value=original),patch.object(m,'private_write'),patch.object(m,'replace') as replace:
 m.activate(s,pathlib.Path('/offline'),copy.deepcopy(state))
 changed=dict(item.split('=',1) for item in replace.call_args.args[1]['Env'])
 assert changed['FINANCE_SERVICE_URL']=='http://host.docker.internal:18001'
 assert changed['FIRSTLEG_SERVICE_URL']=='http://firstleg:9202' and changed['EVENT_MAX_ATTEMPTS']=='12'
row={'entryId':'2','eventId':'00000000-0000-4000-8000-000000000002','payloadHash':'sha256:'+'b'*64,'publishStatus':'DEAD','failedAttempts':12}
retry_state={**state,'operationsRequestId':'00000000-0000-4000-8000-000000000003'}
runtime=copy.deepcopy(original);runtime['Config']['Env'].append('EVENT_OPERATIONS_TOKEN=offline')
dead={'lastleg':[{'sourceVersion':'1','sourceStatus':'ACTIVE'}],'finance':{'source':[{'sourceFeeVersion':'1','status':'VOIDED'}]}}
with patch.object(m,'producer',return_value=row),patch.object(m,'control',return_value=row),patch.object(m,'fee_facts',return_value=dead),patch.object(m,'private_write'),patch.object(m,'inspect',return_value=runtime),patch.object(m.urllib.request,'urlopen',return_value=io.BytesIO(b'{"code":"200"}')) as http:
 assert m.recover(s,pathlib.Path('/offline'),retry_state)['retrySubmitted']
 assert http.call_args.args[0].full_url=='http://172.20.0.3:9303/internal/lastleg/event-operations/2/retry'
 assert retry_state['beforeRetry']['fee']['lastleg'][0]['sourceStatus']=='ACTIVE'
restored=copy.deepcopy(original);restored['Id']='restored'
with patch.object(m,'control',side_effect=OSError('stopped')),patch.object(m,'inspect',return_value=restored),patch.object(m,'private_write'),patch.object(m,'replace') as replace:
 m.restore(pathlib.Path('/offline'),state);assert not state['prepared'];replace.assert_not_called()
state['prepared']=True;restored['Config']['Env'].append('UNEXPECTED=1')
with patch.object(m,'control'),patch.object(m,'inspect',return_value=restored):
 try:m.restore(pathlib.Path('/offline'),state);assert False
 except RuntimeError as e:assert str(e)=='CONCURRENT_PRODUCER_CHANGE'
`],{cwd:new URL('..',import.meta.url),encoding:'utf8',timeout:10000})
  assert.equal(result.status,0,result.stderr||result.error?.message)
})