test(order): add isolated dependency failure and recovery verification
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
import test from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { buildOrderDependencyFaultPlan } from './admin-test-env-modules/orders.mjs'
|
||||
import { scopeFor, renderOrderFaultAction, createOrderFaultRuntime, runOrderFaultWindow } from './order-submit-dependency-fault-executor.mjs'
|
||||
|
||||
const image = 'sha256:' + 'a'.repeat(64)
|
||||
function fixture() {
|
||||
const detail = id => ({order:{id,tenantId:'20',orderNo:`FL${id}`,version:2,status:'DRAFT'},cargoes:[],parties:[],internalNotes:[],statusHistory:[]})
|
||||
const raw={marker:'TEST-DEPENDENCY',customerId:'20',registeredCustomerId:'20',merchant:detail('101'),platform:detail('102')}
|
||||
return {raw,plan:buildOrderDependencyFaultPlan(raw)}
|
||||
}
|
||||
const env={CHONGCHONG_ENV:'test',JENKINS_URL:'https://offline.invalid/',JENKINS_USER:'offline',JENKINS_API_TOKEN:'never-print-this',FBA_CORE_JENKINS_AGENT:'test-agent'}
|
||||
|
||||
test('renderer binds TEST scope and emits no Jenkins credential; ambiguous remote mutation is never retried',async()=>{
|
||||
const {plan}=fixture(),scope=scopeFor(plan)
|
||||
assert.ok(!renderOrderFaultAction('status',scope,env.FBA_CORE_JENKINS_AGENT).includes(env.JENKINS_API_TOKEN))
|
||||
assert.throws(()=>renderOrderFaultAction('delete-all',scope,'agent'))
|
||||
assert.throws(()=>renderOrderFaultAction('prepare',scope,'agent',{routing:{configurationReviewed:true,password:'must-not-embed'}}))
|
||||
assert.throws(()=>createOrderFaultRuntime({...env,CHONGCHONG_ENV:'prod'},scope))
|
||||
for(const mutate of [p=>p.phases[0].expectedCode='A0403',p=>p.phases[0].targetPath+='/other',p=>p.phases[0].expectedMessages.push('anything')]) {
|
||||
const changed=structuredClone(plan);mutate(changed);assert.throws(()=>scopeFor(changed))
|
||||
}
|
||||
let calls=0
|
||||
const runtime=createOrderFaultRuntime(env,scope,async()=>{calls++;throw Error('connection lost with secret')})
|
||||
await assert.rejects(runtime('activate'),/OUTCOME_UNKNOWN_USE_STATUS/)
|
||||
assert.equal(calls,1)
|
||||
})
|
||||
|
||||
function windowFixture(fault='') {
|
||||
const {raw,plan}=fixture(),calls=[],records=[]
|
||||
let mode='PASS'
|
||||
const options={plan,routing:{configurationReviewed:true},expectedImageId:image,wait:async()=>{},
|
||||
async action(action,extra={}) {
|
||||
calls.push(action)
|
||||
if(action==='prepare'&&fault==='prepare-unknown')throw Error('PREPARE_OUTCOME_UNKNOWN')
|
||||
if(action==='restore'&&['restore-unknown','prepare-unknown'].includes(fault))throw Error('UNCERTAIN_MUTATION_REQUIRES_RECONCILE')
|
||||
if(action==='status')return {proxy:{mode:'PASS'}}
|
||||
if(action==='health')return {healthy:true}
|
||||
if(action==='mode'){mode=extra.mode;return {mode}}
|
||||
if(action==='records')return {records}
|
||||
if(action==='verify-restored')return {originalImageId:image,currentImageId:image,originalFingerprint:'same',currentFingerprint:'same',upstreamReads:{IAM:true,FINANCE:true},nacosSourcesUnchanged:true}
|
||||
if(action==='finish')return {restored:true,healthy:true,proxyStopped:true,orderRoutesVerified:true}
|
||||
return {}
|
||||
},
|
||||
async snapshot(role){return {detail:structuredClone(raw[role]),task:null,inbounds:[],inboundPageComplete:true}},
|
||||
async submit(role,request){
|
||||
const phase=plan.phases.find(item=>item.dependency===mode)
|
||||
records.push({requestId:request.headers['X-Request-Id'],dependency:mode,method:'GET',path:phase.targetPath,
|
||||
matched:true,responseStatus:503,forwarded:false,internalTokenPresent:true})
|
||||
return {http:200,body:{code:fault==='wrong-business-code'?'A0403':'S0503',message:phase.expectedMessages[0]}}
|
||||
},
|
||||
async recovery(){calls.push('business-recovery');if(fault==='business-recovery-failed')throw Error('RECOVERY_REPLAY_DRIFT')
|
||||
return {merchant:{orderId:'101',status:'SUBMITTED',sameKeyReplayVerified:true},platform:{orderId:'102',status:'SUBMITTED',sameKeyReplayVerified:true}}},
|
||||
}
|
||||
return {options,calls}
|
||||
}
|
||||
|
||||
test('four observations restore original routes and verify business recovery before stopping proxy',async()=>{
|
||||
const {options,calls}=windowFixture(),report=await runOrderFaultWindow(options)
|
||||
assert.equal(report.success,true);assert.equal(report.observations.length,4)
|
||||
assert.ok(calls.indexOf('restore')<calls.indexOf('verify-restored'))
|
||||
assert.ok(calls.indexOf('verify-restored')<calls.indexOf('business-recovery'))
|
||||
assert.ok(calls.indexOf('business-recovery')<calls.indexOf('finish'))
|
||||
})
|
||||
test('failed business assertion still restores, but never reports fault verification success',async()=>{
|
||||
const {options,calls}=windowFixture('wrong-business-code'),report=await runOrderFaultWindow(options)
|
||||
assert.equal(report.success,false);assert.equal(report.restoration.restored,true)
|
||||
assert.ok(calls.includes('finish'))
|
||||
})
|
||||
for(const fault of ['prepare-unknown','restore-unknown','business-recovery-failed'])test(`unsafe cleanup blocks finish: ${fault}`,async()=>{
|
||||
const {options,calls}=windowFixture(fault),report=await runOrderFaultWindow(options)
|
||||
assert.equal(report.success,false);assert.equal(calls.filter(action=>action==='prepare').length,1)
|
||||
assert.equal(calls.filter(action=>action==='restore').length,1);assert.ok(!calls.includes('finish'))
|
||||
})
|
||||
|
||||
test('offline runtime journals Docker transitions, reconciles observed outcomes and retains original configuration (no Docker/HTTP)',()=>{
|
||||
const script=String.raw`
|
||||
import importlib.util,sys,copy,json,unittest
|
||||
from pathlib import Path
|
||||
spec=importlib.util.spec_from_file_location('runtime',sys.argv[1]);m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
common=m.common_tools(Path(sys.argv[2]).read_text(encoding='utf-8'))
|
||||
class Fake:
|
||||
def __init__(self):
|
||||
self.current={'Id':'original-container-id','Name':'/firstleg-test-order','Image':'sha256:'+'a'*64,
|
||||
'Config':{'Image':'offline/order:tag','Env':['DEPLOY_ENV=test','SERVER_PORT=9204','INTERNAL_SERVICE_TOKEN=offline-secret-long'],
|
||||
'Entrypoint':copy.deepcopy(m.ENTRYPOINT),'Cmd':None,'Labels':{'com.docker.compose.project':'firstleg-test'}},
|
||||
'HostConfig':{'AutoRemove':False,'ExtraHosts':['host.docker.internal:host-gateway']},'Mounts':[],
|
||||
'NetworkSettings':{'Networks':{'test':{'IPAddress':'172.20.0.10','NetworkID':'n1','Aliases':['order','original-con'],'MacAddress':'aa:bb:cc:dd:ee:ff'}}},
|
||||
'State':{'Running':True,'Health':{'Status':'healthy'}}}
|
||||
self.original=copy.deepcopy(self.current);self.calls=[];self.fail=None;self.failBefore=None;self.serial=0
|
||||
def inspect(self,name):
|
||||
if not self.current or name not in ('firstleg-test-order',self.current['Id']):raise RuntimeError('DOCKER_404')
|
||||
return copy.deepcopy(self.current)
|
||||
def docker(self,method,path,body=None):
|
||||
if method=='GET':return {'Id':self.original['Image']}
|
||||
self.calls.append((method,path));result=None
|
||||
if self.failBefore=='stop' and '/stop?' in path:raise RuntimeError('REQUEST_OUTCOME_UNKNOWN')
|
||||
if '/stop?' in path:self.current['State']['Running']=False;op='stop'
|
||||
elif method=='DELETE':self.current=None;op='delete'
|
||||
elif '/create?' in path:
|
||||
self.serial+=1;new_id='new-container-'+str(self.serial);op='create'
|
||||
self.current={'Id':new_id,'Name':'/firstleg-test-order','Image':self.original['Image'],
|
||||
'Config':{k:copy.deepcopy(v) for k,v in body.items() if k not in ('HostConfig','NetworkingConfig')},
|
||||
'HostConfig':copy.deepcopy(body['HostConfig']),'Mounts':[],
|
||||
'NetworkSettings':{'Networks':{}},'State':{'Running':False,'Health':{'Status':'healthy'}}}
|
||||
for name,entry in body['NetworkingConfig']['EndpointsConfig'].items():
|
||||
self.current['NetworkSettings']['Networks'][name]={'IPAddress':entry['IPAMConfig']['IPv4Address'],'NetworkID':'n1',
|
||||
'Aliases':entry['Aliases']+[new_id[:12]],'MacAddress':entry['MacAddress'],'IPAMConfig':entry['IPAMConfig']}
|
||||
result={'Id':new_id}
|
||||
elif path.endswith('/start'):self.current['State']['Running']=True;op='start'
|
||||
else:raise AssertionError(path)
|
||||
if self.fail==op:self.fail=None;raise RuntimeError('RESPONSE_LOST')
|
||||
return result
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def fixture(self):
|
||||
f=Fake();tools=dict(common);tools.update(inspect=f.inspect,docker=f.docker)
|
||||
scope={'marker':'OFFLINE','customerId':'7','orderIds':{'merchant':'10','platform':'11'},'planSha256':'b'*64}
|
||||
r=m.Runtime(scope,tools,Path('offline-not-created'));saved=[];controls=[]
|
||||
r.save=lambda state:saved.append(copy.deepcopy(state));r.control=lambda state,mode=None,ttl=0:(controls.append(mode) or {'mode':'PASS'})
|
||||
r.probes=lambda state:{'IAM':True,'FINANCE':True}
|
||||
r.config_fingerprint=lambda container,routing:'reviewed-config'
|
||||
state={'scope':scope,'phase':'PROXY_STARTED','originalId':f.original['Id'],'originalImageId':f.original['Image'],
|
||||
'originalFingerprint':common['fingerprint'](f.original),'originalBody':common['recreation'](f.original,common['env_of'](f.original),full=True),
|
||||
'proxyPort':12345,'controlPort':12346,'mutation':None,'routing':{},'routingSourceSha256':'reviewed-config'}
|
||||
return f,r,state,saved,controls
|
||||
def test_original_fingerprint_config_and_image_restored(self):
|
||||
f,r,s,saved,controls=self.fixture();r.activate(s)
|
||||
self.assertTrue(r.matches_injected(f.current,s));self.assertEqual(s['phase'],'INJECTED')
|
||||
self.assertTrue(any(x.get('mutation',{}).get('step')=='CREATE_INTENT' for x in saved if x.get('mutation')))
|
||||
r.restore(s);result=r.verify_restored(s)
|
||||
self.assertEqual(result['originalFingerprint'],result['currentFingerprint']);self.assertEqual(f.current['Config'],f.original['Config'])
|
||||
def test_unknown_outcomes_never_auto_repeat_docker_and_can_be_explicitly_reconciled(self):
|
||||
for operation in ['stop','delete','create','start']:
|
||||
f,r,s,saved,controls=self.fixture();f.fail=operation
|
||||
with self.assertRaisesRegex(RuntimeError,'UNCERTAIN'):r.activate(s)
|
||||
count=len(f.calls)
|
||||
with self.assertRaisesRegex(RuntimeError,'RECONCILE'):r.restore(s)
|
||||
self.assertEqual(len(f.calls),count);self.assertEqual(controls[-1],'PASS')
|
||||
r.reconcile(s);self.assertEqual(len(f.calls),count)
|
||||
r.restore(s);self.assertTrue(r.verify_restored(s)['healthy'])
|
||||
def test_unrecognized_configuration_is_never_overwritten(self):
|
||||
f,r,s,saved,controls=self.fixture();r.activate(s);f.current['Config']['Env'].append('UNRELATED_CHANGE=1');count=len(f.calls)
|
||||
with self.assertRaisesRegex(RuntimeError,'UNRECOGNIZED'):r.restore(s)
|
||||
self.assertEqual(len(f.calls),count)
|
||||
def test_unknown_stop_without_observed_stop_is_not_misreported_as_restored(self):
|
||||
f,r,s,saved,controls=self.fixture();f.failBefore='stop'
|
||||
with self.assertRaisesRegex(RuntimeError,'UNCERTAIN'):r.activate(s)
|
||||
count=len(f.calls)
|
||||
with self.assertRaisesRegex(RuntimeError,'NO_BLIND_RECREATE'):r.reconcile(s)
|
||||
self.assertTrue(s['uncertain']);self.assertTrue(f.current['State']['Running']);self.assertEqual(len(f.calls),count)
|
||||
def test_restore_still_runs_when_proxy_control_is_unreachable(self):
|
||||
f,r,s,saved,controls=self.fixture();r.activate(s)
|
||||
def unavailable(*args):raise RuntimeError('CONTROL_LOST')
|
||||
r.control=unavailable;r.restore(s);self.assertTrue(r.verify_restored(s)['healthy'])
|
||||
def test_finish_refuses_unproved_business_route_recovery(self):
|
||||
f,r,s,saved,controls=self.fixture();r.activate(s);r.restore(s)
|
||||
from unittest.mock import patch
|
||||
with patch.object(Path,'read_text',return_value='OFFLINE'),patch.object(Path,'is_symlink',return_value=False):
|
||||
with self.assertRaisesRegex(RuntimeError,'ROUTES_NOT_PROVED'):r.finish(s,{})
|
||||
def test_changed_nacos_configuration_prevents_recovery_claim(self):
|
||||
f,r,s,saved,controls=self.fixture();r.activate(s);r.restore(s);r.config_fingerprint=lambda *args:'changed'
|
||||
with self.assertRaisesRegex(RuntimeError,'NACOS_ROUTE_CONFIGURATION_CHANGED'):r.verify_restored(s)
|
||||
def test_nacos_source_reads_are_explicit_authenticated_and_redacted(self):
|
||||
from unittest.mock import patch
|
||||
f,r,s,saved,controls=self.fixture()
|
||||
f.current['Config']['Env']+=['NACOS_SERVER_ADDR=host.docker.internal:8848','NACOS_USERNAME=offline-user',
|
||||
'NACOS_PASSWORD=offline-password','NACOS_GROUP=BAIYUE','NACOS_NAMESPACE=test-only']
|
||||
routing={'nacos':{'baseUrl':'http://127.0.0.1:8848','loginPath':'/nacos/v1/auth/users/login','configPath':'/nacos/v1/cs/configs'}}
|
||||
calls=[]
|
||||
class Response:
|
||||
status=200
|
||||
def __init__(self,value):self.value=value
|
||||
def __enter__(self):return self
|
||||
def __exit__(self,*args):pass
|
||||
def read(self):return self.value
|
||||
def read(req,timeout):
|
||||
calls.append(req)
|
||||
return Response(b'{"accessToken":"offline-token"}' if not isinstance(req,str) else b'spring:\n test: only\n')
|
||||
with patch.object(m,'http_open',side_effect=read):
|
||||
value=m.Runtime.config_fingerprint(r,f.current,routing)
|
||||
self.assertEqual(len(value),64);self.assertEqual(len(calls),3)
|
||||
self.assertIn(b'password=offline-password',calls[0].data)
|
||||
self.assertTrue(all('accessToken=offline-token' in url for url in calls[1:]))
|
||||
routing['nacos']['baseUrl']='http://other.invalid:8848'
|
||||
with self.assertRaisesRegex(RuntimeError,'EXPLICIT_NACOS'):m.Runtime.config_fingerprint(r,f.current,routing)
|
||||
self.assertEqual(len(calls),3)
|
||||
def test_effective_source_extraction_excludes_all_event_and_sql_functions(self):
|
||||
self.assertEqual(set(common),{'DockerConnection','docker','inspect','env_of','require_test','recreation','fingerprint'})
|
||||
def test_control_token_is_not_sent_to_an_unowned_listener(self):
|
||||
from unittest.mock import patch
|
||||
f,r,s,saved,controls=self.fixture();s.update(proxyPid=777,controlToken='private-control')
|
||||
command=str(r.directory/'proxy.py').encode()+b'\0'
|
||||
tcp='header\n 0: 00000000:3039 00000000:0000 0A 0:0 0:0 0 1000 0 123\n'
|
||||
with patch.object(Path,'read_bytes',return_value=command),patch.object(Path,'iterdir',return_value=iter([Path('fd')])), \
|
||||
patch.object(m.os,'readlink',return_value='socket:[123]'),patch.object(Path,'read_text',return_value=tcp), \
|
||||
patch.object(m,'http_open') as request:
|
||||
with self.assertRaisesRegex(RuntimeError,'LISTENERS_NOT_OWNED'):m.Runtime.control(r,s)
|
||||
request.assert_not_called()
|
||||
result=unittest.TextTestRunner().run(unittest.defaultTestLoader.loadTestsFromTestCase(RuntimeTests))
|
||||
sys.exit(0 if result.wasSuccessful() else 1)
|
||||
`
|
||||
const result=spawnSync(process.env.PYTHON||'python',['-c',script,
|
||||
fileURLToPath(new URL('./order-submit-dependency-fault-runtime.py',import.meta.url)),
|
||||
fileURLToPath(new URL('./test-event-fault-runtime.py',import.meta.url))],{encoding:'utf8',env:{...process.env,PYTHONDONTWRITEBYTECODE:'1'}})
|
||||
assert.equal(result.status,0,result.stderr || result.error?.message)
|
||||
assert.match(result.stderr,/Ran 10 tests/)
|
||||
})
|
||||
Reference in New Issue
Block a user