test(order): verify warehouse faults and await recovery projection

This commit is contained in:
2026-10-04 00:04:36 +08:00
parent 3f6083ff02
commit e8cf5e70ba
8 changed files with 229 additions and 51 deletions
+1 -1
View File
@@ -48,7 +48,7 @@ F40 专项可用 `--signature-mode ROOT_FIRST|CHILD_FIRST|ROOT_SIGNED_REMAINDER_
`DEPARTED_VEHICLE_CREATED` 是独立的显式恢复阶段:只接受已验证成本、唯一接替车辆创建 POST 明确成功、随后仅因车牌大小写断言停止的报告。执行前核对精确报告 SHA、原成功响应、车辆完整当前快照,并通过 Jenkins 一致性只读查询确认该车辆在 `ll_fleet_resource_assignment` 中没有占用;缺少证据、车辆变更或占用均停止。已创建车辆显式传给交接助手,跳过原车辆创建 POST,从新司机及证照继续;原成本和 DEPART 保留。该成功写入不能借用只读失败恢复阶段忽略,也不得自动重发。
`order-submit-dependency-fault-executor.mjs` 是显式 TEST 下单依赖故障入口,内部使用 `order-submit-dependency-fault-proxy.py`、`order-submit-dependency-fault-runtime.py`。`plan --plan <文件>` 只读校验;`preflight --plan <文件> --routing <文件> --confirm-test-runtime yes` 核对测试 Order 镜像、实际依赖和 Nacos v3 配置(固定v3登录与Admin配置读取、校验namespaceId/groupName/dataId后仅记录内容摘要,不回退v1);`run` 另需 `--expected-image <镜像ID>`、全新 `--report <文件>`。凭据仅从明确指定的 TEST 与 Jenkins 环境文件读取,路由文件不得包含凭据。必须事先准备本轮两个独立合法草稿、原始版本与幂等键及完整快照;仅对本轮客户的 IAM 资格和 Finance 钱包 GET 注入短时故障,验证两端提交失败后事实不变。运行会临时重建同镜像 Order 容器并在恢复后真实提交草稿,因此不得与部署或其他共享环境造数并行;不写业务 SQL。结束先恢复直通、原镜像配置、健康与依赖读取,再停止代理;失败保留报告和受限恢复文件,不自动重跑或删除数据。结果未知时使用 `status`、`reconcile` 核对原操作,必要时显式 `restore --plan <文件> --report <新文件> --confirm-test-runtime yes`,不得因观察超时重复创建容器。离线合同为对应 proxy/executor `.test.mjs`,离线通过不代表真实故障注入已验收。
`order-submit-dependency-fault-executor.mjs` 是显式 TEST 下单依赖故障入口,内部使用 `order-submit-dependency-fault-proxy.py`、`order-submit-dependency-fault-runtime.py`。`plan --plan <文件>` 只读校验;`preflight --plan <文件> --routing <文件> --confirm-test-runtime yes` 核对测试 Order 镜像、实际依赖和 Nacos v3 配置(固定v3登录与Admin配置读取、校验namespaceId/groupName/dataId后仅记录内容摘要,不回退v1);`run` 另需 `--expected-image <镜像ID>`、全新 `--report <文件>`。凭据仅从明确指定的 TEST 与 Jenkins 环境文件读取,路由文件不得包含凭据。必须事先准备本轮两个独立合法草稿、原始版本与幂等键及完整快照;仅对本轮客户的 IAM 资格、Finance 钱包及两张草稿共同引用的公共收货仓 GET 注入短时故障;Warehouse 阶段还限定原请求 ID、Order 来源与内部身份,不影响其他客户对同仓的读取。验证三依赖、两端共六次提交失败后事实不变。运行会临时重建同镜像 Order 容器并在恢复后真实提交草稿,因此不得与部署或其他共享环境造数并行;不写业务 SQL。结束先恢复直通、原镜像配置、健康与依赖读取;业务恢复必须等实际唯一入库单的SUBMITTED投影收敛,不能以连续空结果认定完成,随后按原版本原键重放并核对完整快照,再停止代理;失败保留报告和受限恢复文件,不自动重跑或删除数据。结果未知时使用 `status`、`reconcile` 核对原操作,必要时显式 `restore --plan <文件> --report <新文件> --confirm-test-runtime yes`,不得因观察超时重复创建容器。离线合同为对应 proxy/executor `.test.mjs`,离线通过不代表真实故障注入已验收。
`fba-core-regression-test-env.mjs` 为显式 TEST 的 F40 三模式串行 HTTP 专项,默认 `--plan` 只显示计划;`--execute --environment TEST --run-id <新批次>` 执行根先签、子先签、根签收后余量取消。`--tax-mode` 只接受 `BUY_DECLARATION` 或 `TAX_REFUND`,省略时保持买单默认语义。显式 `--tax-mode TAX_REFUND` 将税制传给现有 `context.f40TaxMode`,三种签收顺序均复用正式 F40 依赖编排及退税内部模块,走资料上传、回执确认、正式申报与人工放行、完成文件、尾程签收,并在最终 `COMPLETED` 后验证客户报关资料读取和文件下载;不复制报关流程。计划、manifest、汇总、各模式报告和只读取证日志均标明税制及范围。会通过正式 API 创建客户、单据、文件并使用共享 PDA/托盘,临时 PDA 按原模块 finally 停用。`fba-core-regression-evidence.mjs` 是内部只读取证 helper,Jenkins scriptText 仅在明确 TEST 节点对 TEST 容器所用数据库执行一致性只读 SELECT,不触发部署、不写业务 SQL。凭据通过显式 `node --env-file="<TEST配置>" --env-file="<Jenkins配置>"` 提供,数据库凭据只在执行节点内存读取。
@@ -6,7 +6,7 @@ import { fixtureInboundCode, orderPersistenceFacts, requestedOrderDestinationTyp
buildOrderDependencyFaultPlan, assertOrderDependencyFaultObservation } from './orders.mjs'
function dependencyFixture() {
const detail = id => ({order:{id,orderNo:`FL${id}`,tenantId:'20',status:'DRAFT',version:4},
const detail = id => ({order:{id,orderNo:`FL${id}`,tenantId:'20',receivingWarehouseId:'30',status:'DRAFT',version:4},
cargoes:[{skuNo:'current'}],parties:[],internalNotes:[],statusHistory:[]})
return {marker:'CURRENT-RUN',customerId:'20',registeredCustomerId:'20',merchant:detail('101'),platform:detail('102')}
}
@@ -20,23 +20,28 @@ function dependencyObservation(plan, fixture, dependency, role) {
test('dependency plan binds two new-customer drafts and only named Order upstream properties; it performs no runtime actions',()=>{
const fixture=dependencyFixture(),plan=buildOrderDependencyFaultPlan(fixture)
assert.equal(plan.runtimeChangesExecuted,false)
assert.deepEqual(plan.phases.map(row=>row.urlProperty),['baiyue.clients.iam.url','baiyue.clients.finance.url'])
assert.deepEqual(plan.phases.map(row=>row.urlProperty),['baiyue.clients.iam.url','baiyue.clients.finance.url','baiyue.clients.warehouse.url'])
assert.equal(plan.receivingWarehouseId,'30')
assert.equal(plan.phases[2].targetPath,'/internal/warehouses/0/30/active')
assert.equal(plan.commands.merchant.body.expectedVersion,4)
fixture.merchant.order.version=9
assert.equal(plan.commands.merchant.body.expectedVersion,4)
assert.ok(Object.isFrozen(plan.commands.merchant.body))
for(const change of [f=>f.customerId='0',f=>f.registeredCustomerId='21',f=>f.platform.order.id='101',f=>f.merchant.order.status='SUBMITTED',f=>delete f.merchant.cargoes]) {
for(const change of [f=>f.customerId='0',f=>f.registeredCustomerId='21',f=>f.platform.order.id='101',f=>f.merchant.order.status='SUBMITTED',f=>delete f.merchant.cargoes,f=>f.platform.order.receivingWarehouseId='31',f=>delete f.merchant.order.receivingWarehouseId]) {
const value=dependencyFixture();change(value);assert.throws(()=>buildOrderDependencyFaultPlan(value))
}
})
test('all four dependency-role observations require matching authenticated 503 injection and actual Order S0503 with zero drift',()=>{
test('all six dependency-role observations require matching authenticated 503 injection and actual Order S0503 with zero drift',()=>{
const fixture=dependencyFixture(),plan=buildOrderDependencyFaultPlan(fixture)
for(const dependency of ['IAM','FINANCE'])for(const role of ['merchant','platform']) {
for(const dependency of ['IAM','FINANCE','WAREHOUSE'])for(const role of ['merchant','platform']) {
const evidence=dependencyObservation(plan,fixture,dependency,role)
assert.equal(assertOrderDependencyFaultObservation(plan,dependency,role,evidence).verified,true)
const wrapped=structuredClone(evidence)
wrapped.response.body.message=plan.phases.find(row=>row.dependency===dependency).expectedMessages[1]
assert.equal(assertOrderDependencyFaultObservation(plan,dependency,role,wrapped).verified,true)
const alternative=plan.phases.find(row=>row.dependency===dependency).expectedMessages[1]
if(alternative) {
const wrapped=structuredClone(evidence)
wrapped.response.body.message=alternative
assert.equal(assertOrderDependencyFaultObservation(plan,dependency,role,wrapped).verified,true)
}
for(const mutate of [
value=>value.response.http=503,value=>value.response.body.code='A0403',value=>value.response.body.message='资格未审核',
value=>value.request.body.expectedVersion=5,value=>value.request.headers['Idempotency-Key']='new-key',
+7 -1
View File
@@ -260,12 +260,18 @@ export function buildOrderDependencyFaultPlan(fixture) {
initialOrderSha256:createHash('sha256').update(orderPersistenceFacts(detail)).digest('hex') }
}
if (commands.merchant.orderId === commands.platform.orderId) throw new Error('Use two independent drafts for same-key recovery')
return { marker:fixture.marker, customerId, runtimeChangesExecuted:false, commands,
const receivingWarehouseId = id(fixture.merchant.order.receivingWarehouseId)
if (!/^[1-9]\d*$/.test(receivingWarehouseId) || receivingWarehouseId !== id(fixture.platform.order.receivingWarehouseId)) {
throw new Error('Dependency drafts must reference the same valid public receiving warehouse')
}
return { marker:fixture.marker, customerId, receivingWarehouseId, runtimeChangesExecuted:false, commands,
phases:[
{ dependency:'IAM', urlProperty:'baiyue.clients.iam.url', targetMethod:'GET', targetPath:`/internal/iam/tenants/${customerId}/order-eligibility`,
faultHttp:503, expectedHttp:200, expectedCode:'S0503', expectedMessages:['IAM eligibility service is unavailable','IAM 下单资格响应无效'] },
{ dependency:'FINANCE', urlProperty:'baiyue.clients.finance.url', targetMethod:'GET', targetPath:`/internal/finance/v1/wallet/accounts/${customerId}`,
faultHttp:503, expectedHttp:200, expectedCode:'S0503', expectedMessages:['Finance account service is unavailable','Finance account response is invalid'] },
{ dependency:'WAREHOUSE', urlProperty:'baiyue.clients.warehouse.url', targetMethod:'GET', targetPath:`/internal/warehouses/0/${receivingWarehouseId}/active`,
faultHttp:503, expectedHttp:200, expectedCode:'S0503', expectedMessages:['主数据服务暂时不可用'] },
],
recovery:'Restore original Order image/config/network identity; submit each original frozen command once, then replay it; require one submission history and no duplicate Firstleg facts.' }
}
@@ -29,17 +29,19 @@ export function scopeFor(plan) {
}
assert(plan.commands.merchant.orderId !== plan.commands.platform.orderId, 'Two distinct new drafts required')
assert(plan.runtimeChangesExecuted === false, 'Only a reviewed unexecuted plan is accepted')
assert(Array.isArray(plan.phases) && plan.phases.length === 2, 'Exact two dependency phases required')
assert(/^[1-9]\d{0,19}$/.test(plan.receivingWarehouseId || ''), 'Bounded public receiving warehouse required')
assert(Array.isArray(plan.phases) && plan.phases.length === 3, 'Exact three dependency phases required')
for (const [dependency, property, path, messages] of [
['IAM','baiyue.clients.iam.url',`/internal/iam/tenants/${plan.customerId}/order-eligibility`,['IAM eligibility service is unavailable','IAM 下单资格响应无效']],
['FINANCE','baiyue.clients.finance.url',`/internal/finance/v1/wallet/accounts/${plan.customerId}`,['Finance account service is unavailable','Finance account response is invalid']],
['WAREHOUSE','baiyue.clients.warehouse.url',`/internal/warehouses/0/${plan.receivingWarehouseId}/active`,['主数据服务暂时不可用']],
]) {
const rows = plan.phases.filter(row => row.dependency === dependency)
assert(rows.length === 1 && rows[0].urlProperty === property && rows[0].targetPath === path && rows[0].targetMethod === 'GET'
&& rows[0].faultHttp === 503 && rows[0].expectedHttp === 200 && rows[0].expectedCode === 'S0503'
&& JSON.stringify(rows[0].expectedMessages) === JSON.stringify(messages), 'Dependency failure contract may not be weakened')
}
return { marker: plan.marker, customerId: plan.customerId,
return { marker: plan.marker, customerId: plan.customerId, receivingWarehouseId: plan.receivingWarehouseId,
orderIds: Object.fromEntries(['merchant', 'platform'].map(role => [role, plan.commands[role].orderId])), planSha256: sha(plan) }
}
@@ -50,7 +52,7 @@ export function renderOrderFaultAction(action, scope, agent, extra = {}) {
const routing = extra.routing
assert(routing && Object.keys(routing).every(key => ['configurationReviewed','sourceSha256','upstreams','nacos'].includes(key))
&& routing.configurationReviewed === true && routing.nacos && Object.keys(routing.nacos).every(key => ['baseUrl','loginPath','configPath'].includes(key))
&& routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM', 'Reviewed routing schema only; do not embed credentials')
&& routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM,WAREHOUSE', 'Reviewed routing schema only; do not embed credentials')
for (const address of [routing.nacos.baseUrl,...Object.values(routing.upstreams)]) {
const url = new URL(address)
assert(['http:','https:'].includes(url.protocol) && !url.username && !url.password && !url.search && !url.hash
@@ -60,8 +62,9 @@ export function renderOrderFaultAction(action, scope, agent, extra = {}) {
&& /^[0-9a-f]{64}$/.test(routing.sourceSha256 || ''), 'Reviewed image and Nacos source fingerprint required')
}
const data = encode(JSON.stringify({ ...extra, action, scope, commonSource, ...(action === 'prepare' ? { proxySource } : {}) }))
const expression = value => '[' + encode(value).match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')"
const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),'${data}']).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
const literal = encoded => '[' + encoded.match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')"
const expression = value => literal(encode(value))
const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),${literal(data)}]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null || c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${expression(groovy)}),'UTF-8'),c.channel))`
}
@@ -78,7 +81,10 @@ export function createOrderFaultRuntime(env, scope, fetcher = globalThis.fetch)
})
} catch { throw Error(`RUNTIME_${action.toUpperCase().replaceAll('-', '_')}_OUTCOME_UNKNOWN_USE_STATUS`) }
assert(response.ok, `RUNTIME_HTTP_${response.status}_NO_AUTO_RETRY`)
const rows = (await response.text()).split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON='))
const output = await response.text()
assert(!(output.includes('MultipleCompilationErrorsException') && output.includes('String too long')),
'RUNTIME_RENDERER_COMPILATION_FAILED_NO_ACTION')
const rows = output.split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON='))
assert(rows.length === 1, 'RUNTIME_RESPONSE_AMBIGUOUS_NO_AUTO_RETRY')
const result = JSON.parse(rows[0].slice('ORDER_DEPENDENCY_JSON='.length))
assert(result?.ok === true, /^[A-Z_0-9]{1,90}$/.test(result?.error || '') ? result.error : 'RUNTIME_ACTION_FAILED')
@@ -103,6 +109,7 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti
assert(createHash('sha256').update(orderPersistenceFacts(before.detail)).digest('hex') === plan.commands[role].initialOrderSha256
&& before.detail.order.status === 'DRAFT' && String(before.detail.order.tenantId) === scope.customerId
&& String(before.detail.order.id) === scope.orderIds[role]
&& String(before.detail.order.receivingWarehouseId) === scope.receivingWarehouseId
&& ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(before.detail[field]))
&& before.inboundPageComplete === true, 'PREPARED_DRAFT_CHANGED')
}
@@ -113,7 +120,7 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti
assert(i < 14, 'PROXY_STARTUP_NOT_READY'); await wait(1000)
}
await action('activate'); await health()
for (const dependency of ['IAM','FINANCE']) {
for (const dependency of ['IAM','FINANCE','WAREHOUSE']) {
for (const role of ['merchant','platform']) {
const before = await snapshot(role), command = plan.commands[role]
const request = { method: command.method, path: command.path, body: structuredClone(command.body),
@@ -138,20 +145,51 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti
&& report.restoredRuntime.currentImageId === expectedImageId
&& report.restoredRuntime.originalFingerprint === report.restoredRuntime.currentFingerprint
&& report.restoredRuntime.nacosSourcesUnchanged === true
&& report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true,
&& report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true
&& report.restoredRuntime.upstreamReads?.WAREHOUSE === true,
'RESTORED_IDENTITY_OR_ROUTES_NOT_PROVED')
const businessRecovery = await recovery()
report.businessRecovery = businessRecovery
report.restoration = await action('finish', { businessRecovery })
assert(report.restoration.restored && report.restoration.proxyStopped && report.restoration.orderRoutesVerified, 'RESTORATION_NOT_COMPLETE')
} catch (error) { report.recoveryError = safeError(error); failure ||= error }
} catch (error) {
report.recoveryError = safeError(error)
if (error.recoveryDiagnostics) report.recoveryDiagnostics = error.recoveryDiagnostics
failure ||= error
}
}
}
report.success = !failure && report.observations.length === 4 && Boolean(report.restoration?.restored)
report.success = !failure && report.observations.length === 6 && Boolean(report.restoration?.restored)
if (failure) report.error = safeError(failure)
return report
}
export async function waitForRecoveredSubmission({ plan, role, snapshot, wait = pause }) {
const command = plan.commands[role]
let previous, orderFacts
for (let i = 0; i <= 60; i++) {
const current = await snapshot(role), detail = current.detail
assert(current.inboundPageComplete === true && Array.isArray(current.inbounds) && current.inbounds.length <= 1
&& ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(detail?.[field]))
&& String(detail.order.id) === command.orderId && String(detail.order.tenantId) === plan.customerId
&& detail.order.status === 'SUBMITTED' && Number(detail.order.version) === command.body.expectedVersion + 1
&& detail.statusHistory.filter(row => row.action === 'SUBMITTED').length === 1
&& detail.statusHistory.filter(row => row.action === 'CREATE').length === 1
&& (current.task === null || current.task && String(current.task.orderId) === command.orderId
&& String(current.task.tenantId) === plan.customerId)
&& current.inbounds.every(row => String(row.orderId) === command.orderId && row.orderNo === command.orderNo
&& String(row.tenantId) === plan.customerId), 'RECOVERY_SUBMISSION_FACTS_INVALID')
const facts = orderPersistenceFacts(detail)
orderFacts ??= facts
assert(facts === orderFacts, 'RECOVERY_ORDER_DRIFT')
const projected = current.inbounds.length === 1 && current.inbounds[0].orderStatusSnapshot === 'SUBMITTED'
if (projected && previous && sha(previous) === sha(current)) return current
previous = projected ? current : undefined
if (i < 60) await wait(500)
}
throw Error('RECOVERY_PROJECTION_TIMEOUT')
}
function businessClient(env, plan, fetcher = globalThis.fetch) {
const sessions = {}
const call = async (method, path, body, headers = {}, token) => {
@@ -191,16 +229,19 @@ function businessClient(env, plan, fetcher = globalThis.fetch) {
},
async recovery() {
const evidence={}
for(const role of ['merchant','platform']) {
const command=plan.commands[role]
ok(await submit(role,command))
let before=await snapshot(role)
for(let i=0;i<10;i++) { await pause(500);const next=await snapshot(role);if(sha(before)===sha(next)){before=next;break}before=next }
assert(before.detail.order.status==='SUBMITTED' && before.detail.statusHistory.filter(row=>row.action==='SUBMITTED').length===1
&& before.inbounds.length<=1, 'RECOVERY_SINGLE_SUBMISSION_NOT_PROVED')
ok(await submit(role,command));const after=await snapshot(role)
assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT')
evidence[role]={orderId:command.orderId,status:'SUBMITTED',sameKeyReplayVerified:true,snapshotSha256:sha(after)}
try {
for(const role of ['merchant','platform']) {
const command=plan.commands[role]
ok(await submit(role,command))
const before=await waitForRecoveredSubmission({plan,role,snapshot})
ok(await submit(role,command));const after=await snapshot(role)
evidence[role]={orderId:command.orderId,before,after,snapshotSha256:sha(after)}
assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT')
Object.assign(evidence[role],{status:'SUBMITTED',sameKeyReplayVerified:true})
}
} catch(error) {
error.recoveryDiagnostics=evidence
throw error
}
return evidence
},
@@ -217,7 +258,7 @@ async function main(argv) {
for(let i=0;i<rest.length;i+=2){assert(rest[i]?.startsWith('--') && rest[i+1] && !rest[i+1].startsWith('--'),'Use explicit --name value arguments');args[rest[i].slice(2)]=rest[i+1]}
assert(['plan','preflight','run','status','reconcile','restore'].includes(mode),'Use plan/preflight/run/status/reconcile/restore')
assert(args.plan,'--plan required');const plan=JSON.parse(readFileSync(args.plan,'utf8')),scope=scopeFor(plan)
if(mode==='plan'){console.log(JSON.stringify({scope,actions:['prepare','activate','IAM x 2','FINANCE x 2','finally restore','verify original routes','finish proxy'],requiresExclusiveOrderWindow:true},null,2));return}
if(mode==='plan'){console.log(JSON.stringify({scope,actions:['prepare','activate','IAM x 2','FINANCE x 2','WAREHOUSE x 2','finally restore','verify original routes','finish proxy'],requiresExclusiveOrderWindow:true},null,2));return}
assert(args['confirm-test-runtime']==='yes','--confirm-test-runtime yes required')
const action=createOrderFaultRuntime(process.env,scope)
if(mode==='preflight'){assert(args.routing,'--routing required');console.log(JSON.stringify(await action('preflight',{routing:JSON.parse(readFileSync(args.routing,'utf8'))}),null,2));return}
@@ -3,16 +3,47 @@ import assert from 'node:assert/strict'
import { spawnSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
import { buildOrderDependencyFaultPlan } from './admin-test-env-modules/orders.mjs'
import { scopeFor, renderOrderFaultAction, createOrderFaultRuntime, runOrderFaultWindow } from './order-submit-dependency-fault-executor.mjs'
import { scopeFor, renderOrderFaultAction, createOrderFaultRuntime, runOrderFaultWindow, waitForRecoveredSubmission } from './order-submit-dependency-fault-executor.mjs'
const image = 'sha256:' + 'a'.repeat(64)
function fixture() {
const detail = id => ({order:{id,tenantId:'20',orderNo:`FL${id}`,version:2,status:'DRAFT'},cargoes:[],parties:[],internalNotes:[],statusHistory:[]})
const detail = id => ({order:{id,tenantId:'20',receivingWarehouseId:'30',orderNo:`FL${id}`,version:2,status:'DRAFT'},cargoes:[],parties:[],internalNotes:[],statusHistory:[]})
const raw={marker:'TEST-DEPENDENCY',customerId:'20',registeredCustomerId:'20',merchant:detail('101'),platform:detail('102')}
return {raw,plan:buildOrderDependencyFaultPlan(raw)}
}
const env={CHONGCHONG_ENV:'test',JENKINS_URL:'https://offline.invalid/',JENKINS_USER:'offline',JENKINS_API_TOKEN:'never-print-this',FBA_CORE_JENKINS_AGENT:'test-agent'}
function recoverySnapshot(projected=true) {
const {raw,plan}=fixture(),detail=structuredClone(raw.merchant)
detail.order.status='SUBMITTED';detail.order.version=3
detail.statusHistory=[{action:'CREATE'},{action:'SUBMITTED'}]
return {plan,snapshot:{detail,task:null,inbounds:projected?[{id:'1001',orderId:'101',orderNo:'FL101',tenantId:'20',orderStatusSnapshot:'SUBMITTED'}]:[],inboundPageComplete:true}}
}
test('recovery waits for the actual inbound projection even when two empty reads are stable',async()=>{
const {plan,snapshot}=recoverySnapshot(),empty=recoverySnapshot(false).snapshot
const values=[empty,empty,snapshot,snapshot];let reads=0
const value=await waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>values[Math.min(reads++,values.length-1)],wait:async()=>{}})
assert.equal(reads,4);assert.deepEqual(value,snapshot)
})
test('missing submission projection cannot pass recovery after the bounded wait',async()=>{
const {plan,snapshot}=recoverySnapshot(false);let reads=0
await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>{reads++;return snapshot},wait:async()=>{}}),/RECOVERY_PROJECTION_TIMEOUT/)
assert.equal(reads,61)
})
test('recovery rejects duplicate or foreign projected inbounds instead of waiting them away',async()=>{
for(const change of [s=>s.inbounds.push({...s.inbounds[0],id:'1002'}),s=>s.inbounds[0].tenantId='21',s=>s.inbounds[0].orderId='102']) {
const {plan,snapshot}=recoverySnapshot();change(snapshot)
await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>snapshot,wait:async()=>{}}),/RECOVERY_SUBMISSION_FACTS_INVALID/)
}
})
test('recovery projection wait rejects order fact drift and incomplete pagination',async()=>{
const {plan,snapshot}=recoverySnapshot(),changed=structuredClone(snapshot);changed.detail.order.remark='late mutation'
let reads=0
await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>reads++?changed:snapshot,wait:async()=>{}}),/RECOVERY_ORDER_DRIFT/)
const incomplete=structuredClone(snapshot);incomplete.inboundPageComplete=false
await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>incomplete,wait:async()=>{}}),/RECOVERY_SUBMISSION_FACTS_INVALID/)
})
test('runtime classifies dependency HTTP stages without exposing tokens, URLs or response bodies',()=>{
const script=String.raw`
import importlib.util,sys,urllib.error
@@ -42,6 +73,23 @@ test('renderer binds TEST scope and emits no Jenkins credential; ambiguous remot
await assert.rejects(runtime('activate'),/OUTCOME_UNKNOWN_USE_STATUS/)
assert.equal(calls,1)
})
test('renderer chunks every embedded Groovy literal and preserves the complete runtime payload',()=>{
const {plan}=fixture(),scope=scopeFor(plan)
const routing={configurationReviewed:true,nacos:{baseUrl:'http://nacos.invalid:8848',loginPath:'/nacos/v3/auth/user/login',configPath:'/nacos/v3/admin/cs/config'},upstreams:{IAM:'http://iam.invalid:9201',FINANCE:'http://finance.invalid:9203',WAREHOUSE:'http://warehouse.invalid:9205'}}
const rendered=renderOrderFaultAction('preflight',scope,'test-agent',{routing})
const arrays=text=>Array.from(text.matchAll(/\[('[A-Za-z0-9+/=]+'(?:,'[A-Za-z0-9+/=]+')*)\]\.join\(''\)/g),match=>Array.from(match[1].matchAll(/'([^']+)'/g),part=>part[1]))
const outer=arrays(rendered)
assert.equal(outer.length,1)
const inner=Buffer.from(outer[0].join(''),'base64').toString('utf8')
assert.ok(Array.from(inner.matchAll(/'([^']*)'/g),match=>match[1].length).every(length=>length<=4000))
const embedded=arrays(inner)
assert.equal(embedded.length,2)
const config=JSON.parse(Buffer.from(embedded[1].join(''),'base64').toString('utf8'))
assert.deepEqual(config.scope,scope)
assert.deepEqual(config.routing,routing)
assert.equal(config.action,'preflight')
assert.ok(Buffer.from(config.commonSource,'base64').length>0)
})
function windowFixture(fault='') {
const {raw,plan}=fixture(),calls=[],records=[]
@@ -55,7 +103,7 @@ function windowFixture(fault='') {
if(action==='health')return {healthy:true}
if(action==='mode'){mode=extra.mode;return {mode}}
if(action==='records')return {records}
if(action==='verify-restored')return {originalImageId:image,currentImageId:image,originalFingerprint:'same',currentFingerprint:'same',upstreamReads:{IAM:true,FINANCE:true},nacosSourcesUnchanged:true}
if(action==='verify-restored')return {originalImageId:image,currentImageId:image,originalFingerprint:'same',currentFingerprint:'same',upstreamReads:{IAM:true,FINANCE:true,WAREHOUSE:true},nacosSourcesUnchanged:true}
if(action==='finish')return {restored:true,healthy:true,proxyStopped:true,orderRoutesVerified:true}
return {}
},
@@ -72,9 +120,9 @@ function windowFixture(fault='') {
return {options,calls}
}
test('four observations restore original routes and verify business recovery before stopping proxy',async()=>{
test('six observations restore original routes and verify business recovery before stopping proxy',async()=>{
const {options,calls}=windowFixture(),report=await runOrderFaultWindow(options)
assert.equal(report.success,true);assert.equal(report.observations.length,4)
assert.equal(report.success,true);assert.equal(report.observations.length,6)
assert.ok(calls.indexOf('restore')<calls.indexOf('verify-restored'))
assert.ok(calls.indexOf('verify-restored')<calls.indexOf('business-recovery'))
assert.ok(calls.indexOf('business-recovery')<calls.indexOf('finish'))
@@ -84,6 +132,28 @@ test('failed business assertion still restores, but never reports fault verifica
assert.equal(report.success,false);assert.equal(report.restoration.restored,true)
assert.ok(calls.includes('finish'))
})
test('a redirected public warehouse plan stops before any runtime mutation',async()=>{
const {options,calls}=windowFixture()
options.plan.receivingWarehouseId='31'
options.plan.phases[2].targetPath='/internal/warehouses/0/31/active'
const report=await runOrderFaultWindow(options)
assert.equal(report.success,false)
assert.ok(!calls.includes('prepare'))
assert.ok(!calls.includes('activate'))
})
test('unverified warehouse restoration blocks business recovery and proxy finish',async()=>{
const {options,calls}=windowFixture(),original=options.action
options.action=async(action,extra)=>{
const result=await original(action,extra)
if(action==='verify-restored')result.upstreamReads.WAREHOUSE=false
return result
}
const report=await runOrderFaultWindow(options)
assert.equal(report.success,false)
assert.ok(calls.includes('restore'))
assert.ok(!calls.includes('business-recovery'))
assert.ok(!calls.includes('finish'))
})
for(const fault of ['prepare-unknown','restore-unknown','business-recovery-failed'])test(`unsafe cleanup blocks finish: ${fault}`,async()=>{
const {options,calls}=windowFixture(fault),report=await runOrderFaultWindow(options)
assert.equal(report.success,false);assert.equal(calls.filter(action=>action==='prepare').length,1)
@@ -131,15 +201,37 @@ class Fake:
class RuntimeTests(unittest.TestCase):
def fixture(self):
f=Fake();tools=dict(common);tools.update(inspect=f.inspect,docker=f.docker)
scope={'marker':'OFFLINE','customerId':'7','orderIds':{'merchant':'10','platform':'11'},'planSha256':'b'*64}
scope={'marker':'OFFLINE','customerId':'7','receivingWarehouseId':'30','orderIds':{'merchant':'10','platform':'11'},'planSha256':'b'*64}
r=m.Runtime(scope,tools,Path('offline-not-created'));saved=[];controls=[]
r.save=lambda state:saved.append(copy.deepcopy(state));r.control=lambda state,mode=None,ttl=0:(controls.append(mode) or {'mode':'PASS'})
r.probes=lambda state:{'IAM':True,'FINANCE':True}
r.probes=lambda state:{'IAM':True,'FINANCE':True,'WAREHOUSE':True}
r.config_fingerprint=lambda container,routing:'reviewed-config'
state={'scope':scope,'phase':'PROXY_STARTED','originalId':f.original['Id'],'originalImageId':f.original['Image'],
'originalFingerprint':common['fingerprint'](f.original),'originalBody':common['recreation'](f.original,common['env_of'](f.original),full=True),
'proxyPort':12345,'controlPort':12346,'mutation':None,'routing':{},'routingSourceSha256':'reviewed-config'}
return f,r,state,saved,controls
def test_warehouse_probe_requires_the_exact_active_public_domestic_warehouse(self):
import io
from unittest.mock import patch
f,r,s,saved,controls=self.fixture()
state={'internalToken':'never-public','upstreams':{'IAM':'http://iam.invalid','FINANCE':'http://finance.invalid','WAREHOUSE':'http://warehouse.invalid'}}
warehouse={'id':30,'tenantId':0,'status':'ACTIVE','warehouseType':'DOMESTIC','warehouseName':'Scoped receiving warehouse'}
class Response(io.BytesIO):status=200
def response(request,timeout):
url=request.full_url
if '/internal/iam/' in url:data={'eligible':True}
elif '/internal/finance/' in url:data={'actualBalance':0}
else:
self.assertTrue(url.endswith('/internal/warehouses/0/30/active'))
data={'warehouse':warehouse}
return Response(json.dumps({'code':'200','data':data}).encode())
with patch.object(m,'http_open',side_effect=response):
self.assertEqual(m.Runtime.probes(r,state),{'IAM':True,'FINANCE':True,'WAREHOUSE':True})
for field,value in [('id',31),('tenantId',20),('status','DISABLED'),('warehouseType','FBA'),('warehouseName','')]:
original=warehouse[field];warehouse[field]=value
with self.assertRaisesRegex(RuntimeError,'RECEIVING_WAREHOUSE_NOT_READY'):m.Runtime.probes(r,state)
warehouse[field]=original
self.assertEqual(f.calls,[])
def test_original_fingerprint_config_and_image_restored(self):
f,r,s,saved,controls=self.fixture();r.activate(s)
self.assertTrue(r.matches_injected(f.current,s));self.assertEqual(s['phase'],'INJECTED')
@@ -227,5 +319,5 @@ sys.exit(0 if result.wasSuccessful() else 1)
fileURLToPath(new URL('./order-submit-dependency-fault-runtime.py',import.meta.url)),
fileURLToPath(new URL('./test-event-fault-runtime.py',import.meta.url))],{encoding:'utf8',env:{...process.env,PYTHONDONTWRITEBYTECODE:'1'}})
assert.equal(result.status,0,result.stderr || result.error?.message)
assert.match(result.stderr,/Ran 10 tests/)
assert.match(result.stderr,/Ran 11 tests/)
})
@@ -49,10 +49,14 @@ class Router:
customer = config['customerId']
if not re.fullmatch(r'[1-9][0-9]{0,19}', customer):
raise ValueError('INVALID_CUSTOMER')
warehouse = config.get('receivingWarehouseId', '')
if not re.fullmatch(r'[1-9][0-9]{0,19}', warehouse):
raise ValueError('INVALID_RECEIVING_WAREHOUSE')
if not config.get('requestIds') or any(not re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}', value) for value in config['requestIds']):
raise ValueError('BOUNDED_REQUEST_IDS_REQUIRED')
self.paths = {'IAM': '/internal/iam/tenants/' + customer + '/order-eligibility',
'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer}
'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer,
'WAREHOUSE': '/internal/warehouses/0/' + warehouse + '/active'}
def status(self):
with self.lock:
@@ -67,7 +71,7 @@ class Router:
with self.lock:
# Even an authenticated malformed command disarms first.
self.mode, self.deadline = 'PASS', 0
if mode not in ('PASS', 'IAM', 'FINANCE') or (mode != 'PASS' and
if mode not in ('PASS', 'IAM', 'FINANCE', 'WAREHOUSE') or (mode != 'PASS' and
(type(ttl) is not int or not 1 <= ttl <= 60)):
raise ValueError('INVALID_MODE_OR_TTL')
self.mode = mode
@@ -77,7 +81,7 @@ class Router:
def dispatch(self, method, path, headers, body, peer):
prefix, separator, rest = path.lstrip('/').partition('/')
dependency = {'iam': 'IAM', 'finance': 'FINANCE'}.get(prefix)
dependency = {'iam': 'IAM', 'finance': 'FINANCE', 'warehouse': 'WAREHOUSE'}.get(prefix)
if not dependency or not separator:
return 404, {}, b'Unknown dependency route'
actual_path = '/' + rest
@@ -10,8 +10,8 @@ spec=importlib.util.spec_from_file_location('proxy',sys.argv[1]);p=importlib.uti
class ProxyTests(unittest.TestCase):
def setUp(self):
self.now=[10.0];self.calls=[];self.logs=[]
self.config={'customerId':'7','controlToken':'private-control','internalToken':'private-internal-token',
'orderIps':['172.20.0.10'],'requestIds':['fixture-request'],'upstreams':{'IAM':'http://iam.invalid:9201','FINANCE':'http://finance.invalid:9207'}}
self.config={'customerId':'7','receivingWarehouseId':'30','controlToken':'private-control','internalToken':'private-internal-token',
'orderIps':['172.20.0.10'],'requestIds':['fixture-request'],'upstreams':{'IAM':'http://iam.invalid:9201','FINANCE':'http://finance.invalid:9207','WAREHOUSE':'http://warehouse.invalid:9202'}}
def forward(*args): self.calls.append(args);return 201,{'X-Upstream':'kept'},b'original-response'
self.router=p.Router(self.config,forward=forward,clock=lambda:self.now[0],record=self.logs.append)
self.headers={'X-Internal-Service-Token':'private-internal-token','X-Request-Id':'fixture-request'}
@@ -42,6 +42,24 @@ class ProxyTests(unittest.TestCase):
self.assertEqual(self.send(path)[0],503)
self.now[0]=15.0
self.assertEqual(self.send(path)[0],201);self.assertEqual(self.router.status()['mode'],'PASS')
def test_public_warehouse_fault_is_bounded_by_exact_warehouse_request_and_order_identity(self):
self.router.set_mode('private-control','WAREHOUSE',60)
path='/warehouse/internal/warehouses/0/30/active'
self.assertEqual(self.send(path)[0],503)
for other,method,headers,peer in [
(path.replace('/30/','/31/'),'GET',self.headers,'172.20.0.10'),
(path.replace('/0/','/7/'),'GET',self.headers,'172.20.0.10'),
(path+'?extra=1','GET',self.headers,'172.20.0.10'),
(path,'POST',self.headers,'172.20.0.10'),
(path,'GET',{**self.headers,'X-Request-Id':'another-customer-request'},'172.20.0.10'),
(path,'GET',{**self.headers,'X-Internal-Service-Token':'wrong'},'172.20.0.10'),
(path,'GET',self.headers,'172.20.0.11')]:
self.assertEqual(self.send(other,method,headers,peer)[0],201)
self.now[0]=70.0
self.assertEqual(self.send(path)[0],201)
def test_invalid_receiving_warehouse_scope_is_rejected(self):
for value in ['0','../30','30?all=true','']:
with self.assertRaises(ValueError):p.Router({**self.config,'receivingWarehouseId':value})
def test_control_auth_and_malformed_authenticated_command(self):
self.router.set_mode('private-control','IAM',10)
with self.assertRaises(PermissionError):self.router.set_mode('wrong','PASS')
@@ -65,5 +83,5 @@ sys.exit(0 if result.wasSuccessful() else 1)
fileURLToPath(new URL('./order-submit-dependency-fault-proxy.py', import.meta.url))],
{ encoding:'utf8', env:{...process.env,PYTHONDONTWRITEBYTECODE:'1'} })
assert.equal(result.status,0,result.stderr || result.error?.message)
assert.match(result.stderr,/Ran 6 tests/)
assert.match(result.stderr,/Ran 8 tests/)
})
@@ -24,8 +24,10 @@ from pathlib import Path
ORDER = 'firstleg-test-order'
ENTRYPOINT = ['sh', '-c', 'exec java $JAVA_OPTS org.springframework.boot.loader.launch.JarLauncher']
PROPERTIES = {'IAM': 'baiyue.clients.iam.url', 'FINANCE': 'baiyue.clients.finance.url'}
UPSTREAM_CONTAINERS = {'IAM': 'firstleg-test-iam', 'FINANCE': 'firstleg-test-finance'}
PROPERTIES = {'IAM': 'baiyue.clients.iam.url', 'FINANCE': 'baiyue.clients.finance.url',
'WAREHOUSE': 'baiyue.clients.warehouse.url'}
UPSTREAM_CONTAINERS = {'IAM': 'firstleg-test-iam', 'FINANCE': 'firstleg-test-finance',
'WAREHOUSE': 'firstleg-test-warehouse'}
class NoRedirect(urllib.request.HTTPRedirectHandler):
@@ -63,10 +65,12 @@ def private_write(path, value):
def validate_scope(scope):
if set(scope) != {'marker', 'customerId', 'orderIds', 'planSha256'} or not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', scope['marker']):
if set(scope) != {'marker', 'customerId', 'receivingWarehouseId', 'orderIds', 'planSha256'} or not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', scope['marker']):
raise RuntimeError('INVALID_SCOPE')
if not re.fullmatch(r'[1-9][0-9]{0,19}', scope['customerId']) or set(scope['orderIds']) != {'merchant', 'platform'}:
raise RuntimeError('INVALID_SCOPE')
if not re.fullmatch(r'[1-9][0-9]{0,19}', scope['receivingWarehouseId']):
raise RuntimeError('INVALID_RECEIVING_WAREHOUSE')
if any(not re.fullmatch(r'[1-9][0-9]{0,19}', value) for value in scope['orderIds'].values()) or len(set(scope['orderIds'].values())) != 2:
raise RuntimeError('TWO_DISTINCT_ORDERS_REQUIRED')
if not re.fullmatch(r'[0-9a-f]{64}', scope['planSha256']): raise RuntimeError('INVALID_PLAN_HASH')
@@ -125,7 +129,8 @@ class Runtime:
"""Real authenticated reads, not manufactured business responses. No body/secret is returned."""
customer = self.scope['customerId']; results = {}
targets = {'IAM': '/internal/iam/tenants/' + customer + '/order-eligibility',
'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer}
'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer,
'WAREHOUSE': '/internal/warehouses/0/' + self.scope['receivingWarehouseId'] + '/active'}
for dependency, path in targets.items():
req = urllib.request.Request(state['upstreams'][dependency] + path,
headers={'X-Internal-Service-Token': state['internalToken']})
@@ -136,6 +141,12 @@ class Runtime:
if dependency == 'IAM' and data.get('eligible') is not True: raise RuntimeError('CUSTOMER_NOT_ELIGIBLE')
if dependency == 'FINANCE' and (data.get('actualBalance') is None or float(data['actualBalance']) < 0):
raise RuntimeError('CUSTOMER_BALANCE_NOT_READY')
if dependency == 'WAREHOUSE':
warehouse = data.get('warehouse') or {}
if (str(warehouse.get('id')) != self.scope['receivingWarehouseId']
or str(warehouse.get('tenantId')) != '0' or warehouse.get('status') != 'ACTIVE'
or warehouse.get('warehouseType') != 'DOMESTIC' or not warehouse.get('warehouseName')):
raise RuntimeError('RECEIVING_WAREHOUSE_NOT_READY')
results[dependency] = True
return results
@@ -229,8 +240,9 @@ class Runtime:
proxy = self.directory / 'proxy.py'; proxy.write_bytes(base64.b64decode(config['proxySource'])); os.chmod(proxy, 0o600)
private_write(self.directory / 'proxy-config.json', {k: state[k] for k in
['internalToken', 'controlToken', 'proxyPort', 'controlPort', 'upstreams', 'orderIps']} | {'customerId': self.scope['customerId'],
'receivingWarehouseId': self.scope['receivingWarehouseId'],
'requestIds': ['dep-' + self.scope['marker'] + '-' + dependency + '-' + role
for dependency in ('IAM', 'FINANCE') for role in ('merchant', 'platform')]})
for dependency in PROPERTIES for role in ('merchant', 'platform')]})
state['phase'] = 'PROXY_START_INTENT'; state['proxyLaunchIntent'] = True; self.save(state)
with open(self.directory / 'proxy-output.log', 'ab') as output:
os.chmod(self.directory / 'proxy-output.log', 0o600)
@@ -361,7 +373,7 @@ class Runtime:
if self.config_fingerprint(current, state['routing']) != state['routingSourceSha256']:
raise RuntimeError('NACOS_ROUTE_CONFIGURATION_CHANGED')
checks = self.probes(state)
if checks != {'IAM': True, 'FINANCE': True}: raise RuntimeError('RESTORED_ROUTE_PROBE_INVALID')
if checks != {dependency: True for dependency in PROPERTIES}: raise RuntimeError('RESTORED_ROUTE_PROBE_INVALID')
state['routeReadsVerified'] = checks; self.save(state)
return {'healthy': True, 'originalFingerprint': state['originalFingerprint'], 'currentFingerprint': self.tools['fingerprint'](current),
'originalImageId': state['originalImageId'], 'currentImageId': current['Image'], 'upstreamReads': checks,