diff --git a/scripts/README.md b/scripts/README.md index 8df15425f..8da16712e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -48,7 +48,7 @@ F40 专项可用 `--signature-mode ROOT_FIRST|CHILD_FIRST|ROOT_SIGNED_REMAINDER_ `DEPARTED_VEHICLE_CREATED` 是独立的显式恢复阶段：只接受已验证成本、唯一接替车辆创建 POST 明确成功、随后仅因车牌大小写断言停止的报告。执行前核对精确报告 SHA、原成功响应、车辆完整当前快照，并通过 Jenkins 一致性只读查询确认该车辆在 `ll_fleet_resource_assignment` 中没有占用；缺少证据、车辆变更或占用均停止。已创建车辆显式传给交接助手，跳过原车辆创建 POST，从新司机及证照继续；原成本和 DEPART 保留。该成功写入不能借用只读失败恢复阶段忽略，也不得自动重发。 -`order-submit-dependency-fault-executor.mjs` 是显式 TEST 下单依赖故障入口，内部使用 `order-submit-dependency-fault-proxy.py`、`order-submit-dependency-fault-runtime.py`。`plan --plan <文件>` 只读校验；`preflight --plan <文件> --routing <文件> --confirm-test-runtime yes` 核对测试 Order 镜像、实际依赖和 Nacos v3 配置（固定v3登录与Admin配置读取、校验namespaceId/groupName/dataId后仅记录内容摘要，不回退v1）；`run` 另需 `--expected-image <镜像ID>`、全新 `--report <文件>`。凭据仅从明确指定的 TEST 与 Jenkins 环境文件读取，路由文件不得包含凭据。必须事先准备本轮两个独立合法草稿、原始版本与幂等键及完整快照；仅对本轮客户的 IAM 资格和 Finance 钱包 GET 注入短时故障，验证两端提交失败后事实不变。运行会临时重建同镜像 Order 容器并在恢复后真实提交草稿，因此不得与部署或其他共享环境造数并行；不写业务 SQL。结束先恢复直通、原镜像配置、健康与依赖读取，再停止代理；失败保留报告和受限恢复文件，不自动重跑或删除数据。结果未知时使用 `status`、`reconcile` 核对原操作，必要时显式 `restore --plan <文件> --report <新文件> --confirm-test-runtime yes`，不得因观察超时重复创建容器。离线合同为对应 proxy/executor `.test.mjs`，离线通过不代表真实故障注入已验收。 +`order-submit-dependency-fault-executor.mjs` 是显式 TEST 下单依赖故障入口，内部使用 `order-submit-dependency-fault-proxy.py`、`order-submit-dependency-fault-runtime.py`。`plan --plan <文件>` 只读校验；`preflight --plan <文件> --routing <文件> --confirm-test-runtime yes` 核对测试 Order 镜像、实际依赖和 Nacos v3 配置（固定v3登录与Admin配置读取、校验namespaceId/groupName/dataId后仅记录内容摘要，不回退v1）；`run` 另需 `--expected-image <镜像ID>`、全新 `--report <文件>`。凭据仅从明确指定的 TEST 与 Jenkins 环境文件读取，路由文件不得包含凭据。必须事先准备本轮两个独立合法草稿、原始版本与幂等键及完整快照；仅对本轮客户的 IAM 资格、Finance 钱包及两张草稿共同引用的公共收货仓 GET 注入短时故障；Warehouse 阶段还限定原请求 ID、Order 来源与内部身份，不影响其他客户对同仓的读取。验证三依赖、两端共六次提交失败后事实不变。运行会临时重建同镜像 Order 容器并在恢复后真实提交草稿，因此不得与部署或其他共享环境造数并行；不写业务 SQL。结束先恢复直通、原镜像配置、健康与依赖读取；业务恢复必须等实际唯一入库单的SUBMITTED投影收敛，不能以连续空结果认定完成，随后按原版本原键重放并核对完整快照，再停止代理；失败保留报告和受限恢复文件，不自动重跑或删除数据。结果未知时使用 `status`、`reconcile` 核对原操作，必要时显式 `restore --plan <文件> --report <新文件> --confirm-test-runtime yes`，不得因观察超时重复创建容器。离线合同为对应 proxy/executor `.test.mjs`，离线通过不代表真实故障注入已验收。 `fba-core-regression-test-env.mjs` 为显式 TEST 的 F40 三模式串行 HTTP 专项，默认 `--plan` 只显示计划；`--execute --environment TEST --run-id <新批次>` 执行根先签、子先签、根签收后余量取消。`--tax-mode` 只接受 `BUY_DECLARATION` 或 `TAX_REFUND`，省略时保持买单默认语义。显式 `--tax-mode TAX_REFUND` 将税制传给现有 `context.f40TaxMode`，三种签收顺序均复用正式 F40 依赖编排及退税内部模块，走资料上传、回执确认、正式申报与人工放行、完成文件、尾程签收，并在最终 `COMPLETED` 后验证客户报关资料读取和文件下载；不复制报关流程。计划、manifest、汇总、各模式报告和只读取证日志均标明税制及范围。会通过正式 API 创建客户、单据、文件并使用共享 PDA/托盘，临时 PDA 按原模块 finally 停用。`fba-core-regression-evidence.mjs` 是内部只读取证 helper，Jenkins scriptText 仅在明确 TEST 节点对 TEST 容器所用数据库执行一致性只读 SELECT，不触发部署、不写业务 SQL。凭据通过显式 `node --env-file="" --env-file=""` 提供，数据库凭据只在执行节点内存读取。 diff --git a/scripts/admin-test-env-modules/orders-lifecycle-idempotency-contract.test.mjs b/scripts/admin-test-env-modules/orders-lifecycle-idempotency-contract.test.mjs index 2de2df749..12f364596 100644 --- a/scripts/admin-test-env-modules/orders-lifecycle-idempotency-contract.test.mjs +++ b/scripts/admin-test-env-modules/orders-lifecycle-idempotency-contract.test.mjs @@ -6,7 +6,7 @@ import { fixtureInboundCode, orderPersistenceFacts, requestedOrderDestinationTyp buildOrderDependencyFaultPlan, assertOrderDependencyFaultObservation } from './orders.mjs' function dependencyFixture() { - const detail = id => ({order:{id,orderNo:`FL${id}`,tenantId:'20',status:'DRAFT',version:4}, + const detail = id => ({order:{id,orderNo:`FL${id}`,tenantId:'20',receivingWarehouseId:'30',status:'DRAFT',version:4}, cargoes:[{skuNo:'current'}],parties:[],internalNotes:[],statusHistory:[]}) return {marker:'CURRENT-RUN',customerId:'20',registeredCustomerId:'20',merchant:detail('101'),platform:detail('102')} } @@ -20,23 +20,28 @@ function dependencyObservation(plan, fixture, dependency, role) { test('dependency plan binds two new-customer drafts and only named Order upstream properties; it performs no runtime actions',()=>{ const fixture=dependencyFixture(),plan=buildOrderDependencyFaultPlan(fixture) assert.equal(plan.runtimeChangesExecuted,false) - assert.deepEqual(plan.phases.map(row=>row.urlProperty),['baiyue.clients.iam.url','baiyue.clients.finance.url']) + assert.deepEqual(plan.phases.map(row=>row.urlProperty),['baiyue.clients.iam.url','baiyue.clients.finance.url','baiyue.clients.warehouse.url']) + assert.equal(plan.receivingWarehouseId,'30') + assert.equal(plan.phases[2].targetPath,'/internal/warehouses/0/30/active') assert.equal(plan.commands.merchant.body.expectedVersion,4) fixture.merchant.order.version=9 assert.equal(plan.commands.merchant.body.expectedVersion,4) assert.ok(Object.isFrozen(plan.commands.merchant.body)) - for(const change of [f=>f.customerId='0',f=>f.registeredCustomerId='21',f=>f.platform.order.id='101',f=>f.merchant.order.status='SUBMITTED',f=>delete f.merchant.cargoes]) { + for(const change of [f=>f.customerId='0',f=>f.registeredCustomerId='21',f=>f.platform.order.id='101',f=>f.merchant.order.status='SUBMITTED',f=>delete f.merchant.cargoes,f=>f.platform.order.receivingWarehouseId='31',f=>delete f.merchant.order.receivingWarehouseId]) { const value=dependencyFixture();change(value);assert.throws(()=>buildOrderDependencyFaultPlan(value)) } }) -test('all four dependency-role observations require matching authenticated 503 injection and actual Order S0503 with zero drift',()=>{ +test('all six dependency-role observations require matching authenticated 503 injection and actual Order S0503 with zero drift',()=>{ const fixture=dependencyFixture(),plan=buildOrderDependencyFaultPlan(fixture) - for(const dependency of ['IAM','FINANCE'])for(const role of ['merchant','platform']) { + for(const dependency of ['IAM','FINANCE','WAREHOUSE'])for(const role of ['merchant','platform']) { const evidence=dependencyObservation(plan,fixture,dependency,role) assert.equal(assertOrderDependencyFaultObservation(plan,dependency,role,evidence).verified,true) - const wrapped=structuredClone(evidence) - wrapped.response.body.message=plan.phases.find(row=>row.dependency===dependency).expectedMessages[1] - assert.equal(assertOrderDependencyFaultObservation(plan,dependency,role,wrapped).verified,true) + const alternative=plan.phases.find(row=>row.dependency===dependency).expectedMessages[1] + if(alternative) { + const wrapped=structuredClone(evidence) + wrapped.response.body.message=alternative + assert.equal(assertOrderDependencyFaultObservation(plan,dependency,role,wrapped).verified,true) + } for(const mutate of [ value=>value.response.http=503,value=>value.response.body.code='A0403',value=>value.response.body.message='资格未审核', value=>value.request.body.expectedVersion=5,value=>value.request.headers['Idempotency-Key']='new-key', diff --git a/scripts/admin-test-env-modules/orders.mjs b/scripts/admin-test-env-modules/orders.mjs index d43436d7f..a727e8dd2 100644 --- a/scripts/admin-test-env-modules/orders.mjs +++ b/scripts/admin-test-env-modules/orders.mjs @@ -260,12 +260,18 @@ export function buildOrderDependencyFaultPlan(fixture) { initialOrderSha256:createHash('sha256').update(orderPersistenceFacts(detail)).digest('hex') } } if (commands.merchant.orderId === commands.platform.orderId) throw new Error('Use two independent drafts for same-key recovery') - return { marker:fixture.marker, customerId, runtimeChangesExecuted:false, commands, + const receivingWarehouseId = id(fixture.merchant.order.receivingWarehouseId) + if (!/^[1-9]\d*$/.test(receivingWarehouseId) || receivingWarehouseId !== id(fixture.platform.order.receivingWarehouseId)) { + throw new Error('Dependency drafts must reference the same valid public receiving warehouse') + } + return { marker:fixture.marker, customerId, receivingWarehouseId, runtimeChangesExecuted:false, commands, phases:[ { dependency:'IAM', urlProperty:'baiyue.clients.iam.url', targetMethod:'GET', targetPath:`/internal/iam/tenants/${customerId}/order-eligibility`, faultHttp:503, expectedHttp:200, expectedCode:'S0503', expectedMessages:['IAM eligibility service is unavailable','IAM 下单资格响应无效'] }, { dependency:'FINANCE', urlProperty:'baiyue.clients.finance.url', targetMethod:'GET', targetPath:`/internal/finance/v1/wallet/accounts/${customerId}`, faultHttp:503, expectedHttp:200, expectedCode:'S0503', expectedMessages:['Finance account service is unavailable','Finance account response is invalid'] }, + { dependency:'WAREHOUSE', urlProperty:'baiyue.clients.warehouse.url', targetMethod:'GET', targetPath:`/internal/warehouses/0/${receivingWarehouseId}/active`, + faultHttp:503, expectedHttp:200, expectedCode:'S0503', expectedMessages:['主数据服务暂时不可用'] }, ], recovery:'Restore original Order image/config/network identity; submit each original frozen command once, then replay it; require one submission history and no duplicate Firstleg facts.' } } diff --git a/scripts/order-submit-dependency-fault-executor.mjs b/scripts/order-submit-dependency-fault-executor.mjs index d433d6aa2..a347c2ab1 100644 --- a/scripts/order-submit-dependency-fault-executor.mjs +++ b/scripts/order-submit-dependency-fault-executor.mjs @@ -29,17 +29,19 @@ export function scopeFor(plan) { } assert(plan.commands.merchant.orderId !== plan.commands.platform.orderId, 'Two distinct new drafts required') assert(plan.runtimeChangesExecuted === false, 'Only a reviewed unexecuted plan is accepted') - assert(Array.isArray(plan.phases) && plan.phases.length === 2, 'Exact two dependency phases required') + assert(/^[1-9]\d{0,19}$/.test(plan.receivingWarehouseId || ''), 'Bounded public receiving warehouse required') + assert(Array.isArray(plan.phases) && plan.phases.length === 3, 'Exact three dependency phases required') for (const [dependency, property, path, messages] of [ ['IAM','baiyue.clients.iam.url',`/internal/iam/tenants/${plan.customerId}/order-eligibility`,['IAM eligibility service is unavailable','IAM 下单资格响应无效']], ['FINANCE','baiyue.clients.finance.url',`/internal/finance/v1/wallet/accounts/${plan.customerId}`,['Finance account service is unavailable','Finance account response is invalid']], + ['WAREHOUSE','baiyue.clients.warehouse.url',`/internal/warehouses/0/${plan.receivingWarehouseId}/active`,['主数据服务暂时不可用']], ]) { const rows = plan.phases.filter(row => row.dependency === dependency) assert(rows.length === 1 && rows[0].urlProperty === property && rows[0].targetPath === path && rows[0].targetMethod === 'GET' && rows[0].faultHttp === 503 && rows[0].expectedHttp === 200 && rows[0].expectedCode === 'S0503' && JSON.stringify(rows[0].expectedMessages) === JSON.stringify(messages), 'Dependency failure contract may not be weakened') } - return { marker: plan.marker, customerId: plan.customerId, + return { marker: plan.marker, customerId: plan.customerId, receivingWarehouseId: plan.receivingWarehouseId, orderIds: Object.fromEntries(['merchant', 'platform'].map(role => [role, plan.commands[role].orderId])), planSha256: sha(plan) } } @@ -50,7 +52,7 @@ export function renderOrderFaultAction(action, scope, agent, extra = {}) { const routing = extra.routing assert(routing && Object.keys(routing).every(key => ['configurationReviewed','sourceSha256','upstreams','nacos'].includes(key)) && routing.configurationReviewed === true && routing.nacos && Object.keys(routing.nacos).every(key => ['baseUrl','loginPath','configPath'].includes(key)) - && routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM', 'Reviewed routing schema only; do not embed credentials') + && routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM,WAREHOUSE', 'Reviewed routing schema only; do not embed credentials') for (const address of [routing.nacos.baseUrl,...Object.values(routing.upstreams)]) { const url = new URL(address) assert(['http:','https:'].includes(url.protocol) && !url.username && !url.password && !url.search && !url.hash @@ -60,8 +62,9 @@ export function renderOrderFaultAction(action, scope, agent, extra = {}) { && /^[0-9a-f]{64}$/.test(routing.sourceSha256 || ''), 'Reviewed image and Nacos source fingerprint required') } const data = encode(JSON.stringify({ ...extra, action, scope, commonSource, ...(action === 'prepare' ? { proxySource } : {}) })) - const expression = value => '[' + encode(value).match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')" - const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),'${data}']).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)` + const literal = encoded => '[' + encoded.match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')" + const expression = value => literal(encode(value)) + const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),${literal(data)}]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)` return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null || c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${expression(groovy)}),'UTF-8'),c.channel))` } @@ -78,7 +81,10 @@ export function createOrderFaultRuntime(env, scope, fetcher = globalThis.fetch) }) } catch { throw Error(`RUNTIME_${action.toUpperCase().replaceAll('-', '_')}_OUTCOME_UNKNOWN_USE_STATUS`) } assert(response.ok, `RUNTIME_HTTP_${response.status}_NO_AUTO_RETRY`) - const rows = (await response.text()).split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON=')) + const output = await response.text() + assert(!(output.includes('MultipleCompilationErrorsException') && output.includes('String too long')), + 'RUNTIME_RENDERER_COMPILATION_FAILED_NO_ACTION') + const rows = output.split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON=')) assert(rows.length === 1, 'RUNTIME_RESPONSE_AMBIGUOUS_NO_AUTO_RETRY') const result = JSON.parse(rows[0].slice('ORDER_DEPENDENCY_JSON='.length)) assert(result?.ok === true, /^[A-Z_0-9]{1,90}$/.test(result?.error || '') ? result.error : 'RUNTIME_ACTION_FAILED') @@ -103,6 +109,7 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti assert(createHash('sha256').update(orderPersistenceFacts(before.detail)).digest('hex') === plan.commands[role].initialOrderSha256 && before.detail.order.status === 'DRAFT' && String(before.detail.order.tenantId) === scope.customerId && String(before.detail.order.id) === scope.orderIds[role] + && String(before.detail.order.receivingWarehouseId) === scope.receivingWarehouseId && ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(before.detail[field])) && before.inboundPageComplete === true, 'PREPARED_DRAFT_CHANGED') } @@ -113,7 +120,7 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti assert(i < 14, 'PROXY_STARTUP_NOT_READY'); await wait(1000) } await action('activate'); await health() - for (const dependency of ['IAM','FINANCE']) { + for (const dependency of ['IAM','FINANCE','WAREHOUSE']) { for (const role of ['merchant','platform']) { const before = await snapshot(role), command = plan.commands[role] const request = { method: command.method, path: command.path, body: structuredClone(command.body), @@ -138,20 +145,51 @@ export async function runOrderFaultWindow({ plan, routing, expectedImageId, acti && report.restoredRuntime.currentImageId === expectedImageId && report.restoredRuntime.originalFingerprint === report.restoredRuntime.currentFingerprint && report.restoredRuntime.nacosSourcesUnchanged === true - && report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true, + && report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true + && report.restoredRuntime.upstreamReads?.WAREHOUSE === true, 'RESTORED_IDENTITY_OR_ROUTES_NOT_PROVED') const businessRecovery = await recovery() report.businessRecovery = businessRecovery report.restoration = await action('finish', { businessRecovery }) assert(report.restoration.restored && report.restoration.proxyStopped && report.restoration.orderRoutesVerified, 'RESTORATION_NOT_COMPLETE') - } catch (error) { report.recoveryError = safeError(error); failure ||= error } + } catch (error) { + report.recoveryError = safeError(error) + if (error.recoveryDiagnostics) report.recoveryDiagnostics = error.recoveryDiagnostics + failure ||= error + } } } - report.success = !failure && report.observations.length === 4 && Boolean(report.restoration?.restored) + report.success = !failure && report.observations.length === 6 && Boolean(report.restoration?.restored) if (failure) report.error = safeError(failure) return report } +export async function waitForRecoveredSubmission({ plan, role, snapshot, wait = pause }) { + const command = plan.commands[role] + let previous, orderFacts + for (let i = 0; i <= 60; i++) { + const current = await snapshot(role), detail = current.detail + assert(current.inboundPageComplete === true && Array.isArray(current.inbounds) && current.inbounds.length <= 1 + && ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(detail?.[field])) + && String(detail.order.id) === command.orderId && String(detail.order.tenantId) === plan.customerId + && detail.order.status === 'SUBMITTED' && Number(detail.order.version) === command.body.expectedVersion + 1 + && detail.statusHistory.filter(row => row.action === 'SUBMITTED').length === 1 + && detail.statusHistory.filter(row => row.action === 'CREATE').length === 1 + && (current.task === null || current.task && String(current.task.orderId) === command.orderId + && String(current.task.tenantId) === plan.customerId) + && current.inbounds.every(row => String(row.orderId) === command.orderId && row.orderNo === command.orderNo + && String(row.tenantId) === plan.customerId), 'RECOVERY_SUBMISSION_FACTS_INVALID') + const facts = orderPersistenceFacts(detail) + orderFacts ??= facts + assert(facts === orderFacts, 'RECOVERY_ORDER_DRIFT') + const projected = current.inbounds.length === 1 && current.inbounds[0].orderStatusSnapshot === 'SUBMITTED' + if (projected && previous && sha(previous) === sha(current)) return current + previous = projected ? current : undefined + if (i < 60) await wait(500) + } + throw Error('RECOVERY_PROJECTION_TIMEOUT') +} + function businessClient(env, plan, fetcher = globalThis.fetch) { const sessions = {} const call = async (method, path, body, headers = {}, token) => { @@ -191,16 +229,19 @@ function businessClient(env, plan, fetcher = globalThis.fetch) { }, async recovery() { const evidence={} - for(const role of ['merchant','platform']) { - const command=plan.commands[role] - ok(await submit(role,command)) - let before=await snapshot(role) - for(let i=0;i<10;i++) { await pause(500);const next=await snapshot(role);if(sha(before)===sha(next)){before=next;break}before=next } - assert(before.detail.order.status==='SUBMITTED' && before.detail.statusHistory.filter(row=>row.action==='SUBMITTED').length===1 - && before.inbounds.length<=1, 'RECOVERY_SINGLE_SUBMISSION_NOT_PROVED') - ok(await submit(role,command));const after=await snapshot(role) - assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT') - evidence[role]={orderId:command.orderId,status:'SUBMITTED',sameKeyReplayVerified:true,snapshotSha256:sha(after)} + try { + for(const role of ['merchant','platform']) { + const command=plan.commands[role] + ok(await submit(role,command)) + const before=await waitForRecoveredSubmission({plan,role,snapshot}) + ok(await submit(role,command));const after=await snapshot(role) + evidence[role]={orderId:command.orderId,before,after,snapshotSha256:sha(after)} + assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT') + Object.assign(evidence[role],{status:'SUBMITTED',sameKeyReplayVerified:true}) + } + } catch(error) { + error.recoveryDiagnostics=evidence + throw error } return evidence }, @@ -217,7 +258,7 @@ async function main(argv) { for(let i=0;i ({order:{id,tenantId:'20',orderNo:`FL${id}`,version:2,status:'DRAFT'},cargoes:[],parties:[],internalNotes:[],statusHistory:[]}) + const detail = id => ({order:{id,tenantId:'20',receivingWarehouseId:'30',orderNo:`FL${id}`,version:2,status:'DRAFT'},cargoes:[],parties:[],internalNotes:[],statusHistory:[]}) const raw={marker:'TEST-DEPENDENCY',customerId:'20',registeredCustomerId:'20',merchant:detail('101'),platform:detail('102')} return {raw,plan:buildOrderDependencyFaultPlan(raw)} } const env={CHONGCHONG_ENV:'test',JENKINS_URL:'https://offline.invalid/',JENKINS_USER:'offline',JENKINS_API_TOKEN:'never-print-this',FBA_CORE_JENKINS_AGENT:'test-agent'} +function recoverySnapshot(projected=true) { + const {raw,plan}=fixture(),detail=structuredClone(raw.merchant) + detail.order.status='SUBMITTED';detail.order.version=3 + detail.statusHistory=[{action:'CREATE'},{action:'SUBMITTED'}] + return {plan,snapshot:{detail,task:null,inbounds:projected?[{id:'1001',orderId:'101',orderNo:'FL101',tenantId:'20',orderStatusSnapshot:'SUBMITTED'}]:[],inboundPageComplete:true}} +} +test('recovery waits for the actual inbound projection even when two empty reads are stable',async()=>{ + const {plan,snapshot}=recoverySnapshot(),empty=recoverySnapshot(false).snapshot + const values=[empty,empty,snapshot,snapshot];let reads=0 + const value=await waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>values[Math.min(reads++,values.length-1)],wait:async()=>{}}) + assert.equal(reads,4);assert.deepEqual(value,snapshot) +}) +test('missing submission projection cannot pass recovery after the bounded wait',async()=>{ + const {plan,snapshot}=recoverySnapshot(false);let reads=0 + await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>{reads++;return snapshot},wait:async()=>{}}),/RECOVERY_PROJECTION_TIMEOUT/) + assert.equal(reads,61) +}) +test('recovery rejects duplicate or foreign projected inbounds instead of waiting them away',async()=>{ + for(const change of [s=>s.inbounds.push({...s.inbounds[0],id:'1002'}),s=>s.inbounds[0].tenantId='21',s=>s.inbounds[0].orderId='102']) { + const {plan,snapshot}=recoverySnapshot();change(snapshot) + await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>snapshot,wait:async()=>{}}),/RECOVERY_SUBMISSION_FACTS_INVALID/) + } +}) +test('recovery projection wait rejects order fact drift and incomplete pagination',async()=>{ + const {plan,snapshot}=recoverySnapshot(),changed=structuredClone(snapshot);changed.detail.order.remark='late mutation' + let reads=0 + await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>reads++?changed:snapshot,wait:async()=>{}}),/RECOVERY_ORDER_DRIFT/) + const incomplete=structuredClone(snapshot);incomplete.inboundPageComplete=false + await assert.rejects(waitForRecoveredSubmission({plan,role:'merchant',snapshot:async()=>incomplete,wait:async()=>{}}),/RECOVERY_SUBMISSION_FACTS_INVALID/) +}) + test('runtime classifies dependency HTTP stages without exposing tokens, URLs or response bodies',()=>{ const script=String.raw` import importlib.util,sys,urllib.error @@ -42,6 +73,23 @@ test('renderer binds TEST scope and emits no Jenkins credential; ambiguous remot await assert.rejects(runtime('activate'),/OUTCOME_UNKNOWN_USE_STATUS/) assert.equal(calls,1) }) +test('renderer chunks every embedded Groovy literal and preserves the complete runtime payload',()=>{ + const {plan}=fixture(),scope=scopeFor(plan) + const routing={configurationReviewed:true,nacos:{baseUrl:'http://nacos.invalid:8848',loginPath:'/nacos/v3/auth/user/login',configPath:'/nacos/v3/admin/cs/config'},upstreams:{IAM:'http://iam.invalid:9201',FINANCE:'http://finance.invalid:9203',WAREHOUSE:'http://warehouse.invalid:9205'}} + const rendered=renderOrderFaultAction('preflight',scope,'test-agent',{routing}) + const arrays=text=>Array.from(text.matchAll(/\[('[A-Za-z0-9+/=]+'(?:,'[A-Za-z0-9+/=]+')*)\]\.join\(''\)/g),match=>Array.from(match[1].matchAll(/'([^']+)'/g),part=>part[1])) + const outer=arrays(rendered) + assert.equal(outer.length,1) + const inner=Buffer.from(outer[0].join(''),'base64').toString('utf8') + assert.ok(Array.from(inner.matchAll(/'([^']*)'/g),match=>match[1].length).every(length=>length<=4000)) + const embedded=arrays(inner) + assert.equal(embedded.length,2) + const config=JSON.parse(Buffer.from(embedded[1].join(''),'base64').toString('utf8')) + assert.deepEqual(config.scope,scope) + assert.deepEqual(config.routing,routing) + assert.equal(config.action,'preflight') + assert.ok(Buffer.from(config.commonSource,'base64').length>0) +}) function windowFixture(fault='') { const {raw,plan}=fixture(),calls=[],records=[] @@ -55,7 +103,7 @@ function windowFixture(fault='') { if(action==='health')return {healthy:true} if(action==='mode'){mode=extra.mode;return {mode}} if(action==='records')return {records} - if(action==='verify-restored')return {originalImageId:image,currentImageId:image,originalFingerprint:'same',currentFingerprint:'same',upstreamReads:{IAM:true,FINANCE:true},nacosSourcesUnchanged:true} + if(action==='verify-restored')return {originalImageId:image,currentImageId:image,originalFingerprint:'same',currentFingerprint:'same',upstreamReads:{IAM:true,FINANCE:true,WAREHOUSE:true},nacosSourcesUnchanged:true} if(action==='finish')return {restored:true,healthy:true,proxyStopped:true,orderRoutesVerified:true} return {} }, @@ -72,9 +120,9 @@ function windowFixture(fault='') { return {options,calls} } -test('four observations restore original routes and verify business recovery before stopping proxy',async()=>{ +test('six observations restore original routes and verify business recovery before stopping proxy',async()=>{ const {options,calls}=windowFixture(),report=await runOrderFaultWindow(options) - assert.equal(report.success,true);assert.equal(report.observations.length,4) + assert.equal(report.success,true);assert.equal(report.observations.length,6) assert.ok(calls.indexOf('restore'){ + const {options,calls}=windowFixture() + options.plan.receivingWarehouseId='31' + options.plan.phases[2].targetPath='/internal/warehouses/0/31/active' + const report=await runOrderFaultWindow(options) + assert.equal(report.success,false) + assert.ok(!calls.includes('prepare')) + assert.ok(!calls.includes('activate')) +}) +test('unverified warehouse restoration blocks business recovery and proxy finish',async()=>{ + const {options,calls}=windowFixture(),original=options.action + options.action=async(action,extra)=>{ + const result=await original(action,extra) + if(action==='verify-restored')result.upstreamReads.WAREHOUSE=false + return result + } + const report=await runOrderFaultWindow(options) + assert.equal(report.success,false) + assert.ok(calls.includes('restore')) + assert.ok(!calls.includes('business-recovery')) + assert.ok(!calls.includes('finish')) +}) for(const fault of ['prepare-unknown','restore-unknown','business-recovery-failed'])test(`unsafe cleanup blocks finish: ${fault}`,async()=>{ const {options,calls}=windowFixture(fault),report=await runOrderFaultWindow(options) assert.equal(report.success,false);assert.equal(calls.filter(action=>action==='prepare').length,1) @@ -131,15 +201,37 @@ class Fake: class RuntimeTests(unittest.TestCase): def fixture(self): f=Fake();tools=dict(common);tools.update(inspect=f.inspect,docker=f.docker) - scope={'marker':'OFFLINE','customerId':'7','orderIds':{'merchant':'10','platform':'11'},'planSha256':'b'*64} + scope={'marker':'OFFLINE','customerId':'7','receivingWarehouseId':'30','orderIds':{'merchant':'10','platform':'11'},'planSha256':'b'*64} r=m.Runtime(scope,tools,Path('offline-not-created'));saved=[];controls=[] r.save=lambda state:saved.append(copy.deepcopy(state));r.control=lambda state,mode=None,ttl=0:(controls.append(mode) or {'mode':'PASS'}) - r.probes=lambda state:{'IAM':True,'FINANCE':True} + r.probes=lambda state:{'IAM':True,'FINANCE':True,'WAREHOUSE':True} r.config_fingerprint=lambda container,routing:'reviewed-config' state={'scope':scope,'phase':'PROXY_STARTED','originalId':f.original['Id'],'originalImageId':f.original['Image'], 'originalFingerprint':common['fingerprint'](f.original),'originalBody':common['recreation'](f.original,common['env_of'](f.original),full=True), 'proxyPort':12345,'controlPort':12346,'mutation':None,'routing':{},'routingSourceSha256':'reviewed-config'} return f,r,state,saved,controls + def test_warehouse_probe_requires_the_exact_active_public_domestic_warehouse(self): + import io + from unittest.mock import patch + f,r,s,saved,controls=self.fixture() + state={'internalToken':'never-public','upstreams':{'IAM':'http://iam.invalid','FINANCE':'http://finance.invalid','WAREHOUSE':'http://warehouse.invalid'}} + warehouse={'id':30,'tenantId':0,'status':'ACTIVE','warehouseType':'DOMESTIC','warehouseName':'Scoped receiving warehouse'} + class Response(io.BytesIO):status=200 + def response(request,timeout): + url=request.full_url + if '/internal/iam/' in url:data={'eligible':True} + elif '/internal/finance/' in url:data={'actualBalance':0} + else: + self.assertTrue(url.endswith('/internal/warehouses/0/30/active')) + data={'warehouse':warehouse} + return Response(json.dumps({'code':'200','data':data}).encode()) + with patch.object(m,'http_open',side_effect=response): + self.assertEqual(m.Runtime.probes(r,state),{'IAM':True,'FINANCE':True,'WAREHOUSE':True}) + for field,value in [('id',31),('tenantId',20),('status','DISABLED'),('warehouseType','FBA'),('warehouseName','')]: + original=warehouse[field];warehouse[field]=value + with self.assertRaisesRegex(RuntimeError,'RECEIVING_WAREHOUSE_NOT_READY'):m.Runtime.probes(r,state) + warehouse[field]=original + self.assertEqual(f.calls,[]) def test_original_fingerprint_config_and_image_restored(self): f,r,s,saved,controls=self.fixture();r.activate(s) self.assertTrue(r.matches_injected(f.current,s));self.assertEqual(s['phase'],'INJECTED') @@ -227,5 +319,5 @@ sys.exit(0 if result.wasSuccessful() else 1) fileURLToPath(new URL('./order-submit-dependency-fault-runtime.py',import.meta.url)), fileURLToPath(new URL('./test-event-fault-runtime.py',import.meta.url))],{encoding:'utf8',env:{...process.env,PYTHONDONTWRITEBYTECODE:'1'}}) assert.equal(result.status,0,result.stderr || result.error?.message) - assert.match(result.stderr,/Ran 10 tests/) + assert.match(result.stderr,/Ran 11 tests/) }) diff --git a/scripts/order-submit-dependency-fault-proxy.py b/scripts/order-submit-dependency-fault-proxy.py index 240507e8d..94d2a9738 100644 --- a/scripts/order-submit-dependency-fault-proxy.py +++ b/scripts/order-submit-dependency-fault-proxy.py @@ -49,10 +49,14 @@ class Router: customer = config['customerId'] if not re.fullmatch(r'[1-9][0-9]{0,19}', customer): raise ValueError('INVALID_CUSTOMER') + warehouse = config.get('receivingWarehouseId', '') + if not re.fullmatch(r'[1-9][0-9]{0,19}', warehouse): + raise ValueError('INVALID_RECEIVING_WAREHOUSE') if not config.get('requestIds') or any(not re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}', value) for value in config['requestIds']): raise ValueError('BOUNDED_REQUEST_IDS_REQUIRED') self.paths = {'IAM': '/internal/iam/tenants/' + customer + '/order-eligibility', - 'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer} + 'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer, + 'WAREHOUSE': '/internal/warehouses/0/' + warehouse + '/active'} def status(self): with self.lock: @@ -67,7 +71,7 @@ class Router: with self.lock: # Even an authenticated malformed command disarms first. self.mode, self.deadline = 'PASS', 0 - if mode not in ('PASS', 'IAM', 'FINANCE') or (mode != 'PASS' and + if mode not in ('PASS', 'IAM', 'FINANCE', 'WAREHOUSE') or (mode != 'PASS' and (type(ttl) is not int or not 1 <= ttl <= 60)): raise ValueError('INVALID_MODE_OR_TTL') self.mode = mode @@ -77,7 +81,7 @@ class Router: def dispatch(self, method, path, headers, body, peer): prefix, separator, rest = path.lstrip('/').partition('/') - dependency = {'iam': 'IAM', 'finance': 'FINANCE'}.get(prefix) + dependency = {'iam': 'IAM', 'finance': 'FINANCE', 'warehouse': 'WAREHOUSE'}.get(prefix) if not dependency or not separator: return 404, {}, b'Unknown dependency route' actual_path = '/' + rest diff --git a/scripts/order-submit-dependency-fault-proxy.test.mjs b/scripts/order-submit-dependency-fault-proxy.test.mjs index 73b4c64f6..fccf049b0 100644 --- a/scripts/order-submit-dependency-fault-proxy.test.mjs +++ b/scripts/order-submit-dependency-fault-proxy.test.mjs @@ -10,8 +10,8 @@ spec=importlib.util.spec_from_file_location('proxy',sys.argv[1]);p=importlib.uti class ProxyTests(unittest.TestCase): def setUp(self): self.now=[10.0];self.calls=[];self.logs=[] - self.config={'customerId':'7','controlToken':'private-control','internalToken':'private-internal-token', - 'orderIps':['172.20.0.10'],'requestIds':['fixture-request'],'upstreams':{'IAM':'http://iam.invalid:9201','FINANCE':'http://finance.invalid:9207'}} + self.config={'customerId':'7','receivingWarehouseId':'30','controlToken':'private-control','internalToken':'private-internal-token', + 'orderIps':['172.20.0.10'],'requestIds':['fixture-request'],'upstreams':{'IAM':'http://iam.invalid:9201','FINANCE':'http://finance.invalid:9207','WAREHOUSE':'http://warehouse.invalid:9202'}} def forward(*args): self.calls.append(args);return 201,{'X-Upstream':'kept'},b'original-response' self.router=p.Router(self.config,forward=forward,clock=lambda:self.now[0],record=self.logs.append) self.headers={'X-Internal-Service-Token':'private-internal-token','X-Request-Id':'fixture-request'} @@ -42,6 +42,24 @@ class ProxyTests(unittest.TestCase): self.assertEqual(self.send(path)[0],503) self.now[0]=15.0 self.assertEqual(self.send(path)[0],201);self.assertEqual(self.router.status()['mode'],'PASS') + def test_public_warehouse_fault_is_bounded_by_exact_warehouse_request_and_order_identity(self): + self.router.set_mode('private-control','WAREHOUSE',60) + path='/warehouse/internal/warehouses/0/30/active' + self.assertEqual(self.send(path)[0],503) + for other,method,headers,peer in [ + (path.replace('/30/','/31/'),'GET',self.headers,'172.20.0.10'), + (path.replace('/0/','/7/'),'GET',self.headers,'172.20.0.10'), + (path+'?extra=1','GET',self.headers,'172.20.0.10'), + (path,'POST',self.headers,'172.20.0.10'), + (path,'GET',{**self.headers,'X-Request-Id':'another-customer-request'},'172.20.0.10'), + (path,'GET',{**self.headers,'X-Internal-Service-Token':'wrong'},'172.20.0.10'), + (path,'GET',self.headers,'172.20.0.11')]: + self.assertEqual(self.send(other,method,headers,peer)[0],201) + self.now[0]=70.0 + self.assertEqual(self.send(path)[0],201) + def test_invalid_receiving_warehouse_scope_is_rejected(self): + for value in ['0','../30','30?all=true','']: + with self.assertRaises(ValueError):p.Router({**self.config,'receivingWarehouseId':value}) def test_control_auth_and_malformed_authenticated_command(self): self.router.set_mode('private-control','IAM',10) with self.assertRaises(PermissionError):self.router.set_mode('wrong','PASS') @@ -65,5 +83,5 @@ sys.exit(0 if result.wasSuccessful() else 1) fileURLToPath(new URL('./order-submit-dependency-fault-proxy.py', import.meta.url))], { encoding:'utf8', env:{...process.env,PYTHONDONTWRITEBYTECODE:'1'} }) assert.equal(result.status,0,result.stderr || result.error?.message) - assert.match(result.stderr,/Ran 6 tests/) + assert.match(result.stderr,/Ran 8 tests/) }) diff --git a/scripts/order-submit-dependency-fault-runtime.py b/scripts/order-submit-dependency-fault-runtime.py index eb4014ec1..2ed481f55 100644 --- a/scripts/order-submit-dependency-fault-runtime.py +++ b/scripts/order-submit-dependency-fault-runtime.py @@ -24,8 +24,10 @@ from pathlib import Path ORDER = 'firstleg-test-order' ENTRYPOINT = ['sh', '-c', 'exec java $JAVA_OPTS org.springframework.boot.loader.launch.JarLauncher'] -PROPERTIES = {'IAM': 'baiyue.clients.iam.url', 'FINANCE': 'baiyue.clients.finance.url'} -UPSTREAM_CONTAINERS = {'IAM': 'firstleg-test-iam', 'FINANCE': 'firstleg-test-finance'} +PROPERTIES = {'IAM': 'baiyue.clients.iam.url', 'FINANCE': 'baiyue.clients.finance.url', + 'WAREHOUSE': 'baiyue.clients.warehouse.url'} +UPSTREAM_CONTAINERS = {'IAM': 'firstleg-test-iam', 'FINANCE': 'firstleg-test-finance', + 'WAREHOUSE': 'firstleg-test-warehouse'} class NoRedirect(urllib.request.HTTPRedirectHandler): @@ -63,10 +65,12 @@ def private_write(path, value): def validate_scope(scope): - if set(scope) != {'marker', 'customerId', 'orderIds', 'planSha256'} or not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', scope['marker']): + if set(scope) != {'marker', 'customerId', 'receivingWarehouseId', 'orderIds', 'planSha256'} or not re.fullmatch(r'[A-Za-z0-9_-]{1,80}', scope['marker']): raise RuntimeError('INVALID_SCOPE') if not re.fullmatch(r'[1-9][0-9]{0,19}', scope['customerId']) or set(scope['orderIds']) != {'merchant', 'platform'}: raise RuntimeError('INVALID_SCOPE') + if not re.fullmatch(r'[1-9][0-9]{0,19}', scope['receivingWarehouseId']): + raise RuntimeError('INVALID_RECEIVING_WAREHOUSE') if any(not re.fullmatch(r'[1-9][0-9]{0,19}', value) for value in scope['orderIds'].values()) or len(set(scope['orderIds'].values())) != 2: raise RuntimeError('TWO_DISTINCT_ORDERS_REQUIRED') if not re.fullmatch(r'[0-9a-f]{64}', scope['planSha256']): raise RuntimeError('INVALID_PLAN_HASH') @@ -125,7 +129,8 @@ class Runtime: """Real authenticated reads, not manufactured business responses. No body/secret is returned.""" customer = self.scope['customerId']; results = {} targets = {'IAM': '/internal/iam/tenants/' + customer + '/order-eligibility', - 'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer} + 'FINANCE': '/internal/finance/v1/wallet/accounts/' + customer, + 'WAREHOUSE': '/internal/warehouses/0/' + self.scope['receivingWarehouseId'] + '/active'} for dependency, path in targets.items(): req = urllib.request.Request(state['upstreams'][dependency] + path, headers={'X-Internal-Service-Token': state['internalToken']}) @@ -136,6 +141,12 @@ class Runtime: if dependency == 'IAM' and data.get('eligible') is not True: raise RuntimeError('CUSTOMER_NOT_ELIGIBLE') if dependency == 'FINANCE' and (data.get('actualBalance') is None or float(data['actualBalance']) < 0): raise RuntimeError('CUSTOMER_BALANCE_NOT_READY') + if dependency == 'WAREHOUSE': + warehouse = data.get('warehouse') or {} + if (str(warehouse.get('id')) != self.scope['receivingWarehouseId'] + or str(warehouse.get('tenantId')) != '0' or warehouse.get('status') != 'ACTIVE' + or warehouse.get('warehouseType') != 'DOMESTIC' or not warehouse.get('warehouseName')): + raise RuntimeError('RECEIVING_WAREHOUSE_NOT_READY') results[dependency] = True return results @@ -229,8 +240,9 @@ class Runtime: proxy = self.directory / 'proxy.py'; proxy.write_bytes(base64.b64decode(config['proxySource'])); os.chmod(proxy, 0o600) private_write(self.directory / 'proxy-config.json', {k: state[k] for k in ['internalToken', 'controlToken', 'proxyPort', 'controlPort', 'upstreams', 'orderIps']} | {'customerId': self.scope['customerId'], + 'receivingWarehouseId': self.scope['receivingWarehouseId'], 'requestIds': ['dep-' + self.scope['marker'] + '-' + dependency + '-' + role - for dependency in ('IAM', 'FINANCE') for role in ('merchant', 'platform')]}) + for dependency in PROPERTIES for role in ('merchant', 'platform')]}) state['phase'] = 'PROXY_START_INTENT'; state['proxyLaunchIntent'] = True; self.save(state) with open(self.directory / 'proxy-output.log', 'ab') as output: os.chmod(self.directory / 'proxy-output.log', 0o600) @@ -361,7 +373,7 @@ class Runtime: if self.config_fingerprint(current, state['routing']) != state['routingSourceSha256']: raise RuntimeError('NACOS_ROUTE_CONFIGURATION_CHANGED') checks = self.probes(state) - if checks != {'IAM': True, 'FINANCE': True}: raise RuntimeError('RESTORED_ROUTE_PROBE_INVALID') + if checks != {dependency: True for dependency in PROPERTIES}: raise RuntimeError('RESTORED_ROUTE_PROBE_INVALID') state['routeReadsVerified'] = checks; self.save(state) return {'healthy': True, 'originalFingerprint': state['originalFingerprint'], 'currentFingerprint': self.tools['fingerprint'](current), 'originalImageId': state['originalImageId'], 'currentImageId': current['Image'], 'upstreamReads': checks,