test(lastleg): verify MySQL hold and full PDA picking races
This commit is contained in:
@@ -424,3 +424,6 @@ Dry-run 只校验两个统一 SQL 的入口标记，打印当前 SHA-256、行
|
||||
Admin 全量中的 `phase4-cross-region-events` 使用内部 `test-event-fault-executor.mjs` 和 `test-event-fault-runtime.py`，不是额外 CLI 入口。它们仅在显式 `CHONGCHONG_ENV=test` 和已授权环境中运行：在首个本轮计划完成前启动已有 ACK 丢失代理，短暂重建同镜像 Firstleg 容器，保留数据卷、端口和网络身份，只变更事件目标与本次故障退避参数。仅选定的装柜事件被抑制 ACK；其他事件直通。核对 12 次失败、DEAD、受保护原事件 retry、DUPLICATE 及业务订单零新增，结束或失败都恢复原镜像配置并检查健康，随后停止本任务代理。不修改业务 SQL、载荷、事件 ID 或 hash，不重放已发布事件。恢复失败必须保留阻断和任务元数据；独立后端发布或其他环境造数不得与此窗口并行。凭据仅在执行节点内存与权限600的临时恢复文件中使用，恢复完成后移除临时文件中的凭据。离线合同为 `test-event-fault-executor.test.mjs`。
|
||||
|
||||
`event-outbox-operations.mjs`：单测试环境的内部运维脚本。只连接显式指定的生产服务地址，不通过公网 Gateway，不自动重试或跟随重定向。凭据只从进程环境 `EVENT_OPERATIONS_TOKEN` 读取，另须 `CHONGCHONG_ENV=test`；不要把真实凭据放入参数或仓库。`--service firstleg|lastleg|finance --base <服务源地址> --entry <Outbox主键>` 只读查询状态和摘要，不返回业务负载。追加 `--retry --request-id <稳定UUID> --operator <操作人> --reason <原因> --hash <sha256摘要>` 会写入目标测试服务的 Outbox 与审计表，执行前须已有该环境操作授权。仅 DEAD 原事件可重试，事件身份、负载及摘要保持不变；重复提交使用同一个请求编号，参数变化会被拒绝。三个服务须先部署 `event_delivery_retry_audit` 表，未设置至少32字节的运维凭据时入口关闭。脚本将脱敏结果输出到标准输出，不持久化凭据；离线验证为 `node --test scripts/event-outbox-operations.test.mjs`。
|
||||
|
||||
|
||||
`f32-pda-hold-race.mjs` 是显式 TEST 暂扣/PDA拣货并发验收入口，离线合同 `f32-pda-hold-race.test.mjs`。命令为 `node --env-file="<TEST配置>" --env-file="<Jenkins配置>" --env-file="<本轮自有PDA配置>" scripts/f32-pda-hold-race.mjs --plan <冻结计划> --report <全新报告> --confirm-test-runtime yes`；只消费已实际入仓、4箱可用、无占用、调度审核通过且同仓PDA canPick=true的本轮独立样本，不创建订单或车次。凭据只读明确TEST/Jenkins/自有PDA配置，不写报告；计划冻结客户、来源订单、仓、余额、位置、排车明细、车次、PDA成员和不可变Lastleg镜像ID。通过Jenkins在指定TEST Lastleg所用MySQL建立仅针对原订单的12秒行锁租约（SELECT FOR UPDATE、固定等待、ROLLBACK，无业务DML），用performance_schema实际锁等待证明两条正式HTTP命令同时竞争，再按暂扣先到/拣货先到两种顺序验证。会真实暂扣、解除本轮暂扣和拣货1箱，逐位置、库存流水、拣货及来源归属、原键重放分别核对；两个阶段顺序执行，不并发造数。结束恢复自有PDA停用状态并退出认证会话，测试单据、部分拣货和资源保留。命令结果未知或断言失败时保存原报告及逐步checkpoint，不自动重发命令、重建样本、删数据或回填SQL；租约自动释放后先核对原单和原命令，未证实的暂扣恢复必须单独按原事实处理。离线通过不代表实际MySQL/PDA或四端页面通过。
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// Explicit TEST inventory race. Imports perform no network or environment reads.
|
||||
import fs from 'node:fs'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { buildEvidenceRemoteScript } from './fba-core-regression-evidence.mjs'
|
||||
import { warehouseClaim } from './admin-test-env-modules/unified-pda-session-revocation.mjs'
|
||||
|
||||
const demand=(condition,message)=>{if(!condition)throw Error(message)}
|
||||
const digest=value=>createHash('sha256').update(JSON.stringify(value)).digest('hex')
|
||||
const pause=ms=>new Promise(done=>setTimeout(done,ms))
|
||||
const literal=value=>`CONVERT(0x${Buffer.from(value).toString('hex')} USING utf8mb4) COLLATE utf8mb4_bin`
|
||||
const tx=['SET NAMES utf8mb4 COLLATE utf8mb4_bin;','SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;','START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;']
|
||||
const parse=raw=>JSON.parse(raw.replace(/([:\[,]\s*)(-?\d{16,})(?=\s*[,}\]])/g,'$1"$2"'))
|
||||
export function validateRacePlan(p){
|
||||
const fields=['purpose','marker','customerId','orderId','warehouseId','balanceId','positionId','pdaMembershipId','dispatchItemId','dispatchNo','tripNo','expectedImage','expectedPackages','pickPackages','pickQuantity']
|
||||
demand(p&&Object.keys(p).length===fields.length&&Object.keys(p).every(key=>fields.includes(key)),'INVALID_RACE_SCHEMA')
|
||||
demand(p?.purpose==='F32_OWNED_PDA_HOLD_RACE'&&/^[A-Za-z0-9_-]{1,64}$/.test(p.marker||''),'INVALID_RACE_PLAN')
|
||||
for(const key of ['customerId','orderId','warehouseId','balanceId','positionId','pdaMembershipId','dispatchItemId'])
|
||||
demand(typeof p[key]==='string'&&/^[1-9][0-9]{0,19}$/.test(p[key]),'INVALID_RACE_IDENTIFIER')
|
||||
demand(/^[A-Za-z0-9_-]{1,64}$/.test(p.dispatchNo||'')&&/^[A-Za-z0-9_-]{1,64}$/.test(p.tripNo||''),'INVALID_RACE_BUSINESS_NUMBER')
|
||||
demand(/^sha256:[a-f0-9]{64}$/.test(p.expectedImage||'')&&p.expectedPackages===4&&p.pickPackages===1
|
||||
&&Number.isSafeInteger(p.pickQuantity)&&p.pickQuantity>0,'INVALID_RACE_BASELINE')
|
||||
return p
|
||||
}
|
||||
const b64=value=>'['+Buffer.from(value).toString('base64').match(/.{1,4000}/g).map(x=>`'${x}'`).join(',')+"].join('')"
|
||||
export function buildOrderLockLeaseScript(plan,phase,agent){
|
||||
const p=validateRacePlan(plan)
|
||||
demand(['HOLD_FIRST','PICK_FIRST'].includes(phase)&&/^[A-Za-z0-9_.-]{1,128}$/.test(agent||''),'INVALID_LEASE_SCOPE')
|
||||
const tag=`F32_${p.marker}_${phase}`
|
||||
// The only write-capable transaction contains a locking SELECT and ROLLBACK.
|
||||
// A fixed server-side 12-second lease releases the row even if the observer disconnects.
|
||||
const sql=`SET SESSION innodb_lock_wait_timeout=5;START TRANSACTION;SELECT JSON_OBJECT('rowId',CAST(id AS CHAR),'customerId',customer_id,'connectionId',CONNECTION_ID()) FROM ll_order WHERE id=${p.orderId} AND customer_id=${literal(p.customerId)} AND overseas_warehouse_id=${literal(p.warehouseId)} FOR UPDATE;SELECT JSON_OBJECT('marker','${tag}','leaseElapsed',SLEEP(12));ROLLBACK;SELECT JSON_OBJECT('released',JSON_EXTRACT('true','$'));`
|
||||
const python=`import json,os,subprocess,urllib.parse,re
|
||||
try:
|
||||
c=json.loads(subprocess.check_output(['docker','inspect','firstleg-test-lastleg'],text=True,stderr=subprocess.DEVNULL))[0]
|
||||
e=dict(x.split('=',1) for x in c['Config']['Env'])
|
||||
if c['Name']!='/firstleg-test-lastleg' or c['Config'].get('Labels',{}).get('com.docker.compose.project')!='firstleg-test' or e.get('DEPLOY_ENV')!='test' or c['Image']!='${p.expectedImage}' or c['State'].get('Health',{}).get('Status')!='healthy':raise RuntimeError('TEST_RUNTIME_CHANGED')
|
||||
u=urllib.parse.urlsplit(e['LASTLEG_DB_URL'].removeprefix('jdbc:'));host='127.0.0.1' if u.hostname=='host.docker.internal' else u.hostname
|
||||
childEnv=dict(os.environ,MYSQL_PWD=e['LASTLEG_DB_PASSWORD'])
|
||||
q=subprocess.run(['mysql','--host='+host,'--port='+str(u.port or 3306),'--user='+e['LASTLEG_DB_USERNAME'],'--batch','--raw','--skip-column-names',u.path.lstrip('/')],input=${JSON.stringify(sql)},capture_output=True,text=True,env=childEnv,timeout=22)
|
||||
if q.returncode:
|
||||
code=re.search(r'ERROR ([0-9]{3,5})',q.stderr);print('F32_LEASE_JSON='+json.dumps({'ok':False,'error':'MYSQL_'+(code.group(1) if code else 'UNKNOWN')}))
|
||||
else:
|
||||
rows=[json.loads(x) for x in q.stdout.splitlines() if x.strip()]
|
||||
if len(rows)!=3 or str(rows[0].get('rowId'))!='${p.orderId}' or str(rows[0].get('customerId'))!='${p.customerId}' or rows[-1].get('released')!=True:raise RuntimeError('LEASE_SOURCE_NOT_PROVED')
|
||||
print('F32_LEASE_JSON='+json.dumps({'ok':True,'rows':rows,'imageId':c['Image']}))
|
||||
except Exception as error:
|
||||
print('F32_LEASE_JSON='+json.dumps({'ok':False,'error':str(error) if re.fullmatch('[A-Z_]{1,64}',str(error)) else 'LEASE_FAILED_OR_TIMED_OUT'}))
|
||||
`
|
||||
const groovy=`def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${b64(python)}),'UTF-8')]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
|
||||
return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null||c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${b64(groovy)}),'UTF-8'),c.channel))`
|
||||
}
|
||||
export function buildLockEvidenceSql(plan,phase){
|
||||
const p=validateRacePlan(plan);demand(['HOLD_FIRST','PICK_FIRST'].includes(phase),'INVALID_LEASE_SCOPE')
|
||||
const tag=`F32_${p.marker}_${phase}`
|
||||
return [...tx,
|
||||
`SELECT JSON_OBJECT('collection','lease','connectionId',t.trx_mysql_thread_id) FROM performance_schema.data_locks l JOIN information_schema.innodb_trx t ON t.trx_id=l.ENGINE_TRANSACTION_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME='ll_order' AND l.INDEX_NAME='PRIMARY' AND l.LOCK_STATUS='GRANTED' AND l.LOCK_DATA=${literal(p.orderId)} AND t.trx_query LIKE ${literal('%'+tag+'%')};`,
|
||||
`SELECT JSON_OBJECT('collection','wait','requestingConnectionId',r.trx_mysql_thread_id,'blockingConnectionId',b.trx_mysql_thread_id,'orderId',l.LOCK_DATA) FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID JOIN information_schema.innodb_trx r ON r.trx_id=w.REQUESTING_ENGINE_TRANSACTION_ID JOIN information_schema.innodb_trx b ON b.trx_id=w.BLOCKING_ENGINE_TRANSACTION_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME='ll_order' AND l.INDEX_NAME='PRIMARY' AND l.LOCK_DATA=${literal(p.orderId)};`,'COMMIT;'].join('\n')
|
||||
}
|
||||
export function buildStockEvidenceSql(plan){
|
||||
const p=validateRacePlan(plan),customer=literal(p.customerId)
|
||||
return [...tx,
|
||||
`SELECT JSON_OBJECT('collection','sourceOrder','id',CAST(id AS CHAR),'customerId',customer_id,'warehouseId',overseas_warehouse_id,'picked',COALESCE(picked_package_count,0),'shipped',COALESCE(shipped_package_count,0),'version',version,'detained',detained,'outboundNo',outbound_order_no) FROM ll_order WHERE customer_id=${customer} AND id=${p.orderId};`,
|
||||
`SELECT JSON_OBJECT('collection','hold','id',CAST(id AS CHAR),'key',idempotency_key,'held',held,'version',order_version) FROM ll_order_hold_history WHERE customer_id=${customer} AND order_id=${p.orderId} ORDER BY id;`,
|
||||
`SELECT JSON_OBJECT('collection','pick','id',CAST(id AS CHAR),'key',idempotency_key,'packages',packages,'quantity',product_quantity,'positionId',CAST(inventory_position_id AS CHAR)) FROM ll_dispatch_pda_pick WHERE customer_id=${customer} AND dispatch_item_id=${p.dispatchItemId} ORDER BY id;`,
|
||||
`SELECT JSON_OBJECT('collection','attribution','id',CAST(id AS CHAR),'pickId',CAST(pick_id AS CHAR),'orderId',CAST(order_id AS CHAR),'packages',packages,'quantity',product_quantity) FROM ll_dispatch_pda_pick_order WHERE customer_id=${customer} AND order_id=${p.orderId} ORDER BY id;`,
|
||||
`SELECT JSON_OBJECT('collection','ledger','id',CAST(id AS CHAR),'key',idempotency_key,'positionId',CAST(position_id AS CHAR),'businessType',business_type,'statusFrom',status_from,'statusTo',status_to,'before',before_packages,'delta',delta_packages,'after',after_packages) FROM ll_inventory_ledger WHERE customer_id=${customer} AND balance_id=${p.balanceId} ORDER BY id;`,'COMMIT;'].join('\n')
|
||||
}
|
||||
export function assertContendedRequests(rows,expectedOrderId,firstConnectionId){
|
||||
const leases=rows.filter(x=>x.collection==='lease'),waits=rows.filter(x=>x.collection==='wait')
|
||||
demand(leases.length===1,'LIVE_LEASE_NOT_PROVED')
|
||||
const ids=[...new Set(waits.map(x=>x.requestingConnectionId))]
|
||||
demand(Number.isSafeInteger(leases[0].connectionId)&&leases[0].connectionId>0
|
||||
&&waits.every(x=>x.orderId===expectedOrderId&&Number.isSafeInteger(x.requestingConnectionId)&&x.requestingConnectionId>0)
|
||||
&&!ids.includes(leases[0].connectionId),'LOCK_EVIDENCE_SCOPE_INVALID')
|
||||
demand(ids.length===(firstConnectionId===undefined?1:2)&&waits.some(x=>x.blockingConnectionId===leases[0].connectionId),'EXACT_REQUEST_CONTENTION_NOT_PROVED')
|
||||
if(firstConnectionId!==undefined)demand(ids.includes(firstConnectionId),'FIRST_REQUEST_NO_LONGER_WAITING')
|
||||
return {leaseConnectionId:leases[0].connectionId,requestingConnections:ids,waits}
|
||||
}
|
||||
function stableSnapshot(x){return digest(x)}
|
||||
|
||||
export function assertRaceInventory(plan,phase,before,after){
|
||||
const p=validateRacePlan(plan);demand(['HOLD_FIRST','PICK_FIRST'].includes(phase),'INVALID_RACE_PHASE')
|
||||
const scanned=phase==='PICK_FIRST'?1:0,b=after.balance,order=after.detail.order
|
||||
const bucketNames=['availablePackages','holdPackages','allocatedPackages','pickedPackages','receivingPackages']
|
||||
const buckets=x=>bucketNames.reduce((n,k)=>{demand(Number.isSafeInteger(Number(x[k]))&&Number(x[k])>=0,'INVALID_STOCK_BUCKET');return n+Number(x[k])},0)
|
||||
demand(Number(before.balance.availablePackages)===4&&Number(before.balance.totalPackages)===4
|
||||
&&before.positions.length===2&&after.positions.length===2
|
||||
&&new Set(after.positions.map(x=>x.id)).size===2&&after.positions.every(x=>before.positions.some(y=>y.id===x.id))
|
||||
&&Number(b.availablePackages)===(scanned?3:0)&&Number(b.holdPackages)===(scanned?0:4)
|
||||
&&Number(b.pickedPackages)===0&&Number(b.allocatedPackages)===0&&Number(b.receivingPackages)===0
|
||||
&&Number(b.totalPackages)===4-scanned&&buckets(b)===4-scanned
|
||||
&&after.positions.reduce((n,x)=>n+buckets(x),0)+scanned===4&&order.detained===!scanned,'RACE_INVENTORY_RESULT_INVALID')
|
||||
const sources=after.sql.filter(x=>x.collection==='sourceOrder')
|
||||
demand(sources.length===1&&sources[0].id===p.orderId&&sources[0].customerId===p.customerId&&sources[0].warehouseId===p.warehouseId
|
||||
&&sources[0].picked===scanned&&sources[0].shipped===0&&!sources[0].outboundNo
|
||||
&&sources[0].version===Number(order.version)&&sources[0].detained===(scanned?0:1),'RACE_SOURCE_ORDER_COUNTERS_INVALID')
|
||||
const added=collection=>after.sql.filter(x=>x.collection===collection&&!before.sql.some(y=>y.collection===collection&&y.id===x.id))
|
||||
const pickKey=`${p.marker}-${phase}-PICK`,newPicks=added('pick'),attributed=added('attribution'),ledger=added('ledger'),holds=added('hold')
|
||||
demand(newPicks.length===scanned&&newPicks.every(x=>x.key===pickKey&&x.positionId===p.positionId&&x.packages===1&&x.quantity===p.pickQuantity),'RACE_PICK_FACTS_INVALID')
|
||||
demand(attributed.length===scanned&&attributed.every(x=>x.orderId===p.orderId&&x.packages===1&&x.quantity===p.pickQuantity
|
||||
&&newPicks.some(y=>y.id===x.pickId)),'RACE_SOURCE_ATTRIBUTION_INVALID')
|
||||
demand(ledger.every(x=>Number.isSafeInteger(x.delta)&&x.before+x.delta===x.after),'RACE_LEDGER_ARITHMETIC_INVALID')
|
||||
if(scanned){
|
||||
demand(holds.length===0&&ledger.length===3,'PICK_OR_REJECTED_HOLD_DUPLICATED_FACTS')
|
||||
for(const [type,from,to,delta] of [['ALLOCATE','AVAILABLE','ALLOCATED',1],['PICK','ALLOCATED','PICKED',1],['SHIP','PICKED','SHIPPED',-1]]){
|
||||
const rows=ledger.filter(x=>x.businessType===type)
|
||||
demand(rows.length===1&&rows[0].positionId===p.positionId&&rows[0].statusFrom===from&&rows[0].statusTo===to&&rows[0].delta===delta
|
||||
&&rows[0].key.startsWith(`DISPATCH:${p.dispatchNo}:${p.dispatchItemId}:${pickKey}:${type}:`),'RACE_SCAN_LEDGER_INVALID')
|
||||
}
|
||||
demand(after.positions.every(x=>Number(x.availablePackages)===Number(before.positions.find(y=>y.id===x.id).availablePackages)-(x.id===p.positionId?1:0)
|
||||
&&Number(x.holdPackages)===0&&Number(x.pickedPackages)===0&&Number(x.allocatedPackages)===0),'RACE_POSITION_DEDUCTION_INVALID')
|
||||
}else{
|
||||
demand(holds.length===1&&holds[0].key===`${p.marker}-${phase}-HOLD`&&holds[0].held===1
|
||||
&&holds[0].version===Number(order.version)&&ledger.length===2,'HOLD_OR_REJECTED_PICK_DUPLICATED_FACTS')
|
||||
demand(ledger.every(x=>x.businessType==='HOLD'&&x.statusFrom==='AVAILABLE'&&x.statusTo==='HOLD'
|
||||
&&x.delta===Number(before.positions.find(y=>y.id===x.positionId)?.availablePackages))
|
||||
&&new Set(ledger.map(x=>x.positionId)).size===2&&ledger.reduce((n,x)=>n+x.delta,0)===4,'RACE_HOLD_LEDGER_INVALID')
|
||||
}
|
||||
return{verified:true,scannedOutPackages:scanned,availablePackages:Number(b.availablePackages),heldPackages:Number(b.holdPackages),pickedBucketPackages:0,sourcePickedPackages:sources[0].picked}
|
||||
}
|
||||
|
||||
async function main(argv){
|
||||
const a={};for(let i=0;i<argv.length;i+=2){demand(argv[i]?.startsWith('--')&&argv[i+1],'EXPLICIT_ARGUMENTS_REQUIRED');a[argv[i].slice(2)]=argv[i+1]}
|
||||
demand(a.plan&&a.report&&a['confirm-test-runtime']==='yes'&&String(process.env.CHONGCHONG_ENV).toLowerCase()==='test','EXPLICIT_TEST_SCOPE_REQUIRED')
|
||||
const p=validateRacePlan(JSON.parse(fs.readFileSync(a.plan))),env=process.env
|
||||
for(const key of ['JENKINS_URL','JENKINS_USER','JENKINS_API_TOKEN','CHONGCHONG_ADMIN_API_BASE_URL','CHONGCHONG_ADMIN_USERNAME','CHONGCHONG_ADMIN_PASSWORD','PDA_USERNAME','PDA_PASSWORD','PDA_MEMBERSHIP_ID'])demand(env[key],'EXPLICIT_TEST_CREDENTIALS_REQUIRED')
|
||||
demand(env.PDA_MEMBERSHIP_ID===p.pdaMembershipId,'OWN_PDA_PROFILE_REQUIRED')
|
||||
const report={scope:p,success:false,phases:[],requests:[],at:new Date().toISOString()}
|
||||
const fd=fs.openSync(a.report,'wx',0o600),checkpoint=a.report+'.checkpoint.json'
|
||||
fs.writeFileSync(checkpoint,JSON.stringify(report),{flag:'wx'})
|
||||
const save=()=>{
|
||||
const checkpointFd=fs.openSync(checkpoint,'w',0o600)
|
||||
try{fs.writeFileSync(checkpointFd,JSON.stringify(report,null,2));fs.fsyncSync(checkpointFd)}finally{fs.closeSync(checkpointFd)}
|
||||
}
|
||||
const base=env.CHONGCHONG_ADMIN_API_BASE_URL.replace(/\/$/,''),agent='java-test-agent';let admin,pda,activationAttempted=false
|
||||
const call=async(method,path,body,token=admin?.accessToken)=>{
|
||||
const entry={method,path:path.split('?')[0],phase:'INTENT'};report.requests.push(entry);save()
|
||||
let res;try{res=await fetch(base+path,{method,redirect:'error',signal:AbortSignal.timeout(20000),headers:{Accept:'application/json',...(token?{Authorization:'Bearer '+token}:{}),...(body===undefined?{}:{'Content-Type':'application/json'})},...(body===undefined?{}:{body:JSON.stringify(body)})})}catch{entry.phase='OUTCOME_UNKNOWN';save();throw Error('API_OUTCOME_UNKNOWN_NO_AUTO_RETRY')}
|
||||
const payload=parse(await res.text());Object.assign(entry,{phase:'ACKNOWLEDGED',http:res.status,code:String(payload.code),requestId:res.headers.get('x-request-id')});save();return{http:res.status,body:payload}
|
||||
}
|
||||
const ok=r=>{demand(r.http===200&&String(r.body?.code)==='200','API_SUCCESS_REQUIRED');return r.body.data}
|
||||
const remote=async(script,prefix)=>{
|
||||
const res=await fetch(new URL('scriptText',env.JENKINS_URL.replace(/\/?$/,'/')),{method:'POST',redirect:'error',signal:AbortSignal.timeout(55000),headers:{Authorization:'Basic '+Buffer.from(env.JENKINS_USER+':'+env.JENKINS_API_TOKEN).toString('base64'),'Content-Type':'application/x-www-form-urlencoded'},body:new URLSearchParams({script})})
|
||||
demand(res.ok,'JENKINS_HTTP_FAILURE');const lines=(await res.text()).split(/\r?\n/).filter(x=>x.startsWith(prefix+'='));demand(lines.length===1,'REMOTE_RESULT_AMBIGUOUS');const value=JSON.parse(lines[0].slice(prefix.length+1));demand(value.ok===true,/^[A-Z_0-9]+$/.test(value.error||'')?value.error:'REMOTE_FAILED');return value
|
||||
}
|
||||
const sql=async query=>(await remote(buildEvidenceRemoteScript('lastleg',query,agent),'FBA_CORE_JSON')).rows
|
||||
const snapshot=async()=>{
|
||||
const detail=ok(await call('GET',`/platform/v1/last-leg/orders/${p.orderId}?customerId=${p.customerId}`))
|
||||
const balance=ok(await call('GET',`/platform/v1/last-leg/inventory/balances/${p.balanceId}?customerId=${p.customerId}`))
|
||||
const positions=ok(await call('GET',`/platform/v1/last-leg/inventory/balances/${p.balanceId}/positions?customerId=${p.customerId}`))
|
||||
const task=ok(await call('GET',`/pda/v1/last-leg/dispatches/${p.dispatchNo}`,undefined,pda.accessToken))
|
||||
demand(String(detail.order.id)===p.orderId&&String(detail.order.customerId)===p.customerId&&String(balance.inboundOrderId)===p.orderId
|
||||
&&String(balance.customerId)===p.customerId&&String(balance.overseasWarehouseId)===p.warehouseId
|
||||
&&positions.length===2&&positions.every(x=>String(x.customerId)===p.customerId&&String(x.inboundOrderId)===p.orderId)
|
||||
&&task.tripNo===p.tripNo&&task.dispatchNo===p.dispatchNo,'SNAPSHOT_SCOPE_INVALID')
|
||||
return{detail,balance,positions,task,sql:await sql(buildStockEvidenceSql(p))}
|
||||
}
|
||||
// Explicit headers for immutable command keys; never mint a different key after an unknown result.
|
||||
const command=async(type,body,key)=>{
|
||||
const path=type==='HOLD'?`/platform/v1/last-leg/orders/${p.orderId}/hold?customerId=${p.customerId}`:`/pda/v1/last-leg/dispatches/${p.dispatchNo}/items/${p.dispatchItemId}/pick`
|
||||
const token=type==='HOLD'?admin.accessToken:pda.accessToken,method=type==='HOLD'?'PUT':'POST'
|
||||
const entry={method,path:path.split('?')[0],type,key,body,phase:'INTENT'};report.requests.push(entry);save()
|
||||
let res;try{res=await fetch(base+path,{method,redirect:'error',signal:AbortSignal.timeout(20000),headers:{Authorization:'Bearer '+token,'Content-Type':'application/json',...(type==='HOLD'?{'Idempotency-Key':key}:{})},body:JSON.stringify(body)})}catch{entry.phase='OUTCOME_UNKNOWN';save();throw Error('COMMAND_OUTCOME_UNKNOWN_NO_AUTO_RETRY')}
|
||||
const payload=parse(await res.text());Object.assign(entry,{phase:'ACKNOWLEDGED',http:res.status,code:String(payload.code),message:payload.message,requestId:res.headers.get('x-request-id')});save();return{http:res.status,body:payload}
|
||||
}
|
||||
try{
|
||||
admin=ok(await call('POST','/auth/login',{loginId:env.CHONGCHONG_ADMIN_USERNAME,password:env.CHONGCHONG_ADMIN_PASSWORD},''))
|
||||
const users=ok(await call('GET',`/platform/v1/iam/users?page=1&size=20&keyword=${encodeURIComponent(env.PDA_USERNAME)}`))
|
||||
const user=users.records.find(x=>String(x.membershipId)===p.pdaMembershipId)
|
||||
demand(user&&!user.protectedAccount&&user.status==='DISABLED'&&String(user.overseasWarehouseId)===p.warehouseId,'OWN_DISABLED_PDA_REQUIRED')
|
||||
activationAttempted=true;ok(await call('PUT',`/platform/v1/iam/users/${p.pdaMembershipId}/status`,{status:'ACTIVE'}))
|
||||
pda=ok(await call('POST','/auth/login',{loginId:env.PDA_USERNAME,password:env.PDA_PASSWORD},''));demand(warehouseClaim(pda.accessToken)===p.warehouseId,'PDA_WAREHOUSE_MISMATCH')
|
||||
for(const phase of ['HOLD_FIRST','PICK_FIRST']){
|
||||
const before=await snapshot(),row=before.detail.order,source=before.sql.find(x=>x.collection==='sourceOrder')
|
||||
demand(row.currentStatus==='INBOUND_COMPLETED'&&row.detained===false&&source?.id===p.orderId&&source.customerId===p.customerId
|
||||
&&source.warehouseId===p.warehouseId&&source.version===Number(row.version)&&source.detained===0
|
||||
&&source.picked===0&&source.shipped===0&&!source.outboundNo&&Number(before.balance.availablePackages)===4
|
||||
&&Number(before.balance.holdPackages)===0&&Number(before.balance.pickedPackages)===0&&before.task.canPick===true,'LEGAL_UNPICKED_SOURCE_REQUIRED')
|
||||
const evidence={phase,before,commands:{},lockSamples:[]};report.phases.push(evidence);save()
|
||||
const holdKey=`${p.marker}-${phase}-HOLD`,pickKey=`${p.marker}-${phase}-PICK`
|
||||
const holdBody={held:true,chargeable:false,reason:`${p.marker} ${phase}`,expectedVersion:Number(row.version)}
|
||||
const pickBody={positionId:p.positionId,packages:1,productQuantity:p.pickQuantity,idempotencyKey:pickKey,remark:`${p.marker} ${phase}`}
|
||||
evidence.commands={hold:{key:holdKey,body:holdBody},pick:{key:pickKey,body:pickBody}};save()
|
||||
const pending=[],leasePromise=remote(buildOrderLockLeaseScript(p,phase,agent),'F32_LEASE_JSON').then(x=>({value:x}),error=>({error:error.message}))
|
||||
const waits=async count=>{
|
||||
for(let i=0;i<12;i++){
|
||||
const rows=await sql(buildLockEvidenceSql(p,phase));evidence.lockSamples.push(rows);save()
|
||||
const leases=rows.filter(x=>x.collection==='lease'),connections=[...new Set(rows.filter(x=>x.collection==='wait').map(x=>x.requestingConnectionId))]
|
||||
if(leases.length===1&&connections.length===count)return rows
|
||||
if(connections.length>count)throw Error('FOREIGN_SOURCE_CONTENTION');await pause(100)
|
||||
}throw Error('EXPECTED_MYSQL_LOCK_WAIT_NOT_OBSERVED')
|
||||
}
|
||||
try{
|
||||
await waits(0)
|
||||
const firstType=phase==='HOLD_FIRST'?'HOLD':'PICK',secondType=firstType==='HOLD'?'PICK':'HOLD'
|
||||
const start=type=>command(type,type==='HOLD'?holdBody:pickBody,type==='HOLD'?holdKey:pickKey).then(value=>({type,value}),error=>({type,error:error.message}))
|
||||
pending.push(start(firstType));const one=assertContendedRequests(await waits(1),p.orderId);evidence.firstWait=one;save()
|
||||
pending.push(start(secondType));evidence.twoWaiters=assertContendedRequests(await waits(2),p.orderId,one.requestingConnections[0]);save()
|
||||
}finally{evidence.lease=await leasePromise;evidence.responses=await Promise.all(pending);save()}
|
||||
demand(!evidence.lease.error&&evidence.lease.value.rows.at(-1).released===true&&evidence.responses.length===2
|
||||
&&evidence.lease.value.rows[0].connectionId===evidence.twoWaiters?.leaseConnectionId
|
||||
&&evidence.lease.value.rows[1].marker===`F32_${p.marker}_${phase}`,'LEASE_OR_BOTH_REQUESTS_UNPROVED')
|
||||
const win=evidence.responses.find(x=>x.type===(phase==='HOLD_FIRST'?'HOLD':'PICK')),lose=evidence.responses.find(x=>x!==win)
|
||||
demand(win&&!win.error&&lose&&!lose.error,'RACE_RESPONSE_UNKNOWN');ok(win.value)
|
||||
demand(lose.value.http===200&&String(lose.value.body.code)==='A0409','RACE_LOSER_NOT_CONFLICT')
|
||||
const after=evidence.after=await snapshot(),order=after.detail.order
|
||||
evidence.inventoryProof=assertRaceInventory(p,phase,before,after)
|
||||
ok(await command(win.type,win.type==='HOLD'?holdBody:pickBody,win.type==='HOLD'?holdKey:pickKey));const replay=evidence.replay=await snapshot()
|
||||
demand(stableSnapshot(after)===stableSnapshot(replay),'RACE_REPLAY_DRIFT')
|
||||
if(phase==='HOLD_FIRST'){
|
||||
ok(await command('HOLD',{held:false,chargeable:false,reason:`${p.marker} release own first phase`,expectedVersion:Number(order.version)},`${p.marker}-RELEASE`))
|
||||
evidence.restored=await snapshot();demand(evidence.restored.detail.order.detained===false&&Number(evidence.restored.balance.availablePackages)===4&&Number(evidence.restored.balance.holdPackages)===0,'OWN_HOLD_RELEASE_FAILED')
|
||||
}
|
||||
evidence.verified=true;save()
|
||||
}
|
||||
report.success=report.phases.length===2&&report.phases.every(x=>x.verified)
|
||||
}catch(error){report.success=false;report.error=/^[A-Z_0-9]+$/.test(error.message)?error.message:'F32_RACE_FAILED_INSPECT_CHECKPOINT'}
|
||||
finally{
|
||||
try{if(pda){ok(await call('POST','/auth/logout',{refreshToken:pda.refreshToken},pda.accessToken));report.pdaSessionClosed=true}}catch{report.pdaSessionClosed=false;report.success=false}
|
||||
try{if(activationAttempted){const user=ok(await call('PUT',`/platform/v1/iam/users/${p.pdaMembershipId}/status`,{status:'DISABLED'}));report.operatorRestored=user.status==='DISABLED';demand(report.operatorRestored,'PDA_DISABLE_NOT_PROVED')}}catch{report.operatorRestored=false;report.success=false}
|
||||
try{if(admin){ok(await call('POST','/auth/logout',{refreshToken:admin.refreshToken},admin.accessToken));report.adminSessionClosed=true}}catch{report.adminSessionClosed=false;report.success=false}
|
||||
fs.writeFileSync(fd,JSON.stringify(report,null,2));fs.fsyncSync(fd);fs.closeSync(fd);save()
|
||||
}
|
||||
console.log(JSON.stringify({success:report.success,report:resolve(a.report),phases:report.phases.map(x=>({phase:x.phase,verified:!!x.verified})),error:report.error,operatorRestored:report.operatorRestored}))
|
||||
if(!report.success)process.exitCode=1
|
||||
}
|
||||
if(process.argv[1]&&import.meta.url===pathToFileURL(resolve(process.argv[1])).href)main(process.argv.slice(2)).catch(error=>{console.error(/^[A-Z_0-9]+$/.test(error.message)?error.message:'F32_RACE_ARGUMENT_OR_REPORT_ERROR');process.exitCode=1})
|
||||
@@ -0,0 +1,85 @@
|
||||
import test from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { validateRacePlan,buildOrderLockLeaseScript,buildLockEvidenceSql,buildStockEvidenceSql,assertContendedRequests,assertRaceInventory } from './f32-pda-hold-race.mjs'
|
||||
import {assertReadOnlySql} from './fba-core-regression-evidence.mjs'
|
||||
const plan={purpose:'F32_OWNED_PDA_HOLD_RACE',marker:'OFFLINE',customerId:'21',orderId:'22',warehouseId:'23',balanceId:'24',positionId:'25',pdaMembershipId:'26',dispatchItemId:'27',dispatchNo:'DS-OFFLINE',tripNo:'TR-OFFLINE',expectedImage:'sha256:'+'a'.repeat(64),expectedPackages:4,pickPackages:1,pickQuantity:10}
|
||||
test('explicit owned scope rejects SQL injection, image drift and widened quantities',()=>{
|
||||
assert.equal(validateRacePlan(plan),plan)
|
||||
for(const changed of [{password:'do not persist'},{orderId:'22;DELETE FROM ll_order'},{customerId:21},{positionId:'0'},{expectedImage:'latest'},{marker:'x\';evil'},{expectedPackages:5},{pickPackages:2},{pickQuantity:0}])
|
||||
assert.throws(()=>validateRacePlan({...plan,...changed}))
|
||||
assert.throws(()=>buildOrderLockLeaseScript(plan,'OTHER','agent'))
|
||||
assert.throws(()=>buildOrderLockLeaseScript(plan,'HOLD_FIRST',"agent');evil"))
|
||||
})
|
||||
test('both evidence queries remain consistent read-only transactions with exact original source filters',()=>{
|
||||
for(const phase of ['HOLD_FIRST','PICK_FIRST']){
|
||||
const sql=buildLockEvidenceSql(plan,phase)
|
||||
assert.equal(assertReadOnlySql(sql),sql)
|
||||
assert.match(sql,/OBJECT_NAME='ll_order'/);assert.match(sql,/INDEX_NAME='PRIMARY'/)
|
||||
assert.match(sql,/LOCK_DATA=CONVERT\(0x3232/)
|
||||
}
|
||||
const sql=buildStockEvidenceSql(plan)
|
||||
assert.equal(assertReadOnlySql(sql),sql)
|
||||
assert.match(sql,/dispatch_item_id=27/);assert.match(sql,/order_id=22/);assert.match(sql,/balance_id=24/)
|
||||
})
|
||||
function chunks(source){return Array.from(source.matchAll(/\[('[A-Za-z0-9+/=]+'(?:,'[A-Za-z0-9+/=]+')*)\]\.join\(''\)/g),m=>Array.from(m[1].matchAll(/'([^']+)'/g),x=>x[1]).join(''))}
|
||||
test('lease has a fixed bounded server sleep, rollback and immutable healthy TEST target; Python compiles offline',()=>{
|
||||
const outer=buildOrderLockLeaseScript(plan,'HOLD_FIRST','agent')
|
||||
const inner=Buffer.from(chunks(outer)[0],'base64').toString('utf8')
|
||||
const python=Buffer.from(chunks(inner)[0],'base64').toString('utf8')
|
||||
assert.match(python,/SLEEP\(12\)/);assert.match(python,/ROLLBACK;/);assert.match(python,/timeout=22/)
|
||||
assert.ok(python.includes("JSON_OBJECT('marker','F32_OFFLINE_HOLD_FIRST'"),'Correlation must survive the MySQL client stripping comments')
|
||||
assert.match(python,/JSON_EXTRACT\('true','\$'\)/)
|
||||
assert.match(python,/innodb_lock_wait_timeout=5/);assert.match(python,/FOR UPDATE/)
|
||||
assert.match(python,/DEPLOY_ENV.*test/);assert.match(python,/MYSQL_PWD=e\['LASTLEG_DB_PASSWORD'\]/)
|
||||
assert.ok(python.includes(plan.expectedImage));assert.match(python,/Health.*healthy/)
|
||||
assert.doesNotMatch(python,/INSERT INTO|DELETE FROM|UPDATE ll_order|docker.*(?:stop|restart)/)
|
||||
const result=spawnSync(process.env.PYTHON||'python',['-c',"import sys;compile(sys.stdin.read(),'<offline-lease>','exec')"],{input:python,encoding:'utf8'})
|
||||
assert.equal(result.status,0,result.stderr||result.error?.message)
|
||||
})
|
||||
const rows=[{collection:'lease',connectionId:100},{collection:'wait',requestingConnectionId:101,blockingConnectionId:100,orderId:'22'}]
|
||||
function scanSnapshots(){
|
||||
const bucket=available=>({availablePackages:available,holdPackages:0,allocatedPackages:0,pickedPackages:0,receivingPackages:0})
|
||||
const before={balance:{...bucket(4),totalPackages:4},positions:[{id:'25',...bucket(3)},{id:'28',...bucket(1)}],detail:{order:{detained:false}},sql:[]}
|
||||
const after={balance:{...bucket(3),totalPackages:3},positions:[{id:'25',...bucket(2)},{id:'28',...bucket(1)}],detail:{order:{detained:false,version:1}},sql:[
|
||||
{collection:'sourceOrder',id:'22',customerId:'21',warehouseId:'23',picked:1,shipped:0,detained:0,version:1},
|
||||
{collection:'pick',id:'100',key:'OFFLINE-PICK_FIRST-PICK',positionId:'25',packages:1,quantity:10},
|
||||
{collection:'attribution',id:'101',pickId:'100',orderId:'22',packages:1,quantity:10},
|
||||
...[['ALLOCATE','AVAILABLE','ALLOCATED',1],['PICK','ALLOCATED','PICKED',1],['SHIP','PICKED','SHIPPED',-1]].map(([businessType,statusFrom,statusTo,delta],i)=>({collection:'ledger',id:String(102+i),key:`DISPATCH:DS-OFFLINE:27:OFFLINE-PICK_FIRST-PICK:${businessType}:operation`,positionId:'25',businessType,statusFrom,statusTo,delta,before:delta>0?0:1,after:delta>0?1:0})),
|
||||
]}
|
||||
return{before,after}
|
||||
}
|
||||
test('successful PDA scan accounts for one consumed box and its three immutable ledger steps, without inventing DTO counters',()=>{
|
||||
const {before,after}=scanSnapshots()
|
||||
assert.doesNotThrow(()=>assertRaceInventory(plan,'PICK_FIRST',before,after))
|
||||
})
|
||||
test('missing/duplicate scan ledger, wrong source and a second deduction cannot pass the inventory proof',()=>{
|
||||
for(const change of [x=>x.sql=x.sql.filter(y=>y.businessType!=='SHIP'),x=>x.sql.push({...x.sql.find(y=>y.businessType==='SHIP'),id:'999'}),
|
||||
x=>x.sql.find(y=>y.collection==='sourceOrder').picked=2,x=>x.sql.find(y=>y.collection==='attribution').orderId='other',
|
||||
x=>x.balance.availablePackages=2,x=>x.sql.find(y=>y.businessType==='SHIP').positionId='other',x=>x.sql.find(y=>y.businessType==='SHIP').delta=-2]){
|
||||
const {before,after}=scanSnapshots();change(after)
|
||||
assert.throws(()=>assertRaceInventory(plan,'PICK_FIRST',before,after))
|
||||
}
|
||||
})
|
||||
test('hold winner moves the exact two positions, with one history and no pick or attribution facts',()=>{
|
||||
const {before}=scanSnapshots()
|
||||
const after={balance:{...before.balance,availablePackages:0,holdPackages:4},positions:before.positions.map(x=>({...x,availablePackages:0,holdPackages:x.availablePackages})),
|
||||
detail:{order:{detained:true,version:2}},sql:[{collection:'sourceOrder',id:'22',customerId:'21',warehouseId:'23',picked:0,shipped:0,detained:1,version:2},
|
||||
{collection:'hold',id:'200',key:'OFFLINE-HOLD_FIRST-HOLD',held:1,version:2},
|
||||
...before.positions.map((x,i)=>({collection:'ledger',id:String(201+i),key:'hold',positionId:x.id,businessType:'HOLD',statusFrom:'AVAILABLE',statusTo:'HOLD',before:0,delta:x.availablePackages,after:x.availablePackages}))]}
|
||||
assert.equal(assertRaceInventory(plan,'HOLD_FIRST',before,after).heldPackages,4)
|
||||
for(const change of [x=>x.sql.pop(),x=>x.sql.push({collection:'pick',id:'300',key:'wrong'}),x=>x.sql.find(x=>x.collection==='hold').version=3,x=>x.sql.find(x=>x.collection==='ledger').positionId='other']){
|
||||
const changed=structuredClone(after);change(changed)
|
||||
assert.throws(()=>assertRaceInventory(plan,'HOLD_FIRST',before,changed))
|
||||
}
|
||||
})
|
||||
test('concurrency requires a live original row lease, first observed waiter and two distinct simultaneous waiters',()=>{
|
||||
assert.deepEqual(assertContendedRequests(rows,'22').requestingConnections,[101])
|
||||
const both=[...rows,{collection:'wait',requestingConnectionId:102,blockingConnectionId:100,orderId:'22'}]
|
||||
assert.deepEqual(assertContendedRequests(both,'22',101).requestingConnections,[101,102])
|
||||
assert.throws(()=>assertContendedRequests(rows,'22',101),/EXACT_REQUEST_CONTENTION/)
|
||||
assert.throws(()=>assertContendedRequests(both,'22',999),/FIRST_REQUEST_NO_LONGER_WAITING/)
|
||||
assert.throws(()=>assertContendedRequests(both,'other',101),/LOCK_EVIDENCE_SCOPE/)
|
||||
assert.throws(()=>assertContendedRequests(rows.slice(1),'22'),/LIVE_LEASE/)
|
||||
assert.throws(()=>assertContendedRequests([...both,{collection:'wait',requestingConnectionId:103,blockingConnectionId:100,orderId:'22'}],'22',101),/EXACT_REQUEST_CONTENTION/)
|
||||
})
|
||||
Reference in New Issue
Block a user