134 lines
14 KiB
JavaScript
134 lines
14 KiB
JavaScript
// Explicit TEST-only row-lock fault; importing this file performs no I/O.
|
|
import fs from 'node:fs'
|
|
import { createHash } from 'node:crypto'
|
|
import { resolve } from 'node:path'
|
|
import { pathToFileURL } from 'node:url'
|
|
import { parseCrossCustomerArguments, runCrossCustomer } from './fleet-cross-customer-test-env.mjs'
|
|
import { verifyFleetCrossCustomer } from './admin-test-env-modules/fleet-cross-customer-acceptance.mjs'
|
|
|
|
const demand=(v,message)=>{if(!v)throw Error(message)}
|
|
const literal=v=>`CONVERT(0x${Buffer.from(v).toString('hex')} USING utf8mb4) COLLATE utf8mb4_bin`
|
|
const encoded=v=>'['+Buffer.from(v).toString('base64').match(/.{1,4000}/g).map(x=>`'${x}'`).join(',')+"].join('')"
|
|
const id=v=>String(v??'')
|
|
const rr=['SET NAMES utf8mb4 COLLATE utf8mb4_bin;','SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;','START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;']
|
|
const pause=ms=>new Promise(done=>setTimeout(done,ms))
|
|
const canonical=v=>JSON.stringify(sortFields(v))
|
|
function sortFields(v){return Array.isArray(v)?v.map(sortFields):v&&typeof v==='object'?Object.fromEntries(Object.keys(v).sort().map(k=>[k,sortFields(v[k])])):v}
|
|
export function validateRollbackScope(s) {
|
|
demand(s&&/^[A-Za-z0-9_-]{1,64}$/.test(s.marker||'')&&/^[A-Za-z0-9_-]{1,64}$/.test(s.tripNo||''),'ROLLBACK_SCOPE_REQUIRED')
|
|
for(const key of ['firstCustomerId','secondCustomerId','firstDispatchId','secondDispatchId','firstOutboundId','secondPalletId'])
|
|
demand(typeof s[key]==='string'&&/^[1-9][0-9]{0,19}$/.test(s[key]),'ROLLBACK_IDENTIFIER_INVALID')
|
|
demand(s.firstCustomerId!==s.secondCustomerId&&s.firstDispatchId!==s.secondDispatchId&&/^sha256:[a-f0-9]{64}$/.test(s.expectedImage||''),'ROLLBACK_DISTINCT_CUSTOMERS_AND_RUNTIME_REQUIRED')
|
|
demand(/^[A-Za-z0-9_-]{1,64}$/.test(s.firstDispatchNo||''),'ROLLBACK_DISPATCH_NUMBER_INVALID')
|
|
return s
|
|
}
|
|
export function rollbackSql(scope,mode) {
|
|
const s=validateRollbackScope(scope),second=`customer_id=${literal(s.secondCustomerId)} AND dispatch_order_id=${s.secondDispatchId} AND warehouse_pallet_id=${s.secondPalletId} AND status='ACTIVE'`
|
|
if(mode==='LEASE')return `SET SESSION innodb_lock_wait_timeout=5;START TRANSACTION;SELECT JSON_OBJECT('lockId',CAST(id AS CHAR),'customerId',customer_id,'connectionId',CONNECTION_ID()) FROM ll_dispatch_pallet_lock WHERE ${second} FOR UPDATE;SELECT JSON_OBJECT('marker','ROLLBACK_${s.marker}','elapsed',SLEEP(65));ROLLBACK;SELECT JSON_OBJECT('released',JSON_EXTRACT('true','$'));`
|
|
const lease=`SELECT JSON_OBJECT('collection','lease','connectionId',t.trx_mysql_thread_id,'lockId',l.LOCK_DATA) FROM performance_schema.data_locks l JOIN information_schema.innodb_trx t ON t.trx_id=l.ENGINE_TRANSACTION_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME='ll_dispatch_pallet_lock' AND l.INDEX_NAME='PRIMARY' AND l.LOCK_STATUS='GRANTED' AND l.LOCK_DATA IN (SELECT CAST(id AS CHAR) FROM ll_dispatch_pallet_lock WHERE ${second}) AND t.trx_query LIKE ${literal('%ROLLBACK_'+s.marker+'%')};`
|
|
const waits=`SELECT JSON_OBJECT('collection','wait','requester',r.trx_mysql_thread_id,'blocker',b.trx_mysql_thread_id,'lockId',l.LOCK_DATA) FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID JOIN information_schema.innodb_trx r ON r.trx_id=w.REQUESTING_ENGINE_TRANSACTION_ID JOIN information_schema.innodb_trx b ON b.trx_id=w.BLOCKING_ENGINE_TRANSACTION_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME='ll_dispatch_pallet_lock' AND l.INDEX_NAME='PRIMARY' AND l.LOCK_DATA IN (SELECT CAST(id AS CHAR) FROM ll_dispatch_pallet_lock WHERE ${second});`
|
|
if(mode==='LOCKS')return [...rr,lease,waits,'COMMIT;'].join('\n')
|
|
if(mode==='OBSERVE')return ['SET NAMES utf8mb4 COLLATE utf8mb4_bin;','SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;','START TRANSACTION READ ONLY;',lease,waits,
|
|
`SELECT JSON_OBJECT('collection','firstDispatch','id',CAST(id AS CHAR),'status',status,'shipped',shipped_package_count) FROM ll_dispatch_order WHERE id=${s.firstDispatchId} AND customer_id=${literal(s.firstCustomerId)};`,
|
|
`SELECT JSON_OBJECT('collection','firstOutbound','id',CAST(id AS CHAR),'status',status) FROM ll_outbound_order WHERE id=${s.firstOutboundId} AND customer_id=${literal(s.firstCustomerId)};`,
|
|
`SELECT JSON_OBJECT('collection','firstReplay','count',COUNT(*)) FROM ll_dispatch_pda_operation WHERE customer_id=${literal(s.firstCustomerId)} AND operation_type='DEPART' AND idempotency_key=${literal(s.tripNo+':'+s.firstDispatchNo)};`,'COMMIT;'].join('\n')
|
|
if(mode==='RELEASED')return [...rr,lease,waits,'COMMIT;'].join('\n')
|
|
throw Error('ROLLBACK_MODE_INVALID')
|
|
}
|
|
export function buildRollbackRemoteScript(scope,mode,agent) {
|
|
demand(['RUNTIME','LEASE','LOCKS','OBSERVE','RELEASED'].includes(mode)&&/^[A-Za-z0-9_.-]{1,128}$/.test(agent||''),'ROLLBACK_REMOTE_MODE_REQUIRED')
|
|
const sql=mode==='RUNTIME'?"SELECT JSON_OBJECT('lockWaitSeconds',@@GLOBAL.innodb_lock_wait_timeout);":rollbackSql(scope,mode)
|
|
const imageGuard=mode==='RUNTIME'?'':` or c['Image']!='${scope.expectedImage}'`
|
|
const python=`import json,os,subprocess,urllib.parse,re
|
|
try:
|
|
c=json.loads(subprocess.check_output(['docker','inspect','firstleg-test-lastleg'],text=True,stderr=subprocess.DEVNULL))[0]
|
|
e=dict(x.split('=',1) for x in c['Config']['Env'])
|
|
if c['Name']!='/firstleg-test-lastleg' or c['Config'].get('Labels',{}).get('com.docker.compose.project')!='firstleg-test' or e.get('DEPLOY_ENV')!='test' or c['State'].get('Health',{}).get('Status')!='healthy'${imageGuard}:raise RuntimeError('TEST_RUNTIME_CHANGED')
|
|
u=urllib.parse.urlsplit(e['LASTLEG_DB_URL'].removeprefix('jdbc:'));host='127.0.0.1' if u.hostname=='host.docker.internal' else u.hostname
|
|
q=subprocess.run(['mysql','--host='+host,'--port='+str(u.port or 3306),'--user='+e['LASTLEG_DB_USERNAME'],'--batch','--raw','--skip-column-names',u.path.lstrip('/')],input=${JSON.stringify(sql)},capture_output=True,text=True,env=dict(os.environ,MYSQL_PWD=e['LASTLEG_DB_PASSWORD']),timeout=${mode==='LEASE'?75:25})
|
|
if q.returncode:
|
|
code=re.search(r'ERROR ([0-9]{3,5})',q.stderr);print('TRIP_ROLLBACK_JSON='+json.dumps({'ok':False,'error':'MYSQL_'+(code.group(1) if code else 'UNKNOWN')}))
|
|
else:print('TRIP_ROLLBACK_JSON='+json.dumps({'ok':True,'imageId':c['Image'],'rows':[json.loads(x) for x in q.stdout.splitlines() if x.strip()]}))
|
|
except Exception as error:
|
|
print('TRIP_ROLLBACK_JSON='+json.dumps({'ok':False,'error':str(error) if re.fullmatch('[A-Z_]{1,64}',str(error)) else 'REMOTE_ROLLBACK_FAILED'}))
|
|
`
|
|
const groovy=`def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${encoded(python)}),'UTF-8')]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
|
|
return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null||c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${encoded(groovy)}),'UTF-8'),c.channel))`
|
|
}
|
|
export function assertMidTransactionObservation(rows) {
|
|
const one=name=>rows.filter(x=>x.collection===name)
|
|
const lease=one('lease'),wait=one('wait'),dispatch=one('firstDispatch'),outbound=one('firstOutbound'),replay=one('firstReplay')
|
|
demand(lease.length===1&&wait.length===1&&wait[0].blocker===lease[0].connectionId&&wait[0].lockId===lease[0].lockId&&wait[0].requester!==wait[0].blocker,'OWNED_LATE_ROW_WAIT_NOT_PROVED')
|
|
demand(dispatch.length===1&&dispatch[0].status==='DEPARTED'&&dispatch[0].shipped===1&&outbound.length===1&&outbound[0].status==='OUTBOUNDED'&&replay.length===1&&replay[0].count===1,'FIRST_CUSTOMER_UNCOMMITTED_WRITE_NOT_PROVED')
|
|
return {leaseConnectionId:lease[0].connectionId,requestingConnectionId:wait[0].requester,lockId:lease[0].lockId,dirtyObservationOnly:true}
|
|
}
|
|
async function remote(env,scope,mode) {
|
|
const script=buildRollbackRemoteScript(scope,mode,env.FBA_CORE_JENKINS_AGENT)
|
|
const response=await fetch(new URL('scriptText',env.JENKINS_URL.replace(/\/?$/,'/')),{method:'POST',redirect:'error',signal:AbortSignal.timeout(mode==='LEASE'?95000:45000),headers:{Authorization:'Basic '+Buffer.from(env.JENKINS_USER+':'+env.JENKINS_API_TOKEN).toString('base64'),'Content-Type':'application/x-www-form-urlencoded'},body:new URLSearchParams({script})})
|
|
demand(response.ok,'ROLLBACK_EVIDENCE_HTTP_FAILED')
|
|
const lines=(await response.text()).split(/\r?\n/).filter(x=>x.startsWith('TRIP_ROLLBACK_JSON='));demand(lines.length===1,'ROLLBACK_REMOTE_RESULT_AMBIGUOUS')
|
|
const result=JSON.parse(lines[0].slice('TRIP_ROLLBACK_JSON='.length));demand(result.ok===true,/^[A-Z_0-9]+$/.test(result.error||'')?result.error:'ROLLBACK_REMOTE_FAILED')
|
|
return result
|
|
}
|
|
export function createDepartureRollbackVerifier(env,config,runtime) {
|
|
return async({context,fixture,ready,departPath,departBody,snapshot})=>{
|
|
const output=resolve('reports',`fleet-cross-${config.runId}`,'rollback.json'),fd=fs.openSync(output,'wx',0o600)
|
|
const report={verified:false,scope:'OWNED_TWO_CUSTOMER_DEPARTURE_ROLLBACK',startedAt:new Date().toISOString(),samples:[],command:{path:departPath,body:departBody},runtime}
|
|
const save=()=>{fs.ftruncateSync(fd,0);fs.writeSync(fd,JSON.stringify(report,null,2),0,'utf8');fs.fsyncSync(fd)}
|
|
let leasePromise,pending
|
|
try {
|
|
demand(ready.trip.dispatches?.length===2&&fixture.members.length===2,'TWO_CUSTOMER_READY_SCOPE_REQUIRED')
|
|
const [first,second]=ready.trip.dispatches.map(binding=>fixture.members.find(x=>x.dispatchNo===binding.dispatchNo))
|
|
demand(first&&second&&first.customerId!==second.customerId&&first.dispatchNo===fixture.members[0].dispatchNo,'CURRENT_DEPARTURE_ORDER_REQUIRED')
|
|
const firstState=ready.members[fixture.members.indexOf(first)],secondState=ready.members[fixture.members.indexOf(second)]
|
|
const scope=report.target=validateRollbackScope({marker:context.marker,tripNo:fixture.tripNo,expectedImage:runtime.imageId,
|
|
firstCustomerId:first.customerId,secondCustomerId:second.customerId,firstDispatchId:id(firstState.dispatch.id),secondDispatchId:id(secondState.dispatch.id),
|
|
firstOutboundId:id(firstState.outbound.outbound.id),secondPalletId:second.palletId,firstDispatchNo:first.dispatchNo})
|
|
save();leasePromise=remote(env,scope,'LEASE').then(value=>({value}),error=>({error:error.message}))
|
|
for(let i=0;i<12;i++) {
|
|
const rows=(await remote(env,scope,'LOCKS')).rows;report.samples.push({phase:'LEASE',rows});save()
|
|
if(rows.filter(x=>x.collection==='lease').length===1)break
|
|
demand(i!==11,'LIVE_LEASE_NOT_OBSERVED');await pause(250)
|
|
}
|
|
pending=context.request('POST',departPath,departBody,fixture.pdaToken).then(value=>({value}),error=>({error:error.message}))
|
|
for(let i=0;i<20;i++) {
|
|
const rows=(await remote(env,scope,'OBSERVE')).rows;report.samples.push({phase:'MID_TRANSACTION',rows});save()
|
|
if(rows.some(x=>x.collection==='wait')) {report.observation=assertMidTransactionObservation(rows);save();break}
|
|
demand(i!==19,'MID_TRANSACTION_NOT_OBSERVED');await pause(250)
|
|
}
|
|
report.response=await pending;save()
|
|
demand(!report.response.error,'DEPARTURE_OUTCOME_UNKNOWN_RETAIN_ORIGINAL_COMMAND')
|
|
const response=report.response.value
|
|
demand(response.http===500||(response.http===200&&String(response.body?.code)==='S0500'),'EXPECTED_DATABASE_FAILURE_RESPONSE_REQUIRED')
|
|
report.lease=await leasePromise;save()
|
|
demand(!report.lease.error&&report.lease.value.rows.at(-1)?.released===true,'LEASE_RELEASE_NOT_PROVED')
|
|
const released=report.released=(await remote(env,scope,'RELEASED')).rows
|
|
demand(released.length===0,'OWNED_ROW_LOCK_OR_WAITER_REMAINS')
|
|
const restored=await snapshot('ROLLBACK_AFTER_FAULT')
|
|
report.consistentSnapshot={beforeSha256:createHash('sha256').update(canonical(ready)).digest('hex'),afterSha256:createHash('sha256').update(canonical(restored)).digest('hex')}
|
|
demand(report.consistentSnapshot.beforeSha256===report.consistentSnapshot.afterSha256,'FULL_TWO_CUSTOMER_ROLLBACK_SNAPSHOT_CHANGED')
|
|
// The caller also checks restoration before sending the same original command.
|
|
report.verified=true;return report
|
|
}catch(error){report.error=error.message;throw error}
|
|
finally {
|
|
if(pending&&!report.response)report.response=await pending
|
|
if(leasePromise&&!report.lease)report.lease=await leasePromise
|
|
report.finishedAt=new Date().toISOString();save();fs.closeSync(fd)
|
|
}
|
|
}
|
|
}
|
|
export async function runTripRollback(config,{env=process.env}={}) {
|
|
if(!config.execute)return {...config,scope:'OWNED_TWO_CUSTOMER_DEPARTURE_ROLLBACK',fault:'65-second SELECT FOR UPDATE lease on the second customer pallet-lock row; first customer writes observed as uncommitted, default 50-second lock timeout rolls back the entire original request; then same original command and full normal chain',businessSqlWrites:false,deployment:false}
|
|
demand(env.CHONGCHONG_ENV==='test'&&['JENKINS_URL','JENKINS_USER','JENKINS_API_TOKEN','FBA_CORE_JENKINS_AGENT'].every(key=>env[key]),'EXPLICIT_TEST_EVIDENCE_PROFILE_REQUIRED')
|
|
const runtime=await remote(env,{},'RUNTIME');demand(runtime.rows.length===1&&runtime.rows[0].lockWaitSeconds===50,'DEFAULT_50_SECOND_LOCK_TIMEOUT_REQUIRED')
|
|
const sources=['scripts/fleet-trip-rollback-test-env.mjs','scripts/admin-test-env-modules/fleet-cross-customer-acceptance.mjs'].map(path=>({path,sha256:createHash('sha256').update(fs.readFileSync(path)).digest('hex')}))
|
|
return runCrossCustomer(config,{env,verify:async(context,fixture,options)=>{
|
|
fs.writeFileSync(resolve('reports',`fleet-cross-${config.runId}`,'rollback-source.json'),JSON.stringify({runtime,sources},null,2),{flag:'wx'})
|
|
return verifyFleetCrossCustomer(context,fixture,{...options,verifyDepartureRollback:createDepartureRollbackVerifier(env,config,runtime)})
|
|
}})
|
|
}
|
|
if(process.argv[1]&&import.meta.url===pathToFileURL(resolve(process.argv[1])).href) {
|
|
try {const config=parseCrossCustomerArguments(process.argv.slice(2)),result=await runTripRollback(config);console.log(JSON.stringify(config.execute?{success:result.success,error:result.error,summary:result.summary,requests:result.requestCount,goalComplete:false}:result,null,2));if(config.execute&&!result.success)process.exitCode=1}
|
|
catch(error){console.error(/^[A-Z_0-9]+$/.test(error.message)?error.message:'ROLLBACK_ENTRY_FAILED');process.exitCode=1}
|
|
}
|