baiyue-firstleg/scripts/order-submit-dependency-fault-executor.mjs

// Explicit TEST-only runtime controller. Importing this module performs no HTTP.
import { readFileSync } from 'node:fs'
import { open, mkdir } from 'node:fs/promises'
import { createHash } from 'node:crypto'
import { resolve, dirname } from 'node:path'
import { pathToFileURL } from 'node:url'
import { assertOrderDependencyFaultObservation, orderPersistenceFacts } from './admin-test-env-modules/orders.mjs'

const runtime = readFileSync(new URL('./order-submit-dependency-fault-runtime.py', import.meta.url), 'utf8')
const proxySource = readFileSync(new URL('./order-submit-dependency-fault-proxy.py', import.meta.url)).toString('base64')
const commonSource = readFileSync(new URL('./test-event-fault-runtime.py', import.meta.url)).toString('base64')
const encode = value => Buffer.from(value).toString('base64')
const sha = value => createHash('sha256').update(JSON.stringify(value)).digest('hex')
const assert = (ok, message) => { if (!ok) throw Error(message) }
const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
const safeError = error => /^(?:[A-Z][A-Z_0-9]{1,100}|Dependency fault evidence: [A-Za-z /:-]+)$/.test(error?.message || '')
  ? error.message : 'ORDER_DEPENDENCY_VERIFICATION_FAILED'

export function scopeFor(plan) {
  assert(/^[A-Za-z0-9_-]{1,80}$/.test(plan?.marker || '') && /^[1-9]\d{0,19}$/.test(plan?.customerId || ''), 'Invalid TEST plan scope')
  for (const role of ['merchant', 'platform']) {
    const command = plan.commands?.[role]
    assert(command?.method === 'POST' && /^[1-9]\d{0,19}$/.test(command.orderId || '')
      && command.path === `/api/${role}/v1/orders/first-leg/${command.orderId}/submit${role === 'platform' ? `?tenantId=${plan.customerId}` : ''}`
      && Number.isSafeInteger(command.body?.expectedVersion) && command.body.expectedVersion >= 0
      && Object.keys(command.body).length === 1 && /^[A-Za-z0-9_-]{1,128}$/.test(command.headers?.['Idempotency-Key'] || '')
      && Object.keys(command.headers).length === 1
      && /^[0-9a-f]{64}$/.test(command.initialOrderSha256 || ''), 'Invalid frozen submission command')
  }
  assert(plan.commands.merchant.orderId !== plan.commands.platform.orderId, 'Two distinct new drafts required')
  assert(plan.runtimeChangesExecuted === false, 'Only a reviewed unexecuted plan is accepted')
  assert(/^[1-9]\d{0,19}$/.test(plan.receivingWarehouseId || ''), 'Bounded public receiving warehouse required')
  assert(Array.isArray(plan.phases) && plan.phases.length === 3, 'Exact three dependency phases required')
  for (const [dependency, property, path, messages] of [
    ['IAM','baiyue.clients.iam.url',`/internal/iam/tenants/${plan.customerId}/order-eligibility`,['IAM eligibility service is unavailable','IAM 下单资格响应无效']],
    ['FINANCE','baiyue.clients.finance.url',`/internal/finance/v1/wallet/accounts/${plan.customerId}`,['Finance account service is unavailable','Finance account response is invalid']],
    ['WAREHOUSE','baiyue.clients.warehouse.url',`/internal/warehouses/0/${plan.receivingWarehouseId}/active`,['主数据服务暂时不可用']],
  ]) {
    const rows = plan.phases.filter(row => row.dependency === dependency)
    assert(rows.length === 1 && rows[0].urlProperty === property && rows[0].targetPath === path && rows[0].targetMethod === 'GET'
      && rows[0].faultHttp === 503 && rows[0].expectedHttp === 200 && rows[0].expectedCode === 'S0503'
      && JSON.stringify(rows[0].expectedMessages) === JSON.stringify(messages), 'Dependency failure contract may not be weakened')
  }
  return { marker: plan.marker, customerId: plan.customerId, receivingWarehouseId: plan.receivingWarehouseId,
    orderIds: Object.fromEntries(['merchant', 'platform'].map(role => [role, plan.commands[role].orderId])), planSha256: sha(plan) }
}

export function renderOrderFaultAction(action, scope, agent, extra = {}) {
  assert(['preflight','prepare','activate','health','mode','status','records','restore','reconcile','verify-restored','finish'].includes(action), 'Unknown runtime action')
  assert(/^[A-Za-z0-9_.-]{1,128}$/.test(agent || '') && /^[A-Za-z0-9_-]{1,80}$/.test(scope.marker), 'Invalid agent/scope')
  if (['prepare','preflight'].includes(action)) {
    const routing = extra.routing
    assert(routing && Object.keys(routing).every(key => ['configurationReviewed','sourceSha256','upstreams','nacos'].includes(key))
      && routing.configurationReviewed === true && routing.nacos && Object.keys(routing.nacos).every(key => ['baseUrl','loginPath','configPath'].includes(key))
      && routing.upstreams && Object.keys(routing.upstreams).sort().join(',') === 'FINANCE,IAM,WAREHOUSE', 'Reviewed routing schema only; do not embed credentials')
    for (const address of [routing.nacos.baseUrl,...Object.values(routing.upstreams)]) {
      const url = new URL(address)
      assert(['http:','https:'].includes(url.protocol) && !url.username && !url.password && !url.search && !url.hash
        && ['', '/'].includes(url.pathname), 'Routing origins cannot contain credentials/path/query')
    }
    if (action === 'prepare') assert(/^sha256:[0-9a-f]{64}$/.test(extra.expectedImageId || '')
      && /^[0-9a-f]{64}$/.test(routing.sourceSha256 || ''), 'Reviewed image and Nacos source fingerprint required')
  }
  const data = encode(JSON.stringify({ ...extra, action, scope, commonSource, ...(action === 'prepare' ? { proxySource } : {}) }))
  const literal = encoded => '[' + encoded.match(/.{1,4000}/g).map(part => `'${part}'`).join(',') + "].join('')"
  const expression = value => literal(encode(value))
  const groovy = `def p=new ProcessBuilder(['python3','-c',new String(java.util.Base64.decoder.decode(${expression(runtime)}),'UTF-8'),${literal(data)}]).redirectErrorStream(true).start();def out=p.inputStream.getText('UTF-8');p.waitFor();print(out)`
  return `def c=jenkins.model.Jenkins.instance.getComputer('${agent}');if(c==null || c.channel==null)throw new RuntimeException('TEST agent unavailable');print(hudson.util.RemotingDiagnostics.executeGroovy(new String(java.util.Base64.decoder.decode(${expression(groovy)}),'UTF-8'),c.channel))`
}

export function createOrderFaultRuntime(env, scope, fetcher = globalThis.fetch) {
  assert(env.CHONGCHONG_ENV === 'test', 'CHONGCHONG_ENV=test required')
  assert(env.JENKINS_URL && env.JENKINS_USER && env.JENKINS_API_TOKEN && env.FBA_CORE_JENKINS_AGENT, 'Explicit TEST Jenkins credentials and agent required')
  return async (action, extra = {}) => {
    let response
    try {
      response = await fetcher(new URL('scriptText', env.JENKINS_URL.replace(/\/?$/, '/')), {
        method: 'POST', redirect: 'error', signal: AbortSignal.timeout(55000),
        headers: { Authorization: 'Basic ' + encode(env.JENKINS_USER + ':' + env.JENKINS_API_TOKEN), 'Content-Type': 'application/x-www-form-urlencoded' },
        body: new URLSearchParams({ script: renderOrderFaultAction(action, scope, env.FBA_CORE_JENKINS_AGENT, extra) }),
      })
    } catch { throw Error(`RUNTIME_${action.toUpperCase().replaceAll('-', '_')}_OUTCOME_UNKNOWN_USE_STATUS`) }
    assert(response.ok, `RUNTIME_HTTP_${response.status}_NO_AUTO_RETRY`)
    const output = await response.text()
    assert(!(output.includes('MultipleCompilationErrorsException') && output.includes('String too long')),
      'RUNTIME_RENDERER_COMPILATION_FAILED_NO_ACTION')
    const rows = output.split(/\r?\n/).filter(row => row.startsWith('ORDER_DEPENDENCY_JSON='))
    assert(rows.length === 1, 'RUNTIME_RESPONSE_AMBIGUOUS_NO_AUTO_RETRY')
    const result = JSON.parse(rows[0].slice('ORDER_DEPENDENCY_JSON='.length))
    assert(result?.ok === true, /^[A-Z_0-9]{1,90}$/.test(result?.error || '') ? result.error : 'RUNTIME_ACTION_FAILED')
    return result.data
  }
}

export async function runOrderFaultWindow({ plan, routing, expectedImageId, action, snapshot, submit, recovery, wait = pause }) {
  const scope = scopeFor(plan), report = { scope, success: false, observations: [], restoration: null }
  let prepareAttempted = false, failure
  const health = async () => {
    for (let i = 0; i < 180; i++) {
      const value = await action('health')
      if (value.healthy === true) return
      await wait(1000)
    }
    throw Error('ORDER_HEALTH_TIMEOUT')
  }
  try {
    for (const role of ['merchant','platform']) {
      const before = await snapshot(role)
      assert(createHash('sha256').update(orderPersistenceFacts(before.detail)).digest('hex') === plan.commands[role].initialOrderSha256
        && before.detail.order.status === 'DRAFT' && String(before.detail.order.tenantId) === scope.customerId
        && String(before.detail.order.id) === scope.orderIds[role]
        && String(before.detail.order.receivingWarehouseId) === scope.receivingWarehouseId
        && ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(before.detail[field]))
        && before.inboundPageComplete === true, 'PREPARED_DRAFT_CHANGED')
    }
    prepareAttempted = true
    report.prepared = await action('prepare', { routing, expectedImageId })
    for (let i = 0; i < 15; i++) {
      if ((await action('status')).proxy?.mode === 'PASS') break
      assert(i < 14, 'PROXY_STARTUP_NOT_READY'); await wait(1000)
    }
    await action('activate'); await health()
    for (const dependency of ['IAM','FINANCE','WAREHOUSE']) {
        for (const role of ['merchant','platform']) {
          const before = await snapshot(role), command = plan.commands[role]
          const request = { method: command.method, path: command.path, body: structuredClone(command.body),
            headers: { ...command.headers, 'X-Request-Id': `dep-${plan.marker}-${dependency}-${role}` } }
          let response
          await action('mode', { mode: dependency, ttl: 60 })
          try { response = await submit(role, request) }
          finally { await action('mode', { mode: 'PASS' }) }
          const after = await snapshot(role)
          const proxyRecords = (await action('records')).records.filter(row => row.dependency === dependency)
          report.observations.push(assertOrderDependencyFaultObservation(plan, dependency, role, { request, response, before, after, proxyRecords }))
        }
    }
  } catch (error) { failure = error }
  finally {
    if (prepareAttempted) {
      try {
        await action('restore') // Never retry a mutation after an ambiguous outcome.
        await health()
        report.restoredRuntime = await action('verify-restored')
        assert(report.restoredRuntime.originalImageId === expectedImageId
          && report.restoredRuntime.currentImageId === expectedImageId
          && report.restoredRuntime.originalFingerprint === report.restoredRuntime.currentFingerprint
          && report.restoredRuntime.nacosSourcesUnchanged === true
          && report.restoredRuntime.upstreamReads?.IAM === true && report.restoredRuntime.upstreamReads?.FINANCE === true
          && report.restoredRuntime.upstreamReads?.WAREHOUSE === true,
        'RESTORED_IDENTITY_OR_ROUTES_NOT_PROVED')
        const businessRecovery = await recovery()
        report.businessRecovery = businessRecovery
        report.restoration = await action('finish', { businessRecovery })
        assert(report.restoration.restored && report.restoration.proxyStopped && report.restoration.orderRoutesVerified, 'RESTORATION_NOT_COMPLETE')
      } catch (error) {
        report.recoveryError = safeError(error)
        if (error.recoveryDiagnostics) report.recoveryDiagnostics = error.recoveryDiagnostics
        failure ||= error
      }
    }
  }
  report.success = !failure && report.observations.length === 6 && Boolean(report.restoration?.restored)
  if (failure) report.error = safeError(failure)
  return report
}

export async function waitForRecoveredSubmission({ plan, role, snapshot, wait = pause }) {
  const command = plan.commands[role]
  let previous, orderFacts
  for (let i = 0; i <= 60; i++) {
    const current = await snapshot(role), detail = current.detail
    assert(current.inboundPageComplete === true && Array.isArray(current.inbounds) && current.inbounds.length <= 1
      && ['cargoes','parties','internalNotes','statusHistory'].every(field => Array.isArray(detail?.[field]))
      && String(detail.order.id) === command.orderId && String(detail.order.tenantId) === plan.customerId
      && detail.order.status === 'SUBMITTED' && Number(detail.order.version) === command.body.expectedVersion + 1
      && detail.statusHistory.filter(row => row.action === 'SUBMITTED').length === 1
      && detail.statusHistory.filter(row => row.action === 'CREATE').length === 1
      && (current.task === null || current.task && String(current.task.orderId) === command.orderId
        && String(current.task.tenantId) === plan.customerId)
      && current.inbounds.every(row => String(row.orderId) === command.orderId && row.orderNo === command.orderNo
        && String(row.tenantId) === plan.customerId), 'RECOVERY_SUBMISSION_FACTS_INVALID')
    const facts = orderPersistenceFacts(detail)
    orderFacts ??= facts
    assert(facts === orderFacts, 'RECOVERY_ORDER_DRIFT')
    const projected = current.inbounds.length === 1 && current.inbounds[0].orderStatusSnapshot === 'SUBMITTED'
    if (projected && previous && sha(previous) === sha(current)) return current
    previous = projected ? current : undefined
    if (i < 60) await wait(500)
  }
  throw Error('RECOVERY_PROJECTION_TIMEOUT')
}

function businessClient(env, plan, fetcher = globalThis.fetch) {
  const sessions = {}
  const call = async (method, path, body, headers = {}, token) => {
    const response = await fetcher(new URL(path, env.ADMIN_TEST_GATEWAY), { method, redirect:'error', signal:AbortSignal.timeout(20000),
      headers:{'Content-Type':'application/json', ...headers, ...(token ? {Authorization:`Bearer ${token}`} : {})}, ...(body === undefined ? {} : {body:JSON.stringify(body)}) })
    return {http:response.status,body:await response.json()}
  }
  const ok = value => { assert(value.http === 200 && String(value.body?.code) === '200', 'BUSINESS_READ_OR_RECOVERY_FAILED'); return value.body.data }
  const snapshot = async role => {
    const command = plan.commands[role]
    const detailPath = `/api/${role}/v1/orders/first-leg/${command.orderId}${role === 'platform' ? `?tenantId=${plan.customerId}` : ''}`
    const detail = ok(await call('GET',detailPath,undefined,{},sessions[role].accessToken))
    assert(String(detail?.order?.id)===command.orderId && String(detail.order.tenantId)===plan.customerId
      && ['cargoes','parties','internalNotes','statusHistory'].every(field=>Array.isArray(detail[field])), 'ORDER_SNAPSHOT_INCOMPLETE_OR_FOREIGN')
    const taskResponse = await call('GET',`/api/platform/v1/first-leg/tasks/by-order/${command.orderId}?tenantId=${plan.customerId}`,undefined,{},sessions.platform.accessToken)
    assert(taskResponse.http === 200 && ['200','A0404'].includes(String(taskResponse.body?.code)), 'TASK_SNAPSHOT_FAILED')
    const task = String(taskResponse.body.code) === '200' ? taskResponse.body.data ?? null : null
    const inbounds = []; let total
    for (let page=1;;page++) {
      const value=ok(await call('GET',`/api/platform/inbounds?tenantId=${plan.customerId}&orderNo=${encodeURIComponent(command.orderNo)}&page=${page}&size=200`,undefined,{},sessions.platform.accessToken))
      assert(Array.isArray(value.records) && Number(value.page)===page && Number.isSafeInteger(Number(value.total)), 'INBOUND_PAGE_INVALID')
      total ??= Number(value.total); assert(Number(value.total)===total && value.records.length===Math.min(200,Math.max(0,total-inbounds.length)), 'INBOUND_PAGE_INCOMPLETE')
      for(const row of value.records)assert(row.orderNo===command.orderNo && String(row.tenantId)===plan.customerId && !inbounds.some(before=>String(before.id)===String(row.id)), 'INBOUND_SCOPE_OR_DUPLICATE')
      inbounds.push(...value.records);if(inbounds.length===total)break
    }
    return {detail,task,inbounds,inboundPageComplete:true}
  }
  const submit = (role, request) => call(request.method,request.path,request.body,request.headers,sessions[role].accessToken)
  return { snapshot, submit,
    async login() {
      for(const [role,prefix] of [['platform','ADMIN_TEST'],['merchant','MERCHANT_TEST']]) {
        assert(env[prefix+'_USERNAME'] && env[prefix+'_PASSWORD'], 'EXPLICIT_BUSINESS_CREDENTIALS_REQUIRED')
        sessions[role]=ok(await call('POST','/api/auth/login',{loginId:env[prefix+'_USERNAME'],password:env[prefix+'_PASSWORD']}))
      }
      const profile=ok(await call('GET','/api/merchant/v1/iam/tenant/profile',undefined,{},sessions.merchant.accessToken))
      assert(String(profile.id)===plan.customerId && profile.status==='ACTIVE','REGISTERED_CUSTOMER_SESSION_MISMATCH')
    },
    async recovery() {
      const evidence={}
      try {
        for(const role of ['merchant','platform']) {
          const command=plan.commands[role]
          ok(await submit(role,command))
          const before=await waitForRecoveredSubmission({plan,role,snapshot})
          ok(await submit(role,command));const after=await snapshot(role)
          evidence[role]={orderId:command.orderId,before,after,snapshotSha256:sha(after)}
          assert(sha(before)===sha(after),'RECOVERY_REPLAY_DRIFT')
          Object.assign(evidence[role],{status:'SUBMITTED',sameKeyReplayVerified:true})
        }
      } catch(error) {
        error.recoveryDiagnostics=evidence
        throw error
      }
      return evidence
    },
    async logout() {
      let success=true
      for(const session of Object.values(sessions))try {ok(await call('POST','/api/auth/logout',{refreshToken:session.refreshToken},{},session.accessToken))}catch{success=false}
      return success
    },
  }
}

async function main(argv) {
  const [mode,...rest]=argv, args={}
  for(let i=0;i<rest.length;i+=2){assert(rest[i]?.startsWith('--') && rest[i+1] && !rest[i+1].startsWith('--'),'Use explicit --name value arguments');args[rest[i].slice(2)]=rest[i+1]}
  assert(['plan','preflight','run','status','reconcile','restore'].includes(mode),'Use plan/preflight/run/status/reconcile/restore')
  assert(args.plan,'--plan required');const plan=JSON.parse(readFileSync(args.plan,'utf8')),scope=scopeFor(plan)
  if(mode==='plan'){console.log(JSON.stringify({scope,actions:['prepare','activate','IAM x 2','FINANCE x 2','WAREHOUSE x 2','finally restore','verify original routes','finish proxy'],requiresExclusiveOrderWindow:true},null,2));return}
  assert(args['confirm-test-runtime']==='yes','--confirm-test-runtime yes required')
  const action=createOrderFaultRuntime(process.env,scope)
  if(mode==='preflight'){assert(args.routing,'--routing required');console.log(JSON.stringify(await action('preflight',{routing:JSON.parse(readFileSync(args.routing,'utf8'))}),null,2));return}
  if(['status','reconcile'].includes(mode)){console.log(JSON.stringify(await action(mode),null,2));return}
  assert(args.report && process.env.ADMIN_TEST_GATEWAY,'--report and ADMIN_TEST_GATEWAY required')
  await mkdir(dirname(resolve(args.report)),{recursive:true})
  const reportFile=await open(resolve(args.report),'wx',0o600) // Never overwrite a previous run or recovery report.
  const business=businessClient(process.env,plan);let result
  try {
    await business.login()
    if(mode==='run') {
      assert(args.routing && /^sha256:[0-9a-f]{64}$/.test(args['expected-image']||''),'--routing and immutable --expected-image required')
      result=await runOrderFaultWindow({plan,routing:JSON.parse(readFileSync(args.routing,'utf8')),expectedImageId:args['expected-image'],action,...business})
    } else {
      await action('restore')
      for(let i=0;i<180;i++){if((await action('health')).healthy)break;assert(i<179,'RESTORE_HEALTH_TIMEOUT');await pause(1000)}
      const runtime=await action('verify-restored'),businessRecovery=await business.recovery()
      result={success:true,runtime,businessRecovery,restoration:await action('finish',{businessRecovery})}
    }
  } catch(error){result={success:false,error:/^[A-Z_0-9 :.-]+$/.test(error.message)?error.message:'EXECUTION_FAILED_USE_RUNTIME_STATUS'}}
  finally {
    const sessionsClosed=await business.logout();result ??={success:false};result.scope=scope;result.sessionsClosed=sessionsClosed
    if(!sessionsClosed)result.success=false
    try {await reportFile.writeFile(JSON.stringify(result,null,2)+'\n');await reportFile.sync()}
    finally {await reportFile.close()}
  }
  console.log(JSON.stringify({success:result.success,report:resolve(args.report),restored:result.restoration?.restored===true}))
  if(!result.success)process.exitCode=1
}

if(process.argv[1] && import.meta.url===pathToFileURL(resolve(process.argv[1])).href)main(process.argv.slice(2)).catch(error=>{console.error(/^[A-Z_0-9]{1,90}$/.test(error.message)?error.message:'ORDER_DEPENDENCY_ARGUMENT_OR_CONFIGURATION_ERROR');process.exitCode=1})