test(firstleg): retry lost evidence uploads with the same command

scripts/admin-test-env-modules/phase3-loading.mjs

@@ -108,20 +108,41 @@ export async function uploadEvidence(context, taskId, warehouseId, evidenceType,
   const png = Uint8Array.from([137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82,0,0,0,1,0,0,0,1,8,6,0,0,0,31,21,196,137,0,0,0,13,73,68,65,84,8,215,99,248,207,192,240,31,0,5,0,1,255,137,153,61,29,0,0,0,0,73,69,78,68,174,66,96,130])
   const fileName = `${context.marker}-${String(fileSuffix).toLowerCase()}.png`
   form.append('file', new Blob([png], { type: 'image/png' }), fileName)
-  try {
-    const response = await fetch(`${context.options.gateway}${path}`, {
-      method: 'POST', headers: { Accept: 'application/json', Authorization: `Bearer ${context.pdaToken || context.token}`,
-        'Idempotency-Key': `${context.marker}-p3-evidence-${keySuffix}`.slice(0, 128) },
-      body: form, signal: AbortSignal.timeout(60_000),
-    })
-    const raw = await response.text(); let payload = null
-    try { payload = raw ? JSON.parse(raw) : null } catch { /* bounded raw evidence */ }
-    return { method: 'POST', path, http: response.status, body: payload,
-      raw: payload === null ? raw.slice(0, 500) : null,
-      requestId: payload?.requestId || response.headers.get('x-request-id'),
-      traceId: payload?.traceId || response.headers.get('x-trace-id') }
-  } catch (error) {
-    return { method: 'POST', path, http: 0, body: null, raw: String(error), requestId: null, traceId: null }
+  const url = `${context.options.gateway}${path}`
+  const headers = Object.freeze({ Accept: 'application/json', Authorization: `Bearer ${context.pdaToken || context.token}`,
+    'Idempotency-Key': `${context.marker}-p3-evidence-${keySuffix}`.slice(0, 128) })
+  const attempts = []
+  context.createdResources ??= {}
+  context.createdResources.loadingEvidenceTransport ??= []
+  // No credentials, absolute endpoint or raw error/cause object enters the report.
+  context.createdResources.loadingEvidenceTransport.push({ path, fileName,
+    idempotencyKey: headers['Idempotency-Key'], attempts })
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    let response
+    try {
+      response = await fetch(url, { method: 'POST', headers, body: form, signal: AbortSignal.timeout(60_000) })
+      const raw = await response.text(); let payload = null
+      try { payload = raw ? JSON.parse(raw) : null } catch { /* bounded raw evidence */ }
+      const result = { method: 'POST', path, http: response.status, body: payload,
+        raw: payload === null ? raw.slice(0, 500) : null,
+        requestId: payload?.requestId || response.headers.get('x-request-id'),
+        traceId: payload?.traceId || response.headers.get('x-trace-id') }
+      attempts.push({ attempt, http: result.http, code: payload?.code ?? null, outcome: 'HTTP_RESPONSE',
+        requestId: result.requestId, traceId: result.traceId })
+      return { ...result, transportAttempts: attempts }
+    } catch (error) {
+      const candidate = error?.cause?.code ?? error?.code
+      const causeCode = typeof candidate === 'string' && /^(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|EPIPE|ENETUNREACH|EHOSTUNREACH|UND_ERR_[A-Z_]+)$/.test(candidate)
+        ? candidate : null
+      const errorType = ['TypeError', 'AbortError', 'TimeoutError'].includes(error?.name) ? error.name : 'TransportError'
+      const http = response?.status ?? 0
+      attempts.push({ attempt, http, code: null, outcome: response ? 'RESPONSE_READ_FAILED' : 'NO_HTTP_RESPONSE', errorType, causeCode })
+      if (response || attempt === 3) return { method: 'POST', path, http, body: null,
+        raw: `${errorType}: ${response ? 'response read failed' : 'fetch failed'}${causeCode ? ` (${causeCode})` : ''}`,
+        requestId: response?.headers?.get('x-request-id') || null,
+        traceId: response?.headers?.get('x-trace-id') || null, transportAttempts: attempts }
+      await new Promise(resolve => setTimeout(resolve, 250 * attempt))
+    }
   }
 }