fix(selfop): retry continuous outbound failures with bounded diagnostics

This commit is contained in:
2026-10-16 11:05:37 +08:00
parent a3a5862f89
commit 4e2ea89403
11 changed files with 299 additions and 38 deletions
@@ -0,0 +1,51 @@
package com.baiyue.lastleg.selfoperated.analytics.model;
import java.sql.SQLException;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.dao.CannotAcquireLockException;
import static org.assertj.core.api.Assertions.*;
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.*;
class SelfopFailureTest {
@Test void wrappedDatabaseFailureRetainsDiagnosticsWithoutRawSqlOrCredentials() {
var failure=SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,
new CannotAcquireLockException("SQL customer=private token=secret",new SQLException("password=secret","40001",1213)));
assertThat(failure.code()).isEqualTo("SELFOP_DATABASE_TRANSIENT");assertThat(failure.outboundRetryable()).isTrue();
String stored=failure.stored("MERGE",1,3,true);
assertThat(stored).hasSizeLessThan(500).doesNotContain("private","secret","password","customer");
assertThat(SelfopFailure.publicSummary(failure.code(),stored)).contains("归并出库事实失败","SQLException","SQLState 40001","1213","第 1/3 次自动重试");
}
@Test void permanentDatabaseErrorsAndProviderProtectionDoNotRetry() {
var failure=SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,new RuntimeException("sql",new SQLException("bad syntax","42000",1064)));
assertThat(failure.code()).isEqualTo("SELFOP_DATABASE_FAILED");assertThat(failure.outboundRetryable()).isFalse();
var provider=SelfopFailure.capture("INTEGRATION_TRANSPORT_FAILURE",true,new RuntimeException("private response"));
assertThat(provider.outboundRetryable()).isFalse();
assertThat(SelfopFailure.publicSummary(provider.code(),provider.stored("DIRECTORY",1,3,false)))
.contains("响应结果未确认","不自动重试").doesNotContain("private");
assertThat(SelfopFailure.coded("LINGXING_WMS_SOURCE_REJECTED",true).outboundRetryable()).isTrue();
assertThat(SelfopFailure.coded("LINGXING_WMS_SOURCE_REJECTED",false).outboundRetryable()).isFalse();
}
@Test void diagnosticPhaseUsesOnlyCurrentAttemptCheckpoint() {
var task=new SelfopSyncTask();var run=new SelfopSyncRun();run.setAttemptNo(25);run.setStatus("FETCHING");
task.setProgressJson(json(Map.of("attemptNo",25,"phase","DETAIL")));
assertThat(SelfopFailure.phase(task,run)).isEqualTo("DETAIL");
task.setProgressJson(json(Map.of("attemptNo",24,"phase","PUBLISH")));
assertThat(SelfopFailure.phase(task,run)).isEqualTo("FETCHING");
task.setProgressJson("private invalid json");assertThat(SelfopFailure.phase(task,run)).isEqualTo("FETCHING");
var failure=SelfopFailure.coded("SELFOP_EXECUTION_FAILED",true).atPhase("PUBLISH");
assertThat(SelfopFailure.publicSummary(failure.code(),failure.stored("BUILD",1,3,true))).startsWith("发布统计结果失败");
}
@Test void readSideRejectsUnstructuredMessagesAndIgnoresExtraPayloadFields() {
assertThat(SelfopFailure.publicSummary("FAIL","token=secret")).doesNotContain("secret");
String stored=json(Map.of("v",1,"phase","PUBLISH","type","password=secret","sqlState","credential",
"failures",4,"maxRetries",3,"scheduled",false,"payload","secret"));
assertThat(SelfopFailure.publicSummary("FAIL",stored)).contains("发布统计结果失败","最多 3 次").doesNotContain("secret","credential","password");
assertThat(SelfopFailure.publicSummary("FAIL","x".repeat(501))).doesNotContain("xxx");
}
@Test void boundedCauseTraversalHandlesCyclesAndRetainsUnexpectedExceptionType() {
var a=new RuntimeException("secret");var b=new IllegalStateException("private");a.initCause(b);b.initCause(a);
var failure=SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,a);
assertThat(failure.exceptionType()).isEqualTo("IllegalStateException");assertThat(failure.outboundRetryable()).isTrue();
}
}
@@ -381,6 +381,18 @@ class CollectionQueryServiceTest {
var result=service.runs(q(null,null,null)).records().get(0);
assertEquals("SOURCE_ERROR",result.errorCode());assertFalse(result.failureSummary().contains("credential"));
}
@Test void runHistoryReturnsFailurePhaseCauseAndAutomaticRetryDecision() {
Bound upper=new Bound(now,100L);when(mapper.runBound(any())).thenReturn(upper);
var failure=com.baiyue.lastleg.selfoperated.analytics.model.SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,
new RuntimeException("private SQL",new java.sql.SQLException("secret","40001",1213)));
var r=new Attempt(90L,7L,"LXCOL-TASK",25,"FAILED","FETCHING","BUSINESS",now.minusHours(1),null,null,
now,null,now,null,"PARTIAL",null,null,failure.code(),failure.stored("DETAIL",1,3,true),now,"A",1L,"hash");
when(mapper.runs(any(),eq(upper))).thenReturn(List.of(r));
var result=service.runs(q(null,null,null)).records().get(0);
assertEquals("SELFOP_DATABASE_TRANSIENT",result.errorCode());
assertTrue(result.failureSummary().contains("核验出库数据失败"));assertTrue(result.failureSummary().contains("第 1/3 次自动重试"));
assertTrue(result.failureSummary().contains("SQLState 40001"));assertFalse(result.failureSummary().contains("private"));
}
@Test void inventoryFreshnessUsesCompletedObservationAndBarrierBlocksInitialCoverage() {
Scope s=new Scope(1L,"A","OUTBOUND","hash","Asia/Shanghai","0 0 8,20 * * *",93600,
now.minusMonths(2),now,now.minusMonths(2),now,0,90L,now.minusDays(8),now,7,null,null);
@@ -138,7 +138,7 @@ class SelfopDiscoveryServiceTest {
account.setRequestBudget(10); service.collect(task, run, lease);
verify(client, times(1)).locate(any(SelfopRequestContext.class), eq("target"), eq(1), eq(1)); verify(client, times(1)).locate(any(SelfopRequestContext.class), eq("target"), eq(2), eq(1));
assertEquals("ROUTED", run.getStatus());
verify(taskService, never()).fail(anyLong(), anyLong(), anyString(), anyBoolean(), any());
verify(taskService, never()).fail(anyLong(), anyLong(), any(SelfopFailure.class), any());
}
@Test void stableDirectoryScansYieldAndResumeBeforeDetailsWithoutRepeatingFirstScan() {
directoryTwoOrders(); account.setRequestBudget(1);
@@ -10,6 +10,25 @@ import java.time.LocalDateTime;
import static org.mockito.Mockito.*;
class SelfopExecutionServiceTest {
@Test void unexpectedBusinessFailureReachesDurableRetryHandlerWithSafeCauseMetadata() {
var config=new SelfopAnalyticsProperties();config.setCollectionMode("BUSINESS");config.setEnvironment("production");
var taskService=mock(SelfopTaskService.class);var leases=mock(SelfopLeaseService.class);var tasks=mock(SelfopSyncTaskMapper.class);
var discovery=mock(SelfopDiscoveryService.class);
var task=new SelfopSyncTask();task.setId(1L);task.setTaskRole("ACCOUNT_DISCOVERY");task.setDiscoveryScopeId(3L);
task.setExecutionPurpose("BUSINESS");task.setStatus("READY");task.setTaskNo("LXCOL-TEST");
var run=new SelfopSyncRun();run.setId(2L);var lease=new SelfopLeaseService.Lease(3L,"owner",8L,true);
when(tasks.selectById(1L)).thenReturn(task);when(leases.acquireDiscovery(eq(3L),anyString())).thenReturn(lease);
when(taskService.begin(1L,lease)).thenReturn(run);
doThrow(new org.springframework.dao.CannotAcquireLockException("secret SQL",new java.sql.SQLException("private","40001",1213)))
.when(discovery).collect(task,run,lease);
var service=new SelfopExecutionService(config,taskService,leases,discovery,mock(SelfopInventoryCollector.class),mock(InventorySnapshotService.class),
mock(OutboundFactService.class),mock(OutboundCandidateBuilder.class),mock(SelfopSalesPublisher.class),mock(SelfopSyncScopeMapper.class),tasks,
mock(SelfopSyncStageMapper.class),mock(SelfopOutboundOrderMapper.class),mock(SelfopReferenceRecovery.class),mock(SelfopValidationRunner.class),
mock(SelfopSyncRunMapper.class),mock(SelfopMasterDataCollector.class));
service.execute(1L);
verify(taskService).fail(eq(1L),eq(2L),argThat(f->"SELFOP_DATABASE_TRANSIENT".equals(f.code())&&f.outboundRetryable()&&"40001".equals(f.sqlState())),eq(lease));
verify(leases).release(lease);
}
@Test void completedDayBuildsAndPublishesImmediatelyWithoutBuildIntervalWait() {
var config=new SelfopAnalyticsProperties();config.setCollectionMode("BUSINESS");config.setEnvironment("production");config.setSalesPublicationEnabled(true);
var taskService=mock(SelfopTaskService.class);var leases=mock(SelfopLeaseService.class);var tasks=mock(SelfopSyncTaskMapper.class);var scopes=mock(SelfopSyncScopeMapper.class);
@@ -24,6 +43,9 @@ class SelfopExecutionServiceTest {
service.execute(1L);
var order=inOrder(candidates,publisher);order.verify(candidates).build(eq(3L),eq(2L),anyString(),eq(8L));order.verify(publisher).publish(eq(3L),eq(2L),anyString(),eq(8L));
verify(taskService,never()).waitForBudget(anyLong(),anyLong(),any(),any());verifyNoInteractions(runs);
doThrow(new IllegalStateException("private publication payload")).when(publisher).publish(eq(3L),eq(2L),anyString(),eq(8L));
service.execute(1L);
verify(taskService).fail(eq(1L),eq(2L),argThat(f->"PUBLISH".equals(f.executionPhase())&&"SELFOP_EXECUTION_FAILED".equals(f.code())),eq(lease));
}
@Test void unexpectedValidationFailureTerminatesSessionInsteadOfSchedulingRetry() {
executeValidationFailure(new IllegalStateException("VALIDATION_STRUCTURE_REJECTED"),"VALIDATION_STRUCTURE_REJECTED");
@@ -51,7 +73,7 @@ class SelfopExecutionServiceTest {
recovery,validation,mock(SelfopSyncRunMapper.class),org.mockito.Mockito.mock(SelfopMasterDataCollector.class));
service.execute(1L);
verify(validation).fail(task,run,lease,code);verify(leases).release(lease);
verify(taskService,never()).fail(anyLong(),anyLong(),anyString(),anyBoolean(),any());
verify(taskService,never()).fail(anyLong(),anyLong(),any(SelfopFailure.class),any());
verify(taskService,never()).waitForBudget(anyLong(),anyLong(),any(),any());
verifyNoInteractions(discovery,stock,snapshots,facts,candidates,publisher,recovery);
}
@@ -94,7 +94,7 @@ class SelfopInventoryRecoveryTest {
@Test void genuineFailureClearsCursorAndNewAttemptStartsAtPageOneWithItsOwnStageBatch() {
original.setFenceToken(lease.fence());
when(runs.selectCount(any())).thenReturn(1L);
service.fail(1L, 20L, "SELFOP_STOCK_DUPLICATE", true, lease);
service.fail(1L, 20L,SelfopFailure.coded("SELFOP_STOCK_DUPLICATE", true), lease);
assertEquals("FAILED", original.getStatus()); assertNull(task.getRoutingManifestJson());
assertEquals("RETRY_WAIT", task.getStatus());
// Verifies the final updateById cannot overwrite the explicit SQL-null cursor reset.
@@ -122,7 +122,7 @@ class SelfopInventoryRecoveryTest {
@Test void exhaustedFailuresRetainOldPublishedPointerAndCannotCreateAnotherAttempt() {
original.setFenceToken(lease.fence());
when(runs.selectCount(any())).thenReturn((long) config.getMaxAttempts());
service.fail(1L, 20L, "SOURCE_TIMEOUT", true, lease);
service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TIMEOUT", true), lease);
assertEquals("DEAD", task.getStatus()); assertEquals("FAILED", original.getStatus());
assertEquals(90L, inventory.getPublishedRunId()); assertEquals(11L, original.getBasePublishedRunId());
assertThrows(IllegalStateException.class, () -> service.begin(1L, lease));
@@ -16,6 +16,18 @@ import static org.assertj.core.api.Assertions.*;
import static org.mockito.Mockito.*;
class SelfopSchedulePlannerTest {
@Test void dailyPlanningNeverResetsRetryBudgetByReplacingWaitingOrExhaustedRoot(){
var state=continuousState();var now=LocalDateTime.of(2026,10,13,4,0);
planner.planContinuous(now);
var root=state.stream().filter(t->t.getRequestKey().startsWith("continuous:outbound:2026-10-12:")).findFirst().orElseThrow();
String key=root.getRequestKey();int count=state.size();
for(String status:List.of("RETRY_WAIT","DEAD")) {
root.setStatus(status);root.setAttemptCount(103);
planner.planContinuous(now.plusMinutes(1));
assertThat(state).hasSize(count);assertThat(state.stream().filter(t->key.equals(t.getRequestKey()))).containsExactly(root);
assertThat(root.getStatus()).isEqualTo(status);assertThat(root.getAttemptCount()).isEqualTo(103);
}
}
@Test void masterCatalogIsOneAccountBatchAndAnUnfinishedBatchSpansDays(){
var state=continuousState();
planner.planContinuous(LocalDateTime.of(2026,10,13,4,0));
@@ -306,7 +306,7 @@ class SelfopTaskServiceTest {
@Test void lostLeaseFailCannotTouchCurrentRunOrTask() {
doThrow(new IllegalStateException("SELFOP_FENCE_REJECTED")).when(leases).verify(lease);
assertThatThrownBy(() -> service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease))
assertThatThrownBy(() -> service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease))
.hasMessage("SELFOP_FENCE_REJECTED");
verifyNoInteractions(locks, tasks, runs, scopes, accounts);
}
@@ -317,7 +317,7 @@ class SelfopTaskServiceTest {
oldRun.setFenceToken(6L);
when(locks.lockTask(1L)).thenReturn(task);
when(locks.lockRun(20L)).thenReturn(oldRun);
assertThatThrownBy(() -> service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease))
assertThatThrownBy(() -> service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease))
.hasMessage("SELFOP_RUN_LEASE_MISMATCH");
verify(leases).verify(lease);
assertThat(task.getStatus()).isEqualTo("RUNNING");
@@ -370,7 +370,7 @@ class SelfopTaskServiceTest {
assertThat(query.getParamNameValuePairs()).containsValue("FAILED").doesNotContainValue("YIELDED");
return 2L;
});
service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease);
service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease);
assertThat(task.getStatus()).isEqualTo("RETRY_WAIT");
assertThat(task.getApplyState()).isEqualTo("BLOCKED");
assertThat(run.getStatus()).isEqualTo("FAILED");
@@ -387,7 +387,7 @@ class SelfopTaskServiceTest {
when(locks.lockRun(20L)).thenReturn(run("BUILDING"));
when(scopes.selectById(10L)).thenReturn(scope("OUTBOUND"));
when(runs.selectCount(any())).thenReturn(3L);
service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease);
service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease);
assertThat(task.getStatus()).isEqualTo("DEAD");
assertThat(task.getApplyState()).isEqualTo("BLOCKED");
}
@@ -397,7 +397,7 @@ class SelfopTaskServiceTest {
var attempt=run("FETCHING");attempt.setExecutionPurpose("VALIDATION");
when(locks.lockTask(1L)).thenReturn(child);when(locks.lockRun(20L)).thenReturn(attempt);
when(tasks.selectById(1L)).thenReturn(child);when(locks.lockScope(10L)).thenReturn(scope("OUTBOUND"));
assertThatThrownBy(()->service.fail(1L,20L,"VALIDATION_STRUCTURE_REJECTED",true,lease))
assertThatThrownBy(()->service.fail(1L,20L,SelfopFailure.coded("VALIDATION_STRUCTURE_REJECTED",true),lease))
.hasMessage("SELFOP_BUSINESS_PURPOSE_REQUIRED");
assertThatThrownBy(()->service.retry(1L,"validation-retry","operator","offline retry"))
.hasMessage("SELFOP_BUSINESS_PURPOSE_REQUIRED");
@@ -465,7 +465,7 @@ class SelfopTaskServiceTest {
assertThat(root.getNextRetryAtUtc()).isEqualTo(until);assertThat(root.getBusinessBudgetJson()).isEqualTo(budget);
verify(tasks,never()).update(any(),any());
}
@Test void continuousFailureStopsInsteadOfBecomingAnotherScheduledRetry(){
private SelfopSyncTask continuousOutboundRoot(){
var root=boundedRoot(0,utcNow().plusMinutes(3));config.setContinuousBusinessCollectionEnabled(true);
root.setTaskRole("ACCOUNT_DISCOVERY");root.setTaskType("BACKFILL");root.setScopeId(null);root.setWarehouseCode(null);root.setDiscoveryScopeId(99L);
root.setRequestedBy(SelfopCollectionPolicy.CONTINUOUS_PLANNER);root.setScopeHash("a".repeat(64));
@@ -473,8 +473,59 @@ class SelfopTaskServiceTest {
root.setQueryFromUtc(LocalDateTime.of(2026,10,11,16,0));root.setQueryUntilUtc(LocalDateTime.of(2026,10,12,16,0));
root.setQueryScopeJson(json(new SelfopDiscoveryService.Query("2026-10-12T00:00","2026-10-13T00:00",null,null,null)));
root.setBusinessBudgetJson(com.baiyue.lastleg.selfoperated.analytics.request.SelfopBusinessBudget.freezeContinuous(utcNow()));
var account=new SelfopOutboundDiscoveryScope();account.setId(99L);when(locks.lockDiscovery(99L)).thenReturn(account);
return root;
}
@Test void continuousOutboundRetriesThreeTimesThenStopsWithoutResettingCheckpointOrIdentity(){
var root=continuousOutboundRoot();root.setAttemptCount(99);
root.setRoutingManifestJson("{\"directoryComplete\":true,\"nextPage\":26,\"scan\":2}");
String checkpoint=root.getRoutingManifestJson(),key=root.getRequestKey(),budget=root.getBusinessBudgetJson();
var accountLease=new SelfopLeaseService.Lease(99L,"owner",7L,true);
for(int failures=1;failures<=4;failures++) {
root.setStatus("RUNNING");var failed=run("FETCHING");failed.setAttemptNo(root.getAttemptCount());when(locks.lockRun(20L)).thenReturn(failed);
when(runs.selectCount(any())).thenReturn((long)failures);
var before=utcNow();service.fail(1L,20L,SelfopFailure.coded("SELFOP_EXECUTION_FAILED",true),accountLease);
assertThat(root.getStatus()).isEqualTo(failures<=3?"RETRY_WAIT":"DEAD");
assertThat(root.getRoutingManifestJson()).isEqualTo(checkpoint);assertThat(root.getRequestKey()).isEqualTo(key);
assertThat(root.getBusinessBudgetJson()).isEqualTo(budget);
String summary=SelfopFailure.publicSummary(failed.getErrorCode(),failed.getErrorMessage());
if(failures<=3) {
assertThat(root.getNextRetryAtUtc()).isBetween(before.plusSeconds(5L<<failures),utcNow().plusSeconds(5L<<failures));
assertThat(summary).contains("第 "+failures+"/3 次自动重试");
var next=service.begin(1L,accountLease);
assertThat(next.getTaskId()).isEqualTo(1L);assertThat(root.getRoutingManifestJson()).isEqualTo(checkpoint);
} else {assertThat(root.getNextRetryAtUtc()).isNull();assertThat(summary).contains("自动重试已用尽(最多 3 次)");}
}
verify(tasks,never()).insert(any(SelfopSyncTask.class));
}
@Test void continuousOutboundWarehouseFailureAlsoRetriesLocalProcessing(){
var root=continuousOutboundRoot();root.setStatus("SUCCEEDED");
var child=task("RUNNING");child.setId(2L);child.setBusinessRootTaskId(1L);child.setUpstreamTaskId(1L);child.setApplyState("APPLYING");
child.setRequestKey("route:1:WH-A");child.setTaskType("BACKFILL");
when(tasks.selectById(2L)).thenReturn(child);when(locks.lockTask(2L)).thenReturn(child);
when(scopes.selectById(10L)).thenReturn(scope("OUTBOUND"));
var failed=run("BUILDING");failed.setTaskId(2L);when(locks.lockRun(20L)).thenReturn(failed);when(runs.selectCount(any())).thenReturn(1L);
service.fail(2L,20L,SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,
new org.springframework.dao.CannotAcquireLockException("private SQL",new java.sql.SQLException("private params","40001",1213))),lease);
assertThat(child.getStatus()).isEqualTo("RETRY_WAIT");assertThat(child.getApplyState()).isEqualTo("BLOCKED");
assertThat(root.getStatus()).isEqualTo("SUCCEEDED");assertThat(failed.getErrorCode()).isEqualTo("SELFOP_DATABASE_TRANSIENT");
assertThat(SelfopFailure.publicSummary(failed.getErrorCode(),failed.getErrorMessage())).contains("SQLState 40001","第 1/3").doesNotContain("private");
}
@Test void continuousOutboundProtectionAndContractFailuresNeverRetry(){
for(String code:List.of("LINGXING_WMS_RATE_LIMITED","LINGXING_WMS_AUTHORIZATION_REJECTED","INTEGRATION_TRANSPORT_FAILURE",
"SELFOP_SCOPE_MISMATCH","SELFOP_IDENTITY_INVALID","SELFOP_DATABASE_FAILED")) {
var root=continuousOutboundRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));when(runs.selectCount(any())).thenReturn(1L);
service.fail(1L,20L,SelfopFailure.coded(code,true),new SelfopLeaseService.Lease(99L,"owner",7L,true));
assertThat(root.getStatus()).as(code).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
}
var root=continuousOutboundRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));
service.fail(1L,20L,SelfopFailure.coded("SELFOP_EXECUTION_FAILED",false),new SelfopLeaseService.Lease(99L,"owner",7L,true));
assertThat(root.getStatus()).isEqualTo("DEAD");
}
@Test void continuousFailureStopsInsteadOfBecomingAnotherScheduledRetry(){
var root=continuousOutboundRoot();
var run=run("FETCHING");when(locks.lockRun(20L)).thenReturn(run);
service.fail(1L,20L,"LINGXING_WMS_RATE_LIMITED",true,new SelfopLeaseService.Lease(99L,"owner",7L,true));
service.fail(1L,20L,SelfopFailure.coded("LINGXING_WMS_RATE_LIMITED",true),new SelfopLeaseService.Lease(99L,"owner",7L,true));
assertThat(root.getStatus()).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
assertThat(run.getStatus()).isEqualTo("FAILED");assertThat(run.getErrorCode()).isEqualTo("LINGXING_WMS_RATE_LIMITED");
}
@@ -483,7 +534,7 @@ class SelfopTaskServiceTest {
var root=continuousInventoryRoot();root.setRoutingManifestJson("{\"nextPage\":6}");root.setAttemptCount(99);
var failed=run("FETCHING");when(locks.lockRun(20L)).thenReturn(failed);
when(runs.selectCount(any())).thenReturn(1L);config.setMaxAttempts(3);
var before=utcNow();service.fail(1L,20L,code,true,lease);
var before=utcNow();service.fail(1L,20L,SelfopFailure.coded(code,true),lease);
assertThat(root.getStatus()).isEqualTo("RETRY_WAIT");
assertThat(root.getNextRetryAtUtc()).isBetween(before.plusSeconds(10),utcNow().plusSeconds(10));
assertThat(root.getRoutingManifestJson()).isNull();
@@ -500,7 +551,7 @@ class SelfopTaskServiceTest {
@Test void continuousInventoryScanFailureStopsAtConfiguredLimitAndClearsPersistedRetryTime(){
var root=continuousInventoryRoot();root.setNextRetryAtUtc(utcNow().minusMinutes(1));
when(locks.lockRun(20L)).thenReturn(run("FETCHING"));when(runs.selectCount(any())).thenReturn(3L);config.setMaxAttempts(3);
service.fail(1L,20L,"SELFOP_STOCK_DUPLICATE",true,lease);
service.fail(1L,20L,SelfopFailure.coded("SELFOP_STOCK_DUPLICATE",true),lease);
assertThat(root.getStatus()).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
verify(tasks,atLeastOnce()).update(isNull(),argThat((com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper<SelfopSyncTask> update)->
update.getSqlSet().contains("next_retry_at_utc")));
@@ -509,11 +560,11 @@ class SelfopTaskServiceTest {
for(String code:List.of("LINGXING_WMS_RATE_LIMITED","LINGXING_WMS_AUTHORIZATION_REJECTED",
"INTEGRATION_TRANSPORT_FAILURE","SELFOP_STOCK_SCOPE_LEAK","SELFOP_STOCK_IDENTITY_MISSING","SELFOP_EXECUTION_FAILED")) {
var root=continuousInventoryRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));when(runs.selectCount(any())).thenReturn(1L);
service.fail(1L,20L,code,true,lease);
service.fail(1L,20L,SelfopFailure.coded(code,true),lease);
assertThat(root.getStatus()).as(code).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
}
var root=continuousInventoryRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));
service.fail(1L,20L,"SELFOP_STOCK_DUPLICATE",false,lease);
service.fail(1L,20L,SelfopFailure.coded("SELFOP_STOCK_DUPLICATE",false),lease);
assertThat(root.getStatus()).isEqualTo("DEAD");
}
@Test void commandRetriesDeadContinuousInventoryThroughRealTaskServiceAndRecordsAudit(){
@@ -568,7 +619,7 @@ class SelfopTaskServiceTest {
var root=boundedRoot(7,utcNow().plusMinutes(3));
var budget=read(root.getBusinessBudgetJson(),com.baiyue.lastleg.selfoperated.analytics.request.SelfopBusinessBudget.class);
budget.stopReason="BUSINESS_SESSION_HTTP_LIMIT";root.setBusinessBudgetJson(json(budget));root.setStatus("DEAD");
service.fail(1L,20L,"SELFOP_EXECUTION_FAILED",true,lease);
service.fail(1L,20L,SelfopFailure.coded("SELFOP_EXECUTION_FAILED",true),lease);
verify(runs,never()).updateById(any(SelfopSyncRun.class));verify(tasks,never()).updateById(any(SelfopSyncTask.class));
assertThat(read(root.getBusinessBudgetJson(),com.baiyue.lastleg.selfoperated.analytics.request.SelfopBusinessBudget.class).stopReason).isEqualTo("BUSINESS_SESSION_HTTP_LIMIT");
}