fix(selfop): retry continuous outbound failures with bounded diagnostics
This commit is contained in:
+51
@@ -0,0 +1,51 @@
|
||||
package com.baiyue.lastleg.selfoperated.analytics.model;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.dao.CannotAcquireLockException;
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.*;
|
||||
|
||||
class SelfopFailureTest {
|
||||
@Test void wrappedDatabaseFailureRetainsDiagnosticsWithoutRawSqlOrCredentials() {
|
||||
var failure=SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,
|
||||
new CannotAcquireLockException("SQL customer=private token=secret",new SQLException("password=secret","40001",1213)));
|
||||
assertThat(failure.code()).isEqualTo("SELFOP_DATABASE_TRANSIENT");assertThat(failure.outboundRetryable()).isTrue();
|
||||
String stored=failure.stored("MERGE",1,3,true);
|
||||
assertThat(stored).hasSizeLessThan(500).doesNotContain("private","secret","password","customer");
|
||||
assertThat(SelfopFailure.publicSummary(failure.code(),stored)).contains("归并出库事实失败","SQLException","SQLState 40001","1213","第 1/3 次自动重试");
|
||||
}
|
||||
@Test void permanentDatabaseErrorsAndProviderProtectionDoNotRetry() {
|
||||
var failure=SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,new RuntimeException("sql",new SQLException("bad syntax","42000",1064)));
|
||||
assertThat(failure.code()).isEqualTo("SELFOP_DATABASE_FAILED");assertThat(failure.outboundRetryable()).isFalse();
|
||||
var provider=SelfopFailure.capture("INTEGRATION_TRANSPORT_FAILURE",true,new RuntimeException("private response"));
|
||||
assertThat(provider.outboundRetryable()).isFalse();
|
||||
assertThat(SelfopFailure.publicSummary(provider.code(),provider.stored("DIRECTORY",1,3,false)))
|
||||
.contains("响应结果未确认","不自动重试").doesNotContain("private");
|
||||
assertThat(SelfopFailure.coded("LINGXING_WMS_SOURCE_REJECTED",true).outboundRetryable()).isTrue();
|
||||
assertThat(SelfopFailure.coded("LINGXING_WMS_SOURCE_REJECTED",false).outboundRetryable()).isFalse();
|
||||
}
|
||||
@Test void diagnosticPhaseUsesOnlyCurrentAttemptCheckpoint() {
|
||||
var task=new SelfopSyncTask();var run=new SelfopSyncRun();run.setAttemptNo(25);run.setStatus("FETCHING");
|
||||
task.setProgressJson(json(Map.of("attemptNo",25,"phase","DETAIL")));
|
||||
assertThat(SelfopFailure.phase(task,run)).isEqualTo("DETAIL");
|
||||
task.setProgressJson(json(Map.of("attemptNo",24,"phase","PUBLISH")));
|
||||
assertThat(SelfopFailure.phase(task,run)).isEqualTo("FETCHING");
|
||||
task.setProgressJson("private invalid json");assertThat(SelfopFailure.phase(task,run)).isEqualTo("FETCHING");
|
||||
var failure=SelfopFailure.coded("SELFOP_EXECUTION_FAILED",true).atPhase("PUBLISH");
|
||||
assertThat(SelfopFailure.publicSummary(failure.code(),failure.stored("BUILD",1,3,true))).startsWith("发布统计结果失败");
|
||||
}
|
||||
@Test void readSideRejectsUnstructuredMessagesAndIgnoresExtraPayloadFields() {
|
||||
assertThat(SelfopFailure.publicSummary("FAIL","token=secret")).doesNotContain("secret");
|
||||
String stored=json(Map.of("v",1,"phase","PUBLISH","type","password=secret","sqlState","credential",
|
||||
"failures",4,"maxRetries",3,"scheduled",false,"payload","secret"));
|
||||
assertThat(SelfopFailure.publicSummary("FAIL",stored)).contains("发布统计结果失败","最多 3 次").doesNotContain("secret","credential","password");
|
||||
assertThat(SelfopFailure.publicSummary("FAIL","x".repeat(501))).doesNotContain("xxx");
|
||||
}
|
||||
@Test void boundedCauseTraversalHandlesCyclesAndRetainsUnexpectedExceptionType() {
|
||||
var a=new RuntimeException("secret");var b=new IllegalStateException("private");a.initCause(b);b.initCause(a);
|
||||
var failure=SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,a);
|
||||
assertThat(failure.exceptionType()).isEqualTo("IllegalStateException");assertThat(failure.outboundRetryable()).isTrue();
|
||||
}
|
||||
}
|
||||
+12
@@ -381,6 +381,18 @@ class CollectionQueryServiceTest {
|
||||
var result=service.runs(q(null,null,null)).records().get(0);
|
||||
assertEquals("SOURCE_ERROR",result.errorCode());assertFalse(result.failureSummary().contains("credential"));
|
||||
}
|
||||
@Test void runHistoryReturnsFailurePhaseCauseAndAutomaticRetryDecision() {
|
||||
Bound upper=new Bound(now,100L);when(mapper.runBound(any())).thenReturn(upper);
|
||||
var failure=com.baiyue.lastleg.selfoperated.analytics.model.SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,
|
||||
new RuntimeException("private SQL",new java.sql.SQLException("secret","40001",1213)));
|
||||
var r=new Attempt(90L,7L,"LXCOL-TASK",25,"FAILED","FETCHING","BUSINESS",now.minusHours(1),null,null,
|
||||
now,null,now,null,"PARTIAL",null,null,failure.code(),failure.stored("DETAIL",1,3,true),now,"A",1L,"hash");
|
||||
when(mapper.runs(any(),eq(upper))).thenReturn(List.of(r));
|
||||
var result=service.runs(q(null,null,null)).records().get(0);
|
||||
assertEquals("SELFOP_DATABASE_TRANSIENT",result.errorCode());
|
||||
assertTrue(result.failureSummary().contains("核验出库数据失败"));assertTrue(result.failureSummary().contains("第 1/3 次自动重试"));
|
||||
assertTrue(result.failureSummary().contains("SQLState 40001"));assertFalse(result.failureSummary().contains("private"));
|
||||
}
|
||||
@Test void inventoryFreshnessUsesCompletedObservationAndBarrierBlocksInitialCoverage() {
|
||||
Scope s=new Scope(1L,"A","OUTBOUND","hash","Asia/Shanghai","0 0 8,20 * * *",93600,
|
||||
now.minusMonths(2),now,now.minusMonths(2),now,0,90L,now.minusDays(8),now,7,null,null);
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@ class SelfopDiscoveryServiceTest {
|
||||
account.setRequestBudget(10); service.collect(task, run, lease);
|
||||
verify(client, times(1)).locate(any(SelfopRequestContext.class), eq("target"), eq(1), eq(1)); verify(client, times(1)).locate(any(SelfopRequestContext.class), eq("target"), eq(2), eq(1));
|
||||
assertEquals("ROUTED", run.getStatus());
|
||||
verify(taskService, never()).fail(anyLong(), anyLong(), anyString(), anyBoolean(), any());
|
||||
verify(taskService, never()).fail(anyLong(), anyLong(), any(SelfopFailure.class), any());
|
||||
}
|
||||
@Test void stableDirectoryScansYieldAndResumeBeforeDetailsWithoutRepeatingFirstScan() {
|
||||
directoryTwoOrders(); account.setRequestBudget(1);
|
||||
|
||||
+23
-1
@@ -10,6 +10,25 @@ import java.time.LocalDateTime;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class SelfopExecutionServiceTest {
|
||||
@Test void unexpectedBusinessFailureReachesDurableRetryHandlerWithSafeCauseMetadata() {
|
||||
var config=new SelfopAnalyticsProperties();config.setCollectionMode("BUSINESS");config.setEnvironment("production");
|
||||
var taskService=mock(SelfopTaskService.class);var leases=mock(SelfopLeaseService.class);var tasks=mock(SelfopSyncTaskMapper.class);
|
||||
var discovery=mock(SelfopDiscoveryService.class);
|
||||
var task=new SelfopSyncTask();task.setId(1L);task.setTaskRole("ACCOUNT_DISCOVERY");task.setDiscoveryScopeId(3L);
|
||||
task.setExecutionPurpose("BUSINESS");task.setStatus("READY");task.setTaskNo("LXCOL-TEST");
|
||||
var run=new SelfopSyncRun();run.setId(2L);var lease=new SelfopLeaseService.Lease(3L,"owner",8L,true);
|
||||
when(tasks.selectById(1L)).thenReturn(task);when(leases.acquireDiscovery(eq(3L),anyString())).thenReturn(lease);
|
||||
when(taskService.begin(1L,lease)).thenReturn(run);
|
||||
doThrow(new org.springframework.dao.CannotAcquireLockException("secret SQL",new java.sql.SQLException("private","40001",1213)))
|
||||
.when(discovery).collect(task,run,lease);
|
||||
var service=new SelfopExecutionService(config,taskService,leases,discovery,mock(SelfopInventoryCollector.class),mock(InventorySnapshotService.class),
|
||||
mock(OutboundFactService.class),mock(OutboundCandidateBuilder.class),mock(SelfopSalesPublisher.class),mock(SelfopSyncScopeMapper.class),tasks,
|
||||
mock(SelfopSyncStageMapper.class),mock(SelfopOutboundOrderMapper.class),mock(SelfopReferenceRecovery.class),mock(SelfopValidationRunner.class),
|
||||
mock(SelfopSyncRunMapper.class),mock(SelfopMasterDataCollector.class));
|
||||
service.execute(1L);
|
||||
verify(taskService).fail(eq(1L),eq(2L),argThat(f->"SELFOP_DATABASE_TRANSIENT".equals(f.code())&&f.outboundRetryable()&&"40001".equals(f.sqlState())),eq(lease));
|
||||
verify(leases).release(lease);
|
||||
}
|
||||
@Test void completedDayBuildsAndPublishesImmediatelyWithoutBuildIntervalWait() {
|
||||
var config=new SelfopAnalyticsProperties();config.setCollectionMode("BUSINESS");config.setEnvironment("production");config.setSalesPublicationEnabled(true);
|
||||
var taskService=mock(SelfopTaskService.class);var leases=mock(SelfopLeaseService.class);var tasks=mock(SelfopSyncTaskMapper.class);var scopes=mock(SelfopSyncScopeMapper.class);
|
||||
@@ -24,6 +43,9 @@ class SelfopExecutionServiceTest {
|
||||
service.execute(1L);
|
||||
var order=inOrder(candidates,publisher);order.verify(candidates).build(eq(3L),eq(2L),anyString(),eq(8L));order.verify(publisher).publish(eq(3L),eq(2L),anyString(),eq(8L));
|
||||
verify(taskService,never()).waitForBudget(anyLong(),anyLong(),any(),any());verifyNoInteractions(runs);
|
||||
doThrow(new IllegalStateException("private publication payload")).when(publisher).publish(eq(3L),eq(2L),anyString(),eq(8L));
|
||||
service.execute(1L);
|
||||
verify(taskService).fail(eq(1L),eq(2L),argThat(f->"PUBLISH".equals(f.executionPhase())&&"SELFOP_EXECUTION_FAILED".equals(f.code())),eq(lease));
|
||||
}
|
||||
@Test void unexpectedValidationFailureTerminatesSessionInsteadOfSchedulingRetry() {
|
||||
executeValidationFailure(new IllegalStateException("VALIDATION_STRUCTURE_REJECTED"),"VALIDATION_STRUCTURE_REJECTED");
|
||||
@@ -51,7 +73,7 @@ class SelfopExecutionServiceTest {
|
||||
recovery,validation,mock(SelfopSyncRunMapper.class),org.mockito.Mockito.mock(SelfopMasterDataCollector.class));
|
||||
service.execute(1L);
|
||||
verify(validation).fail(task,run,lease,code);verify(leases).release(lease);
|
||||
verify(taskService,never()).fail(anyLong(),anyLong(),anyString(),anyBoolean(),any());
|
||||
verify(taskService,never()).fail(anyLong(),anyLong(),any(SelfopFailure.class),any());
|
||||
verify(taskService,never()).waitForBudget(anyLong(),anyLong(),any(),any());
|
||||
verifyNoInteractions(discovery,stock,snapshots,facts,candidates,publisher,recovery);
|
||||
}
|
||||
|
||||
+2
-2
@@ -94,7 +94,7 @@ class SelfopInventoryRecoveryTest {
|
||||
@Test void genuineFailureClearsCursorAndNewAttemptStartsAtPageOneWithItsOwnStageBatch() {
|
||||
original.setFenceToken(lease.fence());
|
||||
when(runs.selectCount(any())).thenReturn(1L);
|
||||
service.fail(1L, 20L, "SELFOP_STOCK_DUPLICATE", true, lease);
|
||||
service.fail(1L, 20L,SelfopFailure.coded("SELFOP_STOCK_DUPLICATE", true), lease);
|
||||
assertEquals("FAILED", original.getStatus()); assertNull(task.getRoutingManifestJson());
|
||||
assertEquals("RETRY_WAIT", task.getStatus());
|
||||
// Verifies the final updateById cannot overwrite the explicit SQL-null cursor reset.
|
||||
@@ -122,7 +122,7 @@ class SelfopInventoryRecoveryTest {
|
||||
@Test void exhaustedFailuresRetainOldPublishedPointerAndCannotCreateAnotherAttempt() {
|
||||
original.setFenceToken(lease.fence());
|
||||
when(runs.selectCount(any())).thenReturn((long) config.getMaxAttempts());
|
||||
service.fail(1L, 20L, "SOURCE_TIMEOUT", true, lease);
|
||||
service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TIMEOUT", true), lease);
|
||||
assertEquals("DEAD", task.getStatus()); assertEquals("FAILED", original.getStatus());
|
||||
assertEquals(90L, inventory.getPublishedRunId()); assertEquals(11L, original.getBasePublishedRunId());
|
||||
assertThrows(IllegalStateException.class, () -> service.begin(1L, lease));
|
||||
|
||||
+12
@@ -16,6 +16,18 @@ import static org.assertj.core.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class SelfopSchedulePlannerTest {
|
||||
@Test void dailyPlanningNeverResetsRetryBudgetByReplacingWaitingOrExhaustedRoot(){
|
||||
var state=continuousState();var now=LocalDateTime.of(2026,10,13,4,0);
|
||||
planner.planContinuous(now);
|
||||
var root=state.stream().filter(t->t.getRequestKey().startsWith("continuous:outbound:2026-10-12:")).findFirst().orElseThrow();
|
||||
String key=root.getRequestKey();int count=state.size();
|
||||
for(String status:List.of("RETRY_WAIT","DEAD")) {
|
||||
root.setStatus(status);root.setAttemptCount(103);
|
||||
planner.planContinuous(now.plusMinutes(1));
|
||||
assertThat(state).hasSize(count);assertThat(state.stream().filter(t->key.equals(t.getRequestKey()))).containsExactly(root);
|
||||
assertThat(root.getStatus()).isEqualTo(status);assertThat(root.getAttemptCount()).isEqualTo(103);
|
||||
}
|
||||
}
|
||||
@Test void masterCatalogIsOneAccountBatchAndAnUnfinishedBatchSpansDays(){
|
||||
var state=continuousState();
|
||||
planner.planContinuous(LocalDateTime.of(2026,10,13,4,0));
|
||||
|
||||
+63
-12
@@ -306,7 +306,7 @@ class SelfopTaskServiceTest {
|
||||
|
||||
@Test void lostLeaseFailCannotTouchCurrentRunOrTask() {
|
||||
doThrow(new IllegalStateException("SELFOP_FENCE_REJECTED")).when(leases).verify(lease);
|
||||
assertThatThrownBy(() -> service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease))
|
||||
assertThatThrownBy(() -> service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease))
|
||||
.hasMessage("SELFOP_FENCE_REJECTED");
|
||||
verifyNoInteractions(locks, tasks, runs, scopes, accounts);
|
||||
}
|
||||
@@ -317,7 +317,7 @@ class SelfopTaskServiceTest {
|
||||
oldRun.setFenceToken(6L);
|
||||
when(locks.lockTask(1L)).thenReturn(task);
|
||||
when(locks.lockRun(20L)).thenReturn(oldRun);
|
||||
assertThatThrownBy(() -> service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease))
|
||||
assertThatThrownBy(() -> service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease))
|
||||
.hasMessage("SELFOP_RUN_LEASE_MISMATCH");
|
||||
verify(leases).verify(lease);
|
||||
assertThat(task.getStatus()).isEqualTo("RUNNING");
|
||||
@@ -370,7 +370,7 @@ class SelfopTaskServiceTest {
|
||||
assertThat(query.getParamNameValuePairs()).containsValue("FAILED").doesNotContainValue("YIELDED");
|
||||
return 2L;
|
||||
});
|
||||
service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease);
|
||||
service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease);
|
||||
assertThat(task.getStatus()).isEqualTo("RETRY_WAIT");
|
||||
assertThat(task.getApplyState()).isEqualTo("BLOCKED");
|
||||
assertThat(run.getStatus()).isEqualTo("FAILED");
|
||||
@@ -387,7 +387,7 @@ class SelfopTaskServiceTest {
|
||||
when(locks.lockRun(20L)).thenReturn(run("BUILDING"));
|
||||
when(scopes.selectById(10L)).thenReturn(scope("OUTBOUND"));
|
||||
when(runs.selectCount(any())).thenReturn(3L);
|
||||
service.fail(1L, 20L, "SOURCE_TEMPORARY", true, lease);
|
||||
service.fail(1L, 20L,SelfopFailure.coded("SOURCE_TEMPORARY", true), lease);
|
||||
assertThat(task.getStatus()).isEqualTo("DEAD");
|
||||
assertThat(task.getApplyState()).isEqualTo("BLOCKED");
|
||||
}
|
||||
@@ -397,7 +397,7 @@ class SelfopTaskServiceTest {
|
||||
var attempt=run("FETCHING");attempt.setExecutionPurpose("VALIDATION");
|
||||
when(locks.lockTask(1L)).thenReturn(child);when(locks.lockRun(20L)).thenReturn(attempt);
|
||||
when(tasks.selectById(1L)).thenReturn(child);when(locks.lockScope(10L)).thenReturn(scope("OUTBOUND"));
|
||||
assertThatThrownBy(()->service.fail(1L,20L,"VALIDATION_STRUCTURE_REJECTED",true,lease))
|
||||
assertThatThrownBy(()->service.fail(1L,20L,SelfopFailure.coded("VALIDATION_STRUCTURE_REJECTED",true),lease))
|
||||
.hasMessage("SELFOP_BUSINESS_PURPOSE_REQUIRED");
|
||||
assertThatThrownBy(()->service.retry(1L,"validation-retry","operator","offline retry"))
|
||||
.hasMessage("SELFOP_BUSINESS_PURPOSE_REQUIRED");
|
||||
@@ -465,7 +465,7 @@ class SelfopTaskServiceTest {
|
||||
assertThat(root.getNextRetryAtUtc()).isEqualTo(until);assertThat(root.getBusinessBudgetJson()).isEqualTo(budget);
|
||||
verify(tasks,never()).update(any(),any());
|
||||
}
|
||||
@Test void continuousFailureStopsInsteadOfBecomingAnotherScheduledRetry(){
|
||||
private SelfopSyncTask continuousOutboundRoot(){
|
||||
var root=boundedRoot(0,utcNow().plusMinutes(3));config.setContinuousBusinessCollectionEnabled(true);
|
||||
root.setTaskRole("ACCOUNT_DISCOVERY");root.setTaskType("BACKFILL");root.setScopeId(null);root.setWarehouseCode(null);root.setDiscoveryScopeId(99L);
|
||||
root.setRequestedBy(SelfopCollectionPolicy.CONTINUOUS_PLANNER);root.setScopeHash("a".repeat(64));
|
||||
@@ -473,8 +473,59 @@ class SelfopTaskServiceTest {
|
||||
root.setQueryFromUtc(LocalDateTime.of(2026,10,11,16,0));root.setQueryUntilUtc(LocalDateTime.of(2026,10,12,16,0));
|
||||
root.setQueryScopeJson(json(new SelfopDiscoveryService.Query("2026-10-12T00:00","2026-10-13T00:00",null,null,null)));
|
||||
root.setBusinessBudgetJson(com.baiyue.lastleg.selfoperated.analytics.request.SelfopBusinessBudget.freezeContinuous(utcNow()));
|
||||
var account=new SelfopOutboundDiscoveryScope();account.setId(99L);when(locks.lockDiscovery(99L)).thenReturn(account);
|
||||
return root;
|
||||
}
|
||||
@Test void continuousOutboundRetriesThreeTimesThenStopsWithoutResettingCheckpointOrIdentity(){
|
||||
var root=continuousOutboundRoot();root.setAttemptCount(99);
|
||||
root.setRoutingManifestJson("{\"directoryComplete\":true,\"nextPage\":26,\"scan\":2}");
|
||||
String checkpoint=root.getRoutingManifestJson(),key=root.getRequestKey(),budget=root.getBusinessBudgetJson();
|
||||
var accountLease=new SelfopLeaseService.Lease(99L,"owner",7L,true);
|
||||
for(int failures=1;failures<=4;failures++) {
|
||||
root.setStatus("RUNNING");var failed=run("FETCHING");failed.setAttemptNo(root.getAttemptCount());when(locks.lockRun(20L)).thenReturn(failed);
|
||||
when(runs.selectCount(any())).thenReturn((long)failures);
|
||||
var before=utcNow();service.fail(1L,20L,SelfopFailure.coded("SELFOP_EXECUTION_FAILED",true),accountLease);
|
||||
assertThat(root.getStatus()).isEqualTo(failures<=3?"RETRY_WAIT":"DEAD");
|
||||
assertThat(root.getRoutingManifestJson()).isEqualTo(checkpoint);assertThat(root.getRequestKey()).isEqualTo(key);
|
||||
assertThat(root.getBusinessBudgetJson()).isEqualTo(budget);
|
||||
String summary=SelfopFailure.publicSummary(failed.getErrorCode(),failed.getErrorMessage());
|
||||
if(failures<=3) {
|
||||
assertThat(root.getNextRetryAtUtc()).isBetween(before.plusSeconds(5L<<failures),utcNow().plusSeconds(5L<<failures));
|
||||
assertThat(summary).contains("第 "+failures+"/3 次自动重试");
|
||||
var next=service.begin(1L,accountLease);
|
||||
assertThat(next.getTaskId()).isEqualTo(1L);assertThat(root.getRoutingManifestJson()).isEqualTo(checkpoint);
|
||||
} else {assertThat(root.getNextRetryAtUtc()).isNull();assertThat(summary).contains("自动重试已用尽(最多 3 次)");}
|
||||
}
|
||||
verify(tasks,never()).insert(any(SelfopSyncTask.class));
|
||||
}
|
||||
@Test void continuousOutboundWarehouseFailureAlsoRetriesLocalProcessing(){
|
||||
var root=continuousOutboundRoot();root.setStatus("SUCCEEDED");
|
||||
var child=task("RUNNING");child.setId(2L);child.setBusinessRootTaskId(1L);child.setUpstreamTaskId(1L);child.setApplyState("APPLYING");
|
||||
child.setRequestKey("route:1:WH-A");child.setTaskType("BACKFILL");
|
||||
when(tasks.selectById(2L)).thenReturn(child);when(locks.lockTask(2L)).thenReturn(child);
|
||||
when(scopes.selectById(10L)).thenReturn(scope("OUTBOUND"));
|
||||
var failed=run("BUILDING");failed.setTaskId(2L);when(locks.lockRun(20L)).thenReturn(failed);when(runs.selectCount(any())).thenReturn(1L);
|
||||
service.fail(2L,20L,SelfopFailure.capture("SELFOP_EXECUTION_FAILED",true,
|
||||
new org.springframework.dao.CannotAcquireLockException("private SQL",new java.sql.SQLException("private params","40001",1213))),lease);
|
||||
assertThat(child.getStatus()).isEqualTo("RETRY_WAIT");assertThat(child.getApplyState()).isEqualTo("BLOCKED");
|
||||
assertThat(root.getStatus()).isEqualTo("SUCCEEDED");assertThat(failed.getErrorCode()).isEqualTo("SELFOP_DATABASE_TRANSIENT");
|
||||
assertThat(SelfopFailure.publicSummary(failed.getErrorCode(),failed.getErrorMessage())).contains("SQLState 40001","第 1/3").doesNotContain("private");
|
||||
}
|
||||
@Test void continuousOutboundProtectionAndContractFailuresNeverRetry(){
|
||||
for(String code:List.of("LINGXING_WMS_RATE_LIMITED","LINGXING_WMS_AUTHORIZATION_REJECTED","INTEGRATION_TRANSPORT_FAILURE",
|
||||
"SELFOP_SCOPE_MISMATCH","SELFOP_IDENTITY_INVALID","SELFOP_DATABASE_FAILED")) {
|
||||
var root=continuousOutboundRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));when(runs.selectCount(any())).thenReturn(1L);
|
||||
service.fail(1L,20L,SelfopFailure.coded(code,true),new SelfopLeaseService.Lease(99L,"owner",7L,true));
|
||||
assertThat(root.getStatus()).as(code).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
|
||||
}
|
||||
var root=continuousOutboundRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));
|
||||
service.fail(1L,20L,SelfopFailure.coded("SELFOP_EXECUTION_FAILED",false),new SelfopLeaseService.Lease(99L,"owner",7L,true));
|
||||
assertThat(root.getStatus()).isEqualTo("DEAD");
|
||||
}
|
||||
@Test void continuousFailureStopsInsteadOfBecomingAnotherScheduledRetry(){
|
||||
var root=continuousOutboundRoot();
|
||||
var run=run("FETCHING");when(locks.lockRun(20L)).thenReturn(run);
|
||||
service.fail(1L,20L,"LINGXING_WMS_RATE_LIMITED",true,new SelfopLeaseService.Lease(99L,"owner",7L,true));
|
||||
service.fail(1L,20L,SelfopFailure.coded("LINGXING_WMS_RATE_LIMITED",true),new SelfopLeaseService.Lease(99L,"owner",7L,true));
|
||||
assertThat(root.getStatus()).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
|
||||
assertThat(run.getStatus()).isEqualTo("FAILED");assertThat(run.getErrorCode()).isEqualTo("LINGXING_WMS_RATE_LIMITED");
|
||||
}
|
||||
@@ -483,7 +534,7 @@ class SelfopTaskServiceTest {
|
||||
var root=continuousInventoryRoot();root.setRoutingManifestJson("{\"nextPage\":6}");root.setAttemptCount(99);
|
||||
var failed=run("FETCHING");when(locks.lockRun(20L)).thenReturn(failed);
|
||||
when(runs.selectCount(any())).thenReturn(1L);config.setMaxAttempts(3);
|
||||
var before=utcNow();service.fail(1L,20L,code,true,lease);
|
||||
var before=utcNow();service.fail(1L,20L,SelfopFailure.coded(code,true),lease);
|
||||
assertThat(root.getStatus()).isEqualTo("RETRY_WAIT");
|
||||
assertThat(root.getNextRetryAtUtc()).isBetween(before.plusSeconds(10),utcNow().plusSeconds(10));
|
||||
assertThat(root.getRoutingManifestJson()).isNull();
|
||||
@@ -500,7 +551,7 @@ class SelfopTaskServiceTest {
|
||||
@Test void continuousInventoryScanFailureStopsAtConfiguredLimitAndClearsPersistedRetryTime(){
|
||||
var root=continuousInventoryRoot();root.setNextRetryAtUtc(utcNow().minusMinutes(1));
|
||||
when(locks.lockRun(20L)).thenReturn(run("FETCHING"));when(runs.selectCount(any())).thenReturn(3L);config.setMaxAttempts(3);
|
||||
service.fail(1L,20L,"SELFOP_STOCK_DUPLICATE",true,lease);
|
||||
service.fail(1L,20L,SelfopFailure.coded("SELFOP_STOCK_DUPLICATE",true),lease);
|
||||
assertThat(root.getStatus()).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
|
||||
verify(tasks,atLeastOnce()).update(isNull(),argThat((com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper<SelfopSyncTask> update)->
|
||||
update.getSqlSet().contains("next_retry_at_utc")));
|
||||
@@ -509,11 +560,11 @@ class SelfopTaskServiceTest {
|
||||
for(String code:List.of("LINGXING_WMS_RATE_LIMITED","LINGXING_WMS_AUTHORIZATION_REJECTED",
|
||||
"INTEGRATION_TRANSPORT_FAILURE","SELFOP_STOCK_SCOPE_LEAK","SELFOP_STOCK_IDENTITY_MISSING","SELFOP_EXECUTION_FAILED")) {
|
||||
var root=continuousInventoryRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));when(runs.selectCount(any())).thenReturn(1L);
|
||||
service.fail(1L,20L,code,true,lease);
|
||||
service.fail(1L,20L,SelfopFailure.coded(code,true),lease);
|
||||
assertThat(root.getStatus()).as(code).isEqualTo("DEAD");assertThat(root.getNextRetryAtUtc()).isNull();
|
||||
}
|
||||
var root=continuousInventoryRoot();when(locks.lockRun(20L)).thenReturn(run("FETCHING"));
|
||||
service.fail(1L,20L,"SELFOP_STOCK_DUPLICATE",false,lease);
|
||||
service.fail(1L,20L,SelfopFailure.coded("SELFOP_STOCK_DUPLICATE",false),lease);
|
||||
assertThat(root.getStatus()).isEqualTo("DEAD");
|
||||
}
|
||||
@Test void commandRetriesDeadContinuousInventoryThroughRealTaskServiceAndRecordsAudit(){
|
||||
@@ -568,7 +619,7 @@ class SelfopTaskServiceTest {
|
||||
var root=boundedRoot(7,utcNow().plusMinutes(3));
|
||||
var budget=read(root.getBusinessBudgetJson(),com.baiyue.lastleg.selfoperated.analytics.request.SelfopBusinessBudget.class);
|
||||
budget.stopReason="BUSINESS_SESSION_HTTP_LIMIT";root.setBusinessBudgetJson(json(budget));root.setStatus("DEAD");
|
||||
service.fail(1L,20L,"SELFOP_EXECUTION_FAILED",true,lease);
|
||||
service.fail(1L,20L,SelfopFailure.coded("SELFOP_EXECUTION_FAILED",true),lease);
|
||||
verify(runs,never()).updateById(any(SelfopSyncRun.class));verify(tasks,never()).updateById(any(SelfopSyncTask.class));
|
||||
assertThat(read(root.getBusinessBudgetJson(),com.baiyue.lastleg.selfoperated.analytics.request.SelfopBusinessBudget.class).stopReason).isEqualTo("BUSINESS_SESSION_HTTP_LIMIT");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user