java-project/baiyue-firstleg
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(selfop): batch list checkpoints and publish each completed day

Browse Source
This commit is contained in:
ouyangxq
2026-10-14 10:09:16 +08:00
parent 674914c651
commit 6193043880
8 changed files with 143 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lastleg-service/src/main/java/com/baiyue/lastleg/selfoperated/analytics/outbound/OutboundCandidateBuilder.java
-4
View File
@@ -17,10 +17,6 @@ public class OutboundCandidateBuilder {
if("FINISH".equals(checkpoint.phase()))transaction.finish(scopeId,runId,owner,fence);
}
public Long reusable(Long scopeId,Long runId,String owner,long fence) {return transaction.reusable(scopeId,runId,owner,fence);}
public static boolean regularBuildDue(SelfopSyncTask task,LocalDateTime previousBuild,LocalDateTime now) {
return "REBUILD".equals(task.getTaskType()) || "RECHECK".equals(task.getTaskType()) || task.getAttemptCount()!=null && task.getAttemptCount()>1
|| previousBuild==null || !now.isBefore(previousBuild.plusMinutes(30));
}
public static Coverage contiguous(LocalDateTime initialFrom,List<Coverage> intervals) {
if(initialFrom==null)return new Coverage(null,null);LocalDateTime until=initialFrom;
for(Coverage interval:intervals.stream().filter(i->i.from()!=null && i.until()!=null && i.from().isBefore(i.until()))
lastleg-service/src/main/java/com/baiyue/lastleg/selfoperated/analytics/sync/SelfopDiscoveryService.java
+51 -1
View File
@@ -93,8 +93,12 @@ public class SelfopDiscoveryService {
}
}
route(task,run,manifest,lease,false);
for(var entry:manifest.getObjects().entrySet()) {
var entries=new ArrayList<>(manifest.getObjects().entrySet());
for(int index=0;index<entries.size();index++) {
if(!budget.hasTime()){taskService.yield(task.getId(),run.getId(),lease);return;}
int accepted=acceptListBatch(task,run,lease,manifest,entries,index);
if(accepted>0){index+=accepted-1;continue;}
var entry=entries.get(index);
var ref=entry.getValue();
if(ref.getRefreshTaskId()!=null){
var refresh=tasks.selectById(ref.getRefreshTaskId());require(refresh!=null,"SELFOP_REFRESH_TASK_MISSING");
@@ -211,6 +215,52 @@ public class SelfopDiscoveryService {
.set("catalog_from_utc",account.getCatalogFromUtc()).set("catalog_until_utc",account.getCatalogUntilUtc()).set("updated_at",now));}
});
}
/** Complete list rows need one bounded transaction/checkpoint per batch, not per order.
* Missing data, resumed references and newer revisions still follow the individual resolution path.
* Stages and their manifest references commit together; no network call occurs in this transaction. */
private int acceptListBatch(SelfopSyncTask task,SelfopSyncRun run,SelfopLeaseService.Lease lease,
SelfopRoutingManifest manifest,List<Map.Entry<String,SelfopRoutingManifest.Entry>> entries,int start) {
var candidates=new ArrayList<Map.Entry<String,SelfopRoutingManifest.Entry>>();
for(int i=start;i<Math.min(entries.size(),start+100);i++) {
var e=entries.get(i);var ref=e.getValue();
if(ref.getEffectiveStageId()!=null||ref.getRefreshTaskId()!=null||ref.getDirectoryStageId()==null)break;
candidates.add(e);
}
if(candidates.isEmpty())return 0;
var directory=new HashMap<Long,SelfopSyncStage>();
stages.selectList(new QueryWrapper<SelfopSyncStage>().in("id",candidates.stream().map(e->e.getValue().getDirectoryStageId()).toList()))
.forEach(s->directory.put(s.getId(),s));
var revised=new HashSet<String>();
stages.selectList(new QueryWrapper<SelfopSyncStage>().select("object_key_hash").eq("object_type","OUTBOUND_ORDER")
.in("object_key_hash",candidates.stream().map(Map.Entry::getKey).toList()).eq("validation_status","VALID")
.gt("observation_sequence",task.getObservationBaseline())).forEach(s->revised.add(s.getObjectKeyHash()));
var payloads=new ArrayList<SelfopPayloads.Order>();
for(var e:candidates) {
var s=directory.get(e.getValue().getDirectoryStageId());
if(s==null||!Objects.equals(s.getStageBatchId(),task.getId())||!"PENDING".equals(s.getValidationStatus())||revised.contains(e.getKey()))break;
var p=read(s.getPayloadJson(),SelfopPayloads.Order.class);
if(p==null||LingxingAnalyticsClient.needsDetail(p))break;
require(Objects.equals(p.sourceAccount(),task.getSourceAccount())&&Objects.equals(p.customerCode(),e.getValue().getCustomerCode())
&&Objects.equals(p.deliveryNo(),e.getValue().getDeliveryNo()),"SELFOP_DETAIL_IDENTITY_MISMATCH");
require(p.sourceStatus()!=null,"SELFOP_DETAIL_STRUCTURE_INVALID");identity(p.warehouseCode(),64);
payloads.add(p);
}
if(payloads.isEmpty())return 0;
leases.renew(lease);
tx.executeWithoutResult(s->{
var checkpoint=taskService.checkpoint(task,run,lease);
long first=leases.nextObservations(lease,payloads.size());
for(int i=0;i<payloads.size();i++) {
var ref=candidates.get(i).getValue();var old=directory.get(ref.getDirectoryStageId());var p=payloads.get(i);
var full=stage(task,run,p,old.getFirstPageNo(),first+i,"VALID");
full.setId(old.getId());full.setRunId(old.getRunId());full.setCreatedAt(old.getCreatedAt());
require(stages.updateById(full)==1,"SELFOP_STAGE_MISSING");
ref.setEffectiveStageId(full.getId());ref.setWarehouseCode(p.warehouseCode());
}
saveManifest(checkpoint.task(),checkpoint.run(),manifest,true);
});
return payloads.size();
}
private boolean collectLocated(SelfopSyncTask task,SelfopSyncRun run,SelfopLeaseService.Lease lease,SelfopBudget budget,Query query,SelfopRoutingManifest manifest) {
if(manifest.isDirectoryComplete())return true;
// Identity relocation consumes the full paginated result before trusting one match.
lastleg-service/src/main/java/com/baiyue/lastleg/selfoperated/analytics/sync/SelfopExecutionService.java
-4
View File
@@ -78,10 +78,6 @@ public class SelfopExecutionService {
}
var reused=candidates.reusable(task.getScopeId(),run.getId(),owner,primary.fence());
if(reused!=null){publisher.reuse(task.getScopeId(),run.getId(),reused,owner,primary.fence());return;}
var previousBuild=runs.selectOne(new QueryWrapper<SelfopSyncRun>().eq("scope_id",task.getScopeId()).isNotNull("build_started_at_utc").orderByDesc("build_started_at_utc").last("LIMIT 1"));
if(previousBuild!=null&&!OutboundCandidateBuilder.regularBuildDue(task,previousBuild.getBuildStartedAtUtc(),utcNow())){
taskService.waitForBudget(taskId,run.getId(),previousBuild.getBuildStartedAtUtc().plusMinutes(30),primary);return;
}
candidates.build(task.getScopeId(),run.getId(),owner,primary.fence());
if(config.isSalesPublicationEnabled())publisher.publish(task.getScopeId(),run.getId(),owner,primary.fence());
else taskService.candidateFinished(taskId,run.getId(),primary);
lastleg-service/src/main/java/com/baiyue/lastleg/selfoperated/analytics/sync/SelfopLeaseService.java
+7 -1
View File
@@ -66,9 +66,15 @@ public class SelfopLeaseService {
}
@Transactional
public long nextObservation(Lease lease) {
return nextObservations(lease,1);
}
/** Reserve consecutive observations under the same fence; gaps after a failed batch are harmless. */
@Transactional
public long nextObservations(Lease lease,int count) {
require(count>0&&count<=100,"SELFOP_OBSERVATION_BATCH_INVALID");
require(lease.account(),"SELFOP_ACCOUNT_LEASE_REQUIRED"); var s=locks.lockDiscovery(lease.scopeId());
check(lease,s.getLeaseOwner(),s.getFenceToken(),s.getLeaseUntilUtc(),utcNow());
long seq=s.getNextObservationSequence(); s.setNextObservationSequence(seq+1); s.setUpdatedAt(utcNow());
long seq=s.getNextObservationSequence(); s.setNextObservationSequence(Math.addExact(seq,count)); s.setUpdatedAt(utcNow());
discoveries.updateById(s); return seq;
}
public void verify(Lease lease) {