fix(selfop): discover warehouses with bounded validation and optimize queries

This commit is contained in:
2026-10-12 22:22:08 +08:00
parent e42d358e71
commit b71380de31
20 changed files with 419 additions and 53 deletions
@@ -1,6 +1,10 @@
package com.baiyue.lastleg.selfoperated.analytics.model;
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.require;
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.read;
import static com.baiyue.lastleg.selfoperated.analytics.model.SelfopSupport.identity;
import java.util.Map;
import java.util.List;
/** Purpose is persisted on every task/run. Unknown purpose is never treated as business. */
public final class SelfopCollectionPolicy {
@@ -20,4 +24,27 @@ public final class SelfopCollectionPolicy {
public static String configuredPurpose(SelfopAnalyticsProperties config) {
String purpose=config.getCollectionMode();requireExecution(config,purpose);return purpose;
}
/** The action and warehouse are immutable task inputs, never taken from later configuration. */
public static boolean warehouseDiscovery(SelfopSyncTask task) {
require(task!=null&&"VALIDATION".equals(task.getExecutionPurpose())&&"VALIDATION".equals(task.getTaskType())
&&"ACCOUNT_DISCOVERY".equals(task.getTaskRole()),"SELFOP_VALIDATION_ENTRY_REQUIRED");
Map<?,?> query=read(task.getQueryScopeJson(),Map.class);
require(query!=null&&("WAREHOUSE_DISCOVERY".equals(query.get("action"))||"VALIDATION".equals(query.get("action"))),
"SELFOP_VALIDATION_ENTRY_REQUIRED");
if("WAREHOUSE_DISCOVERY".equals(query.get("action"))) {
require(!query.containsKey("warehouses")&&!query.containsKey("startTime")&&!query.containsKey("endTime"),
"SELFOP_WAREHOUSE_DISCOVERY_PARAMETERS");
return true;
}
return false;
}
public static String validationWarehouse(SelfopSyncTask task) {
require(!warehouseDiscovery(task),"SELFOP_VALIDATION_WAREHOUSE_REQUIRED");
Map<?,?> query=read(task.getQueryScopeJson(),Map.class);
require(query.get("warehouses") instanceof List<?> warehouses&&warehouses.size()==1
&&warehouses.get(0) instanceof String,"SELFOP_VALIDATION_WAREHOUSE_REQUIRED");
String warehouse=(String)((List<?>)query.get("warehouses")).get(0);
identity(warehouse,64);
return warehouse;
}
}
@@ -30,15 +30,12 @@ public final class AnalyticsInventorySql {
+" sales AS (SELECT customer_code,sku_code,SUM(outbound_quantity) observed_quantity,SUM(outbound_order_count) order_count,"
+"MIN(CASE WHEN quantity_unit='PCS' AND outbound_quantity>=0 THEN 1 ELSE 0 END) unit_valid FROM ll_selfop_sku_daily_sales "
+"WHERE sales_run_id=#{p.salesRun} AND"+scope(p)+" AND business_date>=#{p.from} AND business_date<#{p.until} GROUP BY customer_code,sku_code),"
+" last_sales AS (SELECT customer_code,sku_code,MAX(business_date) last_outbound FROM ll_selfop_sku_daily_sales WHERE sales_run_id=#{p.salesRun} AND"+scope(p)
+" AND business_date<=#{p.anchor} AND outbound_quantity>0 AND quantity_unit='PCS' GROUP BY customer_code,sku_code),"
+" values_by_sku AS (SELECT c.customer_code,c.sku_code,c.product_name,c.unit_valid,"
+"CASE WHEN c.sales_evidence IS NULL THEN c.stock_evidence WHEN c.stock_evidence IS NULL THEN c.sales_evidence ELSE LEAST(c.sales_evidence,c.stock_evidence) END evidence_at_utc,"
+stockValue("physical_quantity")+","+stockValue("available_quantity")+","+stockValue("frozen_quantity")+","+stockValue("transit_quantity")+","+stockValue("defective_physical_quantity")+",st.observed_at_utc,"
+"CASE WHEN c.sales_unit_valid=1 AND COALESCE(s.unit_valid,1)=1 AND #{p.complete30}=1 THEN COALESCE(s.observed_quantity,0) END owner_outbound,"
+"CASE WHEN c.sales_unit_valid=1 AND COALESCE(s.unit_valid,1)=1 AND #{p.salesKnown}=1 THEN s.observed_quantity END observed_outbound,s.order_count,l.last_outbound "
+"FROM profiles c LEFT JOIN stock st ON st.customer_code=c.customer_code AND st.sku_code=c.sku_code LEFT JOIN sales s ON s.customer_code=c.customer_code AND s.sku_code=c.sku_code "
+"LEFT JOIN last_sales l ON l.customer_code=c.customer_code AND l.sku_code=c.sku_code),"
+"CASE WHEN c.sales_unit_valid=1 AND COALESCE(s.unit_valid,1)=1 AND #{p.salesKnown}=1 THEN s.observed_quantity END observed_outbound,s.order_count "
+"FROM profiles c LEFT JOIN stock st ON st.customer_code=c.customer_code AND st.sku_code=c.sku_code LEFT JOIN sales s ON s.customer_code=c.customer_code AND s.sku_code=c.sku_code),"
+" qualified AS (SELECT v.*,CASE WHEN evidence_at_utc<=CONVERT_TZ(CAST(#{p.from} AS DATETIME),'+08:00','+00:00') THEN 1 ELSE 0 END observation_eligible,"
+"CASE WHEN unit_valid=1 AND evidence_at_utc<=CONVERT_TZ(CAST(#{p.from} AS DATETIME),'+08:00','+00:00') THEN owner_outbound END outbound_quantity30d FROM values_by_sku v),"
+" calculated AS (SELECT q.*,CASE WHEN available_quantity=0 THEN 0 WHEN available_quantity>0 AND outbound_quantity30d>0 THEN available_quantity*30/outbound_quantity30d END coverage_days,"
@@ -59,7 +56,17 @@ public final class AnalyticsInventorySql {
case "PHYSICAL_DESC" -> "physical_quantity";case "COVERAGE_DAYS_DESC" -> "coverage_days";case "OUTBOUND_30D_ASC" -> "outbound_quantity30d";default -> "available_quantity";};
return field+" IS NULL,"+field+("OUTBOUND_30D_ASC".equals(o.sortValue())?" ASC":" DESC")+",sku_code ASC,customer_code ASC";}
public static String inventoryPage(Map<String,Object> args){var p=(AnalyticsSqlRequest)args.get("p");var o=(AnalyticsInventoryOptions)args.get("o");
return typed(cte(p)+"SELECT customer_code,sku_code,product_name,unit_valid,evidence_at_utc,physical_quantity,available_quantity,frozen_quantity,transit_quantity,defective_physical_quantity,observed_at_utc,owner_outbound,observed_outbound,order_count,last_outbound,observation_eligible,outbound_quantity30d,coverage_days,no_outbound,above_threshold,unassessable FROM result"+filter(o)+" ORDER BY "+order(o)+" LIMIT #{p.size} OFFSET #{p.offset}");}
// The full candidate set is filtered and sorted first. Historical dates enrich only
// those immutable keys selected for this page and never participate in qualification.
String paged=cte(p)+", paged AS (SELECT * FROM result"+filter(o)+" ORDER BY "+order(o)+" LIMIT #{p.size} OFFSET #{p.offset}),"
+" last_sales AS (SELECT d.customer_code,d.sku_code,MAX(d.business_date) last_outbound FROM ll_selfop_sku_daily_sales d "
+"JOIN paged k ON k.customer_code=d.customer_code AND k.sku_code=d.sku_code WHERE d.sales_run_id=#{p.salesRun} "
+"AND d.source_account=#{p.account} AND d.warehouse_code=#{p.warehouse} AND d.business_date<=#{p.anchor} "
+"AND d.outbound_quantity>0 AND d.quantity_unit='PCS' GROUP BY d.customer_code,d.sku_code) ";
String outerOrder=order(o).replace("physical_quantity","k.physical_quantity").replace("coverage_days","k.coverage_days")
.replace("outbound_quantity30d","k.outbound_quantity30d").replace("available_quantity","k.available_quantity")
.replace("sku_code","k.sku_code").replace("customer_code","k.customer_code");
return typed(paged+"SELECT k.customer_code,k.sku_code,k.product_name,k.unit_valid,k.evidence_at_utc,k.physical_quantity,k.available_quantity,k.frozen_quantity,k.transit_quantity,k.defective_physical_quantity,k.observed_at_utc,k.owner_outbound,k.observed_outbound,k.order_count,l.last_outbound,k.observation_eligible,k.outbound_quantity30d,k.coverage_days,k.no_outbound,k.above_threshold,k.unassessable FROM paged k LEFT JOIN last_sales l ON l.customer_code=k.customer_code AND l.sku_code=k.sku_code ORDER BY "+outerOrder);}
public static String inventoryCount(Map<String,Object> args){var p=(AnalyticsSqlRequest)args.get("p");var o=(AnalyticsInventoryOptions)args.get("o");
return typed("ALL".equals(o.attentionValue())?"SELECT COUNT(*) FROM ("+keys(p)+") k":cte(p)+"SELECT COUNT(*) FROM result"+filter(o));}
public static String ownerTotals(Map<String,Object> args){var p=(AnalyticsSqlRequest)args.get("p");StringBuilder sql=new StringBuilder(cte(p)).append("SELECT COUNT(*) sku_count");
@@ -132,10 +132,11 @@ public class AnalyticsQueryService {
}
private AnalyticsResponses.Page<AnalyticsResponses.Metric> metrics(AnalyticsQuery q,String kind,boolean hot){
Snapshot s=context(q,false,false,hot); AnalyticsSqlRequest p=request(s,kind);
long total=mapper.count(p);
List<AnalyticsReadMapper.RankingStat> statistics=mapper.rankingStats(p);
long total=statistics.isEmpty()?0:statistics.get(0).total();
List<AnalyticsResponses.Metric> rows=q.offset()>=total?List.of():mapper.page(p).stream().map(r->metric(s,r)).toList();
Map<String,Long> excluded=new TreeMap<>();
if(!"INVENTORY".equals(kind))for(var e:mapper.excluded(p))excluded.put(e.reason(),e.total());
for(var e:statistics)if(e.reason()!=null)excluded.put(e.reason(),e.excludedCount());
return page(s,rows,total,excluded);
}
private AnalyticsResponses.SalesSummary summary(Snapshot s,Aggregate r){
@@ -20,9 +20,9 @@ public interface AnalyticsReadMapper {
List<AnalyticsReadRows.Aggregate> page(@Param("p") AnalyticsSqlRequest request);
@SelectProvider(type=AnalyticsReadSql.class,method="count")
long count(@Param("p") AnalyticsSqlRequest request);
record Exclusion(String reason,Long total){}
@SelectProvider(type=AnalyticsReadSql.class,method="excluded")
List<Exclusion> excluded(@Param("p") AnalyticsSqlRequest request);
record RankingStat(Long total,String reason,Long excludedCount){}
@SelectProvider(type=AnalyticsReadSql.class,method="rankingStats")
List<RankingStat> rankingStats(@Param("p") AnalyticsSqlRequest request);
@Select("SELECT customer_code,sku_code,product_name,quantity_unit,first_seen_at_utc,earliest_evidence_at_utc,"
+"evidence_type,evidence_run_id,last_seen_run_id FROM ll_selfop_sku_profile WHERE profile_run_id=#{run} "
+"AND source_account=#{account} AND warehouse_code=#{warehouse} AND customer_code=#{customer} AND sku_code=#{sku}")
@@ -29,6 +29,7 @@ public final class AnalyticsReadSql {
return sql;
}
private static String cte(String kind){
if("HOT".equals(kind)||"SLOW".equals(kind))return rankingCte();
String localEvidence="CONVERT_TZ(evidence_at_utc,'+00:00','+08:00')";
String observedStart="CONVERT_TZ(GREATEST(evidence_at_utc,#{p.coveredFrom}),'+00:00','+08:00')";
String observedEnd="CONVERT_TZ(#{p.coveredUntil},'+00:00','+08:00')";
@@ -84,9 +85,14 @@ public final class AnalyticsReadSql {
+"AND st.warehouse_code=#{p.warehouse} AND st.customer_code=c.customer_code AND st.sku_code=c.sku_code AND st.product_quality=0) ";
if("SUMMARY".equals(kind)){
int start=sql.indexOf(" metric_sales AS (");int end=sql.indexOf(" counted AS (",start);
sql=sql.substring(0,start)+" metric_sales AS (SELECT customer_code,sku_code,CAST(NULL AS DECIMAL(20,6)) window_quantity,"
+"CAST(NULL AS DECIMAL(20,0)) window_orders,CAST(NULL AS DECIMAL(20,6)) sales30_quantity,"
+"CAST(NULL AS DECIMAL(20,6)) today_quantity,CAST(NULL AS CHAR(32)) min_unit,CAST(NULL AS CHAR(32)) max_unit FROM bounds WHERE 1=0),"+sql.substring(end);
// An empty aggregate joined back through bounds can still be merged into
// another candidate cross product by MySQL. Summary has no metric join at all.
sql=sql.substring(0,start)+sql.substring(end);
sql=sql.replace("AND (m.min_unit IS NULL OR (m.min_unit=b.quantity_unit AND m.max_unit=b.quantity_unit)) ","")
.replace("m.window_quantity,m.window_orders,m.sales30_quantity,m.today_quantity",
"CAST(NULL AS DECIMAL(20,6)) window_quantity,CAST(NULL AS SIGNED) window_orders,"
+"CAST(NULL AS DECIMAL(20,6)) sales30_quantity,CAST(NULL AS DECIMAL(20,6)) today_quantity")
.replace("LEFT JOIN metric_sales m ON m.customer_code=b.customer_code AND m.sku_code=b.sku_code","");
// A sales chart does not require inventory quantities or freshness. Keep only profile evidence.
int stockJoin=sql.indexOf("LEFT JOIN ll_selfop_inventory_snapshot st");
sql=sql.substring(0,stockJoin)+"LEFT JOIN (SELECT CAST(NULL AS DECIMAL(20,0)) snapshot_run_id,CAST(NULL AS CHAR(64)) source_account,"
@@ -97,6 +103,32 @@ public final class AnalyticsReadSql {
}
return sql;
}
/** Ranking consumes the 30-day aggregate, not chart day-span counts. Its 7/30-day
* window is contained in this scan, so the same unit proof covers both windows. */
private static String rankingCte(){
String valid="c.profile_unit_valid=1 AND (m.min_unit IS NULL OR (m.min_unit=c.quantity_unit AND m.max_unit=c.quantity_unit))";
return candidates(false)+" metric_sales AS (SELECT customer_code,sku_code,"
+"SUM(CASE WHEN business_date>=#{p.windowFrom} AND business_date<#{p.anchor} THEN outbound_quantity ELSE 0 END) window_quantity,"
+"SUM(CASE WHEN business_date>=#{p.windowFrom} AND business_date<#{p.anchor} THEN outbound_order_count ELSE 0 END) window_orders,"
+"SUM(CASE WHEN business_date<#{p.anchor} THEN outbound_quantity ELSE 0 END) sales30_quantity,"
+"SUM(CASE WHEN business_date=#{p.anchor} THEN outbound_quantity ELSE NULL END) today_quantity,MIN(quantity_unit) min_unit,MAX(quantity_unit) max_unit "
+"FROM ll_selfop_sku_daily_sales WHERE sales_run_id=#{p.salesRun} AND"+KEY_FILTER
+" AND business_date>=TIMESTAMPADD(DAY,-30,#{p.anchor}) AND business_date<TIMESTAMPADD(DAY,1,#{p.anchor}) GROUP BY customer_code,sku_code),"
+" result AS (SELECT c.customer_code,c.sku_code,c.product_name,c.quantity_unit,c.evidence_at_utc,CASE WHEN "+valid+" THEN 1 ELSE 0 END unit_valid,"
+"CAST(NULL AS DECIMAL(20,6)) observed_quantity,CAST(NULL AS SIGNED) sku_order_count,0 complete_day_count,0 partial_day_count,0 unknown_day_count,"
+"CASE WHEN "+valid+" AND #{p.completeWindow}=1 THEN COALESCE(m.window_quantity,0) END window_quantity,"
+"CASE WHEN "+valid+" AND #{p.completeWindow}=1 THEN COALESCE(m.window_orders,0) END window_sku_order_count,"
+"CASE WHEN "+valid+" AND #{p.complete30}=1 THEN COALESCE(m.sales30_quantity,0) END sales30_quantity,"
+"CASE WHEN "+valid+" AND #{p.salesKnown}=1 THEN m.today_quantity END today_quantity,"
+"st.physical_quantity,st.available_quantity,st.frozen_quantity,st.transit_quantity,st.observed_at_utc inventory_observed_at_utc,"
+"CASE WHEN st.sku_code IS NULL THEN 'UNKNOWN' WHEN st.quality_status='INVALID' THEN 'INVALID' "
+"WHEN st.quality_status<>'VALID' OR st.available_quantity IS NULL OR st.physical_quantity IS NULL OR st.frozen_quantity IS NULL OR st.transit_quantity IS NULL THEN 'UNKNOWN' "
+"WHEN st.available_quantity<0 OR st.physical_quantity<0 OR st.frozen_quantity<0 OR st.transit_quantity<0 THEN 'INVALID' "
+"WHEN #{p.stockFresh}=0 THEN 'STALE' ELSE 'VALID' END inventory_quality,st.quality_reason inventory_reason "
+"FROM evidence c LEFT JOIN metric_sales m ON m.customer_code=c.customer_code AND m.sku_code=c.sku_code "
+"LEFT JOIN ll_selfop_inventory_snapshot st ON st.snapshot_run_id=#{p.inventoryRun} AND st.source_account=#{p.account} "
+"AND st.warehouse_code=#{p.warehouse} AND st.customer_code=c.customer_code AND st.sku_code=c.sku_code AND st.product_quality=0) ";
}
private static String filter(String kind){
if("HOT".equals(kind))return " WHERE window_quantity>0 ";
if("SLOW".equals(kind))return " WHERE window_quantity IS NOT NULL AND inventory_quality='VALID' AND available_quantity>0 "
@@ -119,18 +151,21 @@ public final class AnalyticsReadSql {
public static String count(java.util.Map<String,Object> parameters){
var p=(AnalyticsSqlRequest)parameters.get("p");
// Candidate totals do not scan daily rows for normal lists.
if(!"HOT".equals(p.kind())&&!"SLOW".equals(p.kind()))return typed("SELECT COUNT(*) FROM (SELECT customer_code,sku_code FROM ll_selfop_sku_profile "
return typed("SELECT COUNT(*) FROM (SELECT customer_code,sku_code FROM ll_selfop_sku_profile "
+"WHERE profile_run_id=#{p.salesRun} AND"+KEY_FILTER+" UNION SELECT customer_code,sku_code FROM ll_selfop_sku_profile "
+"WHERE profile_run_id=#{p.inventoryRun} AND"+KEY_FILTER+") candidate_keys",p);
return typed(cte(p.kind())+"SELECT LEAST(COUNT(*),100) FROM result"+filter(p.kind()),p);
}
public static String excluded(java.util.Map<String,Object> parameters){
public static String rankingStats(java.util.Map<String,Object> parameters){
var p=(AnalyticsSqlRequest)parameters.get("p");
String reason="HOT".equals(p.kind())?"CASE WHEN unit_valid=0 THEN 'UNIT_UNKNOWN_OR_CONFLICT' WHEN window_quantity IS NULL THEN 'SALES_WINDOW_UNAVAILABLE' ELSE 'ZERO_OUTBOUND' END":
"CASE WHEN inventory_quality<>'VALID' THEN CONCAT('INVENTORY_',inventory_quality) WHEN available_quantity<=0 THEN 'NO_AVAILABLE_INVENTORY' "
+"WHEN window_quantity IS NULL THEN 'SALES_WINDOW_UNAVAILABLE' ELSE 'OBSERVATION_INSUFFICIENT' END";
return typed(cte(p.kind())+"SELECT "+reason+" reason,COUNT(*) total FROM result WHERE NOT ("+filter(p.kind()).replace(" WHERE ","").strip()+") OR window_quantity IS NULL"
+("SLOW".equals(p.kind())?" OR evidence_at_utc IS NULL":"")+" GROUP BY reason",p);
String eligible=filter(p.kind()).replace(" WHERE ","").strip();
return typed(cte(p.kind())+", classified AS (SELECT CASE WHEN "+eligible+" THEN 1 ELSE 0 END eligible,"
+reason+" reason FROM result), counts AS (SELECT eligible,CASE WHEN eligible=1 THEN NULL ELSE reason END reason,COUNT(*) amount "
+"FROM classified GROUP BY eligible,CASE WHEN eligible=1 THEN NULL ELSE reason END) "
+"SELECT LEAST(SUM(CASE WHEN eligible=1 THEN amount ELSE 0 END) OVER (),100) total,reason,"
+"CASE WHEN eligible=0 THEN amount ELSE 0 END excluded_count FROM counts",p);
}
private static String typed(String sql,AnalyticsSqlRequest p){
String filters=" source_account=#{p.account} AND warehouse_code=#{p.warehouse} ";
@@ -23,6 +23,12 @@ public class SelfopEgressService {
checkAccount(account,budget,now);
SelfopValidationBudget validation=null;
if("VALIDATION".equals(context.executionPurpose())){
if(SelfopCollectionPolicy.warehouseDiscovery(root)){
if(operation!=Operation.WAREHOUSES)throw reject("VALIDATION_DIRECTORY_OPERATION_ONLY");
}else{
String selected=SelfopCollectionPolicy.validationWarehouse(root);
if((operation==Operation.PRODUCT_INVENTORY||operation==Operation.OUTBOUND_DETAIL)&&!Objects.equals(selected,warehouseCode))throw reject("EGRESS_WAREHOUSE_INVALID");
}
validation=validation(root);String limit=validation.limit(now,operation,pageSize,warehouseCode,detailKey);
if(limit!=null){stop(root,validation,limit,false,now);throw new EgressRejectedException(limit,true,null);}
if(operation==Operation.PRODUCT_INVENTORY&&(warehouseCode==null||!config.getWarehouses().containsKey(warehouseCode)))throw reject("EGRESS_WAREHOUSE_INVALID");
@@ -11,6 +11,7 @@ public final class SelfopValidationBudget {
public Set<String> detailKeys=new LinkedHashSet<>();
public Map<String,Integer> inventoryPages=new LinkedHashMap<>();
public static String freeze(LocalDateTime now){return freeze(now,10,20,40,300);}
public static String freezeWarehouseDiscovery(LocalDateTime now){return freeze(now,1,1,1,60);}
public static String freeze(LocalDateTime now,int pages,int details,int http,int seconds){
if(pages<1||details<1||http<1||seconds<1)throw new IllegalArgumentException("VALIDATION_LIMIT_INVALID");
var b=new SelfopValidationBudget();b.maxPages=Math.min(10,pages);b.maxDetailHttp=Math.min(20,details);b.maxDistinctDetails=Math.min(20,details);b.maxHttp=Math.min(40,http);b.startedAtUtc=now;b.deadlineAtUtc=now.plusSeconds(Math.min(300,seconds));return SelfopSupport.json(b);
@@ -34,7 +34,20 @@ public class SelfopCommandService {
private SelfopSyncTask requestChecked(Command c,String operator){
if(!config.isExecutionEnabled())throw new BizException("采集功能未启用,当前环境不允许执行采集或重试");
if(c==null)throw new BizException("请提供采集命令");
input(c.requestKey(),100,"请求标识");input(c.warehouseCode(),64,"仓库编码");
input(c.requestKey(),100,"请求标识");
if("WAREHOUSE_DISCOVERY".equals(c.action())) {
require("VALIDATION".equals(SelfopCollectionPolicy.configuredPurpose(config)),"SELFOP_VALIDATION_ENTRY_REQUIRED");
require(c.warehouseCode()==null&&c.taskNo()==null&&c.sourceFrom()==null&&c.sourceUntil()==null
&&c.customerCode()==null&&c.deliveryNo()==null&&c.reason()==null,"SELFOP_WAREHOUSE_DISCOVERY_PARAMETERS");
var account=taskService.initializeAccount();
Map<String,Object> query=Map.of("action","WAREHOUSE_DISCOVERY","sourceAccount",config.getSourceAccount());
var task=new SelfopSyncTask();task.setSourceAccount(config.getSourceAccount());task.setRequestedBy(operator);
task.setRequestKey("manual:"+c.requestKey());task.setExecutionPurpose("VALIDATION");task.setTaskType("VALIDATION");
task.setTaskRole("ACCOUNT_DISCOVERY");task.setDiscoveryScopeId(account.getId());
task.setQueryScopeJson(json(query));task.setScopeHash(hash(query));task.setRequestHash(hash(Arrays.asList(c,task.getScopeHash())));
return taskService.create(task);
}
input(c.warehouseCode(),64,"仓库编码");
require(config.getWarehouses().containsKey(c.warehouseCode()),"SELFOP_WAREHOUSE_UNCONFIGURED");
if(c.action()==null||!Set.of("RETRY","INVENTORY","BACKFILL","RECHECK","REBUILD","VALIDATION").contains(c.action()))
throw new BizException("不支持此采集操作");
@@ -68,7 +81,7 @@ public class SelfopCommandService {
task.setTaskRole("ACCOUNT_DISCOVERY");task.setDiscoveryScopeId(account.getId());
if("VALIDATION".equals(c.action())){
require(c.sourceFrom()!=null&&c.sourceUntil()!=null&&c.sourceFrom().isBefore(c.sourceUntil())&&!c.sourceUntil().isAfter(c.sourceFrom().plusDays(1)),"SELFOP_QUERY_WINDOW_INVALID");
task.setTaskType("VALIDATION");task.setQueryScopeJson(json(Map.of("startTime",c.sourceFrom().toString(),"endTime",c.sourceUntil().toString(),"warehouses",config.getWarehouses().keySet())));
task.setTaskType("VALIDATION");task.setQueryScopeJson(json(Map.of("action","VALIDATION","startTime",c.sourceFrom().toString(),"endTime",c.sourceUntil().toString(),"warehouses",List.of(c.warehouseCode()))));
}else if("RECHECK".equals(c.action())){identity(c.customerCode(),64);identity(c.deliveryNo(),128);
task.setQueryScopeJson(json(new SelfopDiscoveryService.Query(null,null,c.deliveryNo(),c.customerCode(),null)));
}else{
@@ -99,6 +112,7 @@ public class SelfopCommandService {
Map.entry("SELFOP_SLICE_MAX_ONE_DAY","单次采集时间范围不能超过一天"),
Map.entry("SELFOP_SOURCE_TIME_AMBIGUOUS","来源时间存在时区歧义,请重新选择时间范围"),
Map.entry("SELFOP_SCOPE_MISSING","采集作用域尚未准备就绪,请联系管理员"),
Map.entry("SELFOP_WAREHOUSE_DISCOVERY_PARAMETERS","仓库目录发现仅接受请求标识和操作类型,不接受仓库或采集范围"),
Map.entry("SELFOP_IDEMPOTENCY_CONFLICT","请求标识已用于其他内容,请刷新后重新提交"),
Map.entry("SELFOP_RETRY_LEASE_BUSY","任务正在执行,暂不能重试"),
Map.entry("SELFOP_TASK_ALREADY_SUCCEEDED","任务已成功完成,无需重试"),
@@ -32,7 +32,10 @@ public class SelfopSchedulePlanner {
public void tick(){
if("OFF".equals(config.getCollectionMode()))return;
if("BUSINESS".equals(config.getCollectionMode()))SelfopCollectionPolicy.requireExecution(config,"BUSINESS");
var configured=taskService.initialize();var now=utcNow();
List<SelfopSyncScope> configured;
if("VALIDATION".equals(config.getCollectionMode())){taskService.initializeAccount();configured=List.of();}
else configured=taskService.initialize();
var now=utcNow();
for(var scope:configured){
if(!"BUSINESS".equals(config.getCollectionMode()))break;
if("INVENTORY".equals(scope.getDataFamily()))plan(scope,now);
@@ -30,14 +30,7 @@ public class SelfopTaskService {
public List<SelfopSyncScope> initialize() {
identity(config.getSourceAccount(),64); require(!config.getWarehouses().isEmpty(),"SELFOP_WAREHOUSE_UNCONFIGURED");
var now=utcNow();
var a=accounts.selectOne(new QueryWrapper<SelfopOutboundDiscoveryScope>().eq("source_account",config.getSourceAccount()));
if(a==null){a=new SelfopOutboundDiscoveryScope();a.setId(IdWorker.getId());a.setSourceAccount(config.getSourceAccount());
a.setNextSequence(1L);a.setNextObservationSequence(1L);a.setFenceToken(0L);
a.setRequestBudget(config.getRequestBudget());a.setTimeBudgetSeconds(config.getTimeBudgetSeconds());
a.setMinRequestIntervalMs(config.getMinRequestIntervalMs());a.setCreatedAt(now);a.setUpdatedAt(now);
require(a.getRequestBudget()>0&&a.getTimeBudgetSeconds()>0&&a.getMinRequestIntervalMs()>0,"SELFOP_BUDGET_INVALID");
try{accounts.insert(a);}catch(DuplicateKeyException ignored){ }
}
initializeAccount();
for(var e:config.getWarehouses().entrySet()) for(String family:List.of("OUTBOUND","INVENTORY")) {
identity(e.getKey(),64);var p=e.getValue();ZoneId.of(p.getBusinessTimezone());
var existing=scopes.selectOne(new QueryWrapper<SelfopSyncScope>().eq("source_account",config.getSourceAccount()).eq("warehouse_code",e.getKey()).eq("data_family",family));
@@ -79,6 +72,23 @@ public class SelfopTaskService {
return scopes.selectList(new QueryWrapper<SelfopSyncScope>().eq("source_account",config.getSourceAccount()).orderByAsc("warehouse_code","data_family"));
}
@Transactional
public SelfopOutboundDiscoveryScope initializeAccount() {
identity(config.getSourceAccount(),64);
var now=utcNow();
var account=accounts.selectOne(new QueryWrapper<SelfopOutboundDiscoveryScope>().eq("source_account",config.getSourceAccount()));
if(account!=null)return account;
account=new SelfopOutboundDiscoveryScope();account.setId(IdWorker.getId());account.setSourceAccount(config.getSourceAccount());
account.setNextSequence(1L);account.setNextObservationSequence(1L);account.setFenceToken(0L);
account.setRequestBudget(config.getRequestBudget());account.setTimeBudgetSeconds(config.getTimeBudgetSeconds());
account.setMinRequestIntervalMs(config.getMinRequestIntervalMs());account.setCreatedAt(now);account.setUpdatedAt(now);
require(account.getRequestBudget()>0&&account.getTimeBudgetSeconds()>0&&account.getMinRequestIntervalMs()>0,"SELFOP_BUDGET_INVALID");
try{accounts.insert(account);}catch(DuplicateKeyException ignored){
account=accounts.selectOne(new QueryWrapper<SelfopOutboundDiscoveryScope>().eq("source_account",config.getSourceAccount()));
require(account!=null,"SELFOP_SCOPE_MISSING");
}
return account;
}
@Transactional
public SelfopSyncTask create(SelfopSyncTask task) {
identity(task.getRequestKey(),128); var existing=tasks.selectOne(new QueryWrapper<SelfopSyncTask>()
.eq("source_account",task.getSourceAccount()).eq("request_key",task.getRequestKey()));
@@ -89,7 +99,12 @@ public class SelfopTaskService {
task.setTaskNo("LXCOL-"+now.atZone(ZoneOffset.UTC).withZoneSameInstant(ZoneId.of("Asia/Shanghai")).toLocalDate().toString().replace("-","")+"-"+UUID.randomUUID().toString().replace("-","").substring(0,16).toUpperCase(Locale.ROOT));
if("VALIDATION".equals(task.getExecutionPurpose())){
require("VALIDATION".equals(task.getTaskType()),"SELFOP_VALIDATION_ENTRY_REQUIRED");
if(task.getValidationRootTaskId()==null){task.setValidationRootTaskId(task.getId());task.setValidationBudgetJson(com.baiyue.lastleg.selfoperated.analytics.request.SelfopValidationBudget.freeze(now));}
if(task.getValidationRootTaskId()==null){task.setValidationRootTaskId(task.getId());
boolean directoryOnly=warehouseDiscovery(task);
if(!directoryOnly)validationWarehouse(task);
task.setValidationBudgetJson(directoryOnly
?com.baiyue.lastleg.selfoperated.analytics.request.SelfopValidationBudget.freezeWarehouseDiscovery(now)
:com.baiyue.lastleg.selfoperated.analytics.request.SelfopValidationBudget.freeze(now));}
task.setApplyState("NOT_APPLICABLE");
}
task.setStatus(task.getStatus()==null?"READY":task.getStatus());
@@ -44,19 +44,30 @@ public class SelfopValidationRunner {
try {
var state=task.getRoutingManifestJson()==null?new Sample():read(task.getRoutingManifestJson(),Sample.class);
Map<?,?> query=read(task.getQueryScopeJson(),Map.class);
var from=LocalDateTime.parse(Objects.toString(query.get("startTime"),""));
var until=LocalDateTime.parse(Objects.toString(query.get("endTime"),""));
require(until.isAfter(from)&&Duration.between(from,until).compareTo(Duration.ofDays(1))<=0,"SELFOP_VALIDATION_WINDOW_INVALID");
boolean directoryOnly=SelfopCollectionPolicy.warehouseDiscovery(task);
String selectedWarehouse=directoryOnly?null:SelfopCollectionPolicy.validationWarehouse(task);
if(!directoryOnly)require(config.getWarehouses().containsKey(selectedWarehouse),"SELFOP_WAREHOUSE_UNCONFIGURED");
var from=directoryOnly?null:LocalDateTime.parse(Objects.toString(query.get("startTime"),""));
var until=directoryOnly?null:LocalDateTime.parse(Objects.toString(query.get("endTime"),""));
if(!directoryOnly)require(until.isAfter(from)&&Duration.between(from,until).compareTo(Duration.ofDays(1))<=0,"SELFOP_VALIDATION_WINDOW_INVALID");
if(!state.warehousesRead) {
leases.renew(lease); var page=request(task,run,lease,()->client.warehouses(context,1,10)); validatePage(page,1);
tx.executeWithoutResult(s->{check(task,run,lease);
Set<String> warehouseCodes=new HashSet<>();
for(var warehouse:page.records()) {
identity(warehouse.warehouseCode(),64);
require(warehouseCodes.add(warehouse.warehouseCode()),"SELFOP_SAMPLE_WAREHOUSE_DUPLICATE");
}
run.setObservedDimensionsJson(json(Map.of("validationWarehouseSamples",page.records(),"truncated",page.pages()>1)));
state.warehousesRead=true;state.pages++;save(task,run,state,"DIRECTORY","PAGE",state.pages,null);
});
}
if(directoryOnly){
require(run.getObservedDimensionsJson()!=null,"SELFOP_WAREHOUSE_DIRECTORY_EVIDENCE_MISSING");
Map<?,?> dimensions=read(run.getObservedDimensionsJson(),Map.class);
finish(task,run,lease,Boolean.TRUE.equals(dimensions.get("truncated"))
?"VALIDATION_WAREHOUSE_DIRECTORY_TRUNCATED":"VALIDATION_WAREHOUSE_DIRECTORY_COMPLETE",false);return;
}
while(!state.directoryRead&&state.outboundPage<=2) {
int current=state.outboundPage;leases.renew(lease);
var page=request(task,run,lease,()->client.discover(context,from,until,current,10));validatePage(page,current);
@@ -64,6 +75,7 @@ public class SelfopValidationRunner {
for(var summary:page.records()) {
require(Objects.equals(task.getSourceAccount(),summary.sourceAccount()),"SELFOP_SAMPLE_ACCOUNT_MISMATCH");
identity(summary.customerCode(),64);identity(summary.warehouseCode(),64);identity(summary.deliveryNo(),128);
if(!selectedWarehouse.equals(summary.warehouseCode()))continue;
String key=hash(List.of(summary.customerCode(),summary.deliveryNo()));
require(state.orders.stream().noneMatch(o->key.equals(hash(List.of(o.customerCode(),o.deliveryNo())))),"SELFOP_SAMPLE_DIRECTORY_DUPLICATE");
state.orders.add(summary);
@@ -75,6 +87,7 @@ public class SelfopValidationRunner {
}
while(state.detailIndex<state.orders.size()) {
var summary=state.orders.get(state.detailIndex);leases.renew(lease);
require(selectedWarehouse.equals(summary.warehouseCode()),"SELFOP_SAMPLE_WAREHOUSE_MISMATCH");
var detail=request(task,run,lease,()->client.detail(context,summary.warehouseCode(),summary.customerCode(),summary.deliveryNo()));
validateOrder(task,summary,detail);
tx.executeWithoutResult(s->{check(task,run,lease);
@@ -85,8 +98,8 @@ public class SelfopValidationRunner {
state.detailIndex++;save(task,run,state,"DETAIL","ORDER",state.detailIndex,(long)state.orders.size());
});
}
// Each configured warehouse is sampled independently, never inferred from directory absence.
for(String warehouse:config.getWarehouses().keySet()) {
// Only the command's frozen warehouse is sampled; other configured warehouses are independent roots.
for(String warehouse:List.of(selectedWarehouse)) {
int current=state.inventoryNextPage.getOrDefault(warehouse,1);
while(current<=2&&!state.inventoryFinished.contains(warehouse)) {
leases.renew(lease);int pageNo=current;