diff --git a/AGENTS.md b/AGENTS.md index 771987cc4..ee0ad0f9e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ | `order/` | 订单主数据、租户运单数、持久化幂等、状态编排、快照与 Outbox | `PlatformFirstLegOrderController.java`、`OrderApplicationService.java`、`TenantOrderCountResponse.java`、`README-ORDER.md` | | `firstleg/` | 履约任务分页/详情、操作分配与分配历史 | `PlatformFulfillmentAssignmentController.java`、`FulfillmentTaskService.java`、`src/main/resources/db/phase1-firstleg-task-list-index.sql` | | `warehouse/` | 仓库、地址、服务能力、往来单位多类型、快递公司与 XLSX 导入 | `controller/`、`PlatformPartnerQueryService.java`、`PartnerImportService.java`、`ExpressCompanyImportService.java` | -| `file/` | 企业资质文件上传、替换、预览与鉴权 | `QualificationFileController.java`、`PlatformQualificationFileController.java`、`LocalQualificationStorage.java`；元数据仍归 IAM，File 自身表结构尚未接入 | +| `file/` | 企业资质文件与公共导出/下载中心 | 资质：`QualificationFileController.java`、`LocalQualificationStorage.java`；导出：`PlatformExportTaskController.java`、`ExportStateService.java`、`mq/`、`GenericXlsxExportRunner.java`、`ExportReconciler.java` | | `print/` | 打印中心边界 | 当前为应用、健康检查和 `print/src/main/resources/db/print.sql` 骨架 | | `audit-service/` | 审计事件接收与持久化 | `AuditIngestController.java`、`AuditIngestService.java`；当前接收接口未实现内部令牌保护 | | `integration-service/` | 重量 OCR、百度 OCR、Amazon 等第三方适配 | `controller/WeightOcrController.java`、`service/ocr/`、`provider/ocr/`、`IntegrationProperties.java` | @@ -56,6 +56,7 @@ - 本轮测试环境数据库增量统一从 `deploy/sql/2026-07-31一期回归测试环境增量迁移.sql` 执行，执行边界与核验方式见 `deploy/sql/README.md`。 - CI 总控入口是 `Jenkinsfile`，负责按变更或勾选项并行调度 11 个独立服务 Job；单服务构建、镜像与部署逻辑在 `Jenkinsfile.service`。Job 名必须以模块名结尾，公共模块或 `deploy/` 变化会触发全部服务。 - 从本目录运行全量校验：`mvn -DskipTests validate`、`mvn test`。单模块修改优先运行 `mvn -pl -am test`。 +- 公共导出中心测试环境真实任务验收使用 `scripts/export-center-test-env-acceptance.ps1`；脚本只调用 Gateway，不负责部署或停止 Broker/File/IAM，故障注入由有权限的运维配合。 - `reports/`、`target/`、日志、上传文件和本地环境文件不属于源码，检索与提交时排除。 ## 通用修改约束 diff --git a/auth/src/main/java/com/baiyue/auth/client/IamOnboardingClient.java b/auth/src/main/java/com/baiyue/auth/client/IamOnboardingClient.java index 704653bb8..b78cc2138 100644 --- a/auth/src/main/java/com/baiyue/auth/client/IamOnboardingClient.java +++ b/auth/src/main/java/com/baiyue/auth/client/IamOnboardingClient.java @@ -1,6 +1,8 @@ package com.baiyue.auth.client; import com.baiyue.common.core.api.ApiResponse; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.cloud.openfeign.FeignClient; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -14,6 +16,11 @@ public interface IamOnboardingClient { @RequestHeader("X-Internal-Service-Token") String internalToken, @RequestBody OnboardingRequest request); + @DeleteMapping("/merchant-registrations/{authUserId}/compensation") + ApiResponse compensate( + @RequestHeader("X-Internal-Service-Token") String internalToken, + @PathVariable("authUserId") Long authUserId); + record OnboardingRequest(Long authUserId, String displayName, String companyName, String legalPerson, String legalPhone, String contactEmail, String countryCode, String address) { } diff --git a/auth/src/main/java/com/baiyue/auth/controller/AuthController.java b/auth/src/main/java/com/baiyue/auth/controller/AuthController.java index ecee6b279..72c8edf2b 100644 --- a/auth/src/main/java/com/baiyue/auth/controller/AuthController.java +++ b/auth/src/main/java/com/baiyue/auth/controller/AuthController.java @@ -47,7 +47,7 @@ public class AuthController { @SecurityRequirements @PostMapping("/register") @Operation(summary = "Merchant registration / 客户注册", - description = "Create a login account and an enterprise profile awaiting one business-license upload.") + description = "Create a login account and an enterprise profile awaiting four required qualification files.") public ApiResponse register(@Valid @RequestBody MerchantRegisterRequest request) { return ApiResponse.success(registrationService.register(request)); } diff --git a/auth/src/main/java/com/baiyue/auth/service/MerchantRegistrationService.java b/auth/src/main/java/com/baiyue/auth/service/MerchantRegistrationService.java index 64287017c..09dd998cd 100644 --- a/auth/src/main/java/com/baiyue/auth/service/MerchantRegistrationService.java +++ b/auth/src/main/java/com/baiyue/auth/service/MerchantRegistrationService.java @@ -11,13 +11,21 @@ import com.baiyue.common.audit.BusinessAudit; import com.baomidou.mybatisplus.core.toolkit.IdWorker; import feign.FeignException; import jakarta.annotation.Resource; +import org.springframework.dao.DuplicateKeyException; import org.springframework.beans.factory.annotation.Value; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import org.springframework.transaction.annotation.Transactional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicBoolean; @Service public class MerchantRegistrationService { + private static final Logger LOGGER = LoggerFactory.getLogger(MerchantRegistrationService.class); @Resource private AuthAccountRepository accountRepository; @Resource private PasswordEncoder passwordEncoder; @Resource private IamOnboardingClient onboardingClient; @@ -31,6 +39,9 @@ public class MerchantRegistrationService { throw new BizException(ResultCode.CONFLICT, "Username already exists"); } long accountId = IdWorker.getId(); + String passwordHash = passwordEncoder.encode(request.password()); + AtomicBoolean compensationCompleted = new AtomicBoolean(false); + registerRollbackCompensation(accountId, compensationCompleted); IamOnboardingClient.OnboardingResponse onboarding; try { var response = onboardingClient.register(internalToken, @@ -51,17 +62,26 @@ public class MerchantRegistrationService { } onboarding = response.data(); } catch (FeignException exception) { - throw new BizException(ResultCode.SERVICE_UNAVAILABLE, "IAM registration service is unavailable"); + BizException failure = new BizException(ResultCode.SERVICE_UNAVAILABLE, + "IAM registration service is unavailable"); + failure.addSuppressed(exception); + compensateOnboarding(accountId, failure, compensationCompleted); + throw failure; } catch (BizException exception) { + compensateOnboarding(accountId, exception, compensationCompleted); throw exception; } catch (RuntimeException exception) { - throw new BizException(ResultCode.SERVICE_UNAVAILABLE, "IAM registration service is unavailable"); + BizException failure = new BizException(ResultCode.SERVICE_UNAVAILABLE, + "IAM registration service is unavailable"); + failure.addSuppressed(exception); + compensateOnboarding(accountId, failure, compensationCompleted); + throw failure; } AuthAccount account = new AuthAccount(); account.setId(accountId); account.setUsername(request.username()); - account.setPasswordHash(passwordEncoder.encode(request.password())); + account.setPasswordHash(passwordHash); account.setDisplayName(request.displayName()); account.setAccountStatus("ACTIVE"); account.setFailedLoginCount(0); @@ -71,8 +91,61 @@ public class MerchantRegistrationService { account.setDefaultMembershipId(onboarding.membershipId()); account.setDefaultAppCode("MERCHANT"); account.setDeleted(0); - accountRepository.insert(account); + try { + accountRepository.insert(account); + } catch (DuplicateKeyException exception) { + BizException failure = new BizException(ResultCode.CONFLICT, "Username already exists"); + failure.addSuppressed(exception); + compensateOnboarding(accountId, failure, compensationCompleted); + throw failure; + } catch (RuntimeException exception) { + compensateOnboarding(accountId, exception, compensationCompleted); + throw exception; + } return new MerchantRegisterResponse(accountId, onboarding.tenantId(), onboarding.membershipId(), - onboarding.tenantStatus(), "LOGIN_AND_UPLOAD_BUSINESS_LICENSE"); + onboarding.tenantStatus(), "LOGIN_AND_COMPLETE_QUALIFICATIONS"); + } + + private void registerRollbackCompensation(long accountId, AtomicBoolean compensationCompleted) { + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + return; + } + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + if (status == STATUS_COMMITTED || compensationCompleted.get()) { + return; + } + IllegalStateException rollbackFailure = new IllegalStateException( + "Auth transaction rolled back after IAM onboarding"); + if (!compensateOnboarding(accountId, rollbackFailure, compensationCompleted)) { + LOGGER.error("IAM registration compensation failed after Auth rollback, authUserId={}", + accountId, rollbackFailure); + } + } + }); + } + + private boolean compensateOnboarding(long accountId, RuntimeException registrationFailure, + AtomicBoolean compensationCompleted) { + if (compensationCompleted.get()) { + return true; + } + try { + var response = onboardingClient.compensate(internalToken, accountId); + if (response != null && response.successful()) { + compensationCompleted.set(true); + return true; + } + { + String code = response == null ? "empty-response" : response.code(); + String message = response == null ? "IAM compensation returned no response" : response.message(); + registrationFailure.addSuppressed(new IllegalStateException( + "IAM registration compensation failed: code=" + code + ", message=" + message)); + } + } catch (RuntimeException compensationFailure) { + registrationFailure.addSuppressed(compensationFailure); + } + return false; } } diff --git a/auth/src/test/java/com/baiyue/auth/service/MerchantRegistrationServiceTest.java b/auth/src/test/java/com/baiyue/auth/service/MerchantRegistrationServiceTest.java new file mode 100644 index 000000000..61ce317ec --- /dev/null +++ b/auth/src/test/java/com/baiyue/auth/service/MerchantRegistrationServiceTest.java @@ -0,0 +1,116 @@ +package com.baiyue.auth.service; + +import com.baiyue.auth.client.IamOnboardingClient; +import com.baiyue.auth.dto.MerchantRegisterRequest; +import com.baiyue.auth.entity.AuthAccount; +import com.baiyue.auth.repository.AuthAccountRepository; +import com.baiyue.common.core.api.ApiResponse; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class MerchantRegistrationServiceTest { + @Test + void registeredAuthAccountUsesTheOwnerIdentityReturnedByIamOnboarding() { + AuthAccountRepository accounts = mock(AuthAccountRepository.class); + PasswordEncoder encoder = mock(PasswordEncoder.class); + IamOnboardingClient onboarding = mock(IamOnboardingClient.class); + MerchantRegistrationService service = new MerchantRegistrationService(); + ReflectionTestUtils.setField(service, "accountRepository", accounts); + ReflectionTestUtils.setField(service, "passwordEncoder", encoder); + ReflectionTestUtils.setField(service, "onboardingClient", onboarding); + ReflectionTestUtils.setField(service, "internalToken", "internal-token"); + when(encoder.encode("Password123")).thenReturn("encoded-password"); + when(onboarding.register(eq("internal-token"), any())).thenReturn(ApiResponse.success( + new IamOnboardingClient.OnboardingResponse(10001L, 30001L, "PROFILE_REQUIRED"))); + + var result = service.register(new MerchantRegisterRequest( + "registered_owner", "Password123", "注册人", "测试企业", "张三", + "13800138000", "owner@example.com", "CN", "深圳市宝安区")); + + ArgumentCaptor onboardingRequest = + ArgumentCaptor.forClass(IamOnboardingClient.OnboardingRequest.class); + ArgumentCaptor storedAccount = ArgumentCaptor.forClass(AuthAccount.class); + verify(onboarding).register(eq("internal-token"), onboardingRequest.capture()); + verify(accounts).insert(storedAccount.capture()); + + assertEquals(onboardingRequest.getValue().authUserId(), storedAccount.getValue().getId()); + assertEquals(result.accountId(), storedAccount.getValue().getId()); + assertEquals(10001L, storedAccount.getValue().getDefaultTenantId()); + assertEquals(10001L, storedAccount.getValue().getOperatorTenantId()); + assertEquals(30001L, storedAccount.getValue().getDefaultMembershipId()); + assertEquals("MERCHANT", storedAccount.getValue().getDefaultAppCode()); + assertEquals("ACTIVE", storedAccount.getValue().getAccountStatus()); + assertEquals("encoded-password", storedAccount.getValue().getPasswordHash()); + assertEquals("LOGIN_AND_COMPLETE_QUALIFICATIONS", result.nextStep()); + } + + @Test + void failedAuthInsertCompensatesTheIamRegistration() { + AuthAccountRepository accounts = mock(AuthAccountRepository.class); + PasswordEncoder encoder = mock(PasswordEncoder.class); + IamOnboardingClient onboarding = mock(IamOnboardingClient.class); + MerchantRegistrationService service = new MerchantRegistrationService(); + ReflectionTestUtils.setField(service, "accountRepository", accounts); + ReflectionTestUtils.setField(service, "passwordEncoder", encoder); + ReflectionTestUtils.setField(service, "onboardingClient", onboarding); + ReflectionTestUtils.setField(service, "internalToken", "internal-token"); + when(encoder.encode("Password123")).thenReturn("encoded-password"); + when(onboarding.register(eq("internal-token"), any())).thenReturn(ApiResponse.success( + new IamOnboardingClient.OnboardingResponse(10001L, 30001L, "PROFILE_REQUIRED"))); + when(onboarding.compensate(eq("internal-token"), any())).thenReturn(ApiResponse.success(null)); + doThrow(new IllegalStateException("auth database unavailable")).when(accounts).insert(any()); + + assertThrows(IllegalStateException.class, () -> service.register(new MerchantRegisterRequest( + "failed_owner", "Password123", "注册人", "测试企业", "张三", + "13800138000", "owner@example.com", "CN", "深圳市宝安区"))); + + ArgumentCaptor onboardingRequest = + ArgumentCaptor.forClass(IamOnboardingClient.OnboardingRequest.class); + ArgumentCaptor compensationUserId = ArgumentCaptor.forClass(Long.class); + verify(onboarding).register(eq("internal-token"), onboardingRequest.capture()); + verify(onboarding).compensate(eq("internal-token"), compensationUserId.capture()); + assertEquals(onboardingRequest.getValue().authUserId(), compensationUserId.getValue()); + } + + @Test + void rolledBackAuthTransactionCompensatesAfterTheLocalTransactionCompletes() { + AuthAccountRepository accounts = mock(AuthAccountRepository.class); + PasswordEncoder encoder = mock(PasswordEncoder.class); + IamOnboardingClient onboarding = mock(IamOnboardingClient.class); + MerchantRegistrationService service = new MerchantRegistrationService(); + ReflectionTestUtils.setField(service, "accountRepository", accounts); + ReflectionTestUtils.setField(service, "passwordEncoder", encoder); + ReflectionTestUtils.setField(service, "onboardingClient", onboarding); + ReflectionTestUtils.setField(service, "internalToken", "internal-token"); + when(encoder.encode("Password123")).thenReturn("encoded-password"); + when(onboarding.register(eq("internal-token"), any())).thenReturn(ApiResponse.success( + new IamOnboardingClient.OnboardingResponse(10001L, 30001L, "PROFILE_REQUIRED"))); + when(onboarding.compensate(eq("internal-token"), any())).thenReturn(ApiResponse.success(null)); + + TransactionSynchronizationManager.initSynchronization(); + try { + service.register(new MerchantRegisterRequest( + "rollback_owner", "Password123", "注册人", "测试企业", "张三", + "13800138000", "owner@example.com", "CN", "深圳市宝安区")); + var synchronizations = TransactionSynchronizationManager.getSynchronizations(); + assertEquals(1, synchronizations.size()); + synchronizations.get(0).afterCompletion(TransactionSynchronization.STATUS_ROLLED_BACK); + verify(onboarding).compensate(eq("internal-token"), any()); + } finally { + TransactionSynchronizationManager.clearSynchronization(); + } + } +} diff --git a/common-core/src/main/java/com/baiyue/common/core/export/ExportContracts.java b/common-core/src/main/java/com/baiyue/common/core/export/ExportContracts.java new file mode 100644 index 000000000..1b6a6cbcc --- /dev/null +++ b/common-core/src/main/java/com/baiyue/common/core/export/ExportContracts.java @@ -0,0 +1,28 @@ +package com.baiyue.common.core.export; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; + +/** Versioned contracts shared by File Service and business export providers. */ +public final class ExportContracts { + private ExportContracts() { } + + public record ExportCommand( + int schemaVersion, String eventId, String taskId, String exportType, + int dispatchVersion, OffsetDateTime occurredAt, String traceId) { } + + public record ExportBatchRequest( + String taskId, String tenantId, String appCode, String requestedBy, + int dispatchVersion, Map parameters, String cursor, + String snapshotToken, int limit) { } + + public record ExportBatchResponse( + int schemaVersion, String schemaHash, String snapshotToken, + List columns, List> rows, + String nextCursor, boolean hasMore, Long totalRows) { } + + public record ExportColumn(String key, String title, ExportColumnType type, Integer width) { } + + public enum ExportColumnType { STRING, INTEGER, DECIMAL, BOOLEAN, DATE, DATETIME } +} diff --git a/common-web/pom.xml b/common-web/pom.xml index 4fdc1b567..fa892aee9 100644 --- a/common-web/pom.xml +++ b/common-web/pom.xml @@ -35,6 +35,10 @@ io.micrometer micrometer-core + + io.micrometer + micrometer-registry-prometheus + org.springframework.boot spring-boot-starter-test diff --git a/config/nacos/cn/file-service.yml b/config/nacos/cn/file-service.yml index 429ab4a59..70b0fd2cc 100644 --- a/config/nacos/cn/file-service.yml +++ b/config/nacos/cn/file-service.yml @@ -7,3 +7,16 @@ spring: # 当前代码只实现本地/共享文件目录，非本地环境必须显式注入并挂载可写目录。 baiyue.file.storage-root: ${FILE_STORAGE_ROOT} +baiyue: + file: + export: + definitions: + CUSTOMER_LIST: + provider-service: iam-service + provider-path: /internal/exports/v1/CUSTOMER_LIST/batches + required-permission: platform:customer:export + app-scope: PLATFORM + enabled: true + max-rows: 200000 + max-attempts: 3 + batch-size: 500 diff --git a/config/nacos/local/file-service.yml b/config/nacos/local/file-service.yml index bfc359e34..a4ba7e706 100644 --- a/config/nacos/local/file-service.yml +++ b/config/nacos/local/file-service.yml @@ -4,3 +4,16 @@ spring.datasource: username: ${FILE_DB_USERNAME:root} password: ${FILE_DB_PASSWORD:123456} baiyue.file.storage-root: ${FILE_STORAGE_ROOT:D:/baiyue-firstleg-runtime/uploads} +baiyue: + file: + export: + definitions: + CUSTOMER_LIST: + provider-service: iam-service + provider-path: /internal/exports/v1/CUSTOMER_LIST/batches + required-permission: platform:customer:export + app-scope: PLATFORM + enabled: true + max-rows: 200000 + max-attempts: 3 + batch-size: 500 diff --git a/deploy/docker-compose.rocketmq.yml b/deploy/docker-compose.rocketmq.yml new file mode 100644 index 000000000..e458e36c4 --- /dev/null +++ b/deploy/docker-compose.rocketmq.yml @@ -0,0 +1,46 @@ +name: firstleg-${DEPLOY_ENV:-test}-rocketmq + +services: + rocketmq-namesrv: + image: apache/rocketmq:5.3.1 + restart: unless-stopped + command: sh mqnamesrv + networks: [infra-network] + + rocketmq-broker: + image: apache/rocketmq:5.3.1 + restart: unless-stopped + depends_on: [rocketmq-namesrv] + environment: + ROCKETMQ_BROKER_IP: ${ROCKETMQ_BROKER_IP:?ROCKETMQ_BROKER_IP is required} + volumes: + - ./rocketmq:/config:ro + - ${ROCKETMQ_DATA_DIR:-/data/firstleg/rocketmq/store}:/home/rocketmq/store + command: sh /config/render-broker-config.sh + networks: [infra-network] + + rocketmq-topic-init: + image: apache/rocketmq:5.3.1 + depends_on: [rocketmq-broker] + environment: + # 必须与 File Service 完全一致；允许生产环境显式传入空字符串。 + MQ_TOPIC_PREFIX: ${MQ_TOPIC_PREFIX?MQ_TOPIC_PREFIX must be set; use test- for test and an empty value for production} + command: >- + sh -eu -c 'until sh mqadmin clusterList -n rocketmq-namesrv:9876 2>/dev/null + | grep -q "BaiyueCluster"; do sleep 2; done; + sh mqadmin updateTopic -n rocketmq-namesrv:9876 -c BaiyueCluster + -t "$${MQ_TOPIC_PREFIX}baiyue-file-export-command-v1" -r 4 -w 4; + sh mqadmin updateSubGroup -n rocketmq-namesrv:9876 -c BaiyueCluster + -g "$${MQ_TOPIC_PREFIX}baiyue-file-export-worker-v1"; + sh mqadmin updateSubGroup -n rocketmq-namesrv:9876 -c BaiyueCluster + -g "$${MQ_TOPIC_PREFIX}baiyue-file-export-dlq-handler-v1"; + sh mqadmin updateTopic -n rocketmq-namesrv:9876 -c BaiyueCluster + -t "%DLQ%$${MQ_TOPIC_PREFIX}baiyue-file-export-worker-v1" -r 1 -w 1 -p 6' + # Broker 注册较慢或任一初始化命令失败时整组重试；所有命令均幂等。 + restart: on-failure + networks: [infra-network] + +networks: + infra-network: + external: true + name: ${INFRA_NETWORK:-firstleg-test-infra-network} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 7c8d5b92a..e7a116f12 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -124,6 +124,14 @@ services: FILE_DB_USERNAME: ${FILE_DB_USERNAME} FILE_DB_PASSWORD: ${FILE_DB_PASSWORD} FILE_STORAGE_ROOT: /data/files + JAVA_TOOL_OPTIONS: "-Xms1g -Xmx2g -XX:+ExitOnOutOfMemoryError -Dfile.encoding=UTF-8" + ROCKETMQ_NAME_SERVER: ${ROCKETMQ_NAME_SERVER} + # 必须显式设置：测试填 test-，生产填空字符串；使用 ? 而不是 :?，允许生产空值。 + MQ_TOPIC_PREFIX: ${MQ_TOPIC_PREFIX?MQ_TOPIC_PREFIX must be set; use test- for test and an empty value for production} + FILE_EXPORT_CONSUMER_ENABLED: ${FILE_EXPORT_CONSUMER_ENABLED:-false} + FILE_EXPORT_CONSUMER_CONCURRENCY: ${FILE_EXPORT_CONSUMER_CONCURRENCY:-4} + FILE_EXPORT_PUBLISHER_ENABLED: ${FILE_EXPORT_PUBLISHER_ENABLED:-false} + FILE_EXPORT_RECONCILER_ENABLED: ${FILE_EXPORT_RECONCILER_ENABLED:-false} print: <<: *common diff --git a/deploy/env/.env.infra-test.example b/deploy/env/.env.infra-test.example index f7899d10c..f0a8cb14a 100644 --- a/deploy/env/.env.infra-test.example +++ b/deploy/env/.env.infra-test.example @@ -5,3 +5,8 @@ MYSQL_DATABASE=baiyue MYSQL_USER=baiyue MYSQL_PASSWORD=CHANGE_ME_TEST_MYSQL_PASSWORD REDIS_PASSWORD=CHANGE_ME_TEST_REDIS_PASSWORD +# RocketMQ 与 File Service 位于同一 Docker 网络，Broker 向客户端公布服务名。 +ROCKETMQ_BROKER_IP=rocketmq-broker +# 测试环境必须显式使用 test-；生产环境在对应凭据文件中显式设置为空字符串。 +MQ_TOPIC_PREFIX=test- +ROCKETMQ_DATA_DIR=/data/firstleg/rocketmq/store diff --git a/deploy/rocketmq/broker.conf.template b/deploy/rocketmq/broker.conf.template new file mode 100644 index 000000000..abd7ae08b --- /dev/null +++ b/deploy/rocketmq/broker.conf.template @@ -0,0 +1,11 @@ +brokerClusterName=BaiyueCluster +brokerName=baiyue-file-export-broker +brokerId=0 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH +deleteWhen=04 +fileReservedTime=72 +autoCreateTopicEnable=false +autoCreateSubscriptionGroup=false +listenPort=10911 +brokerIP1=__BROKER_IP__ diff --git a/deploy/rocketmq/render-broker-config.sh b/deploy/rocketmq/render-broker-config.sh new file mode 100644 index 000000000..ee53cb3b7 --- /dev/null +++ b/deploy/rocketmq/render-broker-config.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +case "${ROCKETMQ_BROKER_IP:-}" in + ""|localhost|127.0.0.1|__BROKER_IP__|*'${'*) + echo "ROCKETMQ_BROKER_IP must be an explicit client-reachable address" >&2 + exit 1 + ;; +esac +sed "s/__BROKER_IP__/${ROCKETMQ_BROKER_IP}/g" /config/broker.conf.template > /tmp/broker.conf +if grep -Eq '__BROKER_IP__|\$\{' /tmp/broker.conf; then + echo "broker.conf still contains an unresolved placeholder" >&2 + grep -nE '__BROKER_IP__|\$\{' /tmp/broker.conf >&2 + exit 1 +fi +exec sh mqbroker -n rocketmq-namesrv:9876 -c /tmp/broker.conf diff --git a/deploy/sql/2026-07-31一期回归测试环境增量迁移.sql b/deploy/sql/2026-07-31一期回归测试环境增量迁移.sql index be5de8e48..f33a89241 100644 --- a/deploy/sql/2026-07-31一期回归测试环境增量迁移.sql +++ b/deploy/sql/2026-07-31一期回归测试环境增量迁移.sql @@ -32,8 +32,15 @@ CALL regression_assert_table('iam_permission'); CALL regression_assert_table('iam_role'); CALL regression_assert_table('iam_role_permission'); CALL regression_assert_table('iam_tenant'); +CALL regression_assert_table('iam_tenant_review'); +CALL regression_assert_table('iam_tenant_qualification'); +CALL regression_assert_table('iam_membership'); DROP PROCEDURE regression_assert_table; +ALTER TABLE iam_tenant + MODIFY COLUMN status VARCHAR(20) NOT NULL DEFAULT 'PROFILE_REQUIRED' + COMMENT '租户状态：PROFILE_REQUIRED、PENDING、ACTIVE、REJECTED、DISABLED'; + SET @iam_permission_built_in_sql = IF( EXISTS( SELECT 1 FROM information_schema.columns @@ -48,6 +55,50 @@ PREPARE iam_permission_built_in_stmt FROM @iam_permission_built_in_sql; EXECUTE iam_permission_built_in_stmt; DEALLOCATE PREPARE iam_permission_built_in_stmt; +-- 审核通过自动开户依赖全局企业管理员模板。通过自然键幂等补齐，避免存量环境 +-- 已存在同编码但 ID 不同的角色时，把权限错误授予固定 ID。 +INSERT INTO iam_role ( + id, tenant_id, app_code, role_code, role_name, role_type, status, + description, created_by, updated_by, deleted, version +) VALUES ( + 4000100, 0, 'MERCHANT', 'MERCHANT_OWNER', '企业管理员', 'SYSTEM', 'ACTIVE', + '企业审核通过后自动创建的管理员角色模板', 1000001, 1000001, 0, 0 +) +ON DUPLICATE KEY UPDATE + role_name = VALUES(role_name), role_type = 'SYSTEM', status = 'ACTIVE', + description = VALUES(description), deleted = 0; + +-- 从一期商户权限基线复制全局模板；不创建演示租户或业务数据。 +INSERT INTO iam_permission ( + id, tenant_id, app_code, permission_type, parent_id, permission_code, + permission_name, route_path, http_method, sort_no, status, built_in, deleted +) +SELECT permission.id + 100000, 0, permission.app_code, permission.permission_type, + CASE WHEN permission.parent_id = 0 THEN 0 ELSE permission.parent_id + 100000 END, + permission.permission_code, permission.permission_name, permission.route_path, + permission.http_method, permission.sort_no, 'ACTIVE', 1, 0 +FROM iam_permission permission +WHERE permission.tenant_id = 10001 AND permission.app_code = 'MERCHANT' + AND permission.permission_type IN ('MENU', 'BUTTON') + AND permission.deleted = 0 +ON DUPLICATE KEY UPDATE + permission_type = VALUES(permission_type), parent_id = VALUES(parent_id), + permission_name = VALUES(permission_name), route_path = VALUES(route_path), + http_method = VALUES(http_method), sort_no = VALUES(sort_no), + status = 'ACTIVE', built_in = 1, deleted = 0; + +INSERT INTO iam_role_permission (role_id, permission_id, tenant_id) +SELECT owner_role.id, permission.id, 0 +FROM iam_role owner_role +JOIN iam_permission permission + ON permission.tenant_id = 0 AND permission.app_code = 'MERCHANT' + AND permission.permission_type IN ('MENU', 'BUTTON') + AND permission.status = 'ACTIVE' AND permission.deleted = 0 +WHERE owner_role.tenant_id = 0 AND owner_role.app_code = 'MERCHANT' + AND owner_role.role_code = 'MERCHANT_OWNER' + AND owner_role.status = 'ACTIVE' AND owner_role.deleted = 0 +ON DUPLICATE KEY UPDATE tenant_id = VALUES(tenant_id); + -- 一期客户档案字段；与 iam/db/phase1-customer-profile.sql 保持一致。 DROP PROCEDURE IF EXISTS regression_add_iam_column; DELIMITER $$ @@ -73,6 +124,7 @@ END$$ DELIMITER ; CALL regression_add_iam_column('contact_wechat', 'VARCHAR(64) NULL COMMENT ''联系微信'' AFTER contact_email'); +CALL regression_add_iam_column('legal_representative', 'VARCHAR(64) NULL COMMENT ''法定代表人'' AFTER legal_name'); CALL regression_add_iam_column('sales_owner_name', 'VARCHAR(64) NULL COMMENT ''销售负责人姓名'' AFTER contact_wechat'); CALL regression_add_iam_column('payment_term_days', 'SMALLINT UNSIGNED NULL COMMENT ''账期天数'' AFTER sales_owner_name'); CALL regression_add_iam_column('enterprise_type', 'VARCHAR(64) NULL COMMENT ''企业类型'' AFTER payment_term_days'); @@ -81,8 +133,40 @@ CALL regression_add_iam_column('registered_capital', 'DECIMAL(18,2) NULL COMMENT CALL regression_add_iam_column('registered_capital_currency', 'CHAR(3) NULL COMMENT ''注册资本币种 ISO 4217'' AFTER registered_capital'); CALL regression_add_iam_column('registered_address', 'VARCHAR(500) NULL COMMENT ''注册地址文本'' AFTER registered_capital_currency'); CALL regression_add_iam_column('internal_remark', 'VARCHAR(1000) NULL COMMENT ''平台内部备注'' AFTER registered_address'); +CALL regression_add_iam_column('approved_at', 'DATETIME(3) NULL COMMENT ''审核通过时间'' AFTER internal_remark'); +CALL regression_add_iam_column('approved_by', 'BIGINT NULL COMMENT ''审核人ID'' AFTER approved_at'); +CALL regression_add_iam_column('reject_reason', 'VARCHAR(500) NULL COMMENT ''驳回原因'' AFTER approved_by'); DROP PROCEDURE regression_add_iam_column; +-- 注册审核和资质替换依赖的辅助字段；存量测试库也必须具备同一契约。 +DROP PROCEDURE IF EXISTS regression_add_iam_related_column; +DELIMITER $$ +CREATE PROCEDURE regression_add_iam_related_column( + IN p_table_name VARCHAR(64), + IN p_column_name VARCHAR(64), + IN p_definition VARCHAR(1000) +) +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = p_table_name + AND column_name = p_column_name + ) THEN + SET @regression_iam_related_sql = CONCAT( + 'ALTER TABLE `', p_table_name, '` ADD COLUMN `', p_column_name, '` ', p_definition + ); + PREPARE regression_iam_related_stmt FROM @regression_iam_related_sql; + EXECUTE regression_iam_related_stmt; + DEALLOCATE PREPARE regression_iam_related_stmt; + END IF; +END$$ +DELIMITER ; +CALL regression_add_iam_related_column('iam_tenant_qualification', 'review_comment', 'VARCHAR(500) NULL COMMENT ''审核意见'''); +CALL regression_add_iam_related_column('iam_membership', 'created_by', 'BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT ''创建人ID'''); +CALL regression_add_iam_related_column('iam_membership', 'updated_by', 'BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT ''最后修改人ID'''); +DROP PROCEDURE regression_add_iam_related_column; + -- 先写入父菜单，再写入按钮，避免新增环境出现悬空父节点。 INSERT INTO iam_permission (id, tenant_id, app_code, permission_type, parent_id, permission_code, @@ -122,6 +206,9 @@ VALUES (5100012, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:edit', '编辑企业客户', NULL, NULL, 20, 'ACTIVE', 1, 0), (5100013, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:status', '启停企业客户', NULL, NULL, 30, 'ACTIVE', 1, 0), (5100015, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:export', '导出企业客户', NULL, NULL, 40, 'ACTIVE', 1, 0), + (5100020, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:review', '客户审核', NULL, NULL, 50, 'ACTIVE', 1, 0), + (5100021, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:approve', '审核通过', NULL, NULL, 60, 'ACTIVE', 1, 0), + (5100022, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:reject', '审核驳回', NULL, NULL, 70, 'ACTIVE', 1, 0), (5101011, 0, 'PLATFORM', 'BUTTON', 5101010, 'platform:user:create', '新增内部用户', NULL, NULL, 10, 'ACTIVE', 1, 0), (5101012, 0, 'PLATFORM', 'BUTTON', 5101010, 'platform:user:edit', '编辑内部用户', NULL, NULL, 20, 'ACTIVE', 1, 0), (5101013, 0, 'PLATFORM', 'BUTTON', 5101010, 'platform:user:status', '启停内部用户', NULL, NULL, 30, 'ACTIVE', 1, 0), @@ -207,6 +294,7 @@ SELECT r.id, p.id, 0 AND p.permission_code IN ( 'platform:customer', 'platform:customer:list', 'platform:customer:create', 'platform:customer:view', 'platform:customer:edit', 'platform:customer:status', 'platform:customer:export', + 'platform:customer:review', 'platform:customer:approve', 'platform:customer:reject', 'platform:system', 'platform:user:list', 'platform:user:create', 'platform:user:edit', 'platform:user:status', 'platform:user:reset-password', 'platform:user:grant-role', 'platform:role:list', 'platform:role:create', 'platform:role:grant', @@ -698,6 +786,7 @@ WHERE deleted = 0 AND permission_code IN ( 'platform:customer', 'platform:customer:list', 'platform:customer:create', 'platform:customer:view', 'platform:customer:edit', 'platform:customer:status', 'platform:customer:export', + 'platform:customer:review', 'platform:customer:approve', 'platform:customer:reject', 'platform:system', 'platform:user:list', 'platform:user:create', 'platform:user:edit', 'platform:user:status', 'platform:user:reset-password', 'platform:user:grant-role', 'platform:role:list', 'platform:role:create', 'platform:role:grant', @@ -719,17 +808,82 @@ WHERE deleted = 0 SELECT 'IAM客户档案字段' AS check_item, COUNT(*) AS actual_count, - 9 AS expected_count, - IF(COUNT(*) = 9, '通过', '请检查客户档案字段迁移') AS check_result + 10 AS expected_count, + IF(COUNT(*) = 10, '通过', '请检查客户档案字段迁移') AS check_result FROM information_schema.columns WHERE table_schema = 'baiyue_iam' AND table_name = 'iam_tenant' AND column_name IN ( - 'contact_wechat', 'sales_owner_name', 'payment_term_days', 'enterprise_type', + 'legal_representative', 'contact_wechat', 'sales_owner_name', 'payment_term_days', 'enterprise_type', 'established_date', 'registered_capital', 'registered_capital_currency', 'registered_address', 'internal_remark' ); +SELECT + 'IAM注册审核字段' AS check_item, + COUNT(*) AS actual_count, + 4 AS expected_count, + IF(COUNT(*) = 4, '通过', '请检查租户审核状态和结果字段') AS check_result +FROM information_schema.columns +WHERE table_schema = 'baiyue_iam' + AND table_name = 'iam_tenant' + AND column_name IN ('status', 'approved_at', 'approved_by', 'reject_reason'); + +SELECT + 'IAM资质与成员审计字段' AS check_item, + ( + SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = 'baiyue_iam' + AND table_name = 'iam_tenant_qualification' + AND column_name = 'review_comment' + ) + ( + SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = 'baiyue_iam' + AND table_name = 'iam_membership' + AND column_name IN ('created_by', 'updated_by') + ) AS actual_count, + 3 AS expected_count, + IF( + ( + SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = 'baiyue_iam' + AND table_name = 'iam_tenant_qualification' + AND column_name = 'review_comment' + ) + ( + SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = 'baiyue_iam' + AND table_name = 'iam_membership' + AND column_name IN ('created_by', 'updated_by') + ) = 3, + '通过', '请检查资质审核备注和成员审计字段' + ) AS check_result; + +SELECT + 'IAM全局企业管理员模板' AS check_item, + COUNT(DISTINCT owner_role.id) AS role_count, + COUNT(DISTINCT role_permission.permission_id) AS actual_permission_count, + (SELECT COUNT(*) FROM baiyue_iam.iam_permission permission + WHERE permission.tenant_id = 0 AND permission.app_code = 'MERCHANT' + AND permission.permission_type IN ('MENU', 'BUTTON') + AND permission.status = 'ACTIVE' AND permission.deleted = 0) AS expected_permission_count, + IF( + COUNT(DISTINCT owner_role.id) = 1 + AND COUNT(DISTINCT role_permission.permission_id) = ( + SELECT COUNT(*) FROM baiyue_iam.iam_permission permission + WHERE permission.tenant_id = 0 AND permission.app_code = 'MERCHANT' + AND permission.permission_type IN ('MENU', 'BUTTON') + AND permission.status = 'ACTIVE' AND permission.deleted = 0 + ) + AND COUNT(DISTINCT role_permission.permission_id) > 0, + '通过', '请检查全局MERCHANT_OWNER角色及授权' + ) AS check_result +FROM baiyue_iam.iam_role owner_role +LEFT JOIN baiyue_iam.iam_role_permission role_permission + ON role_permission.role_id = owner_role.id AND role_permission.tenant_id = 0 +WHERE owner_role.tenant_id = 0 AND owner_role.app_code = 'MERCHANT' + AND owner_role.role_code = 'MERCHANT_OWNER' + AND owner_role.role_type = 'SYSTEM' AND owner_role.status = 'ACTIVE' AND owner_role.deleted = 0; + SELECT table_name, COUNT(*) AS column_count diff --git a/deploy/sql/README.md b/deploy/sql/README.md index d5d634534..3a6cc01c1 100644 --- a/deploy/sql/README.md +++ b/deploy/sql/README.md @@ -16,7 +16,10 @@ 1. IAM 基线表必须已存在（`iam.sql`）。 2. 权限父菜单和既有客户列表/审核权限基线（`phase1-admin-rbac.sql`、`rbac-role-management-v2.sql`）。 3. `phase1-platform-master-data-and-permission-management.sql` 中的 `platform:customer:view`、既有角色继承及管理员授权。 -4. `phase1-customer-profile.sql` 中的九个 `iam_tenant` 客户档案字段。 +4. `phase1-customer-profile.sql` 中的十个 `iam_tenant` 客户档案字段。 +5. `phase1-onboarding.sql` 中的 `PROFILE_REQUIRED` 默认状态、审核结果字段、资质审核备注及成员审计字段。 + +统一入口同时幂等补齐审核自动开户所需的全局 `MERCHANT_OWNER` 角色模板、全局商户菜单/按钮权限和角色授权；执行后应使用 `iam/src/test/resources/db/verify-phase1-admin-rbac.sql` 确认模板存在且权限完整。 测试环境只执行统一入口；模块脚本用于新环境基线构建或单模块升级，不应在统一入口执行后再次串行执行。 @@ -24,4 +27,45 @@ 在 Navicat、DataGrip 或 MySQL 命令行中以“运行脚本”方式完整执行，确保客户端支持 `DELIMITER`。不要只选中局部语句执行。 -执行结束后必须看到 `MIGRATION_COMPLETED`，并确认其前面的 IAM 权限、IAM 客户档案九字段、Warehouse 表结构、Order 幂等表和 Firstleg 索引核验均通过。 +执行结束后必须看到 `MIGRATION_COMPLETED`，并确认其前面的 IAM 权限、IAM 客户档案十字段、IAM 注册审核字段、资质与成员审计字段、Warehouse 表结构、Order 幂等表和 Firstleg 索引核验均通过。 + +## 公共导出中心开发阶段重建 + +统一迁移脚本中的 `file_export_task` 是一期旧基线，不能自动升级已经存在的表。公共导出中心尚未进入生产、测试环境确认没有需要保留的导出任务时，按下列顺序重建；这一步是公共导出中心上线的额外步骤，不得用于已有真实导出数据的生产环境。 + +先只读确认任务数为 0： + +```sql +SELECT COUNT(*) AS export_task_count FROM baiyue_file.file_export_task; +``` + +只有结果为 0 才允许执行： + +```sql +USE baiyue_file; +DROP TABLE IF EXISTS file_export_outbox; +DROP TABLE IF EXISTS file_export_task_guard; +DROP TABLE IF EXISTS file_export_task; +SOURCE file/src/main/resources/db/file.sql; +``` + +`SOURCE` 的相对路径以 MySQL 客户端启动目录为准；图形客户端可直接完整执行 `file/src/main/resources/db/file.sql`。完成后必须核验： + +```sql +SELECT COUNT(*) = 3 AS export_tables_ready + FROM information_schema.tables + WHERE table_schema = 'baiyue_file' + AND table_name IN ('file_export_task', 'file_export_outbox', 'file_export_task_guard'); + +SELECT COUNT(*) = 1 AS creation_guard_ready + FROM baiyue_file.file_export_task_guard + WHERE guard_key = 'GLOBAL_CREATE'; + +SELECT COUNT(*) = 8 AS scheduler_columns_ready + FROM information_schema.columns + WHERE table_schema = 'baiyue_file' AND table_name = 'file_export_task' + AND column_name IN ('dispatch_version', 'attempt_count', 'max_attempts', 'next_retry_at', + 'lease_owner', 'lease_until', 'last_heartbeat_at', 'execution_deadline_at'); +``` + +任一结果不是 1，禁止启动 File Service。后续一旦表内存在生产数据，必须改用可重复执行的增量迁移，不能继续 DROP 重建。 diff --git a/docs/公共导出与下载中心接入说明.md b/docs/公共导出与下载中心接入说明.md index 9e4885f1b..23b135cac 100644 --- a/docs/公共导出与下载中心接入说明.md +++ b/docs/公共导出与下载中心接入说明.md @@ -1,161 +1,151 @@ # 公共导出与下载中心接入说明 -## 1. 目标与边界 +本文描述当前已经实现的 RocketMQ + 事务 Outbox 版本。完整状态机、故障模型和容量门槛见[公共导出中心 RocketMQ 架构设计](./公共导出中心RocketMQ架构设计.md)。 -`file-service` 统一承载异步导出任务、XLSX 文件生成结果、过期清理和下载鉴权。业务服务仍是业务数据唯一事实来源，通过内部、只读、分批接口向 File 提供导出数据。 +## 1. 边界 -- 浏览器不得拉取全部分页后自行生成 Excel。 -- File 不跨库查询 IAM、Order、Warehouse 等业务数据库。 -- 任务不保存用户 Bearer Token；服务间调用使用 `X-Internal-Service-Token`。 -- 当前首个导出类型为 `CUSTOMER_LIST`，数据由 IAM 提供。 -- 当前存储实现沿用 `FILE_STORAGE_ROOT` 下的共享目录；生产多实例必须挂载同一持久化目录，后续可替换为对象存储。 +- `file-service` 统一承载任务、可靠调度、通用 XLSX、存储、过期清理、下载鉴权和指标。 +- File 不跨库读取 IAM、Order、Warehouse 等业务数据；业务服务只实现标准只读 Provider。 +- MQ 只传任务标识，不传筛选条件、业务行、Token 或文件。 +- 一期存储是 `FILE_STORAGE_ROOT` 目录且按单节点部署；多实例前必须使用跨主机共享卷或对象存储。 +- 当前首个类型为 `CUSTOMER_LIST`，由 IAM 提供数据。 -## 2. 调用流程 +## 2. 执行链路 ```text -Admin/Client +Admin -> POST /api/platform/v1/files/export-tasks - -> file_export_task(PENDING) - -> File worker 原子领取(RUNNING) - -> 业务服务 internal export API（最多 500 条/批） - -> SXSSFWorkbook 流式生成 XLSX - -> file_export_task(SUCCEEDED) + -> file_export_task_guard 行锁下复核重复提交及活动上限 + -> 同一事务写 file_export_task + file_export_outbox + -> Publisher 投递 RocketMQ（失败无限退避） + -> Consumer 按 dispatch_version + 数据库租约领取（并发 4） + -> POST /internal/exports/v1/{TYPE}/batches + -> 通用 SXSSFWorkbook 流式写盘 + -> 条件提交 SUCCEEDED -> GET /api/platform/v1/files/export-tasks/{id}/download ``` -任务状态只允许： - ```text -PENDING -> RUNNING -> SUCCEEDED - -> FAILED -> PENDING（手工重试） -SUCCEEDED -> EXPIRED +PENDING -> RUNNING -> SUCCEEDED -> EXPIRED + -> PENDING（自动重试，dispatch_version + 1） + -> FAILED -> PENDING（人工重试，dispatch_version + 1） ``` -服务重启后数据库中的 `PENDING` 会继续处理；超过配置阈值的 `RUNNING` 会重新排队。多个 File 实例通过 `PENDING` 条件更新原子领取，只有一个实例能成功。 +消息补投保持同一版本；只有执行失败、执行租约恢复和人工重试递增版本。独立 watchdog 覆盖 Provider 拉取和最终写盘，最终成功 SQL 同时校验任务状态、版本、租约所有者与数据库 deadline。 -## 3. 平台公开接口 +Reconciler 负责补写缺失 Outbox、同版本重发长期 `SENT` 命令、恢复超时执行租约、释放超时 Outbox 租约及过期文件清理。旧 `ExportTaskWorker` 已删除，避免绕过版本和退避窗口。 -所有接口都经过 Gateway 的现有 `/api/platform/v1/files/**` 路由。 +## 3. 浏览器 API -### 3.1 创建导出任务 +- `POST /api/platform/v1/files/export-tasks` +- `GET /api/platform/v1/files/export-tasks` +- `GET /api/platform/v1/files/export-tasks/{id}` +- `GET /api/platform/v1/files/export-tasks/{id}/download` +- `POST /api/platform/v1/files/export-tasks/{id}/retry` -`POST /api/platform/v1/files/export-tasks` +创建示例： ```json { "exportType": "CUSTOMER_LIST", "format": "XLSX", "taskName": "客户列表", - "parameters": { - "keyword": "柏悦", - "status": "ACTIVE" - } + "parameters": { "keyword": "柏悦", "status": "ACTIVE" } } ``` -创建接口按 `exportType` 校验来源权限。客户导出要求 `platform:customer:export`。筛选参数在创建时形成快照，后续页面修改筛选条件不会改变已提交任务。 +创建按注册表校验访问域、启用状态和来源权限。客户导出要求 `platform:customer:export`；列表和下载要求 `platform:download:list`。默认只能访问本人任务，`platform:download:all` 可访问同一租户与应用域的全部任务。人工重试只允许 `FAILED`，并重新校验类型启用状态和来源权限。 -### 3.2 下载中心列表 +限制：单用户最多 3 个活动任务、单类型最多 5 个活动任务；同一用户 60 秒内相同类型和规范化参数返回已有活动任务。 -`GET /api/platform/v1/files/export-tasks?page=1&size=20&exportType=CUSTOMER_LIST&status=SUCCEEDED&requestedFrom=2026-08-01&requestedTo=2026-08-03` - -需要 `platform:download:list`。默认只返回当前登录用户创建的任务；拥有 `platform:download:all` 时可查看同一租户和应用域内所有用户任务。 - -### 3.3 任务详情、下载和重试 - -- `GET /api/platform/v1/files/export-tasks/{id}` -- `GET /api/platform/v1/files/export-tasks/{id}/download` -- `POST /api/platform/v1/files/export-tasks/{id}/retry` - -下载仅允许 `SUCCEEDED` 且未过期的任务。重试仅允许 `FAILED`，并重新校验该导出类型的来源权限。服务端始终校验任务租户、应用域和申请人归属，前端按钮隐藏不能替代鉴权。 - -## 4. IAM 客户数据提供契约 - -File 调用： - -`GET /internal/iam/exports/customers?cursor={lastId}&limit=500&keyword={keyword}&status={status}` - -请求头： +## 4. Provider 契约 ```http +POST /internal/exports/v1/{exportType}/batches X-Internal-Service-Token: ${INTERNAL_SERVICE_TOKEN} +Content-Type: application/json ``` -响应： +请求使用 `common-core` 的 `ExportBatchRequest`，必须包含 `taskId`、`tenantId`、`appCode`、`requestedBy`、`dispatchVersion`、`parameters`、`cursor`、`snapshotToken` 和 `limit`。Provider 必须按身份字段约束查询，不得信任 parameters 内的租户字段。 -```json -{ - "code": "200", - "data": { - "records": [], - "nextCursor": 190000000000000001, - "hasMore": false - } -} +响应使用 `ExportBatchResponse`：首批返回 `schemaVersion`、`schemaHash`、`snapshotToken`、`columns`、`rows`、游标和估算总行数；后续批次必须保持 `schemaHash`，可省略 `columns`。File 在 Feign 解码流读取阶段限制单批 4MB，再校验游标推进、列/行/文件上限，并支持 `STRING`、`INTEGER`、`DECIMAL`、`BOOLEAN`、`DATE`、`DATETIME` 六类 Excel 单元格。 + +IAM `CUSTOMER_LIST` Provider 强制 `tenantId=0`、`appCode=PLATFORM`，采用 HMAC 签名的最大商户 ID 快照。因此不会混入开始后新增客户，但不冻结期间的修改与删除。 + +新增类型只需：业务服务实现标准 Provider、在 `baiyue.file.export.definitions` 注册受控服务/路径/权限/访问域/容量参数、补权限和测试。不得修改公共 Controller、Consumer 或 Writer。 + +## 5. MQ 与 DLQ + +- Topic：`${MQ_TOPIC_PREFIX}baiyue-file-export-command-v1`，4 个队列。 +- Consumer：`${MQ_TOPIC_PREFIX}baiyue-file-export-worker-v1`，同步并发 4。 +- Producer：`${MQ_TOPIC_PREFIX}baiyue-file-export-publisher-v1`。 +- `eventId`、`taskId`、`dispatchVersion`、`schemaVersion` 同时写入 User Properties。 +- 业务失败从不进入 DLQ；只有无法反序列化或不支持版本的消息不确认消费。 +- DLQ Handler 不解析损坏 body，只从四个属性定位，并在同一事务内把 Outbox 置 `QUARANTINED`、当前 PENDING 任务置 `FAILED`。缺少定位属性只记录 CRITICAL 和 messageId，不误伤任务。 +- DLQ Handler 的数据库处置失败持续重试，不允许因有限次数耗尽进入无人消费的二级 DLQ。 + +## 6. 配置与部署 + +关键环境变量： + +```text +ROCKETMQ_NAME_SERVER +ROCKETMQ_BROKER_IP +MQ_TOPIC_PREFIX +FILE_EXPORT_CONSUMER_CONCURRENCY +FILE_EXPORT_RECONCILER_ENABLED +FILE_EXPORT_PUBLISHER_ENABLED +FILE_EXPORT_CONSUMER_ENABLED +FILE_EXPORT_RETENTION_DAYS +FILE_STORAGE_ROOT ``` -IAM 按租户 ID 升序游标读取，只查询 `tenant_type=MERCHANT`，并复用客户列表的 `keyword/status` 语义。导出 DTO 与平台列表 DTO 分离，因此不会复用列表中的电话和邮箱脱敏结果。 +`MQ_TOPIC_PREFIX` 是部署必填项但允许空值：测试环境显式填 `test-`，生产环境显式填空字符串。不能使用 `:-test-` 作为 Compose 默认值，否则生产空值会被重新替换成测试前缀。 -## 5. 新增其他导出类型 +当前 RocketMQ Compose 不映射 Broker 宿主机端口，File 与 Broker 必须同在 `infra-network`，因此 `ROCKETMQ_BROKER_IP` 填 `rocketmq-broker`。Topic 初始化会等待 `BaiyueCluster` 真正注册，任一幂等初始化命令失败后自动重试。 -以订单导出为例： +File 容器初始 JVM 为 `-Xms1g -Xmx2g -XX:+ExitOnOutOfMemoryError`。Provider 单批连接/读取超时分别为 3 秒/15 秒；执行租约 300 秒、watchdog 60 秒、deadline 30 分钟、消费超时 90 分钟、单文件 50MB。 -1. 数据所属服务提供带内部令牌校验的只读游标接口，不允许 File 直接访问其数据库。 -2. 在 File 增加一个 `ExportTaskHandler` Bean： - - `exportType()` 返回稳定编码，例如 `FIRST_LEG_ORDER_LIST`； - - `requiredPermission()` 返回创建该任务所需的业务权限； - - `export(...)` 分批调用来源服务、流式写入文件并报告进度。 -3. 增加对应 Feign Client 和导出专用 DTO，不复用面向页面的脱敏或分页 DTO。 -4. 添加稳定权限资源、管理员授权和前端任务类型显示。 -5. 增加处理器 XLSX 内容测试、来源接口筛选/游标测试和前端提交测试。 +开发阶段确认无保留任务后，重建并执行 `file/src/main/resources/db/file.sql`。RocketMQ 使用 `deploy/docker-compose.rocketmq.yml`；脚本会先校验并展开 `ROCKETMQ_BROKER_IP`，再显式创建 4 队列命令 Topic、Worker 消费组、DLQ Handler 消费组以及可读写的 Worker DLQ。上线顺序：Broker -> SQL -> Nacos 注册表 -> IAM Provider -> File 三开关全关 -> Reconciler -> Publisher -> Consumer -> Admin。 -不要在公共 Controller 中增加按类型展开的业务查询分支；类型扩展通过 Spring 注入的 `ExportTaskHandler` 注册表完成。 +生产启用并发 4 前必须完成 4 个 20 万行任务并发压测；若 CPU、堆、磁盘、Provider 或 IAM 普通接口延迟不达设计门槛，回退并发 2。 -## 6. 存储、保留与配置 +## 7. 观测与验收 -```yaml -baiyue: - file: - storage-root: ${FILE_STORAGE_ROOT} - export: - retention-days: ${FILE_EXPORT_RETENTION_DAYS:7} - worker-delay-ms: ${FILE_EXPORT_WORKER_DELAY_MS:2000} - stale-running-minutes: ${FILE_EXPORT_STALE_RUNNING_MINUTES:30} +Prometheus 指标包括任务创建/完成/耗时、队列年龄、Outbox 数量/年龄/最大尝试次数、Reconciler 动作与 `export_dlq_messages_total`；日志携带任务与版本标识但不打印完整数据或令牌。 + +最低验收：任务与 Outbox 同事务；Broker 停机恢复后追平；重复消息只生成一个有效文件；watchdog 覆盖最终写盘；deadline 竞态不能提交成功；同版本补投不递增版本；毒消息只隔离一次；IAM 快照不混入新增客户；生成的 XLSX 可真实打开且日期、金额为可计算类型；普通用户不能访问他人任务。 + +### 7.1 测试环境自动验收入口 + +`scripts/export-center-test-env-acceptance.ps1` 通过 Gateway 创建真实 `CUSTOMER_LIST` 任务，轮询终态，下载文件并校验 XLSX 的 ZIP 签名。Token 只通过参数或环境变量注入，脚本不会登录、部署、停止 Broker 或改数据库。 + +单任务冒烟： + +```powershell +$env:ADMIN_ACCESS_TOKEN = '<测试环境平台用户Token>' +.\scripts\export-center-test-env-acceptance.ps1 ` + -GatewayBaseUrl 'http://8.163.30.128:9000' ``` -导出文件位于 `${FILE_STORAGE_ROOT}/exports/{taskId}/{uuid}.xlsx`。`object_key` 必须匹配服务端固定格式，任意绝对路径、`..` 或非 XLSX 后缀都会被拒绝。任务过期后先删除文件，再标记 `EXPIRED`。 +并发 4 容量验收必须使用 4 个不同申请用户，避免 60 秒重复提交合并，并要求每个账号都具备 `platform:customer:export` 与 `platform:download:list`： -## 7. SQL 与部署顺序 +```powershell +$tokens = @($env:EXPORT_TOKEN_1, $env:EXPORT_TOKEN_2, $env:EXPORT_TOKEN_3, $env:EXPORT_TOKEN_4) +.\scripts\export-center-test-env-acceptance.ps1 ` + -GatewayBaseUrl 'http://8.163.30.128:9000' ` + -AccessTokens $tokens -TaskCount 4 -MinimumRows 200000 -TimeoutSeconds 3600 +``` -新环境： +故障注入分两段执行：故障期间使用 `-CreateOnly` 创建并保存任务 ID；恢复 Broker、File 或 IAM 后，用相同申请人的 Token 和 `-TaskIds` 继续轮询与下载。这样停止/恢复基础设施仍由有权限的运维操作，脚本只验证任务是否最终追平： -1. 执行 `iam/src/main/resources/db/phase1-admin-rbac.sql` 及后续 IAM 权限脚本。 -2. 执行 `file/src/main/resources/db/file.sql` 创建 `file_export_task`。 -3. 部署 IAM，再部署 File，最后部署 Admin 静态资源。 +```powershell +.\scripts\export-center-test-env-acceptance.ps1 ` + -GatewayBaseUrl 'http://8.163.30.128:9000' -AccessTokens $tokens -TaskCount 4 -CreateOnly -现有测试环境可执行 `deploy/sql/2026-07-31一期回归测试环境增量迁移.sql`。脚本幂等创建任务表并补齐： +.\scripts\export-center-test-env-acceptance.ps1 ` + -GatewayBaseUrl 'http://8.163.30.128:9000' -AccessTokens $tokens ` + -TaskIds @('', '', '', '') -MinimumRows 200000 +``` -- `platform:customer:export` -- `platform:download:list` -- `platform:download:all` - -File 启动前必须确认 `FILE_STORAGE_ROOT` 是持久化且可写的共享目录。 - -## 8. 错误与追踪 - -- 创建、列表和重试使用统一 `ApiResponse`，失败记录业务码、`requestId`、`traceId`。 -- 下载接口成功返回文件流；失败仍由统一异常处理返回 JSON 错误。 -- 来源服务不可用或返回非成功业务码时任务进入 `FAILED`，下载中心展示可理解的失败原因。 -- File 日志使用任务 ID 和导出类型关联故障；不得打印内部令牌、Bearer Token 或完整导出数据。 - -## 9. 验收清单 - -- 创建任务后浏览器没有批量调用业务列表，也没有生成 HTML `.xls`。 -- 当前筛选条件原样进入 `parameters_json`。 -- 两个 worker 同时领取时只有一个能将任务从 `PENDING` 更新为 `RUNNING`。 -- XLSX 可被标准办公软件打开，表头和数据列匹配。 -- 普通用户看不到、下载不了其他用户任务；`platform:download:all` 用户可以。 -- `FAILED` 可以重试，其他状态不能重试。 -- 过期任务不能下载，文件被清理。 -- 服务重启后 `PENDING` 继续执行，超时 `RUNNING` 能重新排队。 -- Admin 下载中心覆盖加载、空数据、失败、处理中、完成、失败、过期和窄屏操作状态。 +脚本通过只证明 API 创建、数据库状态机、MQ 触发、Provider、XLSX 写入和下载主链路。CPU/堆/磁盘、Provider P95/P99、IAM 普通接口退化比例仍须同时从 Prometheus/Grafana 取证；DLQ 毒消息必须由 `mqadmin` 按 §19.2 投递，不能由普通验收账号伪造。 diff --git a/docs/公共导出中心RocketMQ架构设计.md b/docs/公共导出中心RocketMQ架构设计.md new file mode 100644 index 000000000..1bf2181b5 --- /dev/null +++ b/docs/公共导出中心RocketMQ架构设计.md @@ -0,0 +1,1394 @@ +# 公共导出中心 RocketMQ 架构设计（第四轮快速评审定稿） + +> 状态：设计已实施并通过本地自动化回归，待完成 §19.2/§19.3 测试环境真实验收 +> 日期：2026-08-03 +> 修订说明：第一版曾标记“定稿”，经评审发现 3 个阻断级可靠性问题（Reconciler 导致排队任务饥饿、DEAD 策略自相矛盾且可无限循环、`brokerIP1` 环境变量不会被替换）及 7 项边界缺陷，均已在本稿修正。核心修正是确立 §3 原则 5：**消息补投不产生新版本**。 +> 第四轮快速评审：确认 8C32G 环境以并发 4 作为压测起始值；补齐并发环境变量透传，并将三个调度开关改为默认关闭，避免首次部署绕过分阶段启用顺序。 +> 适用范围：`baiyue-firstleg` 内需要异步导出和下载中心能力的业务服务 +> 容量基准：平台总用户约 1000，导出为低频后台操作。本文所有定量参数按该基准取值，并在 §22 说明放大路径。 + +## 1. 结论摘要 + +将现有 File Service 的“数据库任务表 + 定时扫描并直接执行”升级为： + +```text +任务表 + 事务 Outbox + RocketMQ 触发 + 幂等 Consumer + Reconciler 兜底 + 统一数据 Provider 契约 +``` + +采用至少一次投递，不追求跨数据库与 MQ 的伪“恰好一次”。**数据库是唯一事实来源与唯一可靠性保证；RocketMQ 是加速通道，不是可靠性来源。** 这一条是本文全部设计取舍的根据： + +- 任何“此刻不该执行”的消息一律确认消费，**绝不使用 `RECONSUME_LATER` 做业务重排**。 +- 所有重试、延迟、恢复由数据库的 `next_retry_at`、`lease_until` 与 Reconciler 承担。 +- 因此 DLQ 只可能收到无法反序列化的毒消息，“DLQ 出现任意新消息即告警”这条规则才成立。 + +目标是让 IAM、Order、Warehouse、Finance 等业务服务只提供导出数据，不再分别维护任务表、MQ Consumer、Excel 生成、文件存储、重试、过期清理或下载中心。 + +### 1.1 实施前代码基线（评审时已核实） + +以下表格记录设计启动时用于制定迁移方案的历史基线，不代表当前代码状态；当前已实现能力以 [公共导出与下载中心接入说明](./公共导出与下载中心接入说明.md) 和实际源码为准。 + +| 事项 | 实际状态 | +|---|---| +| 任务执行 | `file/.../service/ExportTaskWorker.java` 定时 2 秒轮询，`processNext()` 每轮只执行一个任务 | +| 原子领取 | `ExportTaskRepository.claim()` 按 `status = PENDING` 条件更新，多实例安全 | +| 失联恢复 | `requeueStale()` 按 `updated_at` 早于阈值粗粒度回队，无租约、无心跳 | +| 自动重试 | 不存在。失败即 `FAILED`，只能人工重试 | +| 导出类型注册 | Java Bean 注册（`ExportTaskHandler` 实现类），列定义硬编码在 `CustomerExportTaskHandler.HEADERS` | +| Provider 调用 | `IamExportClient` 为 IAM 专用 Feign 接口，每增一个类型需复制一个 Client | +| RocketMQ | **仓库无任何可用基础**：`spring-cloud-starter-stream-rocketmq` 只出现在 `firstleg/pom.xml` 且 firstleg 无一行 Stream 代码（死依赖）；`file/pom.xml` 无 MQ 依赖；全仓 `*.yml` 无 rocketmq 配置；`docker-compose-infra.yml` 只有 nacos/mysql/redis，无 NameServer 与 Broker | +| 既有 Outbox | 已有两套且**均无发布器**：`ord_outbox_event`（`order/src/main/resources/db/order.sql:153`，`OrderApplicationService.writeOutbox()` 只写不发）与 `bridge_outbox`（`event-bridge-service`）。全仓 `@Scheduled` 仅 `ExportTaskWorker` 一处 | +| 指标 | 只有 `micrometer-core`（`common-web/pom.xml`），**无 `micrometer-registry-prometheus`**，actuator 的 `prometheus` 端点当前不可用；无 Prometheus 服务端 | +| 测试基建 | 全仓无 Testcontainers 依赖，无容器化测试先例 | +| 存储 | `LocalExportStorage` 为具体类直接注入，对象键 `exports/{taskId}/{uuid}.xlsx`；部署已挂载 `FILE_UPLOADS_HOST_DIR:/data/files` | + +本文的实施清单以上述历史基线为迁移起点，不假设当时已有任何基础设施就绪。 + +## 2. 目标与非目标 + +### 2.1 目标 + +- 导出请求快速返回任务编号，后台异步执行。 +- 数据库提交成功的任务最终一定被调度，不因 MQ 短暂不可用而丢失。 +- MQ 重复投递、Consumer 重启、网络超时不会产生重复有效文件。 +- 失败自动重试、人工重试、租约超时恢复全部可追踪，且不产生 DLQ 噪声。 +- 下载中心继续使用统一任务表，不依赖 MQ 查询状态。 +- 新业务服务只实现公共 Provider 契约并注册导出类型，不写任务、Excel、存储、下载代码。 +- File Service 统一负责 XLSX、存储、下载鉴权、过期清理、可观测性。 +- 代码层面支持 File Service 多实例（租约、条件抢占、`lease_owner` 齐备）；**部署层面一期按单节点运行**，因为存储是宿主机目录（§11.1）。滚动发布期间的短暂双实例由租约保证安全。 + +### 2.2 非目标 + +- 不支持 CSV、PDF 或同步大文件下载。 +- 不允许业务服务提交任意 URL 让 File Service 回调。 +- 不允许 File Service 跨库读取业务数据。 +- 不把完整导出数据写入 MQ。 +- 不以 Redis 作为任务、重试或幂等的事实来源。 +- 不实现用户自定义列编排器。 +- 不实现导出断点续传：每次执行从零重写整个文件（见 §9.5）。 +- 一期不引入 MinIO/OSS（见 §21 决策 6），但存储访问必须走接口抽象。 +- 一期不引入 Testcontainers（见 §21 决策 8）。 + +## 3. 核心原则 + +1. **数据库是事实来源**：任务状态、申请人、租户、筛选条件、文件信息、尝试次数、租约保存在 MySQL。（**注意**：任务表**不**保存权限快照——权限只在创建与人工重试时校验，执行阶段不再校验，理由见 §10.4。） +2. **MQ 只传命令**：消息只含 `eventId`、`taskId`、`dispatchVersion` 等标识，不携带业务数据。 +3. **事务 Outbox 保证不丢任务**：任务与待发送消息在 File 数据库同一本地事务提交。 +4. **至少一次 + 幂等**：允许重复消息，不允许重复有效执行。 +5. **版本只代表执行尝试，不代表投递尝试**：`dispatch_version` 递增**仅**发生在三种情形——执行失败后的自动重试（§9.2）、租约超时恢复（§8.5-2）、人工重试（§9.3）。**消息补投、重发、重复投递一律不递增版本**，只产生同一版本的重复消息，由 Consumer 幂等吸收。 +6. **MQ 不承担重排与延迟**：不使用延迟消息、不使用 `RECONSUME_LATER` 做业务重排；Reconciler 是唯一重派通道。 +7. **业务服务只提供数据**：不生成文件，不维护下载状态，不消费公共导出 Topic。 +8. **注册信息必须受控**：服务名、Provider 路径、权限编码来自受控配置且经代码白名单二次校验，不接受浏览器传入。 +9. **租户维度贯穿全链路**：任务、Provider 请求、Provider 查询、下载鉴权都必须带租户与访问域（AGENTS.md 强制要求）。 +10. **时间以数据库为准**：所有租约领取、续租、过期判断使用数据库 `CURRENT_TIMESTAMP(3)`，不使用各实例的 `LocalDateTime.now()`（见 §8.6）。 + +### 3.1 原则 5 为何是本文的核心 + +第一版设计在“Reconciler 发现命令可能丢失”时递增 `dispatch_version` 并重发。这看似安全，实则制造了一个**活性错误**： + +并发为 4、单任务最长 30 分钟，则第五个任务在 Broker 队列中正常等待 30 分钟期间，任务状态一直是 `PENDING`。若 Reconciler 在第 10 分钟判定“发出太久未执行 = 消息丢失”并递增版本，原消息到达时会因版本过期被丢弃，新消息又排到队尾——**持续负载下这个任务可能永远无法开始执行**。这不是“产生一些垃圾 Outbox 行”，而是任务饥饿。 + +同一个错误还有第二个后果：Broker 长期不可用时，Outbox 发送次数耗尽 → 换新版本重建 → 新行继续耗尽 → 无限循环。因为任务的 `attempt_count` 在这个过程中从不增长，`max_attempts` 这道闸门根本拦不住。 + +根因是把**投递失败**（消息没送到）和**执行失败**（送到了但跑挂了）当成了同一件事。两者的正确处理完全相反： + +| | 语义 | 正确处理 | 是否递增版本 | +|---|---|---|---| +| 投递失败 | 命令没送到 Consumer | 重发**同一条**命令 | **否** | +| 执行失败 | 命令送到了，执行出错 | 作废旧尝试，开始新尝试 | **是** | + +确立原则 5 后，三个问题一次性消解：排队任务不再被作废（补投不动版本，原消息仍有效）、Broker 丢消息可安全补投（重发同版本，Consumer 幂等）、DEAD 无限循环消失（不再有“换版本重建”这条路径）。 + +## 4. 总体架构 + +```mermaid +flowchart LR + UI["Admin / Client"] -->|"创建任务"| API["File Export API"] + API -->|"同一事务"| TASK["file_export_task"] + API -->|"同一事务"| OUTBOX["file_export_outbox"] + PUB["Outbox Publisher"] -->|"到点领取"| OUTBOX + PUB -->|"发送 taskId"| MQ["RocketMQ export command topic"] + MQ --> CONSUMER["File Export Consumer"] + CONSUMER -->|"原子抢占 + 租约"| TASK + CONSUMER -->|"内部令牌 + 游标 + 快照"| PROVIDER["业务服务 Export Provider"] + PROVIDER --> CONSUMER + CONSUMER --> XLSX["通用 XLSX Writer"] + XLSX --> STORAGE["ExportStorage 共享目录"] + CONSUMER --> TASK + REC["Reconciler"] -->|"补发 / 租约恢复 / 过期清理"| TASK + REC --> OUTBOX + REC --> STORAGE + UI -->|"任务列表 / 下载"| API + API --> STORAGE +``` + +### 4.1 服务职责 + +| 组件 | 职责 | +|---|---| +| File Export API | 创建、查询、下载、人工重试；校验登录身份、租户、访问域与业务权限 | +| `file_export_task` | 任务状态与下载中心事实来源，承载调度字段与租约 | +| `file_export_outbox` | 可靠记录需要发送的 MQ 命令，承载发送重试与租约 | +| Outbox Publisher | 到点原子领取 Outbox、发送 MQ、记录发送结果与退避 | +| RocketMQ | 触发 Consumer、吸收突发、按 Consumer 并发天然排队 | +| File Export Consumer | 幂等抢占、watchdog 续租（§8.3）、调用 Provider、生成文件、提交结果 | +| Business Export Provider | 校验租户与筛选条件，按稳定快照游标返回列与行数据 | +| Reconciler | **唯一重派通道**：补发漏发命令、恢复超时租约、释放僵死 Outbox、清理过期文件与孤立文件 | + +### 4.2 为什么 MQ 存在（诚实说明） + +Outbox Publisher 与 Reconciler 仍然轮询数据库，MQ **不消除轮询**。MQ 在本架构中的实际价值是： + +1. 把“任务执行触发”从秒级轮询变为毫秒级推送，Reconciler 因此可以降到 30 秒一轮，数据库压力低于现状。 +2. Consumer 并发即天然的执行队列与背压，突发提交不会压垮 Provider。 +3. 与 `order` / `event-bridge-service` 已规划的事件化方向统一，`file_export_outbox` 的表结构与 Publisher 组件将成为全仓 Outbox 的公共基线（§21 决策 3）。 + +不宣称 MQ 提升了可靠性——可靠性来自数据库与 Reconciler。 + +## 5. 公共接口保持兼容 + +现有浏览器接口不变： + +- `POST /api/platform/v1/files/export-tasks` +- `GET /api/platform/v1/files/export-tasks` +- `GET /api/platform/v1/files/export-tasks/{id}` +- `GET /api/platform/v1/files/export-tasks/{id}/download` +- `POST /api/platform/v1/files/export-tasks/{id}/retry` + +创建任务请求仍为： + +```json +{ + "exportType": "CUSTOMER_LIST", + "format": "XLSX", + "taskName": "客户列表", + "parameters": { + "keyword": "柏悦", + "status": "ACTIVE" + } +} +``` + +浏览器不得提交 Provider 服务名、URL、Topic、消费组或所需权限。File Service 依据 `exportType` 查询受控注册表。响应中所有 `Long` 仍按仓库约定输出为字符串，业务成功与失败统一 HTTP 200 并以响应体 `code == "200"` 判断。 + +任务状态机（新增自动重试路径）： + +```text +PENDING -> RUNNING -> SUCCEEDED -> EXPIRED + -> PENDING（自动重试，dispatch_version + 1） + -> FAILED -> PENDING（人工重试，dispatch_version + 1） +``` + +## 6. RocketMQ 设计 + +### 6.1 客户端选型 + +使用 `org.apache.rocketmq:rocketmq-spring-boot-starter:2.3.3`，**不使用** `spring-cloud-starter-stream-rocketmq`。理由： + +- 本场景需要显式控制“抢不到就确认消费”的返回语义（`ConsumeConcurrentlyStatus.CONSUME_SUCCESS`），Stream 的函数式模型表达这一点绕。 +- 仓库中那个 Stream 依赖是无人使用的死依赖，实施时**一并从 `firstleg/pom.xml` 删除**，避免两套客户端并存。 + +**版本选 2.3.3 而非 2.3.1，且 Spring Boot 3.2.9 兼容性已实证核验**（该 starter 的 parent pom 声明 `spring.boot.version` 为 2.7.18，容易误判为不兼容，故记录证据）： + +| 核验项 | 结果 | +|---|---| +| 自动配置注册方式 | jar 内**同时**包含 `META-INF/spring.factories`（Boot 2）与 `META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports`（Boot 3），Boot 3 下 `RocketMQAutoConfiguration` 能正常加载 | +| `javax.*` 残留 | 全 jar 扫描**无任何** `javax/annotation`、`javax/validation`、`javax/servlet` 引用，不受 javax→jakarta 断裂影响 | +| 字节码版本 | major 52（Java 8），JDK 17 可运行 | +| 传递 `rocketmq-client` | 2.3.3 → **5.3.1**（2.3.1 → 5.3.0）。5.3.1 与 §16.1 的 Broker 镜像 `apache/rocketmq:5.3.1` 版本一致，也与仓库中 `spring-cloud-alibaba 2023.0.3.2` 已解析出的 `rocketmq-client 5.3.1` 相同 | + +starter 的 `spring-boot-starter` / `spring-boot-starter-validation` 依赖未锁版本，在本仓由 Spring Boot 3.2.9 的 `dependencyManagement` 接管，解析为 3.2.9。 + +协议使用 Remoting（4.x 兼容模式）直连 NameServer + Broker，不引入 RocketMQ 5.x 的 Proxy/gRPC 组件——当前量级不需要，且少一个可失效组件。 + +实施第一步仍须编译期验证：加入依赖后跑 `mvn -pl file -am test`，并在本地启动 File Service 确认 `RocketMQAutoConfiguration` 已生效（日志出现 Producer 启动记录）。 + +### 6.2 Topic 与消费组 + +| 项目 | 取值 | +|---|---| +| Topic | `${MQ_TOPIC_PREFIX}baiyue-file-export-command-v1` | +| Tag | `EXPORT_TASK` | +| Consumer Group | `${MQ_TOPIC_PREFIX}baiyue-file-export-worker-v1` | +| Producer Group | `${MQ_TOPIC_PREFIX}baiyue-file-export-publisher-v1` | +| Message Key | `taskId`（便于按任务在控制台检索） | +| 队列数 | 4 | +| 顺序要求 | 无。同一任务的串行由数据库状态与 `dispatch_version` 保证 | +| 投递语义 | At-least-once | +| 延迟消息 | **禁用**。延迟一律由 `file_export_outbox.next_retry_at` 承担（见 §6.3） | + +环境隔离用 `MQ_TOPIC_PREFIX` 前缀（测试 `test-`，生产空）而非 RocketMQ namespace，前缀在 4.x/5.x Broker 上行为一致。部署 Compose 必须显式传入该变量，并使用 `${MQ_TOPIC_PREFIX?...}` 校验“是否设置”而不是 `${MQ_TOPIC_PREFIX:-test-}`；后者会把生产显式空值错误替换为 `test-`。**禁止测试与生产共用 Topic 或消费组。** + +### 6.3 为什么禁用 MQ 延迟消息 + +RocketMQ 4.x 只支持 18 个固定延迟级别（1s/5s/10s/30s/1m/2m/3m/4m/5m/6m/7m/8m/9m/10m/20m/30m/1h/2h）。本文的退避序列（§9.2）含 15 分钟，不在级别表内；未来调整退避也会再次受限。 + +正确做法：Outbox 行自带 `next_retry_at`，Publisher 只领取 `next_retry_at <= now` 的行。延迟精度等于 Publisher 轮询周期（5 秒），任意间隔都可表达，且延迟状态在数据库里可查、可人工干预。 + +### 6.4 消息契约 + +契约类放入 `common-core` 的新包 `com.baiyue.common.core.export`（见 §21 决策 4，**不新建模块**）： + +```json +{ + "schemaVersion": 1, + "eventId": "3f2a7c18-5b41-4d0e-9a6b-2c8d5e1f7a90", + "taskId": "190000000000000001", + "exportType": "CUSTOMER_LIST", + "dispatchVersion": 1, + "occurredAt": "2026-08-03T18:00:00+08:00", + "traceId": "..." +} +``` + +约束： + +- `eventId` 由导出侧直接使用 `UUID.randomUUID().toString()` 生成；不复用 `common-event-contract` 中的 `EventIdentifiers`，避免 `common-core` 反向依赖跨区域事件契约模块。 +- `dispatchVersion` 仅在自动重试、人工重试或 Reconciler 恢复超时执行租约时递增；消息补投不递增。 +- Consumer 只处理与任务当前 `dispatch_version` 相同的消息；不一致即确认消费。 +- 消息不含筛选明细、客户数据、Token、文件内容、租户敏感信息。 +- 反序列化失败或 `schemaVersion` 不支持的消息**不确认**，令其走 MQ 原生重投直至 DLQ——这是 DLQ 的唯一合法来源。落 DLQ 后必须走 §9.4.2 的隔离流程，否则 Reconciler 会重发同一条坏 payload 形成无限循环。 +- Producer 必须同时把 `eventId`、`taskId`、`dispatchVersion`、`schemaVersion` 写入 RocketMQ User Properties。DLQ Handler 从属性定位任务与 Outbox，**不得依赖已经损坏的 JSON body**；属性值必须与 Outbox 列及 payload 中的同名字段一致。 + +## 7. 事务 Outbox + +### 7.1 为什么不能直接发送 MQ + +以下写法不可接受： + +```text +INSERT task -> COMMIT -> SEND MQ +``` + +数据库提交后进程退出，消息不会发送，任务永久停留。反过来先发 MQ 再提交数据库，Consumer 可能收到不存在的任务。 + +### 7.2 表结构 + +```sql +CREATE TABLE IF NOT EXISTS file_export_outbox ( + id BIGINT UNSIGNED NOT NULL COMMENT 'Outbox 记录ID（雪花ID）', + event_id VARCHAR(64) NOT NULL COMMENT '投递意图唯一标识（UUID）', + task_id BIGINT UNSIGNED NOT NULL COMMENT '关联导出任务ID', + dispatch_version INT NOT NULL COMMENT '任务派发版本，与任务表同名字段对应', + topic VARCHAR(128) NOT NULL COMMENT '目标Topic（含环境前缀）', + tag VARCHAR(64) NOT NULL COMMENT '消息Tag', + message_key VARCHAR(64) NOT NULL COMMENT '消息Key，取taskId', + payload_json JSON NOT NULL COMMENT '待发送命令消息体', + status VARCHAR(24) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/CLAIMED/SENT/QUARANTINED；无发送失败终态，QUARANTINED仅用于毒消息隔离，见7.4与9.4', + attempt_count INT NOT NULL DEFAULT 0 COMMENT '累计发送尝试次数', + next_retry_at DATETIME(3) NOT NULL COMMENT '允许发送的最早时间，承担全部延迟语义', + lease_owner VARCHAR(128) NULL COMMENT '领取实例标识', + lease_until DATETIME(3) NULL COMMENT '领取租约截止时间，超时可被其他实例重新领取', + sent_at DATETIME(3) NULL COMMENT '发送成功时间', + last_error VARCHAR(1000) NULL COMMENT '最后一次发送错误', + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '创建时间', + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '更新时间', + PRIMARY KEY (id), + UNIQUE KEY uk_file_export_outbox_event (event_id), + UNIQUE KEY uk_file_export_outbox_task_version (task_id, dispatch_version), + KEY idx_file_export_outbox_publish (status, next_retry_at, id), + KEY idx_file_export_outbox_task (task_id, dispatch_version, status, sent_at) +) ENGINE=InnoDB COMMENT='导出命令事务Outbox表：与任务变更同一本地事务写入，由Publisher可靠投递RocketMQ；不是操作日志'; +``` + +设计说明： + +- **`lease_owner` / `lease_until` 是必需的**，与既有 `bridge_outbox` 对齐。只有 `claimed_at` 时，Publisher 领取后崩溃会让该行永久停留在 `CLAIMED`，没有任何条件能重新领取它。 +- **本表不使用逻辑删除，不声明 `deleted` / `version` 字段。** 仓库 MyBatis-Plus 全局配置了 `logic-delete-field: deleted`，该配置只对声明了同名字段的实体生效，本表实体不声明即不受影响；清理走物理删除（§15）。 +- `uk_file_export_outbox_task_version` 是关键约束：它使“同一任务同一版本只会存在一条待发命令”成为数据库级事实，Reconciler 的补发因此天然幂等（重复插入报唯一键冲突并忽略）。 +- `idx_file_export_outbox_task` 专供 §8.5(1) 的 `NOT EXISTS` 子查询：该子查询按 `(task_id, dispatch_version)` 定位并读取 `status` 与 `sent_at`，走这个覆盖索引可避免回表。虽然唯一键已能定位，但补上 `status`/`sent_at` 让子查询完全走索引。 + +任务创建还需增加只有一行 `GLOBAL_CREATE` 的 `file_export_task_guard` 表，并由初始化 SQL 预置该行；File Service 启动时必须验证它存在，缺失则启动失败。创建事务只对该行执行一次 `SELECT ... FOR UPDATE`（禁止在请求内先 `INSERT IGNORE` 再升级锁，避免并发锁升级死锁），随后在同一事务内完成 60 秒重复提交检查、单用户活动任务上限、单类型活动任务上限以及 Task + Outbox 写入。当前创建量级极低（日任务 < 50），全局单行锁的等待成本可以忽略，却能避免多个实例并发请求同时通过“先 COUNT、后 INSERT”而突破 3/5 上限；该锁只串行化任务创建，不影响 Publisher、Consumer 或下载。 + +### 7.3 任务表字段调整（直接改初始化 SQL） + +**当前处于开发阶段，不产出增量迁移脚本。** 做法：直接在 `file/src/main/resources/db/file.sql` 的 `file_export_task` 建表语句中补齐新字段与索引，测试环境先执行新增 SQL 使其生效，本模块的初始化 SQL 即为唯一事实来源。 + +这样选择的前提与代价： + +- 前提：导出功能尚未有生产数据，测试环境的任务表可以直接重建，不存在必须保留的存量任务。 +- 代价：已经跑过旧版 `file.sql` 的环境不会自动升级。因此测试环境在只读确认任务数为 0 后，按依赖顺序执行 **`DROP TABLE file_export_outbox; DROP TABLE file_export_task_guard; DROP TABLE file_export_task;`，再完整执行 `file.sql`**，而不是 `ALTER TABLE`。具体核验 SQL 见 `deploy/sql/README.md`。 +- 一旦进入生产、表内有真实数据，后续任何字段变更就必须回到 AGENTS.md 要求的可重复执行增量脚本模式（`information_schema` 判断 + `ALTER TABLE`），本节的简化做法**仅适用于当前开发阶段**。 + +`file_export_task` 在现有字段基础上补入以下 7 个字段（位置放在 `failure_message` 之后、`requested_by` 之前，保持读表时调度字段聚集）： + +```sql + dispatch_version INT NOT NULL DEFAULT 1 COMMENT '派发版本，仅自动重试、人工重试或超时执行租约恢复时递增；Consumer只处理版本一致的消息', + attempt_count INT NOT NULL DEFAULT 0 COMMENT '已执行尝试次数', + max_attempts INT NOT NULL DEFAULT 3 COMMENT '自动重试上限，创建任务时按注册表写入', + next_retry_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '允许执行的最早时间，承担全部退避语义', + lease_owner VARCHAR(128) NULL COMMENT '当前执行实例标识', + lease_until DATETIME(3) NULL COMMENT '执行租约截止时间，超时由Reconciler回收', + last_heartbeat_at DATETIME(3) NULL COMMENT '最后心跳时间', +``` + +并在该表的索引区补入两个索引： + +```sql + KEY idx_file_export_dispatch (status, next_retry_at, id), + KEY idx_file_export_lease (status, lease_until) +``` + +同时把 §7.2 的 `file_export_outbox` 建表语句一并加入同一个 `file.sql`，与任务表相邻放置，便于阅读时看到两者的对应关系。 + +**存量 PENDING 任务不做数据迁移**：表是重建的，没有存量。若测试环境选择保留数据，则这些任务的 `next_retry_at` 会取默认当前时间、`dispatch_version` 取 1，Reconciler 会在 30 秒内为它们补发 Outbox 命令并正常执行——无需任何一次性 SQL。 + +### 7.4 Publisher 可靠性 + +轮询周期 5 秒，每轮最多领取 50 条。领取分两步，避免 `UPDATE ... ORDER BY ... LIMIT` 的锁范围问题： + +```text +1) SELECT id FROM file_export_outbox + WHERE status = 'PENDING' AND next_retry_at <= CURRENT_TIMESTAMP(3) + ORDER BY next_retry_at, id LIMIT 50 + +2) 逐条条件更新： + UPDATE file_export_outbox + SET status = 'CLAIMED', lease_owner = :instance, + lease_until = CURRENT_TIMESTAMP(3) + INTERVAL :leaseSeconds SECOND, + attempt_count = attempt_count + 1 + WHERE id = :id AND status = 'PENDING' + -- 影响行数 0 表示已被其他实例领取，跳过 +``` + +发送与结果处理： + +- 发送成功：`status = SENT`，`sent_at = CURRENT_TIMESTAMP(3)`，清空 `lease_owner` / `lease_until`。 +- 发送失败：`status = PENDING`，`next_retry_at = CURRENT_TIMESTAMP(3) + backoff`，记录 `last_error`。退避 5s / 15s / 1m / 5m / 15m，之后**固定 15m 无限重试**。**不得无间隔死循环，也不得放弃。** +- 发送成功但更新数据库前进程退出：租约到期后另一实例重新领取并重发，产生重复消息，由 Consumer 幂等吸收。 + +**不设发送失败终态，Outbox 行永不因发送失败而放弃。** 第一版设计曾规定“`attempt_count > 20` 置 `DEAD`，仅人工恢复”，这与 §8.5 的自动补投逻辑直接冲突，且会导致：Broker 长期不可用 → 所有 Outbox 行耗尽次数变 `DEAD` → 若允许自动换版本重建则无限循环（任务 `attempt_count` 从不增长，`max_attempts` 拦不住）；若不允许则**所有任务静默停滞**，违背“数据库提交成功的任务最终一定被调度”这一核心目标（§2.1）。 + +取而代之：发送失败无限退避重试，靠**告警**而非状态机来驱动人工介入。`export_outbox_oldest_age_seconds > 120` 即告警（§13），Broker 故障期间告警持续存在，运维必然介入；Broker 恢复后所有积压命令自动发出，无需任何人工操作。 + +`status` 共四个值，其中只有 `QUARANTINED` 是终态，且**只用于毒消息隔离，绝不用于发送失败**（§9.4）： + +| 值 | 含义 | 谁写入 | Reconciler 是否补投 | +|---|---|---|---| +| `PENDING` | 待发送 | 创建 / 发送失败退避 / Reconciler 恢复 | — | +| `CLAIMED` | 已被某实例领取，发送中 | Publisher | 租约超时后释放回 `PENDING` | +| `SENT` | 已成功投递到 Broker | Publisher | 超时未执行则改回 `PENDING` 重发 | +| `QUARANTINED` | **消息内容损坏，永久隔离** | DLQ Handler 自动处置 | **否，永久排除** | + +`attempt_count` 仅用于计算退避档位与观测，不再作为放弃条件。 + +**`attempt_count` 的语义是“本 Outbox 行的发送尝试次数”，与任务的 `attempt_count`（执行尝试次数）是两个独立计数器，不可混用。** 发送重试不消耗任务的执行次数（§3 原则 5）。任务重试会写入**新的** Outbox 行，其 `attempt_count` 从 0 开始。 + +退避时长由 `attempt_count` 查表得出，超出表长即取末位（15m）。退避计算基于领取时已递增的值，避免首次失败就跳过 5s 档。 + +Publisher 只负责投递几十字节的小消息，不执行导出，不受导出耗时影响。 + +**Publisher 与 Consumer 必须使用不同的 Spring Bean，且 Producer 的 `sendMessageTimeout`（3 秒）远小于 Outbox 租约（60 秒）**，保证发送调用不会跨越租约边界造成重复领取。 + +## 8. Consumer 幂等、租约与背压 + +### 8.1 执行模型：消费线程同步执行 + +Consumer 采用 `MessageModel.CLUSTERING`、`consumeMessageBatchMaxSize = 1`、`consumeThreadMin = consumeThreadMax = 4`，**消费线程直接同步执行导出**，不再投递到额外线程池。 + +这样选择的原因与代价： + +- 消费线程数即单个 File Service 实例的导出并发上限（4）；超出的任务在 Broker 队列中等待——这正是 MQ 的正当用途，不需要自己实现背压和“线程池满了怎么办”。 +- 并发消费模式下，一个队列可以向同一实例的多个消费线程提交不同批次，**Topic 的 4 个队列不是“最多 4 个线程”的硬上限**。队列数主要决定消费组内实例间的分配粒度；当前单实例部署使用 4 个队列足够，未来扩 File 实例时应保证队列数不少于需要获得分配的实例数，并通过压测决定是否继续扩队列。 +- 长任务会占住消费线程。因此 `consumeTimeout` 必须大于最大任务时长：**最大任务时长 30 分钟，`consumeTimeout` 设 90 分钟**。否则 Broker 判定消费超时并重投，虽有幂等兜底但会产生无意义的抢占尝试。 +- `maxReconsumeTimes = 2`。**仅对毒消息生效**：§8.2 的全部抢占失败分支都返回 `CONSUME_SUCCESS`，业务上永不触发重投；唯一不确认消费的路径是消息反序列化失败（§6.4），该路径重投 2 次后落 DLQ。此参数与抢占竞争无关——第一版文案称其"吸收抢占竞争"是错的，抢占竞争根本不会走到重投。 +- 不使用独立线程池，因此不存在“已抢占但无线程执行、任务卡到租约过期”这个失败模式。 + +导出执行不占用 Servlet 请求线程（MQ 消费线程独立），满足资源隔离要求。 + +### 8.2 原子抢占 + +Consumer 收到命令后执行条件更新（时间一律取自数据库，见 §8.6）： + +```sql +UPDATE file_export_task + SET status = 'RUNNING', + attempt_count = attempt_count + 1, + started_at = CURRENT_TIMESTAMP(3), + lease_owner = :instance, + lease_until = CURRENT_TIMESTAMP(3) + INTERVAL :leaseSeconds SECOND, + last_heartbeat_at = CURRENT_TIMESTAMP(3), + processed_rows = 0, + total_rows = NULL, + failure_message = NULL + WHERE id = :taskId + AND dispatch_version = :dispatchVersion + AND status = 'PENDING' + AND next_retry_at <= CURRENT_TIMESTAMP(3) +``` + +`started_at` 每次抢占都覆盖为当前时间（不用 `COALESCE` 保留首次值），因为 §9.5 规定文件内容对应"最后一次执行开始时刻"的快照，`started_at` 必须与之一致。任务创建时间由 `requested_at` 保留。 + +`processed_rows` / `total_rows` / `failure_message` 在抢占时一并重置，保证下载中心不会显示上一轮的残留进度。 + +**影响行数为 0 时，必须按下表穷尽处理。所有分支一律 `CONSUME_SUCCESS`，无一例外：** + +| 重新读取任务后的状态 | 判定 | 处理 | +|---|---|---| +| 任务不存在 | 极旧消息或数据已清理 | 确认消费，记 WARN | +| `dispatch_version` 大于消息中的值 | 旧版本消息 | 确认消费 | +| `SUCCEEDED` / `FAILED` / `EXPIRED` | 已终结 | 确认消费 | +| `RUNNING` 且 `lease_until > now` | 另一实例正在处理 | 确认消费。**不重投**，由该实例继续；若其崩溃则 Reconciler 恢复 | +| `RUNNING` 且 `lease_until <= now` | 租约已过期 | 确认消费。**不在此处抢占**，交由 Reconciler 统一恢复（见 §8.5） | +| `PENDING` 且 `next_retry_at > now` | 处于退避窗口，命令提前到达 | 确认消费。**不做任何补救**：Publisher 会在到点后发出该版本的命令；即使那条也丢了，Reconciler 的 `SENT` 超时窗口会兜底 | +| `PENDING` 且 `next_retry_at <= now` 但更新仍为 0 | 与其他实例竞争失败 | 确认消费 | + +最后两行是原设计缺失的分支。尤其“`PENDING` 且退避未到”——自动重试会把任务置为 `PENDING` 且 `next_retry_at = CURRENT_TIMESTAMP(3) + 15min`，此时若有重复消息导致命令提前到达，就会落入这个分支。正常流程下不会发生：§9.2 保证新 Outbox 行的 `next_retry_at` 等于任务的 `next_retry_at`，Publisher 到点才发。 + +**这两个分支都只确认消费、不做补救，是刻意的。** 任何“抢不到就重投/就自己补发”的补救逻辑都会与 Reconciler 的重派职责重叠，形成两条并行的重派通道，进而产生版本竞争。§3 原则 5 的“Reconciler 是唯一重派通道”正是这个意思。 + +### 8.3 租约续期：独立 watchdog，覆盖全执行期 + +**续租必须由独立的 watchdog 线程按固定周期执行，不能挂在 Provider 批次循环里。** + +第一版设计把续租放在“每完成一个批次”之后，存在一个致命盲区：**最后一批数据拉完之后，还有 `SXSSFWorkbook.write()`、ZIP 压缩、大文件落盘这些耗时操作，期间一次批次都不会完成，续租因此完全停止。** 20 万行的 XLSX 压缩落盘在慢磁盘上完全可能超过 300 秒租约，导致任务被 Reconciler 判为失联、剥夺租约并重派——而此时它其实马上就要成功了。结果是任务被反复重跑，每次都在最后一步被杀。 + +正确做法：Consumer 抢占成功后启动一个 watchdog，**在整个执行期间**（含 Provider 拉取、XLSX 写入、压缩、落盘、结果提交前的全部阶段）每 60 秒续租一次： + +```sql +UPDATE file_export_task + SET lease_until = CURRENT_TIMESTAMP(3) + INTERVAL :leaseSeconds SECOND, + last_heartbeat_at = CURRENT_TIMESTAMP(3) + WHERE id = :taskId AND status = 'RUNNING' + AND dispatch_version = :dispatchVersion AND lease_owner = :instance +``` + +进度上报（`processed_rows`）仍在批次之间单独更新，与续租解耦——**进度是展示用的，租约是正确性用的，两者不能共用一次写入**。 + +watchdog 的生命周期与约束： + +- 抢占成功后启动，`finally` 块中确保关闭（无论成功、失败、异常）。 +- 续租周期 60 秒，租约 300 秒 → 为 watchdog 发现异常、通知执行线程并安全中止预留约 240 秒租约余量；续租异常仍按下一条立即触发中止，不能据此忽略连续失败。 +- **续租返回 0 行时立即置中止标志**，执行主线程在每个可中断点（批次边界、写入循环内）检查该标志，一旦置位就中止导出、删除半成品文件、放弃提交。 +- watchdog 自身异常不得静默吞掉，必须记 ERROR 并置中止标志——续租线程死了却继续执行，等于失去租约保护。 + +### 8.3.1 执行截止时间：watchdog 不能让任务无限跑 + +**watchdog 会无条件续租，因此它同时消除了"任务卡死自然超时"这一兜底。** §14 声明的"最大任务时长 30 分钟"若没有独立的检查机制，就只是一句无法生效的声明：一个卡在慢 Provider 上的任务会被 watchdog 一直续租下去，占着消费线程直到 `consumeTimeout`（90 分钟）才被 Broker 判超时——而那时它已白占了一个并发位一个半小时。 + +因此抢占成功时必须计算并持久化执行截止时间。在 §8.2 的抢占 SQL 中增加一列： + +```sql + execution_deadline_at = CURRENT_TIMESTAMP(3) + INTERVAL :maxDurationMinutes MINUTE, +``` + +对应 `file_export_task` 增加字段（与 §7.3 的 7 个字段一并落库，共 8 个）： + +```sql + execution_deadline_at DATETIME(3) NULL COMMENT '本次执行的截止时间，超过即判超时失败；抢占时按最大任务时长计算', +``` + +**deadline 必须在三处检查**，缺一不可： + +| 检查点 | 原因 | +|---|---| +| 每个 Provider 批次开始前 | 最常见的超时来源是数据量过大、批次过多 | +| XLSX 写入循环内（每 N 行，如每 10000 行） | 单批数据量大时，写入本身可能耗时很久 | +| 结果提交前（§8.4） | 防止刚好在 deadline 边界完成却仍提交，导致时长统计失真 | + +超过 deadline 按**临时依赖错误**处理（走 §9.2 自动重试），而非永久失败——首次超时可能是 Provider 偶发慢，重试有意义。但 `failure_message` 必须写明"执行超过最大时长 30 分钟"，便于区分真正的超时与其他临时错误。若重试到 `max_attempts` 仍超时，说明数据量确实超出能力，转 `FAILED`，此时应考虑调大 `max-duration-minutes` 或调小 `max-rows`。 + +**deadline 用数据库时间计算与比较**（§8.6）。检查时不要在 Java 端拿 `LocalDateTime.now()` 和它比——用 watchdog 每次续租的返回值携带"是否已超 deadline"，或在续租 SQL 中顺带判断： + +```sql +UPDATE file_export_task + SET lease_until = CURRENT_TIMESTAMP(3) + INTERVAL :leaseSeconds SECOND, + last_heartbeat_at = CURRENT_TIMESTAMP(3) + WHERE id = :taskId AND status = 'RUNNING' + AND dispatch_version = :dispatchVersion AND lease_owner = :instance + AND execution_deadline_at > CURRENT_TIMESTAMP(3) -- 超时则续租失败 +``` + +这样 deadline 与租约共用一次写入：**超过 deadline 后续租自然失败 → 中止标志置位 → 执行主线程在下一个可中断点退出**。不需要额外的时间比较逻辑，也不会因实例时钟偏差而误判。 + +注意这个复用有一个后果：超 deadline 后租约不再续期，300 秒后 Reconciler 会按"租约超时"路径恢复它（§8.5-2，递增版本重试）。这与"超时按临时错误重试"的结论一致，因此**即使 Consumer 侧的中止逻辑因故没跑，Reconciler 也会兜底得到相同结果**——两条路径殊途同归，这是刻意的冗余。 + +租约、续租与超时的约束关系（实施时必须同时满足）： + +```text +续租周期(60s) << 租约(300s) << consumeTimeout(90min) +Provider 读超时(15s) << 租约(300s) +最大任务时长(30min) < consumeTimeout(90min) +``` + +有了 watchdog，Provider 读超时不再是租约正确性的**唯一**保障（续租独立于批次），但仍必须按 §14 配置——否则单批可以无限挂起，任务会一直占着消费线程直到 `consumeTimeout`。 + +### 8.4 结果提交 + +文件写入完成后： + +```sql +UPDATE file_export_task + SET status = 'SUCCEEDED', file_name = ..., object_key = ..., content_type = ..., + size_bytes = ..., total_rows = ..., processed_rows = ..., + completed_at = CURRENT_TIMESTAMP(3), + expires_at = CURRENT_TIMESTAMP(3) + INTERVAL :retentionDays DAY, + lease_owner = NULL, lease_until = NULL, failure_message = NULL + WHERE id = :taskId AND status = 'RUNNING' + AND dispatch_version = :dispatchVersion AND lease_owner = :instance + AND execution_deadline_at > CURRENT_TIMESTAMP(3) +``` + +影响行数为 0 时（版本已变、租约被剥夺或提交时已越过 deadline），**必须删除本次生成的孤立文件**；若仍是本实例持有的同版本 `RUNNING` 且 deadline 已过，则走 §9.2 超时重试，否则只记 WARN 并确认消息。deadline 条件必须进入最终 SQL，不能只在 Java 中预检查，否则检查后跨越截止时间仍可能错误提交成功。 + +### 8.5 Reconciler：唯一重派通道 + +轮询周期 30 秒。四项职责，每项都是幂等的条件更新： + +**(1) 补投疑似丢失的命令** —— **恢复原 Outbox 行重发，绝不递增任务版本**（§3 原则 5）。 + +分两种情形，分别处理： + +**情形 A：当前版本没有任何 Outbox 行**（写入丢失或被误删）。补写一条**同版本**的 Outbox 行： + +```sql +SELECT t.id, t.dispatch_version + FROM file_export_task t + WHERE t.status = 'PENDING' + AND t.next_retry_at <= CURRENT_TIMESTAMP(3) - INTERVAL :redispatchGraceSeconds SECOND + AND NOT EXISTS (SELECT 1 FROM file_export_outbox o + WHERE o.task_id = t.id AND o.dispatch_version = t.dispatch_version) + LIMIT 50 +``` + +插入时 `dispatch_version` 取任务当前值，`event_id` 新生成。唯一键 `uk_file_export_outbox_task_version` 保证并发下只有一条插入成功，其余报冲突并忽略——天然幂等。 + +注意此处 `NOT EXISTS` 不带 `status` 过滤，因此 `QUARANTINED` 行也会阻止补写——这正是需要的：毒消息隔离后不允许再产生同版本命令。 + +**情形 B：Outbox 行已 `SENT` 很久，任务仍是 `PENDING`**（消息大概率已被 Broker 丢弃）。把该行**条件性恢复为 `PENDING` 重发同一版本**： + +```sql +UPDATE file_export_outbox o + JOIN file_export_task t + ON t.id = o.task_id AND t.dispatch_version = o.dispatch_version + SET o.status = 'PENDING', + o.next_retry_at = CURRENT_TIMESTAMP(3), + o.lease_owner = NULL, + o.lease_until = NULL + WHERE o.status = 'SENT' + AND o.sent_at <= CURRENT_TIMESTAMP(3) - INTERVAL :sentStaleSeconds SECOND + AND t.status = 'PENDING' + AND t.next_retry_at <= CURRENT_TIMESTAMP(3) - INTERVAL :redispatchGraceSeconds SECOND + LIMIT 20 +``` + +`WHERE o.status = 'SENT'` 天然排除了 `QUARANTINED` 行——**这是阻断毒消息循环的关键**，见 §9.4。 + +**关键：任务版本不动，Outbox 行也不换版本。** Publisher 下一轮会重发同一条命令，Consumer 收到后正常抢占执行。 + +**为什么这样就不会饿死排队任务**：假设任务因并发已满在 Broker 队列中正常等待，Reconciler 在第 10 分钟把 Outbox 行恢复为 `PENDING` 并重发。此时队列里有**两条同版本命令**： + +- 若原命令先被消费 → 正常抢占执行；随后重复命令看到 `RUNNING` → 确认消费（§8.2 第 4 行）。 +- 若重复命令先被消费 → 同样正常抢占执行；原命令看到 `RUNNING` → 确认消费。 + +**两条命令都是有效的，谁先到谁执行。** 对比第一版“递增版本”的做法：原命令会因版本过期被丢弃，新命令排到队尾，持续负载下任务永远轮不上——这正是被修掉的活性错误。 + +代价：正常排队场景下会产生一条无用的重复消息。这是刻意的取舍——重复消息由幂等吸收，成本仅是一次条件更新；而漏补一条真丢的消息会让任务永久停滞。**宁可多投，不可不投。** + +`sentStaleSeconds` 默认 600 秒。由于重复投递现在是无害的，这个值不再需要精确避开正常排队时长，可以取得比较激进；但也不宜过小，否则积压时会放大 Broker 队列长度。 + +两种情形都**不判断 `attempt_count`**：补投不是新的执行尝试，不受 `max_attempts` 约束。执行次数的闸门在 §8.2 抢占成功时递增、在 §9.2 与 §8.5(2) 判定，与补投无关。 + +**(2) 恢复超时租约**： + +```sql +SELECT id, dispatch_version, attempt_count, max_attempts, object_key + FROM file_export_task + WHERE status = 'RUNNING' AND lease_until <= CURRENT_TIMESTAMP(3) LIMIT 20 +``` + +同一事务内：若 `attempt_count < max_attempts` 则置 `PENDING`、`next_retry_at = CURRENT_TIMESTAMP(3)`、**`dispatch_version + 1`**、清空租约、写新 Outbox；否则置 `FAILED` 并写明“执行实例失联且已达最大尝试次数”。半成品文件按 §9.5 清理。 + +**这里递增版本是正确的**，与 §8.5(1) 的补投不同：租约超时意味着上一次**执行**确实失败了（实例已死或卡死），需要作废那次尝试并开始新尝试——正好符合 §3 原则 5 中“执行失败才递增”的定义。递增后旧实例若“复活”也无法再提交（§8.4 的条件带版本号）。 + +**(3) 释放僵死 Outbox**：`status = 'CLAIMED' AND lease_until <= CURRENT_TIMESTAMP(3)` → 置回 `PENDING`，`next_retry_at = CURRENT_TIMESTAMP(3)`。这是投递侧恢复，**不动任务版本**。 + +**(4) 过期与孤立文件清理**：沿用现有 `expiredArtifacts` + `markExpired` 逻辑（先删文件、删成功才置 `EXPIRED`、失败保持原状并告警），另扫描 `exports/` 下无对应 `SUCCEEDED` 任务且创建超过 24 小时的文件作为孤立文件删除。 + +Reconciler 的所有分支都只做“把任务重新变成可派发的 PENDING + 写 Outbox”或“恢复 Outbox 行重发”，**从不直接执行导出，也从不因投递问题递增任务版本**。 + +### 8.6 时间基准：一律使用数据库时钟 + +**所有与租约、退避、超时判断相关的时间，必须在 SQL 中使用 `CURRENT_TIMESTAMP(3)`，不得由 Java 端传入 `LocalDateTime.now()`。** + +这一条覆盖：抢占的 `next_retry_at <= ?`、租约的 `lease_until` 写入与比较、Outbox 的 `next_retry_at` 与 `sent_at`、Reconciler 的全部时间条件。 + +原因：多实例部署时各容器的系统时钟存在偏差（NTP 同步误差、容器时钟漂移，秒级到分钟级都可能）。若用本地时钟： + +- 时钟快的实例会**提前抢占**：它认为 `lease_until` 已过期，实际上持有者还在正常执行 → 两个实例同时写同一任务。 +- 时钟慢的实例会**延迟恢复**：真正失联的任务迟迟不被回收。 +- 退避窗口在不同实例上长短不一，`next_retry_at` 失去意义。 + +数据库是全局唯一的时间基准，所有实例读到的 `CURRENT_TIMESTAMP(3)` 一致，上述问题全部消失。 + +例外：仅用于日志输出、指标打点、文件名生成的时间可以用本地时钟——它们不参与正确性判断。`file_name` 中的时间戳即属此类。 + +现有代码中 `ExportTaskRepository` 的 `claim(id, LocalDateTime.now())`、`requeueStale(cutoff)` 等签名都是 Java 端传时间，**改造时必须一并改为 SQL 内取时间**，这是从当前实现迁移时容易遗漏的一处。 + +## 9. 重试与 DLQ + +### 9.1 错误分类 + +| 类型 | 示例 | 行为 | +|---|---|---| +| 永久业务错误 | 参数无效、导出类型已停用、列定义不合法、`schemaHash` 批次间不一致、超出 `max-rows` | 立即 `FAILED`，不自动重试 | +| 临时依赖错误 | Provider 503 / 连接超时 / 读超时、存储短暂不可用、游标未推进 | 走 §9.2 自动重试 | +| Consumer 非预期退出 | JVM 退出、实例被杀、心跳失败 | 租约超时 → Reconciler 恢复（§8.5-2） | +| MQ 投递失败 | NameServer / Broker 不可用 | Outbox 退避重发（§7.4） | +| 消息本身损坏 | 反序列化失败、`schemaVersion` 不支持 | 不确认消费 → MQ 原生重投 → DLQ → **§9.4.2 隔离（Outbox 置 `QUARANTINED` + 任务置 `FAILED`）** | + +**只有最后一类会进入 DLQ。** 这是“DLQ 出现任意新消息即告警”能作为有效告警规则的前提。 + +### 9.2 自动重试 + +`max_attempts = 3`（即最多 2 次自动重试）。退避 1 分钟、15 分钟，含 ±20% 随机抖动。退避由数据库承担，任意间隔均可表达。 + +临时失败时，Consumer 在**同一数据库事务**内： + +1. 删除本次半成品文件（见 §9.5），置 `object_key = NULL`； +2. 若 `attempt_count >= max_attempts` → `status = FAILED`，写 `failure_message`，事务结束； +3. 否则 `status = PENDING`、`next_retry_at = CURRENT_TIMESTAMP(3) + backoff`、`dispatch_version + 1`、`processed_rows = 0`、`total_rows = NULL`、清空 `lease_owner` / `lease_until`、保留 `failure_message` 供下载中心展示上次原因； +4. 写入新 Outbox 行，其 `next_retry_at` **等于任务的 `next_retry_at`**（这是 Publisher 不提前发送的保证）。 + +事务提交后确认 MQ 消息。所有重试状态在数据库中可查，不依赖客户端内存，不引入 Redis 重试队列。 + +条件更新同样要带 `dispatch_version = :v AND lease_owner = :instance`；失败则说明租约已被剥夺，放弃写入。 + +### 9.3 人工重试 + +下载中心人工重试只允许 `FAILED`（保持现有 `ExportTaskService.retry()` 的约束），在同一事务内： + +- 重新校验登录身份、租户、访问域与注册表中的 `required-permission`； +- 校验导出类型当前 `enabled = true`（停用类型不得重试）； +- 清空 `failure_message`，`attempt_count = 0`，`processed_rows = 0`，`total_rows = NULL`，`started_at = NULL`，`completed_at = NULL`； +- 删除上次遗留文件并置 `object_key = NULL`； +- `status = PENDING`，`next_retry_at = CURRENT_TIMESTAMP(3)`，`dispatch_version + 1`； +- 写入新 Outbox 行。 + +### 9.4 DLQ 与毒消息隔离 + +#### 9.4.1 为什么必须隔离：否则形成无限毒消息循环 + +确立 §3 原则 5（补投重发同版本）后，出现了一个新的循环路径，**必须显式阻断**： + +```text +消息 payload 损坏（反序列化失败） + -> Consumer 不确认消费 + -> MQ 重投 2 次仍失败 -> 落 DLQ + -> 任务仍是 PENDING，Outbox 行仍是 SENT + -> 超过 sentStaleSeconds 后 Reconciler 判定"消息丢失" + -> 把同一条损坏 payload 改回 PENDING 重发 ← 循环点 + -> 再次落 DLQ ... +``` + +每 10 分钟循环一次，DLQ 持续增长，告警持续触发，且**永远不会自愈**——因为 payload 存在数据库里，重发多少次都是同样的坏数据。Reconciler 的"重发同版本"对 Broker 丢消息是正确的，对内容损坏则是灾难。 + +根因：§8.5 的补投无法区分"消息丢了"和"消息到了但读不懂"。两者在数据库侧的表征完全相同（任务 `PENDING` + Outbox `SENT`），必须由 DLQ 处置流程显式打标。 + +#### 9.4.2 隔离机制 + +新增 Outbox 终态 `QUARANTINED`，**只用于消息内容损坏，绝不用于 Broker 发送失败**（后者永不放弃，§7.4）。 + +File Service 增加独立 DLQ Handler，使用专用消费组订阅 `%DLQ%`。Handler 接收原始 `MessageExt`，从 User Properties 读取 `eventId` / `taskId` / `dispatchVersion` / `schemaVersion`，在下列事务成功提交后才确认 DLQ 消息： + +```sql +-- 1) 隔离该版本的 Outbox 行，阻断 Reconciler 补投 +UPDATE file_export_outbox + SET status = 'QUARANTINED', + last_error = CONCAT('DLQ隔离: ', :reason) + WHERE event_id = :eventId + AND task_id = :taskId AND dispatch_version = :dispatchVersion + AND status IN ('SENT', 'PENDING', 'CLAIMED'); + +-- 2) 同一事务内把对应任务置为 FAILED，让用户在下载中心看到明确失败 +UPDATE file_export_task + SET status = 'FAILED', + failure_message = '导出命令消息损坏，请重新发起导出', + completed_at = CURRENT_TIMESTAMP(3), + lease_owner = NULL, lease_until = NULL + WHERE id = :taskId AND dispatch_version = :dispatchVersion + AND status = 'PENDING'; +``` + +两步必须在**同一事务**内完成，并断言两条更新各影响 1 行。只做第 1 步会让任务永久停在 `PENDING`（Reconciler 已被阻断，无人再驱动它），对用户表现为"一直排队"——比循环更难排查。若任务版本已经变化或 Outbox `event_id` 不匹配，说明是旧消息或外部误投，记录审计后直接确认，不得隔离当前版本。若新协议消息缺少任一定位属性，DLQ Handler 记 CRITICAL、保存 Broker messageId 与原始属性供人工排查并确认消息，避免 DLQ Handler 自身形成二级死信循环。 + +隔离后的恢复路径：**修复消息契约或代码后，由用户或运维走人工重试**（§9.3）。人工重试递增 `dispatch_version` 并写入**新版本**的 Outbox 行，其 payload 由新代码生成，与被隔离的旧版本无关。旧的 `QUARANTINED` 行永久保留作为审计痕迹，因唯一键是 `(task_id, dispatch_version)` 而版本已变，不会冲突。 + +#### 9.4.3 运维要求 + +- Consumer Group 的 DLQ Topic 为 `%DLQ%`，需在部署脚本中显式创建并授予读写权限（§16.2）。 +- DLQ 出现任意消息即告警，因为按 §9.1 它只承载损坏消息。 +- DLQ Handler 必须自动执行 §9.4.2 的隔离；运维收到告警后只负责分析原因与决定是否人工重试。禁止依赖值班人员在 10 分钟补投窗口内手工运行 SQL。 +- DLQ Handler 的数据库处置失败必须持续重试（`maxReconsumeTimes = Integer.MAX_VALUE`），不能在有限次数后进入无人消费的 `%DLQ%` 二级 DLQ；缺少或非法定位属性的消息仍按 §9.4.2 记录 CRITICAL 后确认。 +- 重投前核对任务当前 `dispatch_version`；版本已变说明任务已通过新版本重试，旧消息只允许丢弃。 +- 不允许静默丢弃：每次处置需在运维记录中写明 `taskId`、`eventId`、`dispatchVersion`、损坏原因、处置方式。 +- **任务不会因为消息进 DLQ 而丢失**：要么被隔离流程显式置为 `FAILED`（用户可见、可人工重试），要么在未隔离期间由 Reconciler 补投。两种结局都不是静默停滞。 + +#### 9.4.4 毒消息的真实来源与预防 + +按 §6.4，消息只含 `eventId` / `taskId` / `exportType` / `dispatchVersion` / `occurredAt` / `traceId`，全是定长标识，**正常情况下不可能损坏**。真实来源只有两类： + +1. **消息契约不兼容变更**：如把 `taskId` 从字符串改成数字、删字段、改 `schemaVersion` 语义，而新旧版本 File Service 在滚动发布期间并存。这是最可能的来源。 +2. 手工向 Topic 投递了格式错误的消息（测试误操作）。 + +预防措施：`schemaVersion` 只增不改语义；新增字段必须可选；删除字段需跨两个发布周期（先停用后删除）。滚动发布期间新旧 Consumer 并存是常态，契约必须双向兼容。 + +### 9.5 半成品文件与快照语义 + +- 每次执行从零重写整个文件，不做断点续传。`SXSSFWorkbook` 的临时文件在 `try-with-resources` 中释放。 +- 任何失败路径（永久失败、临时失败、租约丢失、提交竞争失败）都必须删除本次 `object_key` 指向的文件。删除失败只记 WARN，由 Reconciler 的孤立文件扫描兜底。 +- **`snapshotToken` 不跨 `dispatch_version` 复用。** 每一轮执行重新向 Provider 申请首批并获得新 Token。原因：退避 15 分钟后旧 Token 很可能已失效；且新 Token 语义清晰——文件内容对应“最后一次执行开始时刻”的数据快照。 +- 因此下载中心对重试成功的任务应理解为：数据以最后一次执行时刻为准，而非任务创建时刻。任务详情返回的 `startedAt` 即该时刻。 + +## 10. 统一 Business Export Provider 契约 + +### 10.1 接入形态 + +新业务服务不接公共导出 MQ，只提供一个标准内部 HTTP Provider。契约类型放入 `common-core` 的 `com.baiyue.common.core.export` 包，避免每个服务复制 DTO。 + +```http +POST /internal/exports/v1/{exportType}/batches +X-Internal-Service-Token: ... +Content-Type: application/json +``` + +请求： + +```json +{ + "taskId": "190000000000000001", + "tenantId": "0", + "appCode": "PLATFORM", + "requestedBy": "1900000000000000009", + "dispatchVersion": 1, + "parameters": { + "keyword": "柏悦", + "status": "ACTIVE" + }, + "cursor": null, + "snapshotToken": null, + "limit": 500 +} +``` + +**`tenantId`、`appCode`、`requestedBy` 是必填的，这是原设计的严重缺口。** 现有 `file_export_task` 已保存这三项，但原契约未下传，Provider 无法按租户过滤。当前只有平台侧 `CUSTOMER_LIST` 所以未暴露，一旦出现 `app_code = MERCHANT` 的导出类型就会越权导出。AGENTS.md 明确要求“商户数据必须保留租户维度，并由服务端执行租户、访问域、数据范围和按钮权限校验”。 + +Provider 侧强制要求： + +- 必须校验 `X-Internal-Service-Token`，与 `INTERNAL_SERVICE_TOKEN` 一致才响应； +- 必须用 `tenantId` + `appCode` 约束查询，**不得信任 `parameters` 中的任何租户字段**； +- `appCode = MERCHANT` 时必须额外按 `requestedBy` 的数据范围过滤； +- 只读，不得产生副作用。 + +首批响应： + +```json +{ + "code": "200", + "message": "success", + "data": { + "schemaVersion": 1, + "schemaHash": "sha256:...", + "snapshotToken": "provider-defined-opaque-token", + "columns": [ + { "key": "tenantCode", "title": "客户编码", "type": "STRING", "width": 18 }, + { "key": "tenantName", "title": "公司名称", "type": "STRING", "width": 28 } + ], + "rows": [ + { "tenantCode": "TEN101", "tenantName": "柏悦物流" } + ], + "nextCursor": "101", + "hasMore": false, + "totalRows": 1 + } +} +``` + +响应遵循仓库统一约定：HTTP 200 + `code == "200"` 才成功，`Long` 输出为字符串。后续批次必须返回相同 `schemaHash`，可省略 `columns`；`schemaHash` 变化即视为永久业务错误并 `FAILED`（说明 Provider 在导出期间变更了列定义）。 + +`hasMore = true` 但 `nextCursor` 为空或未推进时，File Service 判定为临时依赖错误并终止本轮（沿用现有 `CustomerExportTaskHandler` 的游标推进校验）。 + +### 10.2 稳定快照 + +仅用递增 ID 游标会把导出期间新增的数据混入结果。Provider 首批必须返回不透明 `snapshotToken`，后续批次带回同一 Token。 + +Provider 可将 Token 实现为签名后的最大 ID、截止时间或数据库快照标识。File Service 不解析 Token，只限制长度（≤512 字符）并原样回传。Token 失效时 Provider 返回永久业务错误码，File Service 判为 `FAILED`——不重试，因为重试只会拿到同一个失效 Token（本轮内）。 + +**"稳定快照"的保证范围必须说准，不要过度承诺。** 不同 Token 实现的保证强度差别很大： + +| Token 实现 | 排除导出期间新增 | 冻结导出期间的修改 | 冻结导出期间的删除 | +|---|---|---|---| +| 签名后的最大 ID（最简单，推荐起步） | ✅ | ❌ 后续批次会读到改后的值 | ❌ 已删除的行不再返回，结果行数可能少于 `totalRows` | +| 截止时间戳 + 按 `updated_at` 过滤 | ✅ | ⚠️ 取决于是否有历史版本 | ❌ | +| 数据库快照（如 MySQL 一致性读快照、或导出前落临时表） | ✅ | ✅ | ✅ | + +**一期 `CUSTOMER_LIST` 采用"签名后的最大 ID"，即只保证不混入新增数据，不保证修改与删除的一致性。** 这对客户列表导出是可接受的——导出期间恰好有客户资料被改动的概率极低，且影响仅是个别字段取值偏新。 + +因此对外表述应为“**不会混入导出开始后新增的数据**”，而非笼统的“数据快照一致”。若某个导出类型确实需要完整快照（如财务对账），该 Provider 需自行实现临时表或一致性读方案，本契约不阻止，但也不代为实现。 + +单批响应中的 `totalRows` 为**估算值**，仅用于展示进度百分比，不得用于校验最终行数是否正确——删除会使实际行数偏少，这是上表的直接后果。 + +### 10.3 注册表 + +导出类型通过受控 Nacos 配置注册（`file-service.yml`）： + +```yaml +baiyue: + file: + export: + definitions: + CUSTOMER_LIST: + provider-service: iam-service + provider-path: /internal/exports/v1/CUSTOMER_LIST/batches + required-permission: platform:customer:export + app-scope: PLATFORM + enabled: true + max-rows: 200000 + max-attempts: 3 + batch-size: 500 +``` + +限制与启动校验（任一不通过则**启动失败**，不允许降级运行）： + +- `provider-service` 必须在**代码内置白名单**中（`iam-service`、`order-service`、`warehouse-service`、`firstleg-service`）。只靠 Nacos 校验不够——Nacos 被误改或被入侵即可指向任意服务造成 SSRF；白名单在代码里，配置只能从白名单中选。 +- `provider-path` 必须匹配 `^/internal/exports/v1/[A-Z0-9_]+/batches$`，且路径中的类型名必须等于 map 的 key。 +- `required-permission` 非空且匹配 `^[a-z]+:[a-z-]+:[a-z-]+$`。 +- `app-scope` ∈ `{PLATFORM, MERCHANT}`，用于创建任务时校验请求方访问域。 +- `max-rows` ∈ [1, 500000]，`batch-size` ∈ [50, 1000]，`max-attempts` ∈ [1, 5]。 +- key 不重复、不为空（YAML map 天然去重，但需校验非法字符）。 +- 注册配置由平台发布流程维护，不能由浏览器动态注册。 + +配置支持 Nacos 热刷新，但**刷新时同样执行全量校验，校验失败保留旧配置并告警**，不接受非法配置生效。 + +通过配置注册后，新业务类型不需要修改 File Controller、Consumer、Publisher 或 XLSX Writer。 + +### 10.4 权限与定义在热更新下的语义（必须明确，否则行为不确定） + +任务从创建到执行之间可能间隔数十分钟（排队 + 退避重试），期间 Nacos 注册表可能被改动，用户权限也可能被回收。**必须明确每一项按哪个时点取值**，否则同一个任务在不同时刻执行会得到不同结果。 + +本文的取值规则： + +| 项目 | 取值时点 | 理由 | +|---|---|---| +| `required-permission` 校验 | **创建时** + **人工重试时**，执行阶段不再校验 | 执行发生在后台，此时没有用户会话，无法也不应重新鉴权。任务一旦受理即视为已授权 | +| `enabled` | **创建时** + **人工重试时** | 同上。已受理的任务不因类型临时停用而失败——停用是为了阻止新任务，不是撤销在途任务 | +| `app-scope` | **创建时** | 与权限同批校验 | +| `max_attempts` | **创建时快照进任务表** | 已在 §7.3 落表，重试上限在任务生命周期内恒定，不受配置改动影响 | +| `provider-service` / `provider-path` | **执行时取当前值** | Provider 地址变更（如服务迁移）应当立即对在途任务生效，否则任务会打到已下线的地址 | +| `batch-size` / `max-rows` | **执行时取当前值** | 属于资源保护参数，调小应立即生效以保护下游 | + +因此**任务表不保存权限快照，也不保存定义版本号**——需要冻结的只有 `max_attempts`，已单独落表；其余要么只在创建时校验一次，要么刻意读取最新值。这比引入 `definition_version` 字段更简单，且语义明确。 + +两个必须接受的后果，写明以免日后被当作 bug： + +1. **权限被回收后，用户已提交的排队任务仍会执行完成**，文件可下载（下载时会再校验 `ownedTask`，但不校验导出权限）。若需要立即阻断，应由管理员显式取消任务——**本期不提供取消接口**，属已知缺口，后续需要时再加。 +2. **导出类型被 `enabled: false` 后，在途任务仍会跑完。** 若要强制停止，运维手段是关 `consumer.enabled` 或直接改任务状态为 `FAILED`。 + +如果未来出现"必须严格按执行时权限"的合规要求，则需要在任务表增加权限快照字段并在执行前比对——那是一次有意的设计变更，不是本期的疏漏。 + +## 11. 通用 XLSX Writer + +统一 Writer 替换现有 `CustomerExportTaskHandler` 中硬编码的 `HEADERS` 与 `writeRow`。至少支持： + +- 类型 `STRING`、`INTEGER`、`DECIMAL`、`BOOLEAN`、`DATE`、`DATETIME`，按类型写入对应单元格类型（当前实现把所有值写成字符串，日期与金额在 Excel 中无法排序计算，本次修正）。 +- 空值、超长单元格（截断至 32767 字符并记 WARN）、非法数字、`columns` 中未声明的字段（忽略并记 WARN，不报错）。 +- 冻结表头、列宽上限（≤255 字符宽）、稳定 Sheet 名。 +- 单 Sheet 行数上限 1048575 行；接近上限自动创建下一 Sheet 并重复表头。按 `max-rows: 200000` 的默认值，正常不会触发，但必须实现。 +- `SXSSFWorkbook(100)` 流式写入、`setCompressTempFiles(true)`、临时文件确定性清理。 +- **所有 `STRING` 值以字符串单元格写入并对首字符为 `= + - @ Tab CR` 的值加前导单引号**，防止 CSV/Excel 公式注入。 +- 上限：最大列数 100、最大行数取注册表 `max-rows`、最大任务时长 30 分钟、最大文件 50MB。任一超限即 `FAILED` 并写明具体上限。 + +**文件大小上限必须在写入过程中实时检查并提前中止，不能等文件完整落盘后再判断。** 事后检查有两个问题：磁盘已经被写满（50MB 上限形同虚设，实际可能写出几个 GB）、白白消耗了完整的导出时间才失败。 + +实现方式：在 `SXSSFWorkbook` 的输出流外包一层计数流，累计字节数超过上限立即抛出永久业务错误： + +```java +// 写入路径：SXSSFWorkbook.write(countingStream) +// countingStream 内部累加 written，超限即 throw ExportSizeExceededException +``` + +注意 `SXSSFWorkbook` 的临时文件也会占磁盘（`setCompressTempFiles(true)` 后是压缩的，但仍需空间），行数上限 `max-rows` 是这一侧的主要保护。两道闸门都要有：`max-rows` 防临时文件膨胀，计数流防最终文件膨胀。 + +同理，Provider 单批响应体 ≤4MB 的限制也应在读取时按 `Content-Length` 或计数流判断，而非完整读入后再检查。 + +业务服务不得返回样式、公式、宏或任意文件路径。 + +### 11.1 存储抽象与多实例约束 + +将现有 `LocalExportStorage` 提取为接口 `ExportStorage`（`create` / `size` / `load` / `delete`），实现类 `SharedDirectoryExportStorage` 保持当前行为与对象键格式 `exports/{taskId}/{uuid}.xlsx`。Consumer、API、Reconciler 只依赖接口。 + +一期不引入对象存储（§21 决策 6），但接口就位后未来替换为 MinIO/OSS 不需要改动调用方。**对象键始终由 File Service 生成与校验，业务服务不得提供。** + +**普通宿主机目录挂载不构成"共享存储"，它只在同一宿主机的容器之间共享。** 这是第一版的一处不实表述：部署已有的 `FILE_UPLOADS_HOST_DIR:/data/files` 是宿主机 bind mount，若 File Service 的两个实例被调度到**不同物理机**，各自看到的是各自宿主机上的独立目录——A 实例生成的文件，B 实例既读不到也删不掉，表现为：下载随机 404（请求打到没有该文件的实例）、过期清理留下永久孤儿文件。 + +因此一期存储的明确边界，**二选一，必须在部署前确认**： + +| 方案 | 约束 | 适用 | +|---|---|---| +| **A. 单节点部署**（一期默认） | File Service 只允许 1 个实例；若用编排系统需固定节点亲和，禁止自动扩容 | 1000 用户量级，导出低频，单实例足够 | +| **B. 跨主机共享卷** | 挂载 NFS / CephFS / 云盘共享卷到同一路径 | 需要多实例时 | + +选 A 时必须注意：**代码不得因此引入任何单实例假设**——租约、`lease_owner`、条件更新全部保留，因为滚动发布期间会短暂存在两个实例，且方案 B 随时可能启用。单实例是部署约束，不是设计约束。 + +选 B 时需额外验证：NFS 的 `Files.deleteIfExists` 与 `newOutputStream(CREATE_NEW)` 语义、以及多实例并发写同一目录的表现。 + +`ExportStorage` 接口的价值正在于此：从 A 迁到 B 只是换挂载，从 A/B 迁到 MinIO 只是换实现类，调用方一行不改。 + +## 12. 权限与安全 + +- 创建任务：校验登录身份、`app_code` 与注册表 `app-scope` 匹配、`required-permission` 命中。 +- 列表、详情、下载：沿用现有 `ownedTask()` 的租户 + 访问域 + 申请人 + `platform:download:all` 校验。 +- Provider 调用使用 `X-Internal-Service-Token`，**不转发用户 Bearer Token**；令牌只从配置中心密文注入。 +- File Service 不记录完整行数据、Token 或敏感筛选条件；日志中的 `parameters` 需脱敏。 +- 任务参数限制：键数 ≤20、JSON ≤4KB、单字符串 ≤256 字符、嵌套深度 ≤2。 +- Provider 响应限制：单批 ≤`batch-size` 行、列数 ≤100、单元格 ≤32767 字符、单批响应体 ≤4MB。 +- Provider 服务名与路径必须来自注册表且过代码白名单，防止 SSRF（§10.3）。 +- MQ 消息不含任何敏感数据（§6.4）。当前 Compose 仅支持 File Service 与 Broker 位于同一 `infra-network`，Broker 不映射宿主机端口。 + +## 13. 可观测性 + +**前置依赖**：必须先在 `common-web/pom.xml` 增加 `micrometer-registry-prometheus`，否则 actuator 的 `prometheus` 端点不可用，本节指标全部无处可采。同时需要落地 Prometheus 服务端（§21 决策 5）。 + +指标（1000 用户量级的实用集合）： + +- `export_task_created_total{type}` +- `export_task_completed_total{type,status}` +- `export_task_duration_seconds{type}`（直方图） +- `export_task_queue_age_seconds`（最老 PENDING 任务的等待秒数，Gauge） +- `export_outbox_pending`、`export_outbox_oldest_age_seconds`、`export_outbox_max_attempt`（当前所有待发行中最大的 `attempt_count`，替代原 `export_outbox_dead`） +- `export_provider_request_seconds{service,type}`、`export_provider_failure_total{service,type}` +- `export_reconciler_action_total{action}`（`redispatch` / `lease_recover` / `outbox_release` / `expire` / `orphan_clean`） +- `export_dlq_messages_total`（File Service 的 DLQ Handler 每收到一条消息即递增；无论定位属性是否合法都计数） +- `export_storage_bytes` + +日志统一携带 `taskId`、`eventId`、`dispatchVersion`、`exportType`、`tenantId`、`requestId`、`traceId`，禁止打印完整导出数据。 + +告警： + +| 告警 | 阈值 | 含义 | +|---|---|---| +| Outbox 积压 | `export_outbox_oldest_age_seconds > 120` | Publisher 或 Broker 异常 | +| Outbox 反复发送失败 | `export_outbox_max_attempt > 10` | 单条命令连续失败超 10 次，Broker 或网络异常，需人工排查（行本身会继续重试，不会丢） | +| 任务排队过久 | `export_task_queue_age_seconds > 600` | Consumer 未消费或全部忙 | +| DLQ | 任意新消息 | 出现损坏消息 | +| 失败率 | 连续 10 分钟失败率超阈值 | Provider 或存储异常 | +| Reconciler 频繁恢复 | `export_reconciler_action_total{action="lease_recover"}` 10 分钟内 > 3 | 实例不稳定或租约过短 | +| 存储 | 不可写或剩余空间不足 | 需扩容 | + +## 14. 限流与资源隔离 + +| 项 | 取值 | 说明 | +|---|---|---| +| Consumer 并发（= 导出并发） | 4 | `consumeThreadMin = consumeThreadMax = 4`；队列数不构成并发线程硬上限 | +| 单用户同时活动任务 | 3 | 超出返回业务错误 | +| 单 `exportType` 同时活动任务 | 5 | 保护 Provider | +| 重复提交 | 同一用户 60 秒内相同 `exportType` + 相同参数 JSON → 返回已有活动任务 | 参数 JSON 需规范化后比较 | +| Provider 单批 | ≤500 行 | 注册表 `batch-size` | +| Provider 超时 | 连接 3 秒 / 读 **15 秒** | 见下方说明，**必须按导出 Client 单独配置** | +| 最大任务时长 | 30 分钟 | 超时判临时失败 | +| `consumeTimeout` | 90 分钟 | 必须 > 最大任务时长 | +| 优雅停机 | `server.shutdown: graceful`（已配置）+ Consumer `shutdown()` 先停拉取，等待当前批次到安全点，最长 60 秒 | 停机时不完成的任务由租约超时恢复 | + +默认并发 4 以测试、生产均约 8C32G 为容量基准，但这 8 核由 File、IAM、MySQL 及其他服务共享，不能按 File Service 独占 8 核估算。当前单实例即使不扩 Topic 队列也能把消费线程调到 4 以上，但不得仅凭 CPU/内存规格直接升到 6 或 8；必须复核 Provider、数据库、磁盘和普通接口延迟。若改为多实例部署，还要同时复核队列分配与共享存储。 + +上表前三项创建约束必须在 `file_export_task_guard` 行锁保护的同一数据库事务内复核并写入，Controller/Service 中事务外的预检查只能用于快速失败，不能作为并发正确性保证。 + +File Service 必须显式限制 JVM 堆，不能继续只依赖镜像的 `-XX:MaxRAMPercentage=75.0` 按整机 32GB 计算。测试与生产起步配置为 `-Xms1g -Xmx2g`；若 4 个 20 万行任务并发压测时 GC 后堆占用持续超过 65%，再依据实测调大，不能用扩大堆掩盖未释放的 Workbook、临时文件或响应对象。 + +**Provider 读超时必须按导出 Client 单独配置，不能沿用全局默认值。** 仓库 `config/nacos/*/baiyue-common.yml` 中 `spring.cloud.openfeign.client.config.default.read-timeout` 当前为 **5000ms**，这对普通接口合适，但导出单批要拉 500 行业务数据（含多表关联），5 秒极易超时，会把正常导出误判为临时依赖错误并触发无谓重试。 + +因此在 `file-service.yml` 中按 `contextId` 单独覆盖： + +```yaml +spring: + cloud: + openfeign: + client: + config: + iamExportProviderClient: &export-provider-timeout + connect-timeout: 3000 + read-timeout: 15000 + orderExportProviderClient: *export-provider-timeout + warehouseExportProviderClient: *export-provider-timeout + firstlegExportProviderClient: *export-provider-timeout +``` + +15 秒的取值依据：单批 500 行的正常耗时应在 1~3 秒，15 秒留 5 倍余量吸收抖动。**注意读超时不再是租约的保障**——§8.3 的 watchdog 独立于批次续租，即使单批阻塞 15 秒租约也不会丢；读超时在这里的作用是防止单批无限挂起把消费线程占死到 `consumeTimeout`。 + +## 15. 数据保留与存储 + +- `SUCCEEDED` 文件默认保留 7 天（沿用 `FILE_EXPORT_RETENTION_DAYS`）。 +- 文件删除成功或确认不存在后，任务才能标记 `EXPIRED`；删除失败保持原状并重试 + 告警（沿用现有实现）。 +- `file_export_outbox` 的 `SENT` 行保留 7 天后**物理删除**（每日一次，每次 ≤5000 行分批删除）。清理必须限定 `status = 'SENT'`，绝不能删除 `PENDING` / `CLAIMED` 行——它们是尚未送达的命令。 +- `file_export_task` 终态行长期保留（下载中心需要历史），当前量级无需归档。 +- 存储：一期按 §11.1 方案 A（单节点）或方案 B（跨主机共享卷）二选一，部署前必须确认。宿主机 bind mount 只在同一主机内共享，**不足以支撑跨主机多实例**。 +- 对象键由 File Service 生成和校验，业务服务不得提供。 + +## 16. 基础设施落地 + +### 16.1 测试环境 RocketMQ + +实际部署入口为 `deploy/docker-compose.rocketmq.yml`，包含单 NameServer、单 Broker 和一次性 Topic 初始化容器（1000 用户量级，无主从）： + +```yaml + rocketmq-namesrv: + image: apache/rocketmq:5.3.1 + command: sh mqnamesrv + restart: unless-stopped + networks: [infra-network] + + rocketmq-broker: + image: apache/rocketmq:5.3.1 + depends_on: [rocketmq-namesrv] + environment: + ROCKETMQ_BROKER_IP: ${ROCKETMQ_BROKER_IP:?ROCKETMQ_BROKER_IP is required} + volumes: + - ./rocketmq:/config:ro + - ${ROCKETMQ_DATA_DIR:-/data/firstleg/rocketmq/store}:/home/rocketmq/store + command: sh /config/render-broker-config.sh + restart: unless-stopped + networks: [infra-network] + + rocketmq-topic-init: + image: apache/rocketmq:5.3.1 + depends_on: [rocketmq-broker] + environment: + MQ_TOPIC_PREFIX: ${MQ_TOPIC_PREFIX?MQ_TOPIC_PREFIX must be set} + # 完整的四条幂等 mqadmin 命令见实际 Compose；任一失败由 on-failure 重试 + command: sh -eu -c 'until sh mqadmin clusterList -n rocketmq-namesrv:9876 2>/dev/null | grep -q "BaiyueCluster"; do sleep 2; done; ...' + restart: on-failure + networks: [infra-network] +``` + +**`brokerIP1` 必须在启动时生成，不能在 properties 文件中直接写 `${ROCKETMQ_BROKER_IP}`。** RocketMQ 不会替换该占位符，会把字面量注册到 NameServer，客户端随后表现为发送超时。 + +因此 `deploy/rocketmq/broker.conf.template` 保存模板： + +```properties +brokerClusterName=BaiyueCluster +brokerName=baiyue-file-export-broker +brokerId=0 +brokerIP1=__BROKER_IP__ +listenPort=10911 +deleteWhen=04 +fileReservedTime=72 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH +autoCreateTopicEnable=false +autoCreateSubscriptionGroup=false +``` + +`deploy/rocketmq/render-broker-config.sh` 在启动前使用 `sed` 生成 `/tmp/broker.conf`，并验证地址与残留占位符： + +```bash +#!/bin/sh +set -eu + +case "${ROCKETMQ_BROKER_IP:-}" in + ""|localhost|127.0.0.1|__BROKER_IP__|*'${'*) + echo "ROCKETMQ_BROKER_IP must be an explicit client-reachable address" >&2 + exit 1 + ;; +esac +sed "s/__BROKER_IP__/${ROCKETMQ_BROKER_IP}/g" /config/broker.conf.template > /tmp/broker.conf +if grep -Eq '__BROKER_IP__|\$\{' /tmp/broker.conf; then + echo "broker.conf still contains an unresolved placeholder" >&2 + grep -nE '__BROKER_IP__|\$\{' /tmp/broker.conf >&2 + exit 1 +fi +exec sh mqbroker -n rocketmq-namesrv:9876 -c /tmp/broker.conf +``` + +脚本生成临时运行配置而不覆盖只读模板，并显式拒绝空值、`127.0.0.1`、`localhost` 和未展开占位符。 + +当前部署限定客户端与 Broker 同在 `infra-network`，因此 `ROCKETMQ_BROKER_IP` 固定填 `rocketmq-broker`（服务名）。当前 Compose 不映射 `10911/10909`，不得填写宿主机 IP；如未来确需跨主机客户端，必须先增加受防火墙限制的端口映射并重新完成网络验收，不能只改 `brokerIP1`。 + +`autoCreateTopicEnable=false` 是刻意的：Topic 必须由部署脚本显式创建，避免拼错 Topic 名时静默创建出一个无人消费的新 Topic。 + +**验收项**：Broker 起来后在 `infra-network` 内的容器中执行 `mqadmin clusterList -n `，确认输出的 Broker 地址是 `rocketmq-broker` 而非字面量或 `127.0.0.1`；再从 File Service 容器内确认 `rocketmq-broker:10911` 可达。这一步不通过，后面所有 MQ 功能都无从验证。 + +### 16.2 Topic 初始化容器 + +`deploy/docker-compose.rocketmq.yml` 的 `rocketmq-topic-init` 内联幂等执行： + +```bash +mqadmin updateTopic -n "$NAMESRV" -c BaiyueCluster -t "${PREFIX}baiyue-file-export-command-v1" -r 4 -w 4 +mqadmin updateSubGroup -n "$NAMESRV" -c BaiyueCluster -g "${PREFIX}baiyue-file-export-worker-v1" +mqadmin updateSubGroup -n "$NAMESRV" -c BaiyueCluster -g "${PREFIX}baiyue-file-export-dlq-handler-v1" +mqadmin updateTopic -n "$NAMESRV" -c BaiyueCluster -t "%DLQ%${PREFIX}baiyue-file-export-worker-v1" -r 1 -w 1 -p 6 +``` + +DLQ Topic 默认权限为只写（2），必须显式改为读写（6），否则无法消费 DLQ 排查问题。 + +初始化容器必须使用 `sh -eu`，并等待 `clusterList` 输出中明确出现 `BaiyueCluster` 后才执行上述命令；仅以 `clusterList` 退出码为 0 不能证明 Broker 已注册。任一初始化命令失败时容器按 `restart: on-failure` 重试，避免 NameServer 先就绪、Broker 后注册的首次部署竞态。 + +### 16.3 配置项 + +`file/src/main/resources/application.yml` 新增： + +```yaml +rocketmq: + name-server: ${ROCKETMQ_NAME_SERVER:127.0.0.1:9876} + producer: + # 源码默认 test- 仅服务本地/测试；Compose 部署必须显式设置 MQ_TOPIC_PREFIX，生产传空字符串 + group: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-publisher-v1 + send-message-timeout: 3000 + retry-times-when-send-failed: 2 + retry-times-when-send-async-failed: 2 +spring: + cloud: + openfeign: + client: + config: + iamExportProviderClient: &export-provider-timeout # 覆盖全局 read-timeout:5000，见 §14 + connect-timeout: 3000 + read-timeout: 15000 + orderExportProviderClient: *export-provider-timeout + warehouseExportProviderClient: *export-provider-timeout + firstlegExportProviderClient: *export-provider-timeout +baiyue: + file: + export: + retention-days: ${FILE_EXPORT_RETENTION_DAYS:7} + instance-id: ${HOSTNAME:file-local} + consumer: + # 首次部署必须按 §17/§18 分阶段显式开启；缺少环境变量时保持关闭 + enabled: ${FILE_EXPORT_CONSUMER_ENABLED:false} + concurrency: ${FILE_EXPORT_CONSUMER_CONCURRENCY:4} + consume-timeout-minutes: ${FILE_EXPORT_CONSUME_TIMEOUT_MINUTES:90} + max-reconsume-times: 2 + publisher: + enabled: ${FILE_EXPORT_PUBLISHER_ENABLED:false} + poll-interval-ms: ${FILE_EXPORT_PUBLISHER_POLL_MS:5000} + batch-size: 50 + lease-seconds: 60 + reconciler: + enabled: ${FILE_EXPORT_RECONCILER_ENABLED:false} + poll-interval-ms: ${FILE_EXPORT_RECONCILER_POLL_MS:30000} + redispatch-grace-seconds: 60 + # 命令已发出多久仍未被执行才判定为丢失并补发，见 §8.5(1) + # 必须大于常态排队时长，且与 §13 的 export_task_queue_age_seconds 告警阈值(600s)一致 + sent-stale-seconds: ${FILE_EXPORT_SENT_STALE_SECONDS:600} + task: + lease-seconds: ${FILE_EXPORT_TASK_LEASE_SECONDS:300} + heartbeat-seconds: ${FILE_EXPORT_TASK_HEARTBEAT_SECONDS:60} + max-duration-minutes: ${FILE_EXPORT_MAX_DURATION_MINUTES:30} + max-file-size-mb: ${FILE_EXPORT_MAX_FILE_SIZE_MB:50} + rocketmq: + topic: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-command-v1 + consumer-group: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-worker-v1 + dlq-topic: "%DLQ%${MQ_TOPIC_PREFIX:test-}baiyue-file-export-worker-v1" + dlq-consumer-group: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-dlq-handler-v1 +``` + +`consumer.enabled` / `publisher.enabled` / `reconciler.enabled` 三个独立开关是**部署与回滚的关键**（§17、§18）。`instance-id` 用于 `lease_owner`，容器环境取 `HOSTNAME`。 +Publisher 的退避档位由实现固定为 5s / 15s / 1m / 5m / 15m，超过后固定 15m 持续重试；当前没有 `max-backoff-minutes` 配置项，避免文档出现不会被代码读取的伪配置。源码中的 `MQ_TOPIC_PREFIX:test-` 是本地安全默认值，部署环境仍必须由 Compose 的必填校验显式传入，见 §6.2 与 §18。 + +移除旧配置 `worker-delay-ms`、`stale-running-minutes`（由 Reconciler 参数取代）。 + +### 16.4 依赖变更 + +| 模块 | 变更 | +|---|---| +| `platform-bom` | 新增 `rocketmq-spring-boot-starter:2.3.3` 的版本管理（属性名建议 `rocketmq-spring.version`） | +| `common-core` | 新增包 `com.baiyue.common.core.export`：Provider 请求/响应 DTO、列定义与类型枚举、MQ 命令消息、注册表配置模型。均为 record，不引入新依赖（jackson 由 `spring-web` 传递已可用） | +| `file` | 新增 `rocketmq-spring-boot-starter` | +| `iam` | **无需改 pom**（已通过 `common-web` 传递依赖到 `common-core`），直接实现 Provider 接口 | +| `common-web` | 新增 `micrometer-registry-prometheus` | +| `firstleg` | **删除** 未使用的 `spring-cloud-starter-stream-rocketmq` | +| 根 `pom.xml` | 不变（不新增模块，仍为 18 个） | + +**不新建 `common-export-contract` 模块。** 关键约束：Provider 契约是 file 与 iam 双方共用的 HTTP 结构，若只放在 `file` 模块内，iam 就必须依赖 `file-service`，会把 POI、MyBatis、任务表实体全部拖入 iam 的 classpath，并形成业务服务反向依赖 file 的结构，违反 AGENTS.md 的“服务间调用先找调用方 `client/`、不共享领域 Entity”。放入 `common-core` 既满足双方共用，又不新增模块——iam 与 file 均已通过 `common-web` 传递依赖到 `common-core`，两边 pom 都不用动。 + +## 17. 实施与切换 + +### 阶段 1：基础设施与契约（无行为变更） + +1. 测试环境起 NameServer + Broker，执行 Topic 创建脚本，`mqadmin topicList` 核验。 +2. `common-core` 新增 `com.baiyue.common.core.export` 包（契约 record），`platform-bom` 登记 rocketmq starter 版本。 +3. `common-web` 加 Prometheus registry，核验 `/actuator/prometheus` 返回指标。 +4. 按 §7.3 与 §8.3.1 完善 `file/src/main/resources/db/file.sql`：补齐 `file_export_task` 的 8 个字段与 2 个索引，加入 `file_export_outbox` 与 `file_export_task_guard` 建表语句。测试环境确认任务数为 0 后删除这三张导出表并重新执行，核验方式见 `deploy/sql/README.md`。 +5. 删除 `firstleg` 的死依赖。 +6. 现有 `ExportTaskWorker` 保持运行，行为不变。 + +### 阶段 2：Provider 与 Writer（仍走旧调度） + +1. IAM 实现 `/internal/exports/v1/CUSTOMER_LIST/batches`，含租户校验与 `snapshotToken`。 +2. File 实现通用 `ExportProviderClient` 与通用 XLSX Writer、`ExportStorage` 接口、Nacos 注册表与启动校验。 +3. `CustomerExportTaskHandler` 改为走通用链路，删除硬编码列定义与 `IamExportClient`。 +4. 仍由 `ExportTaskWorker` 调度执行，**先验证数据正确性**：导出结果与旧版逐列比对一致。 + +这一步把“数据链路正确性”与“调度机制正确性”分开验证，任一环节出问题都能独立定位。 + +### 阶段 3：MQ 调度上线 + +1. 部署带 Publisher + Consumer + Reconciler 的 File Service，三个开关**全部关闭**，确认服务健康。 +2. **停用 `ExportTaskWorker.processNext()`**（保留 `expire` 逻辑迁入 Reconciler）。这里不做“新旧双轨执行”——旧 `claim()` 既不检查 `next_retry_at` 也不检查 `dispatch_version`（`ExportTaskRepository.java:56`），双轨期间它会提前执行退避窗口内的任务并绕过版本校验。双轨的复杂度换不来任何安全性。 +3. 开启 `reconciler.enabled` → 观察 `export_reconciler_action_total{action="redispatch"}`，确认存量 PENDING 被补发 Outbox。 +4. 开启 `publisher.enabled` → 观察 Outbox `SENT` 增长、Broker 消息堆积。 +5. 开启 `consumer.enabled` → 任务开始执行，观察端到端。 +6. 逐项跑完 §19 故障注入清单。 + +分步开启的价值：每一步只引入一个变量。若第 4 步后消息发不出去，问题必定在 Publisher 或 Broker，与 Consumer 无关。 + +### 阶段 4：清理 + +1. 删除 `ExportTaskWorker`、`firstPendingId`、`requeueStale` 及其测试。 +2. 更新 `docs/公共导出与下载中心接入说明.md` 与 `AGENTS.md` 的 `file/` 模块条目。 +3. 补充新业务类型接入清单（Provider 实现 + Nacos 注册两步）。 + +### 回滚 + +MQ 调度启用后的安全回退方式是**关配置开关暂停调度**，不需要回滚数据库结构；这不是恢复旧调度器： + +- 关 `consumer.enabled`：停止领取新任务；尚未领取的任务保持 `PENDING`，在途 `RUNNING` 任务按优雅停机完成，未完成的由租约超时后 Reconciler 恢复。 +- 同时关闭 `publisher.enabled` 与 `consumer.enabled`：停止发送新命令并停止执行；Outbox 与任务状态保留，重新开启后自动追平。 +- **禁止恢复旧 `ExportTaskWorker`。** 它不检查 `next_retry_at` 与 `dispatch_version`，会绕过退避和版本保护；阶段 4 删除后也不再具备该回滚路径。如需回滚应用版本，必须先确认旧版本兼容当前表结构与任务状态，并按独立发布回滚方案执行，不能仅靠开关假定回到阶段 2。 +- 数据库中的任务与状态始终完整，Outbox 行不会丢。 + +## 18. 部署顺序 + +```text +RocketMQ NameServer/Broker 就绪 +-> Topic / ConsumerGroup / DLQ 创建 +-> 测试环境执行 file.sql（任务表 + Outbox 表） +-> Nacos 发布导出类型注册表 +-> 业务 Provider（IAM）上线 +-> File Service 上线（三开关全关） +-> 开 Reconciler -> 开 Publisher -> 开 Consumer +-> Admin 验收 +``` + +环境变量需在 `deploy/docker-compose.yml` 与 Jenkins 中透传：`ROCKETMQ_NAME_SERVER`、`MQ_TOPIC_PREFIX`、`ROCKETMQ_BROKER_IP`、`FILE_EXPORT_CONSUMER_ENABLED`、`FILE_EXPORT_CONSUMER_CONCURRENCY`、`FILE_EXPORT_PUBLISHER_ENABLED`、`FILE_EXPORT_RECONCILER_ENABLED`。`MQ_TOPIC_PREFIX` 不得省略：测试显式设为 `test-`，生产显式设为空字符串。测试与生产的 `FILE_EXPORT_CONSUMER_CONCURRENCY` 初始值均显式设为 `4`，不只依赖应用默认值；三个启用开关首次上线均显式设为 `false`，再按本节顺序逐项改为 `true`。File 容器需要 `HOSTNAME` 可用（Docker 默认提供）。 + +## 19. 测试与验收门槛 + +### 19.1 单元测试（必须全绿） + +- 任务与 Outbox 同事务创建；任务插入失败时 Outbox 不落库。 +- 并发创建：两请求同时命中重复提交或活动任务上限时，由 `file_export_task_guard` 串行复核，最终只创建允许数量的任务与 Outbox。 +- Publisher：到点领取、未到点不领取、并发领取只有一个成功、发送成功标记 `SENT`、发送失败退避、**连续失败 30 次后仍保持 `PENDING` 继续重试（无终态放弃）**、租约超时可被重新领取。 +- Consumer 抢占：**§8.2 表格的七个分支逐一覆盖**，每个分支都必须断言返回 `CONSUME_SUCCESS`。 +- Consumer：执行成功、永久失败直接 `FAILED`、临时失败递增版本并写新 Outbox、达到 `max_attempts` 转 `FAILED`。 +- **watchdog 续租覆盖写盘阶段**（§8.3 的核心）：模拟 Provider 拉取结束后 `write()`/压缩耗时超过租约时长，断言 watchdog 期间持续续租、任务未被 Reconciler 剥夺、最终 `SUCCEEDED`。这是第一版按批次续租时会失败的场景。 +- watchdog：租约被剥夺时置中止标志，执行线程在下一个可中断点中止、删除半成品文件、不写任务表；watchdog 自身异常也置中止标志。 +- deadline：抢占时按数据库时钟写入；超过 deadline 后 watchdog 不再续租并触发中止；Java 预检查通过但最终提交 SQL 执行时已超时的竞态必须提交失败、删除文件并进入超时重试。 +- **时间基准**：所有租约与退避判断走数据库 `CURRENT_TIMESTAMP(3)`；构造实例本地时钟快 5 分钟的场景，断言不会提前抢占他人租约。 +- 文件大小：写入超过 50MB 时在**写入过程中**抛错中止，断言磁盘上不存在超限的完整文件。 +- 结果提交：版本已变时提交失败并删除孤立文件。 +- Reconciler：四项职责各自幂等；`redispatch` 只恢复或补写同版本 Outbox，`lease_recover` 才递增版本且达上限转 `FAILED`。 +- **Reconciler 不得跳过退避窗口**：构造一个刚自动重试、`next_retry_at` 在 15 分钟后的 `PENDING` 任务，其 Outbox 行已为 `SENT`；断言 Reconciler 连续多轮都不动它，直到 `next_retry_at` 到点后 60 秒才补投。 +- **Reconciler 补投不递增任务版本**（§3 原则 5，最关键的一条）：构造一个 `PENDING` 任务、其 Outbox 行为 `SENT` 且 `sent_at` 已超窗口；断言 Reconciler 把该行改回 `PENDING` 且 `dispatch_version` **保持不变**。 +- **排队任务不被饿死**（阻断项 1 的回归测试）：模拟并发已满、任务在队列中等待超过 `sentStaleSeconds`；断言补投后原命令仍能成功抢占执行（版本未变故未失效），任务最终 `SUCCEEDED`。 +- **补投两种情形**：无 Outbox 行 → 补写同版本新行；`SENT` 超窗口 → 原行改回 `PENDING`。两者都断言版本不变。 +- **补投不受 `max_attempts` 约束**：`attempt_count` 已达上限的 `PENDING` 任务，其丢失的命令仍应被补投（补投不是执行尝试）。 +- **重复命令幂等**：同版本两条命令并发消费，只有一个抢占成功执行，另一个见 `RUNNING` 后确认消费；最终只有一个有效文件。 +- **执行恢复受重试上限约束**：`RUNNING` 租约超时且 `attempt_count >= max_attempts` 时任务应置 `FAILED`；不得把这条约束误用于 `PENDING` 任务的同版本消息补投。 +- Outbox `attempt_count` 与任务 `attempt_count` 相互独立：任务重试写入的新 Outbox 行，其 `attempt_count` 从 0 开始。 +- 人工重试：只允许 `FAILED`、停用类型拒绝、递增版本、清空上次现场。 +- 注册表：白名单外服务名启动失败、路径格式非法启动失败、路径与 key 不一致启动失败、热刷新非法配置保留旧值。 +- Provider 客户端：租户字段必填、`schemaHash` 变化判永久失败、游标未推进判临时失败、超时判临时失败。 +- XLSX Writer：六种类型写入正确单元格类型、公式注入前缀转义、超长截断、多 Sheet 切换、**真实打开生成的 xlsx 并逐格校验**。 +- Producer：四个定位字段同时写入 payload 与 RocketMQ User Properties，且与 Outbox 列一致。 +- 主 Consumer 反序列化失败时不确认消费；DLQ Handler 使用原始 `MessageExt` 属性自动执行 `QUARANTINED + FAILED` 同事务更新，任一步失败则回滚且不确认；旧版本、eventId 不匹配或缺属性消息不得误伤当前任务。 +- Reconciler 永不补投 `QUARANTINED`；隔离任务人工重试生成新版本 Outbox，旧隔离行保持不变。 + +### 19.2 集成测试（连测试环境真实 RocketMQ） + +不引入 Testcontainers（§21 决策 8）。集成测试通过 profile 连接测试环境 Broker，默认在 CI 中跳过，由人工或独立 Job 触发： + +真实任务的创建、终态轮询、下载和 XLSX 签名校验统一由 `scripts/export-center-test-env-acceptance.ps1` 执行；使用 `-CreateOnly` 与 `-TaskIds` 可把故障期间创建和基础设施恢复后的追平验证拆成两段。脚本不拥有部署或停服权限，Broker/File/IAM/MySQL 的故障注入仍由有权限的运维或独立 Job 执行。 + +- 数据库提交后停 Broker，恢复后任务仍被执行（Outbox 退避生效）。 +- Publisher 发送成功但未标记 `SENT`（注入异常），重复投递只产生一个有效文件。 +- Consumer 执行中 `kill -9`，租约超时后 Reconciler 恢复并最终成功，且不留孤立文件。 +- Provider 503 两次后成功，`attempt_count` 与退避符合预期。 +- Provider 持续 503 至 `max_attempts`，任务 `FAILED`，`export_task_completed_total{status="FAILED"}` 增长，**DLQ 无消息**。 +- 手工投递一条无法反序列化但携带完整 User Properties 的消息，确认进入 DLQ 后被 DLQ Handler 自动隔离：Outbox 为 `QUARANTINED`、任务为 `FAILED`、告警触发，等待超过 `sentStaleSeconds` 后也不会再次进入 DLQ。 +- 投递缺少定位属性的畸形 DLQ 消息，确认 Handler 记录 CRITICAL 与 Broker messageId 后确认，不形成二级 DLQ；不得误伤当前任务版本。 +- 双实例并发消费同一消息，只生成一个有效文件、只有一个 `SUCCEEDED`。 + +### 19.3 测试环境验收 + +- Admin 客户列表创建真实任务，下载中心显示排队 / 执行中（进度递增）/ 重试 / 完成 / 过期。 +- 下载 XLSX 与筛选条件一致，日期与金额列在 Excel 中可排序计算。 +- 导出过程中新增一条客户数据，确认不混入本次结果（快照生效）。 +- 重启 File、停 Broker、停 IAM 各一次，恢复后任务无丢失。 +- 普通用户无法访问他人任务；无 `platform:customer:export` 权限无法创建。 +- 20 万行导出跑通，观察内存、文件大小、耗时在 §14 上限内。 +- 连续提交 10 个任务，确认按并发 4 排队执行，Provider 未被打满。 +- 同时执行 4 个 20 万行导出，确认整机 CPU 持续低于 70%、File Service GC 后堆占用低于 65%、磁盘无持续满负载、Provider P95 < 5 秒且 P99 < 10 秒，IAM 普通接口 P95 相比无导出基线恶化不超过 20%。任一项不满足则生产回退并发 2，不以机器总内存充足替代链路压测。 + +## 20. 通过标准 + +- 不采用“数据库提交后直接发送 MQ”的非可靠方案。 +- **`dispatch_version` 只因执行尝试递增，消息补投不递增版本**（§3 原则 5）——这是本轮评审的核心修正，防止排队任务饥饿与 DEAD 无限循环。 +- Outbox 无终态失败：发送永不放弃，靠告警而非状态机驱动人工介入。 +- 租约续期由独立 watchdog 覆盖全执行期，包含最终写盘与压缩阶段。 +- 所有租约与退避判断使用数据库时钟，不使用实例本地时钟。 +- 任务状态不放入 Redis，Redis 不承担重试队列。 +- 业务服务不自行实现任务、Excel、存储和下载中心。 +- File Service 不跨库读取业务数据。 +- Provider 契约包含 `tenantId` / `appCode` / `requestedBy`，且 Provider 侧强制按其过滤。 +- 权限与定义在热更新下的取值时点已逐项明确（§10.4），无未定义行为。 +- 存储的多实例边界已明确（§11.1），不把宿主机 bind mount 当作跨主机共享存储。 +- MQ 不承担延迟与业务重排；Reconciler 是唯一重派通道；DLQ 只承载损坏消息。 +- Outbox 与任务均有租约，无“领取后崩溃即永久卡死”的路径。 +- 至少一次投递、幂等、租约、Outbox、Reconciler、DLQ、指标与告警全部纳入一期实现。 +- 新业务类型能通过 Provider 契约 + 受控注册接入，不复制客户导出代码。 + +## 21. 已定型决策 + +以下决策已在本文中定型，不再作为评审开放项。列出理由以便后续维护者理解取舍。 + +| # | 决策 | 取值 | 理由 | +|---|---|---|---| +| 1 | 一期是否含 MQ | 含 | 业务方已定。可靠性由数据库承担，MQ 引入不降低稳定性 | +| 2 | MQ 客户端 | `rocketmq-spring-boot-starter` 2.3.3，Remoting 协议 | 需要显式控制消费返回语义；不引入 Proxy 少一个可失效组件；同时删除 `firstleg` 的 Stream 死依赖。Boot 3.2.9 兼容性与 `rocketmq-client 5.3.1` 传递版本已实证核验，见 §6.1 | +| 3 | Outbox 是否抽公共模块 | 一期不抽，`file_export_outbox` 独立建表并作为全仓基线 | 一期只有一个真实使用者，过早抽象会把接口定错。`order` 的 Outbox 目前**根本没有发布器**，是独立待修问题；后续收敛时以本表结构为准（含 `lease_owner`/`lease_until`/`next_retry_at`），命名统一为 `<模块>_outbox` | +| 4 | 契约放哪 | 放入 `common-core` 的 `com.baiyue.common.core.export` 包，**不新建模块**，模块总数仍为 18 | 契约是 file 与 iam 双方共用的 HTTP 结构。放 `file` 内会迫使 iam 依赖 `file-service`（拖入 POI/MyBatis/任务表实体，且业务服务反向依赖 file）；新建模块则多一个模块。放 `common-core` 两全：双方已通过 `common-web` 传递依赖到它，pom 都不用改。与 `common-event-contract` 不合并——后者是跨区域事件信封（含 `sourceRegion`/`payloadHash` 等无关字段），强行复用会互相牵制 | +| 5 | Prometheus | 一期补 `micrometer-registry-prometheus`；服务端由运维落地 | 无 registry 则 §13 全部指标无处可采，等于没有可观测性 | +| 6 | 文件存储 | 一期共享目录挂载 + `ExportStorage` 接口抽象 | 1000 用户、7 天保留、单文件 ≤50MB，总量在 GB 级，共享目录足够。引入 MinIO 多一个有状态组件与备份负担。接口就位后替换不改调用方 | +| 7 | 导出类型注册 | Nacos 受控配置 + 代码内置服务白名单 + 启动强校验 | 纯 Nacos 有 SSRF 面；纯代码注册每加类型都要发版。两者结合：配置只能从白名单里选 | +| 8 | Testcontainers | 不引入，集成测试连测试环境 Broker | 全仓无 Testcontainers 先例，RocketMQ 容器需同时起 NameServer+Broker 并处理 `brokerIP1`，投入产出比低于直接用测试环境 | +| 9 | 新旧双轨 | 不双轨，阶段 3 直接停用旧 Worker | 旧 `claim()` 不识别 `next_retry_at` 与 `dispatch_version`，双轨会提前执行退避中的任务。回滚靠配置开关，不靠双轨 | +| 10 | Consumer 执行模型 | 消费线程同步执行，并发 4，`consumeTimeout` 90 分钟 | 8C32G 共享部署环境下以 4 作为压测起始值；避免自建线程池的“队列满怎么办”与“已抢占无线程执行”两个失败模式；MQ 队列本身就是背压 | +| 11 | 断点续传 | 不做，每次从零重写 | 20 万行 XLSX 生成在分钟级，续传的状态管理复杂度远超收益 | +| 12 | 默认参数 | `max-rows` 20 万、单文件 50MB、并发 4、租约 300 秒、`max_attempts` 3、退避 1m/15m | 见 §14、§22；生产上线前必须通过 4 个最大任务并发压测 | +| 13 | 数据库变更方式 | 开发阶段直接完善 `file/src/main/resources/db/file.sql`，测试环境 `DROP TABLE` 后重建，**不写增量迁移脚本** | 导出功能尚无生产数据，增量脚本的复杂度换不来价值。**限定条件**：一旦上生产、表内有真实数据，后续变更必须回到 AGENTS.md 的可重复执行增量模式 | +| 14 | 版本递增的唯一来源 | 仅执行失败重试、租约恢复、人工重试三处；**补投不递增** | 第二轮评审的核心修正。混淆投递失败与执行失败会造成排队任务饥饿（活性错误）与 DEAD 无限循环，见 §3.1 | +| 15 | Outbox 失败终态 | 取消 `DEAD`，发送无限退避重试 + 告警 | 设终态会让 Broker 长期故障时任务静默停滞，违背 §2.1 的“提交成功的任务最终一定被调度”。靠告警驱动人工介入，Broker 恢复后自动追平 | +| 16 | 续租方式 | 独立 watchdog 线程按 60 秒周期，覆盖全执行期 | 挂在批次循环里会漏掉最终 `write()`/压缩/落盘阶段，慢磁盘下大文件会在临成功时被误判失联并反复重跑 | +| 17 | 时间基准 | 全部走数据库 `CURRENT_TIMESTAMP(3)` | 实例时钟偏差会导致提前抢占（两实例同写一任务）或延迟恢复 | +| 18 | 多实例部署 | 代码支持，**一期部署按单节点**；需多实例时先上跨主机共享卷 | 宿主机 bind mount 只在同主机内共享，跨主机会导致下载随机 404 与永久孤儿文件（§11.1） | +| 19 | 快照强度 | 一期只保证“不混入导出开始后新增的数据”，不冻结修改与删除 | 最大 ID 游标的固有限制。对客户列表可接受；需要完整快照的类型由该 Provider 自行实现临时表或一致性读（§10.2） | +| 20 | 权限取值时点 | 创建与人工重试时校验，执行阶段不校验；不存权限快照 | 执行在后台无用户会话，无法重新鉴权。已受理即视为已授权，代价与已知缺口写明于 §10.4 | +| 21 | 毒消息处置 | 独立 DLQ Handler 自动将匹配的 Outbox 置 `QUARANTINED`、任务置 `FAILED` | 人工隔离无法保证在 Reconciler 10 分钟补投窗口内完成；定位字段放 User Properties，避免依赖损坏 payload | + +仍需运维/管理侧确认（不影响设计定型）： + +- 测试与生产环境 Broker 的机器资源与 NameServer 地址。 +- Prometheus 服务端与告警通道的落地责任人。 +- DLQ 的值班责任人（Outbox 不再有终态失败，靠积压告警驱动，无需单独值班）。 + +## 22. 容量基准与放大路径 + +本文参数按平台总用户约 1000、导出为低频后台操作取值。估算：日导出任务 < 50，峰值并发提交 < 10，单任务行数通常 < 5 万。 + +| 维度 | 当前取值 | 何时需要调整 | 调整方式（不需改架构） | +|---|---|---|---| +| Consumer 并发 | 4 | 排队时长告警持续触发，且 §14 的 CPU、堆、磁盘、Provider 与 IAM 延迟门槛均有余量 | 单实例可直接压测更高 `concurrency`；增加 File 实例前同时复核 Topic 队列分配并启用共享存储 | +| Broker | 单节点 ASYNC_MASTER | 对消息可靠性要求提高 | 加 Slave 或 DLedger；Outbox 保证不丢任务，Broker 丢消息由 Reconciler 补发 | +| Topic 队列数 | 4 | File 实例数接近/超过队列数，或出现队列分配不均、单队列热点 | 用 `mqadmin updateTopic -r/-w` 扩容；并发消费下队列数不需要等于线程总数 | +| 存储 | 共享目录 | 总量超过单机磁盘或需异地备份 | 实现 `ExportStorage` 的 MinIO/OSS 版本，替换 Bean | +| `max-rows` | 20 万 | 出现更大导出需求 | 调注册表；超过 100 万需考虑分文件打包（新需求，非本文范围） | +| Reconciler 周期 | 30 秒 | 任务量增长使单轮扫描变慢 | 缩小 `LIMIT` 并提高频率，或加分片条件 | + +**关键点：以上任何调整都只涉及配置或单个实现类，不触及 Outbox、幂等、租约、Reconciler 这套核心机制。** 这是本文追求“不需要改第二次”的具体含义——不是参数永不变，而是机制不用重做。 + +## 23. 关联文档 + +- 当前已实现版本的接入说明：[公共导出与下载中心接入说明](./公共导出与下载中心接入说明.md)（阶段 4 完成后需同步更新） +- 统一响应与 HTTP 200 约定：[统一响应约定改造方案-HTTP200](./统一响应约定改造方案-HTTP200.md) +- 目录与服务边界：[头程项目目录结构与服务说明](./头程项目目录结构与服务说明.md) +- 本模块表结构事实来源：`file/src/main/resources/db/file.sql`（开发阶段直接维护建表语句，不走增量脚本，见 §7.3） +- 生产化后的增量执行边界：`deploy/sql/README.md` diff --git a/file/AGENTS.md b/file/AGENTS.md index af6b01f9c..79bec75e6 100644 --- a/file/AGENTS.md +++ b/file/AGENTS.md @@ -16,6 +16,10 @@ | 平台资质上传、替换与查看 | `controller/PlatformQualificationFileController.java`；`POST /{tenantId}` 支持 `qualificationType`、`certificateNo`、`validTo`、`replace`，下载为 `GET /{tenantId}/{fileName}` | | 服务状态 | `controller/FileHealthController.java` | | 当前本地磁盘实现 | `service/LocalQualificationStorage.java` | +| 公共导出 API | `controller/PlatformExportTaskController.java`、`service/ExportTaskService.java` | +| 可靠调度 | `service/ExportStateService.java`、`mq/ExportOutboxPublisher.java`、`mq/ExportCommandConsumer.java`、`service/ExportReconciler.java` | +| 通用 Provider/XLSX | `client/ExportProviderClient.java`、`service/GenericXlsxExportRunner.java`、公共契约 `../common-core/.../export/ExportContracts.java` | +| 毒消息隔离 | `mq/ExportDlqHandler.java`；定位必须使用四个 User Properties，不解析损坏 body | | 未被当前 Controller 使用的旧实现 | `service/QualificationFileStorage.java` | | IAM 权限协作 | `client/IamQualificationClient.java`；商户登记 `/qualifications`，平台登记/替换 `/platform-qualifications` | | 目标表结构（尚未接入） | `src/main/resources/db/file.sql` | @@ -26,6 +30,7 @@ ## 修改与验证 - 严格校验租户、文件名/路径、大小、类型和读取权限，防止目录穿越、越权访问和任意文件上传。 +- 企业资质文件只接受 PDF、JPG 和 PNG，单个文件上限 5MB；Client、Admin 与 `LocalQualificationStorage` 的限制和提示必须保持一致。 - 生产存储应使用对象存储或共享文件存储；容器本地目录仅适合当前本地实现，不得硬编码环境路径。 - 接入元数据持久化时需同时实现 Entity、Mapper、Repository、事务和迁移验证，并同步检查 IAM 资质流程及平台/商户 Controller。 - 从父目录运行 `mvn -pl file -am test`，并按接口影响验证商户/平台上传、替换、预览、无权限、签名伪造和磁盘故障场景。 diff --git a/file/pom.xml b/file/pom.xml index 4f70fe49d..7c0efeae4 100644 --- a/file/pom.xml +++ b/file/pom.xml @@ -17,6 +17,7 @@ org.springframework.cloudspring-cloud-starter-openfeign org.springframework.cloudspring-cloud-starter-loadbalancer org.springframework.bootspring-boot-starter-actuator + org.apache.rocketmqrocketmq-spring-boot-starter com.mysqlmysql-connector-jruntime org.springframework.bootspring-boot-starter-testtest diff --git a/file/src/main/java/com/baiyue/file/client/ExportProviderClient.java b/file/src/main/java/com/baiyue/file/client/ExportProviderClient.java new file mode 100644 index 000000000..eaf2752cf --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/ExportProviderClient.java @@ -0,0 +1,47 @@ +package com.baiyue.file.client; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import com.baiyue.file.config.ExportDefinitionRegistry.Definition; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Component +public class ExportProviderClient { + private final Map clients; + private final MeterRegistry meters; + + public ExportProviderClient(IamExportProviderClient iam, OrderExportProviderClient order, + WarehouseExportProviderClient warehouse, FirstlegExportProviderClient firstleg, + MeterRegistry meters) { + this.clients = Map.of("iam-service", iam, "order-service", order, + "warehouse-service", warehouse, "firstleg-service", firstleg); + this.meters = meters; + } + + public ApiResponse batch(String internalToken, String exportType, + Definition definition, ExportBatchRequest request) { + ExportProviderFeignClient client = clients.get(definition.getProviderService()); + if (client == null) throw new IllegalStateException("Export provider is not allowlisted: " + definition.getProviderService()); + long started = System.nanoTime(); + try { + ApiResponse response = client.batch(internalToken, exportType, request); + if (response == null || !response.successful()) failure(definition.getProviderService(), exportType); + return response; + } catch (RuntimeException exception) { + failure(definition.getProviderService(), exportType); + throw exception; + } finally { + meters.timer("export_provider_request_seconds", "service", definition.getProviderService(), "type", exportType) + .record(System.nanoTime() - started, TimeUnit.NANOSECONDS); + } + } + + private void failure(String service, String type) { + meters.counter("export_provider_failure_total", "service", service, "type", type).increment(); + } +} diff --git a/file/src/main/java/com/baiyue/file/client/ExportProviderFeignClient.java b/file/src/main/java/com/baiyue/file/client/ExportProviderFeignClient.java new file mode 100644 index 000000000..d069633f6 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/ExportProviderFeignClient.java @@ -0,0 +1,17 @@ +package com.baiyue.file.client; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; + +interface ExportProviderFeignClient { + @PostMapping("/internal/exports/v1/{exportType}/batches") + ApiResponse batch( + @RequestHeader("X-Internal-Service-Token") String internalToken, + @PathVariable String exportType, + @RequestBody ExportBatchRequest request); +} diff --git a/file/src/main/java/com/baiyue/file/client/ExportProviderFeignConfiguration.java b/file/src/main/java/com/baiyue/file/client/ExportProviderFeignConfiguration.java new file mode 100644 index 000000000..b6df5f054 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/ExportProviderFeignConfiguration.java @@ -0,0 +1,31 @@ +package com.baiyue.file.client; + +import feign.Response; +import feign.codec.Decoder; +import org.springframework.beans.factory.ObjectFactory; +import org.springframework.boot.autoconfigure.http.HttpMessageConverters; +import org.springframework.cloud.openfeign.support.SpringDecoder; +import org.springframework.context.annotation.Bean; + +import java.io.IOException; +import java.io.InputStream; + +public class ExportProviderFeignConfiguration { + static final int MAX_RESPONSE_BYTES = 4 * 1024 * 1024; + + @Bean + Decoder exportProviderDecoder(ObjectFactory converters) { + Decoder delegate = new SpringDecoder(converters); + return (response, type) -> delegate.decode(withLimitedBody(response), type); + } + + private Response withLimitedBody(Response response) throws IOException { + if (response.body() == null) return response; + byte[] bytes; + try (InputStream input = response.body().asInputStream()) { + bytes = input.readNBytes(MAX_RESPONSE_BYTES + 1); + } + if (bytes.length > MAX_RESPONSE_BYTES) throw new ExportProviderResponseTooLargeException(MAX_RESPONSE_BYTES); + return response.toBuilder().body(bytes).build(); + } +} diff --git a/file/src/main/java/com/baiyue/file/client/ExportProviderResponseTooLargeException.java b/file/src/main/java/com/baiyue/file/client/ExportProviderResponseTooLargeException.java new file mode 100644 index 000000000..18b1a5a91 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/ExportProviderResponseTooLargeException.java @@ -0,0 +1,7 @@ +package com.baiyue.file.client; + +public class ExportProviderResponseTooLargeException extends RuntimeException { + public ExportProviderResponseTooLargeException(long limit) { + super("Provider 单批响应超过 " + limit + " 字节"); + } +} diff --git a/file/src/main/java/com/baiyue/file/client/FirstlegExportProviderClient.java b/file/src/main/java/com/baiyue/file/client/FirstlegExportProviderClient.java new file mode 100644 index 000000000..e17e61489 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/FirstlegExportProviderClient.java @@ -0,0 +1,6 @@ +package com.baiyue.file.client; + +import org.springframework.cloud.openfeign.FeignClient; + +@FeignClient(name="firstleg-service", contextId="firstlegExportProviderClient", url="${baiyue.clients.firstleg.url:}", configuration=ExportProviderFeignConfiguration.class) +interface FirstlegExportProviderClient extends ExportProviderFeignClient { } diff --git a/file/src/main/java/com/baiyue/file/client/IamExportClient.java b/file/src/main/java/com/baiyue/file/client/IamExportClient.java deleted file mode 100644 index fc48270c6..000000000 --- a/file/src/main/java/com/baiyue/file/client/IamExportClient.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.baiyue.file.client; - -import com.baiyue.common.core.api.ApiResponse; -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestParam; - -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.List; - -@FeignClient(name = "iam-service", contextId = "iamExportClient", - url = "${baiyue.clients.iam.url:}", path = "/internal/iam") -public interface IamExportClient { - @GetMapping("/exports/customers") - ApiResponse customers( - @RequestHeader("X-Internal-Service-Token") String internalToken, - @RequestParam(required = false) Long cursor, - @RequestParam(defaultValue = "500") int limit, - @RequestParam(required = false) String keyword, - @RequestParam(required = false) String status); - - record CustomerExportBatch(List records, Long nextCursor, boolean hasMore) { } - - record CustomerExportRow( - String id, String tenantCode, String tenantName, String legalName, - String unifiedSocialCreditCode, String enterpriseType, LocalDate establishedDate, - BigDecimal registeredCapital, String registeredCapitalCurrency, String registeredAddress, - String contactName, String contactPhone, String contactEmail, String contactWechat, - String salesOwnerName, String servicePackage, Integer paymentTermDays, - String status, LocalDateTime createdAt, LocalDateTime approvedAt) { } -} diff --git a/file/src/main/java/com/baiyue/file/client/IamExportProviderClient.java b/file/src/main/java/com/baiyue/file/client/IamExportProviderClient.java new file mode 100644 index 000000000..e1af7b346 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/IamExportProviderClient.java @@ -0,0 +1,6 @@ +package com.baiyue.file.client; + +import org.springframework.cloud.openfeign.FeignClient; + +@FeignClient(name="iam-service", contextId="iamExportProviderClient", url="${baiyue.clients.iam.url:}", configuration=ExportProviderFeignConfiguration.class) +interface IamExportProviderClient extends ExportProviderFeignClient { } diff --git a/file/src/main/java/com/baiyue/file/client/OrderExportProviderClient.java b/file/src/main/java/com/baiyue/file/client/OrderExportProviderClient.java new file mode 100644 index 000000000..422d96da9 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/OrderExportProviderClient.java @@ -0,0 +1,6 @@ +package com.baiyue.file.client; + +import org.springframework.cloud.openfeign.FeignClient; + +@FeignClient(name="order-service", contextId="orderExportProviderClient", url="${baiyue.clients.order.url:}", configuration=ExportProviderFeignConfiguration.class) +interface OrderExportProviderClient extends ExportProviderFeignClient { } diff --git a/file/src/main/java/com/baiyue/file/client/WarehouseExportProviderClient.java b/file/src/main/java/com/baiyue/file/client/WarehouseExportProviderClient.java new file mode 100644 index 000000000..23dde5644 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/client/WarehouseExportProviderClient.java @@ -0,0 +1,6 @@ +package com.baiyue.file.client; + +import org.springframework.cloud.openfeign.FeignClient; + +@FeignClient(name="warehouse-service", contextId="warehouseExportProviderClient", url="${baiyue.clients.warehouse.url:}", configuration=ExportProviderFeignConfiguration.class) +interface WarehouseExportProviderClient extends ExportProviderFeignClient { } diff --git a/file/src/main/java/com/baiyue/file/config/ExportDefinitionRegistry.java b/file/src/main/java/com/baiyue/file/config/ExportDefinitionRegistry.java new file mode 100644 index 000000000..3b6bc0713 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/config/ExportDefinitionRegistry.java @@ -0,0 +1,78 @@ +package com.baiyue.file.config; + +import jakarta.annotation.PostConstruct; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +@Component +@ConfigurationProperties(prefix = "baiyue.file.export") +public class ExportDefinitionRegistry { + private static final Set SERVICES = Set.of("iam-service", "order-service", "warehouse-service", "firstleg-service"); + private static final Pattern TYPE = Pattern.compile("[A-Z0-9_]+"); + private static final Pattern PATH = Pattern.compile("/internal/exports/v1/([A-Z0-9_]+)/batches"); + private static final Pattern PERMISSION = Pattern.compile("[a-z]+:[a-z-]+:[a-z-]+"); + private Map definitions = Map.of(); + + @PostConstruct + public void validate() { + if (definitions.isEmpty()) throw new IllegalStateException("At least one export definition is required"); + definitions.forEach(this::validateDefinition); + } + + public Definition require(String exportType) { + Definition definition = definitions.get(exportType); + if (definition == null) throw new IllegalArgumentException("Unsupported export type: " + exportType); + return definition; + } + + private void validateDefinition(String type, Definition d) { + if (!TYPE.matcher(type).matches() || d == null) throw new IllegalStateException("Invalid export type: " + type); + if (!SERVICES.contains(d.providerService)) throw new IllegalStateException("Export provider is not allowlisted: " + d.providerService); + var path = PATH.matcher(d.providerPath == null ? "" : d.providerPath); + if (!path.matches() || !type.equals(path.group(1))) throw new IllegalStateException("Invalid provider path for " + type); + if (d.requiredPermission == null || !PERMISSION.matcher(d.requiredPermission).matches()) throw new IllegalStateException("Invalid export permission for " + type); + if (!Set.of("PLATFORM", "MERCHANT").contains(d.appScope)) throw new IllegalStateException("Invalid app scope for " + type); + if (d.maxRows < 1 || d.maxRows > 500_000 || d.batchSize < 50 || d.batchSize > 1_000 || d.maxAttempts < 1 || d.maxAttempts > 5) { + throw new IllegalStateException("Invalid export limits for " + type); + } + } + + public Map getDefinitions() { return definitions; } + public void setDefinitions(Map definitions) { + Map candidate = definitions == null ? Map.of() : Map.copyOf(definitions); + if (candidate.isEmpty()) throw new IllegalStateException("At least one export definition is required"); + candidate.forEach(this::validateDefinition); + this.definitions = candidate; + } + + public static class Definition { + private String providerService; + private String providerPath; + private String requiredPermission; + private String appScope; + private boolean enabled = true; + private int maxRows = 200_000; + private int maxAttempts = 3; + private int batchSize = 500; + public String getProviderService() { return providerService; } + public void setProviderService(String value) { providerService = value; } + public String getProviderPath() { return providerPath; } + public void setProviderPath(String value) { providerPath = value; } + public String getRequiredPermission() { return requiredPermission; } + public void setRequiredPermission(String value) { requiredPermission = value; } + public String getAppScope() { return appScope; } + public void setAppScope(String value) { appScope = value; } + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean value) { enabled = value; } + public int getMaxRows() { return maxRows; } + public void setMaxRows(int value) { maxRows = value; } + public int getMaxAttempts() { return maxAttempts; } + public void setMaxAttempts(int value) { maxAttempts = value; } + public int getBatchSize() { return batchSize; } + public void setBatchSize(int value) { batchSize = value; } + } +} diff --git a/file/src/main/java/com/baiyue/file/controller/PlatformQualificationFileController.java b/file/src/main/java/com/baiyue/file/controller/PlatformQualificationFileController.java index fc62ba24c..0b574d73a 100644 --- a/file/src/main/java/com/baiyue/file/controller/PlatformQualificationFileController.java +++ b/file/src/main/java/com/baiyue/file/controller/PlatformQualificationFileController.java @@ -45,6 +45,7 @@ public class PlatformQualificationFileController { @RequestParam(required = false) LocalDate validTo, @RequestParam(defaultValue = "false") boolean replace) { requirePlatform(user); + qualificationType = QualificationTypes.requireSupported(qualificationType); var stored = storage.store(tenantId, file); try { var response = iamClient.submitPlatform(internalToken, diff --git a/file/src/main/java/com/baiyue/file/controller/QualificationFileController.java b/file/src/main/java/com/baiyue/file/controller/QualificationFileController.java index 02044c6fe..cbda3cdbc 100644 --- a/file/src/main/java/com/baiyue/file/controller/QualificationFileController.java +++ b/file/src/main/java/com/baiyue/file/controller/QualificationFileController.java @@ -33,11 +33,12 @@ public class QualificationFileController { public ApiResponse upload( @AuthenticationPrincipal AuthenticatedUser user, @RequestPart("file") MultipartFile file, - @RequestParam(defaultValue = "BUSINESS_LICENSE") String qualificationType, + @RequestParam String qualificationType, @RequestParam(value = "certificateNo", required = false) String certificateNo, @RequestParam(required = false) LocalDate validTo, @RequestParam(defaultValue = "false") boolean replace) { long tenantId = merchantTenant(user); + qualificationType = QualificationTypes.requireSupported(qualificationType); var stored = storage.store(tenantId, file); try { var response = iamClient.submit(internalToken, diff --git a/file/src/main/java/com/baiyue/file/controller/QualificationTypes.java b/file/src/main/java/com/baiyue/file/controller/QualificationTypes.java new file mode 100644 index 000000000..6062ca9c6 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/controller/QualificationTypes.java @@ -0,0 +1,22 @@ +package com.baiyue.file.controller; + +import com.baiyue.common.core.exception.BizException; + +import java.util.Set; + +final class QualificationTypes { + private static final Set SUPPORTED = Set.of( + "BUSINESS_LICENSE", + "LEGAL_PERSON_ID", + "FREIGHT_FORWARDER_CERTIFICATE", + "BANK_ACCOUNT_LICENSE"); + + private QualificationTypes() { } + + static String requireSupported(String qualificationType) { + if (qualificationType == null || !SUPPORTED.contains(qualificationType)) { + throw new BizException("Unsupported qualification type / 不支持的资质类型"); + } + return qualificationType; + } +} diff --git a/file/src/main/java/com/baiyue/file/entity/FileExportOutbox.java b/file/src/main/java/com/baiyue/file/entity/FileExportOutbox.java new file mode 100644 index 000000000..9020089a8 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/entity/FileExportOutbox.java @@ -0,0 +1,31 @@ +package com.baiyue.file.entity; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Getter; +import lombok.Setter; + +import java.time.LocalDateTime; + +@Getter +@Setter +@TableName("file_export_outbox") +public class FileExportOutbox { + @TableId private Long id; + private String eventId; + private Long taskId; + private Integer dispatchVersion; + private String topic; + private String tag; + private String messageKey; + private String payloadJson; + private String status; + private Integer attemptCount; + private LocalDateTime nextRetryAt; + private String leaseOwner; + private LocalDateTime leaseUntil; + private LocalDateTime sentAt; + private String lastError; + private LocalDateTime createdAt; + private LocalDateTime updatedAt; +} diff --git a/file/src/main/java/com/baiyue/file/entity/FileExportTask.java b/file/src/main/java/com/baiyue/file/entity/FileExportTask.java index 23fd60421..d2752731f 100644 --- a/file/src/main/java/com/baiyue/file/entity/FileExportTask.java +++ b/file/src/main/java/com/baiyue/file/entity/FileExportTask.java @@ -28,6 +28,14 @@ public class FileExportTask extends BaseEntity { private String contentType; private Long sizeBytes; private String failureMessage; + private Integer dispatchVersion; + private Integer attemptCount; + private Integer maxAttempts; + private LocalDateTime nextRetryAt; + private String leaseOwner; + private LocalDateTime leaseUntil; + private LocalDateTime lastHeartbeatAt; + private LocalDateTime executionDeadlineAt; private Long requestedBy; private String requestedUsername; private LocalDateTime requestedAt; diff --git a/file/src/main/java/com/baiyue/file/mapper/FileExportOutboxMapper.java b/file/src/main/java/com/baiyue/file/mapper/FileExportOutboxMapper.java new file mode 100644 index 000000000..5ed9e1503 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/mapper/FileExportOutboxMapper.java @@ -0,0 +1,55 @@ +package com.baiyue.file.mapper; + +import com.baiyue.file.entity.FileExportOutbox; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; + +import java.util.List; + +@Mapper +public interface FileExportOutboxMapper extends BaseMapper { + @Insert(""" + INSERT INTO file_export_outbox(id,event_id,task_id,dispatch_version,topic,tag,message_key,payload_json, + status,attempt_count,next_retry_at,created_at,updated_at) + VALUES(#{id},#{eventId},#{taskId},#{dispatchVersion},#{topic},#{tag},#{messageKey},CAST(#{payloadJson} AS JSON), + 'PENDING',0,(SELECT next_retry_at FROM file_export_task WHERE id=#{taskId}),CURRENT_TIMESTAMP(3),CURRENT_TIMESTAMP(3)) + """) + int insertCommand(FileExportOutbox row); + @Select("SELECT id FROM file_export_outbox WHERE status='PENDING' AND next_retry_at<=CURRENT_TIMESTAMP(3) ORDER BY next_retry_at,id LIMIT #{limit}") + List publishableIds(@Param("limit") int limit); + + @Update("UPDATE file_export_outbox SET status='CLAIMED',lease_owner=#{owner},lease_until=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{leaseSeconds} SECOND),attempt_count=attempt_count+1 WHERE id=#{id} AND status='PENDING'") + int claim(@Param("id") long id, @Param("owner") String owner, @Param("leaseSeconds") int leaseSeconds); + + @Update("UPDATE file_export_outbox SET status='SENT',sent_at=CURRENT_TIMESTAMP(3),lease_owner=NULL,lease_until=NULL,last_error=NULL WHERE id=#{id} AND status='CLAIMED' AND lease_owner=#{owner}") + int sent(@Param("id") long id, @Param("owner") String owner); + + @Update("UPDATE file_export_outbox SET status='PENDING',next_retry_at=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{backoffSeconds} SECOND),lease_owner=NULL,lease_until=NULL,last_error=#{error} WHERE id=#{id} AND status='CLAIMED' AND lease_owner=#{owner}") + int sendFailed(@Param("id") long id, @Param("owner") String owner, + @Param("backoffSeconds") int backoffSeconds, @Param("error") String error); + + @Update("UPDATE file_export_outbox SET status='PENDING',next_retry_at=CURRENT_TIMESTAMP(3),lease_owner=NULL,lease_until=NULL WHERE status='CLAIMED' AND lease_until<=CURRENT_TIMESTAMP(3) LIMIT 50") + int releaseExpiredClaims(); + + @Update(""" + UPDATE file_export_outbox o JOIN file_export_task t + ON t.id=o.task_id AND t.dispatch_version=o.dispatch_version + SET o.status='PENDING',o.next_retry_at=CURRENT_TIMESTAMP(3),o.lease_owner=NULL,o.lease_until=NULL + WHERE o.status='SENT' AND o.sent_at<=DATE_SUB(CURRENT_TIMESTAMP(3), INTERVAL #{staleSeconds} SECOND) + AND t.status='PENDING' AND t.next_retry_at<=DATE_SUB(CURRENT_TIMESTAMP(3), INTERVAL #{graceSeconds} SECOND) + LIMIT 20 + """) + int redispatchSent(@Param("staleSeconds") int staleSeconds, @Param("graceSeconds") int graceSeconds); + + @Update("UPDATE file_export_outbox SET status='QUARANTINED',last_error=#{reason},lease_owner=NULL,lease_until=NULL WHERE event_id=#{eventId} AND task_id=#{taskId} AND dispatch_version=#{dispatchVersion} AND status<>'QUARANTINED'") + int quarantine(@Param("eventId") String eventId, @Param("taskId") long taskId, + @Param("dispatchVersion") int dispatchVersion, @Param("reason") String reason); + + @Delete("DELETE FROM file_export_outbox WHERE status='SENT' AND sent_at { } +public interface FileExportTaskMapper extends BaseMapper { + @Select("SELECT guard_key FROM file_export_task_guard WHERE guard_key='GLOBAL_CREATE' FOR UPDATE") + String lockCreationGuard(); + + @Select("SELECT guard_key FROM file_export_task_guard WHERE guard_key='GLOBAL_CREATE'") + String findCreationGuard(); + + @Update("UPDATE file_export_task SET next_retry_at=CURRENT_TIMESTAMP(3) WHERE id=#{id} AND status='PENDING' AND dispatch_version=1") + int initializeDispatchTime(@Param("id") long id); + @Update(""" + UPDATE file_export_task SET status='RUNNING', attempt_count=attempt_count+1, + started_at=CURRENT_TIMESTAMP(3), lease_owner=#{owner}, + lease_until=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{leaseSeconds} SECOND), + last_heartbeat_at=CURRENT_TIMESTAMP(3), + execution_deadline_at=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{durationMinutes} MINUTE), + processed_rows=0,total_rows=NULL,failure_message=NULL + WHERE id=#{id} AND dispatch_version=#{dispatchVersion} AND status='PENDING' + AND next_retry_at<=CURRENT_TIMESTAMP(3) + """) + int claim(@Param("id") long id, @Param("dispatchVersion") int dispatchVersion, + @Param("owner") String owner, @Param("leaseSeconds") int leaseSeconds, + @Param("durationMinutes") int durationMinutes); + + @Update(""" + UPDATE file_export_task SET lease_until=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{leaseSeconds} SECOND), + last_heartbeat_at=CURRENT_TIMESTAMP(3) WHERE id=#{id} AND status='RUNNING' + AND dispatch_version=#{dispatchVersion} AND lease_owner=#{owner} + AND execution_deadline_at>CURRENT_TIMESTAMP(3) + """) + int heartbeat(@Param("id") long id, @Param("dispatchVersion") int dispatchVersion, + @Param("owner") String owner, @Param("leaseSeconds") int leaseSeconds); + + @Update("UPDATE file_export_task SET processed_rows=#{rows} WHERE id=#{id} AND status='RUNNING' AND dispatch_version=#{dispatchVersion} AND lease_owner=#{owner}") + int progress(@Param("id") long id, @Param("dispatchVersion") int dispatchVersion, + @Param("owner") String owner, @Param("rows") long rows); + + @Update(""" + UPDATE file_export_task SET status='SUCCEEDED',file_name=#{fileName},object_key=#{objectKey}, + content_type=#{contentType},size_bytes=#{sizeBytes},total_rows=#{rows},processed_rows=#{rows}, + completed_at=CURRENT_TIMESTAMP(3),expires_at=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{retentionDays} DAY), + lease_owner=NULL,lease_until=NULL,failure_message=NULL + WHERE id=#{id} AND status='RUNNING' AND dispatch_version=#{dispatchVersion} + AND lease_owner=#{owner} AND execution_deadline_at>CURRENT_TIMESTAMP(3) + """) + int succeed(@Param("id") long id, @Param("dispatchVersion") int dispatchVersion, + @Param("owner") String owner, @Param("fileName") String fileName, + @Param("objectKey") String objectKey, @Param("contentType") String contentType, + @Param("sizeBytes") long sizeBytes, @Param("rows") long rows, + @Param("retentionDays") int retentionDays); + + @Update(""" + UPDATE file_export_task SET status='PENDING',dispatch_version=dispatch_version+1, + next_retry_at=DATE_ADD(CURRENT_TIMESTAMP(3), INTERVAL #{backoffSeconds} SECOND), + lease_owner=NULL,lease_until=NULL,last_heartbeat_at=NULL,execution_deadline_at=NULL, + failure_message=#{message} WHERE id=#{id} AND status='RUNNING' + AND dispatch_version=#{dispatchVersion} AND lease_owner=#{owner} AND attempt_count=max_attempts + """) + int failStale(@Param("id") long id, @Param("dispatchVersion") int dispatchVersion); + + @Select("SELECT * FROM file_export_task WHERE status='SUCCEEDED' AND expires_at<=CURRENT_TIMESTAMP(3) AND object_key IS NOT NULL ORDER BY expires_at LIMIT 50") + List expiredArtifactsDb(); + + @Select("SELECT COUNT(*) FROM file_export_task WHERE tenant_id=#{tenantId} AND app_code=#{appCode} AND requested_by=#{requestedBy} AND status IN ('PENDING','RUNNING')") + long activeForUser(@Param("tenantId") long tenantId,@Param("appCode") String appCode,@Param("requestedBy") long requestedBy); + + @Select("SELECT COUNT(*) FROM file_export_task WHERE export_type=#{exportType} AND status IN ('PENDING','RUNNING')") + long activeForType(@Param("exportType") String exportType); + + @Select("SELECT * FROM file_export_task WHERE tenant_id=#{tenantId} AND app_code=#{appCode} AND requested_by=#{requestedBy} AND export_type=#{exportType} AND parameters_json=CAST(#{parametersJson} AS JSON) AND status IN ('PENDING','RUNNING') AND requested_at>=DATE_SUB(CURRENT_TIMESTAMP(3),INTERVAL 60 SECOND) ORDER BY requested_at DESC LIMIT 1") + FileExportTask recentDuplicate(@Param("tenantId") long tenantId,@Param("appCode") String appCode,@Param("requestedBy") long requestedBy,@Param("exportType") String exportType,@Param("parametersJson") String parametersJson); + + @Select("SELECT COUNT(*) FROM file_export_task WHERE id=#{id} AND status='SUCCEEDED' AND expires_at>CURRENT_TIMESTAMP(3)") + int downloadable(@Param("id") long id); + + @Select("SELECT COUNT(*) FROM file_export_task WHERE status='SUCCEEDED' AND object_key=#{objectKey}") + int succeededObjectExists(@Param("objectKey") String objectKey); + + @Select("SELECT * FROM file_export_task WHERE status='RUNNING' AND lease_until<=CURRENT_TIMESTAMP(3) ORDER BY lease_until LIMIT 20") + List staleRunning(); + + @Select("SELECT * FROM file_export_task WHERE status='PENDING' AND next_retry_at<=DATE_SUB(CURRENT_TIMESTAMP(3), INTERVAL #{graceSeconds} SECOND) AND NOT EXISTS (SELECT 1 FROM file_export_outbox o WHERE o.task_id=file_export_task.id AND o.dispatch_version=file_export_task.dispatch_version) ORDER BY id LIMIT 50") + List missingOutbox(@Param("graceSeconds") int graceSeconds); + + @Update("UPDATE file_export_task SET status='EXPIRED' WHERE id=#{id} AND status='SUCCEEDED'") + int markExpiredDb(@Param("id") long id); +} diff --git a/file/src/main/java/com/baiyue/file/mq/ExportCommandConsumer.java b/file/src/main/java/com/baiyue/file/mq/ExportCommandConsumer.java new file mode 100644 index 000000000..315ef023f --- /dev/null +++ b/file/src/main/java/com/baiyue/file/mq/ExportCommandConsumer.java @@ -0,0 +1,52 @@ +package com.baiyue.file.mq; + +import com.baiyue.common.core.export.ExportContracts.ExportCommand; +import com.baiyue.file.service.ExportTaskExecutor; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.spring.annotation.MessageModel; +import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; +import org.apache.rocketmq.spring.core.RocketMQListener; +import org.apache.rocketmq.spring.core.RocketMQPushConsumerLifecycleListener; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; + +@Service +@ConditionalOnProperty(prefix="baiyue.file.export.consumer", name="enabled", havingValue="true") +@RocketMQMessageListener(topic="${baiyue.file.export.rocketmq.topic}", consumerGroup="${baiyue.file.export.rocketmq.consumer-group}", + selectorExpression="EXPORT_TASK", messageModel= MessageModel.CLUSTERING, consumeThreadNumber=4, + consumeThreadMax=4, consumeTimeout=90, maxReconsumeTimes=2) +public class ExportCommandConsumer implements RocketMQListener, RocketMQPushConsumerLifecycleListener { + private final ObjectMapper objectMapper; + private final ExportTaskExecutor executor; + private final String instance; + private final int concurrency; + private final long consumeTimeoutMinutes; + private final int maxReconsumeTimes; + public ExportCommandConsumer(ObjectMapper objectMapper, ExportTaskExecutor executor, + @Value("${baiyue.file.export.instance-id:${HOSTNAME:${random.uuid}}}") String instance, + @Value("${baiyue.file.export.consumer.concurrency:4}") int concurrency, + @Value("${baiyue.file.export.consumer.consume-timeout-minutes:90}") long consumeTimeoutMinutes, + @Value("${baiyue.file.export.consumer.max-reconsume-times:2}") int maxReconsumeTimes) { + this.objectMapper=objectMapper;this.executor=executor;this.instance=instance; + if (concurrency < 1 || concurrency > 64) throw new IllegalArgumentException("Export consumer concurrency must be between 1 and 64"); + this.concurrency=concurrency;this.consumeTimeoutMinutes=consumeTimeoutMinutes;this.maxReconsumeTimes=maxReconsumeTimes; + } + @Override public void prepareStart(DefaultMQPushConsumer consumer) { + consumer.setConsumeThreadMin(concurrency); + consumer.setConsumeThreadMax(concurrency); + consumer.setConsumeTimeout(consumeTimeoutMinutes); + consumer.setMaxReconsumeTimes(maxReconsumeTimes); + } + @Override public void onMessage(MessageExt message) { + final ExportCommand command; + try { command = objectMapper.readValue(new String(message.getBody(), StandardCharsets.UTF_8), ExportCommand.class); } + catch (Exception exception) { throw new IllegalArgumentException("Malformed export command", exception); } + if (command.schemaVersion()!=1 || command.taskId()==null || command.eventId()==null) throw new IllegalArgumentException("Unsupported export command"); + executor.execute(Long.parseLong(command.taskId()), command.dispatchVersion(), instance); + } +} diff --git a/file/src/main/java/com/baiyue/file/mq/ExportDlqHandler.java b/file/src/main/java/com/baiyue/file/mq/ExportDlqHandler.java new file mode 100644 index 000000000..c5a3bfbc5 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/mq/ExportDlqHandler.java @@ -0,0 +1,36 @@ +package com.baiyue.file.mq; + +import com.baiyue.file.service.ExportStateService; +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.spring.annotation.MessageModel; +import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; +import org.apache.rocketmq.spring.core.RocketMQListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +@Service +@ConditionalOnProperty(prefix="baiyue.file.export.consumer", name="enabled", havingValue="true") +@RocketMQMessageListener(topic="${baiyue.file.export.rocketmq.dlq-topic}", + consumerGroup="${baiyue.file.export.rocketmq.dlq-consumer-group}", messageModel=MessageModel.CLUSTERING, + consumeThreadNumber=1, consumeThreadMax=1, maxReconsumeTimes=Integer.MAX_VALUE) +public class ExportDlqHandler implements RocketMQListener { + private static final Logger LOGGER = LoggerFactory.getLogger(ExportDlqHandler.class); + private final ExportStateService states; + private final MeterRegistry meters; + public ExportDlqHandler(ExportStateService states, MeterRegistry meters) { this.states=states; this.meters=meters; } + @Override public void onMessage(MessageExt message) { + meters.counter("export_dlq_messages").increment(); + String eventId=message.getUserProperty("eventId"), taskId=message.getUserProperty("taskId"); + String version=message.getUserProperty("dispatchVersion"), schema=message.getUserProperty("schemaVersion"); + if (blank(eventId)||blank(taskId)||blank(version)||blank(schema)) { + LOGGER.error("CRITICAL: 畸形 DLQ 消息缺少定位属性 messageId={}", message.getMsgId()); + return; + } + try { states.quarantine(eventId, Long.parseLong(taskId), Integer.parseInt(version), "消息契约损坏，已自动隔离（schemaVersion="+schema+"）"); } + catch (NumberFormatException exception) { LOGGER.error("CRITICAL: 畸形 DLQ 消息定位属性非法 messageId={}", message.getMsgId(), exception); } + } + private boolean blank(String value) { return value==null||value.isBlank(); } +} diff --git a/file/src/main/java/com/baiyue/file/mq/ExportOutboxPublisher.java b/file/src/main/java/com/baiyue/file/mq/ExportOutboxPublisher.java new file mode 100644 index 000000000..1a86faea3 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/mq/ExportOutboxPublisher.java @@ -0,0 +1,54 @@ +package com.baiyue.file.mq; + +import com.baiyue.file.entity.FileExportOutbox; +import com.baiyue.file.repository.ExportOutboxRepository; +import org.apache.rocketmq.spring.core.RocketMQTemplate; +import org.apache.rocketmq.spring.support.RocketMQHeaders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +@Service +@ConditionalOnProperty(prefix="baiyue.file.export.publisher", name="enabled", havingValue="true") +public class ExportOutboxPublisher { + private static final Logger LOGGER = LoggerFactory.getLogger(ExportOutboxPublisher.class); + private static final int[] BACKOFF = {5, 15, 60, 300, 900}; + private final ExportOutboxRepository outbox; + private final RocketMQTemplate rocketMQ; + private final String owner; + private final int batchSize; + private final int leaseSeconds; + public ExportOutboxPublisher(ExportOutboxRepository outbox, RocketMQTemplate rocketMQ, + @Value("${baiyue.file.export.instance-id:${HOSTNAME:${random.uuid}}}") String owner, + @Value("${baiyue.file.export.publisher.batch-size:50}") int batchSize, + @Value("${baiyue.file.export.publisher.lease-seconds:60}") int leaseSeconds) { + this.outbox=outbox;this.rocketMQ=rocketMQ;this.owner=owner;this.batchSize=batchSize;this.leaseSeconds=leaseSeconds; + } + @Scheduled(fixedDelayString="${baiyue.file.export.publisher.poll-interval-ms:5000}") + public void publish() { + for (Long id : outbox.publishableIds(batchSize)) { + if (!outbox.claim(id, owner, leaseSeconds)) continue; + FileExportOutbox row = outbox.find(id); + try { + var message = MessageBuilder.withPayload(row.getPayloadJson()) + .setHeader(RocketMQHeaders.KEYS, row.getMessageKey()) + .setHeader("eventId", row.getEventId()) + .setHeader("taskId", String.valueOf(row.getTaskId())) + .setHeader("dispatchVersion", String.valueOf(row.getDispatchVersion())) + .setHeader("schemaVersion", "1").build(); + rocketMQ.syncSend(row.getTopic() + ":" + row.getTag(), message, 3000); + outbox.sent(id, owner); + } catch (RuntimeException exception) { + int attempt = Math.max(1, row.getAttemptCount()); + int seconds = BACKOFF[Math.min(attempt - 1, BACKOFF.length - 1)]; + outbox.sendFailed(id, owner, seconds, safe(exception.getMessage())); + LOGGER.error("导出命令发送失败 eventId={} taskId={}", row.getEventId(), row.getTaskId(), exception); + } + } + } + private String safe(String value) { String message=value==null?"RocketMQ send failed":value; return message.length()>1000?message.substring(0,1000):message; } +} diff --git a/file/src/main/java/com/baiyue/file/repository/ExportOutboxRepository.java b/file/src/main/java/com/baiyue/file/repository/ExportOutboxRepository.java new file mode 100644 index 000000000..e731fdfd3 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/repository/ExportOutboxRepository.java @@ -0,0 +1,27 @@ +package com.baiyue.file.repository; + +import com.baiyue.file.entity.FileExportOutbox; +import com.baiyue.file.mapper.FileExportOutboxMapper; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public class ExportOutboxRepository { + private final FileExportOutboxMapper mapper; + public ExportOutboxRepository(FileExportOutboxMapper mapper) { this.mapper = mapper; } + public boolean insert(FileExportOutbox row) { + try { return mapper.insertCommand(row) == 1; } + catch (DuplicateKeyException ignored) { return false; } + } + public FileExportOutbox find(long id) { return mapper.selectById(id); } + public List publishableIds(int limit) { return mapper.publishableIds(limit); } + public boolean claim(long id, String owner, int leaseSeconds) { return mapper.claim(id, owner, leaseSeconds) == 1; } + public boolean sent(long id, String owner) { return mapper.sent(id, owner) == 1; } + public boolean sendFailed(long id, String owner, int seconds, String error) { return mapper.sendFailed(id, owner, seconds, error) == 1; } + public int releaseExpiredClaims() { return mapper.releaseExpiredClaims(); } + public int redispatchSent(int staleSeconds, int graceSeconds) { return mapper.redispatchSent(staleSeconds, graceSeconds); } + public boolean quarantine(String eventId, long taskId, int version, String reason) { return mapper.quarantine(eventId, taskId, version, reason) == 1; } + public int deleteRetainedSent(int retentionDays, int limit) { return mapper.deleteRetainedSent(retentionDays, limit); } +} diff --git a/file/src/main/java/com/baiyue/file/repository/ExportTaskRepository.java b/file/src/main/java/com/baiyue/file/repository/ExportTaskRepository.java index 0455c953f..dcec05c84 100644 --- a/file/src/main/java/com/baiyue/file/repository/ExportTaskRepository.java +++ b/file/src/main/java/com/baiyue/file/repository/ExportTaskRepository.java @@ -6,6 +6,7 @@ import com.baiyue.file.mapper.FileExportTaskMapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Repository; import org.springframework.util.StringUtils; @@ -21,7 +22,60 @@ public class ExportTaskRepository { this.mapper = mapper; } + @PostConstruct + void validateCreationGuard() { + if (!"GLOBAL_CREATE".equals(mapper.findCreationGuard())) { + throw new IllegalStateException("Missing GLOBAL_CREATE export task guard; apply file.sql before starting File Service"); + } + } + + public void lockCreationGuard() { + if (!"GLOBAL_CREATE".equals(mapper.lockCreationGuard())) { + throw new IllegalStateException("Unable to acquire export task creation guard"); + } + } + public void insert(FileExportTask task) { mapper.insert(task); } + public void initializeDispatchTime(long id) { mapper.initializeDispatchTime(id); } + + public boolean claim(long id, int dispatchVersion, String owner, int leaseSeconds, int durationMinutes) { + return mapper.claim(id, dispatchVersion, owner, leaseSeconds, durationMinutes) == 1; + } + + public boolean heartbeat(long id, int dispatchVersion, String owner, int leaseSeconds) { + return mapper.heartbeat(id, dispatchVersion, owner, leaseSeconds) == 1; + } + + public boolean progress(long id, int dispatchVersion, String owner, long rows) { + return mapper.progress(id, dispatchVersion, owner, rows) == 1; + } + + public boolean succeed(long id, int dispatchVersion, String owner, String fileName, String objectKey, + String contentType, long sizeBytes, long rows, int retentionDays) { + return mapper.succeed(id, dispatchVersion, owner, fileName, objectKey, contentType, + sizeBytes, rows, retentionDays) == 1; + } + + public boolean scheduleRetry(long id, int dispatchVersion, String owner, int backoffSeconds, String message) { + return mapper.scheduleRetry(id, dispatchVersion, owner, backoffSeconds, message) == 1; + } + + public boolean fail(long id, int dispatchVersion, String owner, String message) { + return mapper.fail(id, dispatchVersion, owner, message) == 1; + } + public boolean manualRetry(long id) { return mapper.manualRetry(id) == 1; } + public boolean failQuarantined(long id, int dispatchVersion, String reason) { return mapper.failQuarantined(id, dispatchVersion, reason) == 1; } + public boolean recoverStale(long id, int dispatchVersion) { return mapper.recoverStale(id, dispatchVersion) == 1; } + public boolean failStale(long id, int dispatchVersion) { return mapper.failStale(id, dispatchVersion) == 1; } + public List expiredArtifactsDb() { return mapper.expiredArtifactsDb(); } + public long activeForUser(long tenantId,String appCode,long requestedBy){return mapper.activeForUser(tenantId,appCode,requestedBy);} + public long activeForType(String exportType){return mapper.activeForType(exportType);} + public FileExportTask recentDuplicate(long tenantId,String appCode,long requestedBy,String exportType,String json){return mapper.recentDuplicate(tenantId,appCode,requestedBy,exportType,json);} + public boolean downloadable(long id){return mapper.downloadable(id)>0;} + public boolean succeededObjectExists(String objectKey){return mapper.succeededObjectExists(objectKey)>0;} + + public List staleRunning() { return mapper.staleRunning(); } + public List missingOutbox(int graceSeconds) { return mapper.missingOutbox(graceSeconds); } public FileExportTask find(long id) { return mapper.selectById(id); } @@ -43,86 +97,6 @@ public class ExportTaskRepository { return PageResult.from(mapper.selectPage(new Page<>(page, size), query)); } - public Long firstPendingId() { - FileExportTask task = mapper.selectOne(new LambdaQueryWrapper() - .select(FileExportTask::getId) - .eq(FileExportTask::getStatus, "PENDING") - .orderByAsc(FileExportTask::getRequestedAt) - .orderByAsc(FileExportTask::getId) - .last("LIMIT 1")); - return task == null ? null : task.getId(); - } - - public boolean claim(long id, LocalDateTime now) { - return mapper.update(null, new LambdaUpdateWrapper() - .eq(FileExportTask::getId, id) - .eq(FileExportTask::getStatus, "PENDING") - .set(FileExportTask::getStatus, "RUNNING") - .set(FileExportTask::getStartedAt, now) - .set(FileExportTask::getFailureMessage, null)) == 1; - } - - public void progress(long id, long rows) { - mapper.update(null, new LambdaUpdateWrapper() - .eq(FileExportTask::getId, id) - .eq(FileExportTask::getStatus, "RUNNING") - .set(FileExportTask::getProcessedRows, rows)); - } - - public boolean succeed(long id, String fileName, String objectKey, String contentType, - long sizeBytes, long rows, LocalDateTime completedAt, LocalDateTime expiresAt) { - return mapper.update(null, new LambdaUpdateWrapper() - .eq(FileExportTask::getId, id) - .eq(FileExportTask::getStatus, "RUNNING") - .set(FileExportTask::getStatus, "SUCCEEDED") - .set(FileExportTask::getFileName, fileName) - .set(FileExportTask::getObjectKey, objectKey) - .set(FileExportTask::getContentType, contentType) - .set(FileExportTask::getSizeBytes, sizeBytes) - .set(FileExportTask::getTotalRows, rows) - .set(FileExportTask::getProcessedRows, rows) - .set(FileExportTask::getCompletedAt, completedAt) - .set(FileExportTask::getExpiresAt, expiresAt)) == 1; - } - - public void fail(long id, String message, LocalDateTime completedAt) { - mapper.update(null, new LambdaUpdateWrapper() - .eq(FileExportTask::getId, id) - .eq(FileExportTask::getStatus, "RUNNING") - .set(FileExportTask::getStatus, "FAILED") - .set(FileExportTask::getFailureMessage, message) - .set(FileExportTask::getCompletedAt, completedAt)); - } - - public boolean retry(long id) { - return mapper.update(null, new LambdaUpdateWrapper() - .eq(FileExportTask::getId, id) - .eq(FileExportTask::getStatus, "FAILED") - .set(FileExportTask::getStatus, "PENDING") - .set(FileExportTask::getProcessedRows, 0L) - .set(FileExportTask::getTotalRows, null) - .set(FileExportTask::getFailureMessage, null) - .set(FileExportTask::getStartedAt, null) - .set(FileExportTask::getCompletedAt, null)) == 1; - } - - public int requeueStale(LocalDateTime cutoff) { - return mapper.update(null, new LambdaUpdateWrapper() - .eq(FileExportTask::getStatus, "RUNNING") - .lt(FileExportTask::getUpdatedAt, cutoff) - .set(FileExportTask::getStatus, "PENDING") - .set(FileExportTask::getStartedAt, null) - .set(FileExportTask::getFailureMessage, "Previous worker stopped before completion; task requeued")); - } - - public List expiredArtifacts(LocalDateTime now) { - return mapper.selectList(new LambdaQueryWrapper() - .eq(FileExportTask::getStatus, "SUCCEEDED") - .le(FileExportTask::getExpiresAt, now) - .isNotNull(FileExportTask::getObjectKey) - .last("LIMIT 50")); - } - public void markExpired(long id) { mapper.update(null, new LambdaUpdateWrapper() .eq(FileExportTask::getId, id) diff --git a/file/src/main/java/com/baiyue/file/service/CustomerExportTaskHandler.java b/file/src/main/java/com/baiyue/file/service/CustomerExportTaskHandler.java deleted file mode 100644 index da0933569..000000000 --- a/file/src/main/java/com/baiyue/file/service/CustomerExportTaskHandler.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.baiyue.file.service; - -import com.baiyue.common.core.api.ResultCode; -import com.baiyue.common.core.exception.BizException; -import com.baiyue.file.client.IamExportClient; -import com.baiyue.file.entity.FileExportTask; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.poi.ss.usermodel.CellStyle; -import org.apache.poi.ss.usermodel.FillPatternType; -import org.apache.poi.ss.usermodel.IndexedColors; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Service -public class CustomerExportTaskHandler implements ExportTaskHandler { - static final String CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; - private static final List HEADERS = List.of( - "客户ID", "客户编码", "公司名称", "营业执照企业名称", "统一社会信用代码", "企业类型", - "成立日期", "注册资本", "币种", "注册地址", "联系人", "联系电话", "联系邮箱", "联系微信", - "销售人员", "客户套餐", "账期(天)", "状态", "入驻时间", "审批时间"); - - private final IamExportClient iamClient; - private final LocalExportStorage storage; - private final ObjectMapper objectMapper; - private final String internalToken; - - public CustomerExportTaskHandler(IamExportClient iamClient, LocalExportStorage storage, - ObjectMapper objectMapper, - @Value("${baiyue.security.internal-service-token}") String internalToken) { - this.iamClient = iamClient; - this.storage = storage; - this.objectMapper = objectMapper; - this.internalToken = internalToken; - } - - @Override public String exportType() { return "CUSTOMER_LIST"; } - @Override public String requiredPermission() { return "platform:customer:export"; } - - @Override - public void validateParameters(Map parameters) { - if (parameters == null) return; - Object keyword = parameters.get("keyword"); - if (keyword != null && String.valueOf(keyword).length() > 128) { - throw new BizException(ResultCode.VALIDATION_FAILED, "客户导出关键词不能超过 128 个字符"); - } - Object status = parameters.get("status"); - if (status != null && !String.valueOf(status).isBlank() - && !Set.of("PROFILE_REQUIRED", "PENDING", "ACTIVE", "REJECTED", "DISABLED") - .contains(String.valueOf(status))) { - throw new BizException(ResultCode.VALIDATION_FAILED, "客户导出状态无效"); - } - } - - @Override - public ExportResult export(FileExportTask task, ProgressReporter progress) { - CustomerParameters parameters = parameters(task.getParametersJson()); - LocalExportStorage.PendingFile target = storage.create(task.getId()); - long rows = 0; - try (target; SXSSFWorkbook workbook = new SXSSFWorkbook(100)) { - workbook.setCompressTempFiles(true); - Sheet sheet = workbook.createSheet("客户列表"); - writeHeader(workbook, sheet); - Long cursor = null; - boolean hasMore; - do { - var response = iamClient.customers(internalToken, cursor, 500, - blankToNull(parameters.keyword()), blankToNull(parameters.status())); - if (response == null || !response.successful() || response.data() == null) { - throw new BizException(ResultCode.SERVICE_UNAVAILABLE, "IAM customer export is unavailable"); - } - var batch = response.data(); - for (var item : batch.records()) writeRow(sheet.createRow(Math.toIntExact(++rows)), item); - progress.update(rows); - hasMore = batch.hasMore(); - if (hasMore && (batch.nextCursor() == null || batch.nextCursor().equals(cursor))) { - throw new IllegalStateException("IAM customer export cursor did not advance"); - } - cursor = batch.nextCursor(); - } while (hasMore); - for (int index = 0; index < HEADERS.size(); index++) sheet.setColumnWidth(index, columnWidth(index)); - workbook.write(target.output()); - } catch (IOException exception) { - storage.delete(target.objectKey()); - throw new BizException(ResultCode.SERVICE_UNAVAILABLE, "客户导出文件生成失败"); - } catch (RuntimeException exception) { - storage.delete(target.objectKey()); - throw exception; - } - String fileName = "客户列表-" + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")) + ".xlsx"; - return new ExportResult(fileName, target.objectKey(), CONTENT_TYPE, - storage.size(target.objectKey()), rows); - } - - private CustomerParameters parameters(String json) { - try { return objectMapper.readValue(json, CustomerParameters.class); } - catch (Exception exception) { throw new BizException(ResultCode.BAD_REQUEST, "客户导出筛选条件无效"); } - } - - private void writeHeader(Workbook workbook, Sheet sheet) { - CellStyle style = workbook.createCellStyle(); - style.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - style.setFillPattern(FillPatternType.SOLID_FOREGROUND); - var font = workbook.createFont(); - font.setBold(true); - style.setFont(font); - Row row = sheet.createRow(0); - for (int index = 0; index < HEADERS.size(); index++) { - var cell = row.createCell(index); - cell.setCellValue(HEADERS.get(index)); - cell.setCellStyle(style); - } - sheet.createFreezePane(0, 1); - } - - private void writeRow(Row row, IamExportClient.CustomerExportRow item) { - int column = 0; - cell(row, column++, item.id()); cell(row, column++, item.tenantCode()); - cell(row, column++, item.tenantName()); cell(row, column++, item.legalName()); - cell(row, column++, item.unifiedSocialCreditCode()); cell(row, column++, item.enterpriseType()); - cell(row, column++, item.establishedDate()); cell(row, column++, item.registeredCapital()); - cell(row, column++, item.registeredCapitalCurrency()); cell(row, column++, item.registeredAddress()); - cell(row, column++, item.contactName()); cell(row, column++, item.contactPhone()); - cell(row, column++, item.contactEmail()); cell(row, column++, item.contactWechat()); - cell(row, column++, item.salesOwnerName()); cell(row, column++, item.servicePackage()); - cell(row, column++, item.paymentTermDays()); cell(row, column++, statusLabel(item.status())); - cell(row, column++, item.createdAt()); cell(row, column, item.approvedAt()); - } - - private void cell(Row row, int column, Object value) { - row.createCell(column).setCellValue(value == null ? "" : String.valueOf(value)); - } - - private int columnWidth(int index) { - return switch (index) { - case 2, 3, 9 -> 28 * 256; - case 4, 12 -> 24 * 256; - case 18, 19 -> 21 * 256; - default -> 16 * 256; - }; - } - - private String statusLabel(String status) { - if (status == null) return ""; - return switch (status) { - case "PROFILE_REQUIRED" -> "待完善"; - case "PENDING" -> "待审核"; - case "ACTIVE" -> "启用"; - case "REJECTED" -> "已驳回"; - case "DISABLED" -> "已停用"; - default -> status; - }; - } - - private String blankToNull(String value) { return value == null || value.isBlank() ? null : value.trim(); } - - public record CustomerParameters(String keyword, String status) { } -} diff --git a/file/src/main/java/com/baiyue/file/service/ExportCommandFactory.java b/file/src/main/java/com/baiyue/file/service/ExportCommandFactory.java new file mode 100644 index 000000000..f0e7ae1d7 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportCommandFactory.java @@ -0,0 +1,40 @@ +package com.baiyue.file.service; + +import com.baiyue.common.core.export.ExportContracts.ExportCommand; +import com.baiyue.file.entity.FileExportOutbox; +import com.baiyue.file.entity.FileExportTask; +import com.baomidou.mybatisplus.core.toolkit.IdWorker; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.MDC; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.time.OffsetDateTime; +import java.util.UUID; + +@Component +public class ExportCommandFactory { + private final ObjectMapper objectMapper; + private final String topic; + public ExportCommandFactory(ObjectMapper objectMapper, + @Value("${baiyue.file.export.rocketmq.topic:test-baiyue-file-export-command-v1}") String topic) { + this.objectMapper = objectMapper; + this.topic = topic; + } + public FileExportOutbox create(FileExportTask task, int dispatchVersion) { + String eventId = UUID.randomUUID().toString(); + ExportCommand command = new ExportCommand(1, eventId, String.valueOf(task.getId()), task.getExportType(), + dispatchVersion, OffsetDateTime.now(), MDC.get("traceId")); + FileExportOutbox row = new FileExportOutbox(); + row.setId(IdWorker.getId()); row.setEventId(eventId); row.setTaskId(task.getId()); + row.setDispatchVersion(dispatchVersion); row.setTopic(topic); row.setTag("EXPORT_TASK"); + row.setMessageKey(String.valueOf(task.getId())); row.setPayloadJson(json(command)); + row.setStatus("PENDING"); row.setAttemptCount(0); + return row; + } + private String json(ExportCommand command) { + try { return objectMapper.writeValueAsString(command); } + catch (JsonProcessingException exception) { throw new IllegalStateException("Unable to serialize export command", exception); } + } +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportFailure.java b/file/src/main/java/com/baiyue/file/service/ExportFailure.java new file mode 100644 index 000000000..e49b5416b --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportFailure.java @@ -0,0 +1,10 @@ +package com.baiyue.file.service; + +public class ExportFailure extends RuntimeException { + private final boolean retryable; + public ExportFailure(String message, boolean retryable) { super(message); this.retryable = retryable; } + public ExportFailure(String message, boolean retryable, Throwable cause) { super(message, cause); this.retryable = retryable; } + public boolean retryable() { return retryable; } + public static ExportFailure temporary(String message, Throwable cause) { return new ExportFailure(message, true, cause); } + public static ExportFailure permanent(String message) { return new ExportFailure(message, false); } +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportMetrics.java b/file/src/main/java/com/baiyue/file/service/ExportMetrics.java new file mode 100644 index 000000000..4c6954d84 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportMetrics.java @@ -0,0 +1,22 @@ +package com.baiyue.file.service; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +@Component +public class ExportMetrics { + private final MeterRegistry registry; + public ExportMetrics(MeterRegistry registry, JdbcTemplate jdbc) { + this.registry=registry; + Gauge.builder("export_task_queue_age_seconds",jdbc,j->number(j,"SELECT COALESCE(TIMESTAMPDIFF(SECOND,MIN(requested_at),CURRENT_TIMESTAMP(3)),0) FROM file_export_task WHERE status='PENDING'")).register(registry); + Gauge.builder("export_outbox_pending",jdbc,j->number(j,"SELECT COUNT(*) FROM file_export_outbox WHERE status IN ('PENDING','CLAIMED')")).register(registry); + Gauge.builder("export_outbox_oldest_age_seconds",jdbc,j->number(j,"SELECT COALESCE(TIMESTAMPDIFF(SECOND,MIN(created_at),CURRENT_TIMESTAMP(3)),0) FROM file_export_outbox WHERE status IN ('PENDING','CLAIMED')")).register(registry); + Gauge.builder("export_outbox_max_attempt",jdbc,j->number(j,"SELECT COALESCE(MAX(attempt_count),0) FROM file_export_outbox WHERE status IN ('PENDING','CLAIMED')")).register(registry); + Gauge.builder("export_storage_bytes",jdbc,j->number(j,"SELECT COALESCE(SUM(size_bytes),0) FROM file_export_task WHERE status='SUCCEEDED' AND expires_at>CURRENT_TIMESTAMP(3)")).register(registry); + } + public void created(String type){registry.counter("export_task_created_total","type",type).increment();} + public void completed(String type,String status){registry.counter("export_task_completed_total","type",type,"status",status).increment();} + private static double number(JdbcTemplate jdbc,String sql){try{Number value=jdbc.queryForObject(sql,Number.class);return value==null?0:value.doubleValue();}catch(RuntimeException ignored){return 0;}} +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportReconciler.java b/file/src/main/java/com/baiyue/file/service/ExportReconciler.java new file mode 100644 index 000000000..15bc6000b --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportReconciler.java @@ -0,0 +1,51 @@ +package com.baiyue.file.service; + +import com.baiyue.file.entity.FileExportTask; +import com.baiyue.file.repository.ExportOutboxRepository; +import com.baiyue.file.repository.ExportTaskRepository; +import io.micrometer.core.instrument.MeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import java.time.Duration; + +@Service +@ConditionalOnProperty(prefix="baiyue.file.export.reconciler", name="enabled", havingValue="true") +public class ExportReconciler { + private static final Logger LOGGER= LoggerFactory.getLogger(ExportReconciler.class); + private final ExportTaskRepository tasks; private final ExportOutboxRepository outbox; + private final ExportStateService states; private final ExportStorage storage; private final MeterRegistry meters; + private final int graceSeconds, sentStaleSeconds; + private final int retentionDays; + public ExportReconciler(ExportTaskRepository tasks, ExportOutboxRepository outbox, ExportStateService states, + ExportStorage storage, MeterRegistry meters, + @Value("${baiyue.file.export.reconciler.redispatch-grace-seconds:60}") int graceSeconds, + @Value("${baiyue.file.export.reconciler.sent-stale-seconds:600}") int sentStaleSeconds, + @Value("${baiyue.file.export.retention-days:7}") int retentionDays) { + this.tasks=tasks;this.outbox=outbox;this.states=states;this.storage=storage;this.meters=meters; + this.graceSeconds=graceSeconds;this.sentStaleSeconds=sentStaleSeconds;this.retentionDays=retentionDays; + } + @Scheduled(cron="${baiyue.file.export.reconciler.outbox-cleanup-cron:0 15 3 * * *}") + public void cleanupRetainedOutbox() { + count("outbox_clean",outbox.deleteRetainedSent(retentionDays,5000)); + } + @Scheduled(fixedDelayString="${baiyue.file.export.reconciler.poll-interval-ms:30000}") + public void reconcile() { + int released=outbox.releaseExpiredClaims(); count("outbox_release",released); + int redispatched=outbox.redispatchSent(sentStaleSeconds,graceSeconds); count("redispatch",redispatched); + int missing=0; for(FileExportTask task:tasks.missingOutbox(graceSeconds)) if(states.ensureOutbox(task)) missing++; + count("redispatch",missing); + int recovered=0; for(FileExportTask task:tasks.staleRunning()) if(states.recoverStale(task)) recovered++; + count("lease_recover",recovered); + int expired=0; for(FileExportTask task:tasks.expiredArtifactsDb()) { + if(storage.delete(task.getObjectKey())) { tasks.markExpired(task.getId()); expired++; } + else LOGGER.warn("导出文件过期清理失败 taskId={}",task.getId()); + } + count("expire",expired); + count("orphan_clean", storage.cleanupOrphans(Duration.ofHours(24), tasks::succeededObjectExists)); + } + private void count(String action,int value){ if(value>0)meters.counter("export_reconciler_action_total","action",action).increment(value); } +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportStateService.java b/file/src/main/java/com/baiyue/file/service/ExportStateService.java new file mode 100644 index 000000000..8f3c288e8 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportStateService.java @@ -0,0 +1,98 @@ +package com.baiyue.file.service; + +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import com.baiyue.file.entity.FileExportTask; +import com.baiyue.file.repository.ExportOutboxRepository; +import com.baiyue.file.repository.ExportTaskRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + + +@Service +public class ExportStateService { + private final ExportTaskRepository tasks; + private final ExportOutboxRepository outbox; + private final ExportCommandFactory commands; + private final ExportMetrics metrics; + public ExportStateService(ExportTaskRepository tasks, ExportOutboxRepository outbox, ExportCommandFactory commands, + ExportMetrics metrics) { + this.tasks = tasks; this.outbox = outbox; this.commands = commands; this.metrics = metrics; + } + @Transactional + public CreationResult createConstrained(FileExportTask task) { + tasks.lockCreationGuard(); + FileExportTask duplicate = tasks.recentDuplicate(task.getTenantId(), task.getAppCode(), task.getRequestedBy(), + task.getExportType(), task.getParametersJson()); + if (duplicate != null) return new CreationResult(duplicate, false); + if (tasks.activeForUser(task.getTenantId(), task.getAppCode(), task.getRequestedBy()) >= 3) { + throw new BizException(ResultCode.TOO_MANY_REQUESTS, "同一用户最多同时执行 3 个导出任务"); + } + if (tasks.activeForType(task.getExportType()) >= 5) { + throw new BizException(ResultCode.TOO_MANY_REQUESTS, "该导出类型当前任务较多，请稍后重试"); + } + tasks.insert(task); + tasks.initializeDispatchTime(task.getId()); + if (!outbox.insert(commands.create(task, task.getDispatchVersion()))) { + throw new IllegalStateException("Duplicate export command while creating task"); + } + return new CreationResult(task, true); + } + @Transactional + public boolean retryAfterFailure(FileExportTask current, String owner, int seconds, String message) { + if (!tasks.scheduleRetry(current.getId(), current.getDispatchVersion(), owner, seconds, safe(message))) return false; + FileExportTask refreshed = tasks.find(current.getId()); + if (!outbox.insert(commands.create(refreshed, refreshed.getDispatchVersion()))) { + throw new IllegalStateException("Unable to create retry command"); + } + return true; + } + @Transactional + public boolean fail(FileExportTask task, String owner, String message) { + return tasks.fail(task.getId(), task.getDispatchVersion(), owner, safe(message)); + } + @Transactional + public boolean ensureOutbox(FileExportTask task) { + return outbox.insert(commands.create(task, task.getDispatchVersion())); + } + @Transactional + public boolean manualRetry(FileExportTask task) { + if (!tasks.manualRetry(task.getId())) return false; + FileExportTask refreshed = tasks.find(task.getId()); + if (!outbox.insert(commands.create(refreshed, refreshed.getDispatchVersion()))) { + throw new IllegalStateException("Unable to create manual retry command"); + } + return true; + } + @Transactional + public void quarantine(String eventId, long taskId, int version, String reason) { + FileExportTask current = tasks.find(taskId); + if (current == null || current.getDispatchVersion() != version || !"PENDING".equals(current.getStatus())) return; + boolean changed = outbox.quarantine(eventId, taskId, version, safe(reason)); + if (changed) { + if (!tasks.failQuarantined(taskId, version, safe(reason))) { + throw new IllegalStateException("Unable to fail quarantined export task"); + } + metrics.completed(current.getExportType(), "FAILED"); + } + } + @Transactional + public boolean recoverStale(FileExportTask task) { + if (task.getAttemptCount() >= task.getMaxAttempts()) { + boolean failed = tasks.failStale(task.getId(), task.getDispatchVersion()); + if (failed) metrics.completed(task.getExportType(), "FAILED"); + return failed; + } + if (!tasks.recoverStale(task.getId(), task.getDispatchVersion())) return false; + FileExportTask refreshed = tasks.find(task.getId()); + if (!outbox.insert(commands.create(refreshed, refreshed.getDispatchVersion()))) { + throw new IllegalStateException("Unable to create lease recovery command"); + } + return true; + } + private String safe(String message) { + String value = message == null || message.isBlank() ? "导出任务执行失败" : message; + return value.length() > 1000 ? value.substring(0, 1000) : value; + } + public record CreationResult(FileExportTask task, boolean created) { } +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportStorage.java b/file/src/main/java/com/baiyue/file/service/ExportStorage.java new file mode 100644 index 000000000..ddcf35fa2 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportStorage.java @@ -0,0 +1,21 @@ +package com.baiyue.file.service; + +import org.springframework.core.io.Resource; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.time.Duration; +import java.util.function.Predicate; + +public interface ExportStorage { + PendingFile create(long taskId); + long size(String objectKey); + Resource load(String objectKey); + boolean delete(String objectKey); + int cleanupOrphans(Duration minimumAge, Predicate referenced); + + record PendingFile(String objectKey, Path path, OutputStream output) implements AutoCloseable { + @Override public void close() throws IOException { output.close(); } + } +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportTaskExecutor.java b/file/src/main/java/com/baiyue/file/service/ExportTaskExecutor.java new file mode 100644 index 000000000..5790dec11 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/ExportTaskExecutor.java @@ -0,0 +1,111 @@ +package com.baiyue.file.service; + +import com.baiyue.file.config.ExportDefinitionRegistry; +import com.baiyue.file.entity.FileExportTask; +import com.baiyue.file.repository.ExportTaskRepository; +import io.micrometer.core.instrument.MeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import jakarta.annotation.PreDestroy; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +@Service +public class ExportTaskExecutor { + private static final Logger LOGGER = LoggerFactory.getLogger(ExportTaskExecutor.class); + private final ExportTaskRepository tasks; + private final ExportDefinitionRegistry definitions; + private final GenericXlsxExportRunner runner; + private final ExportStateService states; + private final ExportStorage storage; + private final MeterRegistry meters; + private final int leaseSeconds, heartbeatSeconds, maxDurationMinutes, retentionDays; + private final ScheduledExecutorService watchdogs = Executors.newScheduledThreadPool(4, runnable -> { + Thread thread = new Thread(runnable, "file-export-watchdog"); thread.setDaemon(true); return thread; + }); + + public ExportTaskExecutor(ExportTaskRepository tasks, ExportDefinitionRegistry definitions, + GenericXlsxExportRunner runner, ExportStateService states, ExportStorage storage, MeterRegistry meters, + @Value("${baiyue.file.export.task.lease-seconds:300}") int leaseSeconds, + @Value("${baiyue.file.export.task.heartbeat-seconds:60}") int heartbeatSeconds, + @Value("${baiyue.file.export.task.max-duration-minutes:30}") int maxDurationMinutes, + @Value("${baiyue.file.export.retention-days:7}") int retentionDays) { + this.tasks=tasks;this.definitions=definitions;this.runner=runner;this.states=states;this.storage=storage;this.meters=meters; + this.leaseSeconds=leaseSeconds;this.heartbeatSeconds=heartbeatSeconds;this.maxDurationMinutes=maxDurationMinutes;this.retentionDays=retentionDays; + } + + public void execute(long taskId, int dispatchVersion, String owner) { + if (!tasks.claim(taskId, dispatchVersion, owner, leaseSeconds, maxDurationMinutes)) return; + FileExportTask task = tasks.find(taskId); + AtomicBoolean abort = new AtomicBoolean(); + ScheduledFuture watchdog = watchdogs.scheduleAtFixedRate(() -> heartbeat(taskId, dispatchVersion, owner, abort), + heartbeatSeconds, heartbeatSeconds, TimeUnit.SECONDS); + long started = System.nanoTime(); + GenericXlsxExportRunner.Result result = null; + try { + result = runner.run(task, definitions.require(task.getExportType()), abort, + rows -> { if (!tasks.progress(taskId, dispatchVersion, owner, rows)) abort.set(true); }); + if (!tasks.succeed(taskId, dispatchVersion, owner, result.fileName(), result.objectKey(), + result.contentType(), result.sizeBytes(), result.rows(), retentionDays)) { + deleteOrWarn(result.objectKey(), taskId, "最终状态提交未生效"); + FileExportTask latest = tasks.find(taskId); + if (latest != null && "RUNNING".equals(latest.getStatus()) + && latest.getDispatchVersion() == dispatchVersion && owner.equals(latest.getLeaseOwner())) { + retryOrFail(latest, dispatchVersion, owner, ExportFailure.temporary("导出执行超过最大时长 " + maxDurationMinutes + " 分钟", null)); + } + } else { + meters.counter("export_task_completed_total", "type", task.getExportType(), "status", "SUCCEEDED").increment(); + } + } catch (RuntimeException exception) { + if (result != null) deleteOrWarn(result.objectKey(), taskId, "执行异常"); + retryOrFail(tasks.find(taskId), dispatchVersion, owner, exception); + LOGGER.error("导出任务执行失败 taskId={} dispatchVersion={}", taskId, dispatchVersion, exception); + } finally { + watchdog.cancel(true); + meters.timer("export_task_duration_seconds", "type", task.getExportType()) + .record(System.nanoTime() - started, TimeUnit.NANOSECONDS); + } + } + + private void heartbeat(long id, int version, String owner, AtomicBoolean abort) { + try { if (!tasks.heartbeat(id, version, owner, leaseSeconds)) abort.set(true); } + catch (RuntimeException exception) { abort.set(true); LOGGER.error("导出 watchdog 续租失败 taskId={}", id, exception); } + } + + private void deleteOrWarn(String objectKey, long taskId, String reason) { + try { + if (!storage.delete(objectKey)) { + LOGGER.warn("导出结果文件清理失败 taskId={} objectKey={} reason={}", taskId, objectKey, reason); + } + } catch (RuntimeException exception) { + LOGGER.warn("导出结果文件清理异常 taskId={} objectKey={} reason={}", taskId, objectKey, reason, exception); + } + } + + private void retryOrFail(FileExportTask task, int expectedVersion, String owner, RuntimeException exception) { + if (task == null || !"RUNNING".equals(task.getStatus()) || task.getDispatchVersion() != expectedVersion + || !owner.equals(task.getLeaseOwner())) return; + boolean retryable = !(exception instanceof ExportFailure failure) || failure.retryable(); + String message = exception.getMessage() == null ? "导出任务执行失败" : exception.getMessage(); + if (retryable && task.getAttemptCount() < task.getMaxAttempts()) { + int backoff = jitteredBackoffSeconds(task.getAttemptCount() <= 1 ? 60 : 900); + if (states.retryAfterFailure(task, owner, backoff, message)) return; + } + if (states.fail(task, owner, message)) { + meters.counter("export_task_completed_total", "type", task.getExportType(), "status", "FAILED").increment(); + } + } + + static int jitteredBackoffSeconds(int baseSeconds) { + int spread = Math.max(1, baseSeconds / 5); + return ThreadLocalRandom.current().nextInt(baseSeconds - spread, baseSeconds + spread + 1); + } + + @PreDestroy + void shutdownWatchdogs() { + watchdogs.shutdownNow(); + } +} diff --git a/file/src/main/java/com/baiyue/file/service/ExportTaskHandler.java b/file/src/main/java/com/baiyue/file/service/ExportTaskHandler.java deleted file mode 100644 index ba45ebd25..000000000 --- a/file/src/main/java/com/baiyue/file/service/ExportTaskHandler.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.baiyue.file.service; - -import com.baiyue.file.entity.FileExportTask; - -import java.util.Map; - -public interface ExportTaskHandler { - String exportType(); - String requiredPermission(); - default void validateParameters(Map parameters) { } - ExportResult export(FileExportTask task, ProgressReporter progress); - - record ExportResult(String fileName, String objectKey, String contentType, long sizeBytes, long rows) { } - - @FunctionalInterface - interface ProgressReporter { void update(long rows); } -} diff --git a/file/src/main/java/com/baiyue/file/service/ExportTaskService.java b/file/src/main/java/com/baiyue/file/service/ExportTaskService.java index 15b1c6395..ffa345884 100644 --- a/file/src/main/java/com/baiyue/file/service/ExportTaskService.java +++ b/file/src/main/java/com/baiyue/file/service/ExportTaskService.java @@ -8,6 +8,7 @@ import com.baiyue.file.dto.ExportTaskDtos.CreateExportTaskRequest; import com.baiyue.file.dto.ExportTaskDtos.ExportTaskResponse; import com.baiyue.file.entity.FileExportTask; import com.baiyue.file.repository.ExportTaskRepository; +import com.baiyue.file.config.ExportDefinitionRegistry; import com.baomidou.mybatisplus.core.toolkit.IdWorker; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,38 +16,45 @@ import org.springframework.stereotype.Service; import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; +import java.util.TreeMap; @Service public class ExportTaskService { private final ExportTaskRepository repository; - private final LocalExportStorage storage; + private final ExportStorage storage; private final ObjectMapper objectMapper; - private final Map handlers; + private final ExportDefinitionRegistry registry; + private final ExportStateService stateService; + private final ExportMetrics metrics; - public ExportTaskService(ExportTaskRepository repository, LocalExportStorage storage, - ObjectMapper objectMapper, List handlers) { + public ExportTaskService(ExportTaskRepository repository, ExportStorage storage, + ObjectMapper objectMapper, ExportDefinitionRegistry registry, + ExportStateService stateService, ExportMetrics metrics) { this.repository = repository; this.storage = storage; this.objectMapper = objectMapper; - this.handlers = handlers.stream().collect(Collectors.toUnmodifiableMap( - handler -> handler.exportType().toUpperCase(Locale.ROOT), Function.identity())); + this.registry = registry; + this.stateService = stateService; + this.metrics = metrics; } public ExportTaskResponse create(AuthenticatedUser user, CreateExportTaskRequest request) { requirePlatform(user); String type = request.exportType().trim().toUpperCase(Locale.ROOT); String format = request.format().trim().toUpperCase(Locale.ROOT); - ExportTaskHandler handler = handler(type); + var definition = registry.require(type); if (!"XLSX".equals(format)) throw new BizException(ResultCode.VALIDATION_FAILED, "当前仅支持 XLSX 格式"); - if (!user.permissions().contains(handler.requiredPermission())) { + String permission = definition.getRequiredPermission(); + if (!definition.isEnabled() || !definition.getAppScope().equalsIgnoreCase(user.appCode())) { + throw new BizException(ResultCode.VALIDATION_FAILED, "该导出类型当前不可用"); + } + if (!user.permissions().contains(permission)) { throw new BizException(ResultCode.FORBIDDEN, "缺少该导出类型的操作权限"); } - handler.validateParameters(request.parameters() == null ? Map.of() : request.parameters()); + validateParameters(request.parameters()); + String parametersJson = parametersJson(request.parameters()); long id = IdWorker.getId(); LocalDateTime now = LocalDateTime.now(); FileExportTask task = new FileExportTask(); @@ -57,14 +65,19 @@ public class ExportTaskService { task.setExportType(type); task.setFormat(format); task.setTaskName(request.taskName().trim()); - task.setParametersJson(parametersJson(request.parameters())); + task.setParametersJson(parametersJson); task.setStatus("PENDING"); task.setProcessedRows(0L); + task.setDispatchVersion(1); + task.setAttemptCount(0); + task.setMaxAttempts(definition.getMaxAttempts()); + task.setNextRetryAt(now); task.setRequestedBy(user.userId()); task.setRequestedUsername(user.username()); task.setRequestedAt(now); - repository.insert(task); - return response(task); + ExportStateService.CreationResult result = stateService.createConstrained(task); + if (result.created()) metrics.created(type); + return response(result.task()); } public PageResult page(AuthenticatedUser user, long page, long size, @@ -90,7 +103,7 @@ public class ExportTaskService { if (!"SUCCEEDED".equals(task.getStatus())) { throw new BizException(ResultCode.CONFLICT, "导出任务尚未完成或文件已过期"); } - if (task.getExpiresAt() != null && !task.getExpiresAt().isAfter(LocalDateTime.now())) { + if (!repository.downloadable(id)) { throw new BizException(ResultCode.CONFLICT, "导出文件已过期"); } return new DownloadResource(storage.load(task.getObjectKey()), task.getFileName(), task.getContentType()); @@ -98,22 +111,25 @@ public class ExportTaskService { public ExportTaskResponse retry(AuthenticatedUser user, long id) { FileExportTask task = ownedTask(user, id); - ExportTaskHandler handler = handler(task.getExportType()); - if (!user.permissions().contains(handler.requiredPermission())) { + if (!"FAILED".equals(task.getStatus())) { + throw new BizException(ResultCode.CONFLICT, "只有失败的导出任务可以重试"); + } + var definition = registry.require(task.getExportType()); + if (!definition.isEnabled()) throw new BizException(ResultCode.CONFLICT, "该导出类型已停用"); + String permission = definition.getRequiredPermission(); + if (!user.permissions().contains(permission)) { throw new BizException(ResultCode.FORBIDDEN, "缺少该导出类型的操作权限"); } - if (!"FAILED".equals(task.getStatus()) || !repository.retry(id)) { + if (!storage.delete(task.getObjectKey())) { + throw new BizException(ResultCode.SERVICE_UNAVAILABLE, "上次导出文件清理失败，请稍后重试"); + } + boolean retried = stateService.manualRetry(task); + if (!retried) { throw new BizException(ResultCode.CONFLICT, "只有失败的导出任务可以重试"); } return response(repository.find(id)); } - ExportTaskHandler handler(String exportType) { - ExportTaskHandler handler = handlers.get(exportType == null ? "" : exportType.toUpperCase(Locale.ROOT)); - if (handler == null) throw new BizException(ResultCode.VALIDATION_FAILED, "不支持的导出类型"); - return handler; - } - private FileExportTask ownedTask(AuthenticatedUser user, long id) { requirePlatform(user); FileExportTask task = repository.find(id); @@ -128,10 +144,47 @@ public class ExportTaskService { } private String parametersJson(Map parameters) { - try { return objectMapper.writeValueAsString(parameters == null ? Map.of() : parameters); } + try { return objectMapper.writeValueAsString(parameters == null ? Map.of() : new TreeMap<>(parameters)); } catch (JsonProcessingException exception) { throw new BizException(ResultCode.BAD_REQUEST, "导出筛选条件无法序列化"); } } + private void validateParameters(Map parameters) { + Map values = parameters == null ? Map.of() : parameters; + if (values.size() > 20) throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选条件最多 20 项"); + values.forEach((key, value) -> { + if (key == null || key.isBlank() || key.length() > 64) throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选键无效"); + validateParameterValue(value, 1); + }); + try { + if (objectMapper.writeValueAsBytes(values).length > 4096) throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选条件不能超过 4KB"); + } catch (JsonProcessingException exception) { + throw new BizException(ResultCode.BAD_REQUEST, "导出筛选条件无法序列化"); + } + } + + private void validateParameterValue(Object value, int depth) { + if (depth > 2) throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选条件嵌套不能超过 2 层"); + if (value == null || value instanceof Number || value instanceof Boolean) return; + if (value instanceof String text) { + if (text.length() > 256) throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选值最多 256 字符"); + return; + } + if (value instanceof Map map) { + map.forEach((key, nested) -> { + if (!(key instanceof String text) || text.isBlank() || text.length() > 64) { + throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选键无效"); + } + validateParameterValue(nested, depth + 1); + }); + return; + } + if (value instanceof Iterable values) { + values.forEach(nested -> validateParameterValue(nested, depth + 1)); + return; + } + throw new BizException(ResultCode.VALIDATION_FAILED, "导出筛选值类型无效"); + } + private ExportTaskResponse response(FileExportTask task) { return new ExportTaskResponse(String.valueOf(task.getId()), task.getTaskNo(), task.getExportType(), task.getFormat(), task.getTaskName(), task.getStatus(), task.getTotalRows(), diff --git a/file/src/main/java/com/baiyue/file/service/ExportTaskWorker.java b/file/src/main/java/com/baiyue/file/service/ExportTaskWorker.java deleted file mode 100644 index 41168e92a..000000000 --- a/file/src/main/java/com/baiyue/file/service/ExportTaskWorker.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.baiyue.file.service; - -import com.baiyue.file.entity.FileExportTask; -import com.baiyue.file.repository.ExportTaskRepository; -import com.baiyue.common.core.exception.BizException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.time.LocalDateTime; - -@Service -public class ExportTaskWorker { - private static final Logger LOGGER = LoggerFactory.getLogger(ExportTaskWorker.class); - private final ExportTaskRepository repository; - private final ExportTaskService service; - private final LocalExportStorage storage; - private final long retentionDays; - private final long staleRunningMinutes; - - public ExportTaskWorker(ExportTaskRepository repository, ExportTaskService service, - LocalExportStorage storage, - @Value("${baiyue.file.export.retention-days:7}") long retentionDays, - @Value("${baiyue.file.export.stale-running-minutes:30}") long staleRunningMinutes) { - this.repository = repository; - this.service = service; - this.storage = storage; - this.retentionDays = Math.max(1, retentionDays); - this.staleRunningMinutes = Math.max(5, staleRunningMinutes); - } - - @Scheduled(fixedDelayString = "${baiyue.file.export.worker-delay-ms:2000}") - public void poll() { - LocalDateTime now = LocalDateTime.now(); - repository.requeueStale(now.minusMinutes(staleRunningMinutes)); - expire(now); - processNext(); - } - - public boolean processNext() { - Long id = repository.firstPendingId(); - if (id == null || !repository.claim(id, LocalDateTime.now())) return false; - FileExportTask task = repository.find(id); - try { - ExportTaskHandler handler = service.handler(task.getExportType()); - var result = handler.export(task, rows -> repository.progress(id, rows)); - LocalDateTime completed = LocalDateTime.now(); - if (!repository.succeed(id, result.fileName(), result.objectKey(), result.contentType(), - result.sizeBytes(), result.rows(), completed, completed.plusDays(retentionDays))) { - storage.delete(result.objectKey()); - throw new IllegalStateException("Export task state changed before completion"); - } - } catch (Exception exception) { - String message = safeMessage(exception); - repository.fail(id, message, LocalDateTime.now()); - LOGGER.error("导出任务执行失败，taskId={}，exportType={}", id, task.getExportType(), exception); - } - return true; - } - - private void expire(LocalDateTime now) { - for (FileExportTask task : repository.expiredArtifacts(now)) { - storage.delete(task.getObjectKey()); - repository.markExpired(task.getId()); - } - } - - private String safeMessage(Exception exception) { - String message = exception instanceof BizException ? exception.getMessage() : "导出任务执行失败，请稍后重试"; - if (message == null || message.isBlank()) message = "导出任务执行失败"; - return message.length() <= 1000 ? message : message.substring(0, 1000); - } -} diff --git a/file/src/main/java/com/baiyue/file/service/GenericXlsxExportRunner.java b/file/src/main/java/com/baiyue/file/service/GenericXlsxExportRunner.java new file mode 100644 index 000000000..b41e9eb63 --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/GenericXlsxExportRunner.java @@ -0,0 +1,208 @@ +package com.baiyue.file.service; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import com.baiyue.common.core.export.ExportContracts.ExportColumn; +import com.baiyue.file.client.ExportProviderClient; +import com.baiyue.file.client.ExportProviderResponseTooLargeException; +import com.baiyue.file.config.ExportDefinitionRegistry.Definition; +import com.baiyue.file.entity.FileExportTask; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongConsumer; + +@Service +public class GenericXlsxExportRunner { + private static final Logger LOGGER = LoggerFactory.getLogger(GenericXlsxExportRunner.class); + public static final String CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + private static final int EXCEL_SHEET_ROWS = 1_048_575; + private final ExportProviderClient provider; + private final ExportStorage storage; + private final ObjectMapper objectMapper; + private final String internalToken; + private final long maxFileBytes; + + public GenericXlsxExportRunner(ExportProviderClient provider, ExportStorage storage, ObjectMapper objectMapper, + @Value("${baiyue.security.internal-service-token}") String internalToken, + @Value("${baiyue.file.export.task.max-file-size-mb:50}") long maxFileSizeMb) { + this.provider = provider; this.storage = storage; this.objectMapper = objectMapper; + this.internalToken = internalToken; this.maxFileBytes = Math.max(1, maxFileSizeMb) * 1024L * 1024L; + } + + public Result run(FileExportTask task, Definition definition, AtomicBoolean abort, LongConsumer progress) { + Map parameters = parameters(task.getParametersJson()); + ExportStorage.PendingFile target = storage.create(task.getId()); + long rows = 0; + String cursor = null, snapshot = null, schemaHash = null; + List columns = null; + Set columnKeys = Set.of(); + Set warnedUnknownColumns = new HashSet<>(); + Set warnedTruncatedColumns = new HashSet<>(); + try (target; SXSSFWorkbook workbook = new SXSSFWorkbook(100)) { + workbook.setCompressTempFiles(true); + WorkbookStyles styles = new WorkbookStyles(workbook); + SheetState sheet = null; + while (true) { + checkAbort(abort); + ExportBatchRequest request = new ExportBatchRequest(String.valueOf(task.getId()), + String.valueOf(task.getTenantId()), task.getAppCode(), String.valueOf(task.getRequestedBy()), + task.getDispatchVersion(), parameters, cursor, snapshot, definition.getBatchSize()); + ApiResponse response; + try { response = provider.batch(internalToken, task.getExportType(), definition, request); } + catch (RuntimeException exception) { + if (hasCause(exception, ExportProviderResponseTooLargeException.class)) { + throw ExportFailure.permanent("Provider 单批响应超过 4MB"); + } + throw ExportFailure.temporary("导出数据服务暂不可用", exception); + } + if (response == null) throw ExportFailure.temporary("导出数据服务无响应", null); + if (!response.successful()) { + String message = response.message() == null ? "导出数据服务返回失败" : response.message(); + if (response.code() != null && response.code().startsWith("S")) throw ExportFailure.temporary(message, null); + throw ExportFailure.permanent(message); + } + if (response.data() == null) throw ExportFailure.permanent("Provider 成功响应缺少 data"); + ExportBatchResponse batch = response.data(); + try { if (objectMapper.writeValueAsBytes(batch).length > 4 * 1024 * 1024) throw ExportFailure.permanent("Provider 单批响应超过 4MB"); } + catch (IOException exception) { throw ExportFailure.permanent("Provider 响应无法序列化"); } + if (batch.rows() == null || batch.rows().size() > definition.getBatchSize()) throw ExportFailure.permanent("Provider 单批数据超过限制"); + if (batch.totalRows() != null && batch.totalRows() > definition.getMaxRows()) throw ExportFailure.permanent("导出预计行数超过最大限制 " + definition.getMaxRows()); + if (schemaHash == null) { + if (batch.schemaVersion() != 1 || blank(batch.schemaHash()) || blank(batch.snapshotToken()) || batch.snapshotToken().length() > 512) { + throw ExportFailure.permanent("Provider 首批契约无效"); + } + columns = validateColumns(batch.columns()); columnKeys = columns.stream().map(ExportColumn::key).collect(java.util.stream.Collectors.toUnmodifiableSet()); schemaHash = batch.schemaHash(); snapshot = batch.snapshotToken(); + sheet = newSheet(workbook, 1, columns); + } else if (!schemaHash.equals(batch.schemaHash())) { + throw ExportFailure.permanent("Provider 导出列定义在执行期间发生变化"); + } + for (Map values : batch.rows()) { + if (abort.get()) throw ExportFailure.temporary("导出执行租约已失效", null); + if (++rows > definition.getMaxRows()) throw ExportFailure.permanent("导出行数超过最大限制 " + definition.getMaxRows()); + if (sheet.rowIndex > EXCEL_SHEET_ROWS) sheet = newSheet(workbook, sheet.index + 1, columns); + writeRow(sheet.sheet.createRow(sheet.rowIndex++), columns, columnKeys, values, styles, warnedUnknownColumns, warnedTruncatedColumns); + if (rows % 10_000 == 0) checkAbort(abort); + } + progress.accept(rows); + if (!batch.hasMore()) break; + if (blank(batch.nextCursor()) || batch.nextCursor().equals(cursor)) throw ExportFailure.temporary("Provider 游标未推进", null); + cursor = batch.nextCursor(); + } + checkAbort(abort); + workbook.write(new SizeLimitedOutputStream(target.output(), maxFileBytes)); + } catch (SizeLimitedOutputStream.ExportSizeExceededException exception) { + deleteOrWarn(target.objectKey(), task.getId(), "文件超过大小上限"); throw ExportFailure.permanent(exception.getMessage()); + } catch (IOException exception) { + deleteOrWarn(target.objectKey(), task.getId(), "文件写入失败"); throw ExportFailure.temporary("导出文件写入失败", exception); + } catch (RuntimeException exception) { + deleteOrWarn(target.objectKey(), task.getId(), "导出生成失败"); throw exception; + } + String name = task.getTaskName() + "-" + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")) + ".xlsx"; + try { + return new Result(name, target.objectKey(), CONTENT_TYPE, storage.size(target.objectKey()), rows); + } catch (RuntimeException exception) { + deleteOrWarn(target.objectKey(), task.getId(), "读取最终文件大小失败"); + throw ExportFailure.temporary("导出文件读取失败", exception); + } + } + + private void deleteOrWarn(String objectKey, long taskId, String reason) { + try { + if (!storage.delete(objectKey)) { + LOGGER.warn("导出临时文件清理失败 taskId={} objectKey={} reason={}", taskId, objectKey, reason); + } + } catch (RuntimeException exception) { + LOGGER.warn("导出临时文件清理异常 taskId={} objectKey={} reason={}", taskId, objectKey, reason, exception); + } + } + + private List validateColumns(List columns) { + if (columns == null || columns.isEmpty() || columns.size() > 100) throw ExportFailure.permanent("Provider 导出列定义无效"); + Set keys = new HashSet<>(); + for (ExportColumn column : columns) { + if (column == null || blank(column.key()) || blank(column.title()) || column.type() == null || !keys.add(column.key())) { + throw ExportFailure.permanent("Provider 导出列定义无效"); + } + } + return List.copyOf(columns); + } + + private SheetState newSheet(SXSSFWorkbook workbook, int index, List columns) { + Sheet sheet = workbook.createSheet(index == 1 ? "导出数据" : "导出数据-" + index); + CellStyle header = workbook.createCellStyle(); header.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); header.setFillPattern(FillPatternType.SOLID_FOREGROUND); + Font font = workbook.createFont(); font.setBold(true); header.setFont(font); + Row row = sheet.createRow(0); + for (int i = 0; i < columns.size(); i++) { + Cell cell = row.createCell(i); cell.setCellValue(columns.get(i).title()); cell.setCellStyle(header); + sheet.setColumnWidth(i, Math.min(255, Math.max(8, columns.get(i).width() == null ? 18 : columns.get(i).width())) * 256); + } + sheet.createFreezePane(0, 1); + return new SheetState(index, sheet, 1); + } + + private void writeRow(Row row, List columns, Set columnKeys, Map values, + WorkbookStyles styles, Set warnedUnknownColumns, Set warnedTruncatedColumns) { + Map source = values == null ? Map.of() : values; + source.keySet().stream().filter(key -> !columnKeys.contains(key) && warnedUnknownColumns.add(key)) + .forEach(key -> LOGGER.warn("Provider 返回未声明字段，已忽略 column={}", key)); + for (int i = 0; i < columns.size(); i++) writeCell(row.createCell(i), columns.get(i), source.get(columns.get(i).key()), styles, warnedTruncatedColumns); + } + + private void writeCell(Cell cell, ExportColumn column, Object value, WorkbookStyles styles, Set warnedTruncatedColumns) { + if (value == null) return; + try { + switch (column.type()) { + case STRING -> cell.setCellValue(safeString(String.valueOf(value), column.key(), warnedTruncatedColumns)); + case INTEGER -> cell.setCellValue(new BigDecimal(String.valueOf(value)).longValueExact()); + case DECIMAL -> cell.setCellValue(new BigDecimal(String.valueOf(value)).doubleValue()); + case BOOLEAN -> cell.setCellValue(value instanceof Boolean b ? b : Boolean.parseBoolean(String.valueOf(value))); + case DATE -> { cell.setCellValue(LocalDate.parse(String.valueOf(value))); cell.setCellStyle(styles.date); } + case DATETIME -> { cell.setCellValue(LocalDateTime.parse(String.valueOf(value))); cell.setCellStyle(styles.dateTime); } + } + } catch (RuntimeException exception) { throw ExportFailure.permanent("Provider 字段类型与列定义不匹配: " + column.key()); } + } + + private String safeString(String value, String column, Set warnedTruncatedColumns) { + if (value.length() > 32767 && warnedTruncatedColumns.add(column)) { + LOGGER.warn("Provider 字符串超过 Excel 单元格上限，已截断 column={}", column); + } + String text = value.length() > 32767 ? value.substring(0, 32767) : value; + return !text.isEmpty() && "=+-@\t\r\n".indexOf(text.charAt(0)) >= 0 ? "'" + text : text; + } + private Map parameters(String json) { + try { return objectMapper.readValue(json, new TypeReference<>() { }); } + catch (Exception exception) { throw ExportFailure.permanent("导出筛选条件无效"); } + } + private void checkAbort(AtomicBoolean abort) { if (abort.get()) throw ExportFailure.temporary("导出执行超过最大时长或租约已失效", null); } + private boolean blank(String value) { return value == null || value.isBlank(); } + private boolean hasCause(Throwable error, Class type) { + for (Throwable current=error; current!=null; current=current.getCause()) if (type.isInstance(current)) return true; + return false; + } + private static final class WorkbookStyles { + final CellStyle date; + final CellStyle dateTime; + WorkbookStyles(Workbook workbook) { + DataFormat formats = workbook.getCreationHelper().createDataFormat(); + date = workbook.createCellStyle(); date.setDataFormat(formats.getFormat("yyyy-mm-dd")); + dateTime = workbook.createCellStyle(); dateTime.setDataFormat(formats.getFormat("yyyy-mm-dd hh:mm:ss")); + } + } + private static final class SheetState { final int index; final Sheet sheet; int rowIndex; SheetState(int index, Sheet sheet, int rowIndex) { this.index=index;this.sheet=sheet;this.rowIndex=rowIndex; } } + public record Result(String fileName, String objectKey, String contentType, long sizeBytes, long rows) { } +} diff --git a/file/src/main/java/com/baiyue/file/service/LocalExportStorage.java b/file/src/main/java/com/baiyue/file/service/LocalExportStorage.java index 59adb5009..da34ac5a6 100644 --- a/file/src/main/java/com/baiyue/file/service/LocalExportStorage.java +++ b/file/src/main/java/com/baiyue/file/service/LocalExportStorage.java @@ -13,10 +13,14 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.time.Instant; +import java.util.function.Predicate; +import java.util.stream.Stream; import java.util.UUID; @Service -public class LocalExportStorage { +public class LocalExportStorage implements ExportStorage { private final Path root; private final Path exportRoot; @@ -61,14 +65,35 @@ public class LocalExportStorage { catch (Exception exception) { throw new BizException(ResultCode.NOT_FOUND, "导出文件不存在或已清理"); } } - public void delete(String objectKey) { - if (objectKey == null || objectKey.isBlank()) return; + public boolean delete(String objectKey) { + if (objectKey == null || objectKey.isBlank()) return true; Path target = resolve(objectKey); try { Files.deleteIfExists(target); - Path parent = target.getParent(); - if (parent != null && parent.startsWith(exportRoot) && !parent.equals(exportRoot)) Files.deleteIfExists(parent); - } catch (IOException ignored) { } + } catch (IOException ignored) { + return false; + } + Path parent = target.getParent(); + if (parent != null && parent.startsWith(exportRoot) && !parent.equals(exportRoot)) { + try { Files.deleteIfExists(parent); } + catch (IOException ignored) { /* 文件已经删除，空目录清理失败不影响任务过期。 */ } + } + return true; + } + + @Override + public int cleanupOrphans(Duration minimumAge, Predicate referenced) { + int removed = 0; + Instant cutoff = Instant.now().minus(minimumAge); + try (Stream paths = Files.walk(exportRoot)) { + for (Path path : paths.filter(Files::isRegularFile).toList()) { + String objectKey = root.relativize(path).toString().replace('\\', '/'); + if (!objectKey.matches("exports/\\d+/[0-9a-fA-F-]{36}\\.xlsx")) continue; + if (Files.getLastModifiedTime(path).toInstant().isAfter(cutoff) || referenced.test(objectKey)) continue; + if (delete(objectKey)) removed++; + } + } catch (IOException ignored) { /* 下一轮继续清理；存储指标和日志负责暴露持续故障。 */ } + return removed; } private Path resolve(String objectKey) { @@ -80,7 +105,4 @@ public class LocalExportStorage { return target; } - public record PendingFile(String objectKey, Path path, OutputStream output) implements AutoCloseable { - @Override public void close() throws IOException { output.close(); } - } } diff --git a/file/src/main/java/com/baiyue/file/service/LocalQualificationStorage.java b/file/src/main/java/com/baiyue/file/service/LocalQualificationStorage.java index a92add9a1..c25721192 100644 --- a/file/src/main/java/com/baiyue/file/service/LocalQualificationStorage.java +++ b/file/src/main/java/com/baiyue/file/service/LocalQualificationStorage.java @@ -24,7 +24,7 @@ import java.util.UUID; @Service public class LocalQualificationStorage { private static final Logger LOGGER = LoggerFactory.getLogger(LocalQualificationStorage.class); - private static final long MAX_SIZE = 10L * 1024 * 1024; + private static final long MAX_SIZE = 5L * 1024 * 1024; private static final Set TYPES = Set.of("image/jpeg", "image/png", "application/pdf"); private final Path root; private final Path qualificationRoot; @@ -53,7 +53,7 @@ public class LocalQualificationStorage { public StoredFile store(long tenantId, MultipartFile file) { if (file == null || file.isEmpty()) throw new BizException("One qualification file is required / 请选择资质文件"); - if (file.getSize() > MAX_SIZE) throw new BizException("Qualification file must not exceed 10 MB / 资质文件不能超过 10MB"); + if (file.getSize() > MAX_SIZE) throw new BizException("Qualification file must not exceed 5 MB / 资质文件不能超过 5MB"); String contentType = file.getContentType() == null ? "" : file.getContentType().toLowerCase(Locale.ROOT); if (!TYPES.contains(contentType)) throw new BizException("Only PDF, JPG and PNG files are supported / 仅支持 PDF、JPG、PNG 文件"); if (!matchesSignature(file, contentType)) { diff --git a/file/src/main/java/com/baiyue/file/service/SizeLimitedOutputStream.java b/file/src/main/java/com/baiyue/file/service/SizeLimitedOutputStream.java new file mode 100644 index 000000000..e2049c3ba --- /dev/null +++ b/file/src/main/java/com/baiyue/file/service/SizeLimitedOutputStream.java @@ -0,0 +1,19 @@ +package com.baiyue.file.service; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +final class SizeLimitedOutputStream extends FilterOutputStream { + private final long limit; + private long count; + SizeLimitedOutputStream(OutputStream out, long limit) { super(out); this.limit = limit; } + @Override public void write(int value) throws IOException { ensure(1); out.write(value); count++; } + @Override public void write(byte[] bytes, int offset, int length) throws IOException { ensure(length); out.write(bytes, offset, length); count += length; } + private void ensure(long added) throws IOException { + if (count + added > limit) throw new ExportSizeExceededException("导出文件超过最大限制 " + limit + " 字节"); + } + static final class ExportSizeExceededException extends IOException { + ExportSizeExceededException(String message) { super(message); } + } +} diff --git a/file/src/main/resources/application.yml b/file/src/main/resources/application.yml index 56eb0c4a6..80bf94bfa 100644 --- a/file/src/main/resources/application.yml +++ b/file/src/main/resources/application.yml @@ -20,10 +20,30 @@ spring: group: ${NACOS_GROUP:BAIYUE} namespace: ${NACOS_NAMESPACE:public} config.namespace: ${NACOS_NAMESPACE:public} + openfeign: + client: + config: + iamExportProviderClient: + connect-timeout: 3000 + read-timeout: 15000 + orderExportProviderClient: + connect-timeout: 3000 + read-timeout: 15000 + warehouseExportProviderClient: + connect-timeout: 3000 + read-timeout: 15000 + firstlegExportProviderClient: + connect-timeout: 3000 + read-timeout: 15000 servlet: multipart: max-file-size: 10MB max-request-size: 11MB +rocketmq: + name-server: ${ROCKETMQ_NAME_SERVER:127.0.0.1:9876} + producer: + group: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-publisher-v1 + send-message-timeout: 3000 baiyue: security: jwt-secret: ${JWT_SECRET:change-me-to-at-least-32-bytes-long-before-start} @@ -35,8 +55,43 @@ baiyue: storage-root: ${FILE_STORAGE_ROOT:${user.home}/baiyue-firstleg/uploads} export: retention-days: ${FILE_EXPORT_RETENTION_DAYS:7} - worker-delay-ms: ${FILE_EXPORT_WORKER_DELAY_MS:2000} - stale-running-minutes: ${FILE_EXPORT_STALE_RUNNING_MINUTES:30} + legacy-worker-enabled: false + instance-id: ${HOSTNAME:file-local} + task: + lease-seconds: ${FILE_EXPORT_TASK_LEASE_SECONDS:300} + heartbeat-seconds: ${FILE_EXPORT_TASK_HEARTBEAT_SECONDS:60} + max-duration-minutes: ${FILE_EXPORT_MAX_DURATION_MINUTES:30} + max-file-size-mb: ${FILE_EXPORT_MAX_FILE_SIZE_MB:50} + publisher: + enabled: ${FILE_EXPORT_PUBLISHER_ENABLED:false} + poll-interval-ms: ${FILE_EXPORT_PUBLISHER_POLL_MS:5000} + batch-size: 50 + lease-seconds: 60 + consumer: + enabled: ${FILE_EXPORT_CONSUMER_ENABLED:false} + concurrency: ${FILE_EXPORT_CONSUMER_CONCURRENCY:4} + consume-timeout-minutes: ${FILE_EXPORT_CONSUME_TIMEOUT_MINUTES:90} + max-reconsume-times: 2 + reconciler: + enabled: ${FILE_EXPORT_RECONCILER_ENABLED:false} + poll-interval-ms: ${FILE_EXPORT_RECONCILER_POLL_MS:30000} + redispatch-grace-seconds: 60 + sent-stale-seconds: 600 + rocketmq: + topic: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-command-v1 + consumer-group: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-worker-v1 + dlq-topic: "%DLQ%${MQ_TOPIC_PREFIX:test-}baiyue-file-export-worker-v1" + dlq-consumer-group: ${MQ_TOPIC_PREFIX:test-}baiyue-file-export-dlq-handler-v1 + definitions: + CUSTOMER_LIST: + provider-service: iam-service + provider-path: /internal/exports/v1/CUSTOMER_LIST/batches + required-permission: platform:customer:export + app-scope: PLATFORM + enabled: true + max-rows: 200000 + max-attempts: 3 + batch-size: 500 management.endpoints.web.exposure.include: health,info,prometheus logging: diff --git a/file/src/main/resources/db/file.sql b/file/src/main/resources/db/file.sql index aa78343ea..1934219a3 100644 --- a/file/src/main/resources/db/file.sql +++ b/file/src/main/resources/db/file.sql @@ -77,6 +77,14 @@ CREATE TABLE IF NOT EXISTS file_export_task ( content_type VARCHAR(128) NULL COMMENT '文件类型', size_bytes BIGINT UNSIGNED NULL COMMENT '文件字节数', failure_message VARCHAR(1000) NULL COMMENT '失败原因', + dispatch_version INT NOT NULL DEFAULT 1 COMMENT '派发版本，仅执行重试时递增', + attempt_count INT NOT NULL DEFAULT 0 COMMENT '已执行尝试次数', + max_attempts INT NOT NULL DEFAULT 3 COMMENT '自动重试上限', + next_retry_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '允许执行的最早时间', + lease_owner VARCHAR(128) NULL COMMENT '当前执行实例标识', + lease_until DATETIME(3) NULL COMMENT '执行租约截止时间', + last_heartbeat_at DATETIME(3) NULL COMMENT '最后心跳时间', + execution_deadline_at DATETIME(3) NULL COMMENT '本次执行截止时间', requested_by BIGINT UNSIGNED NOT NULL COMMENT '申请用户ID', requested_username VARCHAR(128) NULL COMMENT '申请账号快照', requested_at DATETIME(3) NOT NULL COMMENT '申请时间', @@ -93,5 +101,40 @@ CREATE TABLE IF NOT EXISTS file_export_task ( UNIQUE KEY uk_file_export_task_no (task_no), KEY idx_file_export_owner (tenant_id, app_code, requested_by, requested_at), KEY idx_file_export_status (status, requested_at), + KEY idx_file_export_dispatch (status, next_retry_at, id), + KEY idx_file_export_lease (status, lease_until), KEY idx_file_export_expiry (status, expires_at) ) ENGINE=InnoDB COMMENT='公共异步导出任务表'; + +CREATE TABLE IF NOT EXISTS file_export_task_guard ( + guard_key VARCHAR(64) NOT NULL COMMENT '任务创建串行化锁键', + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + PRIMARY KEY (guard_key) +) ENGINE=InnoDB COMMENT='导出任务创建并发保护表：低频创建入口使用单行锁保证重复提交及活动上限'; + +INSERT IGNORE INTO file_export_task_guard (guard_key) VALUES ('GLOBAL_CREATE'); + +CREATE TABLE IF NOT EXISTS file_export_outbox ( + id BIGINT UNSIGNED NOT NULL COMMENT 'Outbox记录ID', + event_id VARCHAR(64) NOT NULL COMMENT '投递意图唯一标识', + task_id BIGINT UNSIGNED NOT NULL COMMENT '关联导出任务ID', + dispatch_version INT NOT NULL COMMENT '任务派发版本', + topic VARCHAR(128) NOT NULL COMMENT '目标Topic', + tag VARCHAR(64) NOT NULL COMMENT '消息Tag', + message_key VARCHAR(64) NOT NULL COMMENT '消息Key', + payload_json JSON NOT NULL COMMENT '命令消息体', + status VARCHAR(24) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING/CLAIMED/SENT/QUARANTINED', + attempt_count INT NOT NULL DEFAULT 0 COMMENT '累计发送尝试次数', + next_retry_at DATETIME(3) NOT NULL COMMENT '允许发送的最早时间', + lease_owner VARCHAR(128) NULL COMMENT '领取实例标识', + lease_until DATETIME(3) NULL COMMENT '领取租约截止时间', + sent_at DATETIME(3) NULL COMMENT '发送成功时间', + last_error VARCHAR(1000) NULL COMMENT '最后发送错误', + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + PRIMARY KEY (id), + UNIQUE KEY uk_file_export_outbox_event (event_id), + UNIQUE KEY uk_file_export_outbox_task_version (task_id, dispatch_version), + KEY idx_file_export_outbox_publish (status, next_retry_at, id), + KEY idx_file_export_outbox_task (task_id, dispatch_version, status, sent_at) +) ENGINE=InnoDB COMMENT='导出命令事务Outbox表'; diff --git a/file/src/test/java/com/baiyue/file/client/ExportProviderClientTest.java b/file/src/test/java/com/baiyue/file/client/ExportProviderClientTest.java new file mode 100644 index 000000000..01cb5a1a9 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/client/ExportProviderClientTest.java @@ -0,0 +1,33 @@ +package com.baiyue.file.client; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import com.baiyue.file.config.ExportDefinitionRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.*; + +class ExportProviderClientTest { + @Test void routesTheRequestToTheAllowlistedServiceSelectedByTheDefinition() { + IamExportProviderClient iam=mock(IamExportProviderClient.class); + OrderExportProviderClient order=mock(OrderExportProviderClient.class); + WarehouseExportProviderClient warehouse=mock(WarehouseExportProviderClient.class); + FirstlegExportProviderClient firstleg=mock(FirstlegExportProviderClient.class); + var response=ApiResponse.success(new ExportBatchResponse(1,"hash","snapshot",java.util.List.of(),java.util.List.of(),null,false,0L)); + when(order.batch(anyString(),anyString(),any())).thenReturn(response); + var definition=new ExportDefinitionRegistry.Definition();definition.setProviderService("order-service"); + ExportBatchRequest request=new ExportBatchRequest("9","0","PLATFORM","7",1, Map.of(),null,null,500); + + var actual=new ExportProviderClient(iam,order,warehouse,firstleg,new SimpleMeterRegistry()) + .batch("token","ORDER_LIST",definition,request); + + assertSame(response,actual); + verify(order).batch("token","ORDER_LIST",request); + verifyNoInteractions(iam,warehouse,firstleg); + } +} diff --git a/file/src/test/java/com/baiyue/file/client/ExportProviderFeignConfigurationTest.java b/file/src/test/java/com/baiyue/file/client/ExportProviderFeignConfigurationTest.java new file mode 100644 index 000000000..11fbd6567 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/client/ExportProviderFeignConfigurationTest.java @@ -0,0 +1,25 @@ +package com.baiyue.file.client; + +import feign.Request; +import feign.Response; +import feign.codec.Decoder; +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.http.HttpMessageConverters; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ExportProviderFeignConfigurationTest { + @Test void rejectsAnOversizedProviderResponseBeforeJsonDeserialization() { + ExportProviderFeignConfiguration configuration=new ExportProviderFeignConfiguration(); + Decoder decoder=configuration.exportProviderDecoder(HttpMessageConverters::new); + byte[] body=new byte[ExportProviderFeignConfiguration.MAX_RESPONSE_BYTES+1]; + Response response=Response.builder().status(200).reason("OK") + .request(Request.create(Request.HttpMethod.POST,"http://iam-service/internal/exports/v1/CUSTOMER_LIST/batches",Map.of(),null, StandardCharsets.UTF_8,null)) + .body(body).build(); + + assertThrows(ExportProviderResponseTooLargeException.class,() -> decoder.decode(response,Object.class)); + } +} diff --git a/file/src/test/java/com/baiyue/file/config/ExportDefinitionRegistryTest.java b/file/src/test/java/com/baiyue/file/config/ExportDefinitionRegistryTest.java new file mode 100644 index 000000000..3fd4e4ffa --- /dev/null +++ b/file/src/test/java/com/baiyue/file/config/ExportDefinitionRegistryTest.java @@ -0,0 +1,30 @@ +package com.baiyue.file.config; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ExportDefinitionRegistryTest { + @Test void acceptsTheControlledCustomerProvider() { + ExportDefinitionRegistry registry=new ExportDefinitionRegistry(); registry.setDefinitions(Map.of("CUSTOMER_LIST", definition("iam-service","/internal/exports/v1/CUSTOMER_LIST/batches"))); + assertDoesNotThrow(registry::validate); + } + @Test void rejectsSsrfAndPathTypeMismatch() { + ExportDefinitionRegistry registry=new ExportDefinitionRegistry(); + assertThrows(IllegalStateException.class, () -> registry.setDefinitions(Map.of("CUSTOMER_LIST", definition("evil-service","/internal/exports/v1/OTHER/batches")))); + } + @Test void rejectedRefreshKeepsTheLastValidDefinitions() { + ExportDefinitionRegistry registry=new ExportDefinitionRegistry(); + var valid=definition("iam-service","/internal/exports/v1/CUSTOMER_LIST/batches"); + registry.setDefinitions(Map.of("CUSTOMER_LIST", valid)); + assertThrows(IllegalStateException.class, () -> registry.setDefinitions(Map.of("CUSTOMER_LIST", definition("evil-service","/internal/exports/v1/CUSTOMER_LIST/batches")))); + assertDoesNotThrow(() -> registry.require("CUSTOMER_LIST")); + org.junit.jupiter.api.Assertions.assertSame(valid, registry.require("CUSTOMER_LIST")); + } + private ExportDefinitionRegistry.Definition definition(String service,String path){ + var d=new ExportDefinitionRegistry.Definition();d.setProviderService(service);d.setProviderPath(path);d.setRequiredPermission("platform:customer:export");d.setAppScope("PLATFORM");d.setMaxRows(200000);d.setMaxAttempts(3);d.setBatchSize(500);return d; + } +} diff --git a/file/src/test/java/com/baiyue/file/config/ExportInfrastructureContractTest.java b/file/src/test/java/com/baiyue/file/config/ExportInfrastructureContractTest.java new file mode 100644 index 000000000..fb408ae0a --- /dev/null +++ b/file/src/test/java/com/baiyue/file/config/ExportInfrastructureContractTest.java @@ -0,0 +1,57 @@ +package com.baiyue.file.config; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ExportInfrastructureContractTest { + @Test void deploymentPinsConcurrencyFourAndStartsAllSchedulersDisabled() throws Exception { + String compose=read("..","deploy","docker-compose.yml"); + assertTrue(compose.contains("FILE_EXPORT_CONSUMER_CONCURRENCY: ${FILE_EXPORT_CONSUMER_CONCURRENCY:-4}")); + assertTrue(compose.contains("FILE_EXPORT_CONSUMER_ENABLED: ${FILE_EXPORT_CONSUMER_ENABLED:-false}")); + assertTrue(compose.contains("FILE_EXPORT_PUBLISHER_ENABLED: ${FILE_EXPORT_PUBLISHER_ENABLED:-false}")); + assertTrue(compose.contains("FILE_EXPORT_RECONCILER_ENABLED: ${FILE_EXPORT_RECONCILER_ENABLED:-false}")); + assertTrue(compose.contains("-Xms1g -Xmx2g")); + assertTrue(compose.contains("MQ_TOPIC_PREFIX: ${MQ_TOPIC_PREFIX?")); + assertTrue(!compose.contains("MQ_TOPIC_PREFIX: ${MQ_TOPIC_PREFIX:-test-}")); + } + + @Test void rocketMqBootstrapCreatesBothGroupsAndAReadableWorkerDlq() throws Exception { + String compose=read("..","deploy","docker-compose.rocketmq.yml"); + assertTrue(compose.contains("baiyue-file-export-command-v1\" -r 4 -w 4")); + assertTrue(compose.contains("baiyue-file-export-worker-v1\"")); + assertTrue(compose.contains("baiyue-file-export-dlq-handler-v1\"")); + assertTrue(compose.contains("%DLQ%$${MQ_TOPIC_PREFIX}baiyue-file-export-worker-v1\" -r 1 -w 1 -p 6")); + assertTrue(compose.contains("MQ_TOPIC_PREFIX: ${MQ_TOPIC_PREFIX?")); + assertTrue(compose.contains("sh -eu -c 'until sh mqadmin clusterList")); + assertTrue(compose.contains("| grep -q \"BaiyueCluster\"")); + assertTrue(compose.contains("restart: on-failure")); + String environment=read("..","deploy","env",".env.infra-test.example"); + assertTrue(environment.contains("ROCKETMQ_BROKER_IP=rocketmq-broker")); + assertTrue(environment.contains("MQ_TOPIC_PREFIX=test-")); + assertTrue(read("..","deploy","rocketmq","broker.conf.template").contains("autoCreateSubscriptionGroup=false")); + } + + @Test void brokerRendererRejectsUnreachableAndUnexpandedAddresses() throws Exception { + String script=read("..","deploy","rocketmq","render-broker-config.sh"); + assertTrue(script.contains("localhost|127.0.0.1")); + assertTrue(script.contains("*'${'*")); + assertTrue(script.contains("broker.conf still contains an unresolved placeholder")); + } + + @Test void databaseBaselineContainsTaskOutboxAndPreseededCreationGuard() throws Exception { + String sql=read("src","main","resources","db","file.sql"); + assertTrue(sql.contains("CREATE TABLE IF NOT EXISTS file_export_task (")); + assertTrue(sql.contains("CREATE TABLE IF NOT EXISTS file_export_outbox (")); + assertTrue(sql.contains("CREATE TABLE IF NOT EXISTS file_export_task_guard (")); + assertTrue(sql.contains("VALUES ('GLOBAL_CREATE')")); + } + + private String read(String first,String... more) throws Exception { + return Files.readString(Path.of(first,more), StandardCharsets.UTF_8); + } +} diff --git a/file/src/test/java/com/baiyue/file/controller/QualificationFileContractTest.java b/file/src/test/java/com/baiyue/file/controller/QualificationFileContractTest.java index 5d5d9413a..950d3ac81 100644 --- a/file/src/test/java/com/baiyue/file/controller/QualificationFileContractTest.java +++ b/file/src/test/java/com/baiyue/file/controller/QualificationFileContractTest.java @@ -186,6 +186,44 @@ class QualificationFileContractTest { assertTrue(requestCaptor.getValue().replace()); } + @Test + void merchantUploadRejectsMissingOrUnsupportedTypesBeforeWritingTheFile() { + var storage = mock(LocalQualificationStorage.class); + var iamClient = mock(IamQualificationClient.class); + var controller = merchantController(storage, iamClient); + var file = new MockMultipartFile("file", "license.pdf", "application/pdf", "%PDF-1.4".getBytes()); + + assertThrows(BizException.class, + () -> controller.upload(merchantUser(), file, null, null, null, false)); + assertThrows(BizException.class, + () -> controller.upload(merchantUser(), file, "UNKNOWN", null, null, false)); + + verify(storage, never()).store(any(Long.class), any()); + verify(iamClient, never()).submit(any(), any()); + } + + @Test + void everyRequiredQualificationTypeIsAcceptedAndForwarded() { + for (String qualificationType : Set.of( + "BUSINESS_LICENSE", "LEGAL_PERSON_ID", + "FREIGHT_FORWARDER_CERTIFICATE", "BANK_ACCOUNT_LICENSE")) { + var storage = mock(LocalQualificationStorage.class); + var iamClient = mock(IamQualificationClient.class); + var controller = merchantController(storage, iamClient); + var file = new MockMultipartFile("file", "qualification.pdf", "application/pdf", "%PDF-1.4".getBytes()); + when(storage.store(10001L, file)).thenReturn(storedFile()); + when(iamClient.submit(eq("internal-token"), any())).thenReturn(ApiResponse.success( + new IamQualificationClient.QualificationResponse(31L, null, "PROFILE_REQUIRED", null))); + + controller.upload(merchantUser(), file, qualificationType, null, null, false); + + var requestCaptor = org.mockito.ArgumentCaptor.forClass( + IamQualificationClient.QualificationRequest.class); + verify(iamClient).submit(eq("internal-token"), requestCaptor.capture()); + assertEquals(qualificationType, requestCaptor.getValue().qualificationType()); + } + } + private static PlatformQualificationFileController platformController( LocalQualificationStorage storage, IamQualificationClient iamClient) { var controller = new PlatformQualificationFileController(); diff --git a/file/src/test/java/com/baiyue/file/mapper/ExportMapperContractTest.java b/file/src/test/java/com/baiyue/file/mapper/ExportMapperContractTest.java new file mode 100644 index 000000000..af8f29d64 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/mapper/ExportMapperContractTest.java @@ -0,0 +1,54 @@ +package com.baiyue.file.mapper; + +import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.Update; +import org.apache.ibatis.annotations.Delete; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ExportMapperContractTest { + @Test void finalSuccessCommitIsGuardedByVersionLeaseAndDatabaseDeadline() throws Exception { + String sql=update(FileExportTaskMapper.class,"succeed"); + assertTrue(sql.contains("dispatch_version=#{dispatchVersion}")); + assertTrue(sql.contains("lease_owner=#{owner}")); + assertTrue(sql.contains("execution_deadline_at>CURRENT_TIMESTAMP(3)")); + } + + @Test void sentRedispatchKeepsTheSameVersionAndExcludesQuarantinedRows() throws Exception { + String sql=update(FileExportOutboxMapper.class,"redispatchSent"); + assertTrue(sql.contains("o.status='SENT'")); + assertTrue(sql.contains("t.dispatch_version=o.dispatch_version")); + assertFalse(sql.contains("dispatch_version=dispatch_version+1")); + assertFalse(sql.contains("QUARANTINED")); + } + + @Test void missingOutboxDetectionUsesDatabaseTimeAndLetsAnyExistingStatusBlockDuplication() throws Exception { + String sql=select(FileExportTaskMapper.class,"missingOutbox"); + assertTrue(sql.contains("CURRENT_TIMESTAMP(3)")); + assertTrue(sql.contains("NOT EXISTS")); + assertTrue(sql.contains("o.dispatch_version=file_export_task.dispatch_version")); + assertFalse(sql.contains("o.status")); + } + + @Test void retentionCleanupCanOnlyDeleteOldSentRows() throws Exception { + Method target=java.util.Arrays.stream(FileExportOutboxMapper.class.getDeclaredMethods()).filter(candidate -> candidate.getName().equals("deleteRetainedSent")).findFirst().orElseThrow(); + String sql=String.join(" ",target.getAnnotation(Delete.class).value()).replaceAll("\\s+"," "); + assertTrue(sql.contains("status='SENT'")); + assertTrue(sql.contains("CURRENT_TIMESTAMP(3)")); + assertFalse(sql.contains("PENDING")); + assertFalse(sql.contains("CLAIMED")); + } + + private String update(Class type,String method) throws Exception { + Method target=java.util.Arrays.stream(type.getDeclaredMethods()).filter(candidate -> candidate.getName().equals(method)).findFirst().orElseThrow(); + return String.join(" ",target.getAnnotation(Update.class).value()).replaceAll("\\s+"," "); + } + private String select(Class type,String method) throws Exception { + Method target=java.util.Arrays.stream(type.getDeclaredMethods()).filter(candidate -> candidate.getName().equals(method)).findFirst().orElseThrow(); + return String.join(" ",target.getAnnotation(Select.class).value()).replaceAll("\\s+"," "); + } +} diff --git a/file/src/test/java/com/baiyue/file/mq/ExportCommandConsumerTest.java b/file/src/test/java/com/baiyue/file/mq/ExportCommandConsumerTest.java new file mode 100644 index 000000000..ae87ae9a6 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/mq/ExportCommandConsumerTest.java @@ -0,0 +1,38 @@ +package com.baiyue.file.mq; + +import com.baiyue.file.service.ExportTaskExecutor; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.common.message.MessageExt; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +class ExportCommandConsumerTest { + @Test void appliesConfiguredConcurrencyAndDispatchesAValidCommand() { + ExportTaskExecutor executor=mock(ExportTaskExecutor.class); + ExportCommandConsumer consumer=new ExportCommandConsumer(new ObjectMapper(),executor,"file-1",4,90,2); + DefaultMQPushConsumer pushConsumer=new DefaultMQPushConsumer(); + consumer.prepareStart(pushConsumer); + assertEquals(4,pushConsumer.getConsumeThreadMin()); + assertEquals(4,pushConsumer.getConsumeThreadMax()); + assertEquals(90,pushConsumer.getConsumeTimeout()); + assertEquals(2,pushConsumer.getMaxReconsumeTimes()); + + MessageExt message=new MessageExt(); + message.setBody("{\"schemaVersion\":1,\"eventId\":\"event-1\",\"taskId\":\"9\",\"exportType\":\"CUSTOMER_LIST\",\"dispatchVersion\":3}".getBytes(StandardCharsets.UTF_8)); + consumer.onMessage(message); + verify(executor).execute(9L,3,"file-1"); + } + + @Test void rejectsMalformedMessagesSoRocketMqCanRedeliverThem() { + ExportCommandConsumer consumer=new ExportCommandConsumer(new ObjectMapper(),mock(ExportTaskExecutor.class),"file-1",4,90,2); + MessageExt message=new MessageExt();message.setBody("not-json".getBytes(StandardCharsets.UTF_8)); + assertThrows(IllegalArgumentException.class,() -> consumer.onMessage(message)); + } +} diff --git a/file/src/test/java/com/baiyue/file/mq/ExportDlqHandlerTest.java b/file/src/test/java/com/baiyue/file/mq/ExportDlqHandlerTest.java new file mode 100644 index 000000000..573bd7103 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/mq/ExportDlqHandlerTest.java @@ -0,0 +1,29 @@ +package com.baiyue.file.mq; + +import com.baiyue.file.service.ExportStateService; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; +import org.junit.jupiter.api.Test; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +import static org.mockito.Mockito.*; + +class ExportDlqHandlerTest { + @Test void databaseFailuresCannotFallIntoAnUnconsumedSecondaryDlq() { + RocketMQMessageListener listener=ExportDlqHandler.class.getAnnotation(RocketMQMessageListener.class); + org.junit.jupiter.api.Assertions.assertEquals(Integer.MAX_VALUE,listener.maxReconsumeTimes()); + } + @Test void quarantinesUsingUserPropertiesWithoutParsingTheBody() { + ExportStateService states=mock(ExportStateService.class); MessageExt message=new MessageExt(); + SimpleMeterRegistry meters=new SimpleMeterRegistry(); + message.putUserProperty("eventId","event-1");message.putUserProperty("taskId","9");message.putUserProperty("dispatchVersion","3");message.putUserProperty("schemaVersion","1"); + new ExportDlqHandler(states,meters).onMessage(message); + verify(states).quarantine(eq("event-1"),eq(9L),eq(3),contains("自动隔离")); + org.junit.jupiter.api.Assertions.assertEquals(1,meters.counter("export_dlq_messages").count()); + } + @Test void malformedPropertiesAreAcknowledgedWithoutTouchingTasks() { + ExportStateService states=mock(ExportStateService.class); SimpleMeterRegistry meters=new SimpleMeterRegistry(); + new ExportDlqHandler(states,meters).onMessage(new MessageExt()); + verifyNoInteractions(states); org.junit.jupiter.api.Assertions.assertEquals(1,meters.counter("export_dlq_messages").count()); + } +} diff --git a/file/src/test/java/com/baiyue/file/mq/ExportOutboxPublisherTest.java b/file/src/test/java/com/baiyue/file/mq/ExportOutboxPublisherTest.java new file mode 100644 index 000000000..19613f251 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/mq/ExportOutboxPublisherTest.java @@ -0,0 +1,67 @@ +package com.baiyue.file.mq; + +import com.baiyue.file.entity.FileExportOutbox; +import com.baiyue.file.repository.ExportOutboxRepository; +import org.apache.rocketmq.spring.core.RocketMQTemplate; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.messaging.Message; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class ExportOutboxPublisherTest { + @Test void publishesAllDlqLocatorFieldsAsRocketMqUserProperties() { + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + RocketMQTemplate rocket=mock(RocketMQTemplate.class); + FileExportOutbox row=row(); + when(outbox.publishableIds(50)).thenReturn(List.of(1L));when(outbox.claim(1L,"file-1",60)).thenReturn(true);when(outbox.find(1L)).thenReturn(row); + + new ExportOutboxPublisher(outbox,rocket,"file-1",50,60).publish(); + + @SuppressWarnings("rawtypes") ArgumentCaptor message=ArgumentCaptor.forClass(Message.class); + verify(rocket).syncSend(eq("test-export:EXPORT_TASK"),message.capture(),eq(3000L)); + assertEquals("event-1",message.getValue().getHeaders().get("eventId")); + assertEquals("9",message.getValue().getHeaders().get("taskId")); + assertEquals("3",message.getValue().getHeaders().get("dispatchVersion")); + assertEquals("1",message.getValue().getHeaders().get("schemaVersion")); + verify(outbox).sent(1L,"file-1"); + } + + @Test void returnsFailedSendsToPendingWithTheFirstBackoffInsteadOfGivingUp() { + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + RocketMQTemplate rocket=mock(RocketMQTemplate.class); + FileExportOutbox row=row();row.setAttemptCount(1); + when(outbox.publishableIds(50)).thenReturn(List.of(1L));when(outbox.claim(1L,"file-1",60)).thenReturn(true);when(outbox.find(1L)).thenReturn(row); + doThrow(new RuntimeException("broker down")).when(rocket).syncSend(anyString(),any(Message.class),anyLong()); + + new ExportOutboxPublisher(outbox,rocket,"file-1",50,60).publish(); + + verify(outbox).sendFailed(1L,"file-1",5,"broker down"); + verify(outbox,never()).sent(anyLong(),anyString()); + } + + @Test void stillRetriesAfterThirtyConsecutiveBrokerFailuresWithoutADeadState() { + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + RocketMQTemplate rocket=mock(RocketMQTemplate.class); + FileExportOutbox row=row();row.setAttemptCount(30); + when(outbox.publishableIds(50)).thenReturn(List.of(1L)); + when(outbox.claim(1L,"file-1",60)).thenReturn(true); + when(outbox.find(1L)).thenReturn(row); + doThrow(new RuntimeException("broker down")).when(rocket).syncSend(anyString(),any(Message.class),anyLong()); + ExportOutboxPublisher publisher=new ExportOutboxPublisher(outbox,rocket,"file-1",50,60); + + for(int i=0;i<30;i++) publisher.publish(); + + verify(outbox,times(30)).sendFailed(1L,"file-1",900,"broker down"); + verify(outbox,never()).sent(anyLong(),anyString()); + } + + private FileExportOutbox row(){ + FileExportOutbox row=new FileExportOutbox();row.setId(1L);row.setEventId("event-1");row.setTaskId(9L);row.setDispatchVersion(3); + row.setTopic("test-export");row.setTag("EXPORT_TASK");row.setMessageKey("9");row.setPayloadJson("{}");row.setAttemptCount(1);return row; + } +} diff --git a/file/src/test/java/com/baiyue/file/service/CustomerExportTaskHandlerTest.java b/file/src/test/java/com/baiyue/file/service/CustomerExportTaskHandlerTest.java deleted file mode 100644 index 11ce7a8de..000000000 --- a/file/src/test/java/com/baiyue/file/service/CustomerExportTaskHandlerTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.baiyue.file.service; - -import com.baiyue.common.core.api.ApiResponse; -import com.baiyue.file.client.IamExportClient; -import com.baiyue.file.entity.FileExportTask; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class CustomerExportTaskHandlerTest { - @TempDir Path tempDir; - - @Test - void generatesARealXlsxFromIamBatchesAndReportsProgress() throws Exception { - IamExportClient client = mock(IamExportClient.class); - LocalExportStorage storage = new LocalExportStorage(tempDir.toString()); - storage.initialize(); - var row = new IamExportClient.CustomerExportRow( - "101", "TEN101", "柏悦物流", "柏悦物流有限公司", "91440300MA5ABCDE12", - "有限责任公司", null, null, "CNY", "深圳市", "张三", "13800138000", - "ops@example.com", "baiyue", "李四", "基础版", 30, "ACTIVE", null, null); - when(client.customers("internal-token", null, 500, "柏悦", "ACTIVE")) - .thenReturn(ApiResponse.success(new IamExportClient.CustomerExportBatch(List.of(row), 101L, false))); - CustomerExportTaskHandler handler = new CustomerExportTaskHandler( - client, storage, new ObjectMapper().findAndRegisterModules(), "internal-token"); - FileExportTask task = new FileExportTask(); - task.setId(9L); - task.setParametersJson("{\"keyword\":\"柏悦\",\"status\":\"ACTIVE\"}"); - AtomicLong progress = new AtomicLong(); - - var result = handler.export(task, progress::set); - - assertEquals(1L, result.rows()); - assertEquals(1L, progress.get()); - assertTrue(result.fileName().endsWith(".xlsx")); - Path workbookPath = tempDir.resolve(result.objectKey()); - assertTrue(Files.isRegularFile(workbookPath)); - try (XSSFWorkbook workbook = new XSSFWorkbook(Files.newInputStream(workbookPath))) { - assertEquals("客户ID", workbook.getSheetAt(0).getRow(0).getCell(0).getStringCellValue()); - assertEquals("柏悦物流", workbook.getSheetAt(0).getRow(1).getCell(2).getStringCellValue()); - assertEquals("13800138000", workbook.getSheetAt(0).getRow(1).getCell(11).getStringCellValue()); - } - } -} diff --git a/file/src/test/java/com/baiyue/file/service/ExportReconcilerTest.java b/file/src/test/java/com/baiyue/file/service/ExportReconcilerTest.java new file mode 100644 index 000000000..0f5f4dc19 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/service/ExportReconcilerTest.java @@ -0,0 +1,44 @@ +package com.baiyue.file.service; + +import com.baiyue.file.entity.FileExportTask; +import com.baiyue.file.repository.ExportOutboxRepository; +import com.baiyue.file.repository.ExportTaskRepository; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.List; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class ExportReconcilerTest { + @Test void performsEveryRecoveryResponsibilityWithoutExecutingBusinessExports() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + ExportStateService states=mock(ExportStateService.class); + ExportStorage storage=mock(ExportStorage.class); + FileExportTask missing=task(1L,"PENDING"), stale=task(2L,"RUNNING"), expired=task(3L,"SUCCEEDED");expired.setObjectKey("exports/3/file.xlsx"); + when(outbox.releaseExpiredClaims()).thenReturn(2);when(outbox.redispatchSent(600,60)).thenReturn(3); + when(tasks.missingOutbox(60)).thenReturn(List.of(missing));when(states.ensureOutbox(missing)).thenReturn(true); + when(tasks.staleRunning()).thenReturn(List.of(stale));when(states.recoverStale(stale)).thenReturn(true); + when(tasks.expiredArtifactsDb()).thenReturn(List.of(expired));when(storage.delete(expired.getObjectKey())).thenReturn(true); + when(storage.cleanupOrphans(eq(Duration.ofHours(24)),any())).thenReturn(1); + ExportReconciler reconciler=new ExportReconciler(tasks,outbox,states,storage,new SimpleMeterRegistry(),60,600,7); + + reconciler.reconcile(); + + verify(outbox).releaseExpiredClaims();verify(outbox).redispatchSent(600,60); + verify(states).ensureOutbox(missing);verify(states).recoverStale(stale);verify(tasks).markExpired(3L); + verify(storage).cleanupOrphans(eq(Duration.ofHours(24)),any()); + } + + @Test void deletesOnlyTheConfiguredDailyBatchOfRetainedSentOutboxRows() { + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + ExportReconciler reconciler=new ExportReconciler(mock(ExportTaskRepository.class),outbox,mock(ExportStateService.class),mock(ExportStorage.class),new SimpleMeterRegistry(),60,600,7); + reconciler.cleanupRetainedOutbox(); + verify(outbox).deleteRetainedSent(7,5000); + } + + private FileExportTask task(long id,String status){FileExportTask task=new FileExportTask();task.setId(id);task.setStatus(status);return task;} +} diff --git a/file/src/test/java/com/baiyue/file/service/ExportStateServiceTest.java b/file/src/test/java/com/baiyue/file/service/ExportStateServiceTest.java new file mode 100644 index 000000000..c1552dbcc --- /dev/null +++ b/file/src/test/java/com/baiyue/file/service/ExportStateServiceTest.java @@ -0,0 +1,110 @@ +package com.baiyue.file.service; + +import com.baiyue.file.repository.ExportOutboxRepository; +import com.baiyue.file.repository.ExportTaskRepository; +import com.baiyue.file.entity.FileExportTask; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +class ExportStateServiceTest { + @Test void creationLimitsAndDuplicateDetectionRunUnderTheDatabaseGuard() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + ExportCommandFactory commands=mock(ExportCommandFactory.class); + FileExportTask task=task(9L,1,"PENDING"); + task.setTenantId(0L);task.setAppCode("PLATFORM");task.setRequestedBy(7L);task.setExportType("CUSTOMER_LIST");task.setParametersJson("{}"); + when(commands.create(task,1)).thenReturn(mock(com.baiyue.file.entity.FileExportOutbox.class)); + when(outbox.insert(any())).thenReturn(true); + ExportStateService service=new ExportStateService(tasks,outbox,commands,mock(ExportMetrics.class)); + + ExportStateService.CreationResult result=service.createConstrained(task); + + org.junit.jupiter.api.Assertions.assertTrue(result.created()); + InOrder order=inOrder(tasks,outbox); + order.verify(tasks).lockCreationGuard(); + order.verify(tasks).recentDuplicate(0L,"PLATFORM",7L,"CUSTOMER_LIST","{}"); + order.verify(tasks).activeForUser(0L,"PLATFORM",7L); + order.verify(tasks).activeForType("CUSTOMER_LIST"); + order.verify(tasks).insert(task); + order.verify(tasks).initializeDispatchTime(9L); + order.verify(outbox).insert(any()); + } + + @Test void duplicateCreationReturnsTheExistingTaskWithoutWritingAnotherOutbox() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + FileExportTask request=task(9L,1,"PENDING"), existing=task(8L,1,"PENDING"); + request.setTenantId(0L);request.setAppCode("PLATFORM");request.setRequestedBy(7L);request.setExportType("CUSTOMER_LIST");request.setParametersJson("{}"); + when(tasks.recentDuplicate(0L,"PLATFORM",7L,"CUSTOMER_LIST","{}")).thenReturn(existing); + ExportStateService service=new ExportStateService(tasks,outbox,mock(ExportCommandFactory.class),mock(ExportMetrics.class)); + + ExportStateService.CreationResult result=service.createConstrained(request); + + org.junit.jupiter.api.Assertions.assertFalse(result.created()); + org.junit.jupiter.api.Assertions.assertSame(existing,result.task()); + verify(tasks).lockCreationGuard(); + verify(tasks,never()).insert(any()); + verifyNoInteractions(outbox); + } + + @Test void activeTaskLimitsAreEnforcedAfterTakingTheCreationGuard() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + FileExportTask request=task(9L,1,"PENDING"); + request.setTenantId(0L);request.setAppCode("PLATFORM");request.setRequestedBy(7L);request.setExportType("CUSTOMER_LIST");request.setParametersJson("{}"); + when(tasks.activeForUser(0L,"PLATFORM",7L)).thenReturn(3L); + ExportStateService service=new ExportStateService(tasks,outbox,mock(ExportCommandFactory.class),mock(ExportMetrics.class)); + + assertThrows(com.baiyue.common.core.exception.BizException.class,() -> service.createConstrained(request)); + + InOrder order=inOrder(tasks); + order.verify(tasks).lockCreationGuard(); + order.verify(tasks).recentDuplicate(0L,"PLATFORM",7L,"CUSTOMER_LIST","{}"); + order.verify(tasks).activeForUser(0L,"PLATFORM",7L); + verify(tasks,never()).insert(any()); + verifyNoInteractions(outbox); + } + + @Test void quarantineRequiresOutboxAndTaskToChangeAtomically() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + when(tasks.find(9L)).thenReturn(task(9L,3,"PENDING")); + when(outbox.quarantine("event-1",9L,3,"broken")).thenReturn(true); + when(tasks.failQuarantined(9L,3,"broken")).thenReturn(false); + ExportStateService service=new ExportStateService(tasks,outbox,mock(ExportCommandFactory.class),mock(ExportMetrics.class)); + + assertThrows(IllegalStateException.class,() -> service.quarantine("event-1",9L,3,"broken")); + verify(tasks).failQuarantined(9L,3,"broken"); + } + + @Test void oldDlqMessageThatMatchesNoOutboxDoesNotTouchTheCurrentTask() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + when(tasks.find(9L)).thenReturn(task(9L,3,"PENDING")); + when(outbox.quarantine(anyString(),anyLong(),anyInt(),anyString())).thenReturn(false); + ExportStateService service=new ExportStateService(tasks,outbox,mock(ExportCommandFactory.class),mock(ExportMetrics.class)); + + assertDoesNotThrow(() -> service.quarantine("old-event",9L,2,"broken")); + verify(tasks).find(9L); + verifyNoMoreInteractions(tasks); + } + + @Test void matchingDlqMessageDoesNotDamageAnAlreadyRunningTask() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportOutboxRepository outbox=mock(ExportOutboxRepository.class); + when(tasks.find(9L)).thenReturn(task(9L,3,"RUNNING")); + ExportStateService service=new ExportStateService(tasks,outbox,mock(ExportCommandFactory.class),mock(ExportMetrics.class)); + + assertDoesNotThrow(() -> service.quarantine("event-1",9L,3,"broken")); + verifyNoInteractions(outbox); + verify(tasks,never()).failQuarantined(anyLong(),anyInt(),anyString()); + } + + private FileExportTask task(long id,int version,String status){ + FileExportTask task=new FileExportTask();task.setId(id);task.setDispatchVersion(version);task.setStatus(status);return task; + } +} diff --git a/file/src/test/java/com/baiyue/file/service/ExportTaskExecutorTest.java b/file/src/test/java/com/baiyue/file/service/ExportTaskExecutorTest.java new file mode 100644 index 000000000..7b7f48511 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/service/ExportTaskExecutorTest.java @@ -0,0 +1,105 @@ +package com.baiyue.file.service; + +import com.baiyue.file.config.ExportDefinitionRegistry; +import com.baiyue.file.entity.FileExportTask; +import com.baiyue.file.repository.ExportTaskRepository; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class ExportTaskExecutorTest { + @Test void aCommandThatCannotClaimTheTaskIsAcknowledgedWithoutExecutingOrChangingState() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportDefinitionRegistry definitions=mock(ExportDefinitionRegistry.class); + GenericXlsxExportRunner runner=mock(GenericXlsxExportRunner.class); + ExportStateService states=mock(ExportStateService.class); + when(tasks.claim(9L,1,"file-1",300,30)).thenReturn(false); + ExportTaskExecutor executor=new ExportTaskExecutor(tasks,definitions,runner,states,mock(ExportStorage.class),new SimpleMeterRegistry(),300,60,30,7); + + executor.execute(9L,1,"file-1"); + + verifyNoInteractions(definitions,runner,states); + executor.shutdownWatchdogs(); + } + + @Test void watchdogRenewsTheLeaseWhileTheRunnerIsStillWriting() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportDefinitionRegistry definitions=mock(ExportDefinitionRegistry.class); + GenericXlsxExportRunner runner=mock(GenericXlsxExportRunner.class); + ExportStateService states=mock(ExportStateService.class); + FileExportTask running=task(1,"file-1"); + when(tasks.claim(9L,1,"file-1",5,30)).thenReturn(true);when(tasks.find(9L)).thenReturn(running); + when(tasks.heartbeat(9L,1,"file-1",5)).thenReturn(true);when(tasks.succeed(anyLong(),anyInt(),anyString(),anyString(),anyString(),anyString(),anyLong(),anyLong(),anyInt())).thenReturn(true); + when(definitions.require("CUSTOMER_LIST")).thenReturn(new ExportDefinitionRegistry.Definition()); + when(runner.run(eq(running),any(),any(AtomicBoolean.class),any())).thenAnswer(invocation -> { + Thread.sleep(1200); + return new GenericXlsxExportRunner.Result("file.xlsx","exports/9/test.xlsx","application/xlsx",100,1); + }); + ExportTaskExecutor executor=new ExportTaskExecutor(tasks,definitions,runner,states,mock(ExportStorage.class),new SimpleMeterRegistry(),5,1,30,7); + + executor.execute(9L,1,"file-1"); + + verify(tasks,atLeastOnce()).heartbeat(9L,1,"file-1",5); + executor.shutdownWatchdogs(); + } + + @Test void aLateFailureFromAnOldExecutionCannotRetryOrFailANewerVersionOwnedByTheSameInstance() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportDefinitionRegistry definitions=mock(ExportDefinitionRegistry.class); + GenericXlsxExportRunner runner=mock(GenericXlsxExportRunner.class); + ExportStateService states=mock(ExportStateService.class); + ExportStorage storage=mock(ExportStorage.class); + when(tasks.claim(9L,1,"file-1",300,30)).thenReturn(true); + FileExportTask first=task(1,"file-1"), newer=task(2,"file-1"); + when(tasks.find(9L)).thenReturn(first,newer); + when(definitions.require("CUSTOMER_LIST")).thenReturn(new ExportDefinitionRegistry.Definition()); + when(runner.run(eq(first),any(),any(),any())).thenThrow(new RuntimeException("late failure")); + ExportTaskExecutor executor=new ExportTaskExecutor(tasks,definitions,runner,states,storage,new SimpleMeterRegistry(),300,60,30,7); + + executor.execute(9L,1,"file-1"); + + verifyNoInteractions(states); + executor.shutdownWatchdogs(); + } + + @Test void deletesTheGeneratedFileWhenTheFinalDatabaseCommitThrows() { + ExportTaskRepository tasks=mock(ExportTaskRepository.class); + ExportDefinitionRegistry definitions=mock(ExportDefinitionRegistry.class); + GenericXlsxExportRunner runner=mock(GenericXlsxExportRunner.class); + ExportStateService states=mock(ExportStateService.class); + ExportStorage storage=mock(ExportStorage.class); + FileExportTask running=task(1,"file-1"); + when(tasks.claim(9L,1,"file-1",300,30)).thenReturn(true); + when(tasks.find(9L)).thenReturn(running); + when(definitions.require("CUSTOMER_LIST")).thenReturn(new ExportDefinitionRegistry.Definition()); + when(runner.run(eq(running),any(),any(),any())).thenReturn( + new GenericXlsxExportRunner.Result("file.xlsx","exports/9/result.xlsx","application/xlsx",100,1)); + when(tasks.succeed(anyLong(),anyInt(),anyString(),anyString(),anyString(),anyString(),anyLong(),anyLong(),anyInt())) + .thenThrow(new RuntimeException("database unavailable")); + when(states.retryAfterFailure(eq(running),eq("file-1"),intThat(value -> value >= 48 && value <= 72),anyString())).thenReturn(true); + ExportTaskExecutor executor=new ExportTaskExecutor(tasks,definitions,runner,states,storage,new SimpleMeterRegistry(),300,60,30,7); + + executor.execute(9L,1,"file-1"); + + verify(storage).delete("exports/9/result.xlsx"); + verify(states).retryAfterFailure(eq(running),eq("file-1"),intThat(value -> value >= 48 && value <= 72),eq("database unavailable")); + executor.shutdownWatchdogs(); + } + + @Test void retryBackoffAlwaysStaysWithinTheDocumentedTwentyPercentJitter() { + for (int i=0;i<100;i++) { + int first=ExportTaskExecutor.jitteredBackoffSeconds(60); + int second=ExportTaskExecutor.jitteredBackoffSeconds(900); + org.junit.jupiter.api.Assertions.assertTrue(first>=48 && first<=72); + org.junit.jupiter.api.Assertions.assertTrue(second>=720 && second<=1080); + } + } + + private FileExportTask task(int version,String owner){ + FileExportTask task=new FileExportTask();task.setId(9L);task.setExportType("CUSTOMER_LIST");task.setStatus("RUNNING"); + task.setDispatchVersion(version);task.setLeaseOwner(owner);task.setAttemptCount(1);task.setMaxAttempts(3);return task; + } +} diff --git a/file/src/test/java/com/baiyue/file/service/ExportTaskServiceTest.java b/file/src/test/java/com/baiyue/file/service/ExportTaskServiceTest.java index 77f54f0d1..5f4a6be31 100644 --- a/file/src/test/java/com/baiyue/file/service/ExportTaskServiceTest.java +++ b/file/src/test/java/com/baiyue/file/service/ExportTaskServiceTest.java @@ -9,8 +9,9 @@ import com.baiyue.file.repository.ExportTaskRepository; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; -import java.util.List; +import com.baiyue.file.config.ExportDefinitionRegistry; import java.util.Map; +import java.util.List; import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -25,19 +26,18 @@ class ExportTaskServiceTest { @Test void createsPersistedTaskOnlyWhenSourceExportPermissionIsPresent() { ExportTaskRepository repository = mock(ExportTaskRepository.class); - ExportTaskHandler handler = mock(ExportTaskHandler.class); - when(handler.exportType()).thenReturn("CUSTOMER_LIST"); - when(handler.requiredPermission()).thenReturn("platform:customer:export"); - ExportTaskService service = new ExportTaskService(repository, mock(LocalExportStorage.class), - new ObjectMapper(), List.of(handler)); - doAnswer(invocation -> null).when(repository).insert(any()); + ExportStateService states=mock(ExportStateService.class); + when(states.createConstrained(any(FileExportTask.class))).thenAnswer(invocation -> + new ExportStateService.CreationResult(invocation.getArgument(0), true)); + ExportTaskService service = new ExportTaskService(repository, mock(ExportStorage.class), + new ObjectMapper(), registry(), states, mock(ExportMetrics.class)); var response = service.create(user(1L, Set.of("platform:customer:export")), new CreateExportTaskRequest("CUSTOMER_LIST", "XLSX", "客户列表", Map.of("status", "ACTIVE"))); assertEquals("PENDING", response.status()); assertEquals("CUSTOMER_LIST", response.exportType()); - verify(repository).insert(any(FileExportTask.class)); + verify(states).createConstrained(any(FileExportTask.class)); BizException denied = assertThrows(BizException.class, () -> service.create(user(1L, Set.of()), new CreateExportTaskRequest("CUSTOMER_LIST", "XLSX", "客户列表", Map.of()))); assertEquals(ResultCode.FORBIDDEN, denied.getResultCode()); @@ -46,10 +46,8 @@ class ExportTaskServiceTest { @Test void refusesAnotherUsersTaskUnlessDownloadAllIsGranted() { ExportTaskRepository repository = mock(ExportTaskRepository.class); - ExportTaskHandler handler = mock(ExportTaskHandler.class); - when(handler.exportType()).thenReturn("CUSTOMER_LIST"); - ExportTaskService service = new ExportTaskService(repository, mock(LocalExportStorage.class), - new ObjectMapper(), List.of(handler)); + ExportTaskService service = new ExportTaskService(repository, mock(ExportStorage.class), + new ObjectMapper(), registry(), mock(ExportStateService.class), mock(ExportMetrics.class)); FileExportTask task = new FileExportTask(); task.setId(9L); task.setTenantId(0L); task.setAppCode("PLATFORM"); task.setRequestedBy(2L); when(repository.find(9L)).thenReturn(task); @@ -59,7 +57,22 @@ class ExportTaskServiceTest { assertEquals("9", service.get(user(1L, Set.of("platform:download:all")), 9L).id()); } + @Test + void rejectsParametersDeeperThanTwoLevels() { + ExportTaskService service = new ExportTaskService(mock(ExportTaskRepository.class), mock(ExportStorage.class), + new ObjectMapper(), registry(), mock(ExportStateService.class), mock(ExportMetrics.class)); + var request = new CreateExportTaskRequest("CUSTOMER_LIST", "XLSX", "客户列表", + Map.of("level1", Map.of("level2", List.of(Map.of("level3", "value"))))); + + BizException error=assertThrows(BizException.class,() -> service.create(user(1L,Set.of("platform:customer:export")),request)); + assertEquals(ResultCode.VALIDATION_FAILED,error.getResultCode()); + } + private AuthenticatedUser user(long id, Set permissions) { return new AuthenticatedUser(id, 0L, 0L, id, "user" + id, "PLATFORM", "WEB", permissions); } + private ExportDefinitionRegistry registry(){ + var definition=new ExportDefinitionRegistry.Definition();definition.setProviderService("iam-service");definition.setProviderPath("/internal/exports/v1/CUSTOMER_LIST/batches");definition.setRequiredPermission("platform:customer:export");definition.setAppScope("PLATFORM");definition.setMaxRows(200000);definition.setMaxAttempts(3);definition.setBatchSize(500); + ExportDefinitionRegistry registry=mock(ExportDefinitionRegistry.class);when(registry.require("CUSTOMER_LIST")).thenReturn(definition);return registry; + } } diff --git a/file/src/test/java/com/baiyue/file/service/ExportTaskWorkerTest.java b/file/src/test/java/com/baiyue/file/service/ExportTaskWorkerTest.java deleted file mode 100644 index 438ebafdc..000000000 --- a/file/src/test/java/com/baiyue/file/service/ExportTaskWorkerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.baiyue.file.service; - -import com.baiyue.file.entity.FileExportTask; -import com.baiyue.file.repository.ExportTaskRepository; -import org.junit.jupiter.api.Test; - -import java.time.LocalDateTime; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class ExportTaskWorkerTest { - @Test - void onlyExecutesATaskAfterTheAtomicClaimSucceeds() { - ExportTaskRepository repository = mock(ExportTaskRepository.class); - ExportTaskService service = mock(ExportTaskService.class); - LocalExportStorage storage = mock(LocalExportStorage.class); - ExportTaskHandler handler = mock(ExportTaskHandler.class); - FileExportTask task = new FileExportTask(); - task.setId(9L); task.setExportType("CUSTOMER_LIST"); - when(repository.firstPendingId()).thenReturn(9L); - when(repository.claim(eq(9L), any(LocalDateTime.class))).thenReturn(true); - when(repository.find(9L)).thenReturn(task); - when(service.handler("CUSTOMER_LIST")).thenReturn(handler); - when(handler.export(eq(task), any())).thenAnswer(invocation -> { - ExportTaskHandler.ProgressReporter progress = invocation.getArgument(1); - progress.update(12L); - return new ExportTaskHandler.ExportResult("客户列表.xlsx", "exports/9/file.xlsx", - CustomerExportTaskHandler.CONTENT_TYPE, 1024L, 12L); - }); - when(repository.succeed(eq(9L), eq("客户列表.xlsx"), eq("exports/9/file.xlsx"), - eq(CustomerExportTaskHandler.CONTENT_TYPE), eq(1024L), eq(12L), - any(LocalDateTime.class), any(LocalDateTime.class))).thenReturn(true); - ExportTaskWorker worker = new ExportTaskWorker(repository, service, storage, 7, 30); - - worker.processNext(); - - verify(repository).progress(9L, 12L); - verify(repository).succeed(eq(9L), eq("客户列表.xlsx"), eq("exports/9/file.xlsx"), - eq(CustomerExportTaskHandler.CONTENT_TYPE), eq(1024L), eq(12L), - any(LocalDateTime.class), any(LocalDateTime.class)); - } - - @Test - void skipsAContendedTaskThatAnotherWorkerClaimed() { - ExportTaskRepository repository = mock(ExportTaskRepository.class); - when(repository.firstPendingId()).thenReturn(9L); - when(repository.claim(eq(9L), any(LocalDateTime.class))).thenReturn(false); - ExportTaskWorker worker = new ExportTaskWorker(repository, mock(ExportTaskService.class), - mock(LocalExportStorage.class), 7, 30); - - worker.processNext(); - - verify(repository, never()).find(9L); - } -} diff --git a/file/src/test/java/com/baiyue/file/service/GenericXlsxExportRunnerTest.java b/file/src/test/java/com/baiyue/file/service/GenericXlsxExportRunnerTest.java new file mode 100644 index 000000000..09bf40617 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/service/GenericXlsxExportRunnerTest.java @@ -0,0 +1,67 @@ +package com.baiyue.file.service; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.export.ExportContracts.*; +import com.baiyue.file.client.ExportProviderClient; +import com.baiyue.file.config.ExportDefinitionRegistry; +import com.baiyue.file.entity.FileExportTask; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class GenericXlsxExportRunnerTest { + @TempDir Path temp; + @Test void writesRealTypedWorkbookAndCarriesSnapshotAcrossBatches() throws Exception { + ExportProviderClient provider=mock(ExportProviderClient.class); LocalExportStorage storage=new LocalExportStorage(temp.toString());storage.initialize(); + List columns=List.of(new ExportColumn("text","文本",ExportColumnType.STRING,18),new ExportColumn("integer","整数",ExportColumnType.INTEGER,12),new ExportColumn("decimal","金额",ExportColumnType.DECIMAL,12),new ExportColumn("flag","布尔",ExportColumnType.BOOLEAN,10),new ExportColumn("date","日期",ExportColumnType.DATE,12),new ExportColumn("time","时间",ExportColumnType.DATETIME,20)); + List> firstRows=IntStream.range(0,100).mapToObj(index -> Map.of("text",index==0?"=SUM(1,2)":"row-"+index,"integer",12,"decimal","3.50","flag",true,"date","2026-08-03","time","2026-08-03T21:00:00")).toList(); + var first=new ExportBatchResponse(1,"sha256:test","snapshot-9",columns,firstRows,"10",true,101L); + var second=new ExportBatchResponse(1,"sha256:test","snapshot-9",null,List.of(Map.of("text","\n=SUM(1,2)","integer",13,"decimal",4.5,"flag",false,"date","2026-08-04","time","2026-08-04T21:00:00")),null,false,101L); + when(provider.batch(anyString(),eq("CUSTOMER_LIST"),any(),any())).thenReturn(ApiResponse.success(first),ApiResponse.success(second)); + FileExportTask task=new FileExportTask();task.setId(9L);task.setTenantId(0L);task.setAppCode("PLATFORM");task.setRequestedBy(7L);task.setDispatchVersion(1);task.setExportType("CUSTOMER_LIST");task.setTaskName("客户列表");task.setParametersJson("{}"); + var definition=new ExportDefinitionRegistry.Definition();definition.setMaxRows(200000);definition.setBatchSize(500); + var result=new GenericXlsxExportRunner(provider,storage,new ObjectMapper(),"internal-token",50).run(task,definition,new AtomicBoolean(),ignored->{ }); + assertEquals(101,result.rows()); + try(var workbook=new XSSFWorkbook(storage.load(result.objectKey()).getInputStream())){ + var row=workbook.getSheetAt(0).getRow(1);assertEquals("'=SUM(1,2)",row.getCell(0).getStringCellValue()); + assertEquals(CellType.NUMERIC,row.getCell(1).getCellType());assertEquals(CellType.NUMERIC,row.getCell(2).getCellType());assertEquals(CellType.BOOLEAN,row.getCell(3).getCellType());assertEquals(CellType.NUMERIC,row.getCell(4).getCellType());assertEquals(CellType.NUMERIC,row.getCell(5).getCellType()); + assertEquals("'\n=SUM(1,2)",workbook.getSheetAt(0).getRow(101).getCell(0).getStringCellValue()); + assertTrue(workbook.getNumCellStyles()<10,"日期样式必须复用，不能按单元格创建"); + } + ArgumentCaptor requests=ArgumentCaptor.forClass(ExportBatchRequest.class);verify(provider,times(2)).batch(anyString(),eq("CUSTOMER_LIST"),eq(definition),requests.capture()); + assertNull(requests.getAllValues().get(0).snapshotToken());assertEquals("snapshot-9",requests.getAllValues().get(1).snapshotToken()); + } + + @Test void deletesTheCompletedFileWhenReadingItsFinalSizeFails() throws Exception { + ExportProviderClient provider=mock(ExportProviderClient.class); + LocalExportStorage storage=spy(new LocalExportStorage(temp.toString()));storage.initialize(); + var batch=new ExportBatchResponse(1,"sha256:test","snapshot-9", + List.of(new ExportColumn("text","文本",ExportColumnType.STRING,18)), + List.of(Map.of("text","row")),null,false,1L); + when(provider.batch(anyString(),eq("CUSTOMER_LIST"),any(),any())).thenReturn(ApiResponse.success(batch)); + FileExportTask task=new FileExportTask();task.setId(9L);task.setTenantId(0L);task.setAppCode("PLATFORM");task.setRequestedBy(7L);task.setDispatchVersion(1);task.setExportType("CUSTOMER_LIST");task.setTaskName("客户列表");task.setParametersJson("{}"); + var definition=new ExportDefinitionRegistry.Definition();definition.setMaxRows(200000);definition.setBatchSize(500); + doThrow(new RuntimeException("size unavailable")).when(storage).size(anyString()); + + assertThrows(ExportFailure.class,() -> new GenericXlsxExportRunner(provider,storage,new ObjectMapper(),"internal-token",50) + .run(task,definition,new AtomicBoolean(),ignored->{ })); + + try(var paths=Files.walk(temp.resolve("exports"))) { + assertEquals(0,paths.filter(Files::isRegularFile).count()); + } + } +} diff --git a/file/src/test/java/com/baiyue/file/service/LocalQualificationStorageTest.java b/file/src/test/java/com/baiyue/file/service/LocalQualificationStorageTest.java index 939f90b43..587de55a6 100644 --- a/file/src/test/java/com/baiyue/file/service/LocalQualificationStorageTest.java +++ b/file/src/test/java/com/baiyue/file/service/LocalQualificationStorageTest.java @@ -64,6 +64,19 @@ class LocalQualificationStorageTest { storage.load(10001L, stored.fileName()).getFile().toPath().toAbsolutePath()); } + @Test + void rejectsQualificationFilesLargerThanTheClientFiveMegabyteLimit() { + var storage = new LocalQualificationStorage(storageRoot.toString()); + storage.initialize(); + var upload = new MockMultipartFile("file", "oversized.pdf", "application/pdf", + new byte[5 * 1024 * 1024 + 1]); + + BizException exception = assertThrows(BizException.class, + () -> storage.store(10001L, upload)); + + assertTrue(exception.getMessage().contains("5 MB")); + } + @Test void deletesTheReplacedQualificationByItsValidatedDocumentUrl() { var storage = new LocalQualificationStorage(storageRoot.toString()); diff --git a/file/src/test/java/com/baiyue/file/service/SizeLimitedOutputStreamTest.java b/file/src/test/java/com/baiyue/file/service/SizeLimitedOutputStreamTest.java new file mode 100644 index 000000000..f38e8f069 --- /dev/null +++ b/file/src/test/java/com/baiyue/file/service/SizeLimitedOutputStreamTest.java @@ -0,0 +1,19 @@ +package com.baiyue.file.service; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class SizeLimitedOutputStreamTest { + @Test void rejectsTheWriteBeforeAnyBytesBeyondTheLimitReachStorage() throws Exception { + ByteArrayOutputStream target=new ByteArrayOutputStream(); + SizeLimitedOutputStream output=new SizeLimitedOutputStream(target,4); + output.write(new byte[]{1,2,3,4}); + + assertThrows(SizeLimitedOutputStream.ExportSizeExceededException.class,() -> output.write(5)); + assertArrayEquals(new byte[]{1,2,3,4},target.toByteArray()); + } +} diff --git a/firstleg/pom.xml b/firstleg/pom.xml index 467108e07..6dfea9d5c 100644 --- a/firstleg/pom.xml +++ b/firstleg/pom.xml @@ -14,11 +14,9 @@ com.alibaba.cloudspring-cloud-starter-alibaba-nacos-discovery com.alibaba.cloudspring-cloud-starter-alibaba-nacos-config com.alibaba.cloudspring-cloud-starter-alibaba-sentinel - com.alibaba.cloudspring-cloud-starter-stream-rocketmq org.springframework.bootspring-boot-starter-actuator com.mysqlmysql-connector-jruntime org.springframework.bootspring-boot-starter-testtest org.springframework.bootspring-boot-maven-plugin - diff --git a/iam/AGENTS.md b/iam/AGENTS.md index 27b63cf41..daba57927 100644 --- a/iam/AGENTS.md +++ b/iam/AGENTS.md @@ -35,7 +35,7 @@ - 平台客户档案维护使用 `PlatformTenantController` 的列表、详情、新建、编辑、启停接口；详情和维护仅接受 `tenant_type=MERCHANT`。PENDING/REJECTED 只能由审核流程变更，不得由启停接口绕过。 - 平台代客户上传资质的公开 multipart 入口在 File 服务；IAM 的 `/internal/iam/platform-qualifications` 只登记元数据。当前仅 `PROFILE_REQUIRED/REJECTED` 可上传，同类型重复需显式 `replace=true`；替换时旧记录标记 REJECTED 并把旧 `documentUrl` 返回 File 清理。 - 平台资质登记在事务内通过 `ProfileResourceRepository.findQualificationsForUpdate` 锁定目标租户现有资质，避免并发上传或替换产生多个有效同类型记录；该查询不得改回普通无锁列表。 -- 企业送审和审核通过都要求 `BUSINESS_LICENSE` 与 `LEGAL_PERSON_ID`；审核由 `TenantService.review()` 同事务更新审核单、租户和所有 PENDING 资质，条件更新负责并发防重。 +- 企业送审和审核通过都要求 `BUSINESS_LICENSE`、`LEGAL_PERSON_ID`、`FREIGHT_FORWARDER_CERTIFICATE` 与 `BANK_ACCOUNT_LICENSE` 四类资质均为 `PENDING`；审核由 `TenantService.review()` 同事务更新审核单、租户和所有 PENDING 资质，行锁负责并发防重。 - 平台客户列表使用 `platform:customer:list` 并脱敏电话、邮箱；完整详情、平台地址和平台资质只读查询统一使用独立的 `platform:customer:view`，不得复用 `platform:customer:review`。 - 权限资源的 code/type 不可变；只有 `built_in=0` 的自定义资源可编辑、启停或删除，删除前必须无子节点且无角色绑定。 @@ -47,7 +47,7 @@ - 平台内部用户的公开主键是 `membershipId`；列表必须回显 `roleIds`，密码只允许通过 DTO 转交 Auth，禁止进入 IAM 实体、日志或审计快照。 - 平台内部用户角色分配使用 `PUT /api/platform/v1/iam/internal-users/{membershipId}/roles`，仅允许平台租户下已启用的平台角色；空 `roleIds` 表示清空。 - 商户成员新增、编辑采用字段级授权：基础资料分别校验 `merchant:member:create`、`merchant:member:edit`；创建时非空角色、编辑时角色集合发生任何变化（含空数组清空）还必须校验 `merchant:member:grant-role`。详情和列表实时回显当前 `roleIds`，比较和写入均限定当前租户。 -- 企业 `ACTIVE` 只表示平台审核已通过；下单资格还必须至少存在一份 `APPROVED` 资质，因此允许“已激活但暂不可下单”的中间业务状态。 +- 企业 `ACTIVE` 只表示平台审核已通过；下单资格还必须四类必填资质全部为 `APPROVED`。任意一类缺失、驳回、已替换或过期都必须由服务端阻止提交订单。 - SQL 增量保持可重复部署顺序，修改 Entity 时同步检查 Mapper、Repository、DTO 与种子数据。`phase1-platform-master-data-and-permission-management.sql` 必须在 `phase1-admin-rbac.sql` 和 `rbac-role-management-v2.sql` 之后执行；它标记既有系统资源并补齐平台客户、快递公司、往来单位与权限资源管理权限。 - 存量库新增一期客户档案字段时执行 `phase1-customer-profile.sql`；该脚本只扩展 `iam_tenant`，可重复执行，不得把运单数、财务金额、套餐计费、代客登录或物理删除能力并入 IAM。 - `deploy/scripts/initialize-test-databases.sh` 的 IAM 客户档案和 Warehouse 主数据脚本顺序必须与当前文件保持一致；测试环境正式增量仍以 `../deploy/sql/README.md` 指定的统一迁移为准。 diff --git a/iam/README-IAM.md b/iam/README-IAM.md index 5c3b06383..a85df6285 100644 --- a/iam/README-IAM.md +++ b/iam/README-IAM.md @@ -31,6 +31,14 @@ Merge the packages under `src/main/java` into `D:/baiyue/iam/src/main/java`; exe | GET | `/api/{app}/v1/iam/authorization/permissions` | PLATFORM/MERCHANT | Current user's active permission codes / 当前用户有效权限编码 | | GET/POST | `/api/{app}/v1/iam/permissions` | PLATFORM/MERCHANT | Permission tree / 权限树 | +## 审核通过初始化扩展 + +企业审核通过后的默认数据统一通过 `TenantApprovalInitializer` 扩展链初始化，`TenantService` 不直接依赖具体初始化事项。客户端自主注册时，注册账号已经创建 OWNER 成员并绑定 `MERCHANT_OWNER`，审核通过后继续沿用该账号；只有 Admin 平台代建且没有 OWNER 的企业，才由 `TenantAdministratorProvisioningService` 创建管理员账号。后续规则、费用方案或其他默认角色应各自新增初始化器，并通过 `@Order` 声明依赖顺序。 + +初始化器及其补偿必须幂等。任一初始化器失败时，协调器按完成顺序的逆序执行补偿；IAM 本地事务最终回滚时也会触发同一补偿链。初始化器名称必须非空且全局唯一，初始密码等敏感结果只能通过审核响应一次性返回，不得持久化到 IAM 或写入日志。 + +管理员初始化会验证既有 OWNER 成员、IAM 用户及 Auth 账号均处于可用状态且租户、成员和应用范围一致。历史数据只有 OWNER/IAM 身份而缺少 Auth 账号时，复用原身份补建认证账号；停用或范围冲突的数据会阻止审核通过，避免激活后没有可登录管理员。部署前必须执行 `phase1-onboarding.sql`；测试环境统一执行 `deploy/sql/2026-07-31一期回归测试环境增量迁移.sql`，确保全局 `MERCHANT_OWNER` 模板及权限完整。 + OpenAPI is synchronized in `openapi/iam-v1.yaml`. ## 商户成员角色字段级授权 diff --git a/iam/src/main/java/com/baiyue/iam/controller/CustomerListExportProviderController.java b/iam/src/main/java/com/baiyue/iam/controller/CustomerListExportProviderController.java new file mode 100644 index 000000000..8e7eec071 --- /dev/null +++ b/iam/src/main/java/com/baiyue/iam/controller/CustomerListExportProviderController.java @@ -0,0 +1,47 @@ +package com.baiyue.iam.controller; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import com.baiyue.iam.service.CustomerListExportProviderService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +/** Internal provider endpoint consumed only by File Service. */ +@RestController +@RequestMapping("/internal/exports/v1/CUSTOMER_LIST") +public class CustomerListExportProviderController { + private final CustomerListExportProviderService service; + private final byte[] internalToken; + + public CustomerListExportProviderController( + CustomerListExportProviderService service, + @Value("${baiyue.security.internal-service-token}") String internalToken) { + this.service = service; + this.internalToken = internalToken.getBytes(StandardCharsets.UTF_8); + } + + @PostMapping("/batches") + public ApiResponse batch( + @RequestHeader(value = "X-Internal-Service-Token", required = false) String token, + @RequestBody ExportBatchRequest request) { + verifyToken(token); + return ApiResponse.success(service.exportBatch(request)); + } + + private void verifyToken(String token) { + byte[] supplied = token == null ? new byte[0] : token.getBytes(StandardCharsets.UTF_8); + if (!MessageDigest.isEqual(internalToken, supplied)) { + throw new BizException(ResultCode.FORBIDDEN, "Invalid internal service token"); + } + } +} diff --git a/iam/src/main/java/com/baiyue/iam/controller/InternalTenantController.java b/iam/src/main/java/com/baiyue/iam/controller/InternalTenantController.java index 823c8d86b..923ec2ded 100644 --- a/iam/src/main/java/com/baiyue/iam/controller/InternalTenantController.java +++ b/iam/src/main/java/com/baiyue/iam/controller/InternalTenantController.java @@ -48,6 +48,15 @@ public class InternalTenantController { return ApiResponse.success(onboardingService.register(request)); } + @DeleteMapping("/merchant-registrations/{authUserId}/compensation") + public ApiResponse compensateRegistration( + @RequestHeader(value = "X-Internal-Service-Token", required = false) String token, + @PathVariable long authUserId) { + verify(token); + onboardingService.compensateRegistration(authUserId); + return ApiResponse.success(null); + } + @PostMapping("/qualifications") public ApiResponse submitQualification( @RequestHeader(value = "X-Internal-Service-Token", required = false) String token, diff --git a/iam/src/main/java/com/baiyue/iam/controller/PlatformTenantController.java b/iam/src/main/java/com/baiyue/iam/controller/PlatformTenantController.java index 18aeb1712..1999e1ad6 100644 --- a/iam/src/main/java/com/baiyue/iam/controller/PlatformTenantController.java +++ b/iam/src/main/java/com/baiyue/iam/controller/PlatformTenantController.java @@ -8,6 +8,7 @@ import com.baiyue.iam.dto.TenantDtos.ReviewRequest; import com.baiyue.iam.dto.TenantDtos.TenantResponse; import com.baiyue.iam.dto.TenantDtos.UpdateTenantRequest; import com.baiyue.iam.dto.TenantDtos.ChangeTenantStatusRequest; +import com.baiyue.iam.dto.TenantDtos.ApprovalResponse; import com.baiyue.iam.entity.IamTenantReview; import com.baiyue.iam.service.TenantGuard; import com.baiyue.iam.service.TenantService; @@ -80,7 +81,7 @@ public class PlatformTenantController { public ApiResponse register(@Valid @RequestBody RegisterRequest request, @AuthenticationPrincipal AuthenticatedUser user) { TenantGuard.tenant(user, "PLATFORM"); - return ApiResponse.success(service.register(request)); + return ApiResponse.success(service.register(request, user.userId())); } @PutMapping("/tenants/{tenantId:\\d+}") @@ -126,18 +127,17 @@ public class PlatformTenantController { @PostMapping("/reviews/{id}/approve") @PreAuthorize("hasAuthority('platform:customer:approve')") - @Operation(summary = "Approve enterprise registration / 审核通过企业注册", description = "Approve a pending review and activate the tenant. The operation is audited. / 审核通过并激活租户，操作写入审计日志。") + @Operation(summary = "Approve enterprise registration / 审核通过企业注册", description = "Approve and activate the tenant. Client registrations keep their registration account as owner; only platform-created tenants without an owner receive a generated administrator and one-time credentials. / 审核通过并激活租户；客户端注册账号继续作为企业管理员，仅平台代建且没有所有者的企业自动开户并返回一次性凭据。") @ApiResponses({ @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "200", description = "Approved / 审核通过"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "404", description = "Review not found / 审核单不存在"), @io.swagger.v3.oas.annotations.responses.ApiResponse(responseCode = "409", description = "Review already completed / 审核已完成") }) - public ApiResponse approve(@Parameter(description = "Review ticket ID / 审核单 ID", required = true, example = "190000000000000001") @PathVariable long id, + public ApiResponse approve(@Parameter(description = "Review ticket ID / 审核单 ID", required = true, example = "190000000000000001") @PathVariable long id, @RequestBody ReviewRequest request, @AuthenticationPrincipal AuthenticatedUser user) { TenantGuard.tenant(user, "PLATFORM"); - service.review(id, user.userId(), true, request.comment()); - return ApiResponse.success(null); + return ApiResponse.success(service.review(id, user.userId(), true, request.comment())); } @PostMapping("/reviews/{id}/reject") diff --git a/iam/src/main/java/com/baiyue/iam/dto/TenantDtos.java b/iam/src/main/java/com/baiyue/iam/dto/TenantDtos.java index 796bd6c5a..f1303e72e 100644 --- a/iam/src/main/java/com/baiyue/iam/dto/TenantDtos.java +++ b/iam/src/main/java/com/baiyue/iam/dto/TenantDtos.java @@ -25,6 +25,7 @@ public final class TenantDtos { public record RegisterRequest( @Schema(description = "Enterprise display name / 企业名称", example = "深圳虫虫出海科技有限公司") @NotBlank String tenantName, @Schema(description = "Legal entity name / 企业法定名称", example = "深圳虫虫出海科技有限公司") @NotBlank String legalName, + @Schema(description = "Legal representative / 法定代表人", example = "张三") @NotBlank @Size(max = 64) String legalRepresentative, @Schema(description = "18-character unified social credit code / 18 位统一社会信用代码", example = "91440300MA5EXAMPLE") @NotBlank @Pattern(regexp = "[0-9A-Z]{18}") String unifiedSocialCreditCode, @Schema(description = "Primary contact / 联系人", example = "张三") @NotBlank String contactName, @Schema(description = "Contact phone / 联系电话", example = "13800138000") @NotBlank String contactPhone, @@ -45,6 +46,7 @@ public final class TenantDtos { public record UpdateProfileRequest( @Schema(description = "Enterprise display name / 企业名称", example = "深圳虫虫出海科技有限公司") @NotBlank String tenantName, @Schema(description = "Legal entity name / 企业法定名称", example = "深圳虫虫出海科技有限公司") @NotBlank String legalName, + @Schema(description = "Legal representative / 法定代表人", example = "张三") @NotBlank @Size(max = 64) String legalRepresentative, @Schema(description = "Primary contact / 联系人", example = "李四") @NotBlank String contactName, @Schema(description = "Contact phone / 联系电话", example = "13900139000") @NotBlank String contactPhone, @Schema(description = "Contact email / 联系邮箱", example = "service@example.com") @Email String contactEmail @@ -55,6 +57,7 @@ public final class TenantDtos { public record UpdateTenantRequest( @NotBlank String tenantName, @NotBlank String legalName, + @NotBlank @Size(max = 64) String legalRepresentative, @NotBlank String contactName, @NotBlank String contactPhone, @Email String contactEmail, @@ -80,9 +83,17 @@ public final class TenantDtos { @Schema(description = "Approval or rejection comment / 审核意见", example = "资料完整，审核通过") String comment ) { } + @Schema(name = "TenantApprovalResponse", description = "Enterprise approval result and one-time administrator credentials / 企业审核结果及一次性管理员凭据") + public record ApprovalResponse( + @Schema(description = "Tenant ID / 企业租户 ID", example = "190000000000000001") Long tenantId, + @Schema(description = "Whether this approval created the administrator / 本次审核是否新建管理员") boolean administratorCreated, + @Schema(description = "Generated login username; null when an owner already exists / 自动生成的登录账号，已有所有者时为空", nullable = true) String administratorUsername, + @Schema(description = "One-time temporary password; only returned when created / 一次性初始密码，仅新建时返回", format = "password", nullable = true) String temporaryPassword + ) { } + @Schema(name = "TenantResponse", description = "Enterprise tenant summary / 企业租户概要") public record TenantResponse(Long id, String tenantCode, String tenantName, String status, - String legalName, String unifiedSocialCreditCode, String contactName, + String legalName, String legalRepresentative, String unifiedSocialCreditCode, String contactName, String contactPhone, String contactEmail, String contactWechat, String servicePackage, String salesOwnerName, Integer paymentTermDays, String enterpriseType, LocalDate establishedDate, BigDecimal registeredCapital, diff --git a/iam/src/main/java/com/baiyue/iam/entity/IamTenant.java b/iam/src/main/java/com/baiyue/iam/entity/IamTenant.java index 95d0c2172..1761b7a81 100644 --- a/iam/src/main/java/com/baiyue/iam/entity/IamTenant.java +++ b/iam/src/main/java/com/baiyue/iam/entity/IamTenant.java @@ -23,6 +23,7 @@ public class IamTenant extends BaseEntity { private String tenantType; private String status; private String legalName; + private String legalRepresentative; private String unifiedSocialCreditCode; private String contactName; private String contactPhone; diff --git a/iam/src/main/java/com/baiyue/iam/repository/CustomerListExportProviderRepository.java b/iam/src/main/java/com/baiyue/iam/repository/CustomerListExportProviderRepository.java new file mode 100644 index 000000000..aaceaabb4 --- /dev/null +++ b/iam/src/main/java/com/baiyue/iam/repository/CustomerListExportProviderRepository.java @@ -0,0 +1,60 @@ +package com.baiyue.iam.repository; + +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.mapper.IamTenantMapper; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.springframework.stereotype.Repository; +import org.springframework.util.StringUtils; + +import java.util.List; + +/** Read-only customer queries used by the generic export provider. */ +@Repository +public class CustomerListExportProviderRepository { + private final IamTenantMapper tenantMapper; + + public CustomerListExportProviderRepository(IamTenantMapper tenantMapper) { + this.tenantMapper = tenantMapper; + } + + public long maximumId(String keyword, String status) { + LambdaQueryWrapper wrapper = filters(keyword, status) + .select(IamTenant::getId) + .orderByDesc(IamTenant::getId) + .last("LIMIT 1"); + IamTenant tenant = tenantMapper.selectOne(wrapper); + return tenant == null || tenant.getId() == null ? 0L : tenant.getId(); + } + + public long estimateRows(long maximumId, String keyword, String status) { + if (maximumId <= 0) return 0L; + return tenantMapper.selectCount(filters(keyword, status).le(IamTenant::getId, maximumId)); + } + + public List findBatch(Long cursor, long maximumId, int fetchSize, + String keyword, String status) { + if (maximumId <= 0) return List.of(); + return tenantMapper.selectList(filters(keyword, status) + .gt(cursor != null, IamTenant::getId, cursor) + .le(IamTenant::getId, maximumId) + .orderByAsc(IamTenant::getId) + .last("LIMIT " + fetchSize)); + } + + private LambdaQueryWrapper filters(String keyword, String status) { + LambdaQueryWrapper wrapper = new LambdaQueryWrapper() + .eq(IamTenant::getTenantType, "MERCHANT") + .eq(StringUtils.hasText(status), IamTenant::getStatus, status); + if (StringUtils.hasText(keyword)) { + wrapper.and(condition -> condition + .like(IamTenant::getTenantCode, keyword) + .or().like(IamTenant::getTenantName, keyword) + .or().like(IamTenant::getLegalName, keyword) + .or().like(IamTenant::getUnifiedSocialCreditCode, keyword) + .or().like(IamTenant::getContactName, keyword) + .or().like(IamTenant::getContactPhone, keyword) + .or().like(IamTenant::getSalesOwnerName, keyword)); + } + return wrapper; + } +} diff --git a/iam/src/main/java/com/baiyue/iam/repository/MemberRepository.java b/iam/src/main/java/com/baiyue/iam/repository/MemberRepository.java index 8a9eec32f..577a38d27 100644 --- a/iam/src/main/java/com/baiyue/iam/repository/MemberRepository.java +++ b/iam/src/main/java/com/baiyue/iam/repository/MemberRepository.java @@ -39,6 +39,12 @@ public class MemberRepository { return membershipMapper.selectOne(new LambdaQueryWrapper() .eq(IamMembership::getTenantId, tenantId).eq(IamMembership::getId, id).last("LIMIT 1")); } + public IamMembership findOwnerByTenant(long tenantId) { + return membershipMapper.selectOne(new LambdaQueryWrapper() + .eq(IamMembership::getTenantId, tenantId) + .eq(IamMembership::getMemberType, "OWNER") + .last("LIMIT 1")); + } public IamMembership findByTenantAndAuthUser(long tenantId, Long authUserId) { IamUser user = findUserByAuthId(authUserId); if (user == null) return null; diff --git a/iam/src/main/java/com/baiyue/iam/repository/TenantRepository.java b/iam/src/main/java/com/baiyue/iam/repository/TenantRepository.java index 6633a2d1f..c02d0a010 100644 --- a/iam/src/main/java/com/baiyue/iam/repository/TenantRepository.java +++ b/iam/src/main/java/com/baiyue/iam/repository/TenantRepository.java @@ -30,6 +30,12 @@ public class TenantRepository { .eq(IamTenant::getUnifiedSocialCreditCode, creditCode)) > 0; } + public boolean existsByCreditCodeForOtherTenant(String creditCode, long tenantId) { + return tenantMapper.selectCount(new LambdaQueryWrapper() + .eq(IamTenant::getUnifiedSocialCreditCode, creditCode) + .ne(IamTenant::getId, tenantId)) > 0; + } + public IamTenant findTenant(Long tenantId) { return tenantMapper.selectById(tenantId); } /** * Serializes the draft qualification workflow for one enterprise. The file service and the @@ -47,6 +53,11 @@ public class TenantRepository { .orderByAsc(IamTenant::getTenantName)); } public IamTenantReview findReview(Long reviewId) { return reviewMapper.selectById(reviewId); } + public IamTenantReview findReviewForUpdate(Long reviewId) { + return reviewMapper.selectOne(new LambdaQueryWrapper() + .eq(IamTenantReview::getId, reviewId) + .last("FOR UPDATE")); + } public IamTenantReview findPendingReviewByTenant(Long tenantId) { return reviewMapper.selectOne(new LambdaQueryWrapper() .eq(IamTenantReview::getTenantId, tenantId) diff --git a/iam/src/main/java/com/baiyue/iam/service/CustomerListExportProviderService.java b/iam/src/main/java/com/baiyue/iam/service/CustomerListExportProviderService.java new file mode 100644 index 000000000..1589488e2 --- /dev/null +++ b/iam/src/main/java/com/baiyue/iam/service/CustomerListExportProviderService.java @@ -0,0 +1,265 @@ +package com.baiyue.iam.service; + +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import com.baiyue.common.core.export.ExportContracts.ExportColumn; +import com.baiyue.common.core.export.ExportContracts.ExportColumnType; +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.repository.CustomerListExportProviderRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Implements the generic {@code CUSTOMER_LIST} export contract without writing export files. */ +@Service +public class CustomerListExportProviderService { + static final int SCHEMA_VERSION = 1; + private static final int MAX_BATCH_SIZE = 500; + private static final int MAX_SNAPSHOT_TOKEN_LENGTH = 512; + private static final Set STATUSES = Set.of( + "PROFILE_REQUIRED", "PENDING", "ACTIVE", "REJECTED", "DISABLED"); + private static final Set PARAMETER_KEYS = Set.of("keyword", "status"); + private static final List COLUMNS = List.of( + column("id", "客户ID", ExportColumnType.STRING, 20), + column("tenantCode", "客户编码", ExportColumnType.STRING, 18), + column("tenantName", "公司名称", ExportColumnType.STRING, 28), + column("legalName", "营业执照企业名称", ExportColumnType.STRING, 28), + column("unifiedSocialCreditCode", "统一社会信用代码", ExportColumnType.STRING, 24), + column("enterpriseType", "企业类型", ExportColumnType.STRING, 16), + column("establishedDate", "成立日期", ExportColumnType.DATE, 14), + column("registeredCapital", "注册资本", ExportColumnType.DECIMAL, 16), + column("registeredCapitalCurrency", "币种", ExportColumnType.STRING, 10), + column("registeredAddress", "注册地址", ExportColumnType.STRING, 32), + column("contactName", "联系人", ExportColumnType.STRING, 16), + column("contactPhone", "联系电话", ExportColumnType.STRING, 18), + column("contactEmail", "联系邮箱", ExportColumnType.STRING, 24), + column("contactWechat", "联系微信", ExportColumnType.STRING, 18), + column("salesOwnerName", "销售人员", ExportColumnType.STRING, 16), + column("servicePackage", "客户套餐", ExportColumnType.STRING, 16), + column("paymentTermDays", "账期(天)", ExportColumnType.INTEGER, 12), + column("status", "状态", ExportColumnType.STRING, 14), + column("createdAt", "入驻时间", ExportColumnType.DATETIME, 21), + column("approvedAt", "审批时间", ExportColumnType.DATETIME, 21)); + private static final String SCHEMA_HASH = schemaHash(COLUMNS); + + private final CustomerListExportProviderRepository repository; + private final byte[] snapshotSigningKey; + + public CustomerListExportProviderService( + CustomerListExportProviderRepository repository, + @Value("${baiyue.security.internal-service-token}") String internalServiceToken) { + this.repository = repository; + this.snapshotSigningKey = internalServiceToken.getBytes(StandardCharsets.UTF_8); + } + + public ExportBatchResponse exportBatch(ExportBatchRequest request) { + ValidatedRequest validated = validate(request); + boolean firstBatch = validated.snapshotToken() == null; + long maximumId = firstBatch + ? repository.maximumId(validated.keyword(), validated.status()) + : verifySnapshot(validated); + String snapshotToken = firstBatch ? signSnapshot(validated, maximumId) : validated.snapshotToken(); + long totalRows = firstBatch + ? repository.estimateRows(maximumId, validated.keyword(), validated.status()) + : 0L; + + List source = repository.findBatch(validated.cursor(), maximumId, + validated.limit() + 1, validated.keyword(), validated.status()); + boolean hasMore = source.size() > validated.limit(); + List page = hasMore ? source.subList(0, validated.limit()) : source; + String nextCursor = page.isEmpty() + ? validated.cursorText() + : String.valueOf(page.get(page.size() - 1).getId()); + + return new ExportBatchResponse(SCHEMA_VERSION, SCHEMA_HASH, snapshotToken, + firstBatch ? COLUMNS : List.of(), page.stream().map(this::row).toList(), + nextCursor, hasMore, firstBatch ? totalRows : null); + } + + private ValidatedRequest validate(ExportBatchRequest request) { + if (request == null) throw invalid("Export batch request is required"); + requirePositiveId(request.taskId(), "taskId"); + if (!"0".equals(request.tenantId())) { + throw invalid("CUSTOMER_LIST only supports platform tenant 0"); + } + if (!"PLATFORM".equals(request.appCode())) { + throw invalid("CUSTOMER_LIST only supports PLATFORM appCode"); + } + requirePositiveId(request.requestedBy(), "requestedBy"); + if (request.dispatchVersion() <= 0) throw invalid("dispatchVersion must be positive"); + if (request.limit() < 1 || request.limit() > MAX_BATCH_SIZE) { + throw invalid("limit must be between 1 and 500"); + } + + Map parameters = request.parameters() == null ? Map.of() : request.parameters(); + if (!PARAMETER_KEYS.containsAll(parameters.keySet())) { + throw invalid("CUSTOMER_LIST contains unsupported parameters"); + } + String keyword = optionalString(parameters.get("keyword"), "keyword"); + if (keyword != null && keyword.length() > 128) throw invalid("keyword must not exceed 128 characters"); + String status = optionalString(parameters.get("status"), "status"); + if (status != null && !STATUSES.contains(status)) throw invalid("CUSTOMER_LIST status is invalid"); + + String snapshotToken = normalize(request.snapshotToken()); + if (snapshotToken != null && snapshotToken.length() > MAX_SNAPSHOT_TOKEN_LENGTH) { + throw invalid("snapshotToken must not exceed 512 characters"); + } + Long cursor = parseCursor(request.cursor()); + if (snapshotToken == null && cursor != null) { + throw invalid("cursor requires a snapshotToken"); + } + return new ValidatedRequest(request, keyword, status, cursor, normalize(request.cursor()), snapshotToken); + } + + private long verifySnapshot(ValidatedRequest request) { + String[] parts = request.snapshotToken().split("\\.", -1); + if (parts.length != 3 || !"v1".equals(parts[0])) throw invalid("snapshotToken is invalid"); + long maximumId; + byte[] suppliedSignature; + try { + maximumId = Long.parseLong(parts[1]); + suppliedSignature = Base64.getUrlDecoder().decode(parts[2]); + } catch (IllegalArgumentException exception) { + throw invalid("snapshotToken is invalid"); + } + if (maximumId < 0) throw invalid("snapshotToken is invalid"); + byte[] expectedSignature = hmac(snapshotSubject(request, maximumId)); + if (!MessageDigest.isEqual(expectedSignature, suppliedSignature)) { + throw invalid("snapshotToken does not match this export task"); + } + if (request.cursor() != null && request.cursor() > maximumId) { + throw invalid("cursor exceeds the export snapshot"); + } + return maximumId; + } + + private String signSnapshot(ValidatedRequest request, long maximumId) { + String signature = Base64.getUrlEncoder().withoutPadding() + .encodeToString(hmac(snapshotSubject(request, maximumId))); + return "v1." + maximumId + "." + signature; + } + + private String snapshotSubject(ValidatedRequest request, long maximumId) { + ExportBatchRequest source = request.source(); + return String.join("\n", "CUSTOMER_LIST", source.taskId(), source.tenantId(), source.appCode(), + source.requestedBy(), String.valueOf(source.dispatchVersion()), + request.keyword() == null ? "" : request.keyword(), + request.status() == null ? "" : request.status(), String.valueOf(maximumId)); + } + + private byte[] hmac(String value) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(snapshotSigningKey, "HmacSHA256")); + return mac.doFinal(value.getBytes(StandardCharsets.UTF_8)); + } catch (GeneralSecurityException exception) { + throw new IllegalStateException("HmacSHA256 is unavailable", exception); + } + } + + private Map row(IamTenant tenant) { + Map row = new LinkedHashMap<>(); + row.put("id", String.valueOf(tenant.getId())); + row.put("tenantCode", tenant.getTenantCode()); + row.put("tenantName", tenant.getTenantName()); + row.put("legalName", tenant.getLegalName()); + row.put("unifiedSocialCreditCode", tenant.getUnifiedSocialCreditCode()); + row.put("enterpriseType", tenant.getEnterpriseType()); + row.put("establishedDate", tenant.getEstablishedDate()); + row.put("registeredCapital", tenant.getRegisteredCapital()); + row.put("registeredCapitalCurrency", tenant.getRegisteredCapitalCurrency()); + row.put("registeredAddress", tenant.getRegisteredAddress()); + row.put("contactName", tenant.getContactName()); + row.put("contactPhone", tenant.getContactPhone()); + row.put("contactEmail", tenant.getContactEmail()); + row.put("contactWechat", tenant.getContactWechat()); + row.put("salesOwnerName", tenant.getSalesOwnerName()); + row.put("servicePackage", tenant.getServicePackage()); + row.put("paymentTermDays", tenant.getPaymentTermDays()); + row.put("status", statusLabel(tenant.getStatus())); + row.put("createdAt", tenant.getCreatedAt()); + row.put("approvedAt", tenant.getApprovedAt()); + return row; + } + + private String statusLabel(String status) { + if (status == null) return null; + return switch (status) { + case "PROFILE_REQUIRED" -> "待完善"; + case "PENDING" -> "待审核"; + case "ACTIVE" -> "启用"; + case "REJECTED" -> "已驳回"; + case "DISABLED" -> "已停用"; + default -> status; + }; + } + + private static ExportColumn column(String key, String title, ExportColumnType type, int width) { + return new ExportColumn(key, title, type, width); + } + + private static String schemaHash(List columns) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for (ExportColumn column : columns) { + digest.update((column.key() + "\u001f" + column.title() + "\u001f" + column.type() + + "\u001f" + column.width() + "\n").getBytes(StandardCharsets.UTF_8)); + } + return "sha256:" + java.util.HexFormat.of().formatHex(digest.digest()); + } catch (GeneralSecurityException exception) { + throw new ExceptionInInitializerError(exception); + } + } + + private Long parseCursor(String value) { + String normalized = normalize(value); + if (normalized == null) return null; + try { + long parsed = Long.parseLong(normalized); + if (parsed < 0) throw invalid("cursor must not be negative"); + return parsed; + } catch (NumberFormatException exception) { + throw invalid("cursor must be a numeric ID"); + } + } + + private void requirePositiveId(String value, String field) { + try { + if (value == null || Long.parseLong(value) <= 0) throw invalid(field + " must be a positive ID"); + } catch (NumberFormatException exception) { + throw invalid(field + " must be a positive ID"); + } + } + + private String optionalString(Object value, String field) { + if (value == null) return null; + if (!(value instanceof String text)) throw invalid(field + " must be a string"); + return normalize(text); + } + + private String normalize(String value) { + if (value == null || value.isBlank()) return null; + return value.trim(); + } + + private BizException invalid(String message) { + return new BizException(ResultCode.VALIDATION_FAILED, message); + } + + private record ValidatedRequest(ExportBatchRequest source, String keyword, String status, + Long cursor, String cursorText, String snapshotToken) { + int limit() { return source.limit(); } + } +} diff --git a/iam/src/main/java/com/baiyue/iam/service/MerchantOnboardingService.java b/iam/src/main/java/com/baiyue/iam/service/MerchantOnboardingService.java index 7cf003a1a..59e78e689 100644 --- a/iam/src/main/java/com/baiyue/iam/service/MerchantOnboardingService.java +++ b/iam/src/main/java/com/baiyue/iam/service/MerchantOnboardingService.java @@ -9,12 +9,16 @@ import com.baiyue.iam.entity.IamMembershipRole; import com.baiyue.iam.entity.IamRole; import com.baiyue.iam.entity.IamTenant; import com.baiyue.iam.entity.IamTenantAddress; +import com.baiyue.iam.entity.IamTenantQualification; +import com.baiyue.iam.entity.IamTenantReview; import com.baiyue.iam.entity.IamUser; import com.baiyue.iam.mapper.IamMembershipMapper; import com.baiyue.iam.mapper.IamMembershipRoleMapper; import com.baiyue.iam.mapper.IamRoleMapper; import com.baiyue.iam.mapper.IamTenantAddressMapper; import com.baiyue.iam.mapper.IamTenantMapper; +import com.baiyue.iam.mapper.IamTenantQualificationMapper; +import com.baiyue.iam.mapper.IamTenantReviewMapper; import com.baiyue.iam.mapper.IamUserMapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.toolkit.IdWorker; @@ -24,6 +28,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; import java.time.LocalDateTime; +import java.util.List; @Service public class MerchantOnboardingService { @@ -33,6 +38,8 @@ public class MerchantOnboardingService { @Resource private IamRoleMapper roleMapper; @Resource private IamMembershipRoleMapper membershipRoleMapper; @Resource private IamTenantAddressMapper addressMapper; + @Resource private IamTenantQualificationMapper qualificationMapper; + @Resource private IamTenantReviewMapper reviewMapper; @Transactional public MerchantRegistrationResponse register(MerchantRegistrationRequest request) { @@ -61,7 +68,8 @@ public class MerchantOnboardingService { tenant.setTenantType("MERCHANT"); tenant.setStatus("PROFILE_REQUIRED"); tenant.setLegalName(request.companyName()); - tenant.setContactName(request.legalPerson()); + tenant.setLegalRepresentative(request.legalPerson()); + tenant.setContactName(request.displayName()); tenant.setContactPhone(request.legalPhone()); tenant.setContactEmail(request.contactEmail()); tenant.setDeleted(false); @@ -112,4 +120,51 @@ public class MerchantOnboardingService { } return new MerchantRegistrationResponse(tenantId, membershipId, tenant.getStatus()); } + + /** + * Removes a just-created IAM registration when Auth could not persist the paired login account. + * Repeated calls are safe, while progressed onboarding data is deliberately protected. + */ + @Transactional + public void compensateRegistration(long authUserId) { + IamUser user = userMapper.selectOne(new LambdaQueryWrapper() + .eq(IamUser::getAuthUserId, authUserId) + .last("LIMIT 1 FOR UPDATE")); + if (user == null) { + return; + } + List memberships = membershipMapper.selectList(new LambdaQueryWrapper() + .eq(IamMembership::getUserId, user.getId()) + .orderByAsc(IamMembership::getId) + .last("FOR UPDATE")); + if (memberships.size() != 1) { + throw new BizException(ResultCode.CONFLICT, + "IAM registration compensation is unsafe: membership state changed"); + } + IamMembership membership = memberships.get(0); + IamTenant tenant = tenantMapper.selectOne(new LambdaQueryWrapper() + .eq(IamTenant::getId, membership.getTenantId()) + .last("FOR UPDATE")); + if (tenant == null || !"PROFILE_REQUIRED".equals(tenant.getStatus()) + || !"OWNER".equals(membership.getMemberType())) { + throw new BizException(ResultCode.CONFLICT, + "IAM registration compensation is unsafe: tenant state changed"); + } + long qualificationCount = qualificationMapper.selectCount(new LambdaQueryWrapper() + .eq(IamTenantQualification::getTenantId, tenant.getId())); + long reviewCount = reviewMapper.selectCount(new LambdaQueryWrapper() + .eq(IamTenantReview::getTenantId, tenant.getId())); + if (qualificationCount > 0 || reviewCount > 0) { + throw new BizException(ResultCode.CONFLICT, + "IAM registration compensation is unsafe: onboarding has progressed"); + } + + membershipRoleMapper.delete(new LambdaQueryWrapper() + .eq(IamMembershipRole::getMembershipId, membership.getId())); + membershipMapper.deleteById(membership.getId()); + addressMapper.delete(new LambdaQueryWrapper() + .eq(IamTenantAddress::getTenantId, tenant.getId())); + userMapper.deleteById(user.getId()); + tenantMapper.deleteById(tenant.getId()); + } } diff --git a/iam/src/main/java/com/baiyue/iam/service/ProfileResourceService.java b/iam/src/main/java/com/baiyue/iam/service/ProfileResourceService.java index 8a146106b..68d55d7f3 100644 --- a/iam/src/main/java/com/baiyue/iam/service/ProfileResourceService.java +++ b/iam/src/main/java/com/baiyue/iam/service/ProfileResourceService.java @@ -65,6 +65,7 @@ public class ProfileResourceService { throw new BizException(ResultCode.CONFLICT, "Qualifications can only be uploaded before review / 仅待完善或已驳回客户可上传资质"); } + String certificateNo = resolveCertificateNo(tenant, request.qualificationType(), request.certificateNo()); List activeQualifications = repository.findQualificationsForUpdate(tenant.getId()).stream() .filter(item -> request.qualificationType().equals(item.getQualificationType()) && isCurrentQualification(item)) @@ -82,12 +83,12 @@ public class ProfileResourceService { repository.saveQualification(existing); } IamTenantQualification qualification = createQualification(tenant.getId(), - request.qualificationType(), request.certificateNo(), request.documentUrl()); + request.qualificationType(), certificateNo, request.documentUrl()); qualification.setValidTo(request.validTo()); repository.saveQualification(qualification); - if (StringUtils.hasText(request.certificateNo()) - && request.certificateNo().trim().toUpperCase().matches("[0-9A-Z]{18}")) { - tenant.setUnifiedSocialCreditCode(request.certificateNo().trim().toUpperCase()); + if ("BUSINESS_LICENSE".equals(request.qualificationType()) + && !certificateNo.equals(tenant.getUnifiedSocialCreditCode())) { + tenant.setUnifiedSocialCreditCode(certificateNo); tenantRepository.saveTenant(tenant); } return new QualificationSubmissionResponse(qualification.getId(), null, tenant.getStatus(), replacedDocumentUrl); @@ -104,6 +105,7 @@ public class ProfileResourceService { if (!"PROFILE_REQUIRED".equals(tenant.getStatus()) && !"REJECTED".equals(tenant.getStatus())) { throw new BizException(ResultCode.CONFLICT, "Qualifications can only be uploaded before review / 仅待完善或已驳回客户可上传资质"); } + String certificateNo = resolveCertificateNo(tenant, request.qualificationType(), request.certificateNo()); List activeQualifications = repository.findQualificationsForUpdate(tenant.getId()).stream() .filter(item -> request.qualificationType().equals(item.getQualificationType()) && isCurrentQualification(item)) @@ -121,13 +123,36 @@ public class ProfileResourceService { repository.saveQualification(existing); } IamTenantQualification qualification = createQualification(tenant.getId(), request.qualificationType(), - request.certificateNo(), request.documentUrl()); + certificateNo, request.documentUrl()); qualification.setValidTo(request.validTo()); repository.saveQualification(qualification); + if ("BUSINESS_LICENSE".equals(request.qualificationType()) + && !certificateNo.equals(tenant.getUnifiedSocialCreditCode())) { + tenant.setUnifiedSocialCreditCode(certificateNo); + tenantRepository.saveTenant(tenant); + } return new PlatformQualificationRegistrationResponse( qualification.getId(), null, tenant.getStatus(), replacedDocumentUrl); } + private String resolveCertificateNo(IamTenant tenant, String qualificationType, String submittedCertificateNo) { + if (!"BUSINESS_LICENSE".equals(qualificationType)) { + return submittedCertificateNo; + } + String certificateNo = StringUtils.hasText(submittedCertificateNo) + ? submittedCertificateNo.trim().toUpperCase() + : tenant.getUnifiedSocialCreditCode(); + if (!StringUtils.hasText(certificateNo) || !certificateNo.matches("[0-9A-Z]{18}")) { + throw new BizException(ResultCode.BAD_REQUEST, + "A valid 18-character unified social credit code is required / 营业执照必须填写有效的18位统一社会信用代码"); + } + if (tenantRepository.existsByCreditCodeForOtherTenant(certificateNo, tenant.getId())) { + throw new BizException(ResultCode.CONFLICT, + "Unified social credit code already exists / 统一社会信用代码已存在"); + } + return certificateNo; + } + private IamTenantQualification createQualification(long tenant, String type, String certificateNo, String url) { IamTenantQualification x = new IamTenantQualification(); x.setId(IdWorker.getId()); x.setTenantId(tenant); x.setQualificationType(type); diff --git a/iam/src/main/java/com/baiyue/iam/service/TenantAdministratorProvisioningService.java b/iam/src/main/java/com/baiyue/iam/service/TenantAdministratorProvisioningService.java new file mode 100644 index 000000000..1dffe2433 --- /dev/null +++ b/iam/src/main/java/com/baiyue/iam/service/TenantAdministratorProvisioningService.java @@ -0,0 +1,295 @@ +package com.baiyue.iam.service; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import com.baiyue.iam.client.AuthAccountClient; +import com.baiyue.iam.entity.IamMembership; +import com.baiyue.iam.entity.IamRole; +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.entity.IamUser; +import com.baiyue.iam.mapper.IamRoleMapper; +import com.baiyue.iam.repository.MemberRepository; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.IdWorker; +import feign.FeignException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; + +import java.security.SecureRandom; +import java.time.LocalDateTime; +import java.util.List; + +/** Provisions the first enterprise owner while keeping credentials in Auth only. */ +@Service +@Order(100) +public class TenantAdministratorProvisioningService implements TenantApprovalInitializer { + private static final Logger log = LoggerFactory.getLogger(TenantAdministratorProvisioningService.class); + private static final char[] UPPER = "ABCDEFGHJKLMNPQRSTUVWXYZ".toCharArray(); + private static final char[] LOWER = "abcdefghijkmnopqrstuvwxyz".toCharArray(); + private static final char[] DIGITS = "23456789".toCharArray(); + private static final char[] ALL = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789".toCharArray(); + private static final int PASSWORD_LENGTH = 16; + private static final int USERNAME_SUFFIX_LENGTH = 8; + private static final int USERNAME_ATTEMPTS = 3; + + private final MemberRepository members; + private final IamRoleMapper roles; + private final AuthAccountClient authClient; + private final String internalToken; + private final SecureRandom random; + + public TenantAdministratorProvisioningService(MemberRepository members, IamRoleMapper roles, + AuthAccountClient authClient, + @Value("${baiyue.security.internal-service-token}") String internalToken) { + this(members, roles, authClient, internalToken, new SecureRandom()); + } + + TenantAdministratorProvisioningService(MemberRepository members, IamRoleMapper roles, + AuthAccountClient authClient, String internalToken, + SecureRandom random) { + this.members = members; + this.roles = roles; + this.authClient = authClient; + this.internalToken = internalToken; + this.random = random; + } + + @Override + public String name() { return "tenant-administrator"; } + + @Override + public ProvisioningResult initialize(ApprovalContext context) { + IamTenant tenant = context.tenant(); + long operatorId = context.reviewerId(); + IamRole ownerRole = requiredOwnerRole(); + IamMembership existingOwner = members.findOwnerByTenant(tenant.getId()); + if (existingOwner != null) return ensureExistingOwnerCanLogin(tenant, existingOwner, ownerRole); + if (!isPlatformCreatedTenant(tenant)) { + throw inconsistentOwner("客户端注册企业缺少原注册所有者，不能自动创建第二个管理员，请先修复注册身份"); + } + + long authUserId = IdWorker.getId(); + long userId = IdWorker.getId(); + long membershipId = IdWorker.getId(); + String password = temporaryPassword(); + String username = createAuthAccountWithUniqueUsername(authUserId, password, tenant, membershipId); + ProvisioningResult result = ProvisioningResult.created(authUserId, username, password); + + try { + IamUser user = new IamUser(); + user.setId(userId); + user.setAuthUserId(authUserId); + user.setUserType("ENTERPRISE"); + user.setDisplayName(tenant.getContactName()); + user.setPhone(tenant.getContactPhone()); + user.setEmail(tenant.getContactEmail()); + user.setStatus("ACTIVE"); + members.insertUser(user); + + IamMembership membership = new IamMembership(); + membership.setId(membershipId); + membership.setTenantId(tenant.getId()); + membership.setUserId(userId); + membership.setMemberType("OWNER"); + membership.setMemberNo("OWNER-" + membershipId); + membership.setStatus("ACTIVE"); + membership.setJoinedAt(LocalDateTime.now()); + membership.setCreatedBy(operatorId); + membership.setUpdatedBy(operatorId); + members.insert(membership); + members.replaceRoles(tenant.getId(), membershipId, List.of(ownerRole.getId())); + return result; + } catch (RuntimeException localFailure) { + compensate(context, result); + throw localFailure; + } + } + + private IamRole requiredOwnerRole() { + IamRole ownerRole = roles.selectOne(new LambdaQueryWrapper() + .eq(IamRole::getTenantId, 0L) + .eq(IamRole::getAppCode, "MERCHANT") + .eq(IamRole::getRoleCode, "MERCHANT_OWNER") + .eq(IamRole::getStatus, "ACTIVE") + .last("LIMIT 1")); + if (ownerRole == null) { + throw new BizException(ResultCode.SERVICE_UNAVAILABLE, "企业管理员角色模板未初始化，不能审核通过"); + } + return ownerRole; + } + + private boolean isPlatformCreatedTenant(IamTenant tenant) { + // 只有平台创建接口会记录操作人；客户端注册后也会补齐信用代码，不能据此推断来源。 + return tenant.getCreatedBy() != null; + } + + private ProvisioningResult ensureExistingOwnerCanLogin(IamTenant tenant, IamMembership owner, + IamRole ownerRole) { + if (!"ACTIVE".equals(owner.getStatus())) { + throw inconsistentOwner("企业所有者成员已停用，不能审核通过"); + } + IamUser user = members.findUser(owner.getUserId()); + if (user == null || user.getAuthUserId() == null || !"ACTIVE".equals(user.getStatus())) { + throw inconsistentOwner("企业所有者身份不完整或已停用，不能审核通过"); + } + + AuthAccountClient.AccountSummary account = findAuthAccount(user.getAuthUserId()); + if (account == null) { + String password = temporaryPassword(); + String username = createAuthAccountWithUniqueUsername(user.getAuthUserId(), password, + tenant, owner.getId()); + return ProvisioningResult.created(user.getAuthUserId(), username, password); + } + if (!"ACTIVE".equals(account.status()) + || !user.getAuthUserId().equals(account.accountId()) + || !tenant.getId().equals(account.tenantId()) + || !owner.getId().equals(account.membershipId()) + || !"MERCHANT".equals(account.appCode())) { + throw inconsistentOwner("企业所有者认证账号已停用或账号范围不一致，不能审核通过"); + } + List roleIds = members.findRoleIds(tenant.getId(), owner.getId()); + if (!roleIds.contains(ownerRole.getId())) { + List repairedRoleIds = new java.util.ArrayList<>(roleIds); + repairedRoleIds.add(ownerRole.getId()); + members.replaceRoles(tenant.getId(), owner.getId(), repairedRoleIds.stream().distinct().toList()); + } + return ProvisioningResult.existing(); + } + + private AuthAccountClient.AccountSummary findAuthAccount(long authUserId) { + try { + ApiResponse response = authClient.summary(internalToken, authUserId); + if (response == null) throw unavailable(); + if (!response.successful()) { + ResultCode code = ResultCode.of(response.code()); + if (code == ResultCode.NOT_FOUND) return null; + throw new BizException(code, response.message() == null || response.message().isBlank() + ? code.message() : response.message()); + } + if (response.data() == null) throw unavailable(); + return response.data(); + } catch (FeignException exception) { + if (exception.status() == 404) return null; + throw unavailable(); + } catch (BizException exception) { + throw exception; + } catch (RuntimeException exception) { + throw unavailable(); + } + } + + @Override + public void compensate(ApprovalContext context, InitializationResult initialization) { + ProvisioningResult result = (ProvisioningResult) initialization; + if (result.created()) compensate(result.authUserId()); + } + + private String createAuthAccountWithUniqueUsername(long authUserId, String password, + IamTenant tenant, long membershipId) { + for (int attempt = 0; attempt < USERNAME_ATTEMPTS; attempt++) { + String username = username(tenant.getId()); + try { + createAuthAccount(authUserId, username, password, tenant, membershipId); + return username; + } catch (BizException exception) { + if (exception.getResultCode() != ResultCode.CONFLICT || attempt == USERNAME_ATTEMPTS - 1) { + throw exception; + } + } + } + throw new IllegalStateException("Username generation attempts exhausted"); + } + + private void createAuthAccount(long authUserId, String username, String password, + IamTenant tenant, long membershipId) { + try { + ApiResponse response = authClient.create(internalToken, + new AuthAccountClient.CreateAccountRequest(authUserId, username, password, + tenant.getContactName(), tenant.getId(), membershipId, "MERCHANT", "ACTIVE")); + if (response == null) throw unavailable(); + if (!response.successful()) { + ResultCode code = ResultCode.of(response.code()); + throw new BizException(code, response.message() == null || response.message().isBlank() + ? code.message() : response.message()); + } + if (response.data() == null) throw unavailable(); + } catch (FeignException exception) { + if (exception.status() == 409) { + throw new BizException(ResultCode.CONFLICT, "自动生成的企业管理员登录账号已存在"); + } + throw unavailable(); + } catch (BizException exception) { + throw exception; + } catch (RuntimeException exception) { + throw unavailable(); + } + } + + private void compensate(long authUserId) { + try { + ApiResponse response = authClient.compensate(internalToken, authUserId); + if (response == null || !response.successful()) throw unavailable(); + } catch (RuntimeException compensationFailure) { + log.error("企业管理员创建失败且认证账号补偿失败，authUserId={}", authUserId, compensationFailure); + throw new BizException(ResultCode.SERVICE_UNAVAILABLE, + "企业管理员创建未完成，认证账号补偿失败，请按 authUserId=" + authUserId + " 修复一致性"); + } + } + + private String temporaryPassword() { + char[] value = new char[PASSWORD_LENGTH]; + value[0] = pick(UPPER); + value[1] = pick(LOWER); + value[2] = pick(DIGITS); + for (int index = 3; index < value.length; index++) value[index] = pick(ALL); + for (int index = value.length - 1; index > 0; index--) { + int swap = random.nextInt(index + 1); + char current = value[index]; + value[index] = value[swap]; + value[swap] = current; + } + return new String(value); + } + + private String username(long tenantId) { + char[] suffix = new char[USERNAME_SUFFIX_LENGTH]; + for (int index = 0; index < suffix.length; index++) suffix[index] = pick(LOWER); + return "merchant_" + tenantId + "_" + new String(suffix); + } + + private char pick(char[] source) { return source[random.nextInt(source.length)]; } + private BizException unavailable() { + return new BizException(ResultCode.SERVICE_UNAVAILABLE, "认证账号服务暂时不可用，不能审核通过"); + } + private BizException inconsistentOwner(String message) { + return new BizException(ResultCode.SERVICE_UNAVAILABLE, message); + } + + public static final class ProvisioningResult implements InitializationResult { + private final boolean created; + private final Long authUserId; + private final String username; + private final String temporaryPassword; + + private ProvisioningResult(boolean created, Long authUserId, String username, String temporaryPassword) { + this.created = created; + this.authUserId = authUserId; + this.username = username; + this.temporaryPassword = temporaryPassword; + } + + static ProvisioningResult existing() { return new ProvisioningResult(false, null, null, null); } + static ProvisioningResult created(long authUserId, String username, String password) { + return new ProvisioningResult(true, authUserId, username, password); + } + + public boolean created() { return created; } + public Long authUserId() { return authUserId; } + public String username() { return username; } + public String temporaryPassword() { return temporaryPassword; } + } +} diff --git a/iam/src/main/java/com/baiyue/iam/service/TenantApprovalInitializationCoordinator.java b/iam/src/main/java/com/baiyue/iam/service/TenantApprovalInitializationCoordinator.java new file mode 100644 index 000000000..bca36e43b --- /dev/null +++ b/iam/src/main/java/com/baiyue/iam/service/TenantApprovalInitializationCoordinator.java @@ -0,0 +1,123 @@ +package com.baiyue.iam.service; + +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.AnnotationAwareOrderComparator; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Runs all tenant approval initializers and owns their reverse-order compensation lifecycle. */ +@Service +public class TenantApprovalInitializationCoordinator { + private static final Logger log = LoggerFactory.getLogger(TenantApprovalInitializationCoordinator.class); + private final List initializers; + + public TenantApprovalInitializationCoordinator(List initializers) { + List ordered = new ArrayList<>(initializers); + AnnotationAwareOrderComparator.sort(ordered); + Set names = new HashSet<>(); + for (TenantApprovalInitializer initializer : ordered) { + String name = initializer.name(); + if (name == null || name.isBlank() || !names.add(name)) { + throw new IllegalArgumentException("Tenant approval initializer names must be nonblank and unique: " + name); + } + } + this.initializers = List.copyOf(ordered); + } + + public InitializationBatch initialize(TenantApprovalInitializer.ApprovalContext context) { + InitializationBatch batch = new InitializationBatch(context); + registerRollbackCompensation(batch); + try { + for (TenantApprovalInitializer initializer : initializers) { + TenantApprovalInitializer.InitializationResult result = initializer.initialize(context); + if (result == null) { + throw new IllegalStateException("Tenant approval initializer returned null: " + initializer.name()); + } + batch.add(initializer, result); + } + return batch; + } catch (RuntimeException initializationFailure) { + try { + compensate(batch); + } catch (RuntimeException compensationFailure) { + compensationFailure.addSuppressed(initializationFailure); + throw compensationFailure; + } + throw initializationFailure; + } + } + + public InitializationBatch empty(TenantApprovalInitializer.ApprovalContext context) { + return new InitializationBatch(context); + } + + public void compensate(InitializationBatch batch) { + if (!batch.markCompensated()) return; + RuntimeException firstFailure = null; + List completed = new ArrayList<>(batch.completed()); + Collections.reverse(completed); + for (CompletedInitialization item : completed) { + try { + item.initializer().compensate(batch.context(), item.result()); + } catch (RuntimeException exception) { + log.error("企业审核初始化补偿失败，initializer={}, tenantId={}", + item.initializer().name(), batch.context().tenant().getId(), exception); + if (firstFailure == null) firstFailure = exception; + else firstFailure.addSuppressed(exception); + } + } + if (firstFailure != null) { + throw new BizException(ResultCode.SERVICE_UNAVAILABLE, + "企业审核初始化补偿失败，请检查服务日志并修复数据一致性"); + } + } + + private void registerRollbackCompensation(InitializationBatch batch) { + if (!TransactionSynchronizationManager.isSynchronizationActive()) return; + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCompletion(int status) { + if (status == STATUS_ROLLED_BACK) compensate(batch); + } + }); + } + + private record CompletedInitialization(TenantApprovalInitializer initializer, + TenantApprovalInitializer.InitializationResult result) { } + + public static final class InitializationBatch { + private final TenantApprovalInitializer.ApprovalContext context; + private final List completed = new ArrayList<>(); + private final AtomicBoolean compensated = new AtomicBoolean(false); + + private InitializationBatch(TenantApprovalInitializer.ApprovalContext context) { + this.context = context; + } + + private void add(TenantApprovalInitializer initializer, + TenantApprovalInitializer.InitializationResult result) { + completed.add(new CompletedInitialization(initializer, result)); + } + + private TenantApprovalInitializer.ApprovalContext context() { return context; } + private List completed() { return List.copyOf(completed); } + private boolean markCompensated() { return compensated.compareAndSet(false, true); } + + public Optional result(Class type) { + return completed.stream().map(CompletedInitialization::result) + .filter(type::isInstance).map(type::cast).findFirst(); + } + } +} diff --git a/iam/src/main/java/com/baiyue/iam/service/TenantApprovalInitializer.java b/iam/src/main/java/com/baiyue/iam/service/TenantApprovalInitializer.java new file mode 100644 index 000000000..33506b732 --- /dev/null +++ b/iam/src/main/java/com/baiyue/iam/service/TenantApprovalInitializer.java @@ -0,0 +1,22 @@ +package com.baiyue.iam.service; + +import com.baiyue.iam.entity.IamTenant; + +/** + * Extension point for provisioning defaults after an enterprise is approved. + * Implementations may initialize accounts, roles, rules, fee plans or other tenant-scoped defaults. + * Implementations and their compensation must be idempotent. A failed implementation must clean + * up its own partial work before throwing; the coordinator compensates every initializer that + * completed successfully, in reverse order. Use {@code @Order} for explicit dependencies. + */ +public interface TenantApprovalInitializer { + String name(); + + InitializationResult initialize(ApprovalContext context); + + default void compensate(ApprovalContext context, InitializationResult result) { } + + record ApprovalContext(IamTenant tenant, long reviewId, long reviewerId, String comment) { } + + interface InitializationResult { } +} diff --git a/iam/src/main/java/com/baiyue/iam/service/TenantService.java b/iam/src/main/java/com/baiyue/iam/service/TenantService.java index a467b4b03..43f345701 100644 --- a/iam/src/main/java/com/baiyue/iam/service/TenantService.java +++ b/iam/src/main/java/com/baiyue/iam/service/TenantService.java @@ -17,6 +17,7 @@ import com.baiyue.iam.dto.TenantDtos.TenantResponse; import com.baiyue.iam.dto.TenantDtos.UpdateProfileRequest; import com.baiyue.iam.dto.TenantDtos.UpdateTenantRequest; import com.baiyue.iam.dto.TenantDtos.ChangeTenantStatusRequest; +import com.baiyue.iam.dto.TenantDtos.ApprovalResponse; import com.baiyue.iam.dto.CustomerExportDtos.CustomerExportBatch; import com.baiyue.iam.dto.CustomerExportDtos.CustomerExportRow; @@ -39,6 +40,8 @@ public class TenantService { private TenantRepository repository; @Resource private ProfileResourceRepository profileRepository; + @Resource + private TenantApprovalInitializationCoordinator approvalInitialization; @BusinessAudit( @@ -49,7 +52,7 @@ public class TenantService { captureAfter = false ) @Transactional - public TenantResponse register(RegisterRequest request) { + public TenantResponse register(RegisterRequest request, long operatorId) { if (repository.existsByCreditCode(request.unifiedSocialCreditCode())) { throw new BizException(ResultCode.CONFLICT, "Enterprise already exists / 企业已存在"); } @@ -61,6 +64,7 @@ public class TenantService { tenant.setTenantType("MERCHANT"); tenant.setStatus("PROFILE_REQUIRED"); tenant.setLegalName(request.legalName()); + tenant.setLegalRepresentative(request.legalRepresentative()); tenant.setUnifiedSocialCreditCode(request.unifiedSocialCreditCode()); tenant.setContactName(request.contactName()); tenant.setContactPhone(request.contactPhone()); @@ -75,6 +79,8 @@ public class TenantService { tenant.setRegisteredCapitalCurrency(request.registeredCapitalCurrency()); tenant.setRegisteredAddress(request.registeredAddress()); tenant.setInternalRemark(request.internalRemark()); + tenant.setCreatedBy(operatorId); + tenant.setUpdatedBy(operatorId); repository.insertTenant(tenant); return response(tenant); @@ -119,8 +125,8 @@ public class TenantService { captureAfter = false ) @Transactional - public void review(long reviewId, long reviewerId, boolean approve, String comment) { - IamTenantReview review = repository.findReview(reviewId); + public ApprovalResponse review(long reviewId, long reviewerId, boolean approve, String comment) { + IamTenantReview review = repository.findReviewForUpdate(reviewId); if (review == null) { throw new BizException(ResultCode.NOT_FOUND, "Review not found / 审核单不存在"); } @@ -135,28 +141,49 @@ public class TenantService { } } - review.setStatus(approve ? "APPROVED" : "REJECTED"); - review.setReviewerId(reviewerId); - review.setReviewComment(comment); - review.setReviewedAt(LocalDateTime.now()); - repository.saveReview(review); - IamTenant tenant = repository.findTenant(review.getTenantId()); if (tenant == null) { throw new BizException(ResultCode.NOT_FOUND, "Tenant not found / 企业不存在"); } - tenant.setStatus(approve ? "ACTIVE" : "REJECTED"); - tenant.setRejectReason(approve ? null : comment); - tenant.setApprovedAt(approve ? LocalDateTime.now() : null); - tenant.setApprovedBy(approve ? reviewerId : null); - repository.saveTenant(tenant); - for (var qualification : profileRepository.findQualifications(tenant.getId())) { - if ("PENDING".equals(qualification.getStatus())) { - qualification.setStatus(approve ? "APPROVED" : "REJECTED"); - qualification.setReviewComment(comment); - profileRepository.saveQualification(qualification); + + var initializationContext = new TenantApprovalInitializer.ApprovalContext( + tenant, reviewId, reviewerId, comment); + var initialization = approve + ? approvalInitialization.initialize(initializationContext) + : approvalInitialization.empty(initializationContext); + var administrator = initialization + .result(TenantAdministratorProvisioningService.ProvisioningResult.class) + .orElseGet(TenantAdministratorProvisioningService.ProvisioningResult::existing); + try { + review.setStatus(approve ? "APPROVED" : "REJECTED"); + review.setReviewerId(reviewerId); + review.setReviewComment(comment); + review.setReviewedAt(LocalDateTime.now()); + repository.saveReview(review); + + tenant.setStatus(approve ? "ACTIVE" : "REJECTED"); + tenant.setRejectReason(approve ? null : comment); + tenant.setApprovedAt(approve ? LocalDateTime.now() : null); + tenant.setApprovedBy(approve ? reviewerId : null); + repository.saveTenant(tenant); + for (var qualification : profileRepository.findQualifications(tenant.getId())) { + if ("PENDING".equals(qualification.getStatus())) { + qualification.setStatus(approve ? "APPROVED" : "REJECTED"); + qualification.setReviewComment(comment); + profileRepository.saveQualification(qualification); + } } + } catch (RuntimeException localFailure) { + try { + approvalInitialization.compensate(initialization); + } catch (RuntimeException compensationFailure) { + compensationFailure.addSuppressed(localFailure); + throw compensationFailure; + } + throw localFailure; } + return new ApprovalResponse(tenant.getId(), administrator.created(), + administrator.username(), administrator.temporaryPassword()); } public OrderEligibilityResponse orderEligibility(long tenantId) { @@ -168,9 +195,9 @@ public class TenantService { return new OrderEligibilityResponse(false, tenant.getStatus(), "Enterprise information must be approved before ordering"); } - if (profileRepository.countApprovedQualifications(tenantId) < 1) { + if (!hasAllQualificationsWithStatus(profileRepository.findQualifications(tenantId), "APPROVED")) { return new OrderEligibilityResponse(false, tenant.getStatus(), - "At least one approved enterprise qualification is required"); + "All required enterprise qualifications must be approved"); } return new OrderEligibilityResponse(true, tenant.getStatus(), null); } @@ -195,6 +222,7 @@ public class TenantService { IamTenant tenant = requireMerchant(tenantId); tenant.setTenantName(request.tenantName()); tenant.setLegalName(request.legalName()); + tenant.setLegalRepresentative(request.legalRepresentative()); tenant.setContactName(request.contactName()); tenant.setContactPhone(request.contactPhone()); tenant.setContactEmail(request.contactEmail()); @@ -237,6 +265,11 @@ public class TenantService { if (repository.findPendingReviewByTenant(tenantId) != null) { throw new BizException(ResultCode.CONFLICT, "Enterprise information is already under review / 客户资料已在审核中"); } + if (tenant.getUnifiedSocialCreditCode() == null + || !tenant.getUnifiedSocialCreditCode().matches("[0-9A-Z]{18}")) { + throw new BizException(ResultCode.BAD_REQUEST, + "A valid unified social credit code is required / 请先填写有效的18位统一社会信用代码"); + } var qualifications = profileRepository.findQualifications(tenantId); if (!hasAllPendingQualifications(qualifications)) { throw new BizException(ResultCode.BAD_REQUEST, @@ -262,6 +295,7 @@ public class TenantService { } tenant.setTenantName(request.tenantName()); tenant.setLegalName(request.legalName()); + tenant.setLegalRepresentative(request.legalRepresentative()); tenant.setContactName(request.contactName()); tenant.setContactPhone(request.contactPhone()); tenant.setContactEmail(request.contactEmail()); @@ -274,8 +308,12 @@ public class TenantService { } private boolean hasAllPendingQualifications(List qualifications) { + return hasAllQualificationsWithStatus(qualifications, "PENDING"); + } + + private boolean hasAllQualificationsWithStatus(List qualifications, String status) { return REQUIRED_QUALIFICATION_TYPES.stream().allMatch(type -> qualifications.stream() - .anyMatch(item -> type.equals(item.getQualificationType()) && "PENDING".equals(item.getStatus()))); + .anyMatch(item -> type.equals(item.getQualificationType()) && status.equals(item.getStatus()))); } private IamTenant requireMerchant(long tenantId) { @@ -297,7 +335,7 @@ public class TenantService { private TenantResponse response(IamTenant tenant) { return new TenantResponse( tenant.getId(), tenant.getTenantCode(), tenant.getTenantName(), tenant.getStatus(), - tenant.getLegalName(), tenant.getUnifiedSocialCreditCode(), tenant.getContactName(), + tenant.getLegalName(), tenant.getLegalRepresentative(), tenant.getUnifiedSocialCreditCode(), tenant.getContactName(), tenant.getContactPhone(), tenant.getContactEmail(), tenant.getContactWechat(), tenant.getServicePackage(), tenant.getSalesOwnerName(), tenant.getPaymentTermDays(), tenant.getEnterpriseType(), tenant.getEstablishedDate(), tenant.getRegisteredCapital(), tenant.getRegisteredCapitalCurrency(), @@ -308,7 +346,7 @@ public class TenantService { private TenantResponse summaryResponse(IamTenant tenant) { TenantResponse response = response(tenant); return new TenantResponse(response.id(), response.tenantCode(), response.tenantName(), response.status(), - response.legalName(), response.unifiedSocialCreditCode(), response.contactName(), + response.legalName(), response.legalRepresentative(), response.unifiedSocialCreditCode(), response.contactName(), maskPhone(response.contactPhone()), maskEmail(response.contactEmail()), response.contactWechat(), response.servicePackage(), response.salesOwnerName(), response.paymentTermDays(), response.enterpriseType(), response.establishedDate(), response.registeredCapital(), response.registeredCapitalCurrency(), @@ -318,7 +356,7 @@ public class TenantService { private TenantResponse merchantVisibleResponse(IamTenant tenant) { TenantResponse response = response(tenant); return new TenantResponse(response.id(), response.tenantCode(), response.tenantName(), response.status(), - response.legalName(), response.unifiedSocialCreditCode(), response.contactName(), + response.legalName(), response.legalRepresentative(), response.unifiedSocialCreditCode(), response.contactName(), response.contactPhone(), response.contactEmail(), response.contactWechat(), response.servicePackage(), response.salesOwnerName(), response.paymentTermDays(), response.enterpriseType(), response.establishedDate(), response.registeredCapital(), response.registeredCapitalCurrency(), response.registeredAddress(), diff --git a/iam/src/main/resources/db/iam.sql b/iam/src/main/resources/db/iam.sql index 0c9ea0287..5ef63f26f 100644 --- a/iam/src/main/resources/db/iam.sql +++ b/iam/src/main/resources/db/iam.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS iam_tenant ( tenant_type VARCHAR(20) NOT NULL DEFAULT 'MERCHANT' COMMENT '租户类型', status VARCHAR(20) NOT NULL DEFAULT 'PROFILE_REQUIRED' COMMENT '租户状态：PROFILE_REQUIRED、PENDING、ACTIVE、REJECTED、DISABLED', legal_name VARCHAR(128) NULL COMMENT '企业法定名称', + legal_representative VARCHAR(64) NULL COMMENT '法定代表人', unified_social_credit_code VARCHAR(32) NULL COMMENT '统一社会信用代码', contact_name VARCHAR(64) NULL COMMENT '企业联系人', contact_phone VARCHAR(32) NULL COMMENT '联系电话', diff --git a/iam/src/main/resources/db/phase1-admin-rbac.sql b/iam/src/main/resources/db/phase1-admin-rbac.sql index fe20d0941..2f6ab6a40 100644 --- a/iam/src/main/resources/db/phase1-admin-rbac.sql +++ b/iam/src/main/resources/db/phase1-admin-rbac.sql @@ -79,7 +79,6 @@ VALUES (5104000, 0, 'PLATFORM', 'MENU', 0, 'platform:audit', '审计管理', '/audit', NULL, 50, 'ACTIVE', 0), -- Platform: level-two menus. (5100010, 0, 'PLATFORM', 'MENU', 5100000, 'platform:customer:list', '企业客户', '/customers/list', NULL, 10, 'ACTIVE', 0), - (5100020, 0, 'PLATFORM', 'MENU', 5100000, 'platform:customer:review', '企业审核', '/customers/reviews', NULL, 20, 'ACTIVE', 0), (5101010, 0, 'PLATFORM', 'MENU', 5101000, 'platform:user:list', '内部用户', '/system/users', NULL, 10, 'ACTIVE', 0), (5101020, 0, 'PLATFORM', 'MENU', 5101000, 'platform:role:list', '角色管理', '/system/roles', NULL, 20, 'ACTIVE', 0), (5101030, 0, 'PLATFORM', 'MENU', 5101000, 'platform:permission:list', '菜单按钮', '/system/permissions', NULL, 30, 'ACTIVE', 0), @@ -91,6 +90,7 @@ VALUES (5103040, 0, 'PLATFORM', 'MENU', 5103000, 'platform:download:list', '下载中心', '/downloads', NULL, 40, 'ACTIVE', 0), (5104010, 0, 'PLATFORM', 'MENU', 5104000, 'platform:audit:list', '操作日志', '/audit/logs', NULL, 10, 'ACTIVE', 0), -- Platform: buttons. + (5100020, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:review', '客户审核', NULL, NULL, 50, 'ACTIVE', 0), (5100021, 0, 'PLATFORM', 'BUTTON', 5100020, 'platform:customer:approve', '审核通过', NULL, NULL, 10, 'ACTIVE', 0), (5100022, 0, 'PLATFORM', 'BUTTON', 5100020, 'platform:customer:reject', '审核驳回', NULL, NULL, 20, 'ACTIVE', 0), (5100015, 0, 'PLATFORM', 'BUTTON', 5100010, 'platform:customer:export', '导出企业客户', NULL, NULL, 40, 'ACTIVE', 0), diff --git a/iam/src/main/resources/db/phase1-onboarding.sql b/iam/src/main/resources/db/phase1-onboarding.sql index 44ba26145..6060c3934 100644 --- a/iam/src/main/resources/db/phase1-onboarding.sql +++ b/iam/src/main/resources/db/phase1-onboarding.sql @@ -25,6 +25,7 @@ DELIMITER ; CALL phase1_iam_add_column('iam_tenant', 'approved_at', "DATETIME(3) NULL COMMENT 'Approval time'"); CALL phase1_iam_add_column('iam_tenant', 'approved_by', "BIGINT NULL COMMENT 'Reviewer ID'"); CALL phase1_iam_add_column('iam_tenant', 'reject_reason', "VARCHAR(500) NULL COMMENT 'Rejection reason'"); +CALL phase1_iam_add_column('iam_tenant', 'legal_representative', "VARCHAR(64) NULL COMMENT '法定代表人' AFTER legal_name"); CALL phase1_iam_add_column('iam_tenant_qualification', 'review_comment', "VARCHAR(500) NULL COMMENT 'Review comment'"); CALL phase1_iam_add_column('iam_membership', 'created_by', "BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Creator ID'"); CALL phase1_iam_add_column('iam_membership', 'updated_by', "BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Updater ID'"); @@ -55,11 +56,17 @@ ON DUPLICATE KEY UPDATE permission_name=VALUES(permission_name), route_path=VALUES(route_path), parent_id=VALUES(parent_id), sort_no=VALUES(sort_no), status='ACTIVE', deleted=0; -INSERT IGNORE INTO iam_role_permission (role_id, permission_id, tenant_id) -SELECT 4000100, id, 0 -FROM iam_permission -WHERE app_code='MERCHANT' AND tenant_id=0 - AND permission_type IN ('MENU','BUTTON') AND status='ACTIVE' AND deleted=0; +INSERT INTO iam_role_permission (role_id, permission_id, tenant_id) +SELECT owner_role.id, permission.id, 0 +FROM iam_role owner_role +JOIN iam_permission permission + ON permission.app_code='MERCHANT' AND permission.tenant_id=0 + AND permission.permission_type IN ('MENU','BUTTON') + AND permission.status='ACTIVE' AND permission.deleted=0 +WHERE owner_role.tenant_id=0 AND owner_role.app_code='MERCHANT' + AND owner_role.role_code='MERCHANT_OWNER' + AND owner_role.status='ACTIVE' AND owner_role.deleted=0 +ON DUPLICATE KEY UPDATE tenant_id=VALUES(tenant_id); -- 不再写入伪造的资质 URL。资质必须先由 File 服务保存真实图片，再由 IAM 记录文件地址和审核结果。 -- 已部署环境中的 /phase1-seed/ 历史记录按 docs/企业资质文件迁移说明.md 处理，本脚本不得将其改成不存在的 UUID。 diff --git a/iam/src/main/resources/openapi/iam-v1.yaml b/iam/src/main/resources/openapi/iam-v1.yaml index 6d4313c9f..418260627 100644 --- a/iam/src/main/resources/openapi/iam-v1.yaml +++ b/iam/src/main/resources/openapi/iam-v1.yaml @@ -173,7 +173,7 @@ paths: post: tags: [Platform Tenant / 平台租户] summary: Approve enterprise registration / 审核通过企业注册 - description: Approve a pending review and activate the tenant. / 通过待审核申请并激活企业租户。 + description: Approve and activate the tenant. Client registrations keep the registration account as administrator. Only platform-created tenants without an owner receive a generated administrator and one-time credentials. / 审核通过并激活企业；客户端注册账号继续作为企业管理员，仅平台代建且没有所有者的企业自动创建管理员并返回一次性凭据。 parameters: - {$ref: '#/components/parameters/Id'} requestBody: @@ -182,7 +182,11 @@ paths: application/json: schema: {$ref: '#/components/schemas/ReviewRequest'} responses: - '200': {$ref: '#/components/responses/Success'} + '200': + description: 审核通过；data 返回开户结果及一次性初始凭据 + content: + application/json: + schema: {$ref: '#/components/schemas/TenantApprovalApiResponse'} '403': {$ref: '#/components/responses/Forbidden'} '404': {$ref: '#/components/responses/NotFound'} '409': {$ref: '#/components/responses/Conflict'} @@ -595,6 +599,41 @@ paths: '400': {$ref: '#/components/responses/BadRequest'} '403': {$ref: '#/components/responses/Forbidden'} '409': {$ref: '#/components/responses/Conflict'} + /internal/exports/v1/CUSTOMER_LIST/batches: + post: + tags: [Internal Export Provider] + summary: 分批提供企业客户导出数据 + description: 仅供 File Service 调用；按平台访问域、签名最大 ID 快照和游标只读查询商户客户。 + security: [] + parameters: + - name: X-Internal-Service-Token + in: header + required: true + schema: {type: string} + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [taskId, tenantId, appCode, requestedBy, dispatchVersion, parameters, limit] + properties: + taskId: {type: string, pattern: '^\d+$'} + tenantId: {type: string, pattern: '^\d+$'} + appCode: {type: string, enum: [PLATFORM]} + requestedBy: {type: string, pattern: '^\d+$'} + dispatchVersion: {type: integer, minimum: 1} + parameters: {type: object} + cursor: {type: string, nullable: true} + snapshotToken: {type: string, nullable: true, maxLength: 512} + limit: {type: integer, minimum: 1, maximum: 500} + responses: + '200': + description: 统一响应；data 为 ExportBatchResponse，Long 值以字符串传输 + content: + application/json: + schema: {$ref: '#/components/schemas/ApiResponse'} + '403': {$ref: '#/components/responses/Forbidden'} components: securitySchemes: bearerAuth: @@ -677,6 +716,20 @@ components: size: {type: integer, format: int64, minimum: 1} total: {type: integer, format: int64, minimum: 0} pages: {type: integer, format: int64, minimum: 0} + TenantApprovalApiResponse: + allOf: + - {$ref: '#/components/schemas/ApiResponse'} + - type: object + properties: + data: {$ref: '#/components/schemas/TenantApprovalResponse'} + TenantApprovalResponse: + type: object + required: [tenantId, administratorCreated] + properties: + tenantId: {type: string, pattern: '^\d+$', example: '190000000000000001'} + administratorCreated: {type: boolean, description: 本次审核是否新建企业管理员} + administratorUsername: {type: string, nullable: true, example: merchant_190000000000000001_k3mnpqrs} + temporaryPassword: {type: string, format: password, nullable: true, description: 一次性初始密码，仅新建管理员时返回} TenantApiResponse: allOf: - {$ref: '#/components/schemas/ApiResponse'} @@ -704,6 +757,7 @@ components: tenantName: {type: string} status: {type: string, enum: [PROFILE_REQUIRED, PENDING, ACTIVE, REJECTED, DISABLED]} legalName: {type: string} + legalRepresentative: {type: string, maxLength: 64} unifiedSocialCreditCode: {type: string} contactName: {type: string} contactPhone: {type: string} @@ -763,10 +817,11 @@ components: reviewComment: {type: string} RegisterRequest: type: object - required: [tenantName, legalName, unifiedSocialCreditCode, contactName, contactPhone, servicePackage] + required: [tenantName, legalName, legalRepresentative, unifiedSocialCreditCode, contactName, contactPhone, servicePackage] properties: tenantName: {type: string, example: 深圳虫虫出海科技有限公司} - legalName: {type: string, example: 张三} + legalName: {type: string, example: 深圳虫虫出海科技有限公司} + legalRepresentative: {type: string, maxLength: 64, example: 张三} unifiedSocialCreditCode: {type: string, pattern: '^[0-9A-Z]{18}$', example: 91440300MA5ABCDE12} contactName: {type: string, example: 李四} contactPhone: {type: string, example: '13800138000'} @@ -783,19 +838,21 @@ components: internalRemark: {type: string, maxLength: 1000} UpdateProfileRequest: type: object - required: [tenantName, legalName, contactName, contactPhone, servicePackage] + required: [tenantName, legalName, legalRepresentative, contactName, contactPhone] properties: tenantName: {type: string} legalName: {type: string} + legalRepresentative: {type: string, maxLength: 64} contactName: {type: string} contactPhone: {type: string} contactEmail: {type: string, format: email} UpdateTenantRequest: type: object - required: [tenantName, legalName, contactName, contactPhone] + required: [tenantName, legalName, legalRepresentative, contactName, contactPhone] properties: tenantName: {type: string} legalName: {type: string} + legalRepresentative: {type: string, maxLength: 64} contactName: {type: string} contactPhone: {type: string} contactEmail: {type: string, format: email} diff --git a/iam/src/test/java/com/baiyue/iam/controller/CustomerListExportProviderControllerTest.java b/iam/src/test/java/com/baiyue/iam/controller/CustomerListExportProviderControllerTest.java new file mode 100644 index 000000000..86ee0e330 --- /dev/null +++ b/iam/src/test/java/com/baiyue/iam/controller/CustomerListExportProviderControllerTest.java @@ -0,0 +1,57 @@ +package com.baiyue.iam.controller; + +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportBatchResponse; +import com.baiyue.iam.service.CustomerListExportProviderService; +import org.junit.jupiter.api.Test; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class CustomerListExportProviderControllerTest { + @Test + void exposesTheGenericCustomerListProviderPath() throws Exception { + assertEquals("/internal/exports/v1/CUSTOMER_LIST", + CustomerListExportProviderController.class.getAnnotation(RequestMapping.class).value()[0]); + assertEquals("/batches", CustomerListExportProviderController.class + .getMethod("batch", String.class, ExportBatchRequest.class) + .getAnnotation(PostMapping.class).value()[0]); + } + + @Test + void requiresInternalTokenBeforeCallingProvider() { + CustomerListExportProviderService service = mock(CustomerListExportProviderService.class); + CustomerListExportProviderController controller = + new CustomerListExportProviderController(service, "internal-token"); + ExportBatchRequest request = request(); + + BizException missing = assertThrows(BizException.class, () -> controller.batch(null, request)); + BizException wrong = assertThrows(BizException.class, () -> controller.batch("wrong", request)); + assertEquals(ResultCode.FORBIDDEN, missing.getResultCode()); + assertEquals(ResultCode.FORBIDDEN, wrong.getResultCode()); + + ExportBatchResponse batch = new ExportBatchResponse(1, "sha256:test", "snapshot", + List.of(), List.of(), null, false, 0L); + when(service.exportBatch(request)).thenReturn(batch); + var response = controller.batch("internal-token", request); + + assertEquals(ResultCode.SUCCESS.code(), response.code()); + assertEquals(batch, response.data()); + verify(service).exportBatch(request); + } + + private ExportBatchRequest request() { + return new ExportBatchRequest("190000000000000001", "0", "PLATFORM", + "190000000000000009", 1, Map.of(), null, null, 500); + } +} diff --git a/iam/src/test/java/com/baiyue/iam/controller/PlatformTenantControllerContractTest.java b/iam/src/test/java/com/baiyue/iam/controller/PlatformTenantControllerContractTest.java index 9139e81dc..4ee62b7ad 100644 --- a/iam/src/test/java/com/baiyue/iam/controller/PlatformTenantControllerContractTest.java +++ b/iam/src/test/java/com/baiyue/iam/controller/PlatformTenantControllerContractTest.java @@ -11,6 +11,7 @@ import org.springframework.web.bind.annotation.GetMapping; import java.lang.annotation.Annotation; import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; @@ -55,6 +56,16 @@ class PlatformTenantControllerContractTest { method.getAnnotation(PreAuthorize.class).value()); } + @Test + void 审核通过接口返回一次性管理员凭据结果() throws Exception { + Method method = PlatformTenantController.class.getMethod("approve", long.class, + com.baiyue.iam.dto.TenantDtos.ReviewRequest.class, AuthenticatedUser.class); + + ParameterizedType returnType = (ParameterizedType) method.getGenericReturnType(); + assertEquals(com.baiyue.iam.dto.TenantDtos.ApprovalResponse.class, + returnType.getActualTypeArguments()[0]); + } + private T annotation(Method method, int parameter, Class type) { return method.getParameters()[parameter].getAnnotation(type); } diff --git a/iam/src/test/java/com/baiyue/iam/repository/CustomerListExportProviderRepositoryTest.java b/iam/src/test/java/com/baiyue/iam/repository/CustomerListExportProviderRepositoryTest.java new file mode 100644 index 000000000..489358cac --- /dev/null +++ b/iam/src/test/java/com/baiyue/iam/repository/CustomerListExportProviderRepositoryTest.java @@ -0,0 +1,50 @@ +package com.baiyue.iam.repository; + +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.mapper.IamTenantMapper; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.AbstractWrapper; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.junit.jupiter.api.Test; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.mockito.ArgumentCaptor; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class CustomerListExportProviderRepositoryTest { + @Test + void batchQueryAlwaysConstrainsMerchantSnapshotAndAscendingCursor() { + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(new MybatisConfiguration(), "test"), + IamTenant.class); + IamTenantMapper mapper = mock(IamTenantMapper.class); + when(mapper.selectList(any())).thenReturn(List.of()); + CustomerListExportProviderRepository repository = + new CustomerListExportProviderRepository(mapper); + + repository.findBatch(100L, 200L, 501, "柏悦", "ACTIVE"); + + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(Wrapper.class); + verify(mapper).selectList(captor.capture()); + Wrapper query = captor.getValue(); + assertThat(query.getSqlSegment()) + .contains("tenant_type") + .contains("status") + .contains("tenant_code") + .contains("tenant_name") + .contains("id >") + .contains("id <=") + .contains("ORDER BY id ASC") + .endsWith("LIMIT 501"); + assertThat(((AbstractWrapper) query).getParamNameValuePairs().values()) + .contains("MERCHANT", "ACTIVE", 100L, 200L) + .anyMatch(value -> "%柏悦%".equals(value)); + } +} diff --git a/iam/src/test/java/com/baiyue/iam/repository/CustomerProfileMigrationContractTest.java b/iam/src/test/java/com/baiyue/iam/repository/CustomerProfileMigrationContractTest.java index 0b03270da..102de9726 100644 --- a/iam/src/test/java/com/baiyue/iam/repository/CustomerProfileMigrationContractTest.java +++ b/iam/src/test/java/com/baiyue/iam/repository/CustomerProfileMigrationContractTest.java @@ -10,6 +10,21 @@ import java.nio.file.Path; import static org.assertj.core.api.Assertions.assertThat; class CustomerProfileMigrationContractTest { + @Test + void 企业审核初始化按角色自然键为全局管理员模板授权() throws IOException { + String sql; + try (var stream = getClass().getResourceAsStream("/db/phase1-onboarding.sql")) { + assertThat(stream).isNotNull(); + sql = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + + assertThat(sql) + .contains("'MERCHANT_OWNER'") + .contains("SELECT owner_role.id, permission.id, 0") + .contains("owner_role.role_code='MERCHANT_OWNER'") + .doesNotContain("SELECT 4000100, id, 0"); + } + @Test void 客户档案迁移覆盖全部字段并通过元数据判断保持可重复执行() throws IOException { String sql; @@ -46,17 +61,32 @@ class CustomerProfileMigrationContractTest { assertThat(sql) .contains("CALL regression_assert_table('iam_tenant')") + .contains("CALL regression_assert_table('iam_tenant_review')") + .contains("CALL regression_assert_table('iam_tenant_qualification')") + .contains("DEFAULT 'PROFILE_REQUIRED'") .contains("CREATE PROCEDURE regression_add_iam_column") + .contains("regression_add_iam_column('approved_at'") + .contains("regression_add_iam_column('approved_by'") + .contains("regression_add_iam_column('reject_reason'") + .contains("regression_add_iam_related_column('iam_tenant_qualification', 'review_comment'") + .contains("'IAM注册审核字段'") + .contains("'IAM资质与成员审计字段'") .contains("platform:customer:view") .contains("platform:customer:export") .contains("platform:download:list") + .contains("'MERCHANT_OWNER'") + .contains("IAM全局企业管理员模板") + .contains("owner_role.id, permission.id, 0") .contains("'IAM客户档案字段' AS check_item") - .contains("9 AS expected_count") + .contains("10 AS expected_count") .contains("47 AS expected_count") .contains("MIGRATION_COMPLETED"); assertThat(instructions) .contains("phase1-platform-master-data-and-permission-management.sql") .contains("phase1-customer-profile.sql") + .contains("phase1-onboarding.sql") + .contains("IAM 注册审核字段") + .contains("全局 `MERCHANT_OWNER` 角色模板") .contains("测试环境只执行统一入口"); } } diff --git a/iam/src/test/java/com/baiyue/iam/service/CustomerListExportProviderServiceTest.java b/iam/src/test/java/com/baiyue/iam/service/CustomerListExportProviderServiceTest.java new file mode 100644 index 000000000..7c3140634 --- /dev/null +++ b/iam/src/test/java/com/baiyue/iam/service/CustomerListExportProviderServiceTest.java @@ -0,0 +1,161 @@ +package com.baiyue.iam.service; + +import com.baiyue.common.core.api.ResultCode; +import com.baiyue.common.core.exception.BizException; +import com.baiyue.common.core.export.ExportContracts.ExportBatchRequest; +import com.baiyue.common.core.export.ExportContracts.ExportColumnType; +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.repository.CustomerListExportProviderRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class CustomerListExportProviderServiceTest { + private CustomerListExportProviderRepository repository; + private CustomerListExportProviderService service; + + @BeforeEach + void setUp() { + repository = mock(CustomerListExportProviderRepository.class); + service = new CustomerListExportProviderService(repository, + "test-internal-service-token-at-least-32-bytes"); + } + + @Test + void firstBatchReturnsTypedSchemaSignedMaximumIdAndBoundedRows() { + when(repository.maximumId("柏悦", "ACTIVE")).thenReturn(103L); + when(repository.estimateRows(103L, "柏悦", "ACTIVE")).thenReturn(3L); + when(repository.findBatch(null, 103L, 3, "柏悦", "ACTIVE")) + .thenReturn(List.of(tenant(101L), tenant(102L), tenant(103L))); + + var response = service.exportBatch(request(Map.of("keyword", " 柏悦 ", "status", "ACTIVE"), + null, null, 2)); + + assertEquals(1, response.schemaVersion()); + assertThat(response.schemaHash()).matches("sha256:[0-9a-f]{64}"); + assertThat(response.snapshotToken()).matches("v1\\.103\\.[A-Za-z0-9_-]+"); + assertEquals(20, response.columns().size()); + assertEquals("tenantCode", response.columns().get(1).key()); + assertEquals(ExportColumnType.DECIMAL, + response.columns().stream().filter(column -> "registeredCapital".equals(column.key())) + .findFirst().orElseThrow().type()); + assertEquals(2, response.rows().size()); + assertEquals("101", response.rows().get(0).get("id")); + assertEquals(new BigDecimal("1000000.00"), response.rows().get(0).get("registeredCapital")); + assertEquals("启用", response.rows().get(0).get("status")); + assertEquals("102", response.nextCursor()); + assertEquals(true, response.hasMore()); + assertEquals(3L, response.totalRows()); + } + + @Test + void laterBatchReusesVerifiedSnapshotAndOmitsSchemaAndEstimate() { + when(repository.maximumId("柏悦", "ACTIVE")).thenReturn(103L); + when(repository.estimateRows(103L, "柏悦", "ACTIVE")).thenReturn(3L); + when(repository.findBatch(null, 103L, 3, "柏悦", "ACTIVE")) + .thenReturn(List.of(tenant(101L), tenant(102L), tenant(103L))); + var first = service.exportBatch(request(Map.of("keyword", "柏悦", "status", "ACTIVE"), + null, null, 2)); + + when(repository.findBatch(102L, 103L, 3, "柏悦", "ACTIVE")) + .thenReturn(List.of(tenant(103L))); + var later = service.exportBatch(request(Map.of("keyword", "柏悦", "status", "ACTIVE"), + "102", first.snapshotToken(), 2)); + + assertThat(later.columns()).isEmpty(); + assertEquals(first.schemaHash(), later.schemaHash()); + assertEquals(first.snapshotToken(), later.snapshotToken()); + assertEquals(1, later.rows().size()); + assertEquals("103", later.nextCursor()); + assertEquals(false, later.hasMore()); + assertEquals(null, later.totalRows()); + verify(repository).findBatch(102L, 103L, 3, "柏悦", "ACTIVE"); + } + + @Test + void snapshotIsBoundToTaskIdentityAndFilters() { + when(repository.maximumId("柏悦", "ACTIVE")).thenReturn(103L); + when(repository.estimateRows(103L, "柏悦", "ACTIVE")).thenReturn(0L); + when(repository.findBatch(null, 103L, 2, "柏悦", "ACTIVE")).thenReturn(List.of()); + String token = service.exportBatch(request(Map.of("keyword", "柏悦", "status", "ACTIVE"), + null, null, 1)).snapshotToken(); + + BizException changedFilter = assertThrows(BizException.class, () -> service.exportBatch( + request(Map.of("keyword", "其他", "status", "ACTIVE"), "0", token, 1))); + String tampered = token.substring(0, token.length() - 1) + + (token.endsWith("A") ? "B" : "A"); + BizException tamperedToken = assertThrows(BizException.class, () -> service.exportBatch( + request(Map.of("keyword", "柏悦", "status", "ACTIVE"), "0", tampered, 1))); + + assertEquals(ResultCode.VALIDATION_FAILED, changedFilter.getResultCode()); + assertEquals(ResultCode.VALIDATION_FAILED, tamperedToken.getResultCode()); + } + + @Test + void rejectsMissingScopeInvalidLimitsAndUntrustedParametersBeforeQuerying() { + BizException tenant = assertThrows(BizException.class, () -> service.exportBatch(new ExportBatchRequest( + "190000000000000001", "9", "PLATFORM", "190000000000000009", + 1, Map.of(), null, null, 500))); + BizException app = assertThrows(BizException.class, () -> service.exportBatch(new ExportBatchRequest( + "190000000000000001", "0", "MERCHANT", "190000000000000009", + 1, Map.of(), null, null, 500))); + BizException actor = assertThrows(BizException.class, () -> service.exportBatch(new ExportBatchRequest( + "190000000000000001", "0", "PLATFORM", null, + 1, Map.of(), null, null, 500))); + BizException limit = assertThrows(BizException.class, () -> service.exportBatch( + request(Map.of(), null, null, 501))); + BizException untrusted = assertThrows(BizException.class, () -> service.exportBatch( + request(Map.of("tenantId", "9"), null, null, 500))); + BizException cursor = assertThrows(BizException.class, () -> service.exportBatch( + request(Map.of(), "1", null, 500))); + + assertThat(List.of(tenant, app, actor, limit, untrusted, cursor)) + .allMatch(error -> error.getResultCode() == ResultCode.VALIDATION_FAILED); + verify(repository, never()).maximumId(org.mockito.ArgumentMatchers.any(), + org.mockito.ArgumentMatchers.any()); + } + + private ExportBatchRequest request(Map parameters, String cursor, + String snapshotToken, int limit) { + return new ExportBatchRequest("190000000000000001", "0", "PLATFORM", + "190000000000000009", 1, parameters, cursor, snapshotToken, limit); + } + + private IamTenant tenant(long id) { + IamTenant tenant = new IamTenant(); + tenant.setId(id); + tenant.setTenantCode("TEN" + id); + tenant.setTenantName("柏悦物流" + id); + tenant.setLegalName("柏悦物流有限公司"); + tenant.setUnifiedSocialCreditCode("91440000123456789X"); + tenant.setEnterpriseType("有限责任公司"); + tenant.setEstablishedDate(LocalDate.of(2020, 1, 2)); + tenant.setRegisteredCapital(new BigDecimal("1000000.00")); + tenant.setRegisteredCapitalCurrency("CNY"); + tenant.setRegisteredAddress("深圳市"); + tenant.setContactName("张三"); + tenant.setContactPhone("13800138000"); + tenant.setContactEmail("customer@example.com"); + tenant.setContactWechat("wechat"); + tenant.setSalesOwnerName("销售甲"); + tenant.setServicePackage("基础版"); + tenant.setPaymentTermDays(30); + tenant.setStatus("ACTIVE"); + tenant.setCreatedAt(LocalDateTime.of(2026, 8, 1, 10, 0)); + tenant.setApprovedAt(LocalDateTime.of(2026, 8, 2, 10, 0)); + return tenant; + } +} diff --git a/iam/src/test/java/com/baiyue/iam/service/MerchantOnboardingServiceTest.java b/iam/src/test/java/com/baiyue/iam/service/MerchantOnboardingServiceTest.java new file mode 100644 index 000000000..fc8ee4258 --- /dev/null +++ b/iam/src/test/java/com/baiyue/iam/service/MerchantOnboardingServiceTest.java @@ -0,0 +1,178 @@ +package com.baiyue.iam.service; + +import com.baiyue.iam.dto.OnboardingDtos.MerchantRegistrationRequest; +import com.baiyue.iam.entity.IamMembership; +import com.baiyue.iam.entity.IamMembershipRole; +import com.baiyue.iam.entity.IamRole; +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.entity.IamUser; +import com.baiyue.iam.mapper.IamMembershipMapper; +import com.baiyue.iam.mapper.IamMembershipRoleMapper; +import com.baiyue.iam.mapper.IamRoleMapper; +import com.baiyue.iam.mapper.IamTenantAddressMapper; +import com.baiyue.iam.mapper.IamTenantMapper; +import com.baiyue.iam.mapper.IamTenantQualificationMapper; +import com.baiyue.iam.mapper.IamTenantReviewMapper; +import com.baiyue.iam.mapper.IamUserMapper; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class MerchantOnboardingServiceTest { + @Test + void clientRegistrationCreatesOwnerAndAssignsGlobalMerchantOwnerRole() { + IamTenantMapper tenants = mock(IamTenantMapper.class); + IamUserMapper users = mock(IamUserMapper.class); + IamMembershipMapper memberships = mock(IamMembershipMapper.class); + IamRoleMapper roles = mock(IamRoleMapper.class); + IamMembershipRoleMapper membershipRoles = mock(IamMembershipRoleMapper.class); + IamTenantAddressMapper addresses = mock(IamTenantAddressMapper.class); + IamTenantQualificationMapper qualifications = mock(IamTenantQualificationMapper.class); + IamTenantReviewMapper reviews = mock(IamTenantReviewMapper.class); + MerchantOnboardingService service = new MerchantOnboardingService(); + ReflectionTestUtils.setField(service, "tenantMapper", tenants); + ReflectionTestUtils.setField(service, "userMapper", users); + ReflectionTestUtils.setField(service, "membershipMapper", memberships); + ReflectionTestUtils.setField(service, "roleMapper", roles); + ReflectionTestUtils.setField(service, "membershipRoleMapper", membershipRoles); + ReflectionTestUtils.setField(service, "addressMapper", addresses); + ReflectionTestUtils.setField(service, "qualificationMapper", qualifications); + ReflectionTestUtils.setField(service, "reviewMapper", reviews); + + IamRole ownerRole = new IamRole(); + ownerRole.setId(4000100L); + ownerRole.setTenantId(0L); + ownerRole.setAppCode("MERCHANT"); + ownerRole.setRoleCode("MERCHANT_OWNER"); + ownerRole.setStatus("ACTIVE"); + when(roles.selectOne(any())).thenReturn(ownerRole); + + var result = service.register(new MerchantRegistrationRequest( + 9001L, "注册人", "测试企业", "张三", "13800138000", + "owner@example.com", "CN", "深圳市宝安区")); + + ArgumentCaptor tenant = ArgumentCaptor.forClass(IamTenant.class); + ArgumentCaptor user = ArgumentCaptor.forClass(IamUser.class); + ArgumentCaptor membership = ArgumentCaptor.forClass(IamMembership.class); + ArgumentCaptor roleGrant = ArgumentCaptor.forClass(IamMembershipRole.class); + verify(tenants).insert(tenant.capture()); + verify(users).insert(user.capture()); + verify(memberships).insert(membership.capture()); + verify(membershipRoles).insert(roleGrant.capture()); + + assertEquals("PROFILE_REQUIRED", result.tenantStatus()); + assertEquals(result.tenantId(), tenant.getValue().getId()); + assertEquals("测试企业", tenant.getValue().getLegalName()); + assertEquals("张三", tenant.getValue().getLegalRepresentative()); + assertEquals("注册人", tenant.getValue().getContactName()); + assertEquals(9001L, user.getValue().getAuthUserId()); + assertEquals("ACTIVE", user.getValue().getStatus()); + assertEquals(result.tenantId(), membership.getValue().getTenantId()); + assertEquals(user.getValue().getId(), membership.getValue().getUserId()); + assertEquals("OWNER", membership.getValue().getMemberType()); + assertEquals("ACTIVE", membership.getValue().getStatus()); + assertEquals(result.membershipId(), membership.getValue().getId()); + assertEquals(result.membershipId(), roleGrant.getValue().getMembershipId()); + assertEquals(4000100L, roleGrant.getValue().getRoleId()); + assertEquals(result.tenantId(), roleGrant.getValue().getTenantId()); + assertNotNull(result.tenantId()); + } + + @Test + void compensationRemovesOnlyAnUnstartedRegistration() { + IamTenantMapper tenants = mock(IamTenantMapper.class); + IamUserMapper users = mock(IamUserMapper.class); + IamMembershipMapper memberships = mock(IamMembershipMapper.class); + IamRoleMapper roles = mock(IamRoleMapper.class); + IamMembershipRoleMapper membershipRoles = mock(IamMembershipRoleMapper.class); + IamTenantAddressMapper addresses = mock(IamTenantAddressMapper.class); + IamTenantQualificationMapper qualifications = mock(IamTenantQualificationMapper.class); + IamTenantReviewMapper reviews = mock(IamTenantReviewMapper.class); + MerchantOnboardingService service = new MerchantOnboardingService(); + ReflectionTestUtils.setField(service, "tenantMapper", tenants); + ReflectionTestUtils.setField(service, "userMapper", users); + ReflectionTestUtils.setField(service, "membershipMapper", memberships); + ReflectionTestUtils.setField(service, "roleMapper", roles); + ReflectionTestUtils.setField(service, "membershipRoleMapper", membershipRoles); + ReflectionTestUtils.setField(service, "addressMapper", addresses); + ReflectionTestUtils.setField(service, "qualificationMapper", qualifications); + ReflectionTestUtils.setField(service, "reviewMapper", reviews); + + IamUser user = new IamUser(); + user.setId(20001L); + user.setAuthUserId(9001L); + IamMembership membership = new IamMembership(); + membership.setId(30001L); + membership.setTenantId(10001L); + membership.setUserId(20001L); + membership.setMemberType("OWNER"); + IamTenant tenant = new IamTenant(); + tenant.setId(10001L); + tenant.setStatus("PROFILE_REQUIRED"); + when(users.selectOne(any())).thenReturn(user); + when(memberships.selectList(any())).thenReturn(java.util.List.of(membership)); + when(tenants.selectOne(any())).thenReturn(tenant); + when(qualifications.selectCount(any())).thenReturn(0L); + when(reviews.selectCount(any())).thenReturn(0L); + + service.compensateRegistration(9001L); + + verify(membershipRoles).delete(any()); + verify(memberships).deleteById(30001L); + verify(addresses).delete(any()); + verify(users).deleteById((java.io.Serializable) Long.valueOf(20001L)); + verify(tenants).deleteById((java.io.Serializable) Long.valueOf(10001L)); + } + + @Test + void compensationRefusesAfterQualificationUpload() { + IamTenantMapper tenants = mock(IamTenantMapper.class); + IamUserMapper users = mock(IamUserMapper.class); + IamMembershipMapper memberships = mock(IamMembershipMapper.class); + IamRoleMapper roles = mock(IamRoleMapper.class); + IamMembershipRoleMapper membershipRoles = mock(IamMembershipRoleMapper.class); + IamTenantAddressMapper addresses = mock(IamTenantAddressMapper.class); + IamTenantQualificationMapper qualifications = mock(IamTenantQualificationMapper.class); + IamTenantReviewMapper reviews = mock(IamTenantReviewMapper.class); + MerchantOnboardingService service = new MerchantOnboardingService(); + ReflectionTestUtils.setField(service, "tenantMapper", tenants); + ReflectionTestUtils.setField(service, "userMapper", users); + ReflectionTestUtils.setField(service, "membershipMapper", memberships); + ReflectionTestUtils.setField(service, "roleMapper", roles); + ReflectionTestUtils.setField(service, "membershipRoleMapper", membershipRoles); + ReflectionTestUtils.setField(service, "addressMapper", addresses); + ReflectionTestUtils.setField(service, "qualificationMapper", qualifications); + ReflectionTestUtils.setField(service, "reviewMapper", reviews); + + IamUser user = new IamUser(); + user.setId(20001L); + IamMembership membership = new IamMembership(); + membership.setId(30001L); + membership.setTenantId(10001L); + membership.setUserId(20001L); + membership.setMemberType("OWNER"); + IamTenant tenant = new IamTenant(); + tenant.setId(10001L); + tenant.setStatus("PROFILE_REQUIRED"); + when(users.selectOne(any())).thenReturn(user); + when(memberships.selectList(any())).thenReturn(java.util.List.of(membership)); + when(tenants.selectOne(any())).thenReturn(tenant); + when(qualifications.selectCount(any())).thenReturn(1L); + when(reviews.selectCount(any())).thenReturn(0L); + + assertThrows(com.baiyue.common.core.exception.BizException.class, + () -> service.compensateRegistration(9001L)); + verify(membershipRoles, never()).delete(any()); + verify(users, never()).deleteById((java.io.Serializable) any()); + verify(tenants, never()).deleteById((java.io.Serializable) any()); + } +} diff --git a/iam/src/test/java/com/baiyue/iam/service/MerchantQualificationUploadTest.java b/iam/src/test/java/com/baiyue/iam/service/MerchantQualificationUploadTest.java index c68a18c1b..e2ecbf970 100644 --- a/iam/src/test/java/com/baiyue/iam/service/MerchantQualificationUploadTest.java +++ b/iam/src/test/java/com/baiyue/iam/service/MerchantQualificationUploadTest.java @@ -12,11 +12,13 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.baiyue.common.core.exception.BizException; class MerchantQualificationUploadTest { @@ -43,4 +45,31 @@ class MerchantQualificationUploadTest { verify(profileRepository).insertQualification(any()); verify(tenantRepository, never()).insertReview(any()); } + + @Test + void businessLicenseRequiresAndPersistsAUniqueUnifiedSocialCreditCode() { + TenantRepository tenantRepository = mock(TenantRepository.class); + ProfileResourceRepository profileRepository = mock(ProfileResourceRepository.class); + ProfileResourceService service = new ProfileResourceService(); + ReflectionTestUtils.setField(service, "tenantRepository", tenantRepository); + ReflectionTestUtils.setField(service, "repository", profileRepository); + IamTenant tenant = new IamTenant(); + tenant.setId(10001L); + tenant.setTenantType("MERCHANT"); + tenant.setStatus("PROFILE_REQUIRED"); + when(tenantRepository.findTenantForUpdate(10001L)).thenReturn(tenant); + when(profileRepository.findQualificationsForUpdate(10001L)).thenReturn(List.of()); + + assertThrows(BizException.class, () -> service.submitQualification(new QualificationSubmissionRequest( + 10001L, "BUSINESS_LICENSE", null, + "/api/merchant/v1/files/qualifications/license.png", null, false))); + + var response = service.submitQualification(new QualificationSubmissionRequest( + 10001L, "BUSINESS_LICENSE", "91310101abcde12345", + "/api/merchant/v1/files/qualifications/license.png", null, false)); + + assertEquals("PROFILE_REQUIRED", response.tenantStatus()); + assertEquals("91310101ABCDE12345", tenant.getUnifiedSocialCreditCode()); + verify(tenantRepository).saveTenant(tenant); + } } diff --git a/iam/src/test/java/com/baiyue/iam/service/TenantAdministratorProvisioningServiceTest.java b/iam/src/test/java/com/baiyue/iam/service/TenantAdministratorProvisioningServiceTest.java new file mode 100644 index 000000000..eec5877a0 --- /dev/null +++ b/iam/src/test/java/com/baiyue/iam/service/TenantAdministratorProvisioningServiceTest.java @@ -0,0 +1,242 @@ +package com.baiyue.iam.service; + +import com.baiyue.common.core.api.ApiResponse; +import com.baiyue.iam.client.AuthAccountClient; +import com.baiyue.iam.entity.IamMembership; +import com.baiyue.iam.entity.IamRole; +import com.baiyue.iam.entity.IamTenant; +import com.baiyue.iam.entity.IamUser; +import com.baiyue.iam.mapper.IamRoleMapper; +import com.baiyue.iam.repository.MemberRepository; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.security.SecureRandom; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +class TenantAdministratorProvisioningServiceTest { + private final MemberRepository members = mock(MemberRepository.class); + private final IamRoleMapper roles = mock(IamRoleMapper.class); + private final AuthAccountClient auth = mock(AuthAccountClient.class); + private final TenantAdministratorProvisioningService service = + new TenantAdministratorProvisioningService(members, roles, auth, "internal-token", new SecureRandom()); + private IamTenant tenant; + + @BeforeEach + void setUp() { + tenant = new IamTenant(); + tenant.setId(190000000000000001L); + tenant.setCreatedBy(7000L); + tenant.setContactName("张三"); + tenant.setContactPhone("13800138000"); + tenant.setContactEmail("owner@example.com"); + IamRole ownerRole = new IamRole(); + ownerRole.setId(4000100L); + ownerRole.setTenantId(0L); + ownerRole.setAppCode("MERCHANT"); + ownerRole.setRoleCode("MERCHANT_OWNER"); + ownerRole.setStatus("ACTIVE"); + when(roles.selectOne(any())).thenReturn(ownerRole); + when(auth.create(anyString(), any())).thenAnswer(invocation -> { + AuthAccountClient.CreateAccountRequest request = invocation.getArgument(1); + return ApiResponse.success(new AuthAccountClient.AccountSummary( + request.accountId(), request.username(), request.displayName(), "ACTIVE")); + }); + when(auth.compensate(anyString(), anyLong())).thenReturn(ApiResponse.success(null)); + } + + @Test + void missingOwnerCreatesAccountProfileMembershipAndOwnerRole() { + var result = service.initialize(context()); + + assertTrue(result.created()); + assertTrue(result.username().matches("^merchant_190000000000000001_[a-z]{8}$")); + assertNotNull(result.temporaryPassword()); + assertEquals(16, result.temporaryPassword().length()); + assertTrue(result.temporaryPassword().matches("^(?=.*[A-Za-z])(?=.*\\d)[A-Za-z0-9]+$")); + + ArgumentCaptor account = + ArgumentCaptor.forClass(AuthAccountClient.CreateAccountRequest.class); + verify(auth).create(eq("internal-token"), account.capture()); + assertEquals(result.username(), account.getValue().username()); + assertEquals(result.temporaryPassword(), account.getValue().password()); + assertEquals("MERCHANT", account.getValue().appCode()); + assertEquals(tenant.getId(), account.getValue().tenantId()); + verify(members).insertUser(any(IamUser.class)); + verify(members).insert(any(IamMembership.class)); + verify(members).replaceRoles(eq(tenant.getId()), anyLong(), eq(List.of(4000100L))); + } + + @Test + void existingOwnerSkipsProvisioningAndDoesNotReturnCredentials() { + IamMembership owner = activeOwner(); + IamUser user = activeUser(); + when(members.findOwnerByTenant(tenant.getId())).thenReturn(owner); + when(members.findUser(owner.getUserId())).thenReturn(user); + when(auth.summary("internal-token", user.getAuthUserId())).thenReturn(ApiResponse.success( + new AuthAccountClient.AccountSummary(user.getAuthUserId(), "existing.owner", "张三", "ACTIVE", + null, tenant.getId(), owner.getId(), "MERCHANT"))); + when(members.findRoleIds(tenant.getId(), owner.getId())).thenReturn(List.of(4000100L)); + + var result = service.initialize(context()); + + assertFalse(result.created()); + assertEquals(null, result.username()); + assertEquals(null, result.temporaryPassword()); + verify(auth, never()).create(anyString(), any()); + verify(members, never()).replaceRoles(anyLong(), anyLong(), any()); + } + + @Test + void existingRegisteredOwnerKeepsAccountAndRepairsMissingAdministratorRole() { + IamMembership owner = activeOwner(); + IamUser user = activeUser(); + when(members.findOwnerByTenant(tenant.getId())).thenReturn(owner); + when(members.findUser(owner.getUserId())).thenReturn(user); + when(auth.summary("internal-token", user.getAuthUserId())).thenReturn(ApiResponse.success( + new AuthAccountClient.AccountSummary(user.getAuthUserId(), "registered.owner", "张三", "ACTIVE", + null, tenant.getId(), owner.getId(), "MERCHANT"))); + when(members.findRoleIds(tenant.getId(), owner.getId())).thenReturn(List.of(4000200L)); + + var result = service.initialize(context()); + + assertFalse(result.created()); + verify(auth, never()).create(anyString(), any()); + verify(members).replaceRoles(tenant.getId(), owner.getId(), List.of(4000200L, 4000100L)); + } + + @Test + void clientRegistrationWithoutOriginalOwnerIsRejectedInsteadOfCreatingAnotherAdministrator() { + tenant.setCreatedBy(null); + tenant.setUnifiedSocialCreditCode("91330100EXAMPLE001"); + + assertThrows(com.baiyue.common.core.exception.BizException.class, + () -> service.initialize(context())); + + verify(auth, never()).create(anyString(), any()); + verify(members, never()).insertUser(any()); + verify(members, never()).insert(any()); + } + + @Test + void ownerWithoutAuthAccountRepairsAccountUsingExistingIdentity() { + IamMembership owner = activeOwner(); + IamUser user = activeUser(); + when(members.findOwnerByTenant(tenant.getId())).thenReturn(owner); + when(members.findUser(owner.getUserId())).thenReturn(user); + when(auth.summary("internal-token", user.getAuthUserId())).thenReturn( + ApiResponse.failure(com.baiyue.common.core.api.ResultCode.NOT_FOUND, + "账号不存在", null, null)); + + var result = service.initialize(context()); + + assertTrue(result.created()); + ArgumentCaptor account = + ArgumentCaptor.forClass(AuthAccountClient.CreateAccountRequest.class); + verify(auth).create(eq("internal-token"), account.capture()); + assertEquals(user.getAuthUserId(), account.getValue().accountId()); + assertEquals(owner.getId(), account.getValue().membershipId()); + assertEquals(tenant.getId(), account.getValue().tenantId()); + verify(members, never()).insertUser(any()); + verify(members, never()).insert(any()); + verify(members, never()).replaceRoles(anyLong(), anyLong(), any()); + } + + @Test + void disabledOwnerPreventsApprovalInsteadOfSilentlySkippingProvisioning() { + IamMembership owner = activeOwner(); + owner.setStatus("DISABLED"); + when(members.findOwnerByTenant(tenant.getId())).thenReturn(owner); + + assertThrows(com.baiyue.common.core.exception.BizException.class, + () -> service.initialize(context())); + + verify(auth, never()).summary(anyString(), anyLong()); + verify(auth, never()).create(anyString(), any()); + } + + @Test + void mismatchedAuthScopePreventsApproval() { + IamMembership owner = activeOwner(); + IamUser user = activeUser(); + when(members.findOwnerByTenant(tenant.getId())).thenReturn(owner); + when(members.findUser(owner.getUserId())).thenReturn(user); + when(auth.summary("internal-token", user.getAuthUserId())).thenReturn(ApiResponse.success( + new AuthAccountClient.AccountSummary(user.getAuthUserId(), "existing.owner", "张三", "ACTIVE", + null, tenant.getId() + 1, owner.getId(), "MERCHANT"))); + + assertThrows(com.baiyue.common.core.exception.BizException.class, + () -> service.initialize(context())); + + verify(auth, never()).create(anyString(), any()); + } + + @Test + void localWriteFailureCompensatesCreatedAuthAccount() { + doThrow(new RuntimeException("IAM write failed")).when(members).insertUser(any()); + + assertThrows(RuntimeException.class, () -> service.initialize(context())); + + verify(auth).compensate(eq("internal-token"), anyLong()); + verify(members, never()).insert(any(IamMembership.class)); + } + + @Test + void usernameConflictGeneratesAnotherUsernameBeforeCreatingOwner() { + org.mockito.Mockito.doReturn(ApiResponse.failure(com.baiyue.common.core.api.ResultCode.CONFLICT, + "登录账号已存在", null, null)) + .doAnswer(invocation -> { + AuthAccountClient.CreateAccountRequest request = invocation.getArgument(1); + return ApiResponse.success(new AuthAccountClient.AccountSummary( + request.accountId(), request.username(), request.displayName(), "ACTIVE")); + }).when(auth).create(anyString(), any()); + + var result = service.initialize(context()); + + assertTrue(result.created()); + ArgumentCaptor accounts = + ArgumentCaptor.forClass(AuthAccountClient.CreateAccountRequest.class); + verify(auth, times(2)).create(eq("internal-token"), accounts.capture()); + assertFalse(accounts.getAllValues().get(0).username().equals(accounts.getAllValues().get(1).username())); + assertEquals(accounts.getAllValues().get(1).username(), result.username()); + } + + private TenantApprovalInitializer.ApprovalContext context() { + return new TenantApprovalInitializer.ApprovalContext(tenant, 71L, 9001L, "资料完整"); + } + + private IamMembership activeOwner() { + IamMembership owner = new IamMembership(); + owner.setId(7001L); + owner.setTenantId(tenant.getId()); + owner.setUserId(8001L); + owner.setMemberType("OWNER"); + owner.setStatus("ACTIVE"); + return owner; + } + + private IamUser activeUser() { + IamUser user = new IamUser(); + user.setId(8001L); + user.setAuthUserId(9001L); + user.setStatus("ACTIVE"); + return user; + } +} diff --git a/iam/src/test/java/com/baiyue/iam/service/TenantApprovalInitializationCoordinatorTest.java b/iam/src/test/java/com/baiyue/iam/service/TenantApprovalInitializationCoordinatorTest.java new file mode 100644 index 000000000..6483a5283 --- /dev/null +++ b/iam/src/test/java/com/baiyue/iam/service/TenantApprovalInitializationCoordinatorTest.java @@ -0,0 +1,95 @@ +package com.baiyue.iam.service; + +import com.baiyue.iam.entity.IamTenant; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.springframework.core.annotation.Order; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TenantApprovalInitializationCoordinatorTest { + @Test + void initializerFailureCompensatesCompletedStepsInReverseOrder() { + TenantApprovalInitializer first = initializer("rules"); + TenantApprovalInitializer second = initializer("fees"); + TenantApprovalInitializer failing = initializer("roles"); + TenantApprovalInitializer.InitializationResult firstResult = new TestResult("rules"); + TenantApprovalInitializer.InitializationResult secondResult = new TestResult("fees"); + var context = context(); + when(first.initialize(context)).thenReturn(firstResult); + when(second.initialize(context)).thenReturn(secondResult); + when(failing.initialize(context)).thenThrow(new RuntimeException("role initialization failed")); + var coordinator = new TenantApprovalInitializationCoordinator(List.of(first, second, failing)); + + assertThrows(RuntimeException.class, () -> coordinator.initialize(context)); + + InOrder compensationOrder = inOrder(second, first); + compensationOrder.verify(second).compensate(context, secondResult); + compensationOrder.verify(first).compensate(context, firstResult); + } + + @Test + void completedBatchExposesTypedResultsForApprovalResponseAssembly() { + TenantApprovalInitializer initializer = initializer("administrator"); + TestResult result = new TestResult("credential-result"); + var context = context(); + when(initializer.initialize(context)).thenReturn(result); + var coordinator = new TenantApprovalInitializationCoordinator(List.of(initializer)); + + var batch = coordinator.initialize(context); + + assertEquals(result, batch.result(TestResult.class).orElseThrow()); + } + + @Test + void springOrderControlsInitializerExecutionSequence() { + List calls = new ArrayList<>(); + var coordinator = new TenantApprovalInitializationCoordinator( + List.of(new LateInitializer(calls), new EarlyInitializer(calls))); + + coordinator.initialize(context()); + + assertEquals(List.of("early", "late"), calls); + } + + private TenantApprovalInitializer initializer(String name) { + TenantApprovalInitializer initializer = mock(TenantApprovalInitializer.class); + when(initializer.name()).thenReturn(name); + return initializer; + } + + private TenantApprovalInitializer.ApprovalContext context() { + IamTenant tenant = new IamTenant(); + tenant.setId(10001L); + return new TenantApprovalInitializer.ApprovalContext(tenant, 71L, 9001L, "资料完整"); + } + + private record TestResult(String value) implements TenantApprovalInitializer.InitializationResult { } + + @Order(10) + private static final class EarlyInitializer implements TenantApprovalInitializer { + private final List calls; + private EarlyInitializer(List calls) { this.calls = calls; } + public String name() { return "early"; } + public InitializationResult initialize(ApprovalContext context) { + calls.add("early"); return new TestResult("early"); + } + } + + @Order(200) + private static final class LateInitializer implements TenantApprovalInitializer { + private final List calls; + private LateInitializer(List calls) { this.calls = calls; } + public String name() { return "late"; } + public InitializationResult initialize(ApprovalContext context) { + calls.add("late"); return new TestResult("late"); + } + } +} diff --git a/iam/src/test/java/com/baiyue/iam/service/TenantServiceTest.java b/iam/src/test/java/com/baiyue/iam/service/TenantServiceTest.java index 1197047b4..535486d6b 100644 --- a/iam/src/test/java/com/baiyue/iam/service/TenantServiceTest.java +++ b/iam/src/test/java/com/baiyue/iam/service/TenantServiceTest.java @@ -6,6 +6,7 @@ import com.baiyue.iam.dto.TenantDtos.ChangeTenantStatusRequest; import com.baiyue.iam.dto.TenantDtos.UpdateTenantRequest; import com.baiyue.iam.dto.TenantDtos.RegisterRequest; import com.baiyue.iam.entity.IamTenantQualification; +import com.baiyue.iam.entity.IamTenantReview; import com.baiyue.iam.repository.TenantRepository; import com.baiyue.iam.repository.ProfileResourceRepository; import org.junit.jupiter.api.Test; @@ -21,7 +22,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.doThrow; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.junit.jupiter.api.Assertions.assertThrows; import com.baiyue.common.core.exception.BizException; import com.baiyue.common.core.api.ResultCode; @@ -75,12 +79,13 @@ class TenantServiceTest { when(repository.findTenant(10001L)).thenReturn(tenant); var updated = service.updateMerchant(10001L, - new UpdateTenantRequest("New name", "New legal", "Alice", "13900139000", "alice@example.com", + new UpdateTenantRequest("New name", "New legal", "Bob", "Alice", "13900139000", "alice@example.com", "专业版", "alice_wechat", "Sales Zhang", 45, "LIMITED_COMPANY", LocalDate.of(2020, 1, 2), new BigDecimal("5000000.00"), "CNY", "深圳市南山区示例路 1 号", "一期重点客户")); var disabled = service.changeMerchantStatus(10001L, new ChangeTenantStatusRequest("DISABLED")); assertEquals("New name", updated.tenantName()); + assertEquals("Bob", updated.legalRepresentative()); assertEquals("alice_wechat", updated.contactWechat()); assertEquals("专业版", updated.servicePackage()); assertEquals("Sales Zhang", updated.salesOwnerName()); @@ -115,12 +120,16 @@ class TenantServiceTest { ReflectionTestUtils.setField(service, "repository", repository); var created = service.register(new RegisterRequest( - "Example Merchant", "Example Merchant Ltd", "91330100EXAMPLE001", "Alice", "13800138000", + "Example Merchant", "Example Merchant Ltd", "Bob", "91330100EXAMPLE001", "Alice", "13800138000", "alice@example.com", "基础版", "alice_wechat", "Sales Zhang", 30, "LIMITED_COMPANY", - LocalDate.of(2020, 1, 2), new BigDecimal("5000000.00"), "CNY", "深圳市南山区", "草稿客户")); + LocalDate.of(2020, 1, 2), new BigDecimal("5000000.00"), "CNY", "深圳市南山区", "草稿客户"), + 9001L); assertEquals("PROFILE_REQUIRED", created.status()); + assertEquals("Bob", created.legalRepresentative()); assertEquals("基础版", created.servicePackage()); + verify(repository).insertTenant(argThat(tenant -> Long.valueOf(9001L).equals(tenant.getCreatedBy()) + && Long.valueOf(9001L).equals(tenant.getUpdatedBy()))); verify(repository, never()).insertReview(any()); } @@ -162,14 +171,132 @@ class TenantServiceTest { } @Test - void 企业激活不等于具备下单资格仍需已审核资质() { + void explicitSubmissionRejectsMissingUnifiedSocialCreditCodeBeforeCreatingReview() { + TenantRepository repository = mock(TenantRepository.class); + ProfileResourceRepository profileRepository = mock(ProfileResourceRepository.class); + TenantService service = new TenantService(); + ReflectionTestUtils.setField(service, "repository", repository); + ReflectionTestUtils.setField(service, "profileRepository", profileRepository); + IamTenant tenant = merchant(10001L, "PROFILE_REQUIRED"); + tenant.setUnifiedSocialCreditCode(null); + when(repository.findTenantForUpdate(10001L)).thenReturn(tenant); + + BizException exception = assertThrows(BizException.class, () -> service.submitMerchantReview(10001L)); + + assertEquals(ResultCode.BAD_REQUEST, exception.getResultCode()); + verify(repository, never()).insertReview(any()); + verify(profileRepository, never()).findQualifications(10001L); + } + + @Test + void approvalProvisionsMissingAdministratorAndReturnsCredentialsOnce() { + TenantRepository repository = mock(TenantRepository.class); + ProfileResourceRepository profileRepository = mock(ProfileResourceRepository.class); + TenantAdministratorProvisioningService provisioning = mock(TenantAdministratorProvisioningService.class); + when(provisioning.name()).thenReturn("tenant-administrator"); + TenantApprovalInitializationCoordinator initialization = + new TenantApprovalInitializationCoordinator(List.of(provisioning)); + TenantService service = new TenantService(); + ReflectionTestUtils.setField(service, "repository", repository); + ReflectionTestUtils.setField(service, "profileRepository", profileRepository); + ReflectionTestUtils.setField(service, "approvalInitialization", initialization); + IamTenantReview review = review(71L, 10001L); + IamTenant tenant = merchant(10001L, "PENDING"); + when(repository.findReviewForUpdate(71L)).thenReturn(review); + when(repository.findTenant(10001L)).thenReturn(tenant); + when(profileRepository.findQualifications(10001L)).thenReturn(List.of( + qualification("BUSINESS_LICENSE"), qualification("LEGAL_PERSON_ID"), + qualification("FREIGHT_FORWARDER_CERTIFICATE"), qualification("BANK_ACCOUNT_LICENSE"))); + when(provisioning.initialize(any())).thenReturn( + TenantAdministratorProvisioningService.ProvisioningResult.created( + 81L, "merchant_10001", "Password1234")); + + var result = service.review(71L, 9001L, true, "资料完整"); + + assertEquals(10001L, result.tenantId()); + assertEquals(true, result.administratorCreated()); + assertEquals("merchant_10001", result.administratorUsername()); + assertEquals("Password1234", result.temporaryPassword()); + assertEquals("APPROVED", review.getStatus()); + assertEquals("ACTIVE", tenant.getStatus()); + verify(repository).saveReview(review); + verify(repository).saveTenant(tenant); + } + + @Test + void approvalKeepsClientRegisteredAccountAsAdministratorWithoutNewCredentials() { + TenantRepository repository = mock(TenantRepository.class); + ProfileResourceRepository profileRepository = mock(ProfileResourceRepository.class); + TenantAdministratorProvisioningService provisioning = mock(TenantAdministratorProvisioningService.class); + when(provisioning.name()).thenReturn("tenant-administrator"); + TenantApprovalInitializationCoordinator initialization = + new TenantApprovalInitializationCoordinator(List.of(provisioning)); + TenantService service = new TenantService(); + ReflectionTestUtils.setField(service, "repository", repository); + ReflectionTestUtils.setField(service, "profileRepository", profileRepository); + ReflectionTestUtils.setField(service, "approvalInitialization", initialization); + IamTenantReview review = review(71L, 10001L); + IamTenant tenant = merchant(10001L, "PENDING"); + when(repository.findReviewForUpdate(71L)).thenReturn(review); + when(repository.findTenant(10001L)).thenReturn(tenant); + when(profileRepository.findQualifications(10001L)).thenReturn(List.of( + qualification("BUSINESS_LICENSE"), qualification("LEGAL_PERSON_ID"), + qualification("FREIGHT_FORWARDER_CERTIFICATE"), qualification("BANK_ACCOUNT_LICENSE"))); + when(provisioning.initialize(any())).thenReturn( + TenantAdministratorProvisioningService.ProvisioningResult.existing()); + + var result = service.review(71L, 9001L, true, "资料完整"); + + assertEquals(false, result.administratorCreated()); + assertEquals(null, result.administratorUsername()); + assertEquals(null, result.temporaryPassword()); + assertEquals("APPROVED", review.getStatus()); + assertEquals("ACTIVE", tenant.getStatus()); + } + + @Test + void failureAfterProvisioningCompensatesAdministratorAndKeepsApprovalFailed() { + TenantRepository repository = mock(TenantRepository.class); + ProfileResourceRepository profileRepository = mock(ProfileResourceRepository.class); + TenantAdministratorProvisioningService provisioning = mock(TenantAdministratorProvisioningService.class); + when(provisioning.name()).thenReturn("tenant-administrator"); + TenantApprovalInitializationCoordinator initialization = + new TenantApprovalInitializationCoordinator(List.of(provisioning)); + TenantService service = new TenantService(); + ReflectionTestUtils.setField(service, "repository", repository); + ReflectionTestUtils.setField(service, "profileRepository", profileRepository); + ReflectionTestUtils.setField(service, "approvalInitialization", initialization); + IamTenantReview review = review(71L, 10001L); + IamTenant tenant = merchant(10001L, "PENDING"); + var administrator = TenantAdministratorProvisioningService.ProvisioningResult.created( + 81L, "merchant_10001", "Password1234"); + when(repository.findReviewForUpdate(71L)).thenReturn(review); + when(repository.findTenant(10001L)).thenReturn(tenant); + when(profileRepository.findQualifications(10001L)).thenReturn(List.of( + qualification("BUSINESS_LICENSE"), qualification("LEGAL_PERSON_ID"), + qualification("FREIGHT_FORWARDER_CERTIFICATE"), qualification("BANK_ACCOUNT_LICENSE"))); + when(provisioning.initialize(any())).thenReturn(administrator); + doThrow(new RuntimeException("review write failed")).when(repository).saveReview(review); + + assertThrows(RuntimeException.class, () -> service.review(71L, 9001L, true, "资料完整")); + + verify(provisioning).compensate(any(), eq(administrator)); + verify(repository, never()).saveTenant(tenant); + } + + @Test + void 企业激活不等于具备下单资格仍需四类资质全部审核通过() { TenantRepository repository = mock(TenantRepository.class); ProfileResourceRepository profileRepository = mock(ProfileResourceRepository.class); TenantService service = new TenantService(); ReflectionTestUtils.setField(service, "repository", repository); ReflectionTestUtils.setField(service, "profileRepository", profileRepository); when(repository.findTenant(10001L)).thenReturn(merchant(10001L, "ACTIVE")); - when(profileRepository.countApprovedQualifications(10001L)).thenReturn(0L, 1L); + when(profileRepository.findQualifications(10001L)).thenReturn( + List.of(approvedQualification("BUSINESS_LICENSE")), + List.of(approvedQualification("BUSINESS_LICENSE"), approvedQualification("LEGAL_PERSON_ID"), + approvedQualification("FREIGHT_FORWARDER_CERTIFICATE"), + approvedQualification("BANK_ACCOUNT_LICENSE"))); var withoutQualification = service.orderEligibility(10001L); var withQualification = service.orderEligibility(10001L); @@ -198,4 +325,16 @@ class TenantServiceTest { item.setQualificationType(type); item.setStatus("PENDING"); return item; } + + private IamTenantQualification approvedQualification(String type) { + IamTenantQualification item = qualification(type); + item.setStatus("APPROVED"); + return item; + } + + private IamTenantReview review(long id, long tenantId) { + IamTenantReview review = new IamTenantReview(); + review.setId(id); review.setTenantId(tenantId); review.setStatus("PENDING"); + return review; + } } diff --git a/iam/src/test/resources/db/verify-phase1-admin-rbac.sql b/iam/src/test/resources/db/verify-phase1-admin-rbac.sql index b9a2df7b1..e2942472f 100644 --- a/iam/src/test/resources/db/verify-phase1-admin-rbac.sql +++ b/iam/src/test/resources/db/verify-phase1-admin-rbac.sql @@ -36,4 +36,24 @@ WHERE (SELECT COUNT(*) FROM baiyue_iam.iam_role_permission WHERE role_id = 40000 UNION ALL SELECT 'merchant_owner_not_fully_granted' WHERE (SELECT COUNT(*) FROM baiyue_iam.iam_role_permission WHERE role_id = 4000101) - <> (SELECT COUNT(*) FROM baiyue_iam.iam_permission WHERE tenant_id = 10001 AND app_code = 'MERCHANT' AND permission_type IN ('MENU', 'BUTTON')); + <> (SELECT COUNT(*) FROM baiyue_iam.iam_permission WHERE tenant_id = 10001 AND app_code = 'MERCHANT' AND permission_type IN ('MENU', 'BUTTON')) +UNION ALL +SELECT 'missing_global_merchant_owner_template' +WHERE NOT EXISTS ( + SELECT 1 FROM baiyue_iam.iam_role + WHERE tenant_id = 0 AND app_code = 'MERCHANT' AND role_code = 'MERCHANT_OWNER' + AND role_type = 'SYSTEM' AND status = 'ACTIVE' AND deleted = 0 +) +UNION ALL +SELECT 'global_merchant_owner_not_fully_granted' +WHERE ( + SELECT COUNT(*) + FROM baiyue_iam.iam_role_permission role_permission + JOIN baiyue_iam.iam_role owner_role ON owner_role.id = role_permission.role_id + WHERE owner_role.tenant_id = 0 AND owner_role.app_code = 'MERCHANT' + AND owner_role.role_code = 'MERCHANT_OWNER' +) <> ( + SELECT COUNT(*) FROM baiyue_iam.iam_permission + WHERE tenant_id = 0 AND app_code = 'MERCHANT' + AND permission_type IN ('MENU', 'BUTTON') AND status = 'ACTIVE' AND deleted = 0 +); diff --git a/order/src/main/java/com/baiyue/order/service/OrderApplicationService.java b/order/src/main/java/com/baiyue/order/service/OrderApplicationService.java index 05b949560..3a4228183 100644 --- a/order/src/main/java/com/baiyue/order/service/OrderApplicationService.java +++ b/order/src/main/java/com/baiyue/order/service/OrderApplicationService.java @@ -45,7 +45,6 @@ public class OrderApplicationService { } private Long createDraftInternal(long tenantId, long userId, OrderRequests.SaveRequest request) { - eligibilityGuard.requireApproved(tenantId); validatePrototypeFields(request); var snapshots = masterDataGuard.validate(tenantId, request); OrderEntity order = new OrderEntity(); @@ -61,7 +60,6 @@ public class OrderApplicationService { @Transactional public void updateDraft(long tenantId, long userId, long id, OrderRequests.SaveRequest request) { OrderEntity order = require(tenantId, id); - eligibilityGuard.requireApproved(tenantId); validatePrototypeFields(request); var snapshots = masterDataGuard.validate(tenantId, request); if (order.getStatus() != OrderStatus.DRAFT && order.getStatus() != OrderStatus.REJECTED) { diff --git a/order/src/test/java/com/baiyue/order/service/OrderApplicationServiceStateTest.java b/order/src/test/java/com/baiyue/order/service/OrderApplicationServiceStateTest.java index 7844d3e0f..92b2260ee 100644 --- a/order/src/test/java/com/baiyue/order/service/OrderApplicationServiceStateTest.java +++ b/order/src/test/java/com/baiyue/order/service/OrderApplicationServiceStateTest.java @@ -1,6 +1,10 @@ package com.baiyue.order.service; +import com.baiyue.common.core.api.ResultCode; import com.baiyue.common.core.exception.BizException; +import com.baiyue.order.client.PartnerClient; +import com.baiyue.order.client.WarehouseClient; +import com.baiyue.order.dto.OrderRequests; import com.baiyue.order.entity.OrderEntity; import com.baiyue.order.entity.OrderStatus; import com.baiyue.order.repository.OrderRepository; @@ -8,11 +12,76 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.test.util.ReflectionTestUtils; + +import java.math.BigDecimal; +import java.time.LocalTime; +import java.util.List; + import static org.assertj.core.api.Assertions.*; import static org.mockito.Mockito.*; @DisplayName("订单状态流转") class OrderApplicationServiceStateTest { + @Test + void unapprovedTenantCanCreateDraft() { + OrderRepository repository = mock(OrderRepository.class); + OrderEligibilityGuard eligibility = mock(OrderEligibilityGuard.class); + OrderMasterDataGuard masterData = mock(OrderMasterDataGuard.class); + OrderRequests.SaveRequest request = request(); + when(masterData.validate(10001L, request)).thenReturn(snapshots()); + when(repository.completeIdempotency(any(), anyLong())).thenReturn(1); + + Long orderId = service(repository, eligibility, masterData) + .createDraft(10001L, 2L, request, "draft-before-approval"); + + assertThat(orderId).isNotNull(); + verify(eligibility, never()).requireApproved(anyLong()); + verify(repository).insertOrder(argThat(order -> order.getStatus() == OrderStatus.DRAFT)); + } + + @Test + void unapprovedTenantCanUpdateDraft() { + OrderRepository repository = mock(OrderRepository.class); + OrderEligibilityGuard eligibility = mock(OrderEligibilityGuard.class); + OrderMasterDataGuard masterData = mock(OrderMasterDataGuard.class); + OrderRequests.SaveRequest request = request(); + OrderEntity order = new OrderEntity(); + order.setId(1L); + order.setTenantId(10001L); + order.setStatus(OrderStatus.DRAFT); + when(repository.findOwned(10001L, 1L)).thenReturn(order); + when(masterData.validate(10001L, request)).thenReturn(snapshots()); + when(repository.saveOrder(order)).thenReturn(1); + + service(repository, eligibility, masterData).updateDraft(10001L, 2L, 1L, request); + + verify(eligibility, never()).requireApproved(anyLong()); + verify(repository).replaceDraftChildren(10001L, 1L); + verify(repository).saveOrder(argThat(saved -> saved.getStatus() == OrderStatus.DRAFT)); + } + + @Test + void unapprovedTenantStillCannotSubmitDraft() { + OrderRepository repository = mock(OrderRepository.class); + OrderEligibilityGuard eligibility = mock(OrderEligibilityGuard.class); + OrderEntity order = new OrderEntity(); + order.setId(1L); + order.setTenantId(10001L); + order.setStatus(OrderStatus.DRAFT); + when(repository.findOwned(10001L, 1L)).thenReturn(order); + doThrow(new BizException(ResultCode.FORBIDDEN, "企业尚未通过审核")) + .when(eligibility).requireApproved(10001L); + + assertThatThrownBy(() -> service(repository, eligibility, mock(OrderMasterDataGuard.class)) + .submit(10001L, 2L, "u", 1L, "submit-before-approval")) + .isInstanceOf(BizException.class) + .extracting(exception -> ((BizException) exception).getResultCode()) + .isEqualTo(ResultCode.FORBIDDEN); + + verify(eligibility).requireApproved(10001L); + verify(repository, never()).saveOrder(any()); + } + @Test void repeatedSubmissionReturnsConflictWithoutDuplicatePersistence() { OrderRepository repository=mock(OrderRepository.class); @@ -28,4 +97,36 @@ class OrderApplicationServiceStateTest { .isInstanceOf(BizException.class).hasMessageContaining("重复提交"); verify(repository,never()).saveOrder(any()); } + + private OrderApplicationService service(OrderRepository repository, OrderEligibilityGuard eligibility, + OrderMasterDataGuard masterData) { + OrderApplicationService service = new OrderApplicationService(); + ReflectionTestUtils.setField(service, "repository", repository); + ReflectionTestUtils.setField(service, "eligibilityGuard", eligibility); + ReflectionTestUtils.setField(service, "masterDataGuard", masterData); + ReflectionTestUtils.setField(service, "json", new ObjectMapper()); + return service; + } + + private OrderRequests.SaveRequest request() { + var party = new OrderRequests.Party( + "公司", "联系人", "13800138000", null, "CN", null, "深圳", null, "地址", null); + var cargo = new OrderRequests.Cargo( + "货物", null, "CARTON", 1, BigDecimal.ONE, BigDecimal.ONE, + null, null, null, null, null, false, null, null, null, null, + null, null, null, null, null, null, null, null, null); + return new OrderRequests.SaveRequest( + 7L, "客户", "SEA", null, "US", null, "目的仓", null, null, + party, party, List.of(cargo), "PICKUP", 8L, "收货仓", "CH", "渠道", + "BUY_DECLARATION", "LCL", false, "OVERSEAS", null, null, null); + } + + private OrderMasterDataGuard.Snapshots snapshots() { + var partner = new PartnerClient.Partner(7L, 10001L, "客户", "ACTIVE", "SHIPPER", List.of("SHIPPER")); + var warehouse = new WarehouseClient.Warehouse(8L, 10001L, "收货仓", "ACTIVE"); + var address = new WarehouseClient.Address("CN", "广东", "深圳", "南山", "518000", "测试路1号", null); + var capability = new WarehouseClient.Capability("RECEIVING", true, LocalTime.of(16, 0)); + var receiving = new WarehouseClient.Detail(warehouse, List.of(address), List.of(capability)); + return new OrderMasterDataGuard.Snapshots(partner, receiving, null); + } } diff --git a/platform-bom/pom.xml b/platform-bom/pom.xml index a60379d84..508fdb15e 100644 --- a/platform-bom/pom.xml +++ b/platform-bom/pom.xml @@ -19,6 +19,7 @@ com.baiyue.sharedbaiyue-common-openapi${project.version} com.baiyue.sharedbaiyue-common-audit${project.version} com.baiyue.sharedcommon-event-contract${project.version} + org.apache.rocketmqrocketmq-spring-boot-starter2.3.3 diff --git a/pom.xml b/pom.xml index 01ba974a2..6fa5772a2 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 2.5.0 3.13.0 3.4.1 + 2.3.3 @@ -86,6 +87,11 @@ springdoc-openapi-starter-webmvc-ui ${springdoc.version} + + org.apache.rocketmq + rocketmq-spring-boot-starter + ${rocketmq.spring.version} + org.springdoc springdoc-openapi-starter-webflux-ui diff --git a/scripts/export-center-test-env-acceptance.ps1 b/scripts/export-center-test-env-acceptance.ps1 new file mode 100644 index 000000000..8f47229c9 --- /dev/null +++ b/scripts/export-center-test-env-acceptance.ps1 @@ -0,0 +1,171 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern('^https?://')] + [string]$GatewayBaseUrl, + + [string[]]$AccessTokens = @(), + [ValidateRange(1, 4)] + [int]$TaskCount = 1, + [string[]]$TaskIds = @(), + [ValidateRange(1, 7200)] + [int]$TimeoutSeconds = 3600, + [ValidateRange(1, 60)] + [int]$PollSeconds = 2, + [ValidateRange(0, 200000)] + [int]$MinimumRows = 0, + [string]$Keyword = '', + [ValidateSet('', 'PROFILE_REQUIRED', 'PENDING', 'ACTIVE', 'DISABLED', 'REJECTED')] + [string]$Status = '', + [string]$OutputDirectory = '', + [switch]$CreateOnly, + [switch]$SkipDownload +) + +$ErrorActionPreference = 'Stop' +$baseUrl = $GatewayBaseUrl.TrimEnd('/') +$apiPath = "$baseUrl/api/platform/v1/files/export-tasks" + +if ($AccessTokens.Count -eq 0 -and -not [string]::IsNullOrWhiteSpace($env:ADMIN_ACCESS_TOKEN)) { + $AccessTokens = @($env:ADMIN_ACCESS_TOKEN) +} +if ($AccessTokens.Count -eq 0) { + throw '至少提供一个 -AccessTokens，或通过 ADMIN_ACCESS_TOKEN 环境变量注入。' +} +if ($TaskIds.Count -eq 0 -and $AccessTokens.Count -lt $TaskCount) { + throw "创建 $TaskCount 个并发任务需要至少 $TaskCount 个不同用户的 Access Token，避免 60 秒重复提交合并。" +} +if ($TaskIds.Count -gt 0 -and $AccessTokens.Count -lt $TaskIds.Count) { + throw '恢复验收时每个 TaskId 都需要对应申请人的 Access Token。' +} + +function Get-Headers([string]$Token) { + return @{ Authorization = "Bearer $Token" } +} + +function Invoke-ApiJson([string]$Method, [string]$Uri, [string]$Token, [object]$Body = $null) { + $arguments = @{ + Method = $Method + Uri = $Uri + Headers = Get-Headers $Token + TimeoutSec = 60 + } + if ($null -ne $Body) { + $arguments.ContentType = 'application/json' + $arguments.Body = $Body | ConvertTo-Json -Depth 10 -Compress + } + $response = Invoke-RestMethod @arguments + if ($null -eq $response -or [string]$response.code -ne '200') { + $requestId = if ($response.requestId) { " requestId=$($response.requestId)" } else { '' } + $traceId = if ($response.traceId) { " traceId=$($response.traceId)" } else { '' } + throw "API 失败 method=$Method uri=$Uri code=$($response.code) message=$($response.message)$requestId$traceId" + } + return $response.data +} + +function New-ExportTask([string]$Token, [int]$Index) { + $parameters = [ordered]@{} + if (-not [string]::IsNullOrWhiteSpace($Keyword)) { $parameters.keyword = $Keyword.Trim() } + if (-not [string]::IsNullOrWhiteSpace($Status)) { $parameters.status = $Status } + $payload = @{ + exportType = 'CUSTOMER_LIST' + format = 'XLSX' + taskName = "客户列表验收-$Index" + parameters = $parameters + } + return Invoke-ApiJson POST $apiPath $Token $payload +} + +function Wait-ExportTask([string]$TaskId, [string]$Token) { + $deadline = [DateTimeOffset]::UtcNow.AddSeconds($TimeoutSeconds) + $lastStatus = '' + while ([DateTimeOffset]::UtcNow -lt $deadline) { + $task = Invoke-ApiJson GET "$apiPath/$TaskId" $Token + if ($task.status -ne $lastStatus) { + Write-Host "taskId=$TaskId status=$($task.status) processed=$($task.processedRows) total=$($task.totalRows)" + $lastStatus = $task.status + } + if ($task.status -in @('SUCCEEDED', 'FAILED', 'EXPIRED')) { return $task } + Start-Sleep -Seconds $PollSeconds + } + throw "等待导出任务超时 taskId=$TaskId timeoutSeconds=$TimeoutSeconds" +} + +function Save-And-VerifyXlsx([object]$Task, [string]$Token, [string]$Directory) { + $target = Join-Path $Directory "$($Task.id).xlsx" + $response = Invoke-WebRequest -UseBasicParsing -Uri "$apiPath/$($Task.id)/download" ` + -Headers (Get-Headers $Token) -TimeoutSec 600 -OutFile $target -PassThru + $contentType = [string]$response.Headers['Content-Type'] + if ($contentType.ToLowerInvariant().Contains('json')) { + $payload = Get-Content -Raw -LiteralPath $target | ConvertFrom-Json + throw "下载返回业务错误 taskId=$($Task.id) code=$($payload.code) message=$($payload.message) requestId=$($payload.requestId) traceId=$($payload.traceId)" + } + $stream = [IO.File]::OpenRead($target) + try { + $signature = New-Object byte[] 4 + if ($stream.Read($signature, 0, 4) -ne 4 -or $signature[0] -ne 0x50 -or $signature[1] -ne 0x4B ` + -or $signature[2] -ne 0x03 -or $signature[3] -ne 0x04) { + throw "下载结果不是有效 XLSX/ZIP taskId=$($Task.id) contentType=$contentType" + } + } finally { + $stream.Dispose() + } + return [pscustomobject]@{ Path = $target; Bytes = (Get-Item -LiteralPath $target).Length; ContentType = $contentType } +} + +$createdTasks = [System.Collections.Generic.List[object]]::new() +if ($TaskIds.Count -eq 0) { + for ($index = 0; $index -lt $TaskCount; $index++) { + $task = New-ExportTask $AccessTokens[$index] ($index + 1) + $createdTasks.Add([pscustomobject]@{ Task = $task; Token = $AccessTokens[$index] }) + Write-Host "created taskId=$($task.id) owner=$($task.requestedBy) status=$($task.status)" + } + $uniqueIds = @($createdTasks | ForEach-Object { [string]$_.Task.id } | Sort-Object -Unique) + if ($uniqueIds.Count -ne $createdTasks.Count) { + throw '多个创建请求被重复提交规则合并；容量验收需使用不同申请用户，或等待 60 秒后重试。' + } +} else { + for ($index = 0; $index -lt $TaskIds.Count; $index++) { + $createdTasks.Add([pscustomobject]@{ Task = [pscustomobject]@{ id = $TaskIds[$index] }; Token = $AccessTokens[$index] }) + } +} + +if ($CreateOnly) { + $createdTasks | ForEach-Object { [pscustomobject]@{ taskId = [string]$_.Task.id } } | ConvertTo-Json + exit 0 +} + +$temporaryOutput = [string]::IsNullOrWhiteSpace($OutputDirectory) +if ($temporaryOutput) { + $OutputDirectory = Join-Path ([IO.Path]::GetTempPath()) "baiyue-export-acceptance-$([Guid]::NewGuid())" +} +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + +$results = [System.Collections.Generic.List[object]]::new() +try { + foreach ($entry in $createdTasks) { + $task = Wait-ExportTask ([string]$entry.Task.id) $entry.Token + if ($task.status -ne 'SUCCEEDED') { + throw "导出任务未成功 taskId=$($task.id) status=$($task.status) failure=$($task.failureMessage)" + } + if ([long]$task.processedRows -lt $MinimumRows) { + throw "导出行数未达到门槛 taskId=$($task.id) processed=$($task.processedRows) minimum=$MinimumRows" + } + $artifact = $null + if (-not $SkipDownload) { $artifact = Save-And-VerifyXlsx $task $entry.Token $OutputDirectory } + $results.Add([pscustomobject]@{ + taskId = [string]$task.id + status = $task.status + processedRows = [long]$task.processedRows + sizeBytes = [long]$task.sizeBytes + downloadedBytes = if ($artifact) { [long]$artifact.Bytes } else { 0 } + file = if ($artifact) { $artifact.Path } else { $null } + }) + } + $results | ConvertTo-Json -Depth 5 + Write-Host "EXPORT_CENTER_ACCEPTANCE_PASSED=$($results.Count)" +} finally { + if ($temporaryOutput -and (Test-Path -LiteralPath $OutputDirectory)) { + Remove-Item -LiteralPath $OutputDirectory -Recurse -Force + } +} diff --git a/scripts/phase1-onboarding-order-e2e.ps1 b/scripts/phase1-onboarding-order-e2e.ps1 index bd5d7a50d..5b7b5f555 100644 --- a/scripts/phase1-onboarding-order-e2e.ps1 +++ b/scripts/phase1-onboarding-order-e2e.ps1 @@ -3,12 +3,14 @@ param( [string]$IamUrl = 'http://127.0.0.1:9501', [string]$FileUrl = 'http://127.0.0.1:9506', [string]$OrderUrl = 'http://127.0.0.1:9504', + [string]$WarehouseUrl = 'http://127.0.0.1:9505', [string]$ImagePath, [string]$PlatformPassword = 'ChangeMe123!' ) $ErrorActionPreference = 'Stop' -function Invoke-Json($method, $url, $body = $null, $token = $null) { +function Invoke-Json($method, $url, $body = $null, $token = $null, $extraHeaders = @{}) { $headers = @{}; if ($token) { $headers.Authorization = "Bearer $token" } + foreach ($name in $extraHeaders.Keys) { $headers[$name] = $extraHeaders[$name] } $args = @{ Method=$method; Uri=$url; Headers=$headers; TimeoutSec=20 } if ($null -ne $body) { $args.ContentType='application/json'; $args.Body=$body | ConvertTo-Json -Depth 12 } Invoke-RestMethod @args @@ -20,6 +22,11 @@ function Invoke-Status($method, $url, $body, $token) { function Assert-Equal($name, $actual, $expected) { if ($actual -ne $expected) { throw "$name expected=$expected actual=$actual" } } +function Assert-Success($name, $response) { + if ($null -eq $response -or [string]$response.code -notin @('200', '0')) { + throw "$name failed code=$($response.code) message=$($response.message)" + } +} $stamp = Get-Date -Format 'MMddHHmmss' $username = "phase1_$stamp"; $password = 'Phase1Test123!' @@ -29,49 +36,105 @@ $registration = Invoke-Json POST "$AuthUrl/api/auth/register" @{ countryCode='CN'; address='No. 1 Test Road, Nanshan, Shenzhen' } Assert-Equal 'registration status' $registration.data.tenantStatus 'PROFILE_REQUIRED' +Assert-Equal 'registration next step' $registration.data.nextStep 'LOGIN_AND_COMPLETE_QUALIFICATIONS' $tenantId = [long]$registration.data.tenantId $merchantLogin = Invoke-Json POST "$AuthUrl/api/auth/login" @{username=$username;password=$password} +Assert-Success 'merchant login' $merchantLogin $merchantToken = $merchantLogin.data.accessToken +$platformLogin = Invoke-Json POST "$AuthUrl/api/auth/login" @{username='platform_admin';password=$PlatformPassword} +Assert-Success 'platform login' $platformLogin +$platformToken = $platformLogin.data.accessToken +$partner = Invoke-Json POST "$WarehouseUrl/api/platform/v1/partners?tenantId=$tenantId" @{ + partnerCode="E2E-PARTNER-$stamp"; partnerName="Phase One Partner $stamp"; partnerType='SHIPPER' + types=@('SHIPPER','CONSIGNEE'); contactName='Test Partner'; contactPhone='13800138000' + contactEmail="partner_$stamp@example.com"; countryCode='CN'; province='Guangdong'; city='Shenzhen' + district='Nanshan'; addressLine='No. 1 Test Road'; postalCode='518000'; status='ACTIVE'; remark='Phase one E2E' +} $platformToken +Assert-Success 'create tenant partner' $partner +$partnerId = [long]$partner.data +$warehouse = Invoke-Json POST "$WarehouseUrl/api/platform/v1/warehouse-admin" @{ + tenantId=$tenantId; warehouseCode="E2E-WH-$stamp"; warehouseName="Phase One Receiving Warehouse $stamp" + warehouseType='DOMESTIC'; countryCode='CN'; timezone='Asia/Shanghai'; contactName='Test Warehouse' + contactPhone='13800138000'; contactEmail="warehouse_$stamp@example.com"; status='ACTIVE'; remark='Phase one E2E' + address=@{addressType='WAREHOUSE';countryCode='CN';stateProvince='Guangdong';city='Shenzhen';district='Nanshan';postalCode='518000';addressLine1='No. 1 Test Road';addressLine2=$null;latitude=$null;longitude=$null} + capabilities=@(@{capabilityCode='RECEIVING';capabilityName='Receiving';enabled=$true;cutoffTime='16:00:00';description='Phase one E2E receiving'}) +} $platformToken +Assert-Success 'create tenant receiving warehouse' $warehouse +$warehouseId = [long]$warehouse.data + $order = @{ - customerId=$tenantId; customerName="Phase One Enterprise $stamp"; transportMode='SEA'; tradeTerm='FOB' - destinationCountryCode='US'; destinationWarehouseId=1001; destinationWarehouseName='ONT8' + customerId=$partnerId; customerName="Phase One Partner $stamp"; transportMode='SEA'; tradeTerm='FOB' + destinationCountryCode='US'; destinationWarehouseId=$null; destinationWarehouseName='ONT8' expectedShipDate='2026-08-15'; customerRemark='Phase one E2E'; pickupMethod='PICKUP' - receivingWarehouseId=2001; receivingWarehouseName='Shenzhen Receiving Warehouse'; channelCode='SEA-US-FBA' + receivingWarehouseId=$warehouseId; receivingWarehouseName="Phase One Receiving Warehouse $stamp"; channelCode='SEA-US-FBA' channelName='US Sea Channel'; taxMode='BUY_DECLARATION'; containerMode='LCL'; customsInspection=$false destinationType='FBA'; amazonWarehouseCode='ONT8'; shipmentId="FBA$stamp"; referenceId="REF$stamp" sender=@{companyName="Phase One Enterprise $stamp";contactName='Test Sender';phone='13800138000';email="p$stamp@example.com";countryCode='CN';province='Guangdong';city='Shenzhen';district='Nanshan';addressLine='No. 1 Test Road';postalCode='518000'} receiver=@{companyName='Amazon';contactName='ONT8';phone='1234567890';email='ont8@example.com';countryCode='US';province='CA';city='Moreno Valley';district='Riverside';addressLine='24300 Nandina Ave';postalCode='92551'} cargoes=@(@{cargoName='Bluetooth Speaker';hsCode='8518210000';packageType='CARTON';packageCount=2;grossWeightKg=20.5;volumeCbm=0.2;lengthCm=50;widthCm=40;heightCm=30;declaredValue=200;currency='USD';dangerousGoods=$false;cargoRemark='Phase one test';skuNo="SKU$stamp";upcNo='123456789012';cargoNameEn='Bluetooth Speaker';customsHsCode='8518210000';unitsPerPackage=10;totalQuantity=20;netWeightKg=18;unitDeclaredValue=10;material='Plastic';usage='Audio';model='P1';brand='Demo'}) } -$preApprovalStatus = Invoke-Status POST "$OrderUrl/api/merchant/v1/orders/first-leg" $order $merchantToken -Assert-Equal 'pre-approval order status' $preApprovalStatus 403 +$draftKey = "phase1-draft-$stamp" +$submitKey = "phase1-submit-$stamp" +$draft = Invoke-Json POST "$OrderUrl/api/merchant/v1/orders/first-leg" $order $merchantToken @{'Idempotency-Key'=$draftKey} +Assert-Success 'pre-approval draft create' $draft +$orderId = [long]$draft.data +$preApproval = Invoke-Json POST "$OrderUrl/api/merchant/v1/orders/first-leg/$orderId/submit" $null $merchantToken @{'Idempotency-Key'=$submitKey} +Assert-Equal 'pre-approval submit code' $preApproval.code 'A0403' -$uploadJson = & curl.exe -sS -X POST "$FileUrl/api/merchant/v1/files/qualifications" ` - -H "Authorization: Bearer $merchantToken" -F "file=@$ImagePath;type=image/png" ` - -F "certificateNo=91310101$stamp" -if ($LASTEXITCODE -ne 0) { throw 'qualification upload transport failed' } -$upload = $uploadJson | ConvertFrom-Json -Assert-Equal 'upload API code' $upload.code '0' -Assert-Equal 'upload tenant status' $upload.data.tenantStatus 'PENDING' +$creditCode = "91$((Get-Date).ToString('yyyyMMddHHmmss'))A1" +$qualificationTypes = @('BUSINESS_LICENSE', 'LEGAL_PERSON_ID', 'FREIGHT_FORWARDER_CERTIFICATE', 'BANK_ACCOUNT_LICENSE') +$uploads = @{} +foreach ($qualificationType in $qualificationTypes) { + $query = "qualificationType=$qualificationType&replace=false" + if ($qualificationType -eq 'BUSINESS_LICENSE') { $query += "&certificateNo=$creditCode" } + $uploadJson = & curl.exe -sS -X POST "$FileUrl/api/merchant/v1/files/qualifications?$query" ` + -H "Authorization: Bearer $merchantToken" -F "file=@$ImagePath;type=image/png" + if ($LASTEXITCODE -ne 0) { throw "qualification upload transport failed: $qualificationType" } + $upload = $uploadJson | ConvertFrom-Json + Assert-Success "upload $qualificationType" $upload + Assert-Equal "upload $qualificationType tenant status" $upload.data.tenantStatus 'PROFILE_REQUIRED' + $uploads[$qualificationType] = $upload +} + +$merchantQualifications = Invoke-Json GET "$IamUrl/api/merchant/v1/iam/tenant/qualifications" $null $merchantToken +Assert-Success 'merchant qualification list' $merchantQualifications +$pendingTypes = @($merchantQualifications.data | Where-Object { $_.status -eq 'PENDING' } | ForEach-Object { $_.qualificationType } | Sort-Object -Unique) +Assert-Equal 'pending qualification count' $pendingTypes.Count 4 +foreach ($qualificationType in $qualificationTypes) { + if ($qualificationType -notin $pendingTypes) { throw "missing pending qualification: $qualificationType" } +} + +$submitted = Invoke-Json POST "$IamUrl/api/merchant/v1/iam/tenant/qualifications/submit-review" $null $merchantToken +Assert-Success 'submit enterprise review' $submitted +Assert-Equal 'submitted tenant status' $submitted.data.status 'PENDING' -$platformLogin = Invoke-Json POST "$AuthUrl/api/auth/login" @{username='platform_admin';password=$PlatformPassword} -$platformToken = $platformLogin.data.accessToken $reviews = Invoke-Json GET "$IamUrl/api/platform/v1/iam/reviews" $null $platformToken +Assert-Success 'platform review list' $reviews $review = @($reviews.data) | Where-Object { [long]$_.tenantId -eq $tenantId } | Select-Object -First 1 if ($null -eq $review) { throw 'pending review not found' } $qualifications = Invoke-Json GET "$IamUrl/api/platform/v1/iam/tenants/$tenantId/qualifications" $null $platformToken -Assert-Equal 'qualification count' @($qualifications.data).Count 1 -$fileName = Split-Path $upload.data.documentUrl -Leaf +Assert-Success 'platform qualification list' $qualifications +Assert-Equal 'qualification count' @($qualifications.data).Count 4 +$fileName = Split-Path $uploads['BUSINESS_LICENSE'].data.documentUrl -Leaf $preview = Invoke-WebRequest -UseBasicParsing -Uri "$FileUrl/api/platform/v1/files/qualifications/$tenantId/$fileName" ` -Headers @{Authorization="Bearer $platformToken"} -TimeoutSec 20 Assert-Equal 'platform preview status' $preview.StatusCode 200 -Invoke-Json POST "$IamUrl/api/platform/v1/iam/reviews/$($review.id)/approve" @{comment='Phase one E2E approved'} $platformToken | Out-Null +$approval = Invoke-Json POST "$IamUrl/api/platform/v1/iam/reviews/$($review.id)/approve" @{comment='Phase one E2E approved'} $platformToken +Assert-Success 'approve enterprise review' $approval +Assert-Equal 'registered owner retained' $approval.data.administratorCreated $false -$created = Invoke-Json POST "$OrderUrl/api/merchant/v1/orders/first-leg" $order $merchantToken -$orderId = [long]$created.data -Invoke-Json POST "$OrderUrl/api/merchant/v1/orders/first-leg/$orderId/submit" $null $merchantToken | Out-Null +$approvedQualifications = Invoke-Json GET "$IamUrl/api/merchant/v1/iam/tenant/qualifications" $null $merchantToken +Assert-Success 'approved qualification list' $approvedQualifications +$approvedTypes = @($approvedQualifications.data | Where-Object { $_.status -eq 'APPROVED' } | ForEach-Object { $_.qualificationType } | Sort-Object -Unique) +Assert-Equal 'approved qualification count' $approvedTypes.Count 4 +$merchantProfile = Invoke-Json GET "$IamUrl/api/merchant/v1/iam/tenant/profile" $null $merchantToken +Assert-Success 'merchant profile after approval' $merchantProfile +Assert-Equal 'active tenant status' $merchantProfile.data.status 'ACTIVE' + +$approvedSubmit = Invoke-Json POST "$OrderUrl/api/merchant/v1/orders/first-leg/$orderId/submit" $null $merchantToken @{'Idempotency-Key'=$submitKey} +Assert-Success 'approved order submit' $approvedSubmit $detail = Invoke-Json GET "$OrderUrl/api/merchant/v1/orders/first-leg/$orderId" $null $merchantToken Assert-Equal 'submitted order status' $detail.data.order.status 'SUBMITTED' -[pscustomobject]@{username=$username;tenantId=$tenantId;registerStatus='PROFILE_REQUIRED';preApprovalOrderHttp=$preApprovalStatus;uploadStatus='PENDING';reviewId=$review.id;previewHttp=200;orderId=$orderId;finalOrderStatus='SUBMITTED'} | ConvertTo-Json +[pscustomobject]@{username=$username;tenantId=$tenantId;registerStatus='PROFILE_REQUIRED';partnerId=$partnerId;warehouseId=$warehouseId;draftCreatedBeforeApproval=$true;preApprovalSubmitCode=$preApproval.code;pendingQualificationCount=4;reviewId=$review.id;previewHttp=200;approvedQualificationCount=4;orderId=$orderId;finalOrderStatus='SUBMITTED'} | ConvertTo-Json