diff --git a/deploy/scripts/deploy-blue-green-service.sh b/deploy/scripts/deploy-blue-green-service.sh index 6cf70ac73..462bc5527 100644 --- a/deploy/scripts/deploy-blue-green-service.sh +++ b/deploy/scripts/deploy-blue-green-service.sh @@ -33,7 +33,7 @@ env_value() { service_contract() { case "$SERVICE:$COMPOSE_FILE" in - gateway:*) echo "gateway|9000|9000|gateway|gateway" ;; + gateway:*) echo "gateway|9000|9000|gateway|gateway-active" ;; auth:*) echo "auth|9100|9100|auth-service|auth" ;; iam:*) echo "iam|9201|9201|iam-service|iam" ;; firstleg:*) echo "business|9202|9202|firstleg-service|firstleg" ;; @@ -44,11 +44,11 @@ service_contract() { print:*) echo "print|9207|9207|print-service|print" ;; integration-service:*) echo "integration|9210|9210|integration-service|integration-service" ;; audit-service:*) echo "audit|9700|9700|audit-service|audit-service" ;; - gateway-overseas:*) echo "gateway-overseas|9300|9300|gateway-overseas|gateway-overseas" ;; + gateway-overseas:*) echo "gateway-overseas|9300|9300|gateway-overseas|gateway-overseas-active" ;; lastleg-service:*) echo "lastleg|9303|9303|lastleg-service|lastleg-service" ;; warehouse-overseas-service:*) echo "warehouse-overseas|9305|9305|warehouse-overseas-service|warehouse-overseas-service" ;; - event-bridge-service:*overseas*) echo "event-bridge-overseas|9312|9313|event-bridge-service|event-bridge-overseas" ;; - event-bridge-service:*) echo "event-bridge|9312|9312|event-bridge-service|event-bridge-cn" ;; + event-bridge-service:*overseas*) echo "event-bridge-overseas|9312|9313|event-bridge-overseas|event-bridge-overseas" ;; + event-bridge-service:*) echo "event-bridge|9312|9312|event-bridge-cn|event-bridge-cn" ;; *) echo "不支持蓝绿部署的服务: $SERVICE" >&2; exit 2 ;; esac } @@ -173,6 +173,7 @@ start_worker_candidate() { -e "BAIYUE_RELEASE_ID=$IMAGE_TAG" "$SERVICE" >/dev/null; then return 1 fi + docker network connect "$ROUTING_NETWORK" "$worker_candidate" if ! wait_for_container "$worker_candidate" "候选 Worker"; then docker logs --tail 200 "$worker_candidate" 2>&1 || true docker rm -f "$worker_candidate" >/dev/null 2>&1 || true @@ -182,38 +183,50 @@ start_worker_candidate() { [ "$worker_image" = "$IMAGE" ] || return 1 } -connect_routing_alias() { - alias_name="$1" - container="$2" +connect_routing_aliases() { + container="$1" + shift docker network inspect "$ROUTING_NETWORK" --format '{{json .Containers}}' | grep -q "\"$container\"" \ && return 0 - docker network connect --alias "$alias_name" "$ROUTING_NETWORK" "$container" + set -- docker network connect $(for alias_name in "$@"; do printf '%s\n' --alias "$alias_name"; done) + "$@" "$ROUTING_NETWORK" "$container" } bootstrap_legacy_routes() { for pair in \ - gateway:gateway auth:auth iam:iam firstleg:business finance:finance order:order \ - warehouse:warehouse file:file print:print integration-service:integration audit-service:audit \ - gateway-overseas:gateway-overseas lastleg-service:lastleg \ - warehouse-overseas-service:warehouse-overseas event-bridge-cn:event-bridge \ - event-bridge-overseas:event-bridge-overseas + gateway-active,gateway:gateway auth,auth-service:auth iam,iam-service:iam \ + firstleg,firstleg-service:business finance,finance-service:finance order,order-service:order \ + warehouse,warehouse-service:warehouse file,file-service:file print,print-service:print \ + integration-service:integration audit-service:audit \ + gateway-overseas-active,gateway-overseas:gateway-overseas \ + lastleg-service:lastleg warehouse-overseas-service:warehouse-overseas \ + event-bridge-cn:event-bridge event-bridge-overseas:event-bridge-overseas do - alias_name="${pair%%:*}" + aliases="${pair%%:*}" stem="${pair#*:}" container="firstleg-${DEPLOY_ENV}-${stem}" if docker inspect "$container" >/dev/null 2>&1; then - connect_routing_alias "$alias_name" "$container" || true + connect_routing_aliases "$container" $(printf '%s' "$aliases" | tr ',' ' ') || true fi done for container in firstleg-prod-admin-web firstleg-prod-client-web; do if docker inspect "$container" >/dev/null 2>&1; then - connect_routing_alias "${container##*-}" "$container" || true + connect_routing_aliases "$container" "${container##*-}" || true fi done } switch_gateway_proxy() { [ "$SERVICE" = "gateway" ] || [ "$SERVICE" = "gateway-overseas" ] || return 0 + if [ "$SERVICE" = "gateway" ]; then required_alias=gateway-active; else required_alias=gateway-overseas-active; fi + for frontend in firstleg-prod-admin-web-blue firstleg-prod-admin-web-green \ + firstleg-prod-client-web-blue firstleg-prod-client-web-green; do + if docker inspect "$frontend" >/dev/null 2>&1 \ + && ! docker exec "$frontend" nginx -T 2>&1 | grep -q "$required_alias"; then + echo "$frontend 尚未使用稳定上游 $required_alias,拒绝切换 Gateway" >&2 + return 1 + fi + done if [ "$SERVICE" = "gateway" ]; then config="$(env_value GATEWAY_BLUE_GREEN_NGINX_CONFIG)" config="${config:-/www/server/panel/vhost/nginx/api-cn.buggoglobal.com.conf}" @@ -241,6 +254,31 @@ switch_gateway_proxy() { rm -f "$backup" } +reload_frontend_proxies() { + [ "$SERVICE" = "gateway" ] || [ "$SERVICE" = "gateway-overseas" ] || return 0 + for frontend in firstleg-prod-admin-web-blue firstleg-prod-admin-web-green \ + firstleg-prod-client-web-blue firstleg-prod-client-web-green; do + if docker inspect "$frontend" >/dev/null 2>&1; then + docker exec "$frontend" nginx -t + docker exec "$frontend" nginx -s reload + fi + done +} + +rollback_gateway_proxy() { + [ "$SERVICE" = "gateway" ] || [ "$SERVICE" = "gateway-overseas" ] || return 0 + if [ "$SERVICE" = "gateway" ]; then + config="$(env_value GATEWAY_BLUE_GREEN_NGINX_CONFIG)" + config="${config:-/www/server/panel/vhost/nginx/api-cn.buggoglobal.com.conf}" + else + config="$(env_value GATEWAY_OVERSEAS_BLUE_GREEN_NGINX_CONFIG)" + config="${config:-/www/server/panel/vhost/nginx/api-global.buggoglobal.com.conf}" + fi + old_port="${active_port:-$BASE_PORT}" + sed -i "s#127\.0\.0\.1:$target_port#127.0.0.1:$old_port#g" "$config" + nginx -t && nginx -s reload +} + bootstrap_legacy_routes export DEPLOY_ENV IMAGE_REPOSITORY IMAGE_TAG @@ -276,8 +314,11 @@ if [ "$running_image" != "$IMAGE" ]; then exit 1 fi -connect_routing_alias "$ROUTING_ALIAS" "$candidate" -[ "$NACOS_SERVICE" = "$SERVICE" ] || connect_routing_alias "$NACOS_SERVICE" "$candidate" +if [ "$NACOS_SERVICE" = "$ROUTING_ALIAS" ]; then + connect_routing_aliases "$candidate" "$ROUTING_ALIAS" +else + connect_routing_aliases "$candidate" "$ROUTING_ALIAS" "$NACOS_SERVICE" +fi if [ "$STABILIZATION_SECONDS" -gt 0 ]; then echo "候选槽稳定观察 ${STABILIZATION_SECONDS}s" @@ -290,13 +331,60 @@ if ! switch_gateway_proxy; then exit 1 fi -if [ -n "$active_container" ] && docker inspect "$active_container" >/dev/null 2>&1; then - echo "优雅停止旧实例: $active_container" - docker stop --time "$STOP_TIMEOUT" "$active_container" >/dev/null - docker rm "$active_container" >/dev/null +worker_required=false +if needs_worker; then worker_required=true; fi + +if [ "$worker_required" = true ] && [ "$active_slot" != legacy ]; then + if [ -n "$active_worker" ] && docker inspect "$active_worker" >/dev/null 2>&1; then + echo "暂停旧 Worker,避免新旧任务并行: $active_worker" + docker stop --time "$STOP_TIMEOUT" "$active_worker" >/dev/null + fi + if ! start_worker_candidate; then + [ -z "$active_worker" ] || docker start "$active_worker" >/dev/null 2>&1 || true + rollback_gateway_proxy || true + cleanup_candidate + exit 1 + fi fi -printf '%s|%s|%s|%s\n' "$target_slot" "$candidate" "$target_port" "$IMAGE" > "$STATE_FILE.tmp" +old_container_stopped=false +if [ -n "$active_container" ] && docker inspect "$active_container" >/dev/null 2>&1; then + echo "优雅停止旧 Web 实例: $active_container" + docker stop --time "$STOP_TIMEOUT" "$active_container" >/dev/null + old_container_stopped=true +fi + +if [ "$worker_required" = true ] && [ "$active_slot" = legacy ]; then + if ! start_worker_candidate; then + [ "$old_container_stopped" = false ] || docker start "$active_container" >/dev/null 2>&1 || true + rollback_gateway_proxy || true + cleanup_candidate + exit 1 + fi +fi + +if ! reload_frontend_proxies; then + [ "$old_container_stopped" = false ] || docker start "$active_container" >/dev/null 2>&1 || true + if [ "$worker_required" = true ]; then + docker rm -f "$worker_candidate" >/dev/null 2>&1 || true + [ -z "$active_worker" ] || docker start "$active_worker" >/dev/null 2>&1 || true + fi + rollback_gateway_proxy || true + cleanup_candidate + reload_frontend_proxies || true + exit 1 +fi + +[ -z "$active_worker" ] || [ "$active_worker" = "$worker_candidate" ] \ + || docker rm "$active_worker" >/dev/null 2>&1 || true +[ "$old_container_stopped" = false ] || docker rm "$active_container" >/dev/null + +if [ "$worker_required" = true ]; then + next_worker="$worker_candidate" +else + next_worker="" +fi +printf '%s|%s|%s|%s|%s\n' "$target_slot" "$candidate" "$target_port" "$IMAGE" "$next_worker" > "$STATE_FILE.tmp" chmod 640 "$STATE_FILE.tmp" mv -f "$STATE_FILE.tmp" "$STATE_FILE" echo "蓝绿切换完成: service=$SERVICE active=$target_slot container=$candidate port=$target_port" diff --git a/deploy/scripts/deploy-built-service.sh b/deploy/scripts/deploy-built-service.sh index 680ab07a8..e8bcab702 100644 --- a/deploy/scripts/deploy-built-service.sh +++ b/deploy/scripts/deploy-built-service.sh @@ -348,6 +348,16 @@ deploy_local() { if [ "${TARGET_IS_LOCAL:-false}" = "true" ]; then echo "镜像已在目标服务器本机,直接部署: $IMAGE" + if [ "$DEPLOY_ENV" = "production" ]; then + printf '%s' "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST" + trap local_registry_logout EXIT HUP INT TERM + docker pull "$IMAGE" + sh deploy/scripts/deploy-blue-green-service.sh \ + "$DEPLOY_ENV" "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$SERVICE" "$ENV_FILE" "$COMPOSE_FILE" "$PROJECT_NAME" + local_registry_logout + trap - EXIT HUP INT TERM + exit 0 + fi deploy_local exit 0 fi diff --git a/deploy/scripts/deploy-services.sh b/deploy/scripts/deploy-services.sh index c18282116..5b642b59d 100644 --- a/deploy/scripts/deploy-services.sh +++ b/deploy/scripts/deploy-services.sh @@ -188,6 +188,19 @@ if [ "$DEPLOY_ENV" = "development" ]; then "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$SERVICE_CSV" "$ENV_FILE" fi +if [ "$DEPLOY_ENV" = "production" ]; then + docker compose \ + --project-name "$PROJECT_NAME" \ + --env-file "$ENV_FILE" \ + -f "$COMPOSE_FILE" \ + pull $SERVICES + for service in $SERVICES; do + sh deploy/scripts/deploy-blue-green-service.sh \ + "$DEPLOY_ENV" "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$service" "$ENV_FILE" "$COMPOSE_FILE" "$PROJECT_NAME" + done + exit 0 +fi + docker compose \ --project-name "$PROJECT_NAME" \ --env-file "$ENV_FILE" \ diff --git a/event-bridge-service/src/main/resources/application.yml b/event-bridge-service/src/main/resources/application.yml index c71fb2f2b..f16fd66fc 100644 --- a/event-bridge-service/src/main/resources/application.yml +++ b/event-bridge-service/src/main/resources/application.yml @@ -23,6 +23,8 @@ spring: username: ${NACOS_USERNAME:nacos} password: ${NACOS_PASSWORD:nacos} discovery: + enabled: ${NACOS_DISCOVERY_ENABLED:true} + register-enabled: ${NACOS_REGISTER_ENABLED:true} group: ${NACOS_GROUP:BAIYUE} namespace: ${NACOS_NAMESPACE:} config: diff --git a/scripts/blue-green-deployment-contract.test.mjs b/scripts/blue-green-deployment-contract.test.mjs index 27e657bd4..0321df845 100644 --- a/scripts/blue-green-deployment-contract.test.mjs +++ b/scripts/blue-green-deployment-contract.test.mjs @@ -16,17 +16,21 @@ test('all backend business services have blue-green runtime contracts', async () assert.match(script, /target_slot="green"/) assert.match(script, /target_port=\$\(\(BASE_PORT \+ 10000\)\)/) assert.match(script, /compose run -d --no-deps --name "\$candidate"/) - assert.match(script, /wait_for_candidate/) - assert.match(script, /connect_routing_alias "\$ROUTING_ALIAS" "\$candidate"/) + assert.match(script, /wait_for_container "\$candidate"/) + assert.match(script, /BAIYUE_RUNTIME_SCHEDULING_ENABLED=false/) + assert.match(script, /NACOS_REGISTER_ENABLED=false/) + assert.match(script, /start_worker_candidate/) + assert.match(script, /connect_routing_aliases "\$candidate" "\$ROUTING_ALIAS" "\$NACOS_SERVICE"/) + assert.match(script, /file,file-service:file/) assert.match(script, /docker stop --time "\$STOP_TIMEOUT" "\$active_container"/) assert.match(script, /mv -f "\$STATE_FILE\.tmp" "\$STATE_FILE"/) }) test('candidate failure keeps the active container and successful cutover stops it last', async () => { const script = await read('deploy/scripts/deploy-blue-green-service.sh') - const start = script.indexOf('compose run -d') - const healthy = script.indexOf('if ! wait_for_candidate') - const route = script.indexOf('connect_routing_alias "$ROUTING_ALIAS" "$candidate"') + const start = script.lastIndexOf('compose run -d') + const healthy = script.indexOf('if ! wait_for_container "$candidate"') + const route = script.indexOf('connect_routing_aliases "$candidate"') const gateway = script.indexOf('if ! switch_gateway_proxy') const stop = script.indexOf('docker stop --time "$STOP_TIMEOUT" "$active_container"') const state = script.indexOf('mv -f "$STATE_FILE.tmp" "$STATE_FILE"') @@ -39,6 +43,8 @@ test('gateway cutover is atomic and restores nginx config on validation failure' const script = await read('deploy/scripts/deploy-blue-green-service.sh') assert.match(script, /api-cn\.buggoglobal\.com\.conf/) assert.match(script, /api-global\.buggoglobal\.com\.conf/) + assert.match(script, /尚未使用稳定上游 \$required_alias,拒绝切换 Gateway/) + assert.match(script, /reload_frontend_proxies/) assert.match(script, /cp -f "\$config" "\$backup"/) assert.match(script, /nginx -t \|\| ! nginx -s reload/) assert.match(script, /cp -f "\$backup" "\$config"/) @@ -56,9 +62,27 @@ test('compose templates preserve rolling defaults and expose slot metadata', asy } }) +test('every scheduled worker service can disable Nacos registration', async () => { + for (const application of [ + 'iam/src/main/resources/application.yml', + 'firstleg/src/main/resources/application.yml', + 'finance/src/main/resources/application.yml', + 'order/src/main/resources/application.yml', + 'file/src/main/resources/application.yml', + 'event-bridge-service/src/main/resources/application.yml', + 'audit-service/src/main/resources/application.yml', + 'lastleg-service/src/main/resources/application.yml' + ]) { + assert.match(await read(application), /register-enabled: \$\{NACOS_REGISTER_ENABLED:true\}/, application) + } +}) + test('production single-service pipeline dispatches the blue-green executor', async () => { const deploy = await read('deploy/scripts/deploy-built-service.sh') + const batch = await read('deploy/scripts/deploy-services.sh') assert.match(deploy, /run_scp "deploy\/scripts\/deploy-blue-green-service\.sh"/) assert.match(deploy, /if \[ "\$DEPLOY_ENV" = "production" \]/) assert.match(deploy, /run_ssh sh "\$REMOTE_DIR\/\$REMOTE_BLUE_GREEN_NAME"/) + assert.match(deploy, /TARGET_IS_LOCAL[\s\S]*?deploy-blue-green-service\.sh/) + assert.match(batch, /if \[ "\$DEPLOY_ENV" = "production" \][\s\S]*?deploy-blue-green-service\.sh/) }) diff --git a/scripts/blue-green-deployment-runtime.test.mjs b/scripts/blue-green-deployment-runtime.test.mjs index c03b49776..2fa8c975e 100644 --- a/scripts/blue-green-deployment-runtime.test.mjs +++ b/scripts/blue-green-deployment-runtime.test.mjs @@ -21,12 +21,22 @@ case "$command" in exit 0 ;; compose) + container_name='' + previous='' for argument in "$@"; do + if [ "$previous" = --name ]; then container_name="$argument"; fi + previous="$argument" if [ "$argument" = run ]; then - : > "$MOCK_CANDIDATE_EXISTS" - exit 0 + action=run fi done + if [ "\${action:-}" = run ]; then + case "$container_name" in + *-worker) : > "$MOCK_WORKER_EXISTS" ;; + *) : > "$MOCK_CANDIDATE_EXISTS" ;; + esac + exit 0 + fi exit 0 ;; inspect) @@ -43,6 +53,13 @@ case "$command" in *Config.Image*) printf '%s\\n' 'registry.local/group/file:newtag' ;; esac ;; + firstleg-production-file-green-worker) + [ -f "$MOCK_WORKER_EXISTS" ] || exit 1 + case "$*" in + *State.Health*) printf '%s\\n' healthy ;; + *Config.Image*) printf '%s\\n' 'registry.local/group/file:newtag' ;; + esac + ;; *) exit 1 ;; esac ;; @@ -57,6 +74,7 @@ case "$command" in case "$target" in firstleg-production-file) : > "$MOCK_LEGACY_REMOVED" ;; firstleg-production-file-green) rm -f "$MOCK_CANDIDATE_EXISTS" ;; + firstleg-production-file-green-worker) rm -f "$MOCK_WORKER_EXISTS" ;; esac ;; logs) echo 'candidate logs' ;; @@ -79,13 +97,13 @@ async function runBlueGreen(health) { await writeFile(sleep, '#!/usr/bin/env sh\nexit 0\n', 'utf8') await chmod(docker, 0o755) await chmod(sleep, 0o755) - await writeFile(envFile, 'PLACEHOLDER=true\n', 'utf8') + await writeFile(envFile, 'FILE_EXPORT_CONSUMER_ENABLED=true\n', 'utf8') await writeFile(compose, 'services: {}\n', 'utf8') await writeFile(log, '', 'utf8') const paths = { candidate: join(temp, 'candidate'), stopped: join(temp, 'legacy-stopped'), - removed: join(temp, 'legacy-removed') + removed: join(temp, 'legacy-removed'), worker: join(temp, 'worker') } const result = spawnSync(shell, [ script.replaceAll('\\', '/'), 'production', 'registry.local/group', 'newtag', 'file', @@ -100,6 +118,7 @@ async function runBlueGreen(health) { DEPLOY_HEALTH_POLL_SECONDS: '1', MOCK_LOG: log, MOCK_CANDIDATE_EXISTS: paths.candidate, + MOCK_WORKER_EXISTS: paths.worker, MOCK_LEGACY_STOPPED: paths.stopped, MOCK_LEGACY_REMOVED: paths.removed, MOCK_CANDIDATE_HEALTH: health @@ -111,7 +130,8 @@ async function runBlueGreen(health) { const state = existsSync(stateFile) ? await readFile(stateFile, 'utf8') : '' const evidence = { ...result, output, dockerLog, state, - stopped: existsSync(paths.stopped), removed: existsSync(paths.removed) + stopped: existsSync(paths.stopped), removed: existsSync(paths.removed), + worker: existsSync(paths.worker) } await rm(temp, { recursive: true, force: true }) return evidence @@ -123,10 +143,13 @@ test('healthy candidate becomes active only after health and old instance drains assert.equal(result.status, 0, result.output) assert.equal(result.stopped, true) assert.equal(result.removed, true) - assert.match(result.state, /^green\|firstleg-production-file-green\|19206\|registry\.local\/group\/file:newtag/) + assert.equal(result.worker, true) + assert.match(result.state, /^green\|firstleg-production-file-green\|19206\|registry\.local\/group\/file:newtag\|firstleg-production-file-green-worker/) const run = result.dockerLog.indexOf('compose --project-name firstleg-production') const stop = result.dockerLog.indexOf('stop --time 45 firstleg-production-file') assert.ok(run >= 0 && run < stop) + assert.match(result.dockerLog, + /network connect --alias file --alias file-service firstleg-production-routing firstleg-production-file-green/) }) test('unhealthy candidate is removed while old instance and state remain untouched', async (context) => { @@ -135,6 +158,7 @@ test('unhealthy candidate is removed while old instance and state remain untouch assert.equal(result.status, 1, result.output) assert.equal(result.stopped, false) assert.equal(result.removed, false) + assert.equal(result.worker, false) assert.equal(result.state, '') assert.doesNotMatch(result.dockerLog, /stop --time .*firstleg-production-file/) })