feat(ci): rollback unhealthy deployments

deploy/scripts/deploy-built-service.sh

@@ -15,6 +15,18 @@ COMPOSE_FILE="deploy/docker-compose.yml"
 IMAGE="$IMAGE_REPOSITORY/$SERVICE:$IMAGE_TAG"
 REGISTRY_HOST="${IMAGE_REPOSITORY%%/*}"
 PROJECT_NAME="firstleg-$DEPLOY_ENV"
+HEALTH_TIMEOUT_SECONDS="${DEPLOY_HEALTH_TIMEOUT_SECONDS:-240}"
+HEALTH_POLL_SECONDS="${DEPLOY_HEALTH_POLL_SECONDS:-5}"
+
+for value_name in HEALTH_TIMEOUT_SECONDS HEALTH_POLL_SECONDS; do
+  eval "value=\${$value_name}"
+  case "$value" in
+    ''|*[!0-9]*|0)
+      echo "$value_name 必须为正整数，当前: $value" >&2
+      exit 2
+      ;;
+  esac
+done
 
 case "$DEPLOY_ENV" in
   development|test) ;;
@@ -165,23 +177,121 @@ local_registry_logout() {
   docker logout "$REGISTRY_HOST" >/dev/null 2>&1 || true
 }
 
+local_compose() {
+  docker compose \
+    --project-name "$PROJECT_NAME" \
+    --env-file "$ENV_FILE" \
+    -f "$COMPOSE_FILE" \
+    "$@"
+}
+
+local_container_id() {
+  local_compose ps --all --quiet "$SERVICE" | head -n 1
+}
+
+local_container_image() {
+  container_id="$(local_container_id)"
+  if [ -n "$container_id" ]; then
+    docker inspect --format '{{.Config.Image}}' "$container_id"
+  fi
+}
+
+local_health_state() {
+  container_id="$(local_container_id)"
+  if [ -z "$container_id" ]; then
+    echo missing
+    return
+  fi
+  docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-health:{{.State.Status}}{{end}}' "$container_id" 2>/dev/null || echo missing
+}
+
+wait_for_local_health() {
+  elapsed=0
+  while [ "$elapsed" -lt "$HEALTH_TIMEOUT_SECONDS" ]; do
+    health_state="$(local_health_state)"
+    echo "容器健康状态: $health_state（已等待 ${elapsed}s）"
+    case "$health_state" in
+      healthy)
+        return 0
+        ;;
+      no-health:running)
+        echo "容器未配置 HEALTHCHECK，以运行状态作为成功依据" >&2
+        return 0
+        ;;
+      unhealthy|no-health:exited|no-health:dead)
+        return 1
+        ;;
+    esac
+    sleep "$HEALTH_POLL_SECONDS"
+    elapsed=$((elapsed + HEALTH_POLL_SECONDS))
+  done
+  echo "等待容器健康超时: ${HEALTH_TIMEOUT_SECONDS}s" >&2
+  return 1
+}
+
 deploy_local() {
   printf '%s' "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" --password-stdin "$REGISTRY_HOST"
   trap local_registry_logout EXIT HUP INT TERM
+  previous_image="$(local_container_image || true)"
+  if [ -n "$previous_image" ]; then
+    echo "部署前运行镜像: $previous_image"
+  else
+    echo "首次部署，未发现可回滚的旧镜像"
+  fi
   docker pull "$IMAGE"
   export DEPLOY_ENV IMAGE_REPOSITORY IMAGE_TAG
-  docker compose \
-    --project-name "$PROJECT_NAME" \
-    --env-file "$ENV_FILE" \
-    -f "$COMPOSE_FILE" \
-    up -d --remove-orphans --pull never "$SERVICE"
-  docker compose \
-    --project-name "$PROJECT_NAME" \
-    --env-file "$ENV_FILE" \
-    -f "$COMPOSE_FILE" \
-    ps "$SERVICE"
-  local_registry_logout
-  trap - EXIT HUP INT TERM
+  new_started=true
+  if ! local_compose up -d --remove-orphans --pull never "$SERVICE"; then
+    echo "新镜像启动命令失败: $IMAGE" >&2
+    new_started=false
+  fi
+  if [ "$new_started" = "true" ] && wait_for_local_health; then
+    echo "新镜像健康检查通过: $IMAGE"
+    local_compose ps "$SERVICE"
+    local_registry_logout
+    trap - EXIT HUP INT TERM
+    return 0
+  fi
+
+  echo "新镜像健康检查失败: $IMAGE" >&2
+  expected_prefix="$IMAGE_REPOSITORY/$SERVICE:"
+  case "$previous_image" in
+    "$expected_prefix"*)
+      previous_tag="${previous_image#"$expected_prefix"}"
+      ;;
+    '')
+      echo "首次部署没有可回滚镜像，部署失败" >&2
+      local_compose stop "$SERVICE" >/dev/null 2>&1 || true
+      return 1
+      ;;
+    *)
+      echo "旧镜像不属于当前服务，拒绝自动回滚: $previous_image" >&2
+      return 1
+      ;;
+  esac
+  if [ "$previous_image" = "$IMAGE" ]; then
+    echo "旧镜像与新镜像相同，无法自动回滚: $IMAGE" >&2
+    return 1
+  fi
+
+  echo "开始自动回滚到: $previous_image" >&2
+  if ! docker pull "$previous_image"; then
+    echo "自动回滚失败，无法拉取旧镜像: $previous_image" >&2
+    return 1
+  fi
+  IMAGE_TAG="$previous_tag"
+  export IMAGE_TAG
+  if ! local_compose up -d --remove-orphans --pull never "$SERVICE"; then
+    echo "自动回滚失败，旧镜像启动命令失败: $previous_image" >&2
+    return 1
+  fi
+  if wait_for_local_health; then
+    echo "自动回滚成功: $previous_image" >&2
+    local_compose ps "$SERVICE" || true
+  else
+    echo "自动回滚失败，旧镜像未恢复健康: $previous_image" >&2
+  fi
+  return 1
 }
 
 if [ "${TARGET_IS_LOCAL:-false}" = "true" ]; then
@@ -252,7 +362,7 @@ printf '%s' "$REGISTRY_PASSWORD" | run_ssh docker login -u "$REGISTRY_USER" --pa
 trap remote_registry_logout EXIT HUP INT TERM
 run_ssh docker pull "$IMAGE"
 
-run_ssh sh -s -- "$REMOTE_DIR" "$DEPLOY_ENV" "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$SERVICE" "$PROJECT_NAME" "$REMOTE_ENV_NAME" "$REMOTE_COMPOSE_NAME" <<'REMOTE_DEPLOY'
+run_ssh sh -s -- "$REMOTE_DIR" "$DEPLOY_ENV" "$IMAGE_REPOSITORY" "$IMAGE_TAG" "$SERVICE" "$PROJECT_NAME" "$REMOTE_ENV_NAME" "$REMOTE_COMPOSE_NAME" "$HEALTH_TIMEOUT_SECONDS" "$HEALTH_POLL_SECONDS" <<'REMOTE_DEPLOY'
 set -eu
 remote_dir="$1"
 deploy_env="$2"
@@ -262,20 +372,117 @@ service="$5"
 project_name="$6"
 env_file="$7"
 compose_file="$8"
+health_timeout_seconds="$9"
+health_poll_seconds="${10}"
 cd "$remote_dir"
 export DEPLOY_ENV="$deploy_env"
 export IMAGE_REPOSITORY="$image_repository"
 export IMAGE_TAG="$image_tag"
-docker compose \
-  --project-name "$project_name" \
-  --env-file "$env_file" \
-  -f "$compose_file" \
-  up -d --remove-orphans --pull never "$service"
-docker compose \
-  --project-name "$project_name" \
-  --env-file "$env_file" \
-  -f "$compose_file" \
-  ps "$service"
+
+compose() {
+  docker compose \
+    --project-name "$project_name" \
+    --env-file "$env_file" \
+    -f "$compose_file" \
+    "$@"
+}
+
+container_id() {
+  compose ps --all --quiet "$service" | head -n 1
+}
+
+container_image() {
+  id="$(container_id)"
+  if [ -n "$id" ]; then
+    docker inspect --format '{{.Config.Image}}' "$id"
+  fi
+}
+
+health_state() {
+  id="$(container_id)"
+  if [ -z "$id" ]; then
+    echo missing
+    return
+  fi
+  docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-health:{{.State.Status}}{{end}}' "$id" 2>/dev/null || echo missing
+}
+
+wait_for_health() {
+  elapsed=0
+  while [ "$elapsed" -lt "$health_timeout_seconds" ]; do
+    state="$(health_state)"
+    echo "容器健康状态: $state（已等待 ${elapsed}s）"
+    case "$state" in
+      healthy) return 0 ;;
+      no-health:running)
+        echo "容器未配置 HEALTHCHECK，以运行状态作为成功依据" >&2
+        return 0
+        ;;
+      unhealthy|no-health:exited|no-health:dead) return 1 ;;
+    esac
+    sleep "$health_poll_seconds"
+    elapsed=$((elapsed + health_poll_seconds))
+  done
+  echo "等待容器健康超时: ${health_timeout_seconds}s" >&2
+  return 1
+}
+
+new_image="$image_repository/$service:$image_tag"
+previous_image="$(container_image || true)"
+if [ -n "$previous_image" ]; then
+  echo "部署前运行镜像: $previous_image"
+else
+  echo "首次部署，未发现可回滚的旧镜像"
+fi
+
+new_started=true
+if ! compose up -d --remove-orphans --pull never "$service"; then
+  echo "新镜像启动命令失败: $new_image" >&2
+  new_started=false
+fi
+if [ "$new_started" = "true" ] && wait_for_health; then
+  echo "新镜像健康检查通过: $new_image"
+  compose ps "$service"
+  exit 0
+fi
+
+echo "新镜像健康检查失败: $new_image" >&2
+expected_prefix="$image_repository/$service:"
+case "$previous_image" in
+  "$expected_prefix"*) previous_tag="${previous_image#"$expected_prefix"}" ;;
+  '')
+    echo "首次部署没有可回滚镜像，部署失败" >&2
+    compose stop "$service" >/dev/null 2>&1 || true
+    exit 1
+    ;;
+  *)
+    echo "旧镜像不属于当前服务，拒绝自动回滚: $previous_image" >&2
+    exit 1
+    ;;
+esac
+if [ "$previous_image" = "$new_image" ]; then
+  echo "旧镜像与新镜像相同，无法自动回滚: $new_image" >&2
+  exit 1
+fi
+
+echo "开始自动回滚到: $previous_image" >&2
+if ! docker pull "$previous_image"; then
+  echo "自动回滚失败，无法拉取旧镜像: $previous_image" >&2
+  exit 1
+fi
+IMAGE_TAG="$previous_tag"
+export IMAGE_TAG
+if ! compose up -d --remove-orphans --pull never "$service"; then
+  echo "自动回滚失败，旧镜像启动命令失败: $previous_image" >&2
+  exit 1
+fi
+if wait_for_health; then
+  echo "自动回滚成功: $previous_image" >&2
+  compose ps "$service" || true
+else
+  echo "自动回滚失败，旧镜像未恢复健康: $previous_image" >&2
+fi
+exit 1
 REMOTE_DEPLOY
 
 remote_registry_logout