curriculum-project-hub/hub/deploy/deploy_fleet_release.sh

#!/usr/bin/env bash
# Publish one immutable Hub release (tsc + admin-web SPA) to the managed host,
# then repoint every Silo in the chosen fleet to that release and restart it.
#
# Fleet is selected by the middle DNS label of each Silo's HUB_PUBLIC_BASE_URL:
#
#   prod → https://<slug>.educraft.paradigm-edu.net
#   dev  → https://<slug>.educraft-dev.paradigm-edu.net
#
# Example tenant (prod):
#   https://para-26071100.educraft.paradigm-edu.net/auth/feishu/para-26071100
#
# Configure (CI secrets / env):
#   CPH_FLEET                 required: dev | prod
#   PLATFORM_DEPLOY_HOST      default 39.107.254.4
#   PLATFORM_DEPLOY_USER      default root
#   PLATFORM_DEPLOY_SSH_KEY   required (path to private key)
#   PLATFORM_DEPLOY_PORT      default 22
#   PLATFORM_DEPLOY_BASE      default /srv/curriculum-project-hub
#   PLATFORM_DEPLOY_RELEASE   default git HEAD (must be safe for paths)
#   ALLOW_EMPTY_FLEET         optional: if 1, succeed when no silo matches
#
# This is a code-roll for existing silos. It does not create databases, domains,
# or Feishu apps — use new_silo.sh / apply_new_silo.sh for that.
set -euo pipefail

FLEET="${CPH_FLEET:?CPH_FLEET required (dev|prod)}"
case "$FLEET" in
  dev) FLEET_DOMAIN_SUFFIX="educraft-dev.paradigm-edu.net" ;;
  prod) FLEET_DOMAIN_SUFFIX="educraft.paradigm-edu.net" ;;
  *)
    echo "CPH_FLEET must be dev or prod, got: $FLEET" >&2
    exit 1
    ;;
esac

HOST="${PLATFORM_DEPLOY_HOST:-39.107.254.4}"
DEPLOY_USER="${PLATFORM_DEPLOY_USER:-root}"
PORT="${PLATFORM_DEPLOY_PORT:-22}"
SSH_KEY="${PLATFORM_DEPLOY_SSH_KEY:?PLATFORM_DEPLOY_SSH_KEY required}"
BASE="${PLATFORM_DEPLOY_BASE:-/srv/curriculum-project-hub}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
RELEASE_ID="${PLATFORM_DEPLOY_RELEASE:-$(git -C "$REPO_ROOT" rev-parse --verify HEAD)}"
[[ "$RELEASE_ID" =~ ^[A-Za-z0-9._-]+$ ]] || {
  echo "invalid PLATFORM_DEPLOY_RELEASE: $RELEASE_ID" >&2
  exit 1
}
ALLOW_EMPTY_FLEET="${ALLOW_EMPTY_FLEET:-0}"

HUB_DIR="$BASE/releases/$RELEASE_ID/hub"
RELEASE_DIR="$BASE/releases/$RELEASE_ID"
SSH_OPTS=(-i "$SSH_KEY" -p "$PORT" -o BatchMode=yes -o StrictHostKeyChecking=accept-new)

echo "[fleet] fleet=$FLEET domain=*.$FLEET_DOMAIN_SUFFIX host=$DEPLOY_USER@$HOST release=$RELEASE_ID"

# --- 1. Publish immutable release (shared; flock so parallel fleet jobs do not race) ---
release_ready=false
if ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "test -f '$RELEASE_DIR/.complete'"; then
  release_ready=true
  echo "[fleet] release already complete: $RELEASE_DIR"
fi

if [ "$release_ready" = false ]; then
  # Drop a prior incomplete tree for this release id (failed CI leave leftovers).
  ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" bash -s <<REMOTE
set -euo pipefail
flock /var/lock/cph-hub-release-publish bash -c '
  set -euo pipefail
  if [ -f "$RELEASE_DIR/.complete" ]; then
    exit 0
  fi
  if [ -e "$RELEASE_DIR" ]; then
    echo "[fleet] removing incomplete release: $RELEASE_DIR"
    rm -rf "$RELEASE_DIR"
  fi
  mkdir -p "$HUB_DIR"
'
REMOTE

  echo "[fleet] rsync hub → $HUB_DIR"
  rsync -az --delete \
    --exclude node_modules --exclude dist --exclude .env \
    --exclude admin-web/node_modules --exclude admin-web/build --exclude admin-web/.svelte-kit \
    -e "ssh ${SSH_OPTS[*]}" \
    "$REPO_ROOT/hub/" "$DEPLOY_USER@$HOST:$HUB_DIR/"

  echo "[fleet] npm ci + build (tsc + admin-web SPA)"
  ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" bash -s <<REMOTE
set -euo pipefail
flock /var/lock/cph-hub-release-publish bash -c '
  set -euo pipefail
  if [ -f "$RELEASE_DIR/.complete" ]; then
    echo "[fleet] release completed by another job"
    exit 0
  fi
  cd "$HUB_DIR"
  PUPPETEER_SKIP_DOWNLOAD=1 npm ci
  npm ci --prefix admin-web
  npm run audit:production
  npm run build
  test -f admin-web/build/index.html
  touch "$RELEASE_DIR/.complete"
'
REMOTE
fi

# --- 2. Discover silos in this fleet and roll them onto the release ---
ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" \
  env BASE="$BASE" HUB_DIR="$HUB_DIR" FLEET="$FLEET" \
  FLEET_DOMAIN_SUFFIX="$FLEET_DOMAIN_SUFFIX" ALLOW_EMPTY_FLEET="$ALLOW_EMPTY_FLEET" \
  bash -s <<'REMOTE'
set -euo pipefail

fleet_match() {
  local public_url="$1"
  local host="${public_url#https://}"
  host="${host#http://}"
  host="${host%%/*}"
  host="${host%%:*}"
  case "$FLEET" in
    prod)
      [[ "$host" == *."$FLEET_DOMAIN_SUFFIX" || "$host" == "$FLEET_DOMAIN_SUFFIX" ]] || return 1
      # Avoid matching educraft-dev when suffix is educraft.…
      [[ "$host" != *.educraft-dev.paradigm-edu.net && "$host" != educraft-dev.paradigm-edu.net ]] || return 1
      return 0
      ;;
    dev)
      [[ "$host" == *."$FLEET_DOMAIN_SUFFIX" || "$host" == "$FLEET_DOMAIN_SUFFIX" ]]
      return
      ;;
    *)
      return 1
      ;;
  esac
}

env_value() {
  local file="$1"
  local key="$2"
  # Do not source secrets — only read the KEY=value line.
  local line
  line="$(grep -E "^${key}=" "$file" | tail -n1 || true)"
  printf '%s' "${line#"${key}="}"
}

matched=0
shopt -s nullglob
for env_file in "$BASE"/.secrets/*/platform.env; do
  instance_id="$(basename "$(dirname "$env_file")")"
  public_url="$(env_value "$env_file" HUB_PUBLIC_BASE_URL)"
  if [ -z "$public_url" ]; then
    echo "[fleet] skip $instance_id: HUB_PUBLIC_BASE_URL empty"
    continue
  fi
  if ! fleet_match "$public_url"; then
    echo "[fleet] skip $instance_id: $public_url (not $FLEET)"
    continue
  fi

  port="$(env_value "$env_file" PORT)"
  workspace="$(env_value "$env_file" HUB_PROJECT_WORKSPACE_ROOT)"
  unit="cph-hub-${instance_id}.service"
  if [ ! -f "/etc/systemd/system/$unit" ]; then
    echo "[fleet] skip $instance_id: unit missing ($unit)" >&2
    continue
  fi

  # Prefer unit file ceilings (MemoryMax=16G) over systemctl's byte form.
  memory_max="$(sed -n 's/^MemoryMax=//p' "/etc/systemd/system/$unit" | tail -n1)"
  cpu_quota="$(sed -n 's/^CPUQuota=//p' "/etc/systemd/system/$unit" | tail -n1)"
  tasks_max="$(sed -n 's/^TasksMax=//p' "/etc/systemd/system/$unit" | tail -n1)"

  if [ -z "$port" ] || [ -z "$workspace" ] || [ -z "$memory_max" ] || [ -z "$cpu_quota" ] || [ -z "$tasks_max" ]; then
    echo "[fleet] error: $instance_id missing PORT/workspace/ceilings (port=$port workspace=$workspace memory=$memory_max cpu=$cpu_quota tasks=$tasks_max)" >&2
    exit 1
  fi

  echo "[fleet] roll $instance_id → $HUB_DIR ($public_url port=$port)"
  BASE="$BASE" \
    HUB_DIR="$HUB_DIR" \
    INSTANCE_ID="$instance_id" \
    WORKSPACE_ROOT="$workspace" \
    PORT="$port" \
    MEMORY_MAX="$memory_max" \
    CPU_QUOTA="$cpu_quota" \
    TASKS_MAX="$tasks_max" \
    bash "$HUB_DIR/deploy/install_service.sh"

  systemctl restart "$unit"
  systemctl is-active --quiet "$unit"

  health="http://127.0.0.1:${port}/api/healthz"
  ok=false
  for _ in $(seq 1 45); do
    if curl --fail --silent "$health" >/dev/null 2>&1; then
      ok=true
      break
    fi
    sleep 1
  done
  if [ "$ok" != true ]; then
    curl --fail --silent --show-error "$health" >/dev/null
  fi
  echo "[fleet] healthy $instance_id ($health)"
  matched=$((matched + 1))
done

if [ "$matched" -eq 0 ]; then
  msg="[fleet] no silos matched *.$FLEET_DOMAIN_SUFFIX under $BASE/.secrets"
  if [ "$ALLOW_EMPTY_FLEET" = "1" ]; then
    echo "$msg (ALLOW_EMPTY_FLEET=1)"
    exit 0
  fi
  echo "$msg" >&2
  exit 1
fi

echo "[fleet] rolled $matched silo(s) on $FLEET"
REMOTE

echo "[fleet] done fleet=$FLEET release=$RELEASE_ID hub=$HUB_DIR"