feat: secure organization provider credentials

hub/deploy/README.md

@@ -14,13 +14,52 @@ sudo BASE=/srv/curriculum-project-hub \
 ```
 
 On a clean host the first invocation creates
-`/srv/curriculum-project-hub/.secrets/platform.env` with every supported
-production key and exits with status 78. Fill every required blank, set
-`CPH_BIN` to an executable compatible `cph`, keep the file mode at `0600`, and
-run the same command again. No systemd unit is installed until configuration,
+`/srv/curriculum-project-hub/.secrets/platform.env` and a random root-owned
+`secret-keyring.json`, then exits with status 78. Copy the keyring to a
+separately protected recovery location before continuing; it must not live only
+on this host or in the ordinary database/workspace backup. Fill every required
+environment blank, set `CPH_BIN` to an executable compatible `cph`, keep both
+source files root-only at mode `0600`, and run the same command again. No
+systemd unit is installed until configuration,
 paths, Node, bubblewrap, `socat`, `cph --version`, PostgreSQL connectivity and directory
 ownership all pass preflight.
 
+The service account cannot read the keyring source. The installed unit uses
+systemd `LoadCredential` to materialize a read-only `cph-secret-keyring` for the
+running service. Hub refuses production startup if this credential is missing,
+malformed, or does not contain its declared active key. Provider credentials
+are configured through the Organization admin connection API, never through
+process-global `ANTHROPIC_*` variables; preflight rejects those legacy settings.
+
+Rotate the local KEK only from the controlled host console. The command first
+atomically adds a new active key while retaining every previous key, then
+authenticates and rewraps every stored Provider DEK, writes redacted org audit
+events, and verifies the complete set again:
+
+```sh
+sudo bash -c '
+  set -euo pipefail
+  systemctl stop cph-hub.service
+  set -a
+  . /srv/curriculum-project-hub/.secrets/platform.env
+  set +a
+  node /srv/curriculum-project-hub/hub/dist/deployment/rotate-secret-kek.js \
+    --keyring-file /srv/curriculum-project-hub/.secrets/secret-keyring.json
+  systemctl start cph-hub.service
+'
+```
+
+The CLI refuses to rotate while `cph-hub.service` is active; stopping the only
+writer prevents a concurrent BYOK version from being appended under the old
+in-memory keyring. A PostgreSQL transaction advisory lock excludes a second
+rotation and is released automatically if the CLI or host crashes. The
+operation is therefore rerunnable after interruption: old and new keys remain
+in the atomically replaced keyring, so partially rewrapped rows stay
+decryptable. It also refuses success while any envelope still names a
+non-active KEK. Back up the updated keyring to the separate recovery location
+and complete a restore preflight before considering old-key retirement. The
+pilot deliberately does not auto-delete previous keys.
+
 The deployment path also runs `npm run audit:production` from the locked clean
 install before building or restarting the service. A current high or critical
 production advisory therefore blocks deployment instead of becoming a warning

hub/deploy/cph-hub.service

@@ -7,6 +7,7 @@ Wants=network-online.target
 # this start-time assertion keeps later package/host changes fail-closed.
 AssertPathExists=__BWRAP_BIN__
 AssertPathExists=__SOCAT_BIN__
+AssertPathExists=__KEYRING_FILE__
 
 [Service]
 Type=simple
@@ -18,6 +19,9 @@ Environment=HOME=__SERVICE_HOME__
 Environment=XDG_STATE_HOME=__STATE_DIR__
 Environment=XDG_CACHE_HOME=__CACHE_DIR__
 Environment=PATH=__RUNTIME_PATH__
+# ADR-0024: the root-owned source remains unreadable by the service account;
+# systemd materializes a read-only per-unit credential at runtime.
+LoadCredential=cph-secret-keyring:__KEYRING_FILE__
 UMask=0027
 # Run prisma migrations before each start, then launch the Hub.
 ExecStartPre=__NODE_BIN__ __HUB_DIR__/node_modules/prisma/build/index.js migrate deploy --schema __HUB_DIR__/prisma/schema.prisma

hub/deploy/install_service.sh

@@ -21,6 +21,7 @@ WORKSPACE_ROOT="${WORKSPACE_ROOT:-/var/lib/cph-hub/workspaces}"
 HOST="${HOST:-127.0.0.1}"
 PORT="${PORT:-8788}"
 ENV_FILE="${ENV_FILE:-$BASE/.secrets/platform.env}"
+KEYRING_FILE="${KEYRING_FILE:-$BASE/.secrets/secret-keyring.json}"
 SYSTEMD_UNIT_DIR="${SYSTEMD_UNIT_DIR:-/etc/systemd/system}"
 NODE_BIN="${NODE_BIN:-$(command -v node || true)}"
 BWRAP_BIN="${BWRAP_BIN:-$(command -v bwrap || true)}"
@@ -63,7 +64,7 @@ if [ "$SERVICE_USER" = "root" ] || [ "$SERVICE_GROUP" = "root" ]; then
   fail "the Hub service identity must not be root"
 fi
 
-for command in getent groupadd useradd install stat systemctl id; do
+for command in getent groupadd useradd install stat systemctl id openssl date mktemp; do
   require_command "$command"
 done
 for pair in \
@@ -97,6 +98,7 @@ for pair in \
   "CACHE_DIR:$CACHE_DIR" \
   "WORKSPACE_ROOT:$WORKSPACE_ROOT" \
   "ENV_FILE:$ENV_FILE" \
+  "KEYRING_FILE:$KEYRING_FILE" \
   "SYSTEMD_UNIT_DIR:$SYSTEMD_UNIT_DIR" \
   "NODE_BIN:$NODE_BIN" \
   "BWRAP_BIN:$BWRAP_BIN" \
@@ -107,6 +109,32 @@ for pair in \
   require_safe_unit_value "${pair%%:*}" "${pair#*:}"
 done
 
+KEYRING_CREATED=false
+if [ -L "$KEYRING_FILE" ]; then
+  fail "local secret keyring must not be a symlink: $KEYRING_FILE"
+fi
+if [ ! -e "$KEYRING_FILE" ]; then
+  install -d -o root -g root -m 0700 "$(dirname "$KEYRING_FILE")"
+  key_id="local-$(date -u +%Y%m%dT%H%M%SZ)"
+  key_value="$(openssl rand -base64 32)"
+  tmp_keyring="$(mktemp "$(dirname "$KEYRING_FILE")/.secret-keyring.XXXXXX")"
+  trap 'rm -f "${tmp_keyring:-}"' EXIT
+  chmod 0600 "$tmp_keyring"
+  printf '{"version":1,"activeKeyId":"%s","keys":{"%s":"%s"}}\n' \
+    "$key_id" "$key_id" "$key_value" >"$tmp_keyring"
+  install -o root -g root -m 0600 "$tmp_keyring" "$KEYRING_FILE"
+  rm -f "$tmp_keyring"
+  tmp_keyring=""
+  trap - EXIT
+  unset key_value
+  KEYRING_CREATED=true
+  echo "[install] generated local secret keyring: $KEYRING_FILE" >&2
+fi
+[ -f "$KEYRING_FILE" ] || fail "local secret keyring is not a regular file: $KEYRING_FILE"
+keyring_metadata="$(stat -c '%u:%g:%a' -- "$KEYRING_FILE")"
+[ "$keyring_metadata" = "0:0:600" ] || \
+  fail "local secret keyring must be owned by root:root with mode 0600; found $keyring_metadata"
+
 if [ -L "$ENV_FILE" ]; then
   fail "environment file must not be a symlink: $ENV_FILE"
 fi
@@ -118,9 +146,6 @@ if [ ! -e "$ENV_FILE" ]; then
 # rerunning install_service.sh; failed preflight never installs the unit.
 NODE_ENV=production
 DATABASE_URL=
-ANTHROPIC_BASE_URL=https://openrouter.ai/api
-ANTHROPIC_AUTH_TOKEN=
-ANTHROPIC_API_KEY=
 FEISHU_APP_ID=
 FEISHU_APP_SECRET=
 FEISHU_BOT_OPEN_ID=
@@ -132,18 +157,24 @@ HUB_PUBLIC_BASE_URL=
 HUB_SESSION_SECRET=
 HUB_AGENT_MAX_TURNS=25
 HUB_FEISHU_LISTENER_ENABLED=true
-CPH_SANDBOX_EXTRA_DENY_READ=$ENV_FILE
+CPH_SANDBOX_EXTRA_DENY_READ=$ENV_FILE:$KEYRING_FILE
 EOF
   chmod 0600 "$ENV_FILE"
   echo "[install] seeded complete production configuration: $ENV_FILE" >&2
+  echo "[install] copy $KEYRING_FILE to the separately protected keyring backup before continuing" >&2
   echo "[install] fill every required blank, install cph at CPH_BIN, then rerun" >&2
   exit 78
 fi
 [ -f "$ENV_FILE" ] || fail "environment file is not a regular file: $ENV_FILE"
+if [ "$KEYRING_CREATED" = true ]; then
+  echo "[install] copy $KEYRING_FILE to the separately protected keyring backup, then rerun" >&2
+  exit 78
+fi
 
 PREFLIGHT_ARGS=(
   "$NODE_BIN" "$PREFLIGHT_JS"
   --env-file "$ENV_FILE"
+  --keyring-file "$KEYRING_FILE"
   --node-bin "$NODE_BIN"
   --runuser-bin "$RUNUSER_BIN"
   --setpriv-bin "$SETPRIV_BIN"
@@ -249,6 +280,7 @@ sed \
   -e "s|__WORKSPACE_ROOT__|$WORKSPACE_ROOT|g" \
   -e "s|__HUB_DIR__|$HUB_DIR|g" \
   -e "s|__ENV_FILE__|$ENV_FILE|g" \
+  -e "s|__KEYRING_FILE__|$KEYRING_FILE|g" \
   -e "s|__NODE_BIN__|$NODE_BIN|g" \
   -e "s|__BWRAP_BIN__|$BWRAP_BIN|g" \
   -e "s|__SOCAT_BIN__|$SOCAT_BIN|g" \