chore: release v0.0.24

This reverts commit 4f8df12fb0.

.gitea/workflows/checker-check.yml

@@ -19,8 +19,10 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: Install Rust toolchain (stable + clippy + rustfmt)
-        uses: dtolnay/rust-toolchain@stable
+      # Keep local rustup, rustfmt output, clippy, and CI on one reproducible
+      # compiler release. Update this together with rust-toolchain.toml.
+      - name: Install Rust toolchain (1.92.0 + clippy + rustfmt)
+        uses: dtolnay/rust-toolchain@1.92.0
         with:
           components: clippy, rustfmt
 
@@ -35,6 +37,12 @@ jobs:
           restore-keys: |
             cargo-${{ runner.os }}-
 
+      - name: Install cargo-audit 0.22.2
+        run: cargo install cargo-audit --version 0.22.2 --locked
+
+      - name: Audit Rust vulnerabilities and unsoundness
+        run: cargo audit --deny unsound
+
       # Fonts for typst compilation (WU-4 onward): the engineering files are
       # CJK-heavy, so the compile-check / build tests need a CJK font present.
       # Harmless before those crates exist.

.gitea/workflows/hub-check.yml

@@ -36,13 +36,19 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: "20"
+          node-version: "24"
           cache: npm
           cache-dependency-path: hub/package-lock.json
 
       - name: Install dependencies
         run: npm ci
 
+      - name: Audit production Node dependencies
+        run: npm run audit:production
+
+      - name: Install Linux sandbox dependency
+        run: sudo apt-get update && sudo apt-get install --yes bubblewrap socat
+
       - name: Wait for Postgres
         run: |
           node <<'NODE'
@@ -85,6 +91,13 @@ jobs:
           cd ..
           cargo install --path crates/cph-cli --locked
 
+      - name: Prove real Claude SDK Bash sandbox boundary
+        run: |
+          sudo install -d -o "$(id -u)" -g "$(id -g)" -m 0700 /w/t
+          CPH_SANDBOX_TEST_ROOT=/w/t \
+            /usr/bin/setpriv --no-new-privs \
+            npx vitest run test/integration/agent-sandbox-linux.test.ts
+
       - name: Run unit tests
         run: npx vitest run test/unit
 
@@ -93,36 +106,12 @@ jobs:
       - name: Run integration tests (mock provider, real prisma + cph)
         run: |
           npx prisma migrate deploy --schema prisma/schema.prisma
-          npx vitest run test/integration --exclude test/integration/real-model.test.ts
+          npx vitest run test/integration \
+            --exclude test/integration/real-model.test.ts \
+            --exclude test/integration/agent-sandbox-linux.test.ts
         env:
           DATABASE_URL: postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test
 
-      - name: Smoke health endpoint
-        run: |
-          PORT=8878 \
-          NODE_ENV=production \
-          DATABASE_URL=postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test \
-          ANTHROPIC_AUTH_TOKEN=smoke \
-          FEISHU_APP_ID=cli_smoke \
-          FEISHU_APP_SECRET=smoke_secret \
-          FEISHU_BOT_OPEN_ID=ou_smoke \
-          HUB_FEISHU_LISTENER_ENABLED=false \
-          npm start &
-          pid=$!
-          trap 'kill "$pid" 2>/dev/null || true' EXIT
-          for attempt in $(seq 1 30); do
-            if curl --fail --silent "http://127.0.0.1:8878/api/healthz" >/dev/null; then
-              exit 0
-            fi
-            if ! kill -0 "$pid" 2>/dev/null; then
-              echo "hub server exited before health check passed"
-              wait "$pid"
-              exit 1
-            fi
-            sleep 1
-          done
-          curl --fail --silent --show-error "http://127.0.0.1:8878/api/healthz" >/dev/null
-
       # Real-model tests are opt-in: set RUN_REAL_MODEL_TESTS=true and provide
       # OPENROUTER_API_KEY when a branch should hit live OpenRouter.
       - name: Run real-model integration tests (optional, needs secret)

.scratch/saas-production-readiness/assets/backup-migration-disaster-recovery-audit.md

@@ -0,0 +1,325 @@
+# Backup, migration, and disaster-recovery audit
+
+## Verdict
+
+The current Hub has no production backup or restore control plane. PostgreSQL
+can be dumped and restored manually, but the service's authoritative state is
+split between PostgreSQL and mutable Project workspaces, with no shared
+consistency point, backup catalog, off-host retention, restore workflow,
+recovery objectives, or recovery drill.
+
+The current deployment path can also delete customer workspaces when the
+documented relative workspace root is used: `deploy_platform.sh` applies
+`rsync --delete` to the live Hub directory while `.env.example` places
+workspaces below `./data/project-workspaces`. Project creation has the same
+cross-store weakness at a smaller scale: it creates the workspace before the
+database transaction and leaves an orphan directory when that transaction
+fails.
+
+Migration safety is incomplete even though the current PostgreSQL migration
+engine rolled back a deliberately failing migration's DDL in the executed
+probe. The failed migration record blocks later migrations, migrations execute
+inside every service start, no compatibility declaration or production-data
+rehearsal exists, and current history contains destructive changes that make an
+old application query fail after schema upgrade. Application rollback is
+therefore not database rollback.
+
+## Contract and persistent-state baseline
+
+The audit preserves the accepted contracts rather than inventing missing
+product policy:
+
+- `Organization` is the tenant root, but `User` is global and can belong to
+  multiple Organizations. A selective Organization restore is not equivalent
+  to restoring a self-contained database schema.
+- Project workspaces are authoritative curriculum-engineering files bounded by
+  ADR-0018. PostgreSQL stores their absolute `Project.workspaceDir` locations;
+  it does not store their file content.
+- Run, provider cursor, ingress, delivery, audit, and workspace commit/recovery
+  semantics remain split across the unresolved lifecycle tickets. A backup
+  design cannot silently choose their crash behavior.
+- ADR-0022 requires storage ceilings and emergency `DRAIN`/`STOP_NOW`, but does
+  not decide backup frequency, retention, recovery objectives, or tenant-level
+  restore scope.
+- Audit retention is still `OPEN`; regulatory backup retention and deletion
+  obligations require product/legal input already recorded in the map's Fog.
+
+The recovery set has at least four classes:
+
+| State | Current location | Recovery significance |
+| --- | --- | --- |
+| Tenant and workflow state | PostgreSQL | Organizations, identities, membership, permissions, Feishu bindings, sessions, runs, locks, receipts, usage facts, and audit records |
+| Customer artifacts | `HUB_PROJECT_WORKSPACE_ROOT` | Project source files, generated artifacts, inbound attachments, and any workspace-local session/history files |
+| Credentials and configuration | `/srv/.../.secrets/platform.env` today; future org-scoped encrypted connections plus external key material | Database/Feishu/provider access and ability to decrypt restored Organization credentials |
+| Release identity | application release, Prisma migrations, `cph`, runtime/toolchain manifest | Required to start a schema-compatible application and reproduce outputs after restore |
+
+Caches, build output, and reinstallable dependencies should be classified as
+regenerable rather than silently mixed into the protected recovery set.
+
+## Executed evidence
+
+All probes used isolated temporary paths or databases. No production or shared
+test data was modified.
+
+### 1. The deployment rsync pattern deletes an in-tree workspace
+
+A temporary live Hub directory was given the documented workspace shape and a
+sentinel customer file, then the exact deployment rsync include/exclude pattern
+was applied:
+
+```text
+rsync -a --delete \
+  --exclude node_modules --exclude dist --exclude .env \
+  hub/ <temporary-live-hub>/
+
+workspace_survived=false
+reproduced: deploy rsync pattern deletes an in-tree workspace root
+```
+
+This is deterministic: `data/` is absent from the release source and is not an
+exclude, so `--delete` removes it from the destination. The root cause is that
+the documented persistent path is nested inside the mutable release tree and
+the deploy contract does not distinguish release artifacts from state.
+
+### 2. A rejected Project transaction leaves an orphan workspace
+
+A minimal Node harness called the real built `createProjectFromOrgAdmin` with a
+valid Organization admin and a missing Folder. The fake Prisma boundary rejected
+inside the same transaction seam used by production. The harness asserted that
+the Organization workspace directory must remain empty and was run twice:
+
+```text
+node .scratch/saas-production-readiness/probes/orphan-workspace.mjs
+
+orphan_workspace_entries=project_4aa0b6792c4547059d2d92b505b9299e
+exit_code=1
+
+orphan_workspace_entries=project_190e21afb8854244872b1241a600aa58
+exit_code=1
+```
+
+`createManagedProject` calls `mkdir` before `prisma.$transaction`; the failure
+path has no compensating removal, staged rename, or orphan reconciler. The
+throwaway harness was removed after reproduction.
+
+### 3. PostgreSQL rolled back failed migration DDL but retained a blocking failure
+
+An isolated database and throwaway Prisma project applied one migration that
+created a table, divided by zero, then would have created another table. The
+real repository Prisma CLI returned `P3018`:
+
+```text
+migration_exit=1
+before_failure=missing
+after_failure=missing
+20260710000000_partial_failure|finished=false|rolled_back=false|steps=0|logs=true
+```
+
+This specific PostgreSQL/Prisma 6.19.3 path did **not** leave partial DDL, so the
+audit does not claim that it did. It did leave an unfinished
+`_prisma_migrations` row, and Prisma's error explicitly refused new migrations
+until operator recovery. Because systemd runs `migrate deploy` as
+`ExecStartPre`, the application cannot start past that state. The throwaway
+database and migration were removed. This one multi-statement result is not a
+general atomicity guarantee: Prisma's first-party guidance says PostgreSQL
+migrations are not transaction-wrapped by default, so each migration must make
+its transaction and partial-failure behavior explicit.
+
+### 4. The current schema is not backward-compatible with an older application
+
+The application immediately before the typed-principal migration queried
+`PermissionGrant.principal`. That migration drops the column in favor of
+`principalType` plus `principalId`. Running the old read against the current
+test database produced:
+
+```text
+ERROR: column "principal" does not exist
+HINT: Perhaps you meant to reference the column "PermissionGrant.principalId".
+old_query_exit=1
+```
+
+This is direct evidence that switching the application symlink back after that
+migration cannot restore service. The migration history also deletes duplicate
+grant rows and later drops `ExternalPrincipalMembership.source`; there are no
+downgrade migrations.
+
+### 5. A database-only custom dump restores cleanly
+
+As a positive baseline, `pg_dump --format=custom` captured the current
+`cph_hub_test` database and `pg_restore --exit-on-error --no-owner` restored it
+into a new isolated database:
+
+```text
+restored_tables=24
+finished_migrations=9
+failed_migrations=0
+dump_bytes=76113
+```
+
+All nine finished migration checksums in the source database also matched the
+checked-in migration files. This proves a manual logical database round trip on
+the current PostgreSQL 14.20 workstation. It does not prove workspace,
+credentials, global database roles, production volume, cross-version restore,
+application readiness, or an operational RPO/RTO.
+
+## Critical findings
+
+### 1. No backup, retention, or restore mechanism exists
+
+The repository has no backup/restore command, systemd timer, off-host target,
+encryption/immutability policy, retention policy, backup manifest/checksum,
+success/failure alert, restore command, or scheduled recovery drill. The
+installer assumes PostgreSQL and its role/database already exist. A database
+dump alone does not reproduce those global prerequisites or the filesystem and
+secret state above.
+
+There is also no decided RPO, RTO, backup frequency, retention duration,
+geographic copy rule, deletion-in-backups rule, or ownership/escalation path.
+Those values are product and operations decisions, not safe constants to infer
+from the single-host topology.
+
+### 2. Persistent workspaces are inside the documented release tree
+
+`.env.example` recommends `./data/project-workspaces`; systemd's working
+directory is the live Hub directory; deployment mutates that same directory
+with `rsync --delete`. No preflight forbids an in-release workspace root and no
+exclude preserves it. This can turn a routine deploy into irreversible customer
+data loss before build, migration, restart, or readiness runs.
+
+The persistent workspace root must live outside every release directory, have
+explicit ownership/mode and capacity, and be impossible for release retention
+or `rsync --delete` to traverse. Immutable release work already exists as a
+separate ticket, but its acceptance test must include this sentinel case.
+
+### 3. PostgreSQL and workspaces have no consistency boundary
+
+PostgreSQL can provide a transactionally consistent database snapshot, but the
+Hub mutates workspace files independently and has no generation, journal,
+commit marker, snapshot ID, or shared checkpoint that binds filesystem content
+to the corresponding database state. A database dump and filesystem copy taken
+at different instants can disagree about Project existence, active runs,
+provider cursors, delivered messages, and file effects.
+
+Project creation already demonstrates the same missing compensation boundary.
+A replacement host has an additional problem: `Project.workspaceDir` is an
+absolute path and runtime paths from the database are used directly; restore
+has no same-mount validation, safe rebasing operation, or missing/orphan scan.
+
+For the initial single-host topology, a quiesced backup checkpoint is the
+smallest credible route: stop new admission, drain/cancel according to the
+accepted run contract, reach durable zero in-flight work, capture database and
+workspace artifacts under one manifest, then resume. An online scheme would
+need a newly designed journal/generation protocol; it must not be implied by
+two independent copy commands.
+
+### 4. Schema mutation is coupled to service start
+
+Both `npm start` and the systemd unit execute `prisma migrate deploy`; every
+ordinary restart can therefore become a schema rollout. The release path has no
+separate migration lock/gate, `migrate status` check, production-data rehearsal,
+duration/lock budget, pre-migration recovery checkpoint, or compatibility
+manifest. CI only applies the full migration history to a fresh empty
+PostgreSQL database, so it does not exercise real upgrade volume, legacy values,
+old/new application overlap, failure recovery, or rollback compatibility.
+
+The current failed-migration behavior is fail-closed, which is better than
+starting on an unknown schema, but there is no operator procedure for inspecting
+the failure, deciding roll-forward versus mark-rolled-back, restoring the
+checkpoint, or preventing a restart loop. Existing destructive migrations prove
+that application rollback cannot be assumed safe.
+
+### 5. Restore has no isolated verification or cutover gate
+
+There is no procedure that restores into a non-production environment with
+outbound Feishu/provider traffic disabled, verifies backup checksums and
+migration state, validates workspace paths/content/permissions, checks tenant
+relationships, decrypts a sample org connection, reconciles non-terminal work,
+runs application readiness and representative journeys, and measures recovery
+time/point before controlled cutover.
+
+Starting the current server is not a neutral verifier: startup deletes all
+Project locks and independently marks only `ACTIVE` runs failed. Restored data
+must first pass the durable lifecycle reconciliation designed by the run-state
+tickets; otherwise validation itself mutates the evidence and can leave
+`WAITING_FOR_USER` inconsistent.
+
+### 6. Full-service and Organization-selective recovery are different products
+
+A full-service restore can recreate the shared database and all workspaces.
+Restoring one Organization is much harder: users are global and may belong to
+multiple Organizations, permission/audit/history rows span many related tables,
+workspace paths are external, and Feishu/provider connections have external
+state. The repository has no export/import identity mapping or conflict policy.
+
+The initial production gate must at least prove full-service disaster recovery.
+Whether Organization-selective restore/export is promised, and how retained
+backups interact with customer deletion, remains an explicit product/legal
+decision rather than an accidental property of `pg_restore` filters.
+
+## Required recovery contract
+
+Production readiness requires the following guarantees, with exact numerical
+objectives supplied by the pending policy decision:
+
+1. **Declared objectives and ownership.** RPO/RTO, backup frequency, retention,
+   copy locations, encryption/key custody, immutability, deletion handling,
+   full-service versus Organization restore scope, operator ownership, and
+   escalation are documented and testable.
+2. **Complete backup set.** One cataloged manifest identifies release and
+   migration checksums, PostgreSQL artifact and global prerequisites, workspace
+   snapshot, configuration inventory, encrypted org-secret data, external key
+   recovery material, timestamps, sizes, and cryptographic checksums. Backup
+   data and decryption material are not kept only on the protected host or in
+   the same failure domain.
+3. **One consistency point.** A backup is taken only after an observable
+   quiesce/checkpoint protocol, or by a proven online generation protocol that
+   binds database state and workspace effects. Accepted input, active runs,
+   queues, deliveries, locks, and provider cursors have explicit recovery
+   semantics.
+4. **Safe migration rollout.** Migration is a release phase, not an incidental
+   service start. The gate checks status/checksums, rehearses on representative
+   production-shaped data, records old/new application compatibility, uses
+   expand-and-contract for destructive changes, requires a recent verified
+   recovery checkpoint, and provides a tested failed-migration procedure.
+5. **Isolated restore verification.** Every backup class is periodically
+   restored to a clean isolated host/database with outbound integrations off.
+   Verification checks integrity, schema/migration state, global database
+   prerequisites, workspace completeness/path ownership, secret decryption,
+   tenant isolation, lifecycle reconciliation, application readiness, and
+   representative admin/Feishu/agent journeys.
+6. **Observable operation.** Backup age, duration, artifact/checksum status,
+   off-host replication, retention deletion, WAL/archive continuity if used,
+   restore-drill result, measured RPO/RTO, and failures are logged/metricized
+   and alerted with bounded cardinality.
+7. **Controlled cutover and fallback.** Disaster recovery has an explicit
+   authority, traffic/workload brake, DNS/proxy/Feishu reconnect sequence,
+   duplicate/gap checks, rollback/fallback point, verification checklist, and
+   incident evidence record.
+
+## Required release evidence
+
+- A deploy test proves an in-tree or release-overlapping workspace root is
+  rejected before any rsync/switch, while persistent sentinel files survive
+  build failure, successful release, rollback, and release pruning.
+- Project creation, archive/deletion, failed transaction, crash, and restore
+  tests prove database/workspace allocation has no silent missing or orphan
+  state; the reconciler reports rather than hides inconsistencies.
+- Every checked-in migration is rehearsed from the previous production-shaped
+  snapshot, with old/new compatibility and lock/runtime evidence. A deliberately
+  failed migration exercises the documented recovery route.
+- The backup implementation proves completeness, checksum verification,
+  encryption, off-host copy, retention, monitoring, and fail-closed partial
+  artifact cleanup.
+- A clean isolated host restores a representative database, workspaces,
+  secrets, and compatible release; lifecycle reconciliation and critical tenant
+  journeys pass with outbound integrations controlled.
+- Scheduled drills publish achieved recovery point and recovery time against the
+  accepted objectives, and alert when no recent successful drill exists.
+
+## Primary-source interpretation
+
+The official PostgreSQL, Prisma, and systemd behavior supporting this audit is
+collected with claim-level links in
+[Backup and migration primary sources](./backup-migration-primary-sources.md).
+That note distinguishes what a PostgreSQL-consistent dump proves from the
+cross-store guarantees the Hub must build itself.

.scratch/saas-production-readiness/assets/backup-migration-primary-sources.md

@@ -0,0 +1,574 @@
+# Backup, migration, and disaster-recovery primary-source research
+
+This note is the primary-source evidence pack for the Wayfinder ticket
+“Audit backup, migration, and disaster recovery safety.” External claims use
+only PostgreSQL, Prisma, systemd, and rsync first-party documentation. Project
+recommendations are explicitly labelled as derived implications rather than
+upstream guarantees.
+
+PostgreSQL links below use `docs/current`, which was PostgreSQL 18 when this
+research was performed on 2026-07-10. A production implementation must pin the
+deployed PostgreSQL major and verify the same pages for that major. The Hub
+currently depends on Prisma `^6.19.3`, so Prisma 6-specific command examples
+remain relevant even where the live documentation also describes Prisma 7.
+
+## Executive conclusions
+
+1. `pg_dump` can take a consistent **single-database** snapshot while ordinary
+   reads and writes continue, but it does not include the Hub's workspace tree,
+   cluster-global roles/tablespaces, or a WAL chain. PostgreSQL now explicitly
+   says it is generally not the right regular production-backup mechanism
+   outside simple cases. It is a useful logical/export layer, not proof of
+   whole-service disaster recovery.
+   ([`pg_dump` description](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION),
+   [continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
+2. Database PITR requires a physical base backup plus a continuous, unbroken
+   WAL sequence from the backup start through the recovery target. A successful
+   base-backup command alone does not prove that chain exists.
+   ([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING),
+   [making a base backup](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-BASE-BACKUP))
+3. `pg_verifybackup` is necessary but not sufficient: PostgreSQL explicitly
+   requires test restores followed by checks that the restored database works
+   and contains the expected data.
+   ([`pg_verifybackup`](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
+4. Prisma `migrate deploy` applies pending migrations, but it does not detect
+   schema drift and, on PostgreSQL, Prisma does not wrap a migration in a
+   transaction by default. A failed migration can therefore leave partial
+   schema changes requiring explicit operator recovery.
+   ([Prisma production workflow](https://www.prisma.io/docs/orm/prisma-migrate/workflows/development-and-production#production-and-testing-environments),
+   [Prisma transactional behaviour](https://www.prisma.io/blog/prisma-migrate-dx-primitives#what-if-schema-migrations-were-atomic),
+   [failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration))
+5. systemd supplies process lifecycle orchestration, not an application-level
+   consistency barrier. It sends the configured termination signal, waits up
+   to `TimeoutStopSec=`, then can forcibly kill remaining processes; a
+   successful `systemctl stop` does not prove that an `ExecStop=` command or
+   application drain succeeded.
+   ([`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=),
+   [`systemctl stop`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#stop%20PATTERN%E2%80%A6))
+6. The Hub persists one logical service state across two stores: PostgreSQL
+   records `Project.workspaceDir`, runs, messages, locks, and file-change audit
+   rows, while workspaces contain project files. A workspace transcript helper
+   exists but is not wired into the production runner. PostgreSQL's snapshot/
+   PITR guarantees cover only PostgreSQL. A whole-service recovery point
+   therefore requires an application-defined database/workspace checkpoint or
+   a journaled/versioned reconciliation protocol; neither PostgreSQL nor
+   systemd creates one automatically.
+   ([Hub schema](../../../hub/prisma/schema.prisma),
+   [workspace transcript helper](../../../hub/src/agent/transcript.ts),
+   [`pg_dump` scope](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION))
+
+## Current repository facts that make the boundaries material
+
+- The systemd unit runs `prisma migrate deploy` as `ExecStartPre=` before every
+  service start and uses `Restart=on-failure`.
+  ([`cph-hub.service`](../../../hub/deploy/cph-hub.service))
+- The deploy script updates the live Hub directory using
+  `rsync -az --delete`, builds in that same directory, restarts the unit, and
+  accepts a constant liveness endpoint as deployment success.
+  ([`deploy_platform.sh`](../../../hub/deploy/deploy_platform.sh))
+- `Project.workspaceDir` is database state, but the referenced directory holds
+  independently mutable tenant content. A workspace JSONL transcript helper
+  exists but is not wired into the production runner; file contents still live
+  in the workspace while the database separately carries run/message/file-change
+  facts.
+  ([Hub schema](../../../hub/prisma/schema.prisma),
+  [workspace transcript helper](../../../hub/src/agent/transcript.ts))
+
+These are project facts, not upstream guarantees. The sections below establish
+what the upstream tools do and, critically, what they do not do.
+
+## PostgreSQL logical dump and restore
+
+### Snapshot and concurrent-write semantics
+
+- `pg_dump` makes a consistent export even while the database is being used;
+  it does not block ordinary readers or writers. It dumps only one database.
+  Cluster-global objects such as roles and tablespaces require `pg_dumpall` or
+  `pg_dumpall --globals-only`.
+  ([`pg_dump` description](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION),
+  [whole-cluster logical dumps](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-ALL))
+- `pg_dumpall` invokes `pg_dump` once per database. Each database is internally
+  consistent, but the snapshots are not synchronized across databases.
+  ([whole-cluster logical dumps](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-ALL))
+- A parallel dump uses directory format, opens `njobs + 1` connections, and
+  uses synchronized snapshots so every worker sees the same data set. The
+  connection count and additional server load are part of its capacity cost.
+  ([`pg_dump --jobs`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+- Concurrent DDL can still interfere. The parallel-dump leader holds
+  `ACCESS SHARE` locks; if a later exclusive-lock request queues ahead of a
+  worker, that worker uses `NOWAIT` and aborts the dump rather than deadlocking.
+  `--lock-wait-timeout` can bound the initial lock wait.
+  ([`pg_dump --jobs` and lock behaviour](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+- The ordinary snapshot is suitable for disaster-recovery dump semantics, but
+  it need not correspond to a serial execution of all transactions that later
+  commit. `--serializable-deferrable` waits for an anomaly-free snapshot;
+  PostgreSQL says that option is not beneficial for a dump intended only for
+  disaster recovery, but it can matter for a continuously updated reporting
+  copy.
+  ([`--serializable-deferrable`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+- The default waits for dump files to be written safely to disk. Production
+  backups must not use `--no-sync`, because an immediate OS crash can otherwise
+  leave a corrupt dump.
+  ([`pg_dump --no-sync`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+
+Derived Hub implications:
+
+- An online database dump may be internally valid while referring to a
+  `workspaceDir` whose files are from another instant. “`pg_dump` succeeded” is
+  therefore not a whole-Hub consistency claim.
+- Backup scheduling must either serialize against schema migration/DDL or
+  treat any lock wait, abort, non-zero exit, and stderr warning as a failed
+  backup. A partially written archive must never be published as restorable.
+- If the deployed cluster contains more than the Hub database, a single Hub
+  dump is not a cluster backup. Even `pg_dumpall` does not create a single
+  cross-database snapshot.
+
+### Restore error and atomicity semantics
+
+- `pg_restore` defaults to continuing after SQL errors and reports an error
+  count at the end. `--exit-on-error` is required for fail-fast behaviour.
+  `--single-transaction` wraps all emitted commands in `BEGIN`/`COMMIT`, makes
+  the restore all-or-nothing, and implies `--exit-on-error`.
+  ([`pg_restore` error and transaction options](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
+- Parallel restore uses separate connections for data loading, index creation,
+  and constraint creation. It supports only custom/directory archives and
+  cannot be combined with `--single-transaction`; it is not a whole-restore
+  atomic operation.
+  ([`pg_restore --jobs`](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
+- A single giant restore transaction can be impractical because it locks every
+  restored object and may exhaust the lock table. PostgreSQL 18 offers
+  `--transaction-size=N` as a bounded intermediate option; it implies
+  `--exit-on-error` but is not all-or-nothing.
+  ([`pg_restore --transaction-size`](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
+- Plain SQL restored by `psql` also continues after errors unless
+  `ON_ERROR_STOP` is enabled; a single transaction is separately required for
+  all-or-nothing behaviour.
+  ([restoring an SQL dump](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-RESTORE))
+- PostgreSQL recommends restoring into a truly empty database based on
+  `template0`; otherwise local additions in `template1` can cause duplicate
+  definition errors. Required roles must exist before owner/ACL restoration.
+  ([restoring an SQL dump](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-RESTORE))
+- A dump can execute arbitrary code selected by superusers of the source
+  database. Partial dump/restore does not remove that trust boundary.
+  ([`pg_dump` security warning](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION))
+- A `pg_dump` binary can dump older servers but refuses a server newer than its
+  own major version. Output is intended to load into newer PostgreSQL versions,
+  but loading into an older major is not guaranteed.
+  ([`pg_dump` cross-version notes](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-NOTES))
+
+Derived Hub implications:
+
+- Restore should target a new isolated database/cluster. If a non-atomic or
+  parallel restore reports any error, discard that target and start again;
+  never promote a partially restored database.
+- The runbook must record the restore atomicity strategy and exact PostgreSQL
+  client/server versions. A green process exit without explicit fail-fast
+  options is insufficient evidence.
+- A restored database must remain disconnected from real Feishu/provider
+  side-effect endpoints until application-level validation completes.
+
+## Physical backup, WAL, and PITR
+
+### Base backups
+
+- `pg_basebackup` backs up a running PostgreSQL cluster without blocking other
+  clients. A full backup is an exact copy of the entire cluster and can seed
+  PITR or a standby; it cannot select one database or object.
+  ([`pg_basebackup` description](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- It uses the replication protocol. The account needs `REPLICATION` or
+  superuser privilege, `pg_hba.conf` must permit it, and `max_wal_senders`
+  needs capacity for the backup and, when streaming WAL, another connection.
+  ([`pg_basebackup` requirements](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- Default `-X stream` obtains WAL over a second connection while copying. With
+  `-X fetch`, WAL is collected only at the end; if required WAL was recycled,
+  the backup fails and is unusable.
+  ([`pg_basebackup --wal-method`](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- The default waits for backup files to be durable. `--no-sync` is explicitly
+  unsuitable for production because an OS crash can corrupt the backup.
+  ([`pg_basebackup --no-sync`](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- A manifest is generated by default, with per-file checksums. A SHA manifest
+  can detect malicious modification only if the manifest itself is held in a
+  separate trusted location or otherwise authenticated.
+  ([`pg_basebackup` manifest options](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+
+Derived minimum backup record:
+
+- source cluster/system identifier and PostgreSQL version;
+- backup tool version, start/end time, start/end LSN and WAL range;
+- manifest digest and its independently protected location;
+- destination object/version, byte count, exit status, and verification state;
+- an explicit `VALID`, `INVALID`, or `QUARANTINED` state so a failed leftover
+  directory cannot enter the retention set merely because files exist.
+
+### WAL archival and recovery
+
+- PITR combines a physical backup with WAL replay. It can stop at a chosen time
+  and reach a consistent database state. Logical dumps do not contain enough
+  information for WAL replay.
+  ([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
+- Successful recovery requires a continuous sequence of archived WAL extending
+  back at least to the base-backup start. PostgreSQL tells operators to set up
+  and test WAL archiving **before** taking the first base backup.
+  ([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
+- The archive command must return zero if and only if durable archival
+  succeeded. Non-zero causes retry. Archive implementations should refuse to
+  overwrite an existing file; a repeated name may report success only when the
+  existing durable content is identical, and differing content must fail.
+  ([WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- Repeated archive failure accumulates WAL in `pg_wal`; filling that filesystem
+  causes PostgreSQL to panic and remain offline until space is freed. PostgreSQL
+  explicitly advises monitoring archival progress.
+  ([WAL archiving failure behaviour](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- WAL is archived after a segment is completed, so low traffic can make the
+  last committed change wait a long time for archival. `archive_timeout` can
+  bound that age by forcing segment switches, at the cost of archive growth.
+  ([WAL archiving latency](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- WAL archival does not restore manually edited PostgreSQL configuration files
+  (`postgresql.conf`, `pg_hba.conf`, `pg_ident.conf`); those need a separate
+  file backup.
+  ([WAL archiving caveat](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- PostgreSQL's recovery procedure keeps clients out, restores the base backup,
+  configures `restore_command` and `recovery.signal`, replays WAL, and requires
+  inspection of database contents before restoring ordinary access.
+  ([PITR recovery procedure](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-PITR-RECOVERY))
+- Recovery can target a timestamp, LSN, transaction ID, named restore point, or
+  the earliest consistent point. Only one target may be set. Timeline history
+  is required to navigate recovery branches, and a configured target not
+  reached before WAL exhaustion is a fatal recovery failure.
+  ([recovery target settings](https://www.postgresql.org/docs/current/runtime-config-wal.html#RUNTIME-CONFIG-WAL-RECOVERY-TARGET),
+  [timelines](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-TIMELINES))
+
+Derived Hub implications:
+
+- The database RPO is the age of the newest **durably archived, continuous**
+  WAL position, not the age of the newest base backup and not merely the name
+  in `last_archived_wal`.
+- Archive keys need a collision-proof cluster/system-identifier namespace and
+  create-if-absent semantics; unrelated clusters must never overwrite a shared
+  WAL name.
+- Monitoring must cover archive failures, time since durable success, archive
+  lag, WAL continuity, `pg_wal` growth, free space, and base-backup age.
+- PITR restores PostgreSQL only. It does not rewind workspace files. A database
+  recovery target has no whole-Hub meaning until paired with a workspace
+  recovery point and reconciliation rule.
+
+## Verification is a ladder, not a checksum
+
+- `pg_verifybackup` validates the backup manifest and system identifier,
+  detects missing/extra/changed files, checks file checksums, and parses the
+  required WAL with `pg_waldump`. WAL verification is currently supported only
+  for plain-format base backups.
+  ([`pg_verifybackup` checks](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
+- PostgreSQL explicitly says these checks cannot cover everything a running
+  server will encounter and still requires test restores plus checks that the
+  database works and contains correct data.
+  ([`pg_verifybackup` limitation](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
+- `pg_amcheck` can check supported heap/index structures, but currently only
+  ordinary/TOAST tables, materialized views, sequences, and B-tree indexes are
+  supported; other relation types are silently skipped. Stronger B-tree checks
+  can block concurrent modifications.
+  ([`pg_amcheck`](https://www.postgresql.org/docs/current/app-pgamcheck.html#APP-PGAMCHECK))
+
+Derived recovery-evidence ladder:
+
+1. **Artifact verification:** backup command succeeded; manifest and all
+   required WAL validate; any failure marks the artifact unusable.
+2. **Isolated technical restore:** restore to a clean non-production target,
+   reach the intended LSN/time/timeline, and prove PostgreSQL starts without
+   exposing it to users.
+3. **Database validation:** verify migration history/schema identity, tenant
+   counts and relationships, critical constraints/sequences, and appropriate
+   corruption checks.
+4. **Workspace validation:** restore the paired snapshot/version, validate file
+   hashes/ownership/containment, and reconcile database file-change/run state
+   against actual files.
+5. **Application validation:** run read-only tenant-isolation and core workflow
+   smoke tests with external side effects disabled.
+6. **Recovery report:** persist the exact backup, WAL range, workspace point,
+   release/schema versions, checks, observed RPO/RTO, operator, and failures.
+   Only a fully passing report may authorize promotion.
+
+## Prisma Migrate production semantics
+
+### What `migrate deploy` proves and does not prove
+
+- In production, `migrate deploy` compares applied migration checksums with the
+  migration directory, warns when an applied migration was modified, and
+  applies pending migrations.
+  ([Prisma 6 production workflow](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#production-and-testing-environments))
+- It does **not** warn when an already-applied migration is missing, detect
+  production schema drift, reset the database, generate Prisma Client, or use a
+  shadow database.
+  ([Prisma 6 production workflow](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#production-and-testing-environments))
+- The migration directory is the deployable history and must be committed in
+  full, including `migration_lock.toml`. Prisma warns against editing or
+  deleting migrations already applied in production; `migrate deploy` executes
+  migration SQL rather than deriving a new migration from the Prisma schema.
+  ([Prisma migration histories](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/migration-histories))
+- Prisma uses a database advisory lock for production migration commands so
+  multiple migration commands do not run simultaneously. The documented lock
+  timeout is ten seconds and not configurable; after timeout, the caller must
+  retry. The lock can be disabled by environment variable.
+  ([Prisma 6 advisory locking](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#advisory-locking))
+
+Derived Hub implication: `migrate deploy` exit zero proves that this invocation
+applied or found no pending migration. It does not prove the live schema equals
+the expected migration end state, that the new and previous application builds
+are both compatible with it, or that the migration is operationally safe under
+production data volume and traffic.
+
+The advisory lock is a migration-command concurrency guard, not a substitute
+for the repository's deployment lock or application/schema compatibility. As a
+derived safety rule, production preflight should reject
+`PRISMA_SCHEMA_DISABLE_ADVISORY_LOCK` unless an explicit reviewed mechanism
+provides the same serialization guarantee.
+
+### Transactions and failed migrations
+
+- Prisma's first-party migration guidance states that PostgreSQL migrations
+  are not wrapped in a transaction by default. Operators may opt in by adding
+  `BEGIN;`/`COMMIT;` to a migration, but large transactional migrations can hold
+  locks longer and consume more database resources.
+  ([Prisma transactional behaviour](https://www.prisma.io/blog/prisma-migrate-dx-primitives#what-if-schema-migrations-were-atomic))
+- A failed migration records its error in `_prisma_migrations.logs`; further
+  migrations cannot deploy until the failed state is explicitly recovered.
+  Recovery is either (a) revert/repair partial steps, mark the migration rolled
+  back, and redeploy, or (b) manually complete the exact steps and mark it
+  applied.
+  ([Prisma failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration))
+- Prisma error codes `P3009` and `P3018` explicitly report that failed
+  migrations block new migration application until recovery.
+  ([Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference#p3009))
+- `migrate resolve --rolled-back` changes migration-history state so Prisma may
+  apply the migration again; it does not itself undo SQL already executed.
+  `--applied` records success without executing the migration SQL.
+  ([failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration),
+  [`migrate resolve`](https://www.prisma.io/docs/cli/migrate/resolve))
+- Prisma can generate a down SQL script, but its documentation limits the
+  simple rollback workflow to a **failed** up migration. To reverse a successful
+  migration, the supported history is a new forward migration generated from a
+  reverted Prisma schema. A schema down script also cannot restore lost data,
+  application changes, or manually authored SQL automatically.
+  ([generating down migrations](https://www.prisma.io/docs/orm/prisma-migrate/workflows/generating-down-migrations#considerations-when-generating-down-migrations))
+
+Derived Hub implications:
+
+- Every migration needs a declared atomicity choice. If it is intentionally
+  non-transactional, the migration must be resumable/idempotent or ship an
+  exact, tested partial-failure procedure. Blindly rerunning a partially applied
+  script is unsafe.
+- A database backup checkpoint before a destructive or difficult-to-reverse
+  migration is necessary, but restoring that checkpoint is a whole-database
+  recovery event—not a substitute for a compatible rollout design.
+- Service restart cannot be treated as application rollback after a successful
+  schema migration. The previous binary may no longer understand the new
+  schema, and a down migration cannot recreate deleted data.
+
+### Compatibility and rollout ordering
+
+- Prisma documents expand-and-contract as the way to avoid downtime for fields
+  in active use: add the new representation, deploy code that writes both while
+  reading the old, migrate and verify data, switch reads, stop old writes, then
+  remove the old representation.
+  ([Prisma expand-and-contract workflow](https://www.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations#example-use-the-expand-and-contract-pattern-to-evolve-the-schema-without-downtime))
+- Prisma also warns that generated “renames” may be expressed as add/drop and
+  lose data unless the SQL is reviewed and customized before application.
+  ([customizing migrations](https://www.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations#example-rename-a-field))
+
+Derived deployment contract:
+
+1. Migration SQL is immutable after production application and reviewed for
+   destructive operations, lock duration, data volume, and transaction choice.
+2. It is rehearsed against a recent, anonymized production-scale restore and
+   its runtime/locks/disk growth are recorded.
+3. Each release declares a schema compatibility window. Expand releases remain
+   compatible with both old and new application versions; contract/destructive
+   releases occur only after old binaries can no longer run.
+4. Migration status/drift is checked before changing the live release. A
+   failed migration blocks deployment and enters an operator runbook; it must
+   never be hidden by repeated restart attempts.
+5. Rollback means switching to a binary proven compatible with the current
+   schema or deploying a new forward migration. Restoring a backup is the last
+   resort and must also restore/reconcile the workspace recovery point.
+
+## systemd stop, restart, and dependency semantics
+
+- When `ExecStartPre=`, `ExecStart=`, or `ExecStartPost=` fails or times out,
+  service startup does not continue to the normal running process. `ExecStop=`
+  is skipped because startup never succeeded; `ExecStopPost=` is the hook that
+  still runs after a failed startup.
+  ([systemd start/stop command semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStartPre=),
+  [`ExecStopPost=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStopPost=),
+  [permanent upstream man-page source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemd.service.xml#L525-L587))
+- `Restart=on-failure` covers non-zero exits and failures/timeouts in service
+  control processes, including `ExecStartPre=`. An explicit systemd stop does
+  not trigger automatic restart. Restarts are subject to unit start-rate
+  limiting.
+  ([`Restart=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#Restart=),
+  [permanent upstream source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemd.service.xml#L856-L995))
+- With no `ExecStop=`, systemd sends `KillSignal=` (default `SIGTERM`).
+  `TimeoutStopSec=` bounds the wait; after it expires, remaining processes can
+  be forcibly killed with `SIGKILL`/`FinalKillSignal=`.
+  ([`ExecStop=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStop=),
+  [`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=))
+- Default `KillMode=control-group` applies termination to all remaining
+  processes in the unit cgroup. systemd explicitly discourages `process` or
+  `none` because children can escape the service lifecycle and remain running
+  after the service is considered stopped.
+  ([`KillMode=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.kill.html#KillMode=))
+- `systemctl stop` waits for the job unless `--no-block` is used, but the command
+  does not fail merely because an `ExecStop=` command failed: the manager still
+  forcibly terminates the unit.
+  ([`systemctl stop`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#stop%20PATTERN%E2%80%A6),
+  [permanent upstream source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemctl.xml#L444-L459))
+- `systemctl restart` is a stop followed by a start; `ExecStop=` and
+  `ExecStopPost=` participate. It does not necessarily flush every unit resource
+  in the same way as separate completed `stop` then `start` operations.
+  ([restart semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStop=),
+  [`systemctl restart`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#restart%20PATTERN%E2%80%A6))
+- `After=` is only an ordering relationship; it does not pull in another unit.
+  Requirement and ordering dependencies are independent. During shutdown the
+  start order is reversed, so a Hub ordered `After=postgresql.service` stops
+  before PostgreSQL when both are in the shutdown transaction.
+  ([`Requires=` and ordering](https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Requires=),
+  [`Before=`/`After=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Before=))
+- `systemctl freeze` only uses the cgroup freezer to suspend process execution;
+  its documented contract contains no filesystem flush or stable-image
+  guarantee. It is not a substitute for an application drain or a filesystem
+  snapshot barrier.
+  ([permanent `systemctl freeze` man-page source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemctl.xml#L653-L675))
+
+Derived assessment of the current unit:
+
+- A Prisma migration failure in the current `ExecStartPre=` prevents the Hub
+  from starting, then `Restart=on-failure` can retry startup until rate-limited.
+  That is fail-closed, but it is not controlled migration recovery and can
+  repeatedly hit a database already left in Prisma's failed state.
+- `KillSignal=SIGTERM` plus `TimeoutStopSec=30` is only a deadline. Until the Hub
+  installs a real termination handler that makes readiness false, stops ingress,
+  drains/checkpoints active work, closes Feishu and database clients, and reports
+  success, the unit cannot claim a graceful consistency barrier.
+- A backup runbook must issue a controlled drain, verify durable work reached a
+  reconciled terminal/checkpoint state, stop the Hub, verify the unit and its
+  cgroup are inactive, and only then snapshot mutable workspace state. A zero
+  `systemctl stop` exit by itself is insufficient evidence.
+- Recovery should not start the normal unit until the restored database and
+  workspace have been validated. The current unit would run pending migrations
+  immediately on first start, mutating the restored database before validation
+  and possibly destroying the ability to inspect the original recovered state.
+
+## Filesystem and rsync boundaries
+
+- PostgreSQL says a plain filesystem copy of a live data directory is unusable:
+  the database must be shut down, because tools such as `tar` do not create an
+  atomic snapshot and server buffering remains. A correctly implemented frozen
+  snapshot can work while PostgreSQL runs, but all volumes containing data,
+  WAL, and tablespaces must be captured simultaneously; otherwise shut down or
+  use the continuous-archive base-backup protocol.
+  ([PostgreSQL filesystem backups](https://www.postgresql.org/docs/current/backup-file.html))
+- For a PostgreSQL rsync backup, the documented safe pattern is a first pass
+  while running, followed by shutdown and a second `rsync --checksum`; the
+  second pass is what makes the result consistent.
+  ([PostgreSQL filesystem backups](https://www.postgresql.org/docs/current/backup-file.html))
+- At the filesystem layer, `fsfreeze` is documented to let in-progress
+  filesystem operations complete, halt new modifications, and flush dirty data,
+  metadata, and logs before returning. That is still only a filesystem-volume
+  guarantee; it does not replace PostgreSQL's backup protocol and does not make
+  separately frozen volumes simultaneous.
+  ([util-linux `fsfreeze` manual source](https://github.com/util-linux/util-linux/blob/master/sys-utils/fsfreeze.8.adoc))
+- rsync's `--delete` defaults to deletion during transfer on modern peers, so
+  the receiver tree changes incrementally. `--delay-updates` merely holds each
+  updated file and renames files into place rapidly at the end; its own manual
+  says this only attempts to make updates “a little more atomic” and points to
+  a separate parallel-tree/switch approach for stronger atomicity.
+  ([rsync `--delete`](https://rsync.samba.org/ftp/rsync/rsync.1.html#opt--delete),
+  [rsync `--delay-updates`](https://rsync.samba.org/ftp/rsync/rsync.1.html#opt--delay-updates))
+
+Derived Hub implications:
+
+- The current in-place `rsync --delete` deployment is neither an immutable
+  release switch nor a backup snapshot. A failed/interrupted transfer can leave
+  a mixed live tree; service stop/restart cannot reconstruct the deleted prior
+  artifact.
+- The workspace snapshot mechanism remains an explicit design choice. Whichever
+  storage technology is selected must prove atomicity over all workspace
+  volumes or use a quiesced/no-writer interval, then publish an immutable
+  version/checksum manifest. Merely copying a changing directory is not enough.
+
+## Derived minimum whole-Hub recovery contract
+
+The primary sources support the following minimum contract; exact RPO, RTO,
+retention, storage vendor, and numeric schedules remain product/operations
+decisions and must not be invented by implementation.
+
+### Backup path
+
+1. Maintain verified PostgreSQL base backups and a monitored continuous WAL
+   archive, or obtain and test an explicitly equivalent managed-PostgreSQL
+   guarantee. Keep logical dumps as a separate portability/selective-restore
+   layer, not the sole PITR mechanism.
+2. Keep PostgreSQL configuration, deployment/release manifests, migration
+   history, and workspace snapshots in the backup inventory; WAL alone does not
+   cover them.
+3. Create a whole-Hub checkpoint only after ingress is drained and active
+   workspace writers are stopped/reconciled, unless the eventual storage layer
+   supplies a proven atomic snapshot/journal spanning the necessary stores.
+4. Persist one signed/immutable checkpoint manifest joining PostgreSQL
+   backup/WAL position, workspace snapshot/version, application revision,
+   migration digest, tool versions, checksums, timestamps, and validation state.
+5. Treat database/WAL/workspace backups as cross-tenant sensitive data. Apply
+   least-privilege access, encryption, audit, deletion/retention, and an
+   independently protected integrity manifest; WAL contains effectively all
+   database changes and PostgreSQL advises protecting it from unauthorized
+   readers.
+   ([WAL archive confidentiality](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+
+### Migration path
+
+1. Serialize deploys and backup checkpoints; run a drift/status gate before
+   migration, not just `migrate deploy`.
+2. Rehearse each migration on a recent production-scale restore and record
+   runtime, locks, disk/WAL growth, transaction choice, and failure recovery.
+3. Require expand/contract compatibility for online changes. A release must
+   declare which old/new binaries can run against each schema phase.
+4. Separate migration orchestration from ordinary service restart so a failed
+   migration produces one durable incident state and an operator decision—not
+   an opaque restart loop.
+5. Verify a restorable checkpoint before destructive change. Do not describe
+   backup restore as a normal “rollback”; it is whole-service disaster recovery.
+
+### Restore and promotion path
+
+1. Restore into an isolated environment with real external side effects
+   disabled.
+2. Restore the selected PostgreSQL base backup/WAL target and the checkpoint's
+   exact workspace version; do not run pending Prisma migrations implicitly.
+3. Prove artifact integrity, database startup, recovery target/timeline,
+   migration/schema identity, tenant invariants, workspace hashes/containment,
+   and read-only application flows.
+4. Record observed recovery point and recovery time. A restore that starts but
+   misses its target, loses a tenant workspace, has partial migration state, or
+   fails any invariant is not promotable.
+5. Promote explicitly, then enable ingress and external integrations. Preserve
+   the pre-recovery cluster/WAL and recovery report until the incident is closed.
+
+## Decisions and evidence still required
+
+These values are not fixed by the upstream tools and remain `OPEN` for the SaaS
+production contract:
+
+- whole-Hub RPO and RTO, separately for ordinary deletion, database corruption,
+  host loss, region/provider loss, and migration failure;
+- base-backup, logical-dump, workspace-snapshot, WAL, and restore-drill cadence;
+- immutable/off-host storage, encryption/key ownership, retention and verified
+  deletion policy, including org-deletion and legal-retention interactions;
+- exact workspace checkpoint/journal technology and database/workspace
+  reconciliation semantics for in-flight or partially written runs;
+- whether restore promotion is whole-platform only or supports a safe
+  organization/project export-import path (PostgreSQL physical PITR is
+  cluster-wide);
+- migration transaction policy, maximum lock/downtime budget, compatibility
+  window, and who can authorize contract/destructive phases;
+- evidence from at least one production-like full restore and one PITR drill,
+  including deliberate missing-WAL, corrupted-backup, failed-migration,
+  interrupted-workspace-write, and restart-during-recovery probes.

.scratch/saas-production-readiness/assets/clean-host-deployment-audit.md

@@ -0,0 +1,200 @@
+# Clean-host deployment and rollback audit
+
+## Verdict
+
+The current `hub/deploy` path is **not a production deployment mechanism**.
+It can update a manually-prepared host in the happy path, but a clean supported
+host cannot become a runnable Hub from the documented inputs, and an upgrade
+failure can leave the live application tree partially replaced with no
+application rollback path.
+
+The accepted initial topology remains viable: one Linux/systemd host,
+PostgreSQL, Feishu WebSocket ingress, and a non-root Hub service. The blocker is
+the release contract and its verification, not a need to replace that topology.
+
+## Executed evidence
+
+### Static gates
+
+- `bash -n hub/deploy/install_service.sh` and
+  `bash -n hub/deploy/deploy_platform.sh` pass.
+- ShellCheck reports only informational SC2029 findings for intentional
+  client-side expansion of remote paths. It does not find a shell syntax defect.
+- A source-to-installer comparison finds six `requireEnv` settings in
+  `hub/src/server.ts`, but the generated production environment omits
+  `HUB_PROJECT_WORKSPACE_ROOT` and `HUB_SESSION_SECRET`.
+
+### Seeded configuration cannot boot the built server
+
+The installer was run against a temporary `BASE`, with the real built Hub as
+`HUB_DIR`. It exited successfully after creating `platform.env`. The generated
+file was then used as the production configuration while supplying the model and
+Feishu values that its message asks the operator to fill. The built server
+exited with status 1:
+
+```text
+[hub] missing required env: HUB_PROJECT_WORKSPACE_ROOT
+```
+
+After that is supplied, `HUB_SESSION_SECRET` is the next required omission.
+`HUB_PUBLIC_BASE_URL` is also absent; its code fallback is localhost, which is
+not a valid public OAuth callback for a production deployment.
+
+### The service identity is referenced but not provisioned
+
+The installer was executed with `SERVICE_USER=definitely-missing-cph-user` and
+mocked `install`/`systemctl` boundaries. It returned success and rendered:
+
+```text
+User=definitely-missing-cph-user
+WorkingDirectory=.../hub
+EnvironmentFile=.../.secrets/platform.env
+```
+
+There is no `getent`, `id`, `useradd`, or equivalent validation/provisioning
+path. It also creates no service home, cache, state, or project-workspace
+directory. This contradicts ADR-0018's implemented deployment invariant that
+the Hub runs as a dedicated unprivileged service user. A genuinely clean host
+will fail at systemd user resolution; a manually-created user can still fail
+later when the agent SDK, `cph` cache, or project workspace needs a writable
+path.
+
+### A failed build mutates the live tree before failure
+
+`deploy_platform.sh` was run with mocked SSH and rsync boundaries. The first
+remote build was forced to exit 42. The observed order was:
+
+```text
+ssh:mkdir -p '/srv/curriculum-project-hub/hub'
+rsync:live-tree-mutated
+ssh:cd '/srv/curriculum-project-hub/hub' && npm ci && npm run build
+```
+
+The deploy command correctly propagated status 42, but it had already applied
+`rsync --delete` to the one live directory. There is no staged release,
+immutable release identity, current/previous pointer, deployment lock, or
+rollback command. `npm ci` also replaces that live directory's dependency tree
+before the new build has passed.
+
+## Blocking gaps
+
+### 1. Incomplete clean-host bootstrap
+
+`install_service.sh` writes a unit containing `User=cph-hub` without creating or
+validating that account. It does not create owned state/cache/workspace paths,
+and its generated environment omits required settings. On its first invocation
+it deliberately exits after seeding the environment, after which
+`deploy_platform.sh` proceeds to restart a unit that has not yet been installed.
+The documented workflow is therefore neither one-shot nor complete.
+
+### 2. The runtime product is not fully delivered
+
+The deploy uploads only `hub/`. The target prerequisite list does not include a
+preinstalled `cph`, and no release step builds or installs `crates/cph-cli`.
+The Hub starts without checking it: the first agent `cph check` or `cph build`
+returns exit 127 while `/api/healthz` remains green. Bubblewrap is asserted by
+the unit, but Node/npm, curl, PostgreSQL connectivity, service-user namespace
+support, writable storage, and `cph --version` are not checked as one preflight.
+
+### 3. Releases are in-place and have no application rollback
+
+Source, dependencies, build output, and the running service all share one
+directory. A build failure, interrupted rsync, or unsuccessful restart can
+leave a mixed tree. No release manifest records the git revision, Node version,
+dependency lock digest, migration set, or `cph` version that was deployed.
+Concurrent deploys are not serialized.
+
+### 4. The deployment health check is only liveness
+
+`/api/healthz` returns a constant success response after the HTTP server starts.
+It does not verify database access, migration state, workspace writability,
+`cph`, the agent sandbox, or Feishu listener startup. The listener's start
+promise is not awaited or reflected in readiness. A release can therefore be
+declared successful while its defining workflows are unavailable.
+
+The service unit also claims graceful shutdown, but `server.ts` installs no
+`SIGTERM`/`SIGINT` handler and never calls `app.close()` or closes the Feishu
+listener. Node's default SIGTERM handling terminates the process; Fastify's
+documented graceful path begins only when `fastify.close()` is invoked. The
+full data-integrity consequence belongs to the run-lifecycle audit, but the
+current deployment contract must not claim draining behavior it does not have.
+
+References:
+
+- [Fastify shutdown hooks](https://fastify.dev/docs/latest/Reference/Hooks/)
+- [Node.js signal events](https://nodejs.org/api/process.html#signal-events)
+
+### 5. Application rollback and schema migration are not coordinated
+
+The unit runs `prisma migrate deploy` immediately before every start. Current
+migration history includes column drops, index replacement, and data
+deduplication; there is no declaration that the previous application remains
+compatible after each migration. Switching application code back after a
+successful migration is therefore not generally safe.
+
+Prisma documents that `migrate deploy` applies pending migrations but does not
+detect schema drift, and a partially failed migration requires explicit
+operator recovery. Its recommended zero-downtime direction is an
+expand-and-contract sequence. The repository has no pre-deploy migration
+status gate, production-data rehearsal, backup checkpoint, expand/contract
+rule, or failed-migration runbook.
+
+References:
+
+- [Prisma production migration behavior](https://www.prisma.io/docs/orm/prisma-migrate/workflows/development-and-production)
+- [Prisma failed-migration recovery](https://docs.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing)
+- [Prisma expand-and-contract migrations](https://docs.prisma.io/docs/guides/database/data-migration)
+
+### 6. Network binding configuration is misleading
+
+The installer writes `HOST=127.0.0.1`, but `server.ts` ignores it and binds
+`0.0.0.0`. An operator following the generated configuration can unintentionally
+expose the Hub directly instead of binding it behind the intended TLS reverse
+proxy. Public URL validation and proxy topology are not part of deployment
+preflight.
+
+## Required release contract
+
+A production-capable implementation on the accepted topology should establish
+these invariants:
+
+1. **Provisioned identity and storage.** Installation creates or validates a
+   non-login `cph-hub` user and explicit persistent state, cache, home, and
+   workspace paths with tested ownership.
+2. **Complete validated configuration.** A preflight validates every required
+   secret/value, the public HTTPS URL, bind address, PostgreSQL connectivity,
+   writable storage, `bwrap`, and the exact compatible `cph` binary before the
+   live service is touched.
+3. **Immutable staged releases.** Each revision is built in a new release
+   directory, identified by revision and manifest. The live service points to a
+   `current` symlink only after build and preflight pass. Deploys are locked.
+4. **Complete product artifact.** The release carries both the Hub and the
+   matching `cph` executable; readiness checks the binary/version rather than
+   deferring failure to an agent run.
+5. **Real readiness.** Liveness remains cheap, while readiness proves the
+   dependencies needed to accept work. Deployment success waits for readiness,
+   not merely an open HTTP port.
+6. **Application rollback.** A failed switch returns `current` to the previous
+   release and verifies readiness. Database changes are never silently reversed.
+7. **Migration compatibility.** Every release states whether its migration set
+   is backward-compatible with the previous application. Future destructive
+   changes use expand/contract releases; failed migrations follow an explicit
+   recovery runbook tied to backup evidence.
+8. **Observable lifecycle.** Release revision, migration status, readiness
+   failures, restart/drain outcomes, and rollback results are present in command
+   output and service logs.
+
+## Verification route
+
+The implementation is not complete until an automated Linux test drives these
+scenarios with the real scripts and a production-like PostgreSQL instance:
+
+- first install on a host without the service user or directories;
+- incomplete configuration fails before unit installation/restart;
+- staged build failure leaves the current release byte-for-byte unchanged;
+- successful deploy reports revision and passes readiness;
+- failed new release switches back to the previous application;
+- missing/wrong `cph`, unwritable workspace, unavailable database, and failed
+  migration each produce a distinct non-zero deployment failure;
+- repeated installation/deployment is idempotent;
+- concurrent deployments serialize or one fails clearly.

.scratch/saas-production-readiness/assets/initial-abuse-capacity-controls.md

@@ -0,0 +1,125 @@
+# Initial abuse and capacity controls
+
+## Verdict
+
+The initial production boundary requires hard safety controls before customer
+traffic. The current project lock and small in-memory trigger queue are useful
+local protections, but they do not provide multi-tenant admission, durable
+backpressure, storage safety, request isolation, bounded execution, or an
+operator stop mechanism.
+
+The accepted contract is recorded in
+[ADR-0022](../../../docs/adr/0022-layered-capacity-admission.md) and
+[`Spec.System.Capacity`](../../../spec/Spec/System/Capacity.lean). Exact values
+remain deliberately open until production-like calibration; the required
+dimensions and failure behaviors do not.
+
+## Current implementation inventory
+
+| Surface | Existing protection | Production gap |
+| --- | --- | --- |
+| HTTP | Fastify's default 1 MiB body limit | No route-class, user, Org, IP, or administrative-write rate limits; request/connection/handler timeouts are unset |
+| Project execution | Database lock permits one active run per Project | No platform or Org concurrency ceiling and no fair scheduler |
+| Busy Project | Process-local FIFO, default five items and five-minute wait | Restart loses accepted work; expiry is logged and dropped without a durable state or user notification |
+| Agent execution | Global max-turn default of 25 | No wall-clock, tool-call, output, memory, CPU, process, or Org-specific budget |
+| Feishu messages | Message batching limits text count/characters | File/post attachment count and size are unbounded; downloads may buffer whole files |
+| Storage | Workspace path boundary | No Project or Org byte quota, archive-expansion limit, or entity-count ceiling |
+| Usage | Provider-reported token/cost fields and Org aggregation | Missing cost can disappear from totals; no soft-threshold notification or provider-mode attribution contract |
+| Host | systemd lifecycle | No service CPU, memory, or task ceiling and no audited workload brake |
+
+Most Organization member/team/project lists are also unpaginated, and usage
+aggregation can load all matching runs. Those query shapes must be bounded as
+part of enforcement rather than hidden behind larger process memory.
+
+## Accepted contract
+
+### Layered limits
+
+- Every mandatory dimension has a versioned, non-overridable platform ceiling.
+- An Organization OWNER/ADMIN may configure a lower policy limit.
+- Effective limit is `min(platform, organization)`; absent Organization policy
+  means the platform ceiling, never unlimited.
+- Missing or invalid mandatory production ceilings fail startup.
+
+### Admission and fairness
+
+- Project exclusivity remains one active run.
+- Platform and Organization concurrency limits apply before starting work.
+- Available slots are scheduled fairly across Organizations, FIFO within an
+  Organization, so one tenant cannot monopolize a global FIFO.
+- Accepted work is durable and queryable. Queue length and wait are bounded.
+- Overflow is explicit `capacity_exhausted` (`429` plus `Retry-After` for HTTP;
+  a clear busy response for Feishu). Nothing is silently dropped.
+- Wait timeout produces terminal `EXPIRED`, not a later automatic start.
+- The initiator may withdraw their queued request. Project `MANAGE` or an
+  Organization admin may cancel queued requests in their scope. Actor, reason,
+  and notification are recorded.
+
+### Retry and idempotency
+
+- Feishu `event_id` and API idempotency identify one admission across retries.
+- Once a run starts, the whole run is never automatically replayed because tool
+  side effects may already exist.
+- Only bounded transport retries before model output or tool side effects are
+  automatic. Later failure/timeouts require an explicit new request.
+
+### Request, file, storage, and entity safety
+
+- Authenticated Web requests are limited by user, Organization, and endpoint
+  class; signed Feishu events by Organization app, chat, and sender;
+  administrative writes use stricter independent buckets.
+- Admin surfaces require both a valid user session and the corresponding
+  platform/Organization administrator authorization. OAuth callbacks and
+  signed Feishu ingress remain necessary non-session entry points.
+- Hard limits cover request body, single-file bytes, attachment count, archive
+  expanded bytes/file count/depth, Project/Organization storage, and per-type
+  Organization entity counts.
+- Transfer is streamed and counted; partial files are removed after failure.
+- Crossing a ceiling blocks new growth without auto-deleting or hiding existing
+  resources. Read, export, delete, and administrator remediation stay usable.
+
+### Run budget
+
+Every run is bounded by wall time, model turns, tool-call count, per-tool time,
+output/log/event bytes, and process memory/CPU/child-process count. A breached
+budget ends explicitly as `TIMED_OUT` or `LIMIT_EXCEEDED`, records the dimension
+and observed usage, releases capacity, and never masquerades as success.
+
+### Usage and cost
+
+- Token usage, provider-reported cost, run count, and duration are attributed by
+  Organization, Project, Run, model, and Provider Connection.
+- Missing provider cost is `unknown`, never zero.
+- Thresholds are soft alerts in the pilot; they do not automatically stop runs
+  and are not payment settlement.
+- Org admins receive Organization alerts. Platform admins additionally receive
+  anomalous-cost alerts for platform-managed connections; BYOK does not trigger
+  platform financial alerts.
+
+### Emergency control
+
+An audited Platform Administrator control supports Organization or global
+`DRAIN` (reject new work, pause queue starts, finish active runs) and `STOP_NOW`
+(also cancel active runs). Both require an actor, reason, timestamp, explicit
+recovery, and user-visible outcomes. This is workload control, not Organization
+deletion or account suspension.
+
+## Required production evidence
+
+1. Production configuration validation fails closed for every missing or
+   invalid platform ceiling.
+2. Concurrent noisy-neighbor tests prove platform/Org/Project concurrency and
+   Organization-fair scheduling without starvation.
+3. Restart and crash tests prove accepted queue entries, idempotency, expiry,
+   cancellation, and notifications survive process loss.
+4. HTTP and Feishu load tests prove every rate-limit identity and typed overload
+   response, including retry guidance.
+5. File/archive/storage probes prove streaming enforcement, cleanup, and
+   continued read/delete access while over limit.
+6. Hung provider/tool and resource-exhaustion probes prove every run/process
+   budget terminates, records cause, and releases locks/capacity.
+7. Usage tests prove provider-mode attribution, unknown-cost handling, alert
+   routing, and absence of hard monetary blocking.
+8. `DRAIN` and `STOP_NOW` drills prove scoped, audited, reversible behavior.
+9. Production-like calibration publishes the tested numerical defaults and
+   capacity headroom used by the release gate.

.scratch/saas-production-readiness/assets/product-surface-code-inventory.md

@@ -0,0 +1,191 @@
+# Accepted product-surface code inventory
+
+Audit date: 2026-07-10
+
+Scope: source-level inventory for the accepted SaaS product boundary in ADR-0020,
+ADR-0021 and ADR-0022. This is not a claim that tests were executed during this
+inventory. “Test evidence” below means a repository test exercises the named
+path; “no direct test found” means a search of `hub/test/**/*.ts` found no request
+to the named route or corresponding service call.
+
+## Contract baseline
+
+The destination is not a shared-bot, single-credential Hub. The repository's
+current shared language says:
+
+- an Organization owns its memberships, projects, teams, external connections
+  and settings (`CONTEXT.md:7-9`);
+- a Platform Administrator creates and operates Organizations without becoming
+  their member, while an Organization Administrator is only an OWNER/ADMIN of
+  that Organization (`CONTEXT.md:11-17`);
+- the Admin Surface is a private web control plane requiring the corresponding
+  authenticated administrator authority (`CONTEXT.md:19-21`);
+- each Organization has its own Feishu application for login, bot messaging and
+  directory integration (`CONTEXT.md:23-25`);
+- provider connections are Organization-scoped BYOK or an Organization-distinct
+  platform-managed connection (`CONTEXT.md:27-37`);
+- capacity uses non-optional platform ceilings, optional lower Organization
+  limits, durable Organization-fair admission and audited emergency brakes
+  (`CONTEXT.md:39-77`).
+
+ADR-0021 makes `/admin/platform` and `/admin/org/:orgSlug` separate guarded web
+areas (`docs/adr/0021-org-admin-project-onboarding.md:23-40`), requires a
+customer-owned Feishu app with encrypted Organization-scoped secrets
+(`docs/adr/0021-org-admin-project-onboarding.md:42-47`), and pins both provider
+credential modes as Organization-scoped (`docs/adr/0021-org-admin-project-onboarding.md:49-60`).
+ADR-0022 makes its safety controls release requirements, not optional future UI
+(`docs/adr/0022-layered-capacity-admission.md:22-77`).
+
+Status vocabulary in this inventory:
+
+- **HTTP-backed** / **Feishu-backed**: a production route/event handler is wired.
+- **Service/test only**: code can be invoked directly, but there is no production
+  product entry point.
+- **Partial**: part of the journey exists but the accepted end-to-end product
+  surface cannot complete it.
+- **Missing**: no production entry and no suitable persistence/control model.
+- **Contract divergence**: reachable behavior actively uses the wrong tenancy or
+  credential shape.
+
+## Journey matrix
+
+| Target journey | Reachable entry | Authorization | Persistence | Test evidence | Verdict and exact gap |
+|---|---|---|---|---|---|
+| Platform Administrator signs in | None. The admin plugin registers only Feishu auth and Organization routes (`hub/src/admin/plugin.ts:7-8`, `hub/src/admin/plugin.ts:31-46`). The only HTML is the Org Admin login page (`hub/src/admin/routes/authRoutes.ts:171-207`). | `PlatformRoleAssignment` and `PlatformRole.ADMIN` exist in the schema (`hub/prisma/schema.prisma:111-129`), but the HTTP guard explicitly says the platform control plane is not modeled (`hub/src/admin/auth/guards.ts:1-5`). | Role rows can be stored, but there is no platform session/identity binding, invite/bootstrap flow or platform-admin audit surface. | No platform-auth route test exists. | **Missing.** `/admin/platform` from ADR-0021 is not mounted, so no authenticated operator can reach any platform workflow. |
+| Platform Administrator creates/lists/updates/operates Organizations | None. No `/api/platform/*` or Organization CRUD route is registered; the route composition is auth + `/api/org/:orgSlug/*` only (`hub/src/admin/plugin.ts:31-46`, `hub/src/admin/routes/orgRoutes.ts:24-103`). | No `requirePlatformAdmin` implementation; only `requireSession` and `requireOrgRole` exist (`hub/src/admin/auth/guards.ts:43-113`). | `Organization` has slug, name and ACTIVE/SUSPENDED/ARCHIVED status (`hub/prisma/schema.prisma:28-52`). | Organizations are created directly only by test setup, e.g. `prisma.organization.upsert` (`hub/test/integration/helpers.ts:60-77`) and a direct cross-org fixture create (`hub/test/integration/admin-explorer.test.ts:198-210`). | **Service/test only at the database layer.** Manual pilot onboarding has no safe production command, HTTP API or UI; status changes are also unreachable. |
+| Platform Administrator configures an Organization's customer-owned Feishu app and checks readiness | None. Runtime accepts exactly one `feishuAppId`/`feishuAppSecret` pair in `AdminPluginConfig` (`hub/src/admin/plugin.ts:10-19`), and server startup reads one process-global app id, secret and bot open id (`hub/src/server.ts:47-52`). | No platform guard or Organization connection ownership check. | There is no Feishu application/secret model. `ExternalDirectoryConnection` stores provider/tenant/source metadata but no app id, bot id or secret reference (`hub/prisma/schema.prisma:196-214`). | No provisioning/readiness test exists. | **Missing + contract divergence.** There is no Organization-scoped encrypted Feishu connection or readiness workflow; production can operate only the one env-configured app. |
+| Platform Administrator configures an Organization-distinct platform-managed provider key/base URL | None. No platform provider route or UI is registered. | No platform guard. | The accepted Lean type exists only in the semantic contract (`spec/Spec/System/Organization.lean:52-66`); Prisma has no provider-connection or provider-mode model (`hub/prisma/schema.prisma:26-214`). | Runtime tests prove env values are returned, not Organization values (`hub/test/unit/runtime-settings.test.ts:18-55`). | **Missing + contract divergence.** `EnvRuntimeSettings.provider` ignores its project scope and reads one process-global `ANTHROPIC_BASE_URL`/`ANTHROPIC_AUTH_TOKEN` (`hub/src/settings/runtime.ts:20-21`, `hub/src/settings/runtime.ts:46-67`). |
+| Platform Administrator applies Organization/platform `DRAIN` or `STOP_NOW` and later resumes | None. | No platform guard, reason capture or scope check. | No workload-brake model exists; only the Lean contract defines `open`/`drain`/`stopNow` semantics (`spec/Spec/System/Capacity.lean:71-87`). | No workload-brake test exists. | **Missing.** The required audited, reason-bearing, reversible emergency control from ADR-0022 (`docs/adr/0022-layered-capacity-admission.md:69-72`) has no product or persistence surface. |
+| Organization user logs in with that Organization's Feishu app | `GET /auth/feishu` and callback are HTTP-backed (`hub/src/admin/routes/authRoutes.ts:49-128`). The callback upserts a user and signs a cookie (`hub/src/admin/routes/authRoutes.ts:99-123`). | Login itself is public OAuth; Organization APIs later enforce session + active OWNER/ADMIN membership (`hub/src/admin/auth/guards.ts:43-113`). | `User.feishuOpenId` is one process-global unique identifier (`hub/prisma/schema.prisma:89-109`); the signed cookie carries user id/open id but no Organization or Feishu-connection identity (`hub/src/admin/auth/session.ts:18-23`). | Redirect and mocked callback are covered (`hub/test/integration/admin-auth.test.ts:164-230`); OAuth request construction/exchange is covered (`hub/test/unit/feishu-oauth.test.ts:8-81`). | **HTTP-backed but contract-divergent.** OAuth is built once from the process-global app credentials (`hub/src/admin/routes/authRoutes.ts:38-47`), and the implementation explicitly documents this deferral (`hub/src/admin/auth/feishuOAuth.ts:1-10`). There is no org-selecting login route, connection resolver or issuer/app identity in the user key. |
+| Each Organization operates its own Feishu Bot | One WebSocket listener is production-wired (`hub/src/server.ts:76-90`). | Message actions are gated by project grants once a binding resolves (`hub/src/feishu/trigger.ts:715-755`), but there is no initial Organization/connection ownership gate for the inbound app. | Binding stores only `chatId -> projectId`, with no Feishu connection/application id (`hub/prisma/schema.prisma:287-304`). Event receipts store globally unique `eventId`, likewise without connection id (`hub/src/feishu/trigger.ts:687-707`). | Trigger, onboarding and event dedup are integration-tested (`hub/test/integration/trigger.test.ts:211-330`, `hub/test/integration/trigger.test.ts:878-928`). | **Feishu-backed for one bot, contract-divergent for SaaS.** Startup constructs one client and one WS client from one config (`hub/src/feishu/client.ts:854-901`). It cannot operate or route multiple Organization-owned apps. |
+| Suspended/archived Organization is prevented from bot work | No platform status-change entry exists. Org HTTP APIs do reject non-ACTIVE Organizations (`hub/src/admin/auth/guards.ts:79-88`). | The Feishu trigger resolves an active chat binding directly and then project permissions; it does not load or check `Project.organization.status` (`hub/src/feishu/trigger.ts:715-755`). | Organization status exists (`hub/prisma/schema.prisma:30-52`). | No suspended-Organization trigger test exists. | **Partial and unsafe.** Even a database-side status change only blocks web admin; an already-bound chat remains triggerable through the bot path. |
+| Logged-in Organization OWNER/ADMIN reaches the management panel | OAuth success redirects to `/admin/org/:slug` (`hub/src/admin/routes/authRoutes.ts:237-267`). | All Organization APIs default to OWNER/ADMIN through `requireOrgRole` (`hub/src/admin/auth/guards.ts:66-113`), and tests prove MEMBER is denied while ADMIN succeeds (`hub/test/integration/admin-auth.test.ts:95-131`). | Session is an HMAC cookie; user/membership is re-read for each guard (`hub/src/admin/auth/session.ts:31-57`, `hub/src/admin/auth/guards.ts:43-63`). | Guard behavior is integration-tested (`hub/test/integration/admin-auth.test.ts:56-131`). | **API-backed, UI missing.** The plugin says the static SPA is “later” (`hub/src/admin/plugin.ts:1-3`); no route/static handler serves `/admin/org/:slug`. OAuth therefore redirects to an unmounted page. |
+| Organization OWNER/ADMIN manages members and roles | REST endpoints list/add/change/revoke members (`hub/src/admin/routes/membersRoutes.ts:20-98`). | OWNER/ADMIN route guard plus service rules: only OWNER may affect OWNER and the last OWNER cannot be removed (`hub/src/org/members.ts:1-8`, `hub/src/org/members.ts:97-205`). | `OrganizationMembership` persists org, user, role and revocation (`hub/prisma/schema.prisma:54-76`). | Add/list/last-OWNER protection is covered through HTTP (`hub/test/integration/admin-members-teams.test.ts:49-83`). | **HTTP-backed and persisted; management UI missing.** Addition requires a raw `feishuOpenId` and may create the `User` directly (`hub/src/org/members.ts:46-85`); there is no invite/accept or org-connection directory picker. Role-change/revoke variants have less direct HTTP coverage than add/list. |
+| Organization OWNER/ADMIN manages teams and team membership | REST endpoints cover list/create/update/archive and membership add/revoke (`hub/src/admin/routes/teamsRoutes.ts:21-139`). | All routes require Organization OWNER/ADMIN; service verifies the team and added user belong to the same Organization (`hub/src/org/teams.ts:177-215`, `hub/src/org/teams.ts:263-275`). | `Team` and `TeamMembership` are Organization/project-principal persistence (`hub/prisma/schema.prisma:143-177`). | Create/add-member/archive is covered through HTTP (`hub/test/integration/admin-members-teams.test.ts:85-160`). | **HTTP-backed and persisted; management UI missing.** No production route exposes `ExternalDirectoryConnection`/external team binding sync, so the Feishu directory portion remains service-only. |
+| Organization OWNER/ADMIN creates and manages Folders/Projects | REST endpoints cover explorer, folder create/update/archive, project create/detail/rename/move/archive (`hub/src/admin/routes/explorerRoutes.ts:33-212`). | All routes require Organization OWNER/ADMIN. Service operations scope folder/project ids to the Organization (`hub/src/org/explorer.ts:301-341`); HTTP cross-org lookup is tested. | Folder and Project carry `organizationId`; Folder is transparent and Project owns workspace/session/run relations (`hub/prisma/schema.prisma:238-285`). Project creation also creates a real workspace and default permission rows (`hub/src/projectOnboarding.ts:275-348`). | Create/list and move/archive are HTTP-tested (`hub/test/integration/admin-explorer.test.ts:66-196`); cross-org id access is tested (`hub/test/integration/admin-explorer.test.ts:198-223`). | **HTTP-backed and substantially tested; management UI missing.** This is the strongest accepted product slice. |
+| Organization OWNER/ADMIN manages Feishu chat binding | HTTP can archive a binding (`hub/src/admin/routes/explorerRoutes.ts:214-227`). New binding is exposed through the Feishu onboarding card, not web. | Archive requires OWNER/ADMIN at HTTP and then project manager authorization in the shared service (`hub/src/projectOnboarding.ts:231-272`, `hub/src/org/explorer.ts:301-314`). Bind existing requires project MANAGE (`hub/src/projectOnboarding.ts:179-229`). | `ProjectGroupBinding` and the FEISHU_CHAT EDIT grant are transactionally stored (`hub/src/projectOnboarding.ts:190-215`, `hub/prisma/schema.prisma:287-304`). | Bind/one-active/archive/rebind is service-tested (`hub/test/integration/project-onboarding.test.ts:114-160`); card bind is integration-tested (`hub/test/integration/trigger.test.ts:271-330`). No direct HTTP archive test was found. | **Feishu + partial HTTP-backed; UI missing.** Binding identity is not scoped by Feishu application, so the 1:1 uniqueness and lookup are process-global rather than Organization-connection-scoped. |
+| Organization OWNER/ADMIN manages project team access | REST list/grant/revoke endpoints exist (`hub/src/admin/routes/accessRoutes.ts:19-79`). | OWNER/ADMIN route guard; service refuses a team from another Organization (`hub/src/permissions/projectTeamAccess.ts:168-195`). | TEAM -> PROJECT `PermissionGrant` persists the role (`hub/prisma/schema.prisma:428-447`). | Grant/list/archive-induced revoke is HTTP-tested (`hub/test/integration/admin-members-teams.test.ts:123-156`). | **HTTP-backed and tested; UI missing.** Only team grants are surfaced. There is no HTTP management of direct USER/chat grants, `PermissionSettings`, or per-role `RoleTriggerGrant`. |
+| Organization OWNER/ADMIN manages general project settings, roles/models and agent policies | Only `membersCanCreateProjects` is reachable through GET/PATCH `/settings` (`hub/src/admin/routes/orgRoutes.ts:46-81`). | OWNER/ADMIN route guard. | `OrganizationProjectSettings` stores only that Boolean (`hub/prisma/schema.prisma:78-87`). `PermissionSettings` exists, but no admin route references it (`hub/prisma/schema.prisma:449-467`). Model/role registry is explicitly an in-memory skeleton awaiting DB/admin wiring (`hub/src/agent/models.ts:65-79`). | The Boolean setting is HTTP-tested (`hub/test/integration/admin-auth.test.ts:133-162`); model env behavior is unit-tested (`hub/test/unit/runtime-settings.test.ts:4-64`). | **Partial.** The creation policy works, but the accepted “projects and configuration” control plane lacks editable permission settings, model/role settings and run policies. |
+| Organization OWNER/ADMIN chooses BYOK vs platform-managed Provider mode; BYOK admin writes key/base URL | None. | No Organization provider guard because no route/service exists. | No Prisma provider connection, credential mode, encrypted secret handle or rotation metadata. The only persistent `provider` fields are descriptive strings on sessions/runs (`hub/prisma/schema.prisma:308-330`, `hub/prisma/schema.prisma:352-387`). | No provider-mode test. Existing tests assert the global env resolver (`hub/test/unit/runtime-settings.test.ts:18-55`). | **Missing + contract divergence.** Lean/ADR have the mode but production always calls `settings.provider("openrouter", { projectId })` (`hub/src/feishu/trigger.ts:195-199`), whose implementation discards the scope (`hub/src/settings/runtime.ts:46-67`). |
+| Organization OWNER/ADMIN views sessions and runs | HTTP list and detail endpoints exist (`hub/src/admin/routes/sessionsRoutes.ts:14-45`). | OWNER/ADMIN route guard; service scopes project/session through Organization id (`hub/src/org/sessions.ts:6-23`, `hub/src/org/sessions.ts:49-87`). | `AgentSession`/`AgentRun` persist provider/model/status/tokens/cost and relations (`hub/prisma/schema.prisma:308-387`). | No direct test for these HTTP routes or `org/sessions.ts` was found. Trigger tests do prove sessions/runs are written, e.g. separate role sessions (`hub/test/integration/trigger.test.ts:850-876`). | **HTTP-backed and persisted, but untested at the product boundary and UI missing.** |
+| Organization OWNER/ADMIN views Organization/Project/Folder usage | HTTP Organization and Project usage endpoints exist, with optional folder/date filtering (`hub/src/admin/routes/sessionsRoutes.ts:47-79`). | OWNER/ADMIN route guard. | Rollup derives Organization through Project and uses persisted token/cost facts (`hub/src/org/usage.ts:35-77`, `hub/src/org/usage.ts:106-151`). Missing cost increments `runsWithoutCost`, not known cost (`hub/src/org/usage.ts:106-118`). | No direct usage route/service test was found. `/cost` tests cover the Feishu session command, not these admin endpoints (`hub/test/integration/trigger.test.ts:423-480`). | **HTTP-backed and persisted, but incomplete relative to ADR-0022.** It does not attribute by Provider Connection because none exists, has no alert configuration/delivery, no UI, and no product-boundary tests. |
+| Ordinary MEMBER creates a Project from an unbound Feishu group | Mentioning the bot in an unbound chat sends an onboarding card (`hub/src/feishu/trigger.ts:850-887`); card action calls shared project creation (`hub/src/feishu/trigger.ts:554-614`). | User must already exist and have an active Organization membership; ordinary creation is gated by `membersCanCreateProjects` (`hub/src/projectOnboarding.ts:149-177`, `hub/src/projectOnboarding.ts:380-403`). Creator receives USER MANAGE and the chat receives EDIT (`hub/src/projectOnboarding.ts:302-340`). | Project, Folder, workspace, binding, default settings and grants are persisted (`hub/src/projectOnboarding.ts:275-348`). | Card display/create/bind is integration-tested (`hub/test/integration/trigger.test.ts:211-330`); policy denial and grants are service-tested (`hub/test/integration/project-onboarding.test.ts:70-112`). | **Feishu-backed and well-tested for the single-app case.** For a user with multiple active memberships, onboarding refuses and says the org selector does not exist (`hub/src/feishu/trigger.ts:889-929`). With a per-Org app, the inbound connection should supply tenant context; current global bot cannot. |
+| Ordinary MEMBER binds an existing manageable Project | Onboarding card lists unbound projects and card action binds one (`hub/src/feishu/trigger.ts:868-885`, `hub/src/feishu/trigger.ts:932-969`). | Non-admin candidate projects require `collaborator.manage`; bind service rechecks project MANAGE (`hub/src/feishu/trigger.ts:951-960`, `hub/src/projectOnboarding.ts:179-215`). | Binding + FEISHU_CHAT EDIT grant persist transactionally. | Card journey is integration-tested (`hub/test/integration/trigger.test.ts:271-330`). | **Feishu-backed for the single-app case; per-Organization connection routing is absent.** |
+| Ordinary authorized member triggers an Agent Run from Feishu | Bound message event resolves Project, evaluates `agent.trigger`, resolves role/model/provider, creates session/run and starts the runner (`hub/src/feishu/trigger.ts:684-848`, especially `hub/src/feishu/trigger.ts:715-755` and `hub/src/feishu/trigger.ts:140-256`). | Missing sender id is fail-closed; project EDIT+ and optional role grant are enforced (`hub/src/feishu/trigger.ts:171-194`, `hub/src/feishu/trigger.ts:726-755`). | Session/run, messages, file changes, event receipt and audit fragments persist in Prisma (`hub/prisma/schema.prisma:306-405`, `hub/prisma/schema.prisma:496-525`). | Lifecycle, permission denial, dedup, audit, messages and interrupt have integration coverage (`hub/test/integration/trigger.test.ts:116-154`, `hub/test/integration/trigger.test.ts:597-607`, `hub/test/integration/trigger.test.ts:878-928`, `hub/test/integration/trigger.test.ts:983-1100`). | **Feishu-backed and tested for one process-global provider/Feishu connection.** It is not an Organization-resolved run: provider scope is ignored and bot/identity connection is absent. |
+| Accepted work waits under Project/Organization/platform capacity and survives restart | Only a process-local queue per Project exists (`hub/src/feishu/triggerQueue.ts:18-45`). It is drained after the active Project run (`hub/src/feishu/trigger.ts:430-495`). | No Organization admission identity or capacity policy is evaluated. | Queue is a `Map` in memory (`hub/src/feishu/triggerQueue.ts:21-26`), not a Prisma model; accepted requests vanish on process exit. Expired entries are dropped and only logged (`hub/src/feishu/trigger.ts:475-483`). | FIFO/full/expiry behavior is unit-tested (`hub/test/unit/trigger-queue.test.ts:10-78`); locked-trigger enqueue/drain is integration-tested (`hub/test/integration/trigger.test.ts:609-673`). | **Contract divergence.** ADR-0022 requires durable bounded admission, Organization-fair scheduling, explicit EXPIRED/CANCELED states and notifications (`docs/adr/0022-layered-capacity-admission.md:33-46`); none are present. |
+| Platform ceiling + optional lower Organization policy protect request/run/file/storage/entity capacity | None beyond Fastify defaults, a five-item Project queue and global max turns. `Fastify({ logger: true })` has no capacity options/plugins (`hub/src/server.ts:55-55`); dependencies include no rate-limit plugin (`hub/package.json:9-17`). | No admin policy route. | No capacity/limit model. `OrganizationProjectSettings` has only `membersCanCreateProjects` (`hub/prisma/schema.prisma:78-87`). Run policy contains only global `maxTurns` (`hub/src/settings/runtime.ts:24-37`, `hub/src/settings/runtime.ts:74-78`). The systemd unit has no CPU/memory/process controls (`hub/deploy/cph-hub.service:6-26`). | Unit tests cover only global max turns and the local queue (`hub/test/unit/runtime-settings.test.ts:58-64`, `hub/test/unit/trigger-queue.test.ts:5-113`). | **Missing.** None of ADR-0022's mandatory platform/org request rate, body, concurrency, file/archive, storage, entity-count, wall-time/tool/output or process limits are product-reachable or persistently enforced (`spec/Spec/System/Capacity.lean:13-60`). |
+
+## Reachability summary
+
+### Production-wired HTTP surface
+
+The actual production plugin mounts these capability groups:
+
+- global Feishu OAuth/session: `/auth/feishu`, callback, logout, `/api/me`, and
+  the minimal `/admin/login` HTML (`hub/src/admin/routes/authRoutes.ts:49-207`);
+- Organization summary and one onboarding Boolean
+  (`hub/src/admin/routes/orgRoutes.ts:27-81`);
+- Folder/Project explorer and binding archive
+  (`hub/src/admin/routes/explorerRoutes.ts:33-227`);
+- member roles (`hub/src/admin/routes/membersRoutes.ts:20-98`);
+- teams/team membership (`hub/src/admin/routes/teamsRoutes.ts:21-139`);
+- TEAM -> PROJECT grants (`hub/src/admin/routes/accessRoutes.ts:19-79`);
+- sessions and usage (`hub/src/admin/routes/sessionsRoutes.ts:14-79`).
+
+Every `/api/org/:orgSlug/*` route uses the same OWNER/ADMIN guard. This satisfies
+the clarified requirement that a management API cannot be used by an anonymous
+or ordinary Organization member (`hub/src/admin/auth/guards.ts:43-113`). It does
+not provide the accepted management **panel**: only `/admin/login` is rendered,
+and the post-login `/admin/org/:slug` destination is not served.
+
+### Production-wired Feishu surface
+
+The trigger handler is a real end-to-end application path for one Feishu app:
+event receipt/dedup, chat binding, permission checks, unbound-chat onboarding,
+Project creation/binding, sessions/runs, streaming and interrupt. The service
+wires the handler to a single SDK client and WebSocket listener
+(`hub/src/server.ts:76-90`, `hub/src/feishu/client.ts:854-901`). The code and tests
+therefore establish a useful single-app vertical slice, not the required
+per-Organization connection architecture.
+
+### Only database/service/test reachable
+
+- Organization creation is direct Prisma fixture work
+  (`hub/test/integration/helpers.ts:60-77`).
+- `PlatformRoleAssignment` is a stored type with no platform auth/HTTP surface
+  (`hub/prisma/schema.prisma:111-129`).
+- external-directory sync has service code and can create membership rows
+  (`hub/src/permissions/externalSync.ts:40-90`), but no registered production
+  admin route.
+- project `PermissionSettings` and `RoleTriggerGrant` are stored/enforced in
+  parts of the runtime, but have no Organization management API
+  (`hub/prisma/schema.prisma:449-494`).
+
+### Entirely absent from runtime persistence/control plane
+
+- Organization-scoped Feishu application connection and encrypted secret
+  handle;
+- Organization-scoped Provider Connection, BYOK/platform-managed mode, encrypted
+  secret handle, rotation and runtime resolver;
+- platform administrator session/guard, platform Organization CRUD, platform
+  admin invitation, platform-operation audit;
+- versioned platform capacity ceiling loader/validation;
+- Organization policy-limit rows and Organization admin endpoints;
+- durable run request/admission state, fair Organization scheduler, queue cancel
+  and expiry notification;
+- Organization/platform workload brake state and audited actions;
+- usage/cost alert configuration and delivery;
+- the Organization and platform management UI.
+
+## Critical production blockers
+
+1. **No Organization connection control plane.** The running service has one
+   Feishu app/bot and one provider token/base URL. This contradicts the defining
+   SaaS requirement even though Project/Team data is otherwise Organization-
+   scoped (`hub/src/server.ts:44-52`, `hub/src/settings/runtime.ts:46-67`).
+
+2. **No platform control plane.** A Platform Administrator cannot authenticate,
+   create or operate an Organization, provision its Feishu app, configure its
+   platform-managed provider connection, or apply a workload brake. The only
+   apparent Organization creation path is direct database/test code.
+
+3. **No usable management panel.** The backend Organization APIs are meaningful
+   and guarded, but OAuth redirects to an unmounted `/admin/org/:slug`; platform
+   UI does not exist at all (`hub/src/admin/plugin.ts:1-8`,
+   `hub/src/admin/routes/authRoutes.ts:237-267`).
+
+4. **ADR-0022 is almost wholly unimplemented.** The in-memory Project queue is
+   specifically the implementation divergence ADR-0022 rejects; there are no
+   Organization-fair durable admission, mandatory layered limits, hard run/file/
+   storage/process budgets or emergency brakes
+   (`docs/adr/0022-layered-capacity-admission.md:79-88`).
+
+5. **Tenant context is inferred too late and from global identifiers.** OAuth
+   creates a globally keyed `User`, bot events bind globally by `chatId`, and an
+   unbound-chat user with more than one Organization is rejected rather than
+   routed by the customer-owned Feishu connection (`hub/prisma/schema.prisma:89-109`,
+   `hub/src/feishu/trigger.ts:715-723`, `hub/src/feishu/trigger.ts:889-929`). This
+   blocks a correct multi-app implementation and also leaves Organization status
+   unenforced on the bot path.
+
+## What is already worth retaining
+
+The audit does not imply rewriting the working core. The following seams are
+concrete and have useful test evidence:
+
+- Organization ownership on Project/Team/Folder and same-Organization access
+  checks (`hub/prisma/schema.prisma:143-177`, `hub/prisma/schema.prisma:238-285`);
+- OWNER/ADMIN guards and last-OWNER protection
+  (`hub/src/admin/auth/guards.ts:66-113`, `hub/src/org/members.ts:97-205`);
+- reusable service operations for project creation, chat binding and grants
+  (`hub/src/projectOnboarding.ts:128-229`, `hub/src/projectOnboarding.ts:275-348`);
+- Organization-scoped explorer/member/team/team-access HTTP APIs and their
+  integration tests;
+- Feishu onboarding, project permission checks, session/run persistence and
+  dedup in the single-app vertical slice.
+
+The shortest path to the accepted destination is therefore to add the missing
+platform/connection/capacity control planes and real admin UI around these seams,
+then change ingress and runtime resolution to carry an explicit Organization
+connection identity end to end. Treating the existing env credentials as an
+acceptable pilot fallback would preserve the wrong tenant boundary.

.scratch/saas-production-readiness/assets/product-surface-completeness-audit.md

@@ -0,0 +1,259 @@
+# Accepted SaaS product-surface completeness audit
+
+Audit date: 2026-07-10
+
+## Verdict
+
+The current Hub is **not a usable or production-complete SaaS product surface**
+for the accepted destination. It has a worthwhile Organization-scoped backend
+foundation and a well-tested single-Feishu-application Agent Run slice, but the
+running product still has one process-global Feishu app/bot and one global model
+provider credential. It has no platform-administrator control plane, no
+Organization connection/secret plane, and no Organization-scoped Provider
+Credential Mode. Its only HTML product page is a login form; successful OAuth
+redirects an administrator to an unmounted `/admin/org/:orgSlug` route.
+
+ADR-0022's required production safety surface is also almost wholly absent:
+there is no durable Organization-fair admission, layered platform/Organization
+limits, hard file/storage/run/process budgets, usage-alert control plane, or
+audited workload brake. These are release blockers, not optional polish.
+
+The strongest code should be retained rather than rewritten: all current
+`/api/org/:orgSlug/*` routes reauthorize an active OWNER/ADMIN membership;
+Folder/Project, member, Team, Team-to-Project access, session, and usage services
+are Organization-scoped; and the shared project-onboarding services correctly
+create Project/chat/grant relationships for the single-app case. The shortest
+route is to add explicit Organization connection identity and the two private
+admin control planes around those seams.
+
+The claim-by-claim source inventory, including route, schema and test locations,
+is in [accepted product-surface code inventory](product-surface-code-inventory.md).
+Official Feishu identifier, OAuth, event and long-connection semantics are in
+[Feishu product-surface primary sources](product-surface-feishu-primary-sources.md).
+
+## Accepted boundary
+
+The audit treats these as required because they are already accepted by the
+repository's `CONTEXT.md`, ADR-0020, ADR-0021, ADR-0022, and Lean contract:
+
+- a Platform Administrator is separate from Organization membership and uses a
+  private `/admin/platform` area to create and operate Organizations;
+- an Organization OWNER/ADMIN uses a private `/admin/org/:orgSlug` area to
+  manage members, projects, Teams, connections, settings, sessions and usage;
+- every Organization authenticates and exchanges bot/directory traffic through
+  its own customer-owned Feishu Application;
+- every Organization chooses BYOK or an Organization-distinct platform-managed
+  Provider Connection, with different administrators owning the respective
+  plaintext inputs;
+- ordinary members can create/bind a Project from an unbound Feishu group only
+  under the Organization policy, while all Project actions retain Project-level
+  authorization;
+- production has mandatory platform ceilings, optional lower Organization
+  limits, durable fair admission, bounded work and audited emergency brakes.
+
+The audit does not add deferred self-service signup, payment collection,
+folder ACL inheritance, commercial billing, or a live editor for platform
+capacity ceilings. Guided Feishu app setup and manually initiated Organization
+creation by an authenticated Platform Administrator remain valid for the pilot.
+
+## Journey result
+
+| Accepted journey | Current product entry | Result | Owning frontier |
+| --- | --- | --- | --- |
+| Platform Administrator signs in | No platform login/session/guard; only a stored `PlatformRoleAssignment` type | **Missing** | tickets 42, 45 |
+| Platform Administrator creates and operates Organizations | Direct Prisma calls in test fixtures only; no `/api/platform/*` or UI | **Test/database only** | ticket 45 |
+| Platform Administrator configures each customer Feishu app | One env `FEISHU_APP_ID/SECRET/BOT_OPEN_ID`; no connection or secret model | **Missing and contract-divergent** | tickets 43–45 |
+| Platform Administrator supplies a distinct platform-managed provider key/base URL | One process-global `ANTHROPIC_AUTH_TOKEN/BASE_URL`; resolver ignores Project scope | **Missing and contract-divergent** | tickets 43, 45 |
+| Platform Administrator applies and reverses `DRAIN`/`STOP_NOW` | No persistence, guard, API or UI | **Missing** | tickets 37, 45 |
+| Organization user logs in through that Organization's Feishu app | OAuth exists, but is built once from the global app and has no Organization/application issuer | **Single-app only** | tickets 43, 44 |
+| Logged-in Organization OWNER/ADMIN opens the management panel | OAuth redirects to `/admin/org/:slug`; no handler/static application serves it | **Stable 404** | ticket 46 |
+| Organization OWNER/ADMIN manages members, Teams, Projects/Folders and Team access | Guarded Organization JSON APIs and useful integration coverage exist | **Backend present; browser product absent** | ticket 46 |
+| Organization OWNER/ADMIN manages permission/model/role/run policy | Only `membersCanCreateProjects` is exposed; other stored/runtime settings have no control plane | **Partial** | ticket 46 |
+| Organization selects Provider Credential Mode and manages BYOK | No mode/connection model, API or UI | **Missing** | tickets 43, 46 |
+| Organization views sessions and usage | Guarded JSON APIs exist, but have no direct product-boundary tests or UI and cannot attribute Provider Connection | **Partial** | tickets 36, 46 |
+| Ordinary member creates or binds a Project in Feishu | Card/service path is present and well tested | **Works for the one-global-app case** | ticket 44 for SaaS routing |
+| Authorized member runs the agent through its Organization bot | Full single-app trigger path exists; provider/app resolution is global | **Works only as a single-app deployment** | tickets 43, 44 plus lifecycle tickets |
+| Accepted work queues fairly and durably across Organizations | Five-item process-memory Project queue | **Contract-divergent** | tickets 23, 32–35 |
+| Organization sets lower policy limits and receives soft alerts | No policy/alert persistence or product surface | **Missing** | tickets 32, 34–36, 46 |
+
+## Executed defect evidence
+
+The probes used the real Fastify plugin and test PostgreSQL database through a
+temporary Vitest integration file. The file was removed after capture and was
+run twice to separate deterministic behavior from a transient test failure.
+
+### 1. The successful-login destination is not served
+
+The probe seeded an active Organization ADMIN, minted the normal signed session
+cookie, and requested the exact destination selected by
+`resolvePostLoginRedirect`:
+
+```text
+GET /admin/org/test-default
+expected_status=200
+actual_status=404
+```
+
+Both runs returned 404. The root cause is structural: `registerAdminPlugin`
+mounts cookie, auth and Organization API routes only; its own comment says the
+static SPA is “later”. No other repository HTML/TSX/Vue/Svelte/CSS product
+application or deployment-level static handler exists. Existing OAuth tests
+assert only the redirect `Location`, so they stop immediately before the broken
+journey.
+
+This is not an authorization defect in the JSON APIs. Those APIs correctly
+return 401 without a session and 403 to a MEMBER. It is a reachability defect:
+there is no management panel for either an authorized or unauthorized request
+to reach. Ticket 46 must guard both the page shell and its data, so an anonymous
+user may reach only the login entry and never management content.
+
+### 2. Different Organizations start OAuth with the same application
+
+The probe seeded two Organizations and requested the current OAuth entry with
+two different Organization return paths:
+
+```text
+GET /auth/feishu?returnTo=/admin/org/test-default
+client_id=cli_process_global
+
+GET /auth/feishu?returnTo=/admin/org/second
+client_id=cli_process_global
+```
+
+Both runs produced the same result. The root cause is that
+`registerAuthRoutes` constructs one immutable OAuth configuration from plugin
+startup arguments. The `returnTo` path is signed only as a destination; it does
+not resolve or bind an Organization Feishu Application. `server.ts` obtains the
+same app credentials from one environment tuple and also uses them for the only
+WebSocket listener.
+
+Changing only the redirect page cannot fix this. The composition root and every
+identifier-bearing record/action need an explicit Organization connection
+identity, backed by the encrypted resolver in ticket 43.
+
+## Feishu identifier and transport consequence
+
+Feishu's official identifier contract makes the current global identifier
+shape unsafe for customer-owned applications:
+
+- `open_id` identifies a user inside one application and differs across apps;
+- `user_id` is stable inside one tenant but differs across tenants;
+- `union_id` can link apps from the same application service provider, but the
+  service cannot assume independently customer-owned apps share that provider;
+- OAuth authorization identifies the app with `client_id`; the callback brings
+  `code` and `state`, so state and token exchange must remain bound to the
+  selected Organization/application;
+- v2 HTTP event/card headers expose application and tenant identity, while an
+  encrypted outer request must be routed to the correct connection before its
+  verification/decryption key is known; a long connection is itself created by
+  one app credential pair, so the local client/listener identity is the issuer;
+- Feishu rejects cross-tenant chat/operator combinations and does not provide a
+  contract that permits `chat_id` to be treated as a global SaaS key.
+
+The current `User.feishuOpenId @unique`, session `{userId, feishuOpenId}`,
+`ProjectGroupBinding.chatId`, `FeishuEventReceipt.eventId`, principal IDs,
+sender caches, and batching keys omit this issuer namespace. An `open_id`
+collision across customer apps could merge unrelated external identities;
+different `open_id` values for one person across apps cannot be resolved as one
+global identity automatically. A globally looked-up `chatId` or event ID has
+the analogous ambiguity.
+
+The implementation must therefore persist provider-local external identities
+under a Feishu Application Connection, for example conceptually
+`(connectionId, externalIdType, externalId)`, and carry that connection through
+OAuth state/session, ingress, deduplication, chat binding, authorization,
+onboarding/card actions, rate-limit keys, directory sync, outbound delivery and
+audit. Whether two external identities represent the same global `User` is a
+separate explicit linking decision; it must not be inferred from a provider ID
+outside its documented scope.
+
+## Root causes rather than symptoms
+
+### 1. Tenant data was added without tenant-scoping external connections
+
+Organization ownership is correctly present on Projects, Teams, Folders and
+directory-connection metadata, but the process composition still comes from
+the former single-deployment shape. The app/bot/provider credentials and the
+identifiers they issue never crossed an Organization connection boundary.
+`RuntimeScope.projectId` exists but is discarded by the env resolver, making
+the missing seam especially explicit.
+
+### 2. The control-plane work stopped at guarded Organization APIs
+
+The Organization routes provide a credible backend slice, but there is no web
+application, platform route family, platform guard, Organization provisioning
+service, connection configuration, or browser journey test. The comment
+“until admin SPA lands” and redirect-only OAuth assertion document the precise
+unfinished handoff. A small catch-all page would only hide the gap; the fix must
+deliver the authenticated, authorized workflows and their safe states.
+
+### 3. Production capacity semantics exist only in the contract
+
+ADR-0022 intentionally declares the in-memory queue and global-only settings to
+be implementation divergences. There is no common admission identity that can
+join Organization, Feishu connection, Project, user and request, so fair
+scheduling, workload brakes, rate limits, alerts and control-plane operations
+cannot currently compose. The previously created capacity tickets own this
+implementation; the admin tickets must expose their safe operator/customer
+surfaces rather than invent parallel controls.
+
+## What should remain
+
+- Keep `requireOrgRole`'s per-request session, active Organization, active
+  membership and OWNER/ADMIN checks; extend the same principle to page data and
+  add a separate `requirePlatformAdmin` rather than an Organization override.
+- Keep the Organization-scoped explorer/member/Team/access/session/usage APIs
+  as the initial backend for the org panel, while filling their missing tests,
+  pagination/limits, audit and typed-error contracts through existing tickets.
+- Keep shared Project creation and chat-binding services so web and Feishu
+  entries do not drift into two rule sets.
+- Keep the single-app Feishu handler's authorization, onboarding, run/session
+  and dedup behavior, but call it from a connection-bound listener context and
+  move accepted work/delivery into the durable lifecycle tickets.
+- Keep missing provider cost unknown rather than fabricating zero; add Provider
+  Connection attribution before presenting SaaS usage claims.
+
+## Newly clear execution frontier
+
+The product audit creates four implementation/decision slices after the
+existing secret and browser-boundary design work:
+
+1. [Decide the platform-administrator identity and audit boundary](../issues/42-decide-platform-admin-identity-audit.md).
+2. [Implement the Organization connection and secret plane](../issues/43-implement-org-connection-secret-plane.md).
+3. [Route Feishu identity and traffic by Organization connection](../issues/44-route-feishu-by-org-connection.md).
+4. [Build the private platform-administrator control plane](../issues/45-build-private-platform-admin-control-plane.md).
+5. [Build the private Organization-administrator control plane](../issues/46-build-private-org-admin-control-plane.md).
+
+The Feishu routing slice precedes durable inbound/action/outbound and signed
+traffic limiting so those mechanisms use the right issuer namespace from their
+first durable schema. The platform panel precedes the emergency-brake product
+entry so ticket 37 can reuse one authenticated/audited operator boundary.
+Existing tickets 32–38 continue to own capacity enforcement, attribution,
+alerts, brakes and calibration; the new UI tickets do not duplicate them.
+
+## Required production journey evidence
+
+The final release gate must run against at least two configured customer-owned
+Feishu applications and two Organizations, using deliberately colliding fake
+provider-local IDs where the test harness permits. It must prove:
+
+1. an unauthenticated request and an authenticated wrong-role user cannot load
+   either management page data or perform mutations;
+2. a Platform Administrator can create an Organization, bootstrap its OWNER,
+   configure/test its Feishu app and distinct platform-managed Provider
+   Connection, with redacted responses and complete audit history;
+3. an Organization OWNER/ADMIN logs in through that Organization's app, reaches
+   the real org panel, manages the accepted settings/resources, configures BYOK
+   when selected, and cannot read another Organization or platform secret;
+4. each bot receives, binds, authorizes, queues and replies only inside its
+   connection's Organization even when `open_id`, `chat_id`, event or callback
+   values collide; SUSPENDED/ARCHIVED Organizations fail closed;
+5. provider resolution for every run returns that Organization's selected
+   connection and never a process-global fallback;
+6. current member onboarding, Project authorization and Agent Run behavior
+   remain intact after connection namespacing;
+7. layered limits, fair admission, usage alerts and `DRAIN`/`STOP_NOW` are
+   visible only to the corresponding authorized control plane and behave as
+   proven by their dedicated fault/capacity suites;
+8. the complete browser + Feishu journey survives the production-shaped deploy,
+   restart, backup/restore and rollback evidence owned by ticket 08.

.scratch/saas-production-readiness/assets/product-surface-feishu-primary-sources.md

@@ -0,0 +1,333 @@
+# Feishu multi-app identity and event-routing primary sources
+
+This note answers one narrow design question for the production SaaS audit:
+what identity and routing boundaries apply when **every Hub Organization uses
+its own Feishu App and bot**. External claims below use only first-party Feishu
+Open Platform documentation. Statements labelled **Derived requirement** are
+Hub design implications, not guarantees made by Feishu.
+
+Research date: 2026-07-10.
+
+## Executive answer
+
+1. A customer-owned enterprise custom app is a single-tenant app: Feishu says
+   it is developed, reviewed, published, and used inside one tenant and cannot
+   be used by another organization. That fits “one Organization, one Feishu
+   App,” but it means the Hub is operating many independent app security and
+   identity domains, not one shared Feishu identity domain.
+   ([custom apps versus store apps](https://open.feishu.cn/document/home/app-types-introduction/self-built-apps-and-store-apps))
+2. A bare `open_id` is **not** a SaaS-global user key. It identifies a user
+   inside one app, and the same user has different Open IDs in different apps.
+   `user_id` is tenant-scoped; it is stable across apps in one tenant but
+   differs across tenants. `union_id` crosses apps only for apps from the same
+   application provider/developer; it differs between providers.
+   ([user resource overview](https://open.feishu.cn/document/server-docs/contact-v3/user/field-overview?lang=zh-CN),
+   [official ID parameter definitions](https://open.feishu.cn/document/server-docs/contact-v3/department/search?lang=zh-CN),
+   [Union ID guide](https://open.feishu.cn/document/uAjLw4CM/ugTN1YjL4UTN24CO1UjN/trouble-shooting/how-to-obtain-union-id))
+3. OAuth must be routed to the Organization's app **before** authorization.
+   The authorization request contains that app's App ID as `client_id`; the
+   callback contains `code` and the echoed `state`, not an independently
+   trustworthy Organization choice. Exchanging the code requires the matching
+   app's `client_id` and `client_secret`, and the same `redirect_uri`.
+   ([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code),
+   [obtain user access token](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
+4. A version 2 event or callback carries `header.app_id` and
+   `header.tenant_key` after parsing/decryption. An encrypted HTTP delivery,
+   however, exposes only an `encrypt` envelope before decryption, so a shared
+   endpoint cannot safely choose among many apps' Encrypt Keys from the body.
+   ([callback structure](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/callback-overview),
+   [encrypted Webhook delivery](https://open.feishu.cn/document/event-subscription-guide/event-subscriptions/event-subscription-configure-/choose-a-subscription-mode/send-notifications-to-developers-server?lang=zh-CN))
+5. A WebSocket/long-connection client is authenticated with one app's App ID
+   and App Secret. Therefore its local client instance is the pre-authenticated
+   source-app route. It is not a process-global “all customer apps” listener.
+   ([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
+6. Feishu documents `chat_id` as the conversation identifier, but the cited
+   documentation does not promise that a raw `chat_id` is globally unique
+   across every tenant and app. Chat APIs explicitly reject an operator and
+   chat from different tenants and often require the app bot to be in the
+   chat. A bot is represented by its application's `app_id`.
+   ([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology),
+   [get chat members](https://open.feishu.cn/document/server-docs/group/chat-member/get?lang=zh-CN),
+   [message resource](https://open.feishu.cn/document/server-docs/im-v1/message/intro))
+
+## 1. Application and tenant boundary
+
+### Official facts
+
+- `app_id` is the immutable identifier generated by Feishu for an application.
+  `tenant_key` is the enterprise identifier; one `tenant_key` corresponds to
+  one enterprise in actual use.
+  ([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology))
+- An enterprise custom app is restricted to one tenant. Its developers,
+  administrators, and users belong to that tenant, and another organization
+  cannot use it. A store app is the contrasting model that can be installed by
+  multiple tenants.
+  ([custom apps versus store apps](https://open.feishu.cn/document/home/app-types-introduction/self-built-apps-and-store-apps))
+- Access tokens tell Feishu which app/user is calling and which tenant's data
+  is being accessed. A `tenant_access_token` calls as the application and its
+  data range is determined by that application's permissions.
+  ([access-token overview](https://open.feishu.cn/document/server-docs/api-call-guide/calling-process/get-access-token))
+- A custom app obtains its `tenant_access_token` using that app's `app_id` and
+  `app_secret`.
+  ([custom-app tenant access token](https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal?lang=zh-CN))
+
+### Derived requirements
+
+- Model a durable `FeishuAppConnection` (or equivalent) owned by exactly one
+  Hub Organization. At minimum it binds the internal connection ID,
+  `organization_id`, `app_id`, expected `tenant_key`, encrypted `app_secret`,
+  event/callback credentials, lifecycle state, and credential version.
+- Do not select Feishu credentials from a process-global default. Every OAuth
+  exchange, token cache entry, OpenAPI call, Webhook verification, and
+  long-connection client must start from the Organization's connection.
+- Pin the first verified `tenant_key` to the connection. Later login, event,
+  and callback data must match both the expected `app_id` and pinned
+  `tenant_key`; unknown or mismatched pairs must be rejected rather than
+  auto-attached to another Organization.
+
+## 2. User identifier scopes
+
+### Official scope matrix
+
+| Identifier | Feishu's documented scope | Cross-app/tenant behavior | Availability caveat |
+| --- | --- | --- | --- |
+| `open_id` | User identity inside one application | The same user has a different `open_id` in different apps | The normal app-relative user identifier |
+| `user_id` | User identity inside one tenant | Stable across all apps in the same tenant; different in tenant A and tenant B | Reading it is a sensitive field permission and is limited to custom apps in the cited user API |
+| `union_id` | User identity across apps from the same application provider/developer | Same across that provider's apps; different across different providers | It does not make unrelated customer-owned apps one identity domain |
+| `tenant_key` | Enterprise identity | One key corresponds to one enterprise | Returned by login and carried in events/callbacks |
+
+Sources:
+
+- Feishu's current user resource overview states that `user_id` is tenant
+  identity, `open_id` is app identity, and `union_id` is identity across apps
+  developed by the same application provider. It expressly says the same
+  user's Open ID differs between apps.
+  ([user resource overview](https://open.feishu.cn/document/server-docs/contact-v3/user/field-overview?lang=zh-CN))
+- Feishu's API parameter text is more explicit: the same user's User ID differs
+  between tenants but remains the same across apps within one tenant; the same
+  user's Union ID is the same under one developer and different under another.
+  ([official ID parameter definitions](https://open.feishu.cn/document/server-docs/contact-v3/department/search?lang=zh-CN))
+- Feishu describes Union ID as the identity linking one user across multiple
+  applications supplied by the same application service provider.
+  ([Union ID guide](https://open.feishu.cn/document/uAjLw4CM/ugTN1YjL4UTN24CO1UjN/trouble-shooting/how-to-obtain-union-id))
+- Feishu's common-error guidance says not to use an Open ID obtained from app A
+  in app B, and even a test app and its production app produce different Open
+  IDs for the same user. It also reports cross-tenant errors for both User ID
+  and Union ID use.
+  ([common errors](https://open.feishu.cn/document/ukTMukTMukTM/ugjM14COyUjL4ITN?op_tracking=hc))
+- The current login user-info API returns `open_id`, `union_id`, optional
+  sensitive `user_id`, and `tenant_key` from a `user_access_token`.
+  ([get login user information](https://open.feishu.cn/document/server-docs/authentication-management/login-state-management/get?lang=zh-CN))
+
+### Answer: can different customer Apps share one global `open_id` key?
+
+**No.** This directly contradicts Feishu's documented Open ID semantics. Two
+customer-owned custom apps are different applications, so the same person has
+different Open IDs in them. Because those apps are independently owned custom
+apps, the Hub also has no documented basis to assume they are applications of
+one application provider and therefore no basis to auto-merge on `union_id`.
+`user_id` cannot solve cross-customer identity either because it changes across
+tenants.
+
+Derived identity model:
+
+- The safe external login key is `(feishu_app_connection_id, open_id)`; retain
+  `tenant_key`, `user_id`, and `union_id` as scoped attributes, not as an
+  unqualified SaaS-global primary key.
+- If the product needs one Hub human account to participate in multiple
+  Organizations, represent each Feishu login as a separate external identity
+  attached through an explicit, separately verified account-linking flow. Do
+  not silently merge on Open ID, Union ID, email, phone, or display name.
+- Do not make `user_id` mandatory for login unless every customer app is
+  required to obtain its sensitive field permission. `open_id` and
+  `tenant_key` are returned by the user-info flow without using `user_id` as
+  the login key.
+- Replacing an Organization's Feishu App changes the application identity
+  domain and therefore its users' Open IDs. Treat app replacement as an
+  explicit connection migration/re-link operation, not an in-place secret
+  edit.
+
+## 3. OAuth login must be app-routed
+
+### Official facts
+
+- The authorization URL requires `client_id`, which is the selected
+  application's App ID. `redirect_uri` must already appear in that
+  application's security settings. On success Feishu redirects with `code`
+  and, when supplied, the original `state`.
+  ([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code))
+- Feishu says `state` maintains request/callback context and must be compared
+  to prevent CSRF. Its official sample rejects a callback whose state differs
+  from the stored state.
+  ([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code),
+  [user-access-token sample](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
+- The code lasts five minutes and can be used once. Token exchange requires
+  `client_id`, `client_secret`, `code`, and, when used during authorization,
+  the same `redirect_uri`. Feishu defines error 20024 for a code/refresh token
+  that does not match the supplied client ID and error 20071 for a mismatched
+  redirect URI.
+  ([obtain user access token](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
+- Only redirect URLs configured for the selected application pass Feishu's
+  security check; an app may configure multiple URLs.
+  ([access-token acquisition flow](https://open.feishu.cn/document/server-docs/api-call-guide/calling-process/get-access-token))
+
+### Derived request flow
+
+1. Begin at an Organization-qualified route or invitation, resolve exactly one
+   active `FeishuAppConnection`, and create a short-lived, one-time server-side
+   login transaction bound to that Organization, connection, redirect URI,
+   desired return path, and PKCE verifier if used.
+2. Put only a high-entropy opaque transaction nonce in OAuth `state`; generate
+   the authorization URL with that connection's `app_id` as `client_id`.
+3. On callback, atomically consume and validate `state` before selecting
+   credentials. The callback's untrusted query parameters must never be
+   allowed to choose another Organization or App.
+4. Exchange `code` with the connection's `app_id`, decrypted `app_secret`, and
+   the exact stored redirect URI. Then call user-info and require its
+   `tenant_key` to equal the connection's pinned tenant.
+5. Resolve or create the membership through
+   `(feishu_app_connection_id, open_id)`. Never search all Organizations for a
+   bare Open ID and accept the first match.
+
+A single physical callback URL may be configured independently in multiple
+apps, but a shared URL does not remove the need for state-bound app routing.
+
+## 4. HTTP event and callback routing
+
+### Official facts
+
+- A version 2 callback header contains the application's Verification Token,
+  `event_type`, `tenant_key` (the callback application's tenant), and `app_id`
+  (the callback application's App ID).
+  ([callback overview and structure](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/callback-overview))
+- Version 2 event examples likewise place `app_id` and `tenant_key` in the
+  event header.
+  ([user-leaves-chat event](https://open.feishu.cn/document/server-docs/group/chat-member/event/deleted-2))
+- Verification Token is an application validation identifier. Feishu says the
+  server can compare the event's token with that application's token to check
+  that a request is for the specified app. With an Encrypt Key, the request
+  signature is calculated from request timestamp, nonce, that app's Encrypt
+  Key, and the original request body.
+  ([receive and verify events](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/encrypt-key-encryption-configuration-case))
+- When an Encrypt Key is configured, the HTTP request body visible before
+  decryption is an envelope containing only `{"encrypt":"..."}`; `app_id`
+  and `tenant_key` are inside the ciphertext.
+  ([send events to developer server](https://open.feishu.cn/document/event-subscription-guide/event-subscriptions/event-subscription-configure-/choose-a-subscription-mode/send-notifications-to-developers-server?lang=zh-CN))
+
+### Derived routing and verification requirements
+
+- Give every Feishu connection a stable, unguessable callback route such as a
+  connection-specific path token. Resolve the connection from that route,
+  load exactly its Encrypt Key/Verification Token, verify and decrypt, then
+  require the authenticated payload's `app_id` and `tenant_key` to match the
+  route's connection.
+- For an unencrypted common endpoint, `header.app_id` may identify a candidate
+  connection, but it is untrusted until that connection's signature/token has
+  been verified. Never route the side effect solely from an unverified body.
+- A single encrypted URL shared by every customer app has no documented
+  pre-decryption app selector. Trying every tenant's secret is not an identity
+  protocol. Use connection-specific routing instead.
+- Namespace delivery deduplication by the authenticated connection (for
+  example `(connection_id, event_id)`), and dispatch business work only after
+  app/tenant verification. Unknown, disabled, or mismatched connections fail
+  closed and produce an auditable security log.
+
+## 5. Long-connection routing
+
+### Official facts
+
+- Feishu's SDK creates a WebSocket client with one app's required `APP_ID` and
+  `APP_SECRET`. Events subscribed by that application are then delivered over
+  that channel. Authentication is performed when the connection is
+  established.
+  ([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
+- Long-connection event subscription supports enterprise custom apps, not
+  store apps. Each app may establish at most 50 connections. If one app has
+  multiple clients, delivery is cluster-mode rather than broadcast: one random
+  client receives a message.
+  ([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
+- Long-connection callbacks use the same per-app App ID/App Secret client
+  construction and have the same cluster-mode single-recipient behavior.
+  ([long-connection callback subscription](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/step-1-choose-a-subscription-mode/configure-callback-request-address?lang=zh-CN))
+
+### Derived routing requirements
+
+- Create one logical client binding per enabled `FeishuAppConnection`, even if
+  an in-process supervisor manages many clients. Attach immutable
+  `organization_id` and `connection_id` context to each handler; the client
+  that received the frame is the pre-authenticated application route.
+- Never infer the receiving Organization by looking up a bare actor Open ID or
+  chat ID. First use the authenticated client binding; then cross-check any
+  delivered `app_id`/`tenant_key` fields available in the event structure.
+- Scaling several replicas for the same app does not create duplicate
+  broadcast processing according to Feishu's contract, but delivery still
+  requires application-level idempotency and health ownership. Capacity must
+  count clients per app against the documented 50-connection ceiling.
+
+## 6. Bot, chat, and message identifiers
+
+### Official facts
+
+- In the message resource, a sender can be a user or an application. For an
+  application sender, `id_type` is `app_id`; the sender also carries a
+  `tenant_key`. Thus the Feishu bot identity used by messaging is its
+  application identity, not a Hub-global bot ID.
+  ([message management overview](https://open.feishu.cn/document/server-docs/im-v1/message/intro))
+- `chat_id` identifies a conversation, including direct and group chats.
+  Feishu's group resource also exposes the chat's `tenant_key`.
+  ([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology),
+  [group management overview](https://open.feishu.cn/document/server-docs/group/chat/intro?lang=zh-CN))
+- Chat APIs require the operator to be in the same tenant as an internal chat;
+  common errors include `Operator and chat can NOT be in different tenants`.
+  Many operations additionally require the calling app's bot to be in the
+  group and reject an app that is unavailable in that tenant.
+  ([get chat members](https://open.feishu.cn/document/server-docs/group/chat-member/get?lang=zh-CN))
+- The first bot direct-chat event calls `chat_id` the conversation ID between
+  **that robot and user** and includes `app_id`, `tenant_key`, and app-scoped
+  user Open IDs in the same event.
+  ([user and bot first conversation](https://open.feishu.cn/document/server-docs/group/chat-member/event/bot-events?lang=zh-CN))
+
+### What the official sources do not establish
+
+The cited documentation calls `chat_id` a conversation/group identifier but
+does not state that its value is globally unique across every Feishu tenant and
+every independently created app. It does establish tenant and bot-membership
+authorization boundaries. Under the repository rule that unspecified scope
+must not be assumed, raw global uniqueness would be an unsupported inference.
+
+Derived storage/call rules:
+
+- Store chats as `(feishu_app_connection_id, chat_id)` and retain observed
+  `tenant_key` and chat type. Do the same for other resource identifiers unless
+  Feishu explicitly contracts a wider scope.
+- Always call chat/message APIs with the token obtained from the chat's bound
+  connection. Never retry an authorization failure with another Organization's
+  token.
+- Treat bot direct chats as connection-specific. A `chat_id` for user X's
+  conversation with Organization A's bot is not evidence of a conversation
+  with Organization B's bot.
+- For group chats, two customer apps may both be members of a real Feishu
+  group, but that does not authorize the Hub to move configuration, Projects,
+  or memberships between Organizations. The authenticated connection remains
+  the tenant route and Hub Organization remains the authorization root.
+
+## Minimum invariants for implementation review
+
+1. `FeishuIdentity` uniqueness is `(connection_id, open_id)`, never bare
+   `open_id`.
+2. OAuth `state` resolves one server-side transaction bound to one Organization
+   and connection; code exchange uses only that connection's credentials and
+   stored redirect URI.
+3. Login accepts the returned identity only when `tenant_key` matches the
+   connection's pinned tenant.
+4. HTTP Webhooks choose a connection-specific verification/decryption context
+   before trusting payload routing fields; authenticated `app_id` and
+   `tenant_key` must match it.
+5. Every long-connection handler carries the local connection/Organization
+   binding of the App ID/App Secret client that received the frame.
+6. Bot, chat, event-deduplication, token-cache, and user identifiers are all
+   namespaced by the Feishu connection unless Feishu explicitly guarantees a
+   wider scope.
+7. A credential, app, or tenant mismatch is a hard failure with traceable
+   logging; there is no fallback to a default app or search across tenant
+   secrets.

.scratch/saas-production-readiness/assets/production-observability-recovery.md

@@ -0,0 +1,245 @@
+# Production observability and incident-recovery audit
+
+Date: 2026-07-10
+
+Scope: the `hub` HTTP service, Feishu WebSocket ingress and outbound calls, PostgreSQL/Prisma, agent execution, the `cph` subprocess, project workspaces, and the packaged deployment path. This is a release-gate audit, not a proposal to change product semantics. Where the Lean contract is open, this report asks for an operational decision rather than inventing one.
+
+## Verdict
+
+**No-go for unattended SaaS production deployment.** The service exposes a liveness-shaped endpoint, emits some structured component logs, and persists useful run/audit fields, but it cannot currently answer the minimum production questions:
+
+1. Can this instance accept a real request *now* (database reachable, Feishu listener authenticated, required runtime executable and workspace usable)?
+2. Which tenant, request, Feishu event, run, subprocess, and outbound message participated in one failed user action?
+3. How many requests/runs/messages are failing, stuck, delayed, retried, or duplicated?
+4. Which condition should page an operator, and which documented recovery action is safe?
+5. After a restart, deploy, rollback, or dependency outage, what evidence proves that work was neither lost nor silently acknowledged?
+
+Three fault injections demonstrate that the current `/api/healthz` and logging surface can be green while the service is already unable to serve its core paths:
+
+- After startup, cutting PostgreSQL sockets left two consecutive `/api/healthz` requests at HTTP 200.
+- An invalid Feishu app ID was rejected by the installed Feishu SDK while `/api/healthz` remained HTTP 200.
+- With PostgreSQL cut, `/api/me` returned HTTP 400 `bad_request`, exposed a raw Prisma invocation in the response, produced no request-scoped application error log, and only emitted two unscoped Prisma error lines.
+
+Together with the already-confirmed lifecycle and deployment failures, this means green process state, green health checks, and even a visible user error do not prove that the service or its durable workflows are healthy.
+
+## Evidence and positive signals already present
+
+The starting point is not empty. These are useful foundations to retain:
+
+- Fastify's built-in HTTP logging records request/response facts with an internal request ID through the server logger ([`hub/src/server.ts`](../../../hub/src/server.ts)).
+- The same Fastify logger is threaded into the Feishu trigger/runtime, while Prisma emits its configured error/warn events (and optional query events) ([`hub/src/server.ts`](../../../hub/src/server.ts), [`hub/src/db.ts`](../../../hub/src/db.ts)).
+- `AgentRun` persists status, timestamps, error/result fields, token/cost fields, and links to project/session; `ProjectAgentLock` persists a heartbeat; `AuditEntry` and `AgentMessage` provide potential durable trace anchors ([`hub/prisma/schema.prisma`](../../../hub/prisma/schema.prisma)).
+- Agent execution emits intermediate activity to the streaming Feishu card, and lock heartbeat activity is attempted during execution ([`hub/src/agent/runner.ts`](../../../hub/src/agent/runner.ts), [`hub/src/feishu/trigger.ts`](../../../hub/src/feishu/trigger.ts)).
+- Deployment is managed by systemd, so restart policy, exit status, and journal collection have an existing operational host surface ([`hub/deploy/cph-hub.service`](../../../hub/deploy/cph-hub.service)).
+- Database migrations are explicit Prisma artifacts rather than ad-hoc startup DDL ([`hub/prisma/migrations`](../../../hub/prisma/migrations)).
+- Existing tests cover many individual authorization, trigger, lock, queue, runner, and API behaviors. They are valuable regression evidence, although they are not yet a deploy/recovery acceptance suite.
+
+These signals are fragmented rather than joined into an operational contract. Several writes are best-effort or swallowed, identifiers are not propagated end-to-end, there is no metrics/alert surface, and the health endpoint does not test dependencies.
+
+## Failure-to-signal matrix
+
+| Surface | Failure or degraded condition | Current externally visible signal | What is missing / why the signal is unsafe | Minimum production signal and recovery evidence |
+| --- | --- | --- | --- | --- |
+| HTTP | PostgreSQL becomes unreachable after startup | Affected request fails; `/api/healthz` stays 200 | Health is process liveness only. `/api/me` misclassifies the dependency failure as client `400 bad_request`, leaks raw Prisma call text, and creates no request-scoped error record. An LB/operator cannot distinguish bad input from platform outage. | Separate liveness and readiness. Readiness performs a bounded DB probe and returns non-2xx on failure. Unexpected exceptions map to sanitized 5xx with a stable error code and `request_id`; structured log contains the same ID and normalized Prisma category. Alert on readiness failure and 5xx/error-code rate. |
+| HTTP | Handler latency, event-loop stall, exhausted connections, overload | Access log only after a response is completed | No in-flight count, latency histogram, timeout counter, saturation signal, or request correlation. A hanging request may emit no timely evidence. | Request count, duration, in-flight requests, timeout/cancellation count, response class, and event-loop/runtime/process saturation. Bound request duration and log start/end/error with one `request_id`. |
+| Feishu ingress | Invalid/revoked app credentials; WebSocket connect/auth failure | SDK rejects/logs asynchronously; process and `/api/healthz` remain green | Startup success is reported before listener readiness. No listener state, last-connected/last-event timestamp, reconnect counter, or readiness dependency. The installed SDK already exposes `onReady`, `onError`, `onReconnecting`, `onReconnected`, `getConnectionStatus`, and `close`, but the Hub does not retain or wire the `WSClient`. systemd can perpetually supervise a process that cannot receive work. | Listener state machine (`starting/connected/degraded/stopped`), last successful connection/event, reconnect/auth-failure counters, bounded startup readiness gate, and alert on auth failure or prolonged disconnect. Credential contents must never be logged. |
+| Feishu ingress | Duplicate delivery, handler failure after receipt, process restart with queued/batched work | Event receipt/dedup row and selected warning logs | The run-lifecycle audit proved receipt-before-success and in-memory batching/queue loss. There is no durable processing state/lease, retry age, dead-letter view, or alert for accepted-but-unstarted work. | Durable ingress states and timestamps, attempts, last normalized error, event/chat/project correlation, queue depth/oldest age, dead-letter count. Recovery drill proves restart resumes or deliberately redrives exactly the intended work. See [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
+| Feishu outbound | API timeout/ambiguous success, rate limit, invalid credentials, permanent card patch/send failure | Some helpers log a warning; several message helpers catch failure and return `null` | No idempotent delivery record, delivery state, attempt count, provider request/message ID, retry delay, terminal failure metric, or user-visible reconciliation path. Fault injection proved a run can be marked completed although nothing was delivered. | Durable outbox/delivery state; idempotency key; provider message/request ID; attempts, latency and normalized provider error; queue depth/oldest age/dead-letter signals; recovery command to retry or reconcile safely. |
+| PostgreSQL | Startup unavailable | Process startup rejects | This is a useful fail-fast signal, but it is not joined to a deploy gate with bounded logs and rollback criteria. | Installer/supervisor waits for explicit readiness, captures failure reason, and aborts/rolls back without replacing last-known-good service. |
+| PostgreSQL | Runtime outage, pool saturation, slow queries, migration incompatibility | Raw Prisma errors; request symptoms | No readiness failure, pool/saturation metric, query latency/timeout signal, migration compatibility assertion, or tenant/request/run correlation. Raw client internals can cross the HTTP boundary. | DB probe latency/result, Prisma error code/category counters, bounded query timing for critical operations, connection/pool saturation if available, schema migration identifier, sanitized API mapping, and DB outage/runbook drill. Prisma documents structured client error types/codes and configurable event-based logging; use those instead of returning raw exception strings ([Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference), [Prisma logging](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging)). |
+| Agent run | SDK/model error, credentials invalid, provider throttling, malformed stream | Run may become `FAILED`; the card may show an error and trigger finalization may warn | The installed SDK emits API retry, rate-limit, tool-progress, runtime-init/MCP-status, provider request ID, duration, and error-category messages, but `runner.ts` handles only stream/assistant/user/result messages and drops the other lifecycle signals. Provider/model/attempt/latency/normalized error are not consistently available as metrics or correlated logs. Some persistence and audit operations are best-effort. | Run start/finish transition event with `run_id`, `session_id`, tenant/project, provider/model, status, duration, turns, tokens/cost, normalized error and retryability; capture the SDK's bounded retry/rate-limit/tool/runtime signals; aggregate run outcome/latency/stuck gauges without high-cardinality IDs as metric labels. |
+| Agent run | No result/hang, missed heartbeats, process death, terminal DB write failure | Lock heartbeat may stop; restart sweep mutates some state | No wall-clock deadline/watchdog, stale-lock/stale-active-run metric, readiness impact, page, or documented reconciliation. Lifecycle fault injections proved active run without lock, terminal run with lock, and timeout misclassification are possible. | Deadline and durable lease semantics, gauges for active/waiting/stale runs and locks, oldest age, transition-failure counter, reconciliation command and drill. Startup recovery must be transactional and evidence-producing. See [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
+| `cph` subprocess | Missing binary, permission failure, timeout, non-zero exit, malformed output | In production, `cph_check`/`cph_build` grant the Claude SDK's Bash tool; tool results are forwarded only to the in-memory streaming card. The direct `agent/cph.ts` wrapper captures exit/stdout/stderr but is used only by its integration tests, not by the production runner. | There is no production-owned `cph` invocation boundary, startup/readiness validation of the exact deployed artifact, version/build identity, persisted operation result, duration metric, bounded/redacted stderr contract, or reliable distinction between spawn failure and domain diagnostics. | Readiness checks executable path/version and required filesystem permissions without mutating domain state. Introduce an observable production invocation boundary or capture the SDK Bash/tool lifecycle with operation, run/project correlation, duration, exit category and bounded redacted diagnostic; metrics count timeout/spawn/non-zero/protocol errors. Deployment evidence pins and smoke-tests the artifact. |
+| Workspace | Missing/unwritable root, wrong ownership, disk/inode exhaustion, symlink/root escape, partial agent writes | Error appears only when a command/file operation happens | No readiness check, free-space/inode monitoring, workspace mutation journal/commit marker, or recovery procedure. Security audit already proved containment failures; lifecycle audit proved a run can fail after workspace side effects. | Bounded non-destructive readiness check for configured root, owner/mode, read/write/rename/fsync as required, free space and inode signals, per-project mutation correlation, containment validation, backup/restore and partial-write reconciliation drill. See [tenant/auth security audit](./tenant-auth-security-audit.md) and [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
+| Deploy | Bad env/service user/artifact, failed migration/build, unhealthy new process | Shell/systemd exit in some cases; process may restart | Clean-host audit found missing required env/service identity/artifact and in-place, non-atomic release replacement. Health is liveness-only and there is no last-known-good rollback unit. | Immutable versioned release, preflight, migration compatibility gate, explicit readiness deadline, release/build/schema identity endpoint/log, automated rollback criteria, last-known-good artifact and tested rollback. See [clean-host deployment audit](./clean-host-deployment-audit.md). |
+| Deploy | SIGTERM/restart during active work | systemd stops/restarts the process | No graceful drain/stop readiness, bounded shutdown, durable queue resume, or post-restart reconciliation proof. In-memory queues are lost. | On termination: readiness false, stop ingress, drain or checkpoint durable work, enforce timeout, emit shutdown summary. Recovery drill sends work across restart and verifies a single durable outcome and delivery. systemd supports explicit notification/readiness/watchdog protocols, but the current unit does not implement one ([systemd service types](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html), [`sd_notify`](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html)). |
+
+## Logs and end-to-end traceability
+
+### Current gaps
+
+- Fastify generates an internal request ID for its access logs, but it is not returned to the client as a support handle or propagated into application errors, Prisma diagnostics, Feishu work, runs, or subprocess calls.
+- Feishu `event_id`, provider request/message IDs, `chat_id`, `project_id`, `organization_id`, `run_id`, `session_id`, and subprocess invocation are not propagated as one causally connected context.
+- The DB-cut experiment produced raw Prisma diagnostics without an application error record that identified route, request, response class, or tenant context.
+- Several critical failure paths catch and ignore persistence errors or only warn, including audit/message persistence and some card/lock finalization paths. This creates absence of evidence precisely when durable state is least trustworthy.
+- The production runner discards SDK `api_retry`, `rate_limit_event`, `tool_progress`, runtime initialization/MCP status, and related system messages even though the installed SDK exposes the fields needed to diagnose provider and tool degradation.
+- Exception strings can cross the API boundary. Besides leaking implementation details, this makes the client-visible error vocabulary unstable and impossible to alert on reliably.
+- Logs have no documented retention, rotation, ingestion, access-control, redaction, or tenant-support lookup contract. Journald availability alone is not a log strategy.
+
+### Minimum structured event envelope
+
+Every boundary and durable state transition should emit one structured event with a stable `event_name`, `timestamp`, `severity`, `service`, `environment`, `release`, and the applicable correlation fields:
+
+`request_id`, `feishu_event_id`, `organization_id`, `project_id`, `chat_id`, `actor_user_id`, `run_id`, `session_id`, `delivery_id`, `provider_request_id`, and `cph_invocation_id`.
+
+Raw prompt/message/file contents, credentials, cookies, authorization headers, provider tokens, full Prisma invocation text, and unrestricted stdout/stderr must not be logged. Tenant/project/user identifiers are operationally necessary but access-controlled personal/customer metadata; metric labels must not use high-cardinality IDs. OpenTelemetry's HTTP semantic conventions specify the stable `http.server.request.duration` instrument, an opt-in `http.server.active_requests` instrument, and low-cardinality route/error guidance ([HTTP metrics semantic conventions](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/)).
+
+At minimum, events are required for:
+
+- request accepted/completed/failed/canceled;
+- Feishu listener connected/disconnected/auth-failed/reconnecting;
+- ingress received/deduplicated/queued/claimed/started/retried/dead-lettered;
+- run and lock state transitions, transition-write failures, heartbeat stale detection, and recovery decisions;
+- provider/agent start/result/error with normalized category;
+- `cph` spawn/result/timeout/protocol failure;
+- workspace preflight and mutation/commit/reconciliation decisions;
+- outbound delivery queued/attempted/confirmed/retried/dead-lettered;
+- deploy, schema migration, readiness transition, drain, shutdown, startup recovery, and rollback.
+
+Errors must be allowed to fail the affected operation rather than disappear. If a non-critical telemetry sink is unavailable, that telemetry failure itself needs a bounded local signal/counter; durable business audit records are not optional telemetry and must follow their explicit transactional contract.
+
+## Health, readiness, and startup contract
+
+The service needs three distinct concepts:
+
+1. **Liveness**: the process/event loop is responsive. This endpoint must be cheap and must not depend on remote services. The present `/api/healthz` can serve this purpose if it is named/documented as liveness only.
+2. **Readiness**: the instance can accept the traffic assigned to it. A bounded probe must include at least PostgreSQL connectivity/schema compatibility, Feishu listener authenticated/connected state for ingress-serving instances, required runtime configuration, exact `cph` artifact availability/version, and workspace root access/capacity. A failed dependency makes readiness non-2xx without killing the process and losing diagnostics.
+3. **Startup/deploy acceptance**: migrations/preflight complete, listener reaches ready state within a deadline, one authenticated HTTP/database smoke path and one controlled runtime/workspace smoke path pass, and the deployed release/schema identity matches the intended version. Only then is traffic enabled and the previous release eligible for cleanup.
+
+Readiness responses should expose only component names, stable state/error codes, and timestamps—not secrets or raw exceptions. Full diagnostic context belongs in correlated restricted logs. The deployment audit must decide whether Feishu and HTTP run in one instance role; if roles are split later, readiness dependencies become role-specific rather than globally optional.
+
+## Minimum metrics and alerts
+
+Exact SLO values are product/operations decisions and are currently unspecified. The release nevertheless needs the following bounded-cardinality instruments and a reviewed initial threshold for each page/ticket alert.
+
+### Traffic and failures
+
+- HTTP request count, duration, active requests, status/error-code class, timeouts/cancellations.
+- Feishu listener connected state, reconnect/auth failures, time since last successful connection/event.
+- Ingress queue/receipt counts by durable state, processing attempts, oldest pending age, dead-letter count.
+- Outbound delivery attempts/results/latency, retry/dead-letter count, oldest pending age, provider rate-limit responses.
+- Prisma operation failures by normalized code/category and bounded critical-query duration; DB readiness result/latency.
+
+### Workflow integrity and capacity
+
+- Runs started and terminal outcomes, run duration, active/waiting/stale count and oldest age.
+- Locks held/stale/orphaned and transition/reconciliation failures.
+- Agent/provider failures by provider/model/error category; turns/tokens/cost as operational/business counters with tenant detail kept out of metric labels.
+- `cph` invocation count/duration/result category/timeouts.
+- Workspace free bytes/inodes, preflight failures, write/commit/reconciliation failures.
+- Process uptime/restarts, event-loop delay, memory, CPU, file descriptors, unhandled rejection/uncaught exception, graceful shutdown outcome.
+- Release/build/schema identity as an info metric or startup event, never as an unbounded label set.
+
+### Required release-time alerts
+
+Before production traffic, alerts and routing must exist for:
+
+- instance readiness false beyond the rollout/startup grace period;
+- Feishu auth failure or prolonged listener disconnect;
+- sustained server-side HTTP errors or request timeouts;
+- database readiness failure / critical Prisma failure spike;
+- oldest ingress/outbound item, active run, or lock exceeds its decided deadline;
+- dead-letter count non-zero or durable state-transition/reconciliation failure;
+- agent/provider or `cph` failure/timeout spike;
+- workspace disk/inodes below headroom threshold or workspace preflight fails;
+- restart loop, failed deployment, failed migration, or rollback;
+- telemetry pipeline blind (expected heartbeat/scrape/log ingestion absent).
+
+Each alert needs an owner, severity, user impact statement, dashboard/log query, first safe diagnostic command, mitigation, escalation criterion, and recovery verification. A page without a runbook is not a production control.
+
+## Incident recovery runbooks required before release
+
+At minimum, checked-in runbooks must cover these scenarios. Commands must default to read-only inspection; destructive/redrive/force-release actions require explicit confirmation, identify tenant/project/run scope, emit an audit record, and be idempotent or document why they are not.
+
+1. **PostgreSQL unavailable/degraded** — identify connection vs credential vs migration vs capacity failure; stop accepting work through readiness; restore service; reconcile active runs, locks, ingress receipts, and outbox; verify no raw errors escape HTTP.
+2. **Feishu listener disconnected/credentials revoked** — distinguish auth from transient network/reconnect; restore credentials safely; verify last event and controlled inbound round trip; inspect ingress gap and redrive rules.
+3. **Ingress backlog/dead letter** — list oldest/attempts/error by tenant-safe scope; decide retry vs reject; prove dedup/idempotency; verify one resulting run.
+4. **Stuck/orphaned run or lock** — inspect run/session/lock/provider/workspace/audit/outbox together; use lease/reconciliation operation rather than direct row deletion; prove terminal state and next queued work behavior.
+5. **Outbound delivery failure** — reconcile provider message ID and durable outbox; retry with idempotency key; confirm user-visible delivery without duplicating a card/message.
+6. **Agent/provider or `cph` outage** — identify provider credential/rate-limit/runtime/protocol failure; halt or queue new runs; test a canary; resume and reconcile affected work.
+7. **Workspace capacity/corruption/containment event** — quarantine affected project, snapshot evidence, restore/rollback documented unit, verify ownership/containment and repository state before resuming.
+8. **Failed migration/deployment** — preserve old release, collect preflight/migration/readiness diagnostics, follow compatibility-aware application/database rollback, and verify release/schema identity plus canaries.
+9. **Process/host restart during work** — drain when planned; after unplanned restart run durable reconciliation; verify receipts, runs, locks, workspace effects and deliveries converge exactly once at the business level.
+10. **Suspected tenant data exposure** — disable affected path/tenant access without deleting evidence, preserve correlated audit/log records, rotate exposed credentials, determine affected org/project/users, and follow an explicitly owned incident-notification decision process.
+
+## Minimum release evidence
+
+Production readiness is not proved by a green unit suite alone. A release candidate needs an archived, attributable evidence bundle containing all of the following:
+
+### Build and supply identity
+
+- immutable application and `cph` artifact digests, source commit, build timestamp/toolchain, dependency lockfiles and vulnerability policy result;
+- deployed configuration manifest listing required keys and instance role, with secret values redacted;
+- migration set/schema identity and a compatibility statement for both forward deploy and rollback;
+- systemd unit and installer/preflight version used on a clean host.
+
+### Automated gates
+
+- typecheck, complete unit/integration suite, migration-from-supported-version tests, and Lean `lake build` when `spec/` changes;
+- clean-host install test with the real packaged artifacts and declared service user/directories;
+- negative startup tests for missing/invalid env, DB unavailable, invalid Feishu credentials, missing/wrong `cph`, and unwritable/full workspace;
+- readiness tests proving each dependency failure independently makes readiness fail while liveness remains diagnostically available;
+- API error-mapping tests proving unexpected dependency exceptions return sanitized 5xx/stable codes and create a correlated error log;
+- lifecycle fault tests from the run-lifecycle audit, including process kill/restart, DB transition failures, duplicate ingress, ambiguous outbound completion, timeout, and concurrent trigger/session creation;
+- tenant isolation/security regression suite, symlink/root containment tests, and dependency audit policy gate.
+
+### Staging drills against production-shaped dependencies
+
+- authenticated Feishu inbound → durable receipt → one run → workspace effect → durable terminal state → one outbound confirmation, searchable end-to-end by a correlation ID;
+- PostgreSQL socket cut/restoration while serving: readiness changes, alert fires, requests return sanitized server errors, state reconciles, alert clears;
+- Feishu credential rejection/disconnect/reconnect: readiness/alert and controlled recovery are visible;
+- SIGTERM and forced process kill during queued, active, and outbound-pending work: restart reconciliation proves no silent loss and no unintended duplicate user effect;
+- `cph` timeout/non-zero/malformed response, provider throttle/error, workspace disk/permission failure, and outbound rate-limit/permanent failure drills;
+- failed deploy/readiness and rollback drill on a clean host, followed by migration/data compatibility and canary verification;
+- backup restore of representative database and workspace data, with measured recovery point/time and tenant isolation checks.
+
+### Operations acceptance
+
+- dashboards expose all release metrics above and show release/schema identity;
+- every required alert is injected once in staging, reaches the named owner, links to its checked-in runbook, and clears only after verified recovery;
+- log search can pivot from request/event to org/project/run/session/delivery without message content or secrets; retention and access controls are documented and tested;
+- on-call ownership, escalation, incident severity, maintenance/rollback authority, backup ownership, and recovery objectives are explicitly decided;
+- a signed release checklist records who reviewed each piece of evidence and where immutable test/drill outputs are stored.
+
+## Prioritized remediation / release gates
+
+### P0 — before any external production traffic
+
+1. Implement distinct bounded liveness, readiness, and startup/deploy acceptance contracts; make deployment traffic activation depend on readiness.
+2. Install centralized unexpected-error handling that returns sanitized stable 5xx responses and emits request-correlated structured errors. Never return raw Prisma/SDK/process exception text.
+3. Make Feishu connection/auth state observable and readiness-relevant; make listener startup rejection fail the startup gate.
+4. Implement the durable ingress, transactional run/lock/session, durable outbox, workspace commit/reconciliation, and durable audit/history work already opened by the lifecycle audit. Observability cannot compensate for acknowledged work that has no recoverable state.
+5. Add end-to-end correlation and stable structured transition events; remove silent catches from critical paths.
+6. Add the minimum metrics, alerts, dashboards, and checked-in recovery runbooks, then execute the fault/restart/rollback drills.
+7. Replace in-place deployment with the clean-host audit's immutable release/readiness/rollback contract and prove it on a clean host.
+8. Close the critical tenant/agent/workspace/Feishu authorization and dependency findings from the security audit before declaring readiness.
+
+### P1 — required for routine operations immediately after the P0 architecture is in place
+
+- Define initial SLOs/error budgets and tune alert thresholds using staging/load evidence.
+- Add capacity/load/soak tests for DB connections, event-loop/process resources, ingress/outbound backlog, agent concurrency, provider limits, workspace disk/inodes, and log/metric pipelines.
+- Exercise backup/restore, credential rotation, tenant-support trace lookup, and incident evidence preservation.
+- Define data retention/deletion/access-control policy for logs, metrics, traces, audits, messages, receipts, deliveries, runs, and workspaces.
+
+## Decision points that remain OPEN
+
+The code/spec does not currently authorize choosing these product/operations semantics implicitly:
+
+- readiness dependency roles (combined HTTP+Feishu process versus independently deployable roles);
+- maximum acceptable ingress, run, lock, and outbound ages; retry budgets; terminal/dead-letter semantics;
+- workspace mutation commit/rollback unit and what a user should observe after partial agent work;
+- audit durability/availability trade-off and exact event taxonomy;
+- SLOs, recovery point objective, recovery time objective, retention, regional/data-residency constraints, and incident-notification ownership;
+- whether deploy migration compatibility must support application rollback across one or more releases.
+
+These decisions should be recorded in the upstream contract/ADR layer before implementation when they affect domain-visible behavior. Operational mechanisms that only expose already-defined behavior can be implemented and tested without inventing domain semantics.
+
+## Source notes
+
+Primary local evidence:
+
+- [`hub/src/server.ts`](../../../hub/src/server.ts), [`hub/src/admin/errors.ts`](../../../hub/src/admin/errors.ts) — process startup, health/listener/restart behavior, and HTTP exception-to-response mapping.
+- [`hub/src/db.ts`](../../../hub/src/db.ts) — current Prisma event/log configuration.
+- [`hub/src/feishu/client.ts`](../../../hub/src/feishu/client.ts), [`hub/src/feishu/trigger.ts`](../../../hub/src/feishu/trigger.ts), [`hub/src/feishu/messageBatcher.ts`](../../../hub/src/feishu/messageBatcher.ts), [`hub/src/feishu/triggerQueue.ts`](../../../hub/src/feishu/triggerQueue.ts) — ingress/listener, batching, dedup, queue, run and outbound behavior.
+- [`hub/src/agent/runner.ts`](../../../hub/src/agent/runner.ts), [`hub/src/agent/roleTools.ts`](../../../hub/src/agent/roleTools.ts), [`hub/src/agent/cph.ts`](../../../hub/src/agent/cph.ts), [`hub/src/lock.ts`](../../../hub/src/lock.ts), [`hub/src/audit.ts`](../../../hub/src/audit.ts) — production SDK/Bash tool routing, the legacy direct cph wrapper, persistence, heartbeat/lock, and audit behavior.
+- [`hub/prisma/schema.prisma`](../../../hub/prisma/schema.prisma) — durable state and available correlation fields.
+- [`hub/deploy`](../../../hub/deploy) — installer/systemd/deploy behavior.
+- [Clean-host deployment audit](./clean-host-deployment-audit.md), [tenant/auth security audit](./tenant-auth-security-audit.md), and [run-lifecycle integrity audit](./run-lifecycle-integrity-audit.md) — prior deterministic audits and their recorded commands/results.
+
+Official references used only for interface semantics and implementation guidance:
+
+- [Prisma Client error reference](https://www.prisma.io/docs/orm/reference/error-reference) and [Prisma logging](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging).
+- [OpenTelemetry HTTP metric semantic conventions](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/).
+- [systemd service types/readiness](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html) and [`sd_notify`](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html).
+
+No secondary production-readiness checklist was treated as authority. Recommendations are derived from the local failure evidence and the primary interfaces the service actually uses.

.scratch/saas-production-readiness/assets/run-lifecycle-integrity-audit.md

@@ -0,0 +1,272 @@
+# Run lifecycle and data-integrity audit
+
+## Verdict
+
+The current Hub does **not** have a crash-safe or database-failure-safe run
+lifecycle. The normal happy path is covered by tests and the database correctly
+enforces one lock per project, but the surrounding workflow is a chain of
+independent effects: inbound receipt, in-memory batching/queueing, session/run
+creation, lock acquisition, workspace mutation, Feishu delivery, session cursor
+update, run terminal update, audit write, and lock release.
+
+There is no durable coordinator or reconciliation loop across those effects.
+Deterministic fault injection confirmed that the current code can permanently
+lose accepted work, leave a terminal run holding a lock, leave an `ACTIVE` run
+without a lock, split one logical session into duplicate active rows, declare a
+run complete without delivering its result, and create an audit entry referring
+to no run. A hung provider call also has no wall-clock deadline or usable lease,
+so one run can block a project until an operator restarts the service or edits
+the database.
+
+These are production blockers for the accepted single-host topology; they do
+not require HA to reproduce.
+
+## Contract baseline
+
+The audit used the following upstream decisions and intentionally did not
+invent semantics where the contract is open:
+
+- `Spec.System.Run`: `active` and `waitingForUser` are non-terminal;
+  `completed`, `failed`, `timedOut`, and `canceled` are terminal.
+- `Spec.System.Lock` and ADR-0002: each project has at most one lock, its owner
+  is an `AgentRun`, every lock holder is non-terminal, and the lock is released
+  when the run completes, fails, times out, or is canceled.
+- ADR-0002: many runs reuse the same long-lived `AgentSession`.
+- ADR-0017: the active session identity is project + provider + role + model,
+  and its provider cursor is persisted for the next run.
+- `Spec.System.Audit`: every `AuditEntry` is associated with a run. Audit event
+  content, retention, and query dimensions remain `OPEN`.
+- The contract does **not** decide whether filesystem changes made by a failed,
+  canceled, timed-out, or process-killed run should commit, remain as partial
+  work, or roll back. That choice is surfaced below rather than assumed.
+
+## Positive evidence
+
+The core happy path is real:
+
+- PostgreSQL uniqueness enforces at most one `ProjectAgentLock` per project and
+  at most one project lock per run.
+- `currentLockRunId` rejects a terminal run that still holds a lock instead of
+  silently treating it as healthy.
+- Normal trigger completion updates the provider cursor, writes a terminal run
+  state, and releases the lock. Interrupt uses an `AbortController` and maps to
+  `CANCELED` on the normal path.
+- A sequential Feishu redelivery with the same event ID is skipped, so the
+  ordinary duplicate case does not start a second run.
+- The targeted lifecycle baseline passed before fault injection:
+
+```text
+npm run check
+6 test files passed
+71 tests passed
+```
+
+Those protections are necessary, but they only cover operations in which every
+dependency succeeds in the expected order.
+
+## Executed fault-injection evidence
+
+A temporary Vitest harness exercised the real trigger, Prisma test database,
+message batcher, and runner seams. It asserted the production invariant rather
+than the current behavior. The harness was run repeatedly and produced the same
+result each time; the final run was:
+
+```text
+npm test -- --run test/run-lifecycle-audit.repro.test.ts
+
+Test Files  1 failed (1)
+Tests       8 failed (8)
+Duration    918ms
+```
+
+The throwaway harness was removed after diagnosis. Its eight minimal probes and
+observed values were:
+
+| Injected condition | Required invariant | Observed current result |
+| --- | --- | --- |
+| Authorization database error after Feishu receipt creation, followed by the same event redelivery | The failed event is retried | Authorizer calls remained `1`; redelivery was skipped |
+| Batched-run setup callback throws, then `flushAll` retries | The pending batch remains recoverable | Callback attempts remained `1`; batch had already been deleted |
+| Two active sessions with the same project/provider/role/model tuple | Database rejects the duplicate | Second insert succeeded |
+| Audit insert uses a nonexistent run ID | Database enforces `AuditEntry → AgentRun` | Insert succeeded with arbitrary `runId` |
+| Every Feishu message create/reply fails | Completion is not reported as delivered | Run became `COMPLETED`; delivered cards were `0` |
+| First lock release delete fails | Terminal run is eventually unlocked | Lock count remained `1` after run became `COMPLETED` |
+| Every terminal `AgentRun.update` fails | A non-terminal run retains/reacquires its lock | Run remained `ACTIVE`; lock count became `0` |
+| Claude SDK returns `error_max_turns` | Runner returns `length`, trigger stores `TIMED_OUT` | Runner returned `failed` |
+
+This is a tight reproduction at the actual call seams; none of the failures
+depends on timing or external network access.
+
+## Critical findings
+
+### 1. Accepted Feishu work can be permanently lost
+
+`trigger.ts` commits `FeishuEventReceipt` before message validation, binding
+lookup, authorization, file download, command execution, batching, or run
+creation. The receipt has no processing state, lease, attempt count, error, or
+completion marker. Any later exception leaves a permanent row, so Feishu's
+at-least-once redelivery is treated as already completed.
+
+The two in-process buffers compound that behavior:
+
+- `MessageBatcher.flushPendingBatch` deletes the batch before invoking its
+  callback. Timer-triggered callback failures are swallowed with
+  `.catch(() => undefined)`.
+- `TriggerQueue` and `batchContexts` are process-local maps. A restart loses all
+  queued prompts and pending debounce batches. Queue draining removes an item
+  before run setup is durable, so a setup error also loses it.
+
+There is no shutdown hook that flushes or rejects these buffers visibly. The
+service can therefore acknowledge and forget user work even when PostgreSQL is
+healthy again milliseconds later.
+
+### 2. Run state and lock ownership can contradict each other
+
+Run creation, lock acquisition, terminal update, and lock release are separate
+Prisma calls. Their failure handling is asymmetric:
+
+- a lock-acquisition race creates a session and an `AgentRun` first, then marks
+  the losing run failed;
+- terminal update failure is caught, a second best-effort failure update is
+  attempted, and `finally` releases the lock regardless;
+- lock release failure is only logged once, with no retry or reconciler.
+
+Fault injection reached both forbidden outcomes: `COMPLETED + lock` and
+`ACTIVE + no lock`. The first makes `currentLockRunId` throw the explicit
+ADR-0002 invariant error on all later triggers; the second permits another run
+to start while the database still claims the earlier run is live. In the
+injected terminal-write case the earlier execution had already returned, but
+the same unlocked/non-terminal shape created by an overlapping startup can
+coexist with a still-running process.
+
+Startup does not repair these states safely. `server.ts` first deletes **all**
+locks and then, in a separate statement, marks only `ACTIVE` runs failed.
+`WAITING_FOR_USER` is left non-terminal and unlocked, contrary to the contract.
+The two statements are not transactional, and running a second Hub process
+would destructively clear the live first process's locks and fail its runs.
+
+### 3. A provider hang can block a project indefinitely
+
+`maxTurns` is passed to the SDK, but there is no wall-clock run deadline. The
+SDK max-turn result is currently mapped to runner status `failed`, making the
+trigger's `length → TIMED_OUT` branch unreachable for the actual max-turn case.
+
+`ProjectAgentLock.expiresAt` is never populated or consumed. `heartbeatAt` is a
+best-effort update only after an assistant message, not a periodic heartbeat,
+and there is no stale-lock/run sweeper. A query that stalls before its first
+assistant message remains `ACTIVE` while holding the project lock forever.
+`WAITING_FOR_USER` is declared but never written; approval waits remain
+`ACTIVE` in an in-memory `ApprovalManager`.
+
+Systemd sends `SIGTERM`, but the application has no signal handler, listener
+close, in-flight drain, run cancellation, Prisma disconnect, or final recovery
+transaction. A subsequent successful restart happens to fail `ACTIVE` runs,
+but that is not a graceful lifecycle and does not cover all non-terminal
+states.
+
+### 4. Workspace effects have no defined commit or recovery boundary
+
+The Claude SDK's built-in Write/Bash tools mutate the project workspace while
+the run is still `ACTIVE`. A process kill, timeout, cancellation, database
+failure, or later delivery failure can therefore leave real filesystem changes
+behind while the database says the run failed or never completed.
+
+No runtime code writes `AgentFileChange`: the only capture callback is in the
+legacy AI-SDK `writeFileTool`, while the production runner uses Claude SDK
+built-ins and never injects that collector. The schema comment claiming these
+changes are captured for review is not true. The transcript comments are also
+stale: the production runner neither loads nor appends the JSONL transcript;
+continuity comes from the provider cursor.
+
+`AgentMessage` writes are best-effort and silently swallowed. The session
+cursor, message projection, workspace state, run terminal state, and audit
+history can consequently describe different versions of the same execution.
+Before implementing a mechanism, the product/contract must decide whether
+partial workspace changes are committed, quarantined, or rolled back.
+
+## High findings
+
+### 5. Session identity and cursor continuity are not concurrency-safe
+
+`startAgentRun` performs `findFirst` then `create` for the active
+project/provider/role/model session before acquiring the project lock. The
+schema has only a non-unique index for that tuple. The database probe inserted
+two active rows with the same identity.
+
+Concurrent triggers can therefore fork a logical conversation before one loses
+the project-lock race. Future `findFirst` selection is unspecified and may
+resume the wrong provider cursor. On the completion path, the session cursor is
+updated separately and before the run terminal update, so a partial database
+failure can advance the conversation while leaving the originating run active
+or failed.
+
+### 6. Outbound delivery is silent and not recoverable
+
+The Feishu send helpers catch message errors and return `null`; `sendText`
+returns `void`. `StreamingAgentCard.finish` treats a failed create/patch as a
+boolean and normal run completion proceeds. The fault probe recorded a
+`COMPLETED` run with zero delivered cards.
+
+There is no durable outbox, provider message ID record, delivery state,
+idempotency key, retry schedule, or reconciliation job. An operator cannot tell
+whether the user saw the result, and retrying a larger workflow later would
+have no durable key with which to prevent duplicate messages.
+
+### 7. Audit rows violate the pinned relation and audit failures are invisible
+
+`AuditEntry.runId` is nullable, has no foreign key, and accepts arbitrary
+strings. This directly diverges from the pinned `Spec.System.Audit` relation
+that every audit entry belongs to a run. Pre-run permission/security events are
+valid product needs, but they require an explicit event model rather than
+weakening the one relation the contract fixes.
+
+`writeAudit` catches every Prisma error without logging, metric, retry, or
+caller signal, and existing unit tests encode that swallowing as desired.
+Lifecycle history can be absent exactly when PostgreSQL is unhealthy, with no
+evidence that evidence was lost. The content and failure policy are `OPEN` and
+need a decision, but silent unobservable loss is incompatible with operating
+and debugging the service.
+
+## Failure-mode summary
+
+- **Process termination/restart:** loses batches, queued prompts, approvals,
+  active controller handles, and unrecorded message/file history; may leave
+  partial workspace changes; recovery ignores `WAITING_FOR_USER`.
+- **Timeout/hang:** no wall deadline, periodic heartbeat, lease, or sweeper;
+  max-turn termination is mislabeled `FAILED` rather than `TIMED_OUT`.
+- **Duplicate Feishu delivery:** sequential success is deduplicated, but any
+  failure after receipt becomes permanent loss; receipt check/create is also
+  not one transactional claim operation.
+- **Database failure:** can produce either terminal-with-lock or
+  active-without-lock, lose session/message/audit facts, or advance only the
+  provider cursor.
+- **Concurrent requests/processes:** the project-lock unique key prevents two
+  normal lock rows, but active sessions can fork; a second process's startup
+  reset can invalidate the first process's live lifecycle.
+- **Outbound failure:** run completion and user-visible delivery are unrelated,
+  and no durable recovery/idempotency record exists.
+
+## Required release evidence
+
+Production readiness requires automated proof of all of the following:
+
+1. Each accepted inbound event has a durable state machine and can be retried
+   after process/DB failure without duplicate run creation.
+2. Queueing and batching either survive restart or have an explicit, observable
+   rejection/acknowledgement contract that cannot silently lose work.
+3. Every database fault point preserves `lock exists ⇒ holder non-terminal`
+   and prevents a live non-terminal run from continuing without its lock.
+4. A wall deadline, lease/heartbeat, watchdog, and startup reconciler converge
+   every non-terminal state, including human approval waits.
+5. One active session identity is enforced in PostgreSQL and cursor advancement
+   is serialized with the run transition that produced it.
+6. Feishu delivery uses a durable, idempotent outbox/reconciliation path and
+   exposes delivered, pending, and failed delivery states separately from model
+   completion.
+7. The decided failed-run workspace policy is implemented and tested across
+   kill, cancel, timeout, and database failure; file/message history reflects
+   the real effects or explicitly reports missing evidence.
+8. The audit/event model conforms to the pinned run relation, records pre-run
+   security events in an explicitly modeled place, and makes write loss
+   observable and recoverable according to the accepted policy.
+9. Graceful shutdown and two-process/startup-race tests prove that one process
+   cannot clear another live process's locks or accept new work while draining.

.scratch/saas-production-readiness/assets/tenant-auth-security-audit.md

@@ -0,0 +1,262 @@
+# Tenant, authentication, and request security audit
+
+## Verdict
+
+The current Hub has a sound application-level foundation for ordinary org-admin
+CRUD isolation, but it is **not safe to deploy as a multi-tenant SaaS**. Several
+agent and asynchronous-action paths can cross the Organization/Project boundary,
+and the current process-global credential design directly contradicts the
+org-scoped encrypted-secret decision in ADR-0021.
+
+The answer to the audit question is therefore **yes**: current filesystem,
+Feishu-context, deferred authorization, card-action, and credential paths can
+cross or bypass the intended boundary. These are implementation divergences,
+not unspecified product behavior.
+
+## Contract baseline
+
+The audit used these upstream requirements:
+
+- `Spec.System.Organization`: PROJECT and TEAM each have one Organization;
+  TEAM→PROJECT grants are same-org only.
+- `Spec.System.ProjectWorkspace`: folders are transparent; folder/project
+  placement must remain inside one Organization.
+- `Spec.System.Permission`: agent trigger requires EDIT, normal cancel requires
+  MANAGE, and force release is outside the project role lattice.
+- `Spec.System.Memory`: a run may read Feishu context only from its project's
+  bound chat.
+- `Spec.System.AgentSurface`: every agent file operation must stay inside that
+  run's project workspace.
+- ADR-0019: authorization is evaluated against principals resolved at request
+  time so membership changes take effect immediately.
+- ADR-0021: Feishu and model-provider credentials are org-scoped secrets,
+  encrypted at rest, and accessed through a resolver. Provider ownership may be
+  Organization-managed BYOK or a distinct platform-managed connection for that
+  Organization; neither mode permits a process-global shared key.
+
+## Confirmed protections
+
+The following paths are correctly fail-closed today:
+
+- Every `/api/org/:orgSlug/*` route calls `requireOrgRole`; the guard reloads
+  the user, active membership, allowed OWNER/ADMIN role, and ACTIVE org for each
+  request.
+- Project/session/folder/team service functions compare the requested object's
+  Organization before returning or mutating it. Cross-org lookups return not
+  found instead of revealing the foreign object.
+- `PermissionAuthorizer` derives the Organization from the resource before
+  expanding team and external principals. `grantTeamProjectAccess` refuses a
+  TEAM whose Organization differs from the Project and detects corrupt active
+  grants when listing.
+- Feishu messages with no sender `open_id` are denied. Bound-chat triggers run
+  through `agent.trigger` and `role.trigger` gates before creating an AgentRun.
+- OAuth state is signed, expiry-checked, and paired with an HttpOnly nonce
+  cookie. `returnTo` is constrained to same-origin `/admin` paths. Session
+  signatures use HMAC-SHA256 with constant-time comparison, and session users
+  are reloaded on every request.
+- Fastify's current default body limit is 1 MiB, so HTTP bodies are not
+  completely unbounded even though field-level and rate limits are absent.
+
+Executed evidence:
+
+```text
+5 integration files passed
+28 tests passed
+```
+
+Those tests include unauthenticated access, member-vs-admin roles, cross-org
+project IDs, Organization-scoped team slugs, and cross-org TEAM grant refusal.
+
+## Critical findings
+
+### 1. The agent receives service secrets and can read other tenant data
+
+`hub/src/agent/runner.ts` builds the SDK environment as:
+
+```text
+{ ...process.env, ...providerEnv }
+```
+
+The sandboxed agent and its Bash subprocess therefore inherit `DATABASE_URL`,
+`FEISHU_APP_SECRET`, `HUB_SESSION_SECRET`, the model-provider token, and every
+other service variable. A model or prompt injection can run `env`; network
+egress is intentionally open, so exposed values can leave the host.
+
+The same sandbox config has only `allowWrite: [workspaceDir]`. Reads outside
+the workspace are allowed except for a small deny list. All organizations run
+as the same service user, and sibling project workspaces are not denied. This
+violates `AgentFileOp.Authorized`, which covers reads as well as writes.
+
+The installed Claude Agent SDK already exposes two mechanisms that are not
+used: sandbox credential rules can `deny` or proxy-`mask` environment variables,
+and filesystem rules can deny a workspace root while re-allowing the current
+workspace. Production still needs an actual Linux test proving that SDK/Bash
+cannot read another project or service credential.
+
+### 2. MCP file and Feishu-context tools escape their run objects
+
+`resolveDeliverableFile` intentionally adds the repository Git root to allowed
+roots, even when it is above the project workspace. It also checks lexical paths
+with `stat`, which follows symlinks. A deterministic reproduction created
+`workspace/leak.txt` as a symlink to another tenant's file:
+
+```json
+{"symlinkEscapeAccepted":true,"outsideContentReadable":true}
+```
+
+The checked-in unit test currently treats sending a repo-root `README.md` from
+a nested project as desired behavior. That expectation is a direct divergence
+from the AgentSurface contract and must be reversed.
+
+`readFeishuContext` ignores its `ToolContext`. The MCP wrapper closes over the
+bound `chatId`, but the model supplies an arbitrary message/thread ID and the
+reader never verifies the returned message's `chat_id`. A stub Feishu API
+returning a message from `chat-other` produced:
+
+```json
+{"returnedChatId":"chat-other","crossChatContentReturned":true}
+```
+
+The same validation is required for thread parents and every returned item.
+
+Inbound downloads have the inverse symlink problem: the unsandboxed Hub writes
+to `workspace/.cph/inbox`. A prior agent command can replace a path component
+with a symlink, turning the Hub into a confused deputy unless writes use a
+verified real path/no-follow boundary.
+
+### 3. SaaS credentials are process-global instead of org-scoped
+
+`server.ts` creates one process-global Feishu client/listener and passes one
+global app ID/secret to OAuth. `feishuOAuth.ts` explicitly calls per-org
+encrypted credentials "deferred", although ADR-0021 says they **must** be
+org-scoped, encrypted at rest, and accessed through a resolver.
+
+`EnvRuntimeSettings.provider` similarly discards its `scope` argument and uses
+one global OpenRouter token for every project. This erases the accepted
+per-Organization provider mode and makes both BYOK and platform-managed cost
+attribution false. The schema has no encrypted credential store, key version,
+rotation state, or resolver seam.
+
+This is also a product reachability blocker: one customer-owned Feishu app
+cannot receive bot/OAuth traffic for unrelated customer tenants under the
+ADR-0021 model.
+
+## High findings
+
+### 4. Delayed and card actions are not bound to the authorized object
+
+- `onMessage` checks `agent.trigger` before enqueueing/batching, but
+  `startAgentRun` checks only `role.trigger`. If a grant, team membership, or
+  tenant status changes while a request waits, it still starts. This contradicts
+  ADR-0019's request-time effectiveness rule.
+- Interrupt handling authorizes `agent.cancel` against the project bound to the
+  card action's current chat, then calls `activeRuns.get(runId)` without proving
+  that `runId` belongs to that project. A stale, forwarded, or otherwise
+  mismatched action can abort another project's active controller.
+- Approval handling stores the expected `chatId` but never compares it to the
+  action context before resolving the pending approval.
+
+Execution-time authorization must resolve one immutable tuple
+`(organization, project, run/action, actor, chat)` and reject any mismatch
+before mutation.
+
+### 5. SUSPENDED organizations can still execute bound projects
+
+HTTP org-admin guards and unbound-chat onboarding reject non-ACTIVE orgs, but
+`PermissionAuthorizer.organizationForResource` returns only `organizationId`
+and never checks status. A real database reproduction created a SUSPENDED org,
+project, and EDIT grant:
+
+```json
+{"organizationStatus":"SUSPENDED","agentTriggerAllowed":true}
+```
+
+Project onboarding helpers also check membership existence without checking
+Organization status. The same status must be enforced at the shared resource
+authorization/onboarding seam, not independently in selected transports.
+
+### 6. Production dependency graphs contain known vulnerabilities
+
+`npm audit --omit=dev` exits 1 with two high findings:
+
+- `@larksuiteoapi/node-sdk@1.68.0`
+- transitive `axios@1.13.6`
+
+The current latest Feishu SDK (`1.70.0` at audit time) still pins Axios
+`~1.13.3`, while the audited fixed Axios line is newer. The relevant advisories
+include request/credential manipulation, proxy credential leakage, ReDoS, and
+unbounded resource allocation; for example
+[GHSA-3g43-6gmg-66jw](https://github.com/advisories/GHSA-3g43-6gmg-66jw),
+[GHSA-hfxv-24rg-xrqf](https://github.com/advisories/GHSA-hfxv-24rg-xrqf), and
+[GHSA-777c-7fjr-54vf](https://github.com/advisories/GHSA-777c-7fjr-54vf).
+An override is outside the SDK's declared semver range and therefore requires
+integration testing rather than a blind lockfile edit.
+
+`cargo audit` exits 1 with five vulnerability records:
+
+- `crossbeam-epoch@0.9.18`, patched in `>=0.9.20`;
+- `quick-xml@0.38.4` and `0.39.4`, each affected by namespace-allocation and
+  quadratic-attribute DoS advisories, patched in `>=0.41.0`.
+
+It also reports `memmap2@0.9.10` as unsound (patched in `>=0.9.11`) and four
+unmaintained transitive crates. All enter through the Typst 0.15 dependency
+graph. References:
+[RUSTSEC-2026-0204](https://rustsec.org/advisories/RUSTSEC-2026-0204.html),
+[RUSTSEC-2026-0194](https://rustsec.org/advisories/RUSTSEC-2026-0194.html),
+[RUSTSEC-2026-0195](https://rustsec.org/advisories/RUSTSEC-2026-0195.html), and
+[RUSTSEC-2026-0186](https://rustsec.org/advisories/RUSTSEC-2026-0186.html).
+
+No `npm audit` or `cargo audit` gate exists in CI.
+
+## Browser/session and availability gaps
+
+These do not currently prove a cross-org data read, but they are required before
+exposing the service publicly:
+
+- state-changing cookie-auth routes have no CSRF token or Origin/Sec-Fetch-Site
+  enforcement; SameSite=Lax is the only browser-side barrier;
+- there is no session revocation/version store, so a stolen cookie remains valid
+  for up to seven days unless the global secret rotates;
+- startup accepts any non-empty session secret rather than enforcing generated
+  entropy, and there is no key rotation scheme;
+- no security-header policy is registered, and Fastify request timeout is zero;
+- OAuth, login, and org APIs have no per-IP/user/tenant rate limit;
+- Feishu file/post download count, response size, MIME, archive shape, and total
+  workspace quota are unbounded; uploads read entire files into memory.
+
+The exact abuse quotas and file limits remain with
+`Define initial abuse and capacity controls`; they must not be guessed here.
+
+## Secret scan
+
+All 377 tracked files were scanned for private-key blocks, common cloud/API-key
+shapes, and non-empty assignments to the Hub's principal secret variables. No
+private key or real credential was found. The only assignment hits were the
+documented `<OpenRouter API key>` placeholder and local/test PostgreSQL URLs.
+`hub/.env` is ignored and untracked.
+
+This is positive repository hygiene, but it does not mitigate the runtime
+environment leak into the agent process.
+
+## Required security release evidence
+
+Production readiness requires automated proof of all of the following:
+
+1. An agent/Bash process cannot read service credentials, another project
+   workspace, repo-root files, or a symlink target outside its workspace.
+2. Feishu context and file delivery validate the real target after resolution,
+   not only a caller-supplied lexical ID/path.
+3. Every delayed action reauthorizes at execution and binds actor, chat,
+   organization, project, run, and pending action consistently.
+4. ACTIVE Organization status is enforced in the shared authorization seam.
+5. Feishu and provider credentials resolve by Organization, are encrypted at
+   rest with versioned keys, never enter logs/agent tools, and can rotate;
+   provider resolution honors either Organization-managed BYOK or a distinct
+   platform-managed connection for that Organization.
+6. Cross-org HTTP and Prisma service tests remain green, while new negative
+   tests cover every finding above.
+7. Production Node and Rust dependency audits pass with no ignored
+   vulnerability/unsound advisory unless a time-bounded, reviewed exception is
+   explicitly recorded.
+8. Browser/session, request, and file-ingestion limits are verified against the
+   accepted production threat and capacity policy.

.scratch/saas-production-readiness/issues/01-audit-clean-host-deployment.md

@@ -0,0 +1,33 @@
+# Audit the clean-host deployment and rollback contract
+
+Type: research
+Status: resolved
+
+## Question
+
+What concrete defects prevent a clean, supported Linux host from installing,
+starting, health-checking, upgrading, and rolling back the current Hub without
+manual repository knowledge or unsafe partial deployment?
+
+## Answer
+
+The current path is a happy-path updater for a manually prepared host, not a
+production deployment mechanism. The executed audit found an unrunnable seeded
+configuration, an unprovisioned service identity and storage, a missing `cph`
+artifact, in-place mutation before build success, liveness-only verification,
+no coordinated application/schema rollback, and a configured bind host that the
+server ignores.
+
+The complete evidence, operational consequences, required release invariants,
+and verification scenarios are recorded in the
+[clean-host deployment and rollback audit](../assets/clean-host-deployment-audit.md).
+
+The accepted single-host topology remains suitable. The route forward is now
+explicit in these implementation tickets:
+
+- [Provision a runnable non-root Hub service](10-provision-nonroot-service.md)
+- [Package and verify cph with every Hub release](11-package-cph-with-release.md)
+- [Make application releases atomic and rollbackable](12-atomic-releases-and-rollback.md)
+- [Add real readiness and an accurate service lifecycle](13-real-readiness-and-lifecycle.md)
+- [Establish migration compatibility and failed-migration recovery](14-migration-compatibility-and-recovery.md)
+- [Test clean-host deployment and rollback on Linux](15-test-clean-host-deployments-on-linux.md)

.scratch/saas-production-readiness/issues/02-audit-tenant-auth-security.md

@@ -0,0 +1,40 @@
+# Audit tenant, authentication, and request security boundaries
+
+Type: research
+Status: resolved
+
+## Question
+
+Can any current HTTP, Feishu, filesystem, agent, or database path cross an
+Organization boundary, bypass the intended platform/org/project authorization
+seams, expose secrets, or accept unbounded hostile input in the initial
+production topology?
+
+## Answer
+
+Yes. Basic org-admin HTTP/Prisma scoping is correctly enforced and its 28
+targeted integration tests pass, but the agent inherits service secrets and can
+read sibling files, MCP file/context tools accept real targets outside the run
+objects, queued/card actions do not consistently bind or reauthorize their
+target, non-ACTIVE tenants can still trigger agents, and Feishu/provider
+credentials remain global instead of ADR-0021 org-scoped encrypted secrets.
+Current Node and Rust dependency audits also fail on high/vulnerability/unsound
+advisories.
+
+The complete contract mapping, reproductions, positive evidence, advisory
+inventory, and required release proofs are in the
+[tenant, authentication, and request security audit](../assets/tenant-auth-security-audit.md).
+
+The newly-clear implementation and decision frontier is:
+
+- [Confine the agent runtime and protect service credentials](16-confine-agent-and-protect-credentials.md)
+- [Close MCP context and file-delivery escape paths](17-close-mcp-data-egress-escapes.md)
+- [Bind and reauthorize deferred and card actions](18-bind-and-reauthorize-deferred-actions.md)
+- [Design the org-scoped secret and connection control plane](19-design-org-scoped-secret-control-plane.md)
+- [Design the production browser and session boundary](20-design-production-web-session-boundary.md)
+- [Clear dependency advisories and gate releases](21-clear-dependency-security-advisories.md)
+- [Enforce Organization status at the shared authorization seam](22-enforce-organization-status.md)
+
+Inbound Feishu file count/size/MIME and request-rate policy remain owned by
+[Define initial abuse and capacity controls](05-define-abuse-capacity-controls.md)
+rather than being guessed in this audit.

.scratch/saas-production-readiness/issues/03-audit-run-lifecycle-integrity.md

@@ -0,0 +1,38 @@
+# Audit run lifecycle and data integrity under failure
+
+Type: research
+Status: resolved
+
+## Question
+
+Under process termination, restart, timeout, duplicate Feishu delivery,
+database failure, or concurrent requests, which AgentRun, lock, session,
+workspace, audit, and outbound-message invariants can be violated or leave work
+lost, duplicated, or permanently stuck?
+
+## Answer
+
+The happy path and database lock uniqueness work, but the lifecycle is not
+crash-safe. Deterministic fault injection confirmed permanent inbound loss,
+terminal runs retaining locks, active runs losing locks, duplicate active
+sessions, completion without outbound delivery, unconstrained audit run IDs,
+and max-turn termination being stored as failure. Process-local queues,
+non-transactional startup recovery, unused leases, silent audit/history writes,
+and direct workspace mutation add restart and indefinite-hang failure modes.
+
+The complete contract mapping, eight fault probes, recovery analysis, and
+required release evidence are in the
+[run lifecycle and data-integrity audit](../assets/run-lifecycle-integrity-audit.md).
+
+The newly-clear implementation and decision frontier is:
+
+- [Make Feishu intake and queued work durable](23-durable-feishu-inbox-and-queue.md)
+- [Make run, session, lock, and recovery transitions safe](24-transactional-run-lock-recovery.md)
+- [Make outbound Feishu delivery durable and idempotent](25-durable-outbound-delivery.md)
+- [Decide workspace commit semantics for failed runs](26-decide-workspace-run-commit-semantics.md)
+- [Reconcile the audit and run-history contract](27-reconcile-audit-history-contract.md)
+
+Execution-time authorization remains owned by
+[Bind and reauthorize deferred and card actions](18-bind-and-reauthorize-deferred-actions.md),
+while cross-component metrics, alerts, and operator runbooks remain owned by
+[Define production observability and incident recovery](04-define-observability-recovery.md).

.scratch/saas-production-readiness/issues/04-define-observability-recovery.md

@@ -0,0 +1,47 @@
+# Define production observability and incident recovery
+
+Type: research
+Status: resolved
+Blocked by: 01, 02, 03
+
+## Question
+
+Which logs, metrics, readiness signals, audit correlations, alerts, and operator
+recovery actions are missing for a production operator to detect and diagnose
+failures across HTTP, Feishu, PostgreSQL, agent execution, cph builds, and
+workspace storage?
+
+## Answer
+
+The Hub has useful fragments—Fastify access logs, Prisma error/warn output,
+durable AgentRun facts, selected Feishu warnings, explicit migrations, and a
+systemd journal—but no joined operational contract. It has no metrics or alert
+surface, no end-to-end request/event/run/delivery/release correlation, no
+checked-in recovery runbooks, and many critical writes or sends are silently
+best-effort.
+
+Three process-level fault probes confirmed the release gate is actively
+misleading: cutting every PostgreSQL socket left `/api/healthz` at 200 twice;
+an SDK-rejected Feishu app ID also left health at 200; and a runtime database
+outage made `/api/me` return a raw Prisma invocation as `400 bad_request` with
+no request-scoped exception log. The current deploy check can therefore accept
+an instance unable to serve either database or Feishu work, while dependency
+failures evade normal 5xx alerting.
+
+The full failure-to-signal matrix, liveness/readiness/startup contract,
+correlation envelope, metric and alert inventory, required runbooks, executed
+probe evidence, primary sources, and minimum release evidence are in the
+[production observability and incident-recovery audit](../assets/production-observability-recovery.md).
+
+The newly-clear implementation and decision frontier is:
+
+- [Make HTTP dependency failures typed and observable](28-typed-observable-http-errors.md)
+- [Decide the telemetry, SLO, retention, and alerting control plane](29-decide-telemetry-slos-alert-routing.md)
+- [Instrument correlated production runtime telemetry](30-instrument-correlated-runtime-telemetry.md)
+- [Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md)
+
+[Add real readiness and an accurate service lifecycle](13-real-readiness-and-lifecycle.md)
+owns component probes, Feishu connection state and deploy gating. Durable
+ingress/run/outbound recovery stays with tickets 23–25, audit semantics with
+ticket 27, and backup/restore guarantees with ticket 06; this audit does not
+duplicate those implementations.

.scratch/saas-production-readiness/issues/05-define-abuse-capacity-controls.md

@@ -0,0 +1,43 @@
+# Define initial abuse and capacity controls
+
+Type: grilling
+Status: resolved
+Blocked by: 02, 03
+
+## Question
+
+Which request limits, file limits, concurrency limits, agent budgets, tenant
+quotas, and backpressure policies are mandatory for the accepted initial
+production boundary, and which values or behaviors remain product decisions?
+
+## Answer
+
+The initial production service requires layered, fail-closed capacity controls:
+every mandatory dimension has a versioned non-overridable platform ceiling,
+while an Organization may configure only a lower policy limit. Agent work uses
+a durable bounded queue, fair scheduling across Organizations, FIFO within an
+Organization, explicit overload/expiry/cancellation states, and no automatic
+whole-run replay after execution starts.
+
+Hard controls cover request rate/body size, concurrency and queueing, files and
+archive expansion, Project/Organization storage and entity counts, run/tool
+budgets, and process resources. Reaching a limit blocks new growth without
+deleting or hiding existing data. Token and provider cost remain attributed
+soft-alert signals in the pilot, not hard spend controls; unknown cost is not
+zero. Platform administrators also receive audited `DRAIN` and `STOP_NOW`
+workload brakes.
+
+The full current-state inventory, accepted behavior, and release evidence are
+recorded in [Initial abuse and capacity controls](../assets/initial-abuse-capacity-controls.md),
+with the durable decision in ADR-0022 and `Spec.System.Capacity`. Numerical
+ceilings remain open until production-like calibration.
+
+The implementation frontier is:
+
+- [Implement layered limits and multi-dimensional request rate limiting](32-layered-limits-rate-limiting.md)
+- [Build the durable Organization-fair admission scheduler](33-durable-fair-admission-scheduler.md)
+- [Enforce file, archive, storage, and entity budgets](34-file-storage-budgets.md)
+- [Enforce Agent Run and process resource budgets](35-run-runtime-budgets.md)
+- [Implement provider-mode usage attribution and soft alerts](36-usage-soft-alerts.md)
+- [Add audited emergency workload brakes](37-emergency-workload-brakes.md)
+- [Calibrate and prove production capacity ceilings](38-calibrate-production-capacity.md)

.scratch/saas-production-readiness/issues/06-audit-backup-migration-recovery.md

@@ -0,0 +1,48 @@
+# Audit backup, migration, and disaster recovery safety
+
+Type: research
+Status: resolved
+
+## Question
+
+What backup, restore, workspace/database consistency, migration compatibility,
+and disaster-recovery guarantees are needed to deploy current persistent data
+safely, and what evidence or tooling is missing to prove them?
+
+## Answer
+
+The Hub has no production backup/restore control plane. PostgreSQL can be
+logically dumped and restored, but authoritative state is split across the
+database, mutable Project workspaces, credentials/key material, and a
+schema-compatible release with no shared checkpoint or recovery manifest.
+RPO/RTO, retention, off-host/immutable storage, encryption/key custody,
+full-service versus Organization-selective restore, monitoring, and recovery
+drills are all undefined.
+
+Executed probes proved that the current deploy `rsync --delete` removes the
+documented in-tree workspace root, a failed Project transaction leaves an
+orphan workspace, and an older application query fails after a destructive
+migration. A deliberately failed Prisma migration rolled back that probe's DDL
+but left a blocking failed migration record; the official contract does not
+make PostgreSQL migrations transactional by default. A positive custom-format
+round trip restored 24 tables and all nine current migration records, proving
+only database-level portability on the local PostgreSQL 14.20 setup.
+
+The complete state map, probe commands/results, root causes, required recovery
+contract, and release evidence are in the
+[backup, migration, and disaster-recovery audit](../assets/backup-migration-disaster-recovery-audit.md).
+Official PostgreSQL, Prisma, systemd, and rsync semantics with claim-level links
+are in [backup and migration primary sources](../assets/backup-migration-primary-sources.md).
+
+The newly-clear decision and implementation frontier is:
+
+- [Decide recovery objectives, backup retention, and restore scope](39-decide-recovery-objectives.md)
+- [Implement coherent backup sets and verified restore](40-verified-backup-restore.md)
+- [Make the Project workspace lifecycle recoverable](41-recoverable-project-workspace-lifecycle.md)
+- [Establish migration compatibility and failed-migration recovery](14-migration-compatibility-and-recovery.md)
+
+Persistent-path provisioning and release-tree isolation remain owned by
+[Provision a runnable non-root Hub service](10-provision-nonroot-service.md)
+and [Make application releases atomic and rollbackable](12-atomic-releases-and-rollback.md).
+Full recovery/cutover procedures and drill evidence remain owned by
+[Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md).

.scratch/saas-production-readiness/issues/07-audit-product-surface-completeness.md

@@ -0,0 +1,58 @@
+# Audit the accepted product surface for production completeness
+
+Type: research
+Status: resolved
+
+## Question
+
+Against ADR-0020 and ADR-0021, which required platform-admin, org-admin,
+project-onboarding, team/access, session, usage, provider-mode, Organization
+policy-limit, emergency workload-control, and Feishu flows are absent,
+unreachable, or only test-facing rather than usable in the initial production
+service? Treat ADR-0022's capacity control surfaces as part of the accepted
+product boundary.
+
+## Answer
+
+The current Hub is not a usable or production-complete SaaS product surface.
+Its Organization-scoped JSON APIs and single-Feishu-app Agent Run path are
+worth retaining, but the running service has one process-global Feishu
+app/bot, one global provider key/base URL, no Organization connection/secret
+plane, no platform-administrator control plane, and no real Organization or
+platform management UI. OAuth redirects a successful login to an unmounted
+`/admin/org/:orgSlug` route.
+
+Two real Fastify integration probes, each run twice, proved that an authenticated
+Organization ADMIN receives 404 at the post-login destination and two distinct
+Organization return paths produce the same OAuth `client_id`. Official Feishu
+documentation also proves that `open_id` is application-scoped, `user_id` is
+tenant-scoped, and `union_id` only links applications from the same application
+provider. The current globally unique `User.feishuOpenId`, raw `chatId`, event
+identity, OAuth state and one-listener composition therefore cannot safely
+route independently customer-owned applications.
+
+ADR-0022 remains an additional release blocker: beyond a process-memory
+Project queue and global max-turn setting, layered limits, durable
+Organization-fair admission, hard storage/run/process budgets, alerts and
+audited emergency brakes have no complete persistence or product surface.
+
+The full verdict, deterministic probe results, root causes, retained seams and
+required end-to-end release evidence are in the
+[accepted SaaS product-surface completeness audit](../assets/product-surface-completeness-audit.md).
+The detailed route/schema/test matrix is in the
+[accepted product-surface code inventory](../assets/product-surface-code-inventory.md),
+and the Feishu identifier/OAuth/event/long-connection facts are in the
+[Feishu product-surface primary sources](../assets/product-surface-feishu-primary-sources.md).
+
+The newly-clear execution frontier is:
+
+- [Decide the platform-administrator identity and audit boundary](42-decide-platform-admin-identity-audit.md)
+- [Implement the Organization connection and secret plane](43-implement-org-connection-secret-plane.md)
+- [Route Feishu identity and traffic by Organization connection](44-route-feishu-by-org-connection.md)
+- [Build the private platform-administrator control plane](45-build-private-platform-admin-control-plane.md)
+- [Build the private Organization-administrator control plane](46-build-private-org-admin-control-plane.md)
+
+Existing tickets 23 and 32–38 remain the owners of durable ingress/admission,
+capacity enforcement, usage alerts, workload brakes and calibration. Ticket 08
+now requires all of these product slices before claiming the production release
+gate.

.scratch/saas-production-readiness/issues/08-prove-production-release-gate.md

@@ -0,0 +1,12 @@
+# Prove the production release gate
+
+Type: task
+Status: open
+Blocked by: 01, 02, 03, 04, 05, 06, 07, 09, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46
+
+## Question
+
+After the readiness investigations and resulting fixes are resolved, can one
+repeatable release procedure prove build/test/spec health, deploy a clean
+production-like environment, exercise critical tenant and agent journeys,
+verify observability and recovery, and either roll forward or roll back safely?

.scratch/saas-production-readiness/issues/09-restore-checker-format-gate.md

@@ -0,0 +1,40 @@
+# Restore the checker formatting gate
+
+Type: task
+Status: resolved
+
+## Question
+
+Apply the repository's configured rustfmt output to the currently reported
+Rust files, verify that no semantic changes are introduced, and restore
+`cargo fmt --all --check` so the checker CI gate is green.
+
+## Answer
+
+The exact CI command failed deterministically on seven Rust files and an
+isolated single-file rustfmt check reproduced the same output. The source
+changes were formatting-only, but the underlying gate was not reproducible:
+the repository had no `rust-toolchain.toml` and checker CI installed floating
+`stable`, so local rustfmt, clippy and CI could change independently.
+
+The checker toolchain is now pinned to Rust 1.92.0 with `rustfmt` and `clippy`
+in both `rust-toolchain.toml` and the Gitea workflow. The seven files were
+formatted with that toolchain. Running the complete CI path then surfaced three
+test helpers accepting `&PathBuf` instead of `&Path` and a Clippy name collision
+between `Shell` and `PowerShell`; those were corrected without changing CLI
+values or behavior.
+
+Verification:
+
+```text
+cargo fmt --all --check                         PASS (twice)
+cargo clippy --all-targets --all-features -- -D warnings
+                                                  PASS
+cargo test --all-features --workspace            PASS (53 tests)
+cph completions zsh / power-shell                 PASS
+```
+
+The final Rust diff consists of rustfmt layout plus four borrowed-Path
+signature generalizations and the internal `Shell` → `CompletionTarget` type
+rename. No expression, string value, command behavior, or public Rust API was
+changed.

.scratch/saas-production-readiness/issues/10-provision-nonroot-service.md

@@ -0,0 +1,40 @@
+# Provision a runnable non-root Hub service
+
+Type: task
+Status: resolved
+
+## Question
+
+Make clean-host installation create or validate the dedicated service identity,
+home/cache/state/workspace paths and ownership, generate a complete production
+configuration, honor the configured bind host, and fail preflight before unit
+installation when any required value or prerequisite is invalid. The persistent
+workspace root must be outside all release trees, and preflight must reject any
+path that overlaps rsync, release switching, rollback, or release pruning.
+
+## Answer
+
+The installer now creates or strictly validates a dedicated non-root,
+non-login service identity with no supplementary groups; provisions its
+home/state/cache/workspace directories at mode `0750`; and refuses persistent
+paths that overlap the deployment tree after both lexical and physical path
+resolution. A missing production environment file is seeded completely and
+exits with status 78; invalid configuration or prerequisites fail before the
+unit is written or enabled.
+
+Preflight validates the configured bind host/port, authenticated PostgreSQL
+access, Node/cph/Prisma, configured and canonical path traversal, and a real
+bubblewrap namespace. The post-provision phase runs the built Hub dependency
+graph, a database query, cph, socat and bubblewrap as the service user under
+`no_new_privs`, matching the systemd security context. HTTP binding is awaited
+before the Feishu listener starts, so bind failure rejects startup instead of
+leaving a listener-only process alive.
+
+Verification evidence:
+
+- `npm run check`, `npm run build`, and `npm run prisma:validate`
+- 166 unit tests and 86 integration tests passed; one external-model test skipped
+- `bash -n deploy/install_service.sh` and `shellcheck deploy/install_service.sh`
+- `prisma migrate deploy`: 9 migrations found, none pending
+- production-mode HTTP smoke test on `127.0.0.1:8878` returned `/api/healthz`
+- final Standards and Spec reviews found no remaining Issue 10 gap

.scratch/saas-production-readiness/issues/11-package-cph-with-release.md

@@ -0,0 +1,10 @@
+# Package and verify cph with every Hub release
+
+Type: task
+Status: open
+
+## Question
+
+Deliver the compatible `cph` executable as part of the same immutable release
+as the Hub, configure its exact path, and make deployment preflight reject a
+missing, non-executable, or version-incompatible binary before restart.

.scratch/saas-production-readiness/issues/12-atomic-releases-and-rollback.md

@@ -0,0 +1,13 @@
+# Make application releases atomic and rollbackable
+
+Type: task
+Status: open
+Blocked by: 10, 11
+
+## Question
+
+Replace in-place rsync/build with deployment locking, immutable staged release
+directories, a revision manifest, atomic current/previous switching, bounded
+retention, and an explicit application rollback that preserves the live release
+on build, preflight, switch, or readiness failure. Prove deploy, rollback, and
+release pruning cannot traverse or delete the persistent workspace root.

.scratch/saas-production-readiness/issues/13-real-readiness-and-lifecycle.md

@@ -0,0 +1,12 @@
+# Add real readiness and an accurate service lifecycle
+
+Type: task
+Status: open
+Blocked by: 03, 04, 10, 11, 12, 23, 24, 25
+
+## Question
+
+Separate liveness from readiness, verify every dependency needed to accept Hub
+work, make deployment wait on readiness, and align systemd shutdown/restart
+claims with an implemented and observable Fastify, Feishu, Prisma, and in-flight
+run lifecycle.

.scratch/saas-production-readiness/issues/14-migration-compatibility-and-recovery.md

@@ -0,0 +1,15 @@
+# Establish migration compatibility and failed-migration recovery
+
+Type: task
+Status: open
+Blocked by: 06
+
+## Question
+
+Define and enforce the application/schema compatibility rule for each release,
+require expand-and-contract sequencing for destructive changes, add migration
+status/checksum and production-like rehearsal gates, and move schema mutation
+out of ordinary service start into an explicit release phase. Require a recent
+verified recovery checkpoint, bounded lock/runtime evidence, and an operator
+procedure for resolving or restoring a failed Prisma migration without
+pretending that an application rollback reverses the database.

.scratch/saas-production-readiness/issues/15-test-clean-host-deployments-on-linux.md

@@ -0,0 +1,12 @@
+# Test clean-host deployment and rollback on Linux
+
+Type: task
+Status: open
+Blocked by: 10, 11, 12, 13, 14
+
+## Question
+
+Add an automated Linux production-like harness that executes the real install,
+deploy and rollback paths against PostgreSQL and proves first install,
+idempotence, failure atomicity, readiness failures, application rollback, and
+deployment serialization.

.scratch/saas-production-readiness/issues/16-confine-agent-and-protect-credentials.md

@@ -0,0 +1,38 @@
+# Confine the agent runtime and protect service credentials
+
+Type: task
+Status: resolved
+
+## Question
+
+Make the real Claude SDK/Bash execution surface uphold `AgentFileOp.Authorized`:
+deny sibling tenant workspaces and service files, expose only the minimum safe
+process environment, use SDK credential protection for secrets, and add an
+actual Linux sandbox test proving another workspace and every non-provider
+credential are unreadable while required tools still work.
+
+## Answer
+
+The Agent subprocess now receives a replacement environment containing only
+named runtime values and the selected provider protocol values; PostgreSQL,
+Feishu and Hub session credentials never cross the process boundary. Provider
+credentials are additionally denied inside Bash. The SDK must sandbox every
+command (`failIfUnavailable=true`, `allowUnsandboxedCommands=false`), denies
+host reads and writes from root, and re-opens only the canonical project
+workspace plus the reviewed read-only system runtime needed by `cph`.
+
+SDK config/cache/home and general temp storage live inside the workspace. A
+short relative `CLAUDE_CODE_TMPDIR` prevents the pinned SDK from falling back
+to host `/tmp`, while absolute workspace-local `TMPDIR`/`TMP`/`TEMP` remain
+stable after Bash changes directory. Linux production and CI now require both
+bubblewrap and `socat`; preflight probes both as the service user. Bounded,
+credential-redacted SDK stderr is logged with run and project correlation.
+
+The real Linux proof runs the actual Claude SDK, Bash, bubblewrap, `socat` and
+release `cph` as UID 1000 with no capabilities and `no_new_privs`. It proves
+parent, sibling-project, `/tmp` and `/var/tmp` writes are rejected and have no
+host effect; sibling/service-secret reads fail; platform and provider secrets
+are absent from Bash; workspace temp remains anchored after `cd`; and `cph`
+still executes. Final local gates passed: TypeScript check/build, Prisma schema,
+173 unit tests, 85 mock-provider integration tests, deployment shell syntax,
+and the dedicated Linux sandbox test.

.scratch/saas-production-readiness/issues/17-close-mcp-data-egress-escapes.md

@@ -0,0 +1,28 @@
+# Close MCP context and file-delivery escape paths
+
+Type: task
+Status: resolved
+
+## Question
+
+Restrict file delivery to a realpath/no-follow target inside the current project
+workspace, remove Git-root access, make inbound Hub writes symlink-safe, and
+validate every Feishu message/thread result against the run's bound chat before
+returning data to the model.
+
+## Answer
+
+All Agent-facing workspace file reads and writes are anchored to the configured
+workspace root and current project. Linux uses descriptor-relative, no-follow
+traversal so a symlink swap cannot escape after validation; unsupported unsafe
+paths fail closed. Inbound Feishu files use the same boundary. Outbound file
+delivery no longer accepts the repository Git root or arbitrary absolute
+paths: it reads a no-follow snapshot from the current project and uploads that
+snapshot, so the checked object cannot be swapped before delivery.
+
+The Feishu MCP read surface carries the run's project and bound-chat context.
+Message and thread results are rejected unless their returned chat matches that
+binding, and file-delivery MCP calls remain scoped to the same run/project/chat
+tuple. Unit and integration regressions cover symlink paths, sibling projects,
+Git-root delivery attempts, swapped files, and cross-chat results; they are
+included in the final 173-unit/85-integration green gate reported by Issue 16.

.scratch/saas-production-readiness/issues/18-bind-and-reauthorize-deferred-actions.md

@@ -0,0 +1,12 @@
+# Bind and reauthorize deferred and card actions
+
+Type: task
+Status: open
+Blocked by: 44
+
+## Question
+
+At execution time, reauthorize queued/batched triggers and validate one
+actor/chat/organization/project/run-or-pending-action tuple for interrupt,
+approval, and onboarding card actions so revoked permissions and mismatched
+object IDs cannot affect another project.

.scratch/saas-production-readiness/issues/19-design-org-scoped-secret-control-plane.md

@@ -0,0 +1,24 @@
+# Design the org-scoped secret and connection control plane
+
+Type: grilling
+Status: resolved
+
+## Question
+
+Which key-management, envelope-encryption, rotation, connection identity, and
+runtime resolver design will implement ADR-0021's org-scoped Feishu and model
+provider credentials; support one customer-owned Feishu app per Organization;
+support Organization-managed BYOK and a distinct platform-managed provider
+connection per Organization; and ensure plaintext credentials never reach
+business records, logs, or agent tools?
+
+## Answer
+
+Use the local master-key envelope, immutable connection-secret versions,
+writer-authority split, explicit Organization/Project resolver, staged KEK
+rotation, and separately protected keyring recovery contract accepted in
+ADR-0024. Production receives the root-owned keyring only through systemd
+`LoadCredential`; it has no process-global credential fallback. Agent runs use
+a loopback proxy capability rather than the Organization Provider credential;
+the offline rotation command atomically retains old/new KEKs, rewraps stored
+DEKs, audits the changes, and authenticates the complete envelope set.

.scratch/saas-production-readiness/issues/20-design-production-web-session-boundary.md

@@ -0,0 +1,14 @@
+# Design the production browser and session boundary
+
+Type: grilling
+Status: open
+
+## Question
+
+What same-origin/CSRF, proxy/TLS, security-header, session lifetime,
+revocation, secret rotation, and request-timeout contract should the initial
+org-admin web surface enforce, and which parts require server-side session state
+rather than the current seven-day stateless cookie? Apply the same browser
+boundary to ADR-0023's separate server-side Platform Session while preserving
+its distinct cookie/token namespace, immediate revocation, per-request grant
+reload, and bounded recent-authentication requirement.

.scratch/saas-production-readiness/issues/21-clear-dependency-security-advisories.md

@@ -0,0 +1,44 @@
+# Clear dependency advisories and gate releases
+
+Type: task
+Status: resolved
+
+## Question
+
+Remove every current `npm audit --omit=dev` high finding and every `cargo audit`
+vulnerability/unsound finding through tested upgrades, compatible overrides or
+upstream fixes—not ignores—and add reproducible Node and Rust security-audit
+gates to CI and the production release evidence.
+
+## Answer
+
+All current production-high Node advisories and all Rust vulnerability/unsound
+advisories are removed without advisory ignores:
+
+- `@larksuiteoapi/node-sdk` 1.70.0 declares Axios `~1.13.3`, which prevented npm
+  from selecting a patched release. The Hub now has a targeted npm override to
+  Axios 1.18.1. The direct SDK stays current; the override is locked and the
+  full Hub type/build/test gate validates the resolved tree. A deterministic
+  local-HTTP smoke also drives the real SDK/Axios stack through tenant-token,
+  authenticated success and HTTP-error paths without external credentials.
+- `memmap2` is upgraded to 0.9.11 and `crossbeam-epoch` to 0.9.20.
+- `plist` is upgraded to 1.10.0, which moves its XML parser to `quick-xml`
+  0.41.0.
+- crates.io `citationberg` 0.7.0 still pins vulnerable `quick-xml` 0.38.4.
+  Cargo now patches it to the exact upstream citationberg commit that adopts
+  `quick-xml` 0.41.0; the git revision is immutable in both manifest and lock.
+
+`hub-check` runs the checked-in `npm run audit:production` command immediately
+after `npm ci`. `checker-check` installs the pinned cargo-audit 0.22.2 release
+with `--locked`, then runs `cargo audit --deny unsound`; vulnerabilities fail by
+default and unsound warnings are promoted to failures.
+
+Current evidence:
+
+- `npm audit --omit=dev --audit-level=high`: 0 vulnerabilities.
+- `cargo audit --deny unsound`: 0 vulnerabilities and 0 unsound warnings.
+- Four Rust `unmaintained` informational warnings remain visible and are not
+  ignored; they are outside this issue's vulnerability/unsound acceptance
+  criterion and will continue to appear in CI output.
+- Hub: type-check, build and 280 tests pass (10 environment-specific skips).
+- Rust: fmt, clippy with warnings denied, and all 53 workspace tests pass.

.scratch/saas-production-readiness/issues/22-enforce-organization-status.md

@@ -0,0 +1,50 @@
+# Enforce Organization status at the shared authorization seam
+
+Type: task
+Status: resolved
+
+## Question
+
+Make non-ACTIVE Organizations fail closed for project authorization,
+onboarding, queued execution, and Feishu triggers through one shared seam, with
+negative tests proving a SUSPENDED or ARCHIVED tenant cannot create, bind,
+trigger, resume, or mutate work while operator recovery remains explicit.
+
+## Answer
+
+Organization lifecycle is now enforced at both authorization and mutation
+linearization points:
+
+- Project and project-group authorization resolves the owning Organization and
+  denies every customer action before principal resolution when status is not
+  `ACTIVE`. The legacy role helper follows the same rule.
+- Every customer mutation service (project/folder/binding/policy, explorer,
+  members, teams, team grants, external-directory sync and slash-session
+  mutation) verifies `ACTIVE` while holding a PostgreSQL `FOR SHARE` lock on
+  the Organization row. A concurrent status `UPDATE` therefore orders before
+  the operation (which fails closed) or after its transaction commits.
+- Agent session/run/project-lock admission is one transaction behind that same
+  lifecycle lock. Queued triggers reauthorize when drained. Slash-command races
+  return an explicit user-facing rejection instead of becoming deduplicated,
+  silent failures.
+- Feishu attachments download into a private
+  `HUB_PROJECT_WORKSPACE_ROOT/.cph-staging` batch outside every project. A
+  durably committed ACTIVE run/project-lock admission is the linearization
+  point; publication then uses exclusive same-filesystem hard links rather than
+  copying bytes under a database lock. Failure compensates inode-tracked links,
+  fails the admitted run and releases its lock. Startup removes abandoned
+  staging batches and refuses staging symlinks.
+- Project creation holds the lifecycle lock across workspace allocation and DB
+  creation. A later transaction failure removes the new leaf workspace; cleanup
+  failure is surfaced together with the original error as `AggregateError`.
+- External-directory sync uses short lifecycle-serialized transactions per
+  unit rather than holding the Organization row lock across an unbounded sync.
+
+No customer API can reactivate or otherwise change Organization lifecycle
+state. Operator recovery remains a separate platform-control-plane task.
+
+Evidence includes deterministic status/update lock ordering, direct-service
+negative tests for both `SUSPENDED` and `ARCHIVED`, queued-trigger and
+post-authorization races, DB-triggered workspace compensation (including
+cleanup failure), full local Hub tests, and the real Linux attachment trigger
+path.

.scratch/saas-production-readiness/issues/23-durable-feishu-inbox-and-queue.md

@@ -0,0 +1,16 @@
+# Make Feishu intake and queued work durable
+
+Type: task
+Status: open
+Blocked by: 18, 44
+
+## Question
+
+Replace receipt-before-success and process-local batching/queueing with one
+durable inbound work lifecycle that atomically claims duplicate events, records
+processing/completion/failure, survives restart, retries recoverable failures,
+and proves concurrent redelivery cannot duplicate a run or silently lose an
+accepted prompt. Preserve execution-time authorization from ticket 18 when
+deferred work is finally started, and expose the durable identity/state needed
+by ADR-0022's Organization-fair admission scheduler rather than recreating a
+second queue lifecycle.

.scratch/saas-production-readiness/issues/24-transactional-run-lock-recovery.md

@@ -0,0 +1,12 @@
+# Make run, session, lock, and recovery transitions safe
+
+Type: task
+Status: open
+
+## Question
+
+Implement and fault-test a centralized AgentRun lifecycle that enforces one
+active project/provider/role/model session, serializes provider-cursor updates,
+preserves the ADR-0002 lock invariant across every database failure, applies a
+real wall deadline and renewable lease, reconciles every non-terminal state on
+startup/watchdog, and cannot let a second Hub process invalidate live work.

.scratch/saas-production-readiness/issues/25-durable-outbound-delivery.md

@@ -0,0 +1,13 @@
+# Make outbound Feishu delivery durable and idempotent
+
+Type: task
+Status: open
+Blocked by: 44
+
+## Question
+
+Give status cards, final responses, files, and recovery notices a durable
+outbox/delivery lifecycle with stable idempotency keys, provider message IDs,
+bounded retry and reconciliation, so model completion is distinguishable from
+pending/failed/delivered user-visible output and an operator can safely replay
+ambiguous failures without duplicates.

.scratch/saas-production-readiness/issues/26-decide-workspace-run-commit-semantics.md

@@ -0,0 +1,13 @@
+# Decide workspace commit semantics for failed runs
+
+Type: grilling
+Status: open
+
+## Question
+
+When an agent run fails, is canceled, times out, loses the database, or is
+killed mid-tool, should its workspace changes remain as partial collaborative
+work, be quarantined for review, or roll back atomically? Decide the contract,
+including the relationship among provider cursor, AgentMessage,
+AgentFileChange, git/filesystem state, and the next run, then create the
+smallest implementation tickets and crash tests that enforce it.

.scratch/saas-production-readiness/issues/27-reconcile-audit-history-contract.md

@@ -0,0 +1,16 @@
+# Reconcile the audit and run-history contract
+
+Type: grilling
+Status: open
+Blocked by: 04
+
+## Question
+
+Separate or unify run-bound audit entries, pre-run security/permission events,
+structured messages, and operational recovery events without weakening
+`Spec.System.Audit`'s pinned AuditEntry-to-run relation. Decide durability,
+failure, retention, and query semantics; then enforce referential integrity and
+observable/recoverable writes instead of silently swallowing lost evidence.
+Do not merge these customer Project/Run records with ADR-0023's already-decided
+separate, append-only, fail-closed Platform Audit; decide only any shared
+retention/export infrastructure without weakening either semantic boundary.

.scratch/saas-production-readiness/issues/28-typed-observable-http-errors.md

@@ -0,0 +1,13 @@
+# Make HTTP dependency failures typed and observable
+
+Type: task
+Status: open
+
+## Question
+
+Replace message-substring HTTP error classification with typed validation,
+domain, authorization, Prisma/dependency, and unexpected-error boundaries.
+Return only sanitized stable client codes, preserve true 5xx/availability
+semantics, emit one request-scoped structured error with a support-visible
+request ID, redact secrets and raw internals, and add a real database-outage
+regression proving Prisma failures can never become client 4xx responses.

.scratch/saas-production-readiness/issues/29-decide-telemetry-slos-alert-routing.md

@@ -0,0 +1,16 @@
+# Decide the telemetry, SLO, retention, and alerting control plane
+
+Type: grilling
+Status: open
+
+## Question
+
+For the accepted initial single-host service, decide the log/metric/trace
+collection and storage path, instance roles, release and correlation identity,
+redaction/access/retention policy, initial service-level indicators and
+objectives, error budgets, alert thresholds, on-call destinations, escalation
+ownership, and telemetry-blind detection. Prefer the smallest operable control
+plane, but record every domain-visible availability or retry assumption in the
+appropriate ADR/spec rather than hiding it in dashboard configuration. Consume
+ADR-0022's already-decided usage/cost soft-alert ownership as input; do not
+reopen its hard-limit or provider-mode financial-alert semantics here.

.scratch/saas-production-readiness/issues/30-instrument-correlated-runtime-telemetry.md

@@ -0,0 +1,15 @@
+# Instrument correlated production runtime telemetry
+
+Type: task
+Status: open
+Blocked by: 13, 23, 24, 25, 27, 28, 29
+
+## Question
+
+Implement the decided telemetry control plane with one redacted structured
+context from HTTP/Feishu ingress through durable work, run/session/lock,
+provider and SDK retry/rate-limit/tool events, cph/workspace operations,
+outbound delivery, audit, startup/drain/recovery, and release/schema identity.
+Expose bounded-cardinality metrics, dashboards and alert rules; eliminate
+silent critical-path failures; and prove logs can pivot end-to-end while metric
+labels contain no tenant/user/run cardinality or secret/message content.

.scratch/saas-production-readiness/issues/31-write-and-drill-incident-runbooks.md

@@ -0,0 +1,22 @@
+# Write and drill production incident recovery runbooks
+
+Type: task
+Status: open
+Blocked by: 06, 12, 13, 14, 23, 24, 25, 26, 27, 28, 29, 30, 39, 40, 41
+
+## Question
+
+Check in operator runbooks for PostgreSQL outage/migration failure, Feishu
+disconnect and credential rejection, ingress/outbound backlog or dead letter,
+stuck run/lock, provider or cph failure, workspace capacity/corruption,
+deploy/rollback, host restart during work, and suspected tenant exposure. Give
+each alert an owner and safe read-first commands, require confirmed/audited
+scope for destructive or replay actions, execute every runbook against
+production-shaped staging, and archive evidence that detection, mitigation,
+reconciliation, verification, escalation, and alert clearing all work. Include
+database/workspace backup failure, failed migration, full-host restore and
+cutover, duplicate/gap review, fallback, and achieved RPO/RTO evidence.
+Also drill ADR-0023 platform-administrator lockout and Platform-owned Feishu
+Application credential loss through the dual-factor offline recovery command,
+expiring Emergency Platform Grant, session revocation, immutable Platform Audit,
+and explicit recovery closeout; never introduce a standing break-glass account.

.scratch/saas-production-readiness/issues/32-layered-limits-rate-limiting.md

@@ -0,0 +1,13 @@
+# Implement layered limits and multi-dimensional request rate limiting
+
+Type: task
+Status: open
+Blocked by: 20, 22, 44
+
+## Question
+
+Implement ADR-0022's fail-closed platform ceilings, Organization-lowerable
+policy limits, bounded list/query behavior, and typed multi-dimensional rate
+limits for authenticated Web, signed Feishu, OAuth, and administrative writes.
+Prove effective limits never exceed platform ceilings and every overload result
+is observable, scoped, and carries retry guidance where applicable.

.scratch/saas-production-readiness/issues/33-durable-fair-admission-scheduler.md

@@ -0,0 +1,13 @@
+# Build the durable Organization-fair admission scheduler
+
+Type: task
+Status: open
+Blocked by: 18, 23, 24, 32
+
+## Question
+
+Extend the durable inbound lifecycle into ADR-0022's bounded Agent Admission
+Queue with platform/Organization/Project concurrency, fair scheduling across
+Organizations, FIFO within each Organization, durable position/state,
+expiration notification, scoped cancellation, idempotent admission, explicit
+capacity rejection, and no automatic replay after a run has started.

.scratch/saas-production-readiness/issues/34-file-storage-budgets.md

@@ -0,0 +1,13 @@
+# Enforce file, archive, storage, and entity budgets
+
+Type: task
+Status: open
+Blocked by: 17, 23, 32
+
+## Question
+
+Enforce ADR-0022's request/file/attachment/archive-expansion,
+Project/Organization storage, and Organization entity-count limits with
+streaming accounting, atomic reservation, cleanup of incomplete artifacts,
+bounded queries, explicit errors, and continued read/export/delete/admin
+remediation while an Organization is at or above a ceiling.

.scratch/saas-production-readiness/issues/35-run-runtime-budgets.md

@@ -0,0 +1,13 @@
+# Enforce Agent Run and process resource budgets
+
+Type: task
+Status: open
+Blocked by: 16, 24, 32
+
+## Question
+
+Implement ADR-0022's wall-time, turn, tool-count, per-tool-time,
+output/log/event-size, memory, CPU, and child-process budgets. Termination must
+be fail-fast and observable, persist `TIMED_OUT` or `LIMIT_EXCEEDED` with the
+triggered dimension and actual usage, cancel the provider/process safely, and
+release run locks and admission capacity under every injected failure.

.scratch/saas-production-readiness/issues/36-usage-soft-alerts.md

@@ -0,0 +1,13 @@
+# Implement provider-mode usage attribution and soft alerts
+
+Type: task
+Status: open
+Blocked by: 19, 24, 29, 32, 43
+
+## Question
+
+Attribute token usage, provider-reported cost, run count, and duration by
+Organization, Project, Run, model, and Provider Connection; preserve missing
+cost as unknown; expose Organization-configurable soft thresholds; and route
+Organization alerts plus platform-managed anomalous-cost alerts without hard
+spend blocking, BYOK financial alerts, or payment-settlement semantics.

.scratch/saas-production-readiness/issues/37-emergency-workload-brakes.md

@@ -0,0 +1,13 @@
+# Add audited emergency workload brakes
+
+Type: task
+Status: open
+Blocked by: 22, 23, 24, 27, 30, 32, 33, 45
+
+## Question
+
+Implement reversible, reason-bearing, audited `DRAIN` and `STOP_NOW` controls at
+Organization and platform scope. Prove new admissions, queued work, active runs,
+locks, user notifications, authorization, telemetry, restart persistence, and
+explicit recovery follow ADR-0022 without deleting or silently suspending
+Organization data.

.scratch/saas-production-readiness/issues/38-calibrate-production-capacity.md

@@ -0,0 +1,14 @@
+# Calibrate and prove production capacity ceilings
+
+Type: research
+Status: open
+Blocked by: 30, 32, 33, 34, 35, 36, 37
+
+## Question
+
+On the accepted production-like host and provider path, measure safe numerical
+defaults and headroom for every ADR-0022 platform ceiling. Exercise normal and
+noisy-neighbor traffic, provider stalls and rate limits, queue saturation,
+large files/archives, storage pressure, process budgets, and both workload
+brakes; publish reproducible evidence and the checked-in configuration values
+required by the production release gate.

.scratch/saas-production-readiness/issues/39-decide-recovery-objectives.md

@@ -0,0 +1,14 @@
+# Decide recovery objectives, backup retention, and restore scope
+
+Type: grilling
+Status: open
+
+## Question
+
+For the initial SaaS service, decide measurable RPO/RTO targets, backup and
+restore-drill frequency, retention and deletion-in-backups policy, off-host
+copy/failure-domain requirements, logical export versus physical base-backup
+and continuous-WAL/PITR coverage, encryption and key custody, operator ownership,
+and whether the supported promise is full-service disaster recovery only or
+also Organization-selective export/restore. Record compliance-dependent items
+as explicit OPEN inputs rather than inventing defaults.

.scratch/saas-production-readiness/issues/40-verified-backup-restore.md

@@ -0,0 +1,16 @@
+# Implement coherent backup sets and verified restore
+
+Type: task
+Status: open
+Blocked by: 10, 12, 14, 19, 23, 24, 25, 26, 29, 39, 41, 43
+
+## Question
+
+Implement the decided recovery policy as cataloged, checksummed, encrypted,
+off-host backup sets covering PostgreSQL logical exports plus physical base
+backup/continuous WAL or a proven managed equivalent, Project workspaces,
+compatible release/migration identity, configuration inventory, org-secret
+ciphertext, and separately recoverable key material. Establish a proven
+cross-store checkpoint, retention and alerting, partial-artifact cleanup,
+isolated restore with outbound integrations disabled, lifecycle reconciliation,
+critical SaaS journey verification, and measured RPO/RTO drills.

.scratch/saas-production-readiness/issues/41-recoverable-project-workspace-lifecycle.md

@@ -0,0 +1,14 @@
+# Make the Project workspace lifecycle recoverable
+
+Type: task
+Status: open
+Blocked by: 10, 26, 34
+
+## Question
+
+Move persistent workspaces outside every mutable release tree and make Project
+workspace allocation, database commit, archive/deletion, relocation, restore,
+and orphan/missing-path reconciliation explicit and crash-safe. Prove failed
+Project creation leaves no silent orphan, deployment/rollback/pruning cannot
+delete customer files, restored absolute paths are validated or safely rebased,
+and every inconsistency is observable rather than auto-hidden.

.scratch/saas-production-readiness/issues/42-decide-platform-admin-identity-audit.md

@@ -0,0 +1,54 @@
+# Decide the platform-administrator identity and audit boundary
+
+Type: grilling
+Status: resolved
+
+## Question
+
+Which identity issuer, bootstrap and recovery procedure, invite/allowlist model,
+session claims, role lifecycle, separation from Organization membership, and
+append-only audit contract will authenticate Platform Administrators for the
+initial service? The decision must keep `/admin/platform` private, make every
+Organization/connection/workload-control mutation attributable, and define a
+reviewable break-glass path without treating platform staff as customer members.
+
+## Answer
+
+Platform Administrators authenticate only through a dedicated Platform-owned
+Feishu Application and a separate, server-side Platform Session. Platform
+Identity, standing grant, invitation, session, audit and emergency grant are
+independent of customer `User`, `OrganizationMembership`, Project permission
+and Project/Run audit. The initial control plane has one peer role only.
+
+The first administrator is created by a versioned, idempotent, transactional
+database bootstrap procedure that runs only when no standing administrator is
+active and atomically records the Platform Audit Entry. Later administrators
+receive short-lived, single-use invitations bound to a specific platform
+Feishu account. The last active standing administrator cannot be revoked;
+revocation immediately invalidates that identity's sessions and invitations.
+
+Every platform request reloads the revocable session and current standing or
+unexpired emergency grant. Administrator, connection/credential,
+Organization-lifecycle, workload-brake and emergency-grant actions require
+recent Feishu OAuth step-up.
+
+Every successful platform mutation commits with a separate append-only
+Platform Audit Entry or fails. Organization Administrators receive a redacted
+projection of platform actions affecting their own Organization. No standing
+break-glass account exists: offline recovery requires controlled host access,
+an off-host recovery key, an incident and reason, and issues only an expiring
+Emergency Platform Grant.
+
+The complete accepted decision and implementation divergences are in
+[ADR-0023](../../../docs/adr/0023-platform-administrator-identity-and-audit.md).
+The pinned semantic invariants are in
+[`Spec.System.PlatformAdministration`](../../../spec/Spec/System/PlatformAdministration.lean),
+and the canonical terms are in [`CONTEXT.md`](../../../CONTEXT.md).
+
+Exact numeric session/invitation/step-up limits and browser mechanics remain
+with [Design the production browser and session boundary](20-design-production-web-session-boundary.md).
+Customer Project/Run audit retention remains with
+[Reconcile the audit and run-history contract](27-reconcile-audit-history-contract.md),
+without reopening Platform Audit's separate fail-closed boundary. Implementation
+is owned by [Build the private platform-administrator control plane](45-build-private-platform-admin-control-plane.md),
+and recovery rehearsal by [Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md).

.scratch/saas-production-readiness/issues/43-implement-org-connection-secret-plane.md

@@ -0,0 +1,16 @@
+# Implement the Organization connection and secret plane
+
+Type: task
+Status: open
+Blocked by: none
+
+## Question
+
+Implement encrypted, versioned, Organization-scoped Feishu Application and
+model Provider Connection records plus one resolver boundary. Support one
+customer-owned Feishu app per Organization, Organization-admin-managed BYOK,
+and one distinct platform-managed key/base URL per Organization; enforce which
+administrator may write each mode, readiness and rotation lifecycle, runtime
+resolution from Organization/Project, redacted observability, backup/key
+recovery hooks, and no process-global credential fallback or plaintext leakage
+into business rows, logs, HTTP responses, or agent environments.

.scratch/saas-production-readiness/issues/44-route-feishu-by-org-connection.md

@@ -0,0 +1,17 @@
+# Route Feishu identity and traffic by Organization connection
+
+Type: task
+Status: open
+Blocked by: 20, 22, 43
+
+## Question
+
+Make every Feishu OAuth, WebSocket/event, card action, directory sync, chat
+binding, API call, and outbound message start from an explicit active
+Organization Feishu Application Connection. Model external user, chat, event,
+and callback identifiers in that connection's namespace; bind OAuth state to
+the intended Organization/application; run and observe one listener/client
+lifecycle per configured app; and prove two customer-owned apps with colliding
+provider-local identifiers cannot authenticate, bind, authorize, deduplicate,
+rate-limit, or deliver across Organizations. Do not infer tenant from a global
+`open_id`, `chat_id`, or from the user's membership count.

.scratch/saas-production-readiness/issues/45-build-private-platform-admin-control-plane.md

@@ -0,0 +1,25 @@
+# Build the private platform-administrator control plane
+
+Type: task
+Status: open
+Blocked by: 20, 27, 28, 42, 43, 44
+
+## Question
+
+Implement authenticated `requirePlatformAdmin` APIs and a private
+`/admin/platform` web area for Platform Administrators to create/list/update
+Organizations, bootstrap their OWNER, view/change lifecycle state, configure
+and test each customer-owned Feishu Application, configure the Organization's
+distinct platform-managed Provider Connection, and manage platform-admin
+invitations and revocations. Implement ADR-0023's separate Platform Identity,
+single standing role, guarded bootstrap procedure, identity-bound single-use
+invitations, revocable server-side Platform Session, recent-authentication
+step-up, last-administrator protection, fail-closed append-only Platform Audit,
+Organization-safe audit projection, and dual-factor offline recovery with
+expiring Emergency Platform Grants. Replace rather than reuse the legacy
+customer-`User`-bound `PlatformRoleAssignment`/`TEACHER` model. Prove anonymous
+users, ordinary members, and Organization admins cannot load either page data
+or mutations; every sensitive action is validated, redacted, audited,
+observable, and covered by browser-level critical-journey tests. Emergency
+workload controls remain integrated through
+[Add audited emergency workload brakes](37-emergency-workload-brakes.md).

.scratch/saas-production-readiness/issues/46-build-private-org-admin-control-plane.md

@@ -0,0 +1,17 @@
+# Build the private Organization-administrator control plane
+
+Type: task
+Status: open
+Blocked by: 20, 27, 28, 32, 36, 43, 44
+
+## Question
+
+Turn the guarded Organization JSON APIs into a usable private
+`/admin/org/:orgSlug` web control plane for OWNER/ADMIN users. Cover members and
+roles, teams and Project access, Folders/Projects/chat unbinding, sessions and
+usage, model/role/run policy, Organization policy limits and alerts, Feishu
+connection readiness, Provider Credential Mode selection, and BYOK key/base URL
+management without ever displaying stored plaintext. Preserve server-side
+authorization on every request, define safe loading/error/empty states, and
+prove login redirect plus browser journeys, cross-Organization denial, MEMBER
+denial, secret redaction, and logout/session expiry end to end.

.scratch/saas-production-readiness/map.md

@@ -0,0 +1,62 @@
+# Initial Production SaaS Readiness
+
+Label: wayfinder:map
+
+## Destination
+
+The repository can be deployed from a clean checkout as the initial production
+multi-tenant Curriculum Project Hub: platform staff manage Organizations; every
+Organization authenticates through and operates its own customer-owned Feishu
+app/bot, and selects either an Organization-managed BYOK provider connection or
+a distinct platform-managed provider connection; Organization owners/admins
+manage their own members, projects and settings. The ADR-0018/0020/0021
+single-host Linux topology is operated with repeatable release evidence,
+rollback and recovery, and no known critical security or data-integrity gaps.
+
+## Notes
+
+- `spec/` remains the semantic source of truth; implementation drift must be
+  surfaced rather than silently resolved.
+- The initial product boundary follows the accepted ADRs: Organization is the
+  tenant root and customer onboarding may be manual. Full billing, tenant
+  suspension, and self-service platform administration remain deferred;
+  ADR-0022's safety limits and workload brakes are required for production.
+- Manual guided setup may provision each Organization's customer-owned Feishu
+  app during the pilot; one process-global Feishu/OAuth credential shared by
+  unrelated Organizations is not an acceptable destination state.
+- Use the `diagnosing-bugs` loop for every reproduced defect. Prefer explicit
+  failures and observable critical paths over fallback behavior.
+- Work directly on `main` per developer authorization; make small conventional
+  commits after verified, independently reversible milestones. Do not push
+  without explicit authorization.
+- Local `lake build` currently requires a configured elan toolchain; CI remains
+  the authoritative configured Lean build environment until that local toolchain
+  is available.
+
+## Decisions so far
+
+- [Audit the clean-host deployment and rollback contract](issues/01-audit-clean-host-deployment.md) — keep the accepted single-host topology, but replace the incomplete in-place updater with a provisioned, immutable, readiness-gated and rollbackable release contract.
+- [Audit tenant, authentication, and request security boundaries](issues/02-audit-tenant-auth-security.md) — retain the working org-admin/application authorization core, but production is blocked on agent/MCP isolation, execution-time object binding, org-scoped encrypted credentials, tenant status, browser sessions, and clean dependency audits.
+- [Audit run lifecycle and data integrity under failure](issues/03-audit-run-lifecycle-integrity.md) — retain the database-enforced project-lock uniqueness and normal happy path, but replace receipt-before-success, process-local queues, independent run/session/lock effects, silent delivery/audit loss, and partial startup reset with durable, recoverable state machines; explicitly decide failed-run workspace semantics.
+- [Define production observability and incident recovery](issues/04-define-observability-recovery.md) — keep the existing Fastify/Prisma/systemd signal fragments, but production requires dependency-aware readiness, typed and correlated errors, durable-work telemetry, an explicit SLO/retention/alerting control plane, and recovery runbooks proven by fault drills.
+- [Define initial abuse and capacity controls](issues/05-define-abuse-capacity-controls.md) — require layered fail-closed limits, durable Organization-fair admission, explicit backpressure, bounded files/storage/runs, soft usage alerts, and audited emergency workload brakes; calibrate numerical ceilings under production-like load.
+- [Audit backup, migration, and disaster recovery safety](issues/06-audit-backup-migration-recovery.md) — separate persistent state from releases, define a database/workspace checkpoint and recovery objectives, make migrations compatibility-gated, and prove complete off-host backup sets through isolated restore and cutover drills.
+- [Audit the accepted product surface for production completeness](issues/07-audit-product-surface-completeness.md) — retain the guarded Organization APIs and single-app Feishu vertical slice, but require explicit Organization-scoped Feishu/provider connections and external-identity namespaces, private platform and Organization admin control planes, and the full ADR-0022 capacity surface before release.
+- [Decide the platform-administrator identity and audit boundary](issues/42-decide-platform-admin-identity-audit.md) — use a dedicated platform Feishu issuer, one peer administrator role, guarded bootstrap and identity-bound invitations, revocable server-side sessions, atomic append-only Platform Audit, and dual-factor offline recovery without a standing break-glass account.
+- [Restore the checker formatting gate](issues/09-restore-checker-format-gate.md) — pin local and CI Rust/rustfmt/clippy to 1.92.0, apply the canonical workspace format, and keep the full fmt + clippy + 53-test checker gate green.
+- [Provision a runnable non-root Hub service](issues/10-provision-nonroot-service.md) — install a strictly validated service identity and persistent layout, reject release/workspace overlap, validate complete production configuration and real bind settings, and execute Hub/Prisma/cph/bubblewrap probes as the service user under `no_new_privs` before writing the unit.
+- [Confine the agent runtime and protect service credentials](issues/16-confine-agent-and-protect-credentials.md) — replace the Agent environment, deny host reads/writes from root, keep every writable SDK path inside the canonical project, require bubblewrap plus socat, emit bounded correlated diagnostics, and prove the boundary with the real Linux SDK under a capability-free non-root identity.
+- [Close MCP context and file-delivery escape paths](issues/17-close-mcp-data-egress-escapes.md) — use no-follow project-rooted file operations and delivery snapshots, remove Git-root delivery, and bind every Feishu MCP read to the run's project and chat.
+- [Enforce Organization status at the shared authorization seam](issues/22-enforce-organization-status.md) — deny non-ACTIVE tenants before principal resolution, serialize every customer mutation and run admission against Organization lifecycle updates, reauthorize queued work, and stage/publish attachments with crash recovery and explicit filesystem compensation.
+- [Clear dependency advisories and gate releases](issues/21-clear-dependency-security-advisories.md) — resolve Node production-high and Rust vulnerability/unsound advisories with locked compatible upgrades plus an exact upstream citationberg fix, and make both audits mandatory CI gates without ignores.
+
+## Fog
+
+- The production topology after the initial single-host pilot (HA, horizontal
+  workers, multi-region, or managed container orchestration) depends on measured
+  demand and failure experience.
+- Commercial self-service concerns such as billing plans and automated tenant
+  provisioning remain beyond the accepted ADR boundary; revisit after the
+  initial safety and operability route is clear.
+- Regulatory retention, residency, and customer-specific compliance controls
+  require product/legal inputs that the repository does not currently contain.

AGENTS.md

@@ -11,6 +11,36 @@
 - org 后台 project explorer 里 `Folder` 是透明组织节点,不是权限资源;project 仍是权限边界。
   普通老师可在飞书群自助建 project 但受 org policy 控制(见 ADR-0021 /
   `Spec.System.ProjectWorkspace`)。
+- 每个 org 自选 BYOK 或平台托管 model provider connection;平台托管也必须是该 org
+  独享的 key/base URL,不得让无关 org 共用 process-global provider key(见 ADR-0021 /
+  `Spec.System.Organization`)。
+- Feishu/provider secret 使用本地版本化 master-key keyring 的信封加密;生产由 systemd
+  credential 注入,运行时只允许显式 org/project scope 的 fail-closed resolver,不得回退
+  process-global credential;Agent child 只接收 run-scoped loopback proxy capability,
+  不接收 org provider credential(见 ADR-0024 / `Spec.System.Organization`)。
+- 生产容量按不可突破的 platform ceiling 与 org 可下调 policy 分层;有效限制取两者较低值。
+  Agent admission 必须持久、有界、跨 org 公平且显式背压(见 ADR-0022 /
+  `Spec.System.Capacity`)。
+- 平台管理员只通过独立的 platform-owned 飞书应用与可撤销 Platform Session 认证,不复用
+  客户 `User`/org membership;平台写操作与 append-only audit 同事务,break-glass 只走
+  双因子的离线恢复流程(见 ADR-0023 / `Spec.System.PlatformAdministration`)。
+- 受控 alpha 暂采用一 Organization 一具名 systemd Silo:独立 database role/database、
+  service identity、workspace、keyring 与 Feishu/provider connection；进程必须由
+  `HUB_SILO_ORGANIZATION_ID` fail-closed 绑定唯一 org，平台后台不开放。共享 SaaS
+  控制面与 Docker adapter 后置(见 ADR-0025)。
+- Agent role 与 skill 是 Organization-scoped 动态运行配置：role 组合 model、system prompt、
+  tools 与已安装 skill；skill 版本进入 content-addressed 持久存储，run 只读加载所选快照。
+  每个 Organization 必须且只能有一个启用中的默认 role；新建群绑定从该默认值初始化，之后
+  `ProjectGroupBinding` 持久化群内当前 role，run 在接纳时冻结该 role。飞书公开 slash 协议只含
+  `/project`、`/usage`、`/help`；role、会话、目录操作走 `/project` 卡片，Claude 原生
+  `/compact` 只能由卡片动作以未经包装的精确 prompt 转发。
+  `settingSources: []` 继续禁用项目/用户配置加载，不得把任意 workspace `.claude` 配置变成
+  运行时能力(见 ADR-0018)。
+- 项目发现由 `ProjectDiscovery` 模块统一承载：PostgreSQL `pg_trgm` 搜索派生文档、项目编号
+  归一化、完整 Folder breadcrumb、MANAGE 授权过滤与分页都在该模块内；飞书卡片只是 adapter。
+  `Project`/`Folder` 仍是事实来源，搜索文档必须可重建且由数据库触发器同步，禁止调用方双写。
+  系统 `Inbox` 只作为未分类项目的内部落点，不作为业务 folder 暴露；已绑定群通过
+  `@bot /project` 随时打开项目管理卡片，重命名仍走 org-scoped MANAGE 授权与审计。
 
 ## 纪律
 

CONTEXT.md

@@ -0,0 +1,101 @@
+# Curriculum Project Hub
+
+The shared language for the multi-tenant service that lets customer organizations manage curriculum projects and operate agents through their own Feishu connection.
+
+## Language
+
+**Organization**:
+The customer tenant root that owns its memberships, projects, teams, external connections, and settings.
+_Avoid_: Account, workspace
+
+**Platform Administrator**:
+An internal operator who creates and operates Organizations without becoming a member of them.
+_Avoid_: Organization admin, super member
+
+**Organization Administrator**:
+An Organization OWNER or ADMIN who manages only that Organization's members, projects, connections, and settings.
+_Avoid_: Platform admin
+
+**Admin Surface**:
+The private web control plane available only to an authenticated user with the corresponding Platform Administrator or Organization Administrator authority.
+_Avoid_: Public management page, member portal
+
+**Customer-owned Feishu Application**:
+The Organization-scoped Feishu application used for that Organization's login, bot messages, and directory integration; unrelated Organizations never share it.
+_Avoid_: Global Feishu app, platform bot
+
+**Platform-owned Feishu Application**:
+The dedicated Feishu application used only to authenticate Platform Administrators; it is separate from every Customer-owned Feishu Application and grants no Organization or Project authority.
+_Avoid_: Customer-owned Feishu Application, Organization Feishu app
+
+**Platform Administrator Bootstrap**:
+The one-time, guarded grant of Platform Administrator authority to a verified Platform Identity when no active Platform Administrator exists.
+_Avoid_: Seed administrator, standing bootstrap account
+
+**Platform Administrator Invitation**:
+A short-lived, single-use invitation issued by a Platform Administrator and bound to one expected identity from the Platform-owned Feishu Application.
+_Avoid_: Open administrator enrollment, unbound administrator invitation
+
+**Platform Session**:
+A revocable authentication session bound only to a Platform Identity and its active Platform Administrator authority; it carries no Organization membership or Project authority.
+_Avoid_: Organization admin session, customer session
+
+**Platform Audit Entry**:
+An immutable, durable record of a platform control-plane decision or action, separate from customer Project and Agent Run audit and mandatory for every platform mutation. Organization Administrators receive only a redacted projection of entries affecting their Organization.
+_Avoid_: Best-effort log, Project audit entry
+
+**Emergency Platform Grant**:
+Short-lived Platform Administrator authority issued only by the offline recovery procedure for a declared incident and automatically expired afterward.
+_Avoid_: Standing break-glass account, permanent emergency administrator
+
+**Provider Credential Mode**:
+The Organization's choice between a BYOK Provider Connection and a Platform-managed Provider Connection.
+_Avoid_: Global provider setting
+
+**BYOK Provider Connection**:
+An Organization-scoped model-provider connection whose key and base URL are supplied and managed by an Organization Administrator.
+_Avoid_: Platform key
+
+**Platform-managed Provider Connection**:
+An Organization-scoped model-provider connection whose distinct key and base URL are supplied and managed by a Platform Administrator for that Organization.
+_Avoid_: Shared platform key, BYOK
+
+**Platform Capacity Ceiling**:
+A non-overridable upper bound imposed by the platform to protect shared service capacity.
+_Avoid_: Organization quota
+
+**Organization Policy Limit**:
+An optional lower bound configured by an Organization Administrator to govern that Organization's own usage.
+_Avoid_: Platform limit
+
+**Effective Limit**:
+The lower of the Platform Capacity Ceiling and the Organization Policy Limit; when no Organization Policy Limit exists, the Platform Capacity Ceiling applies.
+_Avoid_: Unlimited default
+
+**Organization-fair Scheduling**:
+The policy that allocates available agent execution capacity fairly across Organizations, while preserving first-in-first-out order within each Organization.
+_Avoid_: Global FIFO
+
+**Agent Admission Queue**:
+The durable, bounded set of accepted agent run requests waiting for execution capacity.
+_Avoid_: In-memory trigger queue, unlimited backlog
+
+**Expired Run Request**:
+An accepted agent run request that reached its maximum queue wait before execution; it is terminal and will never execute automatically.
+_Avoid_: Delayed request, automatic retry
+
+**Run Request Initiator**:
+The Organization member whose action created an agent run request.
+_Avoid_: Queue owner, Project owner
+
+**Explicit Run Retry**:
+A new agent run request deliberately created after an earlier run failed or timed out.
+_Avoid_: Automatic replay, implicit rerun
+
+**Run Budget**:
+The deterministic resource boundary for one agent run, independent of monetary or token usage alerts.
+_Avoid_: Cost budget, unlimited run
+
+**Emergency Workload Brake**:
+An audited Platform Administrator control that prevents new agent work for one Organization or the whole platform and may explicitly stop active work during an incident.
+_Avoid_: Organization deletion, service restart

Cargo.lock

@@ -264,10 +264,9 @@ dependencies = [
 [[package]]
 name = "citationberg"
 version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "756ff1e3d43a9ecc8183932fb4d9fd3971236f3ce4acb62fe51d1cd43297547d"
+source = "git+https://github.com/typst/citationberg?rev=06a591e2f237d25e1dfdedac3f3d1494c496c52d#06a591e2f237d25e1dfdedac3f3d1494c496c52d"
 dependencies = [
- "quick-xml 0.38.4",
+ "quick-xml",
  "serde",
  "serde_path_to_error",
 ]
@@ -427,6 +426,7 @@ version = "0.0.2"
 dependencies = [
  "cph-diag",
  "serde",
+ "tempfile",
  "toml",
 ]
 
@@ -480,9 +480,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1402,9 +1402,9 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "memmap2"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
 dependencies = [
  "libc",
 ]
@@ -1641,13 +1641,13 @@ checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315"
 
 [[package]]
 name = "plist"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1"
+checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85"
 dependencies = [
  "base64",
  "indexmap",
- "quick-xml 0.39.4",
+ "quick-xml",
  "serde",
  "time",
 ]
@@ -1755,23 +1755,14 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
 
 [[package]]
 name = "quick-xml"
-version = "0.38.4"
+version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
 dependencies = [
  "memchr",
  "serde",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.39.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.45"

Cargo.toml

@@ -21,3 +21,8 @@ serde = { version = "1", features = ["derive"] }
 toml = "0.8"
 cph-diag = { path = "crates/cph-diag" }
 cph-model = { path = "crates/cph-model" }
+
+# citationberg 0.7.0 on crates.io pins vulnerable quick-xml 0.38. Upstream has
+# adopted quick-xml 0.41 but has not published a replacement release yet.
+[patch.crates-io]
+citationberg = { git = "https://github.com/typst/citationberg", rev = "06a591e2f237d25e1dfdedac3f3d1494c496c52d" }

README.md

@@ -41,7 +41,8 @@ crates/            ← 实现:rule-based checker(向 spec 对齐)。见 crates/R
   cph-check / cph-cli                             ← checker 本体 + `cph` 命令行
 render/            ← typst 渲染包 cph-render(母本的渲染后端之一,ADR-0005)
 examples/          ← 样例工程文件(如 TH-141),流水线的真实输入
-(hub/ exporter/ …) ← 将来的其他部件,平级于 spec/。尚未创建
+hub/              ← SaaS Hub:飞书协作、org 管理、agent runtime 与生产部署
+(exporter/ …)     ← 将来的其他部件,平级于 spec/
 ```
 
 `spec/` 与实现部件**物理分离、平级共存**:谁是上游、谁向谁对齐,一眼可见。
@@ -72,3 +73,7 @@ examples/          ← 样例工程文件(如 TH-141),流水线的真实输入
 ## CI
 
 `.gitea/workflows/spec-check.yml` 在每次 push / PR 时于 `spec/` 下跑 `lake build`,确保契约始终 type-check 通过(从第一天起就是"绿"的)。这是良构 gate,见宪法第 2 条。
+
+Rust checker 的本地与 CI 工具链由根 `rust-toolchain.toml` 固定;`.gitea/workflows/checker-check.yml`
+必须安装同一精确版本并执行 `cargo fmt --all --check`、Clippy `-D warnings` 与 workspace
+全测试。升级 Rust 时这两处必须在同一提交更新并通过完整 checker gate。

crates/cph-check/src/lib.rs

@@ -360,8 +360,13 @@ pub fn target_is_shell(root: &Path, target: &str) -> bool {
         .iter()
         .find(|t| t.name == target)
         .is_some_and(|t| {
-            t.steps.iter().any(|s| matches!(s, cph_model::Step::Shell { .. }))
-                && !t.steps.iter().any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
+            t.steps
+                .iter()
+                .any(|s| matches!(s, cph_model::Step::Shell { .. }))
+                && !t
+                    .steps
+                    .iter()
+                    .any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
         })
 }
 
@@ -370,9 +375,17 @@ fn run_one_shell(cwd: &Path, run: &str) -> ShellStepOutcome {
     use std::process::Command;
 
     let output = if cfg!(target_os = "windows") {
-        Command::new("cmd").arg("/C").arg(run).current_dir(cwd).output()
+        Command::new("cmd")
+            .arg("/C")
+            .arg(run)
+            .current_dir(cwd)
+            .output()
     } else {
-        Command::new("sh").arg("-c").arg(run).current_dir(cwd).output()
+        Command::new("sh")
+            .arg("-c")
+            .arg(run)
+            .current_dir(cwd)
+            .output()
     };
 
     match output {
@@ -704,7 +717,11 @@ fn collect_image_assets(eng_root: &Path, build_root: &Path, body: &str) -> Resul
             }
         }
         if let Err(e) = std::fs::copy(&src, &dst) {
-            return Err(format!("failed to copy '{}' → '{}': {e}", src.display(), dst.display()));
+            return Err(format!(
+                "failed to copy '{}' → '{}': {e}",
+                src.display(),
+                dst.display()
+            ));
         }
     }
     Ok(())
@@ -763,7 +780,8 @@ pub fn target_is_markdown_assemble(root: &Path, target: &str) -> bool {
             t.steps
                 .iter()
                 .any(|s| matches!(s, cph_model::Step::AssembleMarkdown { .. }))
-                && !t.steps
+                && !t
+                    .steps
                     .iter()
                     .any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
         })
@@ -833,7 +851,11 @@ fn compile_targets(lesson: &cph_model::Lesson) -> Vec<&str> {
     lesson
         .targets
         .iter()
-        .filter(|t| t.steps.iter().any(|s| matches!(s, cph_model::Step::TypstCompile { .. })))
+        .filter(|t| {
+            t.steps
+                .iter()
+                .any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
+        })
         .map(|t| t.name.as_str())
         .collect()
 }

crates/cph-check/tests/pipeline.rs

@@ -5,7 +5,7 @@
 //! fixtures built in a temp dir. The typst Engine is pointed at the repo's real
 //! `render/` package via `Engine::with_render_dir`.
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use cph_diag::DiagCode;
 use cph_typst::Engine;
@@ -138,7 +138,7 @@ path = "segments/does-not-exist"
 
 /// Write a throwaway one-segment lesson into `tmp`, with the given `[targets.x]`
 /// body spliced in. Returns nothing; the caller runs `check`.
-fn write_segment_lesson_with_target(tmp: &PathBuf, target_body: &str) {
+fn write_segment_lesson_with_target(tmp: &Path, target_body: &str) {
     std::fs::write(
         tmp.join("manifest.toml"),
         format!(
@@ -264,7 +264,7 @@ template = "exports/handout.typ"
 
 /// Write a one-segment lesson with a `file-tree` shell target whose `run` is the
 /// given command. Returns nothing; the caller runs `run_shell_target`.
-fn write_shell_target_lesson(tmp: &PathBuf, run: &str) {
+fn write_shell_target_lesson(tmp: &Path, run: &str) {
     std::fs::write(
         tmp.join("manifest.toml"),
         format!(
@@ -365,7 +365,10 @@ run = "printf SHOULD_NOT_RUN > build/ran.txt"
 
     let report = cph_check::run_shell_target(&tmp, &engine(), "assets");
     assert!(!report.ok);
-    assert!(report.outcomes.is_empty(), "no command should run on a broken lesson");
+    assert!(
+        report.outcomes.is_empty(),
+        "no command should run on a broken lesson"
+    );
     assert!(report.check.has_errors());
     assert!(
         !tmp.join("build").join("ran.txt").exists(),
@@ -380,7 +383,7 @@ run = "printf SHOULD_NOT_RUN > build/ran.txt"
 /// content file, plus a `slides` target that assembles them. The `which` slice
 /// controls which segments get a `slides.md` (so the "skip absent" path can be
 /// exercised). Returns nothing; the caller runs `run_markdown_assemble_target`.
-fn write_markdown_assemble_target_lesson(tmp: &PathBuf, slides: &[(&str, &str)]) {
+fn write_markdown_assemble_target_lesson(tmp: &Path, slides: &[(&str, &str)]) {
     let mut parts = String::new();
     for (i, (name, _)) in slides.iter().enumerate() {
         if i > 0 {
@@ -424,10 +427,7 @@ field = "slides"
 #[test]
 fn target_is_markdown_assemble_detects_shape() {
     let tmp = tempdir();
-    write_markdown_assemble_target_lesson(
-        &tmp,
-        &[("a", "# A"), ("b", "# B")],
-    );
+    write_markdown_assemble_target_lesson(&tmp, &[("a", "# A"), ("b", "# B")]);
     assert!(cph_check::target_is_markdown_assemble(&tmp, "slides"));
     // The student target doesn't exist here → falls through to typst path → false.
     assert!(!cph_check::target_is_markdown_assemble(&tmp, "student"));
@@ -439,7 +439,10 @@ fn run_markdown_assemble_target_concatenates_in_parts_order() {
     let tmp = tempdir();
     write_markdown_assemble_target_lesson(
         &tmp,
-        &[("intro", "# 规则一\n- 范围"), ("rule2", "# 规则二\n$8\\times$")],
+        &[
+            ("intro", "# 规则一\n- 范围"),
+            ("rule2", "# 规则二\n$8\\times$"),
+        ],
     );
 
     let report = cph_check::run_markdown_assemble_target(&tmp, &engine(), "slides");
@@ -667,6 +670,6 @@ fn tempdir() -> PathBuf {
     p
 }
 
-fn cleanup(p: &PathBuf) {
+fn cleanup(p: &Path) {
     let _ = std::fs::remove_dir_all(p);
 }

crates/cph-cli/src/main.rs

@@ -51,12 +51,12 @@ enum Command {
     /// completion file, e.g. `cph completions zsh > ~/.zfunc/_cph`.
     Completions {
         /// Which shell to generate completions for.
-        shell: Shell,
+        shell: CompletionTarget,
     },
 }
 
 #[derive(Debug, Clone, Copy, clap::ValueEnum)]
-enum Shell {
+enum CompletionTarget {
     Bash,
     Zsh,
     Fish,
@@ -82,16 +82,16 @@ fn main() -> ExitCode {
 /// Emit a shell-completion script for `shell` to stdout. The script is built
 /// from the same `Cli` clap definition above, so it tracks subcommands/flags as
 /// they evolve.
-fn run_completions(shell: Shell) -> ExitCode {
+fn run_completions(shell: CompletionTarget) -> ExitCode {
     use clap::CommandFactory;
     use clap_complete::Shell as CompShell;
 
     let sh = match shell {
-        Shell::Bash => CompShell::Bash,
-        Shell::Zsh => CompShell::Zsh,
-        Shell::Fish => CompShell::Fish,
-        Shell::PowerShell => CompShell::PowerShell,
-        Shell::Elvish => CompShell::Elvish,
+        CompletionTarget::Bash => CompShell::Bash,
+        CompletionTarget::Zsh => CompShell::Zsh,
+        CompletionTarget::Fish => CompShell::Fish,
+        CompletionTarget::PowerShell => CompShell::PowerShell,
+        CompletionTarget::Elvish => CompShell::Elvish,
     };
     let mut cmd = Cli::command();
     clap_complete::generate(sh, &mut cmd, "cph", &mut std::io::stdout());
@@ -188,7 +188,10 @@ fn run_build(
 /// explicit `cph build --target <name>`, and each command is printed before it
 /// runs so the user sees exactly what is executed.
 fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> ExitCode {
-    eprintln!("running shell-step target '{target}' (commands execute in {})", path.display());
+    eprintln!(
+        "running shell-step target '{target}' (commands execute in {})",
+        path.display()
+    );
     let report = cph_check::run_shell_target(path, engine, target);
     print_diagnostics(&report.check);
 
@@ -196,7 +199,10 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
         // Refused before running anything (check errors / unknown target / no
         // shell steps): the diagnostics above explain why.
         if report.check.has_errors() {
-            eprintln!("build refused: {} errors (fix the lesson first)", report.check.error_count());
+            eprintln!(
+                "build refused: {} errors (fix the lesson first)",
+                report.check.error_count()
+            );
         } else {
             eprintln!("build failed: target '{target}' has no shell steps to run");
         }
@@ -212,13 +218,19 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
             eprint!("{}", outcome.stderr);
             eprintln!(
                 "step failed (exit {})",
-                outcome.status.map(|c| c.to_string()).unwrap_or_else(|| "signal".into())
+                outcome
+                    .status
+                    .map(|c| c.to_string())
+                    .unwrap_or_else(|| "signal".into())
             );
         }
     }
 
     if report.ok {
-        println!("shell-step target '{target}' completed: {} step(s) ok", report.outcomes.len());
+        println!(
+            "shell-step target '{target}' completed: {} step(s) ok",
+            report.outcomes.len()
+        );
         ExitCode::SUCCESS
     } else {
         eprintln!("shell-step target '{target}' failed");
@@ -231,13 +243,19 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
 /// shell path, this is opt-in — only on an explicit `cph build --target <name>`,
 /// and never in `check`.
 fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target: &str) -> ExitCode {
-    eprintln!("assembling markdown target '{target}' (reading parts under {})", path.display());
+    eprintln!(
+        "assembling markdown target '{target}' (reading parts under {})",
+        path.display()
+    );
     let report = cph_check::run_markdown_assemble_target(path, engine, target);
     print_diagnostics(&report.check);
 
     if report.outcomes.is_empty() && !report.ok {
         if report.check.has_errors() {
-            eprintln!("build refused: {} errors (fix the lesson first)", report.check.error_count());
+            eprintln!(
+                "build refused: {} errors (fix the lesson first)",
+                report.check.error_count()
+            );
         } else {
             eprintln!("build failed: target '{target}' has no assemble-markdown steps to run");
         }
@@ -250,7 +268,11 @@ fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target:
             outcome.field, outcome.parts_read
         );
         if let Some(out_rel) = &outcome.written {
-            println!("wrote {} ({} bytes)", path.join(out_rel).display(), outcome.body.len());
+            println!(
+                "wrote {} ({} bytes)",
+                path.join(out_rel).display(),
+                outcome.body.len()
+            );
         }
         if let Some(err) = &outcome.error {
             eprintln!("assemble failed: {err}");
@@ -258,7 +280,10 @@ fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target:
     }
 
     if report.ok {
-        println!("assemble-markdown target '{target}' completed: {} step(s) ok", report.outcomes.len());
+        println!(
+            "assemble-markdown target '{target}' completed: {} step(s) ok",
+            report.outcomes.len()
+        );
         ExitCode::SUCCESS
     } else {
         eprintln!("assemble-markdown target '{target}' failed");

crates/cph-model/Cargo.toml

@@ -10,3 +10,6 @@ serde = { workspace = true }
 # `preserve_order` keeps `[targets.*]` tables in TOML document order, which is
 # the declared order this crate exposes via `Lesson.targets` (ADR-0009).
 toml = { workspace = true, features = ["preserve_order"] }
+
+[dev-dependencies]
+tempfile = "3"

crates/cph-model/src/lib.rs

@@ -737,7 +737,11 @@ fn parse_target(name: String, value: toml::Value, diags: &mut Vec<Diagnostic>) -
 /// A well-formed but empty `covers = []` yields `Some(vec![])`: a target that
 /// explicitly renders no kind (every used kind then draws a `renderIgnored`
 /// warning) — distinct from an absent key.
-fn parse_covers(name: &str, value: toml::Value, diags: &mut Vec<Diagnostic>) -> Option<Vec<String>> {
+fn parse_covers(
+    name: &str,
+    value: toml::Value,
+    diags: &mut Vec<Diagnostic>,
+) -> Option<Vec<String>> {
     let items = match value {
         toml::Value::Array(items) => items,
         _ => {

crates/cph-model/tests/load.rs

@@ -279,16 +279,15 @@ fn malformed_target_config_is_non_fatal_with_schema_violations() {
 /// match exactly (ADR-0016's MVP rule).
 const CPH_VERSION: &str = env!("CARGO_PKG_VERSION");
 
-/// Write a one-part lesson into a temp dir, optionally with a `.cph-version`
-/// file. Returns the temp dir so the caller runs `load`.
-fn tmp_lesson_with_version(version: Option<&str>) -> PathBuf {
-    let mut p = std::env::temp_dir();
-    let nanos = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap()
-        .as_nanos();
-    p.push(format!("cph-version-test-{nanos}"));
-    std::fs::create_dir_all(&p).unwrap();
+/// Write a one-part lesson into a uniquely-created temp dir, optionally with a
+/// `.cph-version` file. `TempDir` owns cleanup so parallel tests cannot delete
+/// one another's fixture after a timestamp collision.
+fn tmp_lesson_with_version(version: Option<&str>) -> tempfile::TempDir {
+    let tmp = tempfile::Builder::new()
+        .prefix("cph-version-test-")
+        .tempdir()
+        .unwrap();
+    let p = tmp.path();
     std::fs::write(
         p.join("manifest.toml"),
         "[project]\nid = \"v\"\nname = \"v\"\n[info]\ntitle = \"v\"\n[[parts]]\nkind = \"segment\"\npath = \"segments/a\"\n",
@@ -301,14 +300,13 @@ fn tmp_lesson_with_version(version: Option<&str>) -> PathBuf {
     if let Some(v) = version {
         std::fs::write(p.join(".cph-version"), v).unwrap();
     }
-    p
+    tmp
 }
 
 #[test]
 fn cph_version_matching_is_not_a_diagnostic() {
     let tmp = tmp_lesson_with_version(Some(CPH_VERSION));
-    let (_lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (_lesson, diags) = load(tmp.path());
     assert!(
         diags.iter().all(|d| d.code != DiagCode::CphVersionMismatch),
         "a matching .cph-version must not warn, got {diags:?}"
@@ -318,8 +316,7 @@ fn cph_version_matching_is_not_a_diagnostic() {
 #[test]
 fn cph_version_mismatch_is_an_error_diagnostic() {
     let tmp = tmp_lesson_with_version(Some("99.99.99"));
-    let (lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (lesson, diags) = load(tmp.path());
     // The lesson still loads (so other defects could surface), but the
     // mismatch is an error diagnostic — which alone makes it illegal.
     assert!(lesson.is_some(), "a mismatch should not halt loading");
@@ -327,7 +324,11 @@ fn cph_version_mismatch_is_an_error_diagnostic() {
         .iter()
         .filter(|d| d.code == DiagCode::CphVersionMismatch)
         .collect();
-    assert_eq!(mm.len(), 1, "expected one CphVersionMismatch, got {diags:?}");
+    assert_eq!(
+        mm.len(),
+        1,
+        "expected one CphVersionMismatch, got {diags:?}"
+    );
     assert!(
         mm[0].message.contains("99.99.99") && mm[0].message.contains(CPH_VERSION),
         "diagnostic should name both versions, got: {}",
@@ -339,8 +340,7 @@ fn cph_version_mismatch_is_an_error_diagnostic() {
 fn cph_version_missing_is_skipped() {
     // No `.cph-version` file at all → skipped (ADR-0016 migration period; OPEN).
     let tmp = tmp_lesson_with_version(None);
-    let (_lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (_lesson, diags) = load(tmp.path());
     assert!(
         diags.iter().all(|d| d.code != DiagCode::CphVersionMismatch),
         "a missing .cph-version must be skipped, got {diags:?}"
@@ -350,8 +350,7 @@ fn cph_version_missing_is_skipped() {
 #[test]
 fn cph_version_empty_is_an_error() {
     let tmp = tmp_lesson_with_version(Some("   \n"));
-    let (_lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (_lesson, diags) = load(tmp.path());
     assert!(
         diags
             .iter()

crates/cph-schema/src/lib.rs

@@ -526,7 +526,11 @@ mod tests {
         // ADR-0015: `slides`/`transcript` are markdown content fields (`x-cph-content: "md"`),
         // distinct from the `.typ` content fields.
         let s = schema_for("segment").unwrap();
-        let slides = s.content_fields.iter().find(|f| f.name == "slides").unwrap();
+        let slides = s
+            .content_fields
+            .iter()
+            .find(|f| f.name == "slides")
+            .unwrap();
         assert_eq!(slides.ext, "md");
         assert!(!slides.required);
         let transcript = s

crates/cph-typst/src/embedded.rs

@@ -48,7 +48,8 @@ pub fn resolve_render_dir() -> PathBuf {
     ensure_extracted().unwrap_or_else(|_| {
         // Last-resort: extract under the OS temp dir. Still correct, just not
         // cached across processes.
-        let fallback = std::env::temp_dir().join(format!("cph-render-{}", env!("CARGO_PKG_VERSION")));
+        let fallback =
+            std::env::temp_dir().join(format!("cph-render-{}", env!("CARGO_PKG_VERSION")));
         let _ = extract_to(&fallback);
         fallback
     })

docs/adr/0017-agent-session-is-provider-bound.md

@@ -34,8 +34,36 @@ provider runtime cursor needed to continue a conversation. For Claude Code SDK,
 that cursor is the `result.session_id`; store it in `AgentSession.metadata` as
 `claudeSessionId` and pass it back to the next `query()` call as
 `options.resume`. Role is part of the session binding because role prompts and
-tool surfaces can differ even when the underlying model is the same; `/draft`
-and `/review` must not resume the same Claude runtime cursor by accident.
+tool surfaces can differ even when the underlying model is the same. A Feishu
+project group's active binding selects one Organization role for ordinary
+messages. Switching that selection routes future messages to the selected
+role's own session; it never mutates or merges provider cursors across roles.
+Work freezes the selected role when accepted so queued requests cannot drift
+after a later switch.
+
+Role definitions are Organization-scoped runtime data. A role bundle selects
+its default model, system prompt, tool allowlist and installed Agent skill
+versions. PostgreSQL is authoritative for role composition and skill metadata;
+skill bytes live in a content-addressed persistent store selected only by the
+recorded SHA-256 digest. Updating a role or binding skills takes effect without
+a Hub release or process restart. A change to the role's execution surface
+(model, prompt, tools, selected skill content) archives its active sessions so
+the next run cannot resume a provider context created under stale instructions;
+label and ordering-only changes preserve conversational continuity.
+
+Exactly one active role per Organization is the default used when a project
+group is first bound. The default is configuration data, not a hard-coded role
+name. Roles are selected through the Hub project console; role ids are not
+public slash commands. A project participant may change the group's shared
+selection only when both `agent.trigger` for the project and `role.trigger` for
+the target role authorize that actor. The selection affects every participant's
+future messages, while already accepted work keeps its frozen role.
+
+Hub slash commands are a closed control-plane protocol. Unknown commands fail
+explicitly and are never downgraded to Agent text. Claude SDK session commands
+are invoked only through typed Hub actions. In particular, compaction resumes
+the selected role session and sends the exact `/compact` prompt without
+prepending Feishu context.
 
 Environment variables:
 ```

docs/adr/0018-agent-execution-surface-bounded-by-workspace.md

@@ -86,18 +86,52 @@ discipline).
 ### The mechanism: SDK sandbox (OS-level, per agent process)
 
 The boundary is enforced by the Claude Code SDK's built-in sandbox
-(bubblewrap on Linux, seatbelt on macOS), configured via `query()` options in
+(bubblewrap plus `socat` on Linux, seatbelt on macOS), configured via `query()` options in
 `hub/src/agent/runner.ts`:
 
 - `sandbox.enabled: true` + `failIfUnavailable: true` — the agent process and
   its Bash subprocesses run sandboxed; if the sandbox can't start, `query()`
   emits an error and exits rather than running unsandboxed.
-- `sandbox.filesystem.allowWrite: [workspaceDir]` — writes confined to the
-  workspace (the ADR-0007 directory tree). Reads outside are allowed by
-  default (the agent may read system files for context), but sensitive host
-  paths are denied via `denyRead` (`/etc`, `/root`, `~/.ssh`, `~/.aws`, …).
-- `sandbox.filesystem.denyRead: SENSITIVE_READ_PATHS` — defense-in-depth;
-  extends via `CPH_SANDBOX_EXTRA_DENY_READ` for deployment-specific secrets.
+- `sandbox.allowUnsandboxedCommands: false` — a tool cannot opt out with the
+  SDK's `dangerouslyDisableSandbox` input. Claude Code 2.1.202 does not enforce
+  that option reliably, so a host-side `PreToolUse` hook also denies every
+  Bash request whose input explicitly sets `dangerouslyDisableSandbox: true`
+  before a process can start.
+- `sandbox.filesystem.denyRead: ["/"]` with `allowRead` for the canonical
+  current workspace and a small named system-runtime set — normal reads stay
+  in the run's workspace while `/bin`, shared libraries, CA certificates,
+  fonts and the configured `cph` executable remain available as the external
+  tool exception described above.
+- `sandbox.filesystem.allowWrite: [workspaceDir]` confines persistent host
+  effects to the canonical ADR-0007 workspace. Config/cache/home stay beneath
+  `.cph/agent-runtime/`; `TMPDIR`, `TMP`, `TEMP`, and `CLAUDE_CODE_TMPDIR` all
+  point at the workspace-local `.cph/t`. The workspace allocator uses stable
+  compact Organization/Project path segments, deployment requires a short
+  workspace root, and the canonical temp prefix fails fast above 56 bytes so
+  the SDK can append randomized `socat` bridge socket names without exceeding
+  Linux `sockaddr_un.sun_path`. Bubblewrap shadows non-allowlisted host trees
+  with disposable tmpfs mounts: a shell write there may succeed inside that
+  private namespace, but it cannot mutate the corresponding host path. The
+  Linux proof checks host state after the sandbox exits.
+- The SDK subprocess environment replaces rather than spreads `process.env`.
+  Only provider protocol variables and non-secret runtime variables cross the
+  boundary; database, Feishu and Hub session credentials never enter it.
+  `sandbox.credentials` denies provider token variables inside Bash and denies
+  configured credential files. `CPH_SANDBOX_EXTRA_DENY_READ` names additional
+  deployment-specific secret paths on the Hub side only.
+- `settingSources: []` and strict MCP configuration prevent an untrusted
+  workspace or service-user config from widening tools, hooks, MCP servers, or
+  sandbox paths.
+- Agent skills are Organization-scoped runtime configuration, not Hub release
+  assets. A controlled host-console installer imports each version into a
+  content-addressed persistent store and records its digest in PostgreSQL. A
+  role selects enabled Organization skills alongside its model, system prompt
+  and tool allowlist. Each run copies only those selected immutable versions
+  into a run-scoped plugin outside the project workspace; the sandbox exposes
+  that snapshot read-only and deletes it after the run. SDK-bundled skills and
+  filesystem setting sources remain disabled, so project `.claude` content
+  cannot register skills or widen tools. Requested skill versions are recorded
+  on `run.created`; SDK initialization remains authoritative loading evidence.
 - Network: open (see Open Questions).
 
 `bypassPermissions` is kept (headless server — no interactive prompts); the
@@ -146,21 +180,24 @@ as it upholds `AgentFileOp.Authorized`.
   tradeoff for CLI integration simplicity. This is revisitable — a network
   allowlist (`sandbox.network.allowedDomains`) can be added without a spec
   change if the threat model tightens.
-- **Sensitive read path coverage.** `SENSITIVE_READ_PATHS` in `runner.ts`
-  covers `/etc`, `/root`, `/var/log`, `/proc`, `/sys`, and common
-  credential dirs (`~/.ssh`, `~/.aws`, `~/.config/gcloud`, `~/.gnupg`).
-  Deployments with additional secret locations extend via
-  `CPH_SANDBOX_EXTRA_DENY_READ` (colon-separated). Whether this set should be
-  spec-pinned rather than implementation-chosen is `OPEN` — it's plumbing, not
-  a semantic divergence, so currently left out of `spec/`.
+- **Named system-runtime read set.** The implementation allows only the
+  platform runtime directories/files needed for shell, dynamic libraries,
+  fonts, TLS/DNS and `cph` execution in addition to the project workspace.
+  Expanding that set requires an explicit reviewed code change. Deployments
+  name credential files with `CPH_SANDBOX_EXTRA_DENY_READ`; these paths are
+  protected through the SDK credential layer and are not passed to the Agent
+  environment. The exact runtime set remains plumbing and is not spec-pinned.
 - **Shell command vocabulary.** Whether the agent's Bash is restricted to a
   whitelist (e.g. `cph`, `ls`, `grep`) or allowed arbitrary commands whose
   *effects* are then confined, is a policy choice left to implementation. The
   invariant governs effects either way; current choice is arbitrary commands
   (sandbox confines effects).
-- **Per-user rate limiting and token budgets.** Orthogonal to the surface
-  boundary (this ADR is about *what* the agent may touch, not *how often* it
-  may run). Deferred to a separate concern.
-- **Inbound file size / MIME limits.** The inbox lives inside the boundary,
-  but unbounded file size is a DoS vector distinct from the boundary
-  invariant. Deferred.
+
+## Related Decisions
+
+- **Rate limiting and token usage.** ADR-0022 requires multi-dimensional hard
+  request limits while keeping token thresholds as attributed soft alerts in
+  the initial release.
+- **Inbound file limits.** ADR-0022 requires hard file, attachment, archive,
+  Project-storage, and Organization-storage limits with explicit failure and
+  partial-file cleanup.

docs/adr/0020-organization-is-saas-tenant-root.md

@@ -83,8 +83,10 @@ separate control plane and must not be modeled as project `MANAGE`.
 
 ## Open Questions / Deferred
 
-- Full platform admin UI, billing, limits, and tenant suspension workflows are
-  deferred.
+- Full platform admin UI, billing, and tenant suspension workflows are
+  deferred. Production safety limits and workload brakes are no longer
+  deferred; ADR-0022 defines them without treating a workload brake as tenant
+  suspension.
 - Whether direct `USER` project grants should require active
   `OrganizationMembership` is deferred; explicit direct grants remain allowed
   for now.

docs/adr/0021-org-admin-project-onboarding.md

@@ -11,8 +11,8 @@ surface is not only the Feishu bot trigger path, but also the control planes
 around it:
 
 - platform staff need a private platform admin area to create and operate orgs;
-- customer org owners/admins need an org admin area for teams, roles, model
-  provider configuration, projects, folders, sessions, and usage accounting;
+- customer org owners/admins need an org admin area for teams, roles, provider
+  mode and BYOK configuration, projects, folders, sessions, and usage accounting;
 - ordinary teachers should be able to start work from the natural Feishu group
   flow without entering the web backend.
 
@@ -46,6 +46,19 @@ OAuth login, bot messages, and directory sync. App secrets are org-scoped
 secrets and must be encrypted at rest; business code should access them through
 a secret/connection resolver, not raw plaintext columns.
 
+Each Organization chooses one model-provider credential mode:
+
+- `BYOK`: an Organization OWNER/ADMIN supplies and manages that Organization's
+  provider key and base URL in the org admin area;
+- `PLATFORM_MANAGED`: a platform admin supplies and manages one distinct key
+  and base URL for that Organization in the platform admin area. Organization
+  users cannot read the plaintext credential.
+
+Provider connections remain Organization-scoped in both modes. Unrelated
+Organizations never share a process-global provider key. Provider secrets must
+be encrypted at rest and resolved at runtime through the same connection/secret
+boundary as Feishu credentials.
+
 Project management has two creation paths:
 
 - org owner/admin creates projects in the org web backend;
@@ -77,9 +90,10 @@ aggregation:
 Project permissions remain on `PROJECT` resources. Moving a project between
 folders does not change grants.
 
-Usage accounting is org-wise and project/folder aggregatable. It is not payment
-collection in the pilot because customers supply their own model provider API
-keys and base URLs.
+Usage accounting is org-wise and project/folder aggregatable for both provider
+modes. It is not payment collection in the pilot: BYOK spend belongs to the
+customer's provider account, while platform-managed spend is attributed to the
+Organization for operational reporting; commercial billing remains deferred.
 
 ## Consequences
 
@@ -89,8 +103,8 @@ keys and base URLs.
   explorer without creating a second ACL system.
 - Service code should expose project creation and chat binding as reusable
   backend operations so Feishu cards and future web APIs call the same rules.
-- Org role/model/provider/billing panels can be added on top of the org tenant
-  root without changing project authorization.
+- Org role/model/provider-mode panels and the platform-managed connection panel
+  can be added on top of the org tenant root without changing project authorization.
 
 ## Open Questions / Deferred
 
@@ -98,5 +112,6 @@ keys and base URLs.
   and readiness checks.
 - Folder-level permissions are deferred until there is a concrete customer need.
 - Self-serve org signup and payment collection are deferred beyond pilot.
-- The exact platform admin identity store and audit schema are separate from
-  this ADR and should be modeled before exposing the platform admin panel.
+- The platform admin identity, session, bootstrap, invitation, audit, and
+  offline recovery boundary is decided by ADR-0023 and must be implemented
+  before exposing the platform admin panel.

docs/adr/0022-layered-capacity-admission.md

@@ -0,0 +1,88 @@
+# ADR 0022: Layered Capacity Admission And Workload Safety
+
+## Status
+
+Accepted.
+
+## Context
+
+The initial SaaS deployment shares one production host across Organizations.
+The current Hub only has a one-run-per-project lock, a process-local five-item
+trigger queue, a five-minute queue wait, a global max-turn setting, and
+Fastify's default request-body limit. It has no global or Organization agent
+capacity, durable backpressure, fair scheduling, storage quotas, run wall-clock
+budget, multi-dimensional rate limiting, or emergency workload control.
+
+Leaving these limits implicit lets one Organization monopolize the host, lets
+accepted work disappear on restart, and makes overload look like success. A
+single global FIFO would preserve arrival order while still permitting one
+tenant to occupy every execution slot. Unlimited queues or automatic whole-run
+replay would hide overload and can duplicate tool side effects.
+
+## Decision
+
+Every mandatory capacity dimension has a non-optional **Platform Capacity
+Ceiling**. An Organization OWNER/ADMIN may configure a lower **Organization
+Policy Limit**; the **Effective Limit** is the lower value, or the platform
+ceiling when the Organization value is absent. There is no unlimited fallback.
+Production refuses to start when a mandatory platform ceiling is absent or
+invalid. Platform ceilings are changed through versioned deployment
+configuration in the initial release; live platform-admin editing is deferred
+until audited approval and rollback exist.
+
+Agent work uses a durable, bounded admission queue. Scheduling is fair across
+Organizations and FIFO within an Organization, while the existing one-active-
+run-per-Project rule remains. Capacity saturation queues only within effective
+length and wait limits. Queue overflow is an explicit `capacity_exhausted`
+response with retry guidance; an accepted request that waits too long becomes
+`EXPIRED`, is never started later, and notifies its initiator. Initiators may
+withdraw their own queued requests; Project `MANAGE` and Organization admins
+may cancel queued requests in their scope. Every transition is auditable.
+
+The platform never automatically replays a whole Agent Run after execution has
+started. Limited transport retries are allowed only before model output or tool
+side effects. Otherwise the run ends explicitly and a retry is a new user-
+initiated request. Feishu event identity and API idempotency prevent delivery
+retries from creating duplicate admissions.
+
+The first production gate requires hard limits for request rate/body size,
+agent concurrency, queue length/wait, files and attachment counts, archive
+expansion, Project and Organization storage, Organization entity counts, run
+wall time/turns/tool calls/output, individual tool duration, and execution
+process memory/CPU/process count. Reaching an entity or storage ceiling blocks
+new growth but does not delete, hide, or lock existing data; read, export,
+delete, and administrator remediation remain available.
+
+Admin surfaces require an authenticated user plus the corresponding platform
+or Organization administrator authorization. Authenticated Web traffic is
+limited by user, Organization, and endpoint class; signed Feishu traffic is
+limited by Organization application, chat, and sender; administrative writes
+use separate stricter buckets. Rejections and truncation are never silent.
+
+Token, provider-reported cost, run count, and duration are attributed by
+Organization, Project, Run, model, and Provider Connection. They are statistics
+and soft alerts in the initial release, not hard spend controls or settlement.
+Missing provider cost remains unknown rather than zero. Organization admins
+receive their alerts; platform admins additionally receive anomalous-cost
+alerts for platform-managed Provider Connections, not BYOK financial alerts.
+
+Platform administrators have audited, reason-bearing, reversible workload
+brakes at Organization and platform scope. `DRAIN` rejects new work and pauses
+queue starts while allowing active runs to finish. `STOP_NOW` also cancels
+active runs. These controls do not delete or suspend the Organization.
+
+Exact numerical ceilings remain open until production-like capacity tests
+measure the supported host and provider behavior. Defaults must be checked in,
+validated, observable, and proven at the release gate rather than guessed as
+domain constants.
+
+## Consequences
+
+- The current in-memory `TriggerQueue` and global-only agent settings are
+  implementation divergences, not acceptable production fallbacks.
+- Durable ingress, run/lock recovery, authorization, telemetry, and capacity
+  enforcement must share one Organization-scoped admission identity.
+- HTTP overload uses typed `429` responses with `Retry-After`; Feishu reports a
+  clear busy/expired/canceled outcome to the initiating user.
+- Capacity calibration is a release prerequisite and must include noisy-neighbor,
+  restart, queue expiry, file/archive, provider-stall, and workload-brake tests.

docs/adr/0023-platform-administrator-identity-and-audit.md

@@ -0,0 +1,136 @@
+# ADR 0023: Platform Administrator Identity And Audit Boundary
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0021 separated `/admin/platform` from Organization administration but left
+the exact platform identity store and audit schema open. That choice must be
+settled before exposing the panel: customer Feishu identities are scoped by
+their customer-owned applications, Organization membership must not confer
+platform authority, and a platform operator can create tenants, change
+connections, rotate credentials, or stop workloads across tenant boundaries.
+
+The current implementation is not the accepted platform model.
+`PlatformRoleAssignment` points at the customer-side `User`, includes a legacy
+`TEACHER` value, and has no platform login or guard. The current browser session
+is a seven-day stateless customer cookie, while `AuditEntry` is Project/Run
+oriented and deliberately best-effort. Reusing any of these would couple the
+two control planes and make immediate revocation or fail-closed privileged
+audit impossible.
+
+## Decision
+
+### Identity issuer and authority
+
+Platform Administrators authenticate only through one dedicated
+**Platform-owned Feishu Application**. It is separate from every Organization's
+Customer-owned Feishu Application and cannot grant Organization membership or
+Project permission.
+
+A `PlatformIdentity` is scoped by that platform application and its verified
+external Feishu user identity. It is not a customer `User`. The same person may
+also have customer identities, but there is no implicit link, shared session,
+or authority propagation between them.
+
+The initial platform control plane has exactly one standing role:
+`Platform Administrator`. There is no Platform Owner, Super Admin, or Teacher
+hierarchy. The first administrator has no permanent special authority after
+another administrator is active.
+
+### Bootstrap, invitation, and revocation
+
+The first Platform Administrator is granted through a checked-in, versioned,
+idempotent and transactional bootstrap database procedure. It accepts only a
+verified Platform Identity, refuses to run while any standing Platform
+Administrator is active, and atomically writes the grant plus its Platform
+Audit Entry. Ad-hoc manual `INSERT` statements and permanent environment
+allowlists are not supported bootstrap paths.
+
+Later administrators join through short-lived, single-use invitation links
+created by an active Platform Administrator. An invitation is bound to one
+specific Feishu account in the Platform-owned Feishu Application; possessing a
+link without authenticating as that account is insufficient. The invitation
+records its inviter and reason and becomes terminal after acceptance, expiry,
+or revocation. One active Platform Administrator approval is sufficient for the
+initial service.
+
+All standing Platform Administrators are peers. An administrator may invite or
+revoke another administrator, but the last active standing administrator cannot
+be revoked. Revocation immediately invalidates all Platform Sessions and all
+unclaimed invitations belonging to the revoked identity.
+
+### Platform Session and step-up authentication
+
+`Platform Session` is a separate, server-side, revocable session with a distinct
+cookie/token namespace. The browser receives an opaque random token; persistence
+stores only its hash and binds the session to a Platform Identity. Every request
+reloads the session and active grant. The session carries no Organization,
+membership, Project, or customer Feishu authority.
+
+Administrator lifecycle changes, platform or customer Feishu connection
+changes, platform-managed Provider credential changes, Organization lifecycle
+changes, workload-brake changes, and Emergency Platform Grant changes require
+recent Feishu OAuth reauthentication. Exact session/idle/step-up durations,
+CSRF, cookie and proxy/TLS mechanics belong to the production browser/session
+boundary decision; they must be configurable under checked maximums rather than
+silently inheriting the current seven-day customer cookie.
+
+### Platform audit
+
+`Platform Audit Entry` is a separate append-only record from customer Project
+and Agent Run audit. Every successful platform mutation and its audit entry
+commit in the same database transaction. If the audit entry cannot be written,
+the mutation fails. Platform audit is durable business evidence, not
+best-effort telemetry.
+
+An entry identifies the Platform Identity and Platform Session, action, target
+kind and identifier, affected Organization when applicable, mandatory reason
+for security-sensitive actions, request/correlation identity, outcome, time,
+and redacted before/after facts. It never stores credential, token, recovery-key
+or other plaintext secret material. The normal service path can append and read
+but cannot update or delete history.
+
+Platform Administrators can read the complete redacted platform audit.
+Organization Administrators can read a customer-safe projection of entries
+affecting their Organization, including Organization lifecycle, Feishu/provider
+connection, and workload-brake changes; they cannot read other Organizations,
+platform administrator lifecycle, recovery material, or internal security
+events.
+
+### Offline recovery and break-glass
+
+There is no standing break-glass web account or shared emergency password.
+Recovery uses a checked-in offline command from a controlled production-host
+console. The supported procedure requires both privileged host access and a
+recovery key held outside the host, environment file, and ordinary backup set.
+It also requires an incident identifier, reason, and one explicit recovery
+action.
+
+Recovery may revoke Platform Sessions, repair the Platform-owned Feishu
+connection, or issue an `Emergency Platform Grant` to a verified Platform
+Identity. An emergency grant is time-bounded, automatically expires, is not a
+standing role, and must be explicitly closed when recovery finishes. Every
+recovery action is represented in the append-only Platform Audit. Two factors
+are mandatory for the initial service; two separate human approvers are not.
+
+## Consequences
+
+- Platform identity, invitation, standing grant, server-side session, platform
+  audit, and emergency-grant persistence must be distinct from customer
+  `User`, `OrganizationMembership`, Project permission, and Project/Run audit.
+- `requirePlatformAdmin` must resolve the dedicated session and an active
+  standing or unexpired emergency grant on every request. It is never an
+  Organization-role override.
+- The legacy `PlatformRoleAssignment`/`TEACHER` schema is not a compatible
+  shortcut and must be migrated or replaced before the platform panel ships.
+- Bootstrap and recovery remain direct database/control-plane operations, but
+  only through versioned, asserted, auditable procedures rather than arbitrary
+  SQL.
+- Exact numeric invitation/session/step-up/emergency durations, audit retention
+  and export policy, recovery-key custody rotation, and future multi-party
+  approval remain follow-on operational decisions. They may not weaken the
+  identity separation, last-administrator protection, fail-closed audit, or
+  two-factor recovery invariants fixed here.

docs/adr/0024-local-secret-envelope-and-connection-resolution.md

@@ -0,0 +1,143 @@
+# ADR 0024: Local Secret Envelope And Connection Resolution
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0021 requires every customer Feishu application and model provider
+credential to belong to one Organization. It forbids a process-global provider
+credential and requires business code to obtain plaintext only through a
+resolver. Before the pilot can store real customer credentials, the deployment
+also needs a concrete encryption, rotation, recovery, writer-authority, and
+connection-identity contract.
+
+The initial production shape is one Hub service and one PostgreSQL database on
+a controlled host. An external KMS would add another remote availability and
+bootstrap dependency before the pilot has operational support for it. Keeping
+plaintext credentials in environment variables, configuration files readable
+by the service account, database columns, logs, or Agent environments is not an
+acceptable substitute.
+
+## Decision
+
+### Local master-key keyring
+
+The pilot uses a versioned **local master-key keyring**. Each key is a random
+256-bit key-encryption key (KEK) with a stable, non-secret key identifier. The
+keyring names exactly one active KEK and may retain previous KEKs while rotation
+is in progress. It is not stored in PostgreSQL, a release directory, an
+environment variable, or the ordinary database/workspace backup set.
+
+In production, the source keyring is owned by root, mode `0600`, and delivered
+to the unprivileged Hub service by systemd `LoadCredential`. Hub reads the
+read-only file under `CREDENTIALS_DIRECTORY`; production startup rejects a
+direct arbitrary file path. Development and tests may use an explicit local
+keyring file. Startup fails when the keyring is absent, malformed, contains an
+invalid key, has no active key, or names an active key that is not present.
+
+### Envelope format and binding
+
+Every immutable secret version receives a new random 256-bit data-encryption
+key (DEK). AES-256-GCM encrypts the secret payload with that DEK. AES-256-GCM
+also wraps the DEK with the keyring's active KEK. Nonces are random and never
+reused with the same key.
+
+The versioned envelope persists only algorithm/version metadata, the KEK
+identifier, nonces, authentication tags, the wrapped DEK, and ciphertext.
+Authenticated additional data binds both layers to the secret purpose,
+Organization identifier, Connection identifier, immutable secret-version
+identifier, and envelope version. Copying ciphertext between rows,
+Organizations, connections, purposes, or versions therefore fails
+authentication. Unknown algorithms or versions, missing KEKs, malformed
+envelopes, and failed authentication are hard errors; there is no fallback to
+another credential source.
+
+The encrypted payload is a versioned object specific to its connection type.
+For a Provider Connection, provider base URL and authentication material are
+inside the payload. For a Feishu Application Connection, application ID,
+application secret, verification/encryption material, and bot identity are
+inside the payload. HTTP responses, business records, logs, metrics, traces,
+audit facts, exceptions, and Agent process environments receive only redacted
+metadata, never those plaintext fields or a decrypted DEK.
+
+### Connection identity, lifecycle, and writer authority
+
+A connection has a stable opaque identifier and belongs to exactly one
+Organization. Secret versions are immutable; the connection points to one
+active version. Rotation creates and validates a new version, atomically moves
+the active pointer, and retains the previous encrypted version for rollback and
+audit until retention policy removes it. A connection is `DRAFT`, `ACTIVE`, or
+`DISABLED`; runtime resolution accepts only an active connection with a valid
+active secret version.
+
+There is one customer-owned Feishu Application Connection per Organization.
+Its provider-local user, chat, event, and callback identifiers are meaningful
+only together with that connection identity. Organization OWNER/ADMIN may
+create and rotate that customer-owned connection. Platform repair remains a
+separate stepped-up, reason-bearing, fail-closed-audited operation under
+ADR-0023, not an Organization API bypass.
+
+A model Provider Connection is unique by Organization and provider identity.
+In `BYOK` mode an Organization OWNER/ADMIN creates and rotates it. In
+`PLATFORM_MANAGED` mode only a stepped-up Platform Administrator may create or
+rotate it, with the platform mutation and redacted Platform Audit Entry in one
+transaction. Switching modes creates or activates the correctly governed
+connection version; it never re-labels plaintext or grants the other control
+plane write authority.
+
+### Resolver boundary
+
+Business code asks one connection resolver for an active connection using an
+explicit Organization, or using a Project that is first resolved to its active
+Organization. Provider resolution also requires the provider identity; Feishu
+resolution requires the intended Feishu Connection identity whenever external
+traffic enters the system. Resolution fails for missing, draft, disabled,
+cross-Organization, corrupt, or undecryptable records.
+
+Decrypted values exist only in the resolver result for the duration of the
+outbound provider/Feishu operation. They are not cached across connections,
+returned through administrative reads, passed to Agent tools, or written back
+to business rows. Production has no process-global Feishu or model-provider
+credential fallback.
+
+The Claude Agent child never receives an Organization's Provider credential.
+For each run, Hub opens a loopback-only proxy with a random, short-lived run
+capability. The child receives only that local endpoint and capability; Hub
+validates the capability, replaces it with the Organization credential at the
+upstream hop, refuses automatic redirects, and destroys the listener when the
+run ends. Agent sandbox credential rules hide even the run capability from
+tool subprocesses.
+
+### KEK rotation, backup, and recovery
+
+KEK rotation is staged: install a new KEK alongside the old keys, mark it
+active, rewrap each stored DEK under the new KEK without decrypting its payload,
+verify every envelope, then retire the old KEK only after the separately
+protected recovery copy and restore drill are current. Secret rotation and KEK
+rotation are distinct operations.
+
+A recoverable deployment requires both the ordinary database/workspace backup
+and a separately protected backup of the local keyring. Neither copy alone can
+recover plaintext credentials. Restore preflight verifies key identifiers and
+authenticates sample/all envelopes before traffic is enabled. A missing or
+incorrect keyring leaves the service unavailable and produces redacted
+diagnostics; operators must restore the correct keyring rather than reset or
+silently discard credentials.
+
+## Consequences
+
+- The database may be restored or copied without exposing plaintext secrets,
+  but losing the separately held keyring makes encrypted credentials
+  unrecoverable.
+- The service unit, installer, preflight, backup, restore, and rotation tooling
+  must treat the keyring as an explicit production dependency.
+- Administrative writes must accept secret material as write-only input and
+  return only connection state, version, timestamps, and redacted fingerprints.
+- Tests must prove cross-Organization/row substitution failure, wrong/missing
+  key failure, version rotation, absence of global fallback, and absence of
+  plaintext from persistence, responses, logs, and Agent environments.
+- External KMS/HSM adoption is deferred. It may replace the local KEK provider
+  without changing connection identity, immutable secret versions, AAD
+  binding, writer authority, or resolver behavior.

docs/adr/0025-alpha-silo-deployment.md

@@ -0,0 +1,57 @@
+# ADR 0025: Alpha Uses Managed Single-Organization Silos
+
+## Status
+
+Accepted for the supervised alpha.
+
+## Context
+
+The shared SaaS control plane, durable cross-Organization admission scheduler,
+and customer administration SPA are not complete. Real customer data must not
+depend on partially migrated application-level tenant filtering merely to start
+the supervised alpha.
+
+## Decision
+
+Each alpha Organization runs as a separately named Silo deployment. A Silo has
+one Hub systemd unit and service identity, one logical PostgreSQL database and
+database role, one workspace root, one environment/session secret, one local
+master-key keyring, one Feishu Application Connection, and Organization-local
+Provider Connections. The process is pinned by `HUB_SILO_ORGANIZATION_ID` and
+refuses startup unless its database contains exactly that one active
+Organization.
+
+Application releases remain shared and versioned; customer state does not live
+inside a release directory. Host-console bootstrap, upgrade, backup, restore,
+and KEK rotation always name a Silo instance explicitly. The alpha exposes no
+platform administration surface. Operators provision and support Silos through
+the controlled host console and retain an external change record.
+
+The process-local trigger queue is not an accepted-work queue in the Silo
+alpha. A busy project or instance rejects new work explicitly. Each instance
+has checked-in application limits and explicit systemd CPU, memory, and process
+ceilings. Workspace filesystem quotas are a host provisioning prerequisite.
+
+This topology changes deployment, not the semantic tenant root:
+`Organization` remains the tenant root and every authorization, secret, audit,
+Project, Team, and Run remains Organization-scoped. A later pooled or bridge
+adapter may host multiple Organizations only after ADR-0022's durable admission
+and platform control requirements are implemented.
+
+## Consequences
+
+- Cross-Organization data access is prevented by database credentials and
+  process/filesystem identity in addition to application authorization.
+- Provisioning, migrations, backups, health, and upgrades are repeated per
+  Silo and therefore must be automated before tenant count grows materially.
+- Cross-Organization fairness is not applicable inside one Silo; host cgroup
+  ceilings prevent one Silo from exhausting the machine.
+- Database/workspace backups and the keyring/environment recovery copy are
+  produced to distinct protected destinations and must pass a restore drill.
+
+## Deferred
+
+- Shared platform identity and administration from ADR-0023.
+- Pooled durable fair scheduling from ADR-0022.
+- Customer org administration SPA and self-service onboarding.
+- Docker or Kubernetes as an alternative deployment adapter.

docs/para-26071100-feishu-setup.md

@@ -0,0 +1,221 @@
+# Educraft 组织接入与飞书应用配置指南
+
+本文供准备接入 Educraft Alpha Silo 的学校、教培机构和组织管理员使用。完成本文后，请把末尾的“部署信息交付单”交给 Educraft 部署人员；我们会为组织创建独立的服务账号、数据库、运行目录和域名入口。
+
+> **安全提醒：** App Secret、模型 Provider Token 属于密钥，禁止粘贴到飞书群、普通云文档、工单正文或截图中。请只通过双方约定的安全渠道传递。
+
+## 1. 双方分别负责什么
+
+| 角色 | 负责事项 |
+| --- | --- |
+| 组织管理员 | 创建企业自建应用、启用机器人、开通最小权限、配置事件、回调和 OAuth 重定向 URL、发布应用、提供 OWNER 身份 |
+| Educraft 部署人员 | 分配组织 slug 和域名、部署独立 Silo、加密保存应用及模型密钥、初始化 OWNER、联调和验收 |
+| 试点 OWNER | 把机器人加入试点群、创建或绑定项目、组织首轮验收 |
+
+## 2. 创建企业自建应用
+
+1. 打开[飞书开放平台开发者后台](https://open.feishu.cn/app)。
+2. 在目标企业下点击“创建企业自建应用”。应用名称建议填写“Educraft + 组织简称”。
+3. 进入“凭证与基础信息”，记录 App ID 和 App Secret。
+4. 进入“添加应用能力”，添加并启用“机器人”。
+
+![凭证与基础信息页面；App Secret 默认以星号隐藏](assets/feishu-setup/01-credentials.png)
+
+App ID 通常以 `cli_` 开头，可以写入交付单。App Secret 必须通过安全渠道单独发送。Bot Open ID 不需要管理员手工查找；部署程序会用 App ID/App Secret 调用 Bot Info API 获取并校验归属。
+
+## 3. 开通最小权限
+
+进入“权限管理”，点击“开通权限”，搜索并申请以下应用身份权限。控制台中文名称可能调整，请优先核对 scope。
+
+| 用途 | Scope |
+| --- | --- |
+| 接收群聊中 @ 机器人的消息 | `im:message.group_at_msg:readonly` |
+| 以应用身份发送消息 | `im:message:send_as_bot` |
+| 读取触发消息和线程上下文 | `im:message:readonly` |
+| 获取与上传图片或文件 | `im:resource` |
+| 添加、删除消息表情回复 | `im:message.reactions:write_only` |
+| 获取用户基本信息 | `contact:user.base:readonly` |
+| 获取用户基本资料 | `contact:user.basic_profile:readonly` |
+| 通过手机号或邮箱查询 OWNER Open ID | `contact:user.id:readonly` |
+
+### 批量导入权限（推荐）
+
+在“权限管理”页面点击“批量处理 → 导入”，粘贴以下 JSON 后确认。导入只会新增本次列出的权限，不会删除或影响应用已经申请、开通的其他权限。
+
+```json
+{
+  "scopes": {
+    "tenant": [
+      "im:message.group_at_msg:readonly",
+      "im:message:send_as_bot",
+      "im:message:readonly",
+      "im:resource",
+      "im:message.reactions:write_only",
+      "contact:user.base:readonly",
+      "contact:user.basic_profile:readonly",
+      "contact:user.id:readonly"
+    ],
+    "user": []
+  }
+}
+```
+
+Educraft 机器人以应用身份调用上述 API，因此这些 scope 全部放在 `tenant`，不要为了省事把相同权限重复放进 `user`。
+
+![权限管理入口与已开通权限列表](assets/feishu-setup/02-permissions.png)
+
+如果 API 调试台提示缺少更细粒度权限，请把错误提示和发生时间截图给部署人员。不要自行开通通讯录全量读取等超出本表的权限。
+
+## 4. 配置事件与卡片回调
+
+进入“事件与回调”。
+
+1. 在“事件配置”中将订阅方式设为“使用长连接接收事件”。
+2. 添加事件“接收消息” `im.message.receive_v1`。
+3. 添加事件“解散群” `im.chat.disbanded_v1`，用于立即归档该群的项目绑定。
+4. 添加事件“机器人被移出群” `im.chat.member.bot.deleted_v1`，用于立即归档该群的项目绑定。
+5. 在“回调配置”中同样选择长连接。
+6. 添加回调“卡片回传交互” `card.action.trigger`，用于审批、运行中断和项目创建/绑定按钮。
+
+![长连接与消息事件配置](assets/feishu-setup/03-events.png)
+
+![卡片交互回调配置](assets/feishu-setup/04-callbacks.png)
+
+这里不需要填写公网 Event Callback URL。Educraft Hub 使用飞书官方 SDK 的长连接模式。
+
+## 5. 配置用户 OAuth 重定向 URL（必需）
+
+普通群成员首次使用前，需要通过飞书 OAuth 建立其在本应用下的用户身份。进入“安全设置 → 重定向 URL”，添加组织专属 callback：
+
+```text
+https://<organization-slug>.educraft.paradigm-edu.net/auth/feishu/callback
+```
+
+例如组织 slug 为 `example-school`：
+
+```text
+https://example-school.educraft.paradigm-edu.net/auth/feishu/callback
+```
+
+![在安全设置中添加组织专属 OAuth 重定向 URL](assets/feishu-setup/05-security.png)
+
+必须使用 Educraft 部署人员最终确认的 slug；不要直接照抄示例。该 URL 用于 OAuth 返回并创建应用作用域下的飞书用户身份，不代表当前已经开放组织管理台。
+
+组织专属 OAuth 同时完成身份建立和入组：首次成功登录的用户会自动成为当前 Organization 的 `MEMBER`，回到群聊即可使用。`OWNER` 和 `ADMIN` 仍只能由部署人员或管理员显式授予；曾被移除的成员重新登录不会自动恢复资格。
+
+## 6. 发布并安装应用
+
+1. 进入“版本管理与发布”，点击“创建版本”。
+2. 将应用可用范围至少覆盖试点 OWNER 和试点群成员。
+3. 提交企业管理员审核并发布。
+4. 发布成功后，将机器人加入准备试用的群。
+
+![版本管理与发布页面](assets/feishu-setup/06-publish.png)
+
+仅保存开发配置但未发布时，新增权限、事件和可用范围通常不会对试点用户生效。
+
+## 7. 获取首位 OWNER 身份
+
+首次部署必须指定一位组织 OWNER。OWNER 是 Educraft 组织内的初始管理员，不等同于飞书应用所有者；两者可以是同一个人，也可以不同。部署所需的 Open ID 必须由本次创建的企业自建应用查询，因为同一用户在不同应用下的 Open ID 不同，不能复用其他应用查到的值。
+
+### 7.1 确认 OWNER
+
+先确认哪一位企业成员将担任 OWNER。记录其飞书显示名称，并准备在飞书的成员选择器中按姓名找到本人。若企业内有同名成员，选择前须通过部门等信息核对身份。
+
+### 7.2 开通查询权限和数据范围
+
+确认应用已开通上文列出的用户基本信息和用户 ID 权限。如果使用上文的批量导入 JSON，这些权限已包含在内。
+
+应用的通讯录数据范围还必须覆盖这位 OWNER。最小做法是把 OWNER 加入应用可用范围；不需要为此开放全企业通讯录。
+
+### 7.3 在官方接口页面获取 Open ID
+
+1. 打开飞书开放平台的[“获取单个用户信息”接口页面](https://open.feishu.cn/document/server-docs/contact-v3/user/get)。如果使用带 `appId` 参数的页面链接，可以直接进入对应应用；本文不提供固定 App ID，请在页面顶部选择本组织刚创建的企业自建应用，并核对 App ID 与交付单一致。
+2. 找到路径参数 `user_id`，点击参数输入框旁的“获取”。
+3. 在成员选择器中找到并选择 OWNER；如有同名成员，依据部门等信息确认本人。
+4. ID 类型选择 `open_id`。将选择器返回的值填入 `user_id`，并保持查询参数 `user_id_type=open_id`。
+5. 以应用身份（`tenant_access_token`）调用接口，核对成功响应中 `data.user.name` 与 OWNER 本人一致。
+6. 复制完整的 `data.user.open_id` 交给 Educraft 部署人员。Open ID 通常以 `ou_` 开头。
+
+参数旁的“获取”是飞书文档调试台提供的成员选择功能，不是要求管理员预先知道 Open ID。不要复用其他应用查到的 Open ID；同一用户在不同应用下的 Open ID 不同。若无法选择成员或接口调用失败，依次检查：页面当前选择的 App ID、应用可用范围和通讯录数据范围是否覆盖 OWNER、用户基本信息与用户 ID 权限是否已开通并随应用版本发布。
+
+### 7.4 核对并交付
+
+交付前完成以下检查：
+
+- 返回用户的姓名与 OWNER 本人一致；
+- Open ID 来自本次组织的这一个 App ID；
+- Open ID 完整复制，没有空格或省略号；
+- 显示名称使用组织希望在 Educraft 中展示的姓名；
+- Union ID 不是必填项，查不到可以留空。
+
+最终向部署人员提供：
+
+```text
+OWNER Open ID：ou_...
+OWNER 显示名称：
+OWNER Union ID：（可选）
+用于查询的 App ID：cli_...
+```
+
+Open ID 和显示名称可以放在普通交付单中；不要把 App Secret 一起粘贴进去。
+
+## 8. 部署信息交付单
+
+请复制下面的模板填写。标注“安全渠道”的字段不要与普通字段放在同一条群消息或云文档中。
+
+```text
+【组织信息】
+组织正式名称：
+组织简称：
+期望 organization slug：（小写字母、数字和连字符，例如 example-school）
+期望机器人显示名称：
+
+【飞书应用】
+App ID：cli_...
+App Secret：（通过安全渠道单独发送）
+应用已发布：是 / 否
+机器人能力已启用：是 / 否
+消息事件（含解散群、机器人被移出群）和卡片回调已配置：是 / 否
+OAuth 重定向 URL 已配置：是 / 否
+
+【首位 OWNER】
+OWNER Open ID：ou_...
+OWNER 显示名称：
+OWNER Union ID：（可选）
+
+【试点范围】
+试点群名称：（可选，用于验收定位）
+初始 Team 名称：（可选；没有 Team 不影响首次部署）
+预计试用人数：
+
+【模型配置】
+Provider 名称：（例如 OpenRouter）
+Provider Base URL：
+Provider Token：（通过安全渠道单独发送）
+启用的模型 ID：
+```
+
+Educraft 部署人员收到信息后，会回传最终 organization slug、访问域名、部署窗口和验收时间。若期望 slug 已被占用或不符合命名规则，会在部署前协调调整。
+
+## 9. 上线验收
+
+部署人员通知服务就绪后，由 OWNER 完成：
+
+1. OWNER 在试点群中 @机器人发送一条纯文本消息。
+2. 如果群尚未绑定项目，确认机器人返回项目创建/绑定卡片。
+3. 创建项目后再次 @机器人，确认出现处理状态、流式卡片和最终回答。
+4. 选择一位非 OWNER 试点成员完成 OAuth 登录，确认其自动以 MEMBER 身份加入组织。
+5. 该成员在同一群中 @机器人，确认能够进入已绑定项目。
+6. 测试一个小文件附件、一次运行中断，以及一个需要生成文档的任务。
+
+出现问题时，请保留发生时间、群名、消息截图和飞书 request/log ID。截图前确认其中不包含 App Secret、Provider Token 或其他密钥。
+
+## 10. Alpha 阶段边界
+
+- 每个组织运行在独立的系统用户、服务实例、数据库和持久化目录中。
+- 组织的 role、system prompt、tools 和 skills 是运行时配置，不需要跟随版本发布。
+- 同一项目同一时间只执行一个任务，避免并发修改同一个 workspace；组织级并发上限由部署配置决定。
+- 当前由 Educraft 人工创建组织、OWNER、Provider Connection 和初始 Team，并通过服务器上的受控管理命令运维；组织管理台尚未开放。
+- 非 OWNER 试点成员通过组织专属 OAuth 首次登录后自动成为 MEMBER；OWNER/ADMIN 提权和被移除成员的恢复仍需人工操作。
+- Alpha 不提供开放注册、自助密钥管理或跨组织资源共享。

hub/.env.example

@@ -1,14 +1,14 @@
 # Hub runtime configuration. Copy to .env and fill in.
 
+# Production preflight requires the explicit production runtime mode.
+NODE_ENV="production"
+
 # PostgreSQL connection string.
 DATABASE_URL="postgresql://paradigm:paradigm@127.0.0.1:5432/paradigm"
 
-# Claude Code SDK auth via OpenRouter's Anthropic Skin.
-# ANTHROPIC_AUTH_TOKEN = your OpenRouter API key (https://openrouter.ai/keys)
-# ANTHROPIC_API_KEY must be explicitly empty to avoid auth conflicts.
-ANTHROPIC_BASE_URL="https://openrouter.ai/api"
-ANTHROPIC_AUTH_TOKEN=""
-ANTHROPIC_API_KEY=""
+# Provider URL and credentials are write-only Organization Provider Connection
+# data in PostgreSQL, encrypted by ADR-0024. Process-global ANTHROPIC_* provider
+# settings are rejected in production and must not be added here.
 
 # Model override (optional). Defaults to anthropic/claude-sonnet-5.
 # Use an OpenRouter model slug, for example:
@@ -18,21 +18,34 @@ ANTHROPIC_API_KEY=""
 # ANTHROPIC_DEFAULT_OPUS_MODEL="z-ai/glm-4.7"
 # ANTHROPIC_DEFAULT_HAIKU_MODEL="z-ai/glm-4.6"
 
-# Agent run policy (optional). Defaults to 25.
+# Alpha Silo safety limits. max turns may use its default; every other value is
+# mandatory in production and should be calibrated on the target host.
 # HUB_AGENT_MAX_TURNS=25
+HUB_AGENT_MAX_CONCURRENT_RUNS="1"
+HUB_AGENT_MAX_RUN_SECONDS="900"
+HUB_HTTP_BODY_LIMIT_BYTES="1048576"
+HUB_MAX_FILES_PER_MESSAGE="8"
+HUB_MAX_FILE_BYTES="26214400"
+HUB_HTTP_REQUESTS_PER_MINUTE="120"
+HUB_FEISHU_EVENTS_PER_MINUTE="120"
 
-# System-managed root for project workspaces created from org admin / Feishu onboarding.
-HUB_PROJECT_WORKSPACE_ROOT="./data/project-workspaces"
+# Persistent system-managed root for project workspaces. Production must use an
+# absolute path outside the deployment/release tree; install_service.sh defaults
+# to this path and rejects any overlap before installing the unit.
+HUB_PROJECT_WORKSPACE_ROOT="/var/lib/cph-hub/workspaces"
 
-# Feishu (lark) bot credentials.
-FEISHU_APP_ID=""
-FEISHU_APP_SECRET=""
-FEISHU_BOT_OPEN_ID=""
+# This process is pinned to exactly one Organization. Feishu credentials are
+# resolved from that Organization's encrypted ACTIVE connection.
+HUB_SILO_ORGANIZATION_ID=""
+HUB_SYSTEMD_UNIT="cph-hub-example.service"
 
-# Path to the `cph` binary (ADR-0016). Defaults to `cph` on PATH.
-# CPH_BIN="/usr/local/bin/cph"
+# Absolute path to the `cph` binary (ADR-0016). Production preflight requires
+# the file to be executable and `cph --version` to succeed.
+CPH_BIN="/usr/local/bin/cph"
 
-# Hub server port. Defaults to 8788.
+# Hub bind address and port. Production defaults to loopback for a local TLS
+# reverse proxy; both values are validated and honored by the HTTP server.
+HOST="127.0.0.1"
 PORT=8788
 
 # --- Org admin web (ADR-0021) ---------------------------------------------
@@ -40,7 +53,7 @@ PORT=8788
 # redirect_uri = ${HUB_PUBLIC_BASE_URL}/auth/feishu/callback
 # Configure the same callback URL in the Feishu developer console under
 # Security Settings → Redirect URLs.
-HUB_PUBLIC_BASE_URL="http://127.0.0.1:8788"
+HUB_PUBLIC_BASE_URL="https://hub.example.com"
 
 # HMAC secret for signed session + OAuth state cookies. Generate with:
 #   openssl rand -base64 32
@@ -49,3 +62,7 @@ HUB_SESSION_SECRET=""
 # Optional OAuth scope (space-separated). Default: contact:user.base:readonly
 # Apply matching scopes in the Feishu developer console.
 # HUB_OAUTH_SCOPE="contact:user.base:readonly"
+
+# Local development/tests only. Production must not set this: systemd injects
+# the fixed cph-secret-keyring credential through CREDENTIALS_DIRECTORY.
+# HUB_SECRET_KEYRING_FILE="/absolute/path/to/dev-keyring.json"

hub/deploy/NEW_SILO_RUNBOOK.md

@@ -0,0 +1,133 @@
+# New Alpha Silo runbook
+
+Use this runbook for a new Organization. A Silo is not merely another row in
+the existing database: it has an independent PostgreSQL role/database, Linux
+service identity, systemd unit, secret directory/keyring, workspace, skill
+store, loopback port, domain, Feishu app and provider credential.
+
+The repeatable entry point is:
+
+```sh
+bash hub/deploy/new_silo.sh
+```
+
+After collecting the Organization inputs, the wizard shows the assigned
+resources and asks once before applying them directly over SSH. The generated
+bundle is only a root-secret-safe retry and audit checkpoint; the operator does
+not execute it manually during the normal path.
+
+It gathers values and writes a private deployment bundle below
+`~/.cph-silo-plans/<instance-id>/`. The directory and all generated files are
+mode `0700`/`0600`. Never commit, paste into chat, or copy that directory into
+an immutable release. Run the wizard once per Organization; do not edit a copy
+from another Organization.
+
+## Inputs to collect
+
+The wizard derives the instance/Organization id from the slug, uses the current
+release and managed Alpha defaults, connects to the managed host (currently
+`39.107.254.4`), derives
+`https://<organization-slug>.educraft.paradigm-edu.net` from the wildcard DNS,
+and allocates an unused loopback port plus short workspace path by inspecting
+existing Silo environments, listening sockets and workspace paths over
+read-only SSH. The Organization administrator supplies:
+
+- Organization display name and slug;
+- Feishu App ID and App Secret (the wizard resolves the bot Open ID);
+- the first OWNER's Open ID and display name;
+- an Organization-exclusive OpenRouter token.
+
+Everything else is platform-managed or derived: instance/Organization id,
+server, SSH settings, release, resource ceilings, database coordinates and
+generated password, domain, port, workspace, provider/base URL, model/role,
+curated skills, concurrency, request/file limits and the managed Mihomo proxy
+environment. `NODE_USE_ENV_PROXY=1` is required on Node.js 24 so Hub's built-in
+`fetch` actually uses that proxy; merely setting `HTTP_PROXY`/`HTTPS_PROXY` is
+not sufficient.
+
+The Feishu app is scoped to this Silo. OAuth users authenticated by that app are
+automatically admitted to this Organization; OWNER remains the initial
+privileged membership used for controlled administration and bootstrap. An
+empty initial team list does not block the Alpha.
+
+To target a replacement platform-managed host, the platform operator may set
+`CPH_ALPHA_HOST`, `CPH_ALPHA_DEPLOY_USER`, `CPH_ALPHA_SSH_PORT` and
+`CPH_ALPHA_BASE_DOMAIN` before running the wizard. These are fleet controls,
+not Organization setup questions.
+
+Obtain a person's Open ID from the Feishu user-get documentation page by
+clicking the `user_id` value picker and selecting the person. Configure the
+redirect URL shown by the generated `OPERATE.md`; it is required for first-time
+OAuth admission.
+
+## Host prerequisites
+
+Before the first Silo on a host, install Node.js 24+, npm, rsync, PostgreSQL
+server/client, `pg_isready`, systemd, bubblewrap, socat, `runuser`, `setpriv`,
+`pg_dump`, tar, sha256sum, Nginx, Certbot, Typst and a compatible `cph` binary.
+Configure outbound proxying independently at host/service level and verify both
+GitHub and the selected model provider through it. The wizard does not install
+or select proxy nodes.
+
+Use a deployment account with only the required passwordless sudo operations.
+The application itself always runs as the installer-created non-root
+`cph-<instance-id>` user. PostgreSQL must have a separate login role and logical
+database per Silo even when all Silo databases share one PostgreSQL server.
+
+## Execute a generated bundle
+
+Open the bundle's `OPERATE.md` and perform its numbered gates in order:
+
+1. DNS, Feishu redirect URL, permissions and event subscription.
+2. Dedicated PostgreSQL role and database.
+3. Immutable release publication. Exit 78 is expected only when the first
+   installer call seeds this instance's keyring and environment template.
+4. Root-owned secret installation and off-host keyring recovery copy.
+5. Prisma migration, stopped service installation and idempotent bootstrap.
+6. Explicit runtime role/skill installation.
+7. Nginx/TLS, service start, internal/external health and Feishu acceptance.
+8. First off-host backup.
+
+Every command must fail fast. Do not add `|| true` around install, migration,
+bootstrap, Nginx validation, health, or backup checks. If a check fails, retain
+the unit logs and the exact failed stage before changing configuration.
+
+## Default runtime role and skills
+
+Roles and skills are dynamic Silo state, not release contents. The release
+contains only the management CLI. Stage approved skill directories on the host
+and install them with `agent_config.sh`; PostgreSQL records role definitions and
+skill selections while the versioned skill content lives in the Silo state
+directory and is included in backups.
+
+For the current Alpha, upsert the default role with the agreed education
+assistant system prompt, model selection, tools policy and approved skill list.
+Keep the prompt in a root-controlled staging file, pass it via
+`--system-prompt-file`, then verify with `agent_config.sh list`. Do not sync an
+operator's entire personal skill directory: each enabled skill must be reviewed
+and named explicitly. Typst being installed on the host and the Typst skill
+being enabled are separate gates.
+
+## Acceptance gate
+
+A Silo is ready only when all of the following pass:
+
+- its systemd service is active and both loopback and TLS health endpoints pass;
+- startup preflight sees exactly the configured Organization plus active Feishu
+  and provider connections;
+- OWNER completes OAuth and can interact with the bot;
+- a non-OWNER completes OAuth and can interact with the same app/Organization;
+- two Feishu groups bind distinct projects/sessions;
+- a restart preserves persisted session cursor behavior;
+- the configured provider/model succeeds through the host proxy;
+- every enabled skill is listed, and a Typst task succeeds if Typst is enabled;
+- the first business backup and separate recovery backup are stored off-host.
+
+## Rollback boundary
+
+For a failed code release, point only this instance back to the previous
+immutable release and rerun its installer with the same instance parameters.
+Do not roll back a database after migrations unless that release's documented
+database compatibility permits it. Preserve the environment, keyring, database,
+workspace and skill store. For destructive recovery, stop traffic and use the
+separate restore procedure; never substitute another Silo's state.

hub/deploy/README.md

@@ -0,0 +1,199 @@
+# Alpha Silo service installation
+
+For a brand-new Organization, start with the repeatable wizard and end-to-end
+operator runbook:
+
+```sh
+bash hub/deploy/new_silo.sh
+```
+
+See [NEW_SILO_RUNBOOK.md](NEW_SILO_RUNBOOK.md). The remainder of this document
+describes the individual installer and maintenance primitives used by the
+generated bundle.
+
+The supervised alpha runs one Organization per named Silo. The supported host
+has systemd, PostgreSQL, Node.js 24+, `pg_isready`, `runuser`, `setpriv`,
+bubblewrap, `socat`, `pg_dump`, `tar`, `sha256sum`, and a compatible `cph`.
+
+Each Silo needs its own logical database/role, instance id, service user,
+environment, keyring, workspace, port, domain, and host filesystem quota.
+Application code lives in immutable versioned directories under
+`/srv/curriculum-project-hub/releases/`; releases contain no customer state.
+
+## Install an instance
+
+```sh
+sudo BASE=/srv/curriculum-project-hub \
+  HUB_DIR=/srv/curriculum-project-hub/releases/<release-id>/hub \
+  INSTANCE_ID=org-a \
+  WORKSPACE_ROOT=/w/997 \
+  PORT=8788 \
+  MEMORY_MAX=16G CPU_QUOTA=400% TASKS_MAX=512 \
+  bash /srv/curriculum-project-hub/releases/<release-id>/hub/deploy/install_service.sh
+```
+
+The first invocation creates root-owned `0600` files below
+`/srv/curriculum-project-hub/.secrets/org-a/` and exits 78. Copy the keyring to
+a separately protected recovery destination and fill every blank in
+`platform.env`. Before rerunning the installer, migrate the empty Silo database:
+
+```sh
+sudo bash -c '
+  set -euo pipefail
+  set -a; . /srv/curriculum-project-hub/.secrets/org-a/platform.env; set +a
+  node /srv/curriculum-project-hub/releases/<release-id>/hub/node_modules/prisma/build/index.js \
+    migrate deploy \
+    --schema /srv/curriculum-project-hub/releases/<release-id>/hub/prisma/schema.prisma
+'
+```
+
+Then rerun the installer so it can complete preflight and install the stopped
+unit. Resource numbers above are examples, not defaults; the operator must set
+measured ceilings explicitly.
+
+The installer refuses overlapping release/persistent paths, root service users,
+missing Node/cph/PostgreSQL/bubblewrap prerequisites, invalid credentials, and
+unreachable database roles. The installed unit is `cph-hub-org-a.service` and
+uses systemd `LoadCredential`; the service user cannot read the source keyring.
+
+Default state paths are:
+
+```text
+/var/lib/cph-hub/org-a/home
+/var/lib/cph-hub/org-a/state
+/w/997
+/var/cache/cph-hub/org-a
+```
+
+Organization Agent roles and skills are runtime configuration. Skill versions
+are stored below the Silo state directory (`state/skills`) and are included in
+`backup_silo.sh` as `agent-skills.tar`; PostgreSQL stores role bundles, skill
+metadata and role-to-skill selection. Operate them as the Silo service user so
+content ownership remains correct:
+
+```sh
+sudo INSTANCE_ID=org-a \
+  ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
+  bash hub/deploy/agent_config.sh install-skill \
+  --organization org-a --source /staging/typst --version 1
+sudo INSTANCE_ID=org-a \
+  ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
+  bash hub/deploy/agent_config.sh upsert-role \
+  --organization org-a --role draft --label 草稿 --tools-json null
+sudo INSTANCE_ID=org-a \
+  ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
+  bash hub/deploy/agent_config.sh set-role-skills \
+  --organization org-a --role draft --skills outline,lesson-project,typst
+sudo INSTANCE_ID=org-a \
+  ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
+  bash hub/deploy/agent_config.sh list --organization org-a
+```
+
+`--tools-json null` means the full registered tool surface; `[]` means no
+ordinary tools. SDK-bundled skills and workspace/user setting sources remain
+disabled regardless of runtime configuration.
+
+## Bootstrap the only Organization
+
+Prepare a root-owned `0600` JSON file containing
+`organization`, `owner`, `feishu`, `provider`, and optionally `teams`. Secrets
+never appear in command-line arguments.
+
+```json
+{
+  "organization": { "id": "org_a", "slug": "org-a", "name": "Org A" },
+  "owner": { "openId": "ou_owner", "displayName": "Owner" },
+  "feishu": {
+    "appId": "cli_xxx",
+    "appSecret": "...",
+    "botOpenId": "ou_bot"
+  },
+  "provider": {
+    "providerId": "openrouter",
+    "baseUrl": "https://openrouter.ai/api",
+    "authToken": "..."
+  },
+  "teams": [{ "slug": "teachers", "name": "Teachers" }]
+}
+```
+
+```sh
+sudo bash -c '
+  set -euo pipefail
+  set -a; . /srv/curriculum-project-hub/.secrets/org-a/platform.env; set +a
+  node /srv/curriculum-project-hub/releases/<release-id>/hub/dist/deployment/bootstrap-silo-cli.js \
+    --config-file /root/org-a-bootstrap.json \
+    --keyring-file /srv/curriculum-project-hub/.secrets/org-a/secret-keyring.json
+'
+```
+
+Bootstrap encrypts and activates both connection records and is rerunnable when
+provider configuration fails after Organization creation. Hub startup refuses
+to become healthy unless the sole Organization, active provider, active Feishu
+application, and Feishu listener are ready. Delete the bootstrap input after
+the recovery copy is current. Production has no process-global Feishu/provider
+credential fallback.
+
+## Start and rotate
+
+```sh
+sudo systemctl start cph-hub-org-a.service
+sudo systemctl status cph-hub-org-a.service
+```
+
+KEK rotation requires the named unit to be stopped. Source its instance env so
+`HUB_SYSTEMD_UNIT` and `DATABASE_URL` identify the same Silo:
+
+```sh
+sudo bash -c '
+  set -euo pipefail
+  systemctl stop cph-hub-org-a.service
+  set -a; . /srv/curriculum-project-hub/.secrets/org-a/platform.env; set +a
+  node /srv/curriculum-project-hub/hub/dist/deployment/rotate-secret-kek.js \
+    --keyring-file /srv/curriculum-project-hub/.secrets/org-a/secret-keyring.json
+  systemctl start cph-hub-org-a.service
+'
+```
+
+## Backup and restore drill (recommended after the demo is running)
+
+Stop the unit and run `backup_silo.sh` with distinct root-owned destinations:
+
+```sh
+sudo INSTANCE_ID=org-a \
+  ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
+  KEYRING_FILE=/srv/curriculum-project-hub/.secrets/org-a/secret-keyring.json \
+  BUSINESS_BACKUP_DIR=/backup/business \
+  RECOVERY_BACKUP_DIR=/separate-recovery \
+  bash hub/deploy/backup_silo.sh
+```
+
+The business set contains the PostgreSQL custom dump, workspace archive and
+`agent-skills.tar`. The
+separate recovery set contains the keyring and environment. Both include
+checksums; neither destination may be the live host's only disk.
+
+Restore into a separate drill database, workspace and skill-store directory;
+verify checksums before extracting both tar archives, then run:
+
+```sh
+set -a; . /path/to/restored/platform.env; set +a
+mkdir -p "$HUB_PROJECT_WORKSPACE_ROOT" "$HUB_SKILL_STORE_ROOT"
+tar -xf /path/to/business/workspaces.tar -C "$HUB_PROJECT_WORKSPACE_ROOT"
+tar -xf /path/to/business/agent-skills.tar -C "$HUB_SKILL_STORE_ROOT"
+node hub/dist/deployment/restore-preflight.js \
+  --keyring-file /path/to/restored/secret-keyring.json
+sudo INSTANCE_ID=org-a ENV_FILE=/path/to/restored/platform.env \
+  HUB_DIR=/path/to/restored/release/hub \
+  bash hub/deploy/agent_config.sh verify-store --organization org-a
+```
+
+Traffic stays disabled until the sole Organization and every Feishu/provider
+envelope authenticate, the workspace exists, and an end-to-end test run passes.
+
+For the first supervised demo, install → bootstrap → start → health check is the
+deployment gate. A completed off-host restore drill is an Alpha hardening item,
+not a prerequisite for bringing up that first controlled Silo.
+
+The default bind remains loopback; expose it only through a TLS reverse proxy.
+The platform admin surface is not part of the alpha and must not be exposed.

hub/deploy/agent_config.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+INSTANCE_ID="${INSTANCE_ID:?INSTANCE_ID required}"
+ENV_FILE="${ENV_FILE:?ENV_FILE required}"
+SERVICE_USER="${SERVICE_USER:-cph-$INSTANCE_ID}"
+HUB_DIR="${HUB_DIR:-/srv/curriculum-project-hub/current/hub}"
+
+[ "$(id -u)" -eq 0 ] || { echo "agent config console must run as root" >&2; exit 1; }
+[ -r "$ENV_FILE" ] || { echo "environment file is not readable: $ENV_FILE" >&2; exit 1; }
+[ -f "$HUB_DIR/dist/deployment/agent-config-cli.js" ] || { echo "Agent config CLI missing below $HUB_DIR" >&2; exit 1; }
+id "$SERVICE_USER" >/dev/null 2>&1 || { echo "service user missing: $SERVICE_USER" >&2; exit 1; }
+
+set -a
+# shellcheck disable=SC1090
+. "$ENV_FILE"
+set +a
+: "${DATABASE_URL:?DATABASE_URL missing from ENV_FILE}"
+: "${HUB_SILO_ORGANIZATION_ID:?HUB_SILO_ORGANIZATION_ID missing from ENV_FILE}"
+
+exec runuser --user "$SERVICE_USER" -- \
+  env -i \
+    DATABASE_URL="$DATABASE_URL" \
+    HUB_SILO_ORGANIZATION_ID="$HUB_SILO_ORGANIZATION_ID" \
+    HUB_SKILL_STORE_ROOT="${HUB_SKILL_STORE_ROOT:-/var/lib/cph-hub/$INSTANCE_ID/state/skills}" \
+    XDG_STATE_HOME="${XDG_STATE_HOME:-/var/lib/cph-hub/$INSTANCE_ID/state}" \
+    PATH="${PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}" \
+    node "$HUB_DIR/dist/deployment/agent-config-cli.js" "$@"

hub/deploy/apply_new_silo.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Apply a bundle produced by new_silo.sh to the managed Alpha host.
+set -euo pipefail
+
+BUNDLE="${1:?usage: apply_new_silo.sh BUNDLE_DIR}"
+ANSWERS="$BUNDLE/answers.env"
+[ -f "$ANSWERS" ] || { echo "missing $ANSWERS" >&2; exit 1; }
+
+value() { sed -n "s/^$1=//p" "$ANSWERS" | tail -n1; }
+INSTANCE_ID="$(value INSTANCE_ID)"
+ORG_ID="$(value ORGANIZATION_ID)"
+HOST="$(value DEPLOY_HOST)"
+SSH_USER="$(value DEPLOY_USER)"
+SSH_PORT="$(value DEPLOY_SSH_PORT)"
+SSH_KEY="$(value DEPLOY_SSH_KEY)"
+BASE="$(value DEPLOY_BASE)"
+RELEASE="$(value RELEASE_ID)"
+HUB_PORT="$(value HUB_PORT)"
+WORKSPACE="$(value WORKSPACE_ROOT)"
+MEMORY="$(value MEMORY_MAX)"
+CPU="$(value CPU_QUOTA)"
+TASKS="$(value TASKS_MAX)"
+DB_NAME="$(value DATABASE_NAME)"
+DB_USER="$(value DATABASE_USER)"
+DB_PASSWORD="$(value DATABASE_PASSWORD)"
+PUBLIC_URL="$(value PUBLIC_BASE_URL)"
+DOMAIN="${PUBLIC_URL#https://}"
+HUB_DIR="$BASE/releases/$RELEASE/hub"
+ENV_PATH="$BASE/.secrets/$INSTANCE_ID/platform.env"
+KEYRING_PATH="$BASE/.secrets/$INSTANCE_ID/secret-keyring.json"
+UNIT="cph-hub-$INSTANCE_ID.service"
+SSH=(ssh -i "$SSH_KEY" -p "$SSH_PORT" -o BatchMode=yes "$SSH_USER@$HOST")
+SCP=(scp -i "$SSH_KEY" -P "$SSH_PORT")
+
+for file in platform.env bootstrap.json default-role-prompt.md; do
+  [ -f "$BUNDLE/$file" ] || { echo "missing bundle file: $file" >&2; exit 1; }
+done
+
+echo "[1/8] Verify immutable release"
+"${SSH[@]}" "test -f '$BASE/releases/$RELEASE/.complete'"
+
+echo "[2/8] Create dedicated database"
+if ! "${SSH[@]}" "sudo -u postgres psql -Atqc \"select 1 from pg_database where datname='$DB_NAME'\"" | grep -qx 1; then
+  printf "CREATE ROLE %s LOGIN PASSWORD '%s';\nCREATE DATABASE %s OWNER %s;\n" \
+    "$DB_USER" "$DB_PASSWORD" "$DB_NAME" "$DB_USER" | "${SSH[@]}" sudo -u postgres psql -v ON_ERROR_STOP=1
+fi
+
+echo "[3/8] Seed instance keyring and service template"
+set +e
+"${SSH[@]}" "BASE='$BASE' HUB_DIR='$HUB_DIR' INSTANCE_ID='$INSTANCE_ID' WORKSPACE_ROOT='$WORKSPACE' PORT='$HUB_PORT' MEMORY_MAX='$MEMORY' CPU_QUOTA='$CPU' TASKS_MAX='$TASKS' bash '$HUB_DIR/deploy/install_service.sh'"
+status=$?
+set -e
+[ "$status" -eq 0 ] || [ "$status" -eq 78 ] || exit "$status"
+
+echo "[4/8] Upload root-only configuration"
+remote_stage="/root/.cph-bootstrap-$INSTANCE_ID"
+"${SSH[@]}" "install -d -o root -g root -m 0700 '$remote_stage'"
+"${SCP[@]}" "$BUNDLE/platform.env" "$BUNDLE/bootstrap.json" "$BUNDLE/default-role-prompt.md" "$SSH_USER@$HOST:$remote_stage/"
+"${SSH[@]}" "install -o root -g root -m 0600 '$remote_stage/platform.env' '$ENV_PATH'; chmod 0600 '$remote_stage/bootstrap.json' '$remote_stage/default-role-prompt.md'"
+
+echo "[5/8] Migrate, install, and bootstrap"
+"${SSH[@]}" "set -euo pipefail; set -a; . '$ENV_PATH'; set +a; node '$HUB_DIR/node_modules/prisma/build/index.js' migrate deploy --schema '$HUB_DIR/prisma/schema.prisma'; BASE='$BASE' HUB_DIR='$HUB_DIR' INSTANCE_ID='$INSTANCE_ID' WORKSPACE_ROOT='$WORKSPACE' PORT='$HUB_PORT' MEMORY_MAX='$MEMORY' CPU_QUOTA='$CPU' TASKS_MAX='$TASKS' bash '$HUB_DIR/deploy/install_service.sh'; node '$HUB_DIR/dist/deployment/bootstrap-silo-cli.js' --config-file '$remote_stage/bootstrap.json' --keyring-file '$KEYRING_PATH'"
+
+echo "[6/8] Copy curated skills and configure default role"
+"${SSH[@]}" "set -euo pipefail; stage='/var/lib/cph-hub/$INSTANCE_ID/state/operator-staging'; install -d -o cph-$INSTANCE_ID -g cph-$INSTANCE_ID -m 0700 \"\$stage/skills\"; install -o cph-$INSTANCE_ID -g cph-$INSTANCE_ID -m 0600 '$remote_stage/default-role-prompt.md' \"\$stage/default-role-prompt.md\"; for source in /var/lib/cph-hub/para-26071100/state/skills/versions/*; do name=\$(sed -n 's/^name: *//p' \"\$source/SKILL.md\" | head -n1); case \"\$name\" in outline|lesson-project|data-processing-spec|typst) cp -a \"\$source\" \"\$stage/skills/\$name\"; chown -R cph-$INSTANCE_ID:cph-$INSTANCE_ID \"\$stage/skills/\$name\";; esac; done; for name in outline lesson-project data-processing-spec typst; do INSTANCE_ID='$INSTANCE_ID' ENV_FILE='$ENV_PATH' HUB_DIR='$HUB_DIR' bash '$HUB_DIR/deploy/agent_config.sh' install-skill --organization '$ORG_ID' --source \"\$stage/skills/\$name\" --version 1; done; INSTANCE_ID='$INSTANCE_ID' ENV_FILE='$ENV_PATH' HUB_DIR='$HUB_DIR' bash '$HUB_DIR/deploy/agent_config.sh' upsert-role --organization '$ORG_ID' --role draft --label '智能助手' --system-prompt-file \"\$stage/default-role-prompt.md\" --tools-json null; INSTANCE_ID='$INSTANCE_ID' ENV_FILE='$ENV_PATH' HUB_DIR='$HUB_DIR' bash '$HUB_DIR/deploy/agent_config.sh' set-role-skills --organization '$ORG_ID' --role draft --skills outline,lesson-project,data-processing-spec,typst; rm -rf \"\$stage\""
+
+echo "[7/8] Configure Nginx and TLS"
+printf 'server { listen 80; listen [::]:80; server_name %s; client_max_body_size 2m; location / { proxy_pass http://127.0.0.1:%s; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_read_timeout 3600s; proxy_send_timeout 3600s; proxy_buffering off; } }\n' "$DOMAIN" "$HUB_PORT" | "${SSH[@]}" "install -o root -g root -m 0644 /dev/stdin '/etc/nginx/sites-available/$INSTANCE_ID'; ln -sfn '/etc/nginx/sites-available/$INSTANCE_ID' '/etc/nginx/sites-enabled/$INSTANCE_ID'; nginx -t; systemctl reload nginx; certbot --nginx --non-interactive --agree-tos --redirect --register-unsafely-without-email -d '$DOMAIN'"
+
+echo "[8/8] Start and verify"
+"${SSH[@]}" "systemctl enable --now '$UNIT'; systemctl is-active --quiet '$UNIT'; curl --fail --silent 'http://127.0.0.1:$HUB_PORT/api/healthz' >/dev/null; rm -f '$remote_stage/bootstrap.json'"
+curl --fail --silent --show-error "$PUBLIC_URL/api/healthz" >/dev/null
+echo "Deployed $INSTANCE_ID at $PUBLIC_URL"

hub/deploy/backup_silo.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Create a stopped-instance backup. Business data and recovery credentials are
+# deliberately written to different root-owned destinations.
+set -euo pipefail
+
+INSTANCE_ID="${INSTANCE_ID:?INSTANCE_ID required}"
+ENV_FILE="${ENV_FILE:?ENV_FILE required}"
+KEYRING_FILE="${KEYRING_FILE:?KEYRING_FILE required}"
+BUSINESS_BACKUP_DIR="${BUSINESS_BACKUP_DIR:?BUSINESS_BACKUP_DIR required}"
+RECOVERY_BACKUP_DIR="${RECOVERY_BACKUP_DIR:?RECOVERY_BACKUP_DIR required}"
+SERVICE_UNIT="cph-hub-$INSTANCE_ID.service"
+SKILL_STORE_ROOT="${SKILL_STORE_ROOT:-/var/lib/cph-hub/$INSTANCE_ID/state/skills}"
+
+[ "$(id -u)" -eq 0 ] || { echo "backup must run as root" >&2; exit 1; }
+umask 077
+
+validate_root_secret_file() {
+  local label="$1" path="$2" metadata
+  [[ "$path" = /* ]] || { echo "$label must be an absolute path" >&2; exit 1; }
+  [ ! -L "$path" ] || { echo "$label must not be a symlink: $path" >&2; exit 1; }
+  [ -f "$path" ] || { echo "$label must be a regular file: $path" >&2; exit 1; }
+  metadata="$(stat -c '%u:%g:%a' -- "$path")"
+  [ "$metadata" = "0:0:600" ] || {
+    echo "$label must be root:root mode 0600; found $metadata" >&2
+    exit 1
+  }
+}
+
+validate_root_secret_file "ENV_FILE" "$ENV_FILE"
+validate_root_secret_file "KEYRING_FILE" "$KEYRING_FILE"
+state="$(systemctl show --property=ActiveState --value "$SERVICE_UNIT")"
+case "$state" in inactive|failed) ;; *) echo "$SERVICE_UNIT must be stopped; state=$state" >&2; exit 1;; esac
+
+set -a
+# shellcheck disable=SC1090
+. "$ENV_FILE"
+set +a
+: "${DATABASE_URL:?DATABASE_URL missing from ENV_FILE}"
+: "${HUB_PROJECT_WORKSPACE_ROOT:?HUB_PROJECT_WORKSPACE_ROOT missing from ENV_FILE}"
+[ -d "$HUB_PROJECT_WORKSPACE_ROOT" ] || { echo "workspace root missing" >&2; exit 1; }
+[ -d "$SKILL_STORE_ROOT" ] || { echo "skill store root missing" >&2; exit 1; }
+
+install -d -o root -g root -m 0700 "$BUSINESS_BACKUP_DIR" "$RECOVERY_BACKUP_DIR"
+business_root="$(realpath -m "$BUSINESS_BACKUP_DIR")"
+recovery_root="$(realpath -m "$RECOVERY_BACKUP_DIR")"
+workspace_root="$(realpath -m "$HUB_PROJECT_WORKSPACE_ROOT")"
+secret_root="$(realpath -m "$(dirname "$KEYRING_FILE")")"
+skill_root="$(realpath -m "$SKILL_STORE_ROOT")"
+paths_overlap() {
+  local left="$1" right="$2"
+  [ "$left" = "$right" ] || [[ "$left/" == "$right/"* ]] || [[ "$right/" == "$left/"* ]]
+}
+for pair in \
+  "$business_root|$recovery_root" \
+  "$business_root|$workspace_root" \
+  "$recovery_root|$workspace_root" \
+  "$recovery_root|$secret_root"; do
+  left="${pair%%|*}"; right="${pair#*|}"
+  if paths_overlap "$left" "$right"; then
+    echo "backup destinations must not overlap each other or live state: $left <> $right" >&2
+    exit 1
+  fi
+done
+if paths_overlap "$business_root" "$skill_root" || paths_overlap "$recovery_root" "$skill_root" || paths_overlap "$workspace_root" "$skill_root"; then
+  echo "backup destinations, workspace and skill store must not overlap: $skill_root" >&2
+  exit 1
+fi
+stamp="$(date -u +%Y%m%dT%H%M%SZ)"
+business="$business_root/$INSTANCE_ID-$stamp"
+recovery="$recovery_root/$INSTANCE_ID-$stamp"
+install -d -o root -g root -m 0700 "$business" "$recovery"
+
+pg_dump --format=custom --file="$business/database.dump" "$DATABASE_URL"
+tar --create --file="$business/workspaces.tar" --directory="$HUB_PROJECT_WORKSPACE_ROOT" .
+tar --create --file="$business/agent-skills.tar" --directory="$SKILL_STORE_ROOT" .
+cp --preserve=mode,ownership,timestamps "$KEYRING_FILE" "$recovery/secret-keyring.json"
+cp --preserve=mode,ownership,timestamps "$ENV_FILE" "$recovery/platform.env"
+chmod 0600 "$recovery/secret-keyring.json" "$recovery/platform.env"
+
+(
+  cd "$business"
+  sha256sum database.dump workspaces.tar agent-skills.tar > SHA256SUMS
+)
+(
+  cd "$recovery"
+  sha256sum secret-keyring.json platform.env > SHA256SUMS
+)
+echo "business backup: $business"
+echo "separate recovery backup: $recovery"

hub/deploy/build_legacy_project_manifest.mjs

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+import { readdir, readFile, realpath } from "node:fs/promises";
+import { dirname, relative, resolve, sep } from "node:path";
+
+const rootArgument = process.argv[2];
+if (!rootArgument) throw new Error("usage: build_legacy_project_manifest.mjs <legacy-workspaces-root>");
+const root = await realpath(rootArgument);
+const projectFiles = await findProjectFiles(root);
+const seenIds = new Set();
+const manifest = [];
+for (const projectFile of projectFiles) {
+  const metadata = JSON.parse(await readFile(projectFile, "utf8"));
+  if (typeof metadata.id !== "string" || metadata.id.trim() === "") {
+    throw new Error(`project metadata has no id: ${projectFile}`);
+  }
+  if (seenIds.has(metadata.id)) throw new Error(`duplicate project id: ${metadata.id}`);
+  seenIds.add(metadata.id);
+  if (typeof metadata.name !== "string" || metadata.name.trim() === "") {
+    throw new Error(`project metadata has no name: ${projectFile}`);
+  }
+  const projectRoot = dirname(projectFile);
+  const sourceRelativePath = relative(root, projectRoot).split(sep).join("/");
+  const physicalFolderPath = dirname(sourceRelativePath) === "."
+    ? []
+    : dirname(sourceRelativePath).split("/");
+  if (metadata.folderPath !== undefined && (
+    !Array.isArray(metadata.folderPath)
+    || metadata.folderPath.some((part) => typeof part !== "string")
+    || JSON.stringify(metadata.folderPath) !== JSON.stringify(physicalFolderPath)
+  )) {
+    process.stderr.write(`[legacy-manifest] stale metadata folderPath; using physical path: ${projectFile}\n`);
+  }
+  manifest.push({
+    legacyId: metadata.id,
+    name: metadata.name,
+    folderPath: physicalFolderPath,
+    sourceRelativePath,
+  });
+}
+manifest.sort((left, right) => left.sourceRelativePath.localeCompare(right.sourceRelativePath, "zh-CN"));
+process.stdout.write(`${JSON.stringify(manifest, null, 2)}\n`);
+
+async function findProjectFiles(directory) {
+  const entries = await readdir(directory, { withFileTypes: true });
+  const projectMetadata = entries.find((entry) => entry.isFile() && entry.name === "project.json");
+  if (projectMetadata !== undefined) return [resolve(directory, projectMetadata.name)];
+  const found = [];
+  for (const entry of entries) {
+    if (entry.name === ".trash") continue;
+    const path = resolve(directory, entry.name);
+    if (entry.isSymbolicLink()) {
+      process.stderr.write(`[legacy-manifest] skip untracked symbolic link: ${path}\n`);
+      continue;
+    }
+    if (entry.isDirectory()) found.push(...await findProjectFiles(path));
+  }
+  return found;
+}

hub/deploy/cph-hub.service

@@ -1,24 +1,41 @@
 [Unit]
-Description=Curriculum Project Hub (Feishu agent + cph)
+Description=Curriculum Project Hub Silo (__INSTANCE_ID__)
 After=network-online.target postgresql.service
 Wants=network-online.target
+# ADR-0018: refuse to start when the configured OS sandbox binary disappears
+# after installation. Deployment preflight validates the executable itself;
+# this start-time assertion keeps later package/host changes fail-closed.
+AssertPathExists=__BWRAP_BIN__
+AssertPathExists=__SOCAT_BIN__
+AssertPathExists=__KEYRING_FILE__
 
 [Service]
 Type=simple
 User=__SERVICE_USER__
+Group=__SERVICE_GROUP__
 WorkingDirectory=__HUB_DIR__
 EnvironmentFile=__ENV_FILE__
+Environment=HOME=__SERVICE_HOME__
+Environment=XDG_STATE_HOME=__STATE_DIR__
+Environment=XDG_CACHE_HOME=__CACHE_DIR__
+Environment=HUB_SKILL_STORE_ROOT=__SKILL_STORE_ROOT__
+Environment=PATH=__RUNTIME_PATH__
+# ADR-0024: the root-owned source remains unreadable by the service account;
+# systemd materializes a read-only per-unit credential at runtime.
+LoadCredential=cph-secret-keyring:__KEYRING_FILE__
+UMask=0027
 # Run prisma migrations before each start, then launch the Hub.
 ExecStartPre=__NODE_BIN__ __HUB_DIR__/node_modules/prisma/build/index.js migrate deploy --schema __HUB_DIR__/prisma/schema.prisma
 ExecStart=__NODE_BIN__ __HUB_DIR__/dist/server.js
 Restart=on-failure
 RestartSec=5
+MemoryMax=__MEMORY_MAX__
+CPUQuota=__CPU_QUOTA__
+TasksMax=__TASKS_MAX__
+OOMPolicy=stop
 # Graceful shutdown: lark ws close + in-flight runs.
 KillSignal=SIGTERM
 TimeoutStopSec=30
-# ADR-0018: the agent sandbox (bubblewrap) must be available on the host —
-# failIfUnavailable makes the Hub refuse to run unsandboxed. Install bwrap.
-AssertPathExists=/usr/bin/bwrap
 # Hardening: no new privileges. ProtectSystem/ProtectHome are NOT set here —
 # they would break cph's render-cache extraction (~/.cache/cph/) and the
 # bwrap user-namespace setup. The OS-level sandbox (ADR-0018) is the hard

hub/deploy/deploy_platform.sh

@@ -5,49 +5,77 @@
 #   PLATFORM_DEPLOY_HOST     required
 #   PLATFORM_DEPLOY_SSH_KEY  required (private key for the deploy user)
 #   PLATFORM_DEPLOY_USER     optional, defaults to deploy
-#   PLATFORM_DEPLOY_PORT     optional, defaults to 22
+#   PLATFORM_DEPLOY_PORT     optional SSH port, defaults to 22
+#   PLATFORM_DEPLOY_HUB_PORT required Silo listener port
 #   PLATFORM_DEPLOY_BASE     optional, defaults to /srv/curriculum-project-hub
+#   PLATFORM_DEPLOY_RELEASE  optional immutable release id, defaults to git HEAD
+#   PLATFORM_DEPLOY_INSTANCE required, Silo instance id
+#   PLATFORM_DEPLOY_WORKSPACE_ROOT required short per-Silo path (for example /w/997)
+#   PLATFORM_DEPLOY_MEMORY_MAX / CPU_QUOTA / TASKS_MAX required ceilings
 #   PLATFORM_DEPLOY_HEALTH_URL optional, defaults to http://127.0.0.1:8788/api/healthz
-# The target host must have Node.js, npm, rsync, PostgreSQL, systemd, bubblewrap
-# (`bwrap`, for the ADR-0018 agent sandbox), and a writable $PLATFORM_DEPLOY_BASE.
+# The target host must have Node.js 24+, npm, rsync, PostgreSQL client tools
+# (`pg_isready`), systemd, bubblewrap (`bwrap`) and `socat` (for the ADR-0018 agent sandbox),
+# and a compatible `cph` binary named by platform.env. The service installer
+# provisions the dedicated non-root identity and persistent directories.
 # Use a dedicated `deploy` user that has passwordless sudo for systemctl and
-# writing /etc/systemd/system/cph-hub.service through deploy/install_service.sh.
+# writing the named /etc/systemd/system/cph-hub-<instance>.service through the installer.
 set -euo pipefail
 
 HOST="${PLATFORM_DEPLOY_HOST:?PLATFORM_DEPLOY_HOST required}"
 SSH_KEY="${PLATFORM_DEPLOY_SSH_KEY:?PLATFORM_DEPLOY_SSH_KEY required}"
 DEPLOY_USER="${PLATFORM_DEPLOY_USER:-deploy}"
 PORT="${PLATFORM_DEPLOY_PORT:-22}"
+HUB_PORT="${PLATFORM_DEPLOY_HUB_PORT:?PLATFORM_DEPLOY_HUB_PORT required}"
 BASE="${PLATFORM_DEPLOY_BASE:-/srv/curriculum-project-hub}"
-HEALTH_URL="${PLATFORM_DEPLOY_HEALTH_URL:-http://127.0.0.1:8788/api/healthz}"
-HUB_DIR="$BASE/hub"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+RELEASE_ID="${PLATFORM_DEPLOY_RELEASE:-$(git -C "$REPO_ROOT" rev-parse --verify HEAD)}"
+[[ "$RELEASE_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { echo "invalid PLATFORM_DEPLOY_RELEASE" >&2; exit 1; }
+INSTANCE_ID="${PLATFORM_DEPLOY_INSTANCE:?PLATFORM_DEPLOY_INSTANCE required}"
+WORKSPACE_ROOT="${PLATFORM_DEPLOY_WORKSPACE_ROOT:?PLATFORM_DEPLOY_WORKSPACE_ROOT required}"
+SERVICE_UNIT="cph-hub-$INSTANCE_ID.service"
+MEMORY_MAX="${PLATFORM_DEPLOY_MEMORY_MAX:?PLATFORM_DEPLOY_MEMORY_MAX required}"
+CPU_QUOTA="${PLATFORM_DEPLOY_CPU_QUOTA:?PLATFORM_DEPLOY_CPU_QUOTA required}"
+TASKS_MAX="${PLATFORM_DEPLOY_TASKS_MAX:?PLATFORM_DEPLOY_TASKS_MAX required}"
+HEALTH_URL="${PLATFORM_DEPLOY_HEALTH_URL:-http://127.0.0.1:$HUB_PORT/api/healthz}"
+HUB_DIR="$BASE/releases/$RELEASE_ID/hub"
+RELEASE_DIR="$BASE/releases/$RELEASE_ID"
 
 SSH_OPTS=(-i "$SSH_KEY" -p "$PORT" -o StrictHostKeyChecking=accept-new)
 
-ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "mkdir -p '$HUB_DIR'"
+release_ready=false
+if ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "test -f '$RELEASE_DIR/.complete'"; then
+  release_ready=true
+elif ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "test -e '$RELEASE_DIR'"; then
+  echo "incomplete release already exists: $RELEASE_DIR" >&2
+  exit 1
+else
+  ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "mkdir -p '$HUB_DIR'"
+fi
 
-# 1. Rsync the hub/ directory (exclude node_modules/dist, they rebuild on host).
-rsync -az --delete \
-  --exclude node_modules --exclude dist --exclude .env \
-  -e "ssh ${SSH_OPTS[*]}" \
-  "$REPO_ROOT/hub/" "$DEPLOY_USER@$HOST:$HUB_DIR/"
+if [ "$release_ready" = false ]; then
+  # 1. Populate a new release directory. A completed release is never mutated.
+  rsync -az --delete \
+    --exclude node_modules --exclude dist --exclude .env \
+    -e "ssh ${SSH_OPTS[*]}" \
+    "$REPO_ROOT/hub/" "$DEPLOY_USER@$HOST:$HUB_DIR/"
 
-# 2. Install deps + build on the host.
-ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "cd '$HUB_DIR' && npm ci && npm run build"
+  # 2. Install deps, audit and build, then atomically mark the release complete.
+  ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" \
+    "cd '$HUB_DIR' && npm ci && npm run audit:production && npm run build && touch '$RELEASE_DIR/.complete'"
+fi
 
 # 3. Ensure the service is installed (idempotent), then restart.
 ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "
   set -euo pipefail
   if [ \"\$(id -u)\" = \"0\" ]; then
-    BASE='$BASE' HUB_DIR='$HUB_DIR' bash '$HUB_DIR/deploy/install_service.sh'
-    systemctl restart cph-hub.service
-    systemctl is-active --quiet cph-hub.service
+    BASE='$BASE' HUB_DIR='$HUB_DIR' INSTANCE_ID='$INSTANCE_ID' WORKSPACE_ROOT='$WORKSPACE_ROOT' PORT='$HUB_PORT' MEMORY_MAX='$MEMORY_MAX' CPU_QUOTA='$CPU_QUOTA' TASKS_MAX='$TASKS_MAX' bash '$HUB_DIR/deploy/install_service.sh'
+    systemctl restart '$SERVICE_UNIT'
+    systemctl is-active --quiet '$SERVICE_UNIT'
   else
-    sudo -n BASE='$BASE' HUB_DIR='$HUB_DIR' bash '$HUB_DIR/deploy/install_service.sh'
-    sudo -n systemctl restart cph-hub.service
-    sudo -n systemctl is-active --quiet cph-hub.service
+    sudo -n BASE='$BASE' HUB_DIR='$HUB_DIR' INSTANCE_ID='$INSTANCE_ID' WORKSPACE_ROOT='$WORKSPACE_ROOT' PORT='$HUB_PORT' MEMORY_MAX='$MEMORY_MAX' CPU_QUOTA='$CPU_QUOTA' TASKS_MAX='$TASKS_MAX' bash '$HUB_DIR/deploy/install_service.sh'
+    sudo -n systemctl restart '$SERVICE_UNIT'
+    sudo -n systemctl is-active --quiet '$SERVICE_UNIT'
   fi
   for attempt in \$(seq 1 30); do
     if curl --fail --silent '$HEALTH_URL' >/dev/null 2>&1; then

hub/deploy/install_service.sh

@@ -1,76 +1,323 @@
 #!/usr/bin/env bash
-# Install the cph-hub systemd service on a host. Idempotent.
-#
-# Prerequisites already on the host: Node.js, npm, PostgreSQL, systemd.
-# The Hub repo is expected at HUB_DIR (default /srv/curriculum-project-hub/hub)
-# with `npm ci` + `npm run build` already run by deploy_platform.sh.
+# Provision and install the cph-hub systemd service on a supported Linux host.
+# The script is idempotent: existing identities/directories are validated and
+# never silently rewritten. An absent environment file is seeded completely,
+# then the script exits EX_CONFIG so no unit is installed with placeholders.
 #
 # Usage:
-#   sudo BASE=/srv/curriculum-project-hub bash deploy/install_service.sh
+#   sudo BASE=/srv/curriculum-project-hub \
+#     HUB_DIR=/srv/curriculum-project-hub/hub \
+#     bash deploy/install_service.sh
 set -euo pipefail
 
 BASE="${BASE:-/srv/curriculum-project-hub}"
 HUB_DIR="${HUB_DIR:-$BASE/hub}"
-SERVICE_USER="${SERVICE_USER:-cph-hub}"
+INSTANCE_ID="${INSTANCE_ID:?INSTANCE_ID is required (lowercase letters, digits, hyphens)}"
+if [[ ! "$INSTANCE_ID" =~ ^[a-z0-9]([a-z0-9-]{0,22}[a-z0-9])?$ ]]; then
+  echo "[install] error: invalid INSTANCE_ID: $INSTANCE_ID" >&2
+  exit 1
+fi
+SERVICE_UNIT="cph-hub-$INSTANCE_ID.service"
+SERVICE_USER="${SERVICE_USER:-cph-$INSTANCE_ID}"
+SERVICE_GROUP="${SERVICE_GROUP:-$SERVICE_USER}"
+SERVICE_HOME="${SERVICE_HOME:-/var/lib/cph-hub/$INSTANCE_ID/home}"
+STATE_DIR="${STATE_DIR:-/var/lib/cph-hub/$INSTANCE_ID/state}"
+CACHE_DIR="${CACHE_DIR:-/var/cache/cph-hub/$INSTANCE_ID}"
+SKILL_STORE_ROOT="${SKILL_STORE_ROOT:-$STATE_DIR/skills}"
+WORKSPACE_ROOT="${WORKSPACE_ROOT:?WORKSPACE_ROOT required (use a short per-Silo path such as /w/997)}"
 HOST="${HOST:-127.0.0.1}"
-PORT="${PORT:-8788}"
-ENV_FILE="${ENV_FILE:-$BASE/.secrets/platform.env}"
+PORT="${PORT:?PORT is required and must be unique on the host}"
+ENV_FILE="${ENV_FILE:-$BASE/.secrets/$INSTANCE_ID/platform.env}"
+KEYRING_FILE="${KEYRING_FILE:-$BASE/.secrets/$INSTANCE_ID/secret-keyring.json}"
+MEMORY_MAX="${MEMORY_MAX:?MEMORY_MAX is required, for example 16G}"
+CPU_QUOTA="${CPU_QUOTA:?CPU_QUOTA is required, for example 400%}"
+TASKS_MAX="${TASKS_MAX:?TASKS_MAX is required, for example 512}"
+SYSTEMD_UNIT_DIR="${SYSTEMD_UNIT_DIR:-/etc/systemd/system}"
 NODE_BIN="${NODE_BIN:-$(command -v node || true)}"
+BWRAP_BIN="${BWRAP_BIN:-$(command -v bwrap || true)}"
+SOCAT_BIN="${SOCAT_BIN:-$(command -v socat || true)}"
+PG_ISREADY_BIN="${PG_ISREADY_BIN:-$(command -v pg_isready || true)}"
+RUNUSER_BIN="${RUNUSER_BIN:-$(command -v runuser || true)}"
+SETPRIV_BIN="${SETPRIV_BIN:-$(command -v setpriv || true)}"
+CPH_BIN_DEFAULT="${CPH_BIN_DEFAULT:-/usr/local/bin/cph}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PREFLIGHT_JS="$HUB_DIR/dist/deployment/preflight-cli.js"
+RUNTIME_PATH="$(dirname "$NODE_BIN"):$(dirname "$BWRAP_BIN"):$(dirname "$SOCAT_BIN"):$(dirname "$SETPRIV_BIN"):/usr/local/bin:/usr/bin:/bin"
 
-if [ -z "$NODE_BIN" ]; then
-  echo "node must be on PATH, or set NODE_BIN." >&2
+fail() {
+  echo "[install] error: $*" >&2
   exit 1
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || fail "required command is missing: $1"
+}
+
+require_safe_unit_value() {
+  local label="$1"
+  local value="$2"
+  if [[ ! "$value" =~ ^[A-Za-z0-9_./:@+-]+$ ]]; then
+    fail "$label contains characters that cannot be rendered safely into the systemd unit: $value"
+  fi
+}
+
+if [ "$(id -u)" -ne 0 ]; then
+  fail "must run as root (use sudo)"
 fi
-if [ ! -d "$HUB_DIR" ]; then
-  echo "HUB_DIR not found: $HUB_DIR (set HUB_DIR or clone the repo there)." >&2
-  exit 1
+if [[ ! "$SERVICE_USER" =~ ^[a-z_][a-z0-9_-]*[$]?$ ]]; then
+  fail "invalid SERVICE_USER: $SERVICE_USER"
 fi
-if [ ! -f "$HUB_DIR/dist/server.js" ]; then
-  echo "build missing: run 'npm ci && npm run build' in $HUB_DIR first." >&2
-  exit 1
+if [[ ! "$SERVICE_GROUP" =~ ^[a-z_][a-z0-9_-]*[$]?$ ]]; then
+  fail "invalid SERVICE_GROUP: $SERVICE_GROUP"
 fi
-if [ ! -f "$SCRIPT_DIR/cph-hub.service" ]; then
-  echo "service template missing: $SCRIPT_DIR/cph-hub.service" >&2
-  exit 1
+if [ "$SERVICE_USER" = "root" ] || [ "$SERVICE_GROUP" = "root" ]; then
+  fail "the Hub service identity must not be root"
 fi
 
-# Seed the env file if absent. Secrets stay on the server, never in the repo.
-if [ ! -f "$ENV_FILE" ]; then
-  install -d -m 0700 "$(dirname "$ENV_FILE")"
+for command in getent groupadd useradd install stat systemctl id openssl date mktemp; do
+  require_command "$command"
+done
+for pair in \
+  "NODE_BIN:$NODE_BIN" \
+  "BWRAP_BIN:$BWRAP_BIN" \
+  "SOCAT_BIN:$SOCAT_BIN" \
+  "PG_ISREADY_BIN:$PG_ISREADY_BIN"; do
+  label="${pair%%:*}"
+  value="${pair#*:}"
+  [ -n "$value" ] || fail "$label is required and its executable was not found on PATH"
+  [ -x "$value" ] || fail "$label is not executable: $value"
+done
+[ -n "$RUNUSER_BIN" ] || fail "RUNUSER_BIN is required and runuser was not found on PATH"
+[ -x "$RUNUSER_BIN" ] || fail "RUNUSER_BIN is not executable: $RUNUSER_BIN"
+[ -n "$SETPRIV_BIN" ] || fail "SETPRIV_BIN is required and setpriv was not found on PATH"
+[ -x "$SETPRIV_BIN" ] || fail "SETPRIV_BIN is not executable: $SETPRIV_BIN"
+
+[ -d "$HUB_DIR" ] || fail "HUB_DIR not found: $HUB_DIR"
+[ -f "$HUB_DIR/dist/server.js" ] || fail "Hub build missing: $HUB_DIR/dist/server.js"
+[ -f "$PREFLIGHT_JS" ] || fail "deployment preflight missing from Hub build: $PREFLIGHT_JS"
+[ -f "$HUB_DIR/dist/deployment/service-runtime-probe.js" ] || fail "service runtime probe missing from Hub build"
+[ -f "$HUB_DIR/node_modules/prisma/build/index.js" ] || fail "production dependencies missing: run npm ci in $HUB_DIR"
+[ -f "$HUB_DIR/prisma/schema.prisma" ] || fail "Prisma schema missing: $HUB_DIR/prisma/schema.prisma"
+[ -f "$SCRIPT_DIR/cph-hub.service" ] || fail "service template missing: $SCRIPT_DIR/cph-hub.service"
+
+for pair in \
+  "BASE:$BASE" \
+  "HUB_DIR:$HUB_DIR" \
+  "SERVICE_HOME:$SERVICE_HOME" \
+  "STATE_DIR:$STATE_DIR" \
+  "CACHE_DIR:$CACHE_DIR" \
+  "SKILL_STORE_ROOT:$SKILL_STORE_ROOT" \
+  "WORKSPACE_ROOT:$WORKSPACE_ROOT" \
+  "ENV_FILE:$ENV_FILE" \
+  "KEYRING_FILE:$KEYRING_FILE" \
+  "SYSTEMD_UNIT_DIR:$SYSTEMD_UNIT_DIR" \
+  "NODE_BIN:$NODE_BIN" \
+  "BWRAP_BIN:$BWRAP_BIN" \
+  "SOCAT_BIN:$SOCAT_BIN" \
+  "RUNUSER_BIN:$RUNUSER_BIN" \
+  "SETPRIV_BIN:$SETPRIV_BIN" \
+  "RUNTIME_PATH:$RUNTIME_PATH"; do
+  require_safe_unit_value "${pair%%:*}" "${pair#*:}"
+done
+[[ "$MEMORY_MAX" =~ ^[1-9][0-9]*[KMGT]$ ]] || fail "MEMORY_MAX must be a systemd byte size such as 16G"
+[[ "$CPU_QUOTA" =~ ^[1-9][0-9]*%$ ]] || fail "CPU_QUOTA must be a positive percentage such as 400%"
+[[ "$TASKS_MAX" =~ ^[1-9][0-9]*$ ]] || fail "TASKS_MAX must be a positive integer"
+[ "${#WORKSPACE_ROOT}" -le 16 ] || fail "WORKSPACE_ROOT must be at most 16 bytes for Agent sandbox sockets: $WORKSPACE_ROOT"
+
+KEYRING_CREATED=false
+if [ -L "$KEYRING_FILE" ]; then
+  fail "local secret keyring must not be a symlink: $KEYRING_FILE"
+fi
+if [ ! -e "$KEYRING_FILE" ]; then
+  install -d -o root -g root -m 0700 "$(dirname "$KEYRING_FILE")"
+  key_id="local-$(date -u +%Y%m%dT%H%M%SZ)"
+  key_value="$(openssl rand -base64 32)"
+  tmp_keyring="$(mktemp "$(dirname "$KEYRING_FILE")/.secret-keyring.XXXXXX")"
+  trap 'rm -f "${tmp_keyring:-}"' EXIT
+  chmod 0600 "$tmp_keyring"
+  printf '{"version":1,"activeKeyId":"%s","keys":{"%s":"%s"}}\n' \
+    "$key_id" "$key_id" "$key_value" >"$tmp_keyring"
+  install -o root -g root -m 0600 "$tmp_keyring" "$KEYRING_FILE"
+  rm -f "$tmp_keyring"
+  tmp_keyring=""
+  trap - EXIT
+  unset key_value
+  KEYRING_CREATED=true
+  echo "[install] generated local secret keyring: $KEYRING_FILE" >&2
+fi
+[ -f "$KEYRING_FILE" ] || fail "local secret keyring is not a regular file: $KEYRING_FILE"
+keyring_metadata="$(stat -c '%u:%g:%a' -- "$KEYRING_FILE")"
+[ "$keyring_metadata" = "0:0:600" ] || \
+  fail "local secret keyring must be owned by root:root with mode 0600; found $keyring_metadata"
+
+if [ -L "$ENV_FILE" ]; then
+  fail "environment file must not be a symlink: $ENV_FILE"
+fi
+if [ ! -e "$ENV_FILE" ]; then
+  install -d -o root -g root -m 0700 "$(dirname "$ENV_FILE")"
+  umask 077
   cat >"$ENV_FILE" <<EOF
+# cph-hub production configuration. Required blanks must be filled before
+# rerunning install_service.sh; failed preflight never installs the unit.
 NODE_ENV=production
-DATABASE_URL=postgresql://paradigm:change-me@127.0.0.1:5432/paradigm
-# Claude Code SDK auth via OpenRouter's Anthropic Skin (ADR-0017).
-# ANTHROPIC_AUTH_TOKEN = your OpenRouter API key; ANTHROPIC_API_KEY must be empty.
-ANTHROPIC_BASE_URL=https://openrouter.ai/api
-ANTHROPIC_AUTH_TOKEN=
-ANTHROPIC_API_KEY=
-FEISHU_APP_ID=
-FEISHU_APP_SECRET=
-FEISHU_BOT_OPEN_ID=
-PORT=$PORT
+DATABASE_URL=
+HUB_SILO_ORGANIZATION_ID=
+HUB_SYSTEMD_UNIT=$SERVICE_UNIT
+CPH_BIN=$CPH_BIN_DEFAULT
 HOST=$HOST
+PORT=$PORT
+HUB_PROJECT_WORKSPACE_ROOT=$WORKSPACE_ROOT
+HUB_PUBLIC_BASE_URL=
+HUB_SESSION_SECRET=
+HUB_AGENT_MAX_TURNS=25
+HUB_AGENT_MAX_CONCURRENT_RUNS=
+HUB_AGENT_MAX_RUN_SECONDS=
+HUB_HTTP_BODY_LIMIT_BYTES=
+HUB_MAX_FILES_PER_MESSAGE=
+HUB_MAX_FILE_BYTES=
+HUB_HTTP_REQUESTS_PER_MINUTE=
+HUB_FEISHU_EVENTS_PER_MINUTE=
+HUB_FEISHU_LISTENER_ENABLED=true
+CPH_SANDBOX_EXTRA_DENY_READ=$ENV_FILE:$KEYRING_FILE
 EOF
   chmod 0600 "$ENV_FILE"
-  echo "Seeded $ENV_FILE — edit it to fill in ANTHROPIC_AUTH_TOKEN and Feishu creds, then re-run."
-  exit 0
+  echo "[install] seeded complete production configuration: $ENV_FILE" >&2
+  echo "[install] copy $KEYRING_FILE to the separately protected keyring backup before continuing" >&2
+  echo "[install] fill every required blank, install cph at CPH_BIN, then rerun" >&2
+  exit 78
+fi
+[ -f "$ENV_FILE" ] || fail "environment file is not a regular file: $ENV_FILE"
+if [ "$KEYRING_CREATED" = true ]; then
+  echo "[install] copy $KEYRING_FILE to the separately protected keyring backup, then rerun" >&2
+  exit 78
 fi
 
-# Render the unit with concrete paths.
-UNIT=/etc/systemd/system/cph-hub.service
+PREFLIGHT_ARGS=(
+  "$NODE_BIN" "$PREFLIGHT_JS"
+  --env-file "$ENV_FILE"
+  --keyring-file "$KEYRING_FILE"
+  --node-bin "$NODE_BIN"
+  --runuser-bin "$RUNUSER_BIN"
+  --setpriv-bin "$SETPRIV_BIN"
+  --service-user "$SERVICE_USER"
+  --base-dir "$BASE"
+  --hub-dir "$HUB_DIR"
+  --service-home "$SERVICE_HOME"
+  --state-dir "$STATE_DIR"
+  --cache-dir "$CACHE_DIR"
+  --workspace-root "$WORKSPACE_ROOT"
+  --service-unit "$SERVICE_UNIT"
+  --bwrap-bin "$BWRAP_BIN"
+  --socat-bin "$SOCAT_BIN"
+  --pg-isready-bin "$PG_ISREADY_BIN"
+)
+
+# Required configuration, path isolation, binaries and PostgreSQL must pass
+# before identity/directory provisioning or unit installation begins.
+"${PREFLIGHT_ARGS[@]}"
+
+NOLOGIN_SHELL=""
+for candidate in /usr/sbin/nologin /sbin/nologin /usr/bin/false /bin/false; do
+  if [ -x "$candidate" ]; then
+    NOLOGIN_SHELL="$candidate"
+    break
+  fi
+done
+[ -n "$NOLOGIN_SHELL" ] || fail "no non-login shell found"
+
+group_entry="$(getent group "$SERVICE_GROUP" || true)"
+if [ -z "$group_entry" ]; then
+  groupadd --system "$SERVICE_GROUP"
+  group_entry="$(getent group "$SERVICE_GROUP" || true)"
+  [ -n "$group_entry" ] || fail "created service group cannot be resolved: $SERVICE_GROUP"
+fi
+IFS=: read -r resolved_group _ SERVICE_GID _ <<<"$group_entry"
+[ "$resolved_group" = "$SERVICE_GROUP" ] || fail "service group resolved to unexpected name: $resolved_group"
+[[ "$SERVICE_GID" =~ ^[0-9]+$ ]] || fail "service group has invalid gid: $SERVICE_GID"
+[ "$SERVICE_GID" -ne 0 ] || fail "service group must not have gid 0"
+
+passwd_entry="$(getent passwd "$SERVICE_USER" || true)"
+if [ -z "$passwd_entry" ]; then
+  useradd \
+    --system \
+    --gid "$SERVICE_GROUP" \
+    --home-dir "$SERVICE_HOME" \
+    --shell "$NOLOGIN_SHELL" \
+    --no-create-home \
+    "$SERVICE_USER"
+  passwd_entry="$(getent passwd "$SERVICE_USER" || true)"
+  [ -n "$passwd_entry" ] || fail "created service user cannot be resolved: $SERVICE_USER"
+fi
+IFS=: read -r resolved_user _ SERVICE_UID user_gid _ user_home user_shell <<<"$passwd_entry"
+[ "$resolved_user" = "$SERVICE_USER" ] || fail "service user resolved to unexpected name: $resolved_user"
+[[ "$SERVICE_UID" =~ ^[0-9]+$ ]] || fail "service user has invalid uid: $SERVICE_UID"
+[ "$SERVICE_UID" -ne 0 ] || fail "service user must not have uid 0"
+[ "$user_gid" = "$SERVICE_GID" ] || fail "service user primary gid $user_gid does not match $SERVICE_GROUP ($SERVICE_GID)"
+[ "$user_home" = "$SERVICE_HOME" ] || fail "service user home is $user_home; expected $SERVICE_HOME"
+case "$user_shell" in
+  /usr/sbin/nologin | /sbin/nologin | /usr/bin/false | /bin/false) ;;
+  *) fail "service user must have a non-login shell; found $user_shell" ;;
+esac
+service_gids="$(id -G "$SERVICE_USER")" || fail "cannot resolve groups for service user $SERVICE_USER"
+for service_gid in $service_gids; do
+  if [ "$service_gid" != "$SERVICE_GID" ]; then
+    fail "service user $SERVICE_USER must not have supplementary groups; found gid $service_gid in: $service_gids"
+  fi
+done
+
+provision_directory() {
+  local path="$1"
+  if [ -L "$path" ]; then
+    fail "service directory must not be a symlink: $path"
+  fi
+  if [ -e "$path" ]; then
+    [ -d "$path" ] || fail "service path exists but is not a directory: $path"
+    actual="$(stat -c '%u:%g:%a' -- "$path")"
+    expected="$SERVICE_UID:$SERVICE_GID:750"
+    [ "$actual" = "$expected" ] || fail "service directory $path has $actual; expected $expected"
+    return
+  fi
+  install -d -o "$SERVICE_UID" -g "$SERVICE_GID" -m 0750 "$path"
+}
+
+provision_directory "$SERVICE_HOME"
+provision_directory "$STATE_DIR"
+provision_directory "$CACHE_DIR"
+provision_directory "$SKILL_STORE_ROOT"
+provision_directory "$WORKSPACE_ROOT"
+
+# Resolve every provisioned path again and verify uid/gid/mode before writing
+# the unit. This catches symlink or ownership changes between the two phases.
+"${PREFLIGHT_ARGS[@]}" --service-uid "$SERVICE_UID" --service-gid "$SERVICE_GID"
+
+UNIT="$SYSTEMD_UNIT_DIR/$SERVICE_UNIT"
+install -d -o root -g root -m 0755 "$SYSTEMD_UNIT_DIR"
 TMP_UNIT="$(mktemp)"
+trap 'rm -f "$TMP_UNIT"' EXIT
 sed \
   -e "s|__SERVICE_USER__|$SERVICE_USER|g" \
+  -e "s|__SERVICE_GROUP__|$SERVICE_GROUP|g" \
+  -e "s|__SERVICE_HOME__|$SERVICE_HOME|g" \
+  -e "s|__STATE_DIR__|$STATE_DIR|g" \
+  -e "s|__CACHE_DIR__|$CACHE_DIR|g" \
+  -e "s|__SKILL_STORE_ROOT__|$SKILL_STORE_ROOT|g" \
+  -e "s|__WORKSPACE_ROOT__|$WORKSPACE_ROOT|g" \
   -e "s|__HUB_DIR__|$HUB_DIR|g" \
-  -e "s|__BASE_DIR__|$BASE|g" \
   -e "s|__ENV_FILE__|$ENV_FILE|g" \
+  -e "s|__KEYRING_FILE__|$KEYRING_FILE|g" \
   -e "s|__NODE_BIN__|$NODE_BIN|g" \
+  -e "s|__BWRAP_BIN__|$BWRAP_BIN|g" \
+  -e "s|__SOCAT_BIN__|$SOCAT_BIN|g" \
+  -e "s|__RUNTIME_PATH__|$RUNTIME_PATH|g" \
+  -e "s|__INSTANCE_ID__|$INSTANCE_ID|g" \
+  -e "s|__MEMORY_MAX__|$MEMORY_MAX|g" \
+  -e "s|__CPU_QUOTA__|$CPU_QUOTA|g" \
+  -e "s|__TASKS_MAX__|$TASKS_MAX|g" \
   "$SCRIPT_DIR/cph-hub.service" >"$TMP_UNIT"
-install -m 0644 "$TMP_UNIT" "$UNIT"
-rm -f "$TMP_UNIT"
+install -o root -g root -m 0644 "$TMP_UNIT" "$UNIT"
 
 systemctl daemon-reload
-systemctl enable cph-hub.service
-echo "Installed cph-hub.service. Start with: systemctl start cph-hub"
-echo "Check health: curl http://127.0.0.1:$PORT/api/healthz"
+systemctl enable "$SERVICE_UNIT"
+echo "[install] installed $SERVICE_UNIT for $SERVICE_USER:$SERVICE_GROUP"
+echo "[install] home=$SERVICE_HOME state=$STATE_DIR cache=$CACHE_DIR skills=$SKILL_STORE_ROOT workspaces=$WORKSPACE_ROOT"
+echo "[install] start with: systemctl start $SERVICE_UNIT"

hub/deploy/new_silo.sh

@@ -0,0 +1,412 @@
+#!/usr/bin/env bash
+#
+# A wizard — walks a human through a manual procedure step by step.
+# Generated by the /wizard skill.
+#
+# Everything above the "STAGES" marker is the wizard library: do not hand-edit
+# it. Author the per-step stages below the marker.
+
+set -euo pipefail
+
+# ──────────────────────────────────────────────────────────────────────────
+# Wizard library — delightful, consistent UX. Identical across every wizard.
+# ──────────────────────────────────────────────────────────────────────────
+
+if [[ -t 1 ]] && command -v tput >/dev/null 2>&1 && [[ "$(tput colors 2>/dev/null || echo 0)" -ge 8 ]]; then
+  BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0)
+  BLUE=$(tput setaf 4); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); RED=$(tput setaf 1)
+else
+  BOLD=""; DIM=""; RESET=""; BLUE=""; GREEN=""; YELLOW=""; RED=""
+fi
+
+# Author sets these two at the top of the stages section.
+TOTAL_STAGES=0
+TOTAL_MINUTES=0
+
+_STAGE_INDEX=0
+_MINUTES_ELAPSED=0
+ENV_FILE="${ENV_FILE:-.env}"
+WRITTEN_ENV=()    # KEYs written to ENV_FILE this run
+WRITTEN_SECRET=() # secret NAMEs set this run
+SKIPPED=()        # things we couldn't do (e.g. gh missing)
+
+# _clear — wipe the terminal so only the current step is on screen. No-op when
+# output isn't a terminal, so piped logs stay readable.
+_clear() {
+  [[ -t 1 ]] || return 0
+  if command -v tput >/dev/null 2>&1; then tput clear; else printf '\033[2J\033[3J\033[H'; fi
+}
+
+# banner "Title" — opening frame: what this wizard does and how long it takes.
+banner() {
+  _clear
+  printf '\n%s%s  %s%s\n' "$BOLD" "$BLUE" "$1" "$RESET"
+  printf '%s  %s stages · about %s minutes%s\n\n' \
+    "$DIM" "$TOTAL_STAGES" "$TOTAL_MINUTES" "$RESET"
+  printf '%s  You drive the browser; this wizard tells you exactly what to do and\n' "$DIM"
+  printf '  captures the values you copy back. Stop any time with Ctrl-C and re-run\n'
+  printf '  later — it remembers values already saved.%s\n' "$RESET"
+  pause "Ready to start?"
+}
+
+# stage "Name" <minutes> — clear the screen, then announce a stage and show
+# progress + time remaining. Clearing keeps only the current step on screen.
+stage() {
+  _clear
+  _STAGE_INDEX=$((_STAGE_INDEX + 1))
+  local remaining=$((TOTAL_MINUTES - _MINUTES_ELAPSED))
+  (( remaining < 0 )) && remaining=0
+  _MINUTES_ELAPSED=$((_MINUTES_ELAPSED + ${2:-0}))
+  printf '\n%s%s▸ Stage %s/%s · %s%s  %s(~%s min left)%s\n' \
+    "$BOLD" "$BLUE" "$_STAGE_INDEX" "$TOTAL_STAGES" "$1" "$RESET" "$DIM" "$remaining" "$RESET"
+}
+
+# say "..." — a plain instruction line.
+say()  { printf '  %s\n' "$1"; }
+# step "..." — a numbered-feeling action the human takes in the browser.
+step() { printf '  %s•%s %s\n' "$BLUE" "$RESET" "$1"; }
+note() { printf '  %s%s%s\n' "$DIM" "$1" "$RESET"; }
+warn() { printf '  %s⚠ %s%s\n' "$YELLOW" "$1" "$RESET"; }
+
+# open_url URL — open in the human's browser, cross-platform incl. WSL.
+open_url() {
+  local url="$1"
+  printf '  %s↗ opening%s %s\n' "$GREEN" "$RESET" "$url"
+  { if   command -v wslview     >/dev/null 2>&1; then wslview "$url"
+    elif command -v explorer.exe >/dev/null 2>&1; then explorer.exe "$url"
+    elif command -v xdg-open    >/dev/null 2>&1; then xdg-open "$url"
+    elif command -v open        >/dev/null 2>&1; then open "$url"
+    else warn "couldn't open a browser — visit it manually: $url"; fi
+  } >/dev/null 2>&1 || warn "couldn't open a browser — visit it manually: $url"
+}
+
+# pause "msg" — wait for the human to confirm they've done the manual part.
+pause() {
+  printf '  %s%s%s ' "$DIM" "${1:-Press Enter to continue}" "$RESET"
+  read -r _ || true
+}
+
+# confirm "question" — y/N gate; returns success on yes.
+confirm() {
+  local reply=""
+  printf '  %s? %s [y/N] ' "$YELLOW" "$1"
+  read -r reply || true
+  [[ "$reply" =~ ^[Yy] ]]
+}
+
+# _existing KEY — current value of KEY in ENV_FILE, if any.
+_existing() {
+  [[ -f "$ENV_FILE" ]] || return 1
+  local line; line=$(grep -E "^${1}=" "$ENV_FILE" | tail -n1) || return 1
+  printf '%s' "${line#*=}"
+}
+
+# ask KEY "Prompt" — read a value into $KEY. Offers the existing .env value as
+# a default on re-runs (Enter keeps it). Visible input (non-secret).
+ask() {
+  local key="$1" prompt="$2" current input
+  current=$(_existing "$key" || true)
+  if [[ -n "$current" ]]; then
+    printf '  %s%s%s %s[Enter keeps current]%s ' "$BOLD" "$prompt" "$RESET" "$DIM" "$RESET"
+  else
+    printf '  %s%s%s ' "$BOLD" "$prompt" "$RESET"
+  fi
+  read -r input || true
+  [[ -z "$input" && -n "$current" ]] && input="$current"
+  printf -v "$key" '%s' "$input"
+}
+
+# ask_secret KEY "Prompt" — like ask, but input is hidden.
+ask_secret() {
+  local key="$1" prompt="$2" current input
+  current=$(_existing "$key" || true)
+  if [[ -n "$current" ]]; then
+    printf '  %s%s%s %s[Enter keeps current]%s ' "$BOLD" "$prompt" "$RESET" "$DIM" "$RESET"
+  else
+    printf '  %s%s%s ' "$BOLD" "$prompt" "$RESET"
+  fi
+  read -rs input || true
+  printf '\n'
+  [[ -z "$input" && -n "$current" ]] && input="$current"
+  printf -v "$key" '%s' "$input"
+}
+
+# write_env KEY VALUE — upsert KEY=VALUE into ENV_FILE (creates it; replaces
+# any existing line). Idempotent.
+write_env() {
+  local key="$1" value="$2" tmp
+  touch "$ENV_FILE"
+  tmp=$(mktemp)
+  grep -vE "^${key}=" "$ENV_FILE" > "$tmp" || true
+  printf '%s=%s\n' "$key" "$value" >> "$tmp"
+  mv "$tmp" "$ENV_FILE"
+  WRITTEN_ENV+=("$key")
+  printf '  %s✓ wrote%s %s → %s\n' "$GREEN" "$RESET" "$key" "$ENV_FILE"
+}
+
+# set_secret NAME VALUE — set a GitHub Actions repo secret via gh. Falls back
+# to a warning (and records it) if gh is unavailable or unauthenticated.
+set_secret() {
+  local name="$1" value="$2"
+  if command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; then
+    if printf '%s' "$value" | gh secret set "$name" >/dev/null 2>&1; then
+      WRITTEN_SECRET+=("$name")
+      printf '  %s✓ set%s GitHub secret %s\n' "$GREEN" "$RESET" "$name"
+      return
+    fi
+  fi
+  SKIPPED+=("GitHub secret $name (set it manually: gh secret set $name)")
+  warn "skipped GitHub secret $name — gh not ready; set it later"
+}
+
+# set_var NAME VALUE — set a GitHub Actions repo variable (non-secret).
+set_var() {
+  local name="$1" value="$2"
+  if command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; then
+    if gh variable set "$name" --body "$value" >/dev/null 2>&1; then
+      printf '  %s✓ set%s GitHub variable %s\n' "$GREEN" "$RESET" "$name"
+      return
+    fi
+  fi
+  SKIPPED+=("GitHub variable $name")
+  warn "skipped GitHub variable $name — gh not ready; set it later"
+}
+
+# finish — clear, then a closing summary of everything configured.
+finish() {
+  _clear
+  printf '\n%s%s  ✓ Setup complete%s\n' "$BOLD" "$GREEN" "$RESET"
+  (( ${#WRITTEN_ENV[@]} ))    && note "wrote ${#WRITTEN_ENV[@]} value(s) to $ENV_FILE: ${WRITTEN_ENV[*]}"
+  (( ${#WRITTEN_SECRET[@]} )) && note "set ${#WRITTEN_SECRET[@]} GitHub secret(s): ${WRITTEN_SECRET[*]}"
+  if (( ${#SKIPPED[@]} )); then
+    printf '\n'; warn "still to do by hand:"
+    for s in "${SKIPPED[@]}"; do note "  - $s"; done
+  fi
+  printf '\n'
+}
+
+# ──────────────────────────────────────────────────────────────────────────
+# STAGES — author this section. One stage() per step the human takes.
+# Replace the example below. Set the two totals to match the stages you write.
+# ──────────────────────────────────────────────────────────────────────────
+
+TOTAL_STAGES=7
+TOTAL_MINUTES=35
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+PLAN_ROOT="${CPH_SILO_PLAN_ROOT:-$HOME/.cph-silo-plans}"
+umask 077
+
+require_value() {
+  local key="$1" value="$2"
+  [[ -n "$value" ]] || { warn "$key is required"; exit 1; }
+  [[ "$value" != *$'\n'* && "$value" != *$'\r'* ]] || {
+    warn "$key must be a single line"
+    exit 1
+  }
+}
+
+seed_default() {
+  local key="$1" value="$2"
+  _existing "$key" >/dev/null 2>&1 || write_env "$key" "$value"
+}
+
+capture() {
+  local key="$1" prompt="$2"
+  ask "$key" "$prompt"
+  require_value "$key" "${!key}"
+  write_env "$key" "${!key}"
+}
+
+capture_secret() {
+  local key="$1" prompt="$2"
+  ask_secret "$key" "$prompt"
+  require_value "$key" "${!key}"
+  write_env "$key" "${!key}"
+}
+
+allocate_host_slot() {
+  local allocation local_reserved="" answers_file reserved_port
+  for answers_file in "$PLAN_ROOT"/*/answers.env; do
+    [ -f "$answers_file" ] || continue
+    reserved_port="$(sed -n 's/^HUB_PORT=//p' "$answers_file" | tail -n1)"
+    [[ "$reserved_port" =~ ^[0-9]+$ ]] || continue
+    local_reserved="${local_reserved:+$local_reserved,}$reserved_port"
+  done
+  allocation="$(ssh \
+    -i "$DEPLOY_SSH_KEY" \
+    -p "$DEPLOY_SSH_PORT" \
+    -o BatchMode=yes \
+    -o StrictHostKeyChecking=accept-new \
+    "$DEPLOY_USER@$DEPLOY_HOST" \
+    bash -s -- "$local_reserved" <<'REMOTE'
+      set -euo pipefail
+      local_reserved=",${1:-},"
+      for candidate in $(seq 8788 8999); do
+        reserved=false
+        if [[ "$local_reserved" == *",$candidate,"* ]]; then
+          continue
+        fi
+        for env_file in /srv/curriculum-project-hub/.secrets/*/platform.env; do
+          [ -f "$env_file" ] || continue
+          if [ "$(sed -n "s/^PORT=//p" "$env_file")" = "$candidate" ]; then
+            reserved=true
+            break
+          fi
+        done
+        workspace="/w/$candidate"
+        if [ "$reserved" = false ] && ! ss -H -ltn "sport = :$candidate" | grep -q . && [ ! -e "$workspace" ]; then
+          printf "%s %s\n" "$candidate" "$workspace"
+          exit 0
+        fi
+      done
+      echo "no free Alpha Silo slot in 8788..8999" >&2
+      exit 1
+REMOTE
+  )"
+  read -r HUB_PORT WORKSPACE_ROOT <<<"$allocation"
+  require_value HUB_PORT "$HUB_PORT"
+  require_value WORKSPACE_ROOT "$WORKSPACE_ROOT"
+  write_env HUB_PORT "$HUB_PORT"
+  write_env WORKSPACE_ROOT "$WORKSPACE_ROOT"
+}
+
+banner "New Alpha Silo"
+
+stage "Organization identity and private plan" 3
+say "One run creates one Organization's private deployment bundle."
+ask ORGANIZATION_SLUG "Organization slug (lowercase, max 24 chars; e.g. school-a):"
+require_value ORGANIZATION_SLUG "$ORGANIZATION_SLUG"
+[[ "$ORGANIZATION_SLUG" =~ ^[a-z0-9]([a-z0-9-]{0,22}[a-z0-9])?$ ]] || {
+  warn "invalid Organization slug"
+  exit 1
+}
+INSTANCE_ID="$ORGANIZATION_SLUG"
+ORGANIZATION_ID="$ORGANIZATION_SLUG"
+OUTPUT_DIR="$PLAN_ROOT/$INSTANCE_ID"
+mkdir -p "$OUTPUT_DIR"
+chmod 0700 "$OUTPUT_DIR"
+ENV_FILE="$OUTPUT_DIR/answers.env"
+touch "$ENV_FILE"
+chmod 0600 "$ENV_FILE"
+write_env INSTANCE_ID "$INSTANCE_ID"
+write_env ORGANIZATION_ID "$ORGANIZATION_ID"
+write_env ORGANIZATION_SLUG "$ORGANIZATION_SLUG"
+capture ORGANIZATION_NAME "Organization display name:"
+
+stage "Host, release, and isolation" 5
+say "The Alpha host is platform-managed. Port and short workspace path are allocated from live host state."
+DEPLOY_HOST="${CPH_ALPHA_HOST:-39.107.254.4}"
+DEPLOY_USER="${CPH_ALPHA_DEPLOY_USER:-root}"
+DEPLOY_SSH_PORT="${CPH_ALPHA_SSH_PORT:-22}"
+write_env DEPLOY_HOST "$DEPLOY_HOST"
+write_env DEPLOY_USER "$DEPLOY_USER"
+write_env DEPLOY_SSH_PORT "$DEPLOY_SSH_PORT"
+seed_default DEPLOY_BASE "/srv/curriculum-project-hub"
+seed_default DEPLOY_SSH_KEY "$HOME/.ssh/id_ed25519"
+write_env RELEASE_ID "v$(node -p 'require(process.argv[1]).version' "$REPO_ROOT/hub/package.json")"
+seed_default MEMORY_MAX "16G"
+seed_default CPU_QUOTA "400%"
+seed_default TASKS_MAX "512"
+seed_default CPH_BIN "/usr/local/bin/cph"
+note "Managed Alpha host: $DEPLOY_USER@$DEPLOY_HOST:$DEPLOY_SSH_PORT"
+DEPLOY_SSH_KEY="$(_existing DEPLOY_SSH_KEY)"
+MEMORY_MAX="$(_existing MEMORY_MAX)"
+CPU_QUOTA="$(_existing CPU_QUOTA)"
+TASKS_MAX="$(_existing TASKS_MAX)"
+[ -f "$DEPLOY_SSH_KEY" ] || { warn "managed SSH key is missing: $DEPLOY_SSH_KEY"; exit 1; }
+if [[ "$(_existing HUB_PORT || true)" =~ ^(878[8-9]|87[9][0-9]|8[89][0-9]{2})$ ]] && \
+   [ "$(_existing WORKSPACE_ROOT || true)" = "/w/$(_existing HUB_PORT)" ]; then
+  HUB_PORT="$(_existing HUB_PORT)"
+  WORKSPACE_ROOT="$(_existing WORKSPACE_ROOT)"
+  note "Keeping allocated host slot: port $HUB_PORT, workspace $WORKSPACE_ROOT"
+else
+  say "Checking existing Silo environments, listening sockets, and workspace paths..."
+  allocate_host_slot
+  note "Allocated host slot: port $HUB_PORT, workspace $WORKSPACE_ROOT"
+fi
+note "Platform ceilings: MemoryMax=$MEMORY_MAX, CPUQuota=$CPU_QUOTA, TasksMax=$TASKS_MAX"
+
+stage "Dedicated PostgreSQL database" 4
+say "A PostgreSQL server may be shared, but this Silo gets a distinct login role and database."
+seed_default DATABASE_HOST "127.0.0.1"
+seed_default DATABASE_PORT "5432"
+seed_default DATABASE_NAME "cph_${INSTANCE_ID//-/_}"
+seed_default DATABASE_USER "cph_${INSTANCE_ID//-/_}"
+DATABASE_NAME="$(_existing DATABASE_NAME)"
+if ! _existing DATABASE_PASSWORD >/dev/null 2>&1; then
+  DATABASE_PASSWORD="$(openssl rand -base64 36 | tr -d '\n')"
+  write_env DATABASE_PASSWORD "$DATABASE_PASSWORD"
+fi
+note "The generated OPERATE.md uses an interactive/protected SQL path; the password is never put in a command argument."
+
+stage "Public URL and Feishu app" 9
+open_url "https://open.feishu.cn/app"
+say "Create or open the Organization's own app. Copy credentials from Credentials & Basic Info."
+PUBLIC_BASE_URL="https://${ORGANIZATION_SLUG}.${CPH_ALPHA_BASE_DOMAIN:-educraft.paradigm-edu.net}"
+write_env PUBLIC_BASE_URL "$PUBLIC_BASE_URL"
+note "Platform-assigned public URL: $PUBLIC_BASE_URL"
+capture FEISHU_APP_ID "Feishu App ID:"
+capture_secret FEISHU_APP_SECRET "Feishu App Secret:"
+say "Resolving the bot Open ID from Feishu..."
+FEISHU_BOT_OPEN_ID="$(printf '%s\0%s\0' "$FEISHU_APP_ID" "$FEISHU_APP_SECRET" | node "$SCRIPT_DIR/resolve_feishu_bot.mjs")"
+write_env FEISHU_BOT_OPEN_ID "$FEISHU_BOT_OPEN_ID"
+note "Resolved bot identity: $FEISHU_BOT_OPEN_ID"
+open_url "https://open.feishu.cn/document/server-docs/contact-v3/user/get"
+step "In the user/get page, click the user_id value picker, select the first OWNER, and copy the returned open_id."
+capture OWNER_OPEN_ID "OWNER Open ID (ou_...):"
+capture OWNER_DISPLAY_NAME "OWNER display name:"
+write_env OWNER_UNION_ID ""
+say "The exact redirect URL and acceptance steps will be written to OPERATE.md."
+
+stage "Provider and Alpha limits" 5
+say "Use a provider credential exclusive to this Organization. Host proxy setup is a separate prerequisite."
+seed_default PROVIDER_ID "openrouter"
+seed_default PROVIDER_BASE_URL "https://openrouter.ai/api"
+seed_default DEFAULT_MODEL "anthropic/claude-sonnet-5"
+seed_default DEFAULT_ROLE_ID "draft"
+seed_default DEFAULT_ROLE_LABEL "智能助手"
+seed_default MAX_TURNS "25"
+seed_default MAX_CONCURRENT_RUNS "4"
+seed_default MAX_RUN_SECONDS "900"
+seed_default HTTP_BODY_LIMIT_BYTES "1048576"
+seed_default MAX_FILES_PER_MESSAGE "8"
+seed_default MAX_FILE_BYTES "26214400"
+seed_default HTTP_REQUESTS_PER_MINUTE "120"
+seed_default FEISHU_EVENTS_PER_MINUTE "120"
+capture_secret PROVIDER_AUTH_TOKEN "Provider auth token:"
+for key in PROVIDER_ID PROVIDER_BASE_URL DEFAULT_MODEL DEFAULT_ROLE_ID DEFAULT_ROLE_LABEL \
+  MAX_TURNS MAX_CONCURRENT_RUNS MAX_RUN_SECONDS HTTP_BODY_LIMIT_BYTES \
+  MAX_FILES_PER_MESSAGE MAX_FILE_BYTES HTTP_REQUESTS_PER_MINUTE FEISHU_EVENTS_PER_MINUTE; do
+  printf -v "$key" '%s' "$(_existing "$key")"
+done
+write_env APPROVED_SKILLS "outline,lesson-project,data-processing-spec,typst"
+note "Platform runtime: OpenRouter, concurrency 4, default education role, curated skills."
+if ! _existing HUB_SESSION_SECRET >/dev/null 2>&1; then
+  command -v openssl >/dev/null 2>&1 || { warn "openssl is required"; exit 1; }
+  HUB_SESSION_SECRET="$(openssl rand -hex 32)"
+  write_env HUB_SESSION_SECRET "$HUB_SESSION_SECRET"
+fi
+
+stage "Render the private deployment bundle" 2
+say "This renders platform.env, bootstrap.json, deploy.env, nginx.conf and OPERATE.md."
+command -v node >/dev/null 2>&1 || { warn "Node.js is required to render safely"; exit 1; }
+node "$SCRIPT_DIR/render_new_silo_bundle.mjs" "$ENV_FILE" "$OUTPUT_DIR"
+chmod 0700 "$OUTPUT_DIR"
+chmod 0600 "$OUTPUT_DIR"/*
+say "Bundle: $OUTPUT_DIR"
+warn "It contains database, Feishu, provider and session secrets. Never commit or paste it."
+
+stage "Operator handoff and gates" 7
+say "The deployment package is an internal retry/audit checkpoint; you do not operate it manually."
+step "Target: $PUBLIC_BASE_URL → $DEPLOY_HOST:$HUB_PORT"
+step "Resources: database $DATABASE_NAME, workspace $WORKSPACE_ROOT, service cph-hub-$INSTANCE_ID"
+warn "Confirmation will create server, database, TLS, and runtime state."
+if confirm "Deploy this Organization now?"; then
+  bash "$SCRIPT_DIR/apply_new_silo.sh" "$OUTPUT_DIR"
+else
+  warn "deployment skipped; rerun the wizard later and keep existing answers"
+fi
+
+finish

hub/deploy/render_new_silo_bundle.mjs

@@ -0,0 +1,217 @@
+#!/usr/bin/env node
+
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+const [answersPath, outputPath] = process.argv.slice(2);
+if (!answersPath || !outputPath) {
+  throw new Error("usage: render_new_silo_bundle.mjs ANSWERS_ENV OUTPUT_DIR");
+}
+
+function parseAnswers(source) {
+  const values = new Map();
+  for (const [index, line] of source.split("\n").entries()) {
+    if (!line || line.startsWith("#")) continue;
+    const separator = line.indexOf("=");
+    if (separator < 1) throw new Error(`invalid answers line ${index + 1}`);
+    const key = line.slice(0, separator);
+    const value = line.slice(separator + 1);
+    if (!/^[A-Z][A-Z0-9_]*$/.test(key)) {
+      throw new Error(`invalid answers key on line ${index + 1}: ${key}`);
+    }
+    if (value.includes("\r") || value.includes("\n")) {
+      throw new Error(`multiline value is not supported: ${key}`);
+    }
+    values.set(key, value);
+  }
+  return values;
+}
+
+function required(values, key) {
+  const value = values.get(key);
+  if (!value) throw new Error(`missing required answer: ${key}`);
+  return value;
+}
+
+function optional(values, key, fallback = "") {
+  return values.get(key) || fallback;
+}
+
+function assertMatch(label, value, pattern) {
+  if (!pattern.test(value)) throw new Error(`invalid ${label}: ${value}`);
+}
+
+function envLine(key, value) {
+  if (/[\r\n]/.test(value)) throw new Error(`unsafe newline in ${key}`);
+  return `${key}=${value}`;
+}
+
+const answers = parseAnswers(await readFile(resolve(answersPath), "utf8"));
+const instanceId = required(answers, "INSTANCE_ID");
+const orgId = required(answers, "ORGANIZATION_ID");
+const orgSlug = required(answers, "ORGANIZATION_SLUG");
+const port = required(answers, "HUB_PORT");
+const workspaceRoot = required(answers, "WORKSPACE_ROOT");
+const publicBaseUrl = required(answers, "PUBLIC_BASE_URL").replace(/\/$/, "");
+const domain = new URL(publicBaseUrl).hostname;
+const databasePassword = required(answers, "DATABASE_PASSWORD");
+const databaseUrl = `postgresql://${encodeURIComponent(required(answers, "DATABASE_USER"))}:${encodeURIComponent(databasePassword)}@${required(answers, "DATABASE_HOST")}:${required(answers, "DATABASE_PORT")}/${encodeURIComponent(required(answers, "DATABASE_NAME"))}`;
+
+assertMatch("INSTANCE_ID", instanceId, /^[a-z0-9](?:[a-z0-9-]{0,22}[a-z0-9])?$/);
+assertMatch("organization slug", orgSlug, /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/);
+assertMatch("port", port, /^[1-9][0-9]{1,4}$/);
+assertMatch("database name", required(answers, "DATABASE_NAME"), /^[a-z_][a-z0-9_]*$/);
+assertMatch("database user", required(answers, "DATABASE_USER"), /^[a-z_][a-z0-9_]*$/);
+assertMatch("default role id", required(answers, "DEFAULT_ROLE_ID"), /^[a-z][a-z0-9-]*$/);
+assertMatch("release id", required(answers, "RELEASE_ID"), /^[A-Za-z0-9._-]+$/);
+if (!publicBaseUrl.startsWith("https://")) throw new Error("PUBLIC_BASE_URL must use https");
+for (const [label, value] of [
+  ["WORKSPACE_ROOT", workspaceRoot],
+  ["DEPLOY_BASE", required(answers, "DEPLOY_BASE")],
+  ["DEPLOY_SSH_KEY", required(answers, "DEPLOY_SSH_KEY")],
+  ["CPH_BIN", required(answers, "CPH_BIN")],
+]) {
+  if (!value.startsWith("/") || /\s/.test(value)) {
+    throw new Error(`${label} must be an absolute path without whitespace: ${value}`);
+  }
+}
+if (Number(port) > 65535) throw new Error(`invalid port: ${port}`);
+if (Buffer.byteLength(workspaceRoot) > 16) {
+  throw new Error(`WORKSPACE_ROOT exceeds the 16-byte sandbox socket limit: ${workspaceRoot}`);
+}
+
+const outputDir = resolve(outputPath);
+await mkdir(outputDir, { recursive: true, mode: 0o700 });
+await chmod(outputDir, 0o700);
+
+const base = required(answers, "DEPLOY_BASE");
+const release = required(answers, "RELEASE_ID");
+const releaseHub = `${base}/releases/${release}/hub`;
+const secretDir = `${base}/.secrets/${instanceId}`;
+const envPath = `${secretDir}/platform.env`;
+const keyringPath = `${secretDir}/secret-keyring.json`;
+const unit = `cph-hub-${instanceId}.service`;
+
+const platformEnv = [
+  "# Generated by hub/deploy/new_silo.sh. Install root:root mode 0600.",
+  envLine("NODE_ENV", "production"),
+  envLine("DATABASE_URL", databaseUrl),
+  envLine("HUB_SILO_ORGANIZATION_ID", orgId),
+  envLine("HUB_SYSTEMD_UNIT", unit),
+  envLine("CPH_BIN", required(answers, "CPH_BIN")),
+  envLine("HOST", "127.0.0.1"),
+  envLine("PORT", port),
+  envLine("HUB_PROJECT_WORKSPACE_ROOT", workspaceRoot),
+  envLine("HUB_PUBLIC_BASE_URL", publicBaseUrl),
+  envLine("HUB_SESSION_SECRET", required(answers, "HUB_SESSION_SECRET")),
+  envLine("HUB_AGENT_MAX_TURNS", required(answers, "MAX_TURNS")),
+  envLine("HUB_AGENT_MAX_CONCURRENT_RUNS", required(answers, "MAX_CONCURRENT_RUNS")),
+  envLine("HUB_AGENT_MAX_RUN_SECONDS", required(answers, "MAX_RUN_SECONDS")),
+  envLine("HUB_HTTP_BODY_LIMIT_BYTES", required(answers, "HTTP_BODY_LIMIT_BYTES")),
+  envLine("HUB_MAX_FILES_PER_MESSAGE", required(answers, "MAX_FILES_PER_MESSAGE")),
+  envLine("HUB_MAX_FILE_BYTES", required(answers, "MAX_FILE_BYTES")),
+  envLine("HUB_HTTP_REQUESTS_PER_MINUTE", required(answers, "HTTP_REQUESTS_PER_MINUTE")),
+  envLine("HUB_FEISHU_EVENTS_PER_MINUTE", required(answers, "FEISHU_EVENTS_PER_MINUTE")),
+  envLine("HUB_FEISHU_LISTENER_ENABLED", "true"),
+  envLine("HTTP_PROXY", "http://127.0.0.1:7890"),
+  envLine("HTTPS_PROXY", "http://127.0.0.1:7890"),
+  envLine("ALL_PROXY", "socks5h://127.0.0.1:7890"),
+  envLine("NO_PROXY", "127.0.0.1,localhost,::1"),
+  envLine("NODE_USE_ENV_PROXY", "1"),
+  envLine("ANTHROPIC_DEFAULT_SONNET_MODEL", "anthropic/claude-sonnet-5"),
+  envLine("CPH_SANDBOX_EXTRA_DENY_READ", `${envPath}:${keyringPath}`),
+  "",
+].join("\n");
+
+const bootstrap = {
+  organization: {
+    id: orgId,
+    slug: orgSlug,
+    name: required(answers, "ORGANIZATION_NAME"),
+  },
+  owner: {
+    openId: required(answers, "OWNER_OPEN_ID"),
+    displayName: required(answers, "OWNER_DISPLAY_NAME"),
+  },
+  feishu: {
+    appId: required(answers, "FEISHU_APP_ID"),
+    appSecret: required(answers, "FEISHU_APP_SECRET"),
+    botOpenId: required(answers, "FEISHU_BOT_OPEN_ID"),
+  },
+  provider: {
+    providerId: required(answers, "PROVIDER_ID"),
+    baseUrl: required(answers, "PROVIDER_BASE_URL"),
+    authToken: required(answers, "PROVIDER_AUTH_TOKEN"),
+  },
+  teams: [],
+};
+const ownerUnionId = optional(answers, "OWNER_UNION_ID");
+if (ownerUnionId) bootstrap.owner.unionId = ownerUnionId;
+
+const deployEnv = [
+  "# Source this file locally before deploy_platform.sh (contains no app/provider secrets).",
+  envLine("PLATFORM_DEPLOY_HOST", required(answers, "DEPLOY_HOST")),
+  envLine("PLATFORM_DEPLOY_SSH_KEY", required(answers, "DEPLOY_SSH_KEY")),
+  envLine("PLATFORM_DEPLOY_USER", required(answers, "DEPLOY_USER")),
+  envLine("PLATFORM_DEPLOY_PORT", required(answers, "DEPLOY_SSH_PORT")),
+  envLine("PLATFORM_DEPLOY_HUB_PORT", port),
+  envLine("PLATFORM_DEPLOY_BASE", base),
+  envLine("PLATFORM_DEPLOY_RELEASE", release),
+  envLine("PLATFORM_DEPLOY_INSTANCE", instanceId),
+  envLine("PLATFORM_DEPLOY_WORKSPACE_ROOT", workspaceRoot),
+  envLine("PLATFORM_DEPLOY_MEMORY_MAX", required(answers, "MEMORY_MAX")),
+  envLine("PLATFORM_DEPLOY_CPU_QUOTA", required(answers, "CPU_QUOTA")),
+  envLine("PLATFORM_DEPLOY_TASKS_MAX", required(answers, "TASKS_MAX")),
+  envLine("PLATFORM_DEPLOY_HEALTH_URL", `http://127.0.0.1:${port}/api/healthz`),
+  "",
+].join("\n");
+
+const nginx = `# Install as /etc/nginx/conf.d/${instanceId}.conf after obtaining TLS certificates.\nserver {\n  listen 80;\n  server_name ${domain};\n  return 301 https://$host$request_uri;\n}\n\nserver {\n  listen 443 ssl http2;\n  server_name ${domain};\n\n  ssl_certificate /etc/letsencrypt/live/${domain}/fullchain.pem;\n  ssl_certificate_key /etc/letsencrypt/live/${domain}/privkey.pem;\n\n  location / {\n    proxy_pass http://127.0.0.1:${port};\n    proxy_http_version 1.1;\n    proxy_set_header Host $host;\n    proxy_set_header X-Real-IP $remote_addr;\n    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n    proxy_set_header X-Forwarded-Proto https;\n  }\n}\n`;
+
+const defaultRolePrompt = `你是一位教育机构智能助手，服务于学校和教培机构的日常管理与协作。
+## 核心能力
+**教务管理**：熟悉排课调课、班级管理、课时统计、师资调度等常见管理流程。
+**教研支持**：理解课程设计、教案编写、例题/变式题、定理证明、随堂练习、阶段性测验、知识点拆解、大纲对标等教学业务概念。
+**数字化操作**：熟练使用shell命令行进行文件管理、批量处理、脚本编写、文本处理（grep/sed/awk等）、自动化任务；能协助处理Markdown、Typst等文档、表格数据、格式转换等。
+## 行为准则
+- **实事求是**：不确定时明确说明，能力边界外的任务如实告知，不为完成目标而编造信息或勉强输出
+- **简洁优先**：直接给出答案或方案，省略铺垫和客套
+- **按需展开**：仅在问题复杂或用户明确要求时提供详细说明
+- **结构清晰**：多步骤内容使用编号，便于飞书阅读
+- **可操作性**：涉及操作时给出具体命令或步骤
+保持专业、高效，像一位熟悉业务的同事。
+`;
+
+const operate = `# ${instanceId} deployment commands\n\nThis file contains no application/provider secrets. Run commands deliberately; do not source answers.env.\n\n## 1. DNS and Feishu\n\n- Point \`${domain}\` to \`${required(answers, "DEPLOY_HOST")}\`.\n- In the Feishu app, configure redirect URL: \`${publicBaseUrl}/auth/feishu/callback\`.\n- Enable the required bot/message/contact permissions and event subscription described in the customer setup document.\n\n## 2. Database (run as the PostgreSQL administrator)\n\nCreate one role and one database for this Silo. The password is in \`answers.env\`; use an interactive client or a protected SQL file, never a command-line argument.\n\n\`\`\`sql\nCREATE ROLE ${required(answers, "DATABASE_USER")} LOGIN PASSWORD '<copy DATABASE_PASSWORD from answers.env>';\nCREATE DATABASE ${required(answers, "DATABASE_NAME")} OWNER ${required(answers, "DATABASE_USER")};\n\`\`\`\n\n## 3. Publish the immutable release\n\nThe normal deploy script seeds the first-instance secrets and exits 78. That exit is expected only on this first pass.\n\n\`\`\`sh\nset -a; . ./deploy.env; set +a\nbash hub/deploy/deploy_platform.sh\n\`\`\`\n\n## 4. Install the generated secrets\n\nCopy \`platform.env\` and \`bootstrap.json\` to the server through a protected channel. On the server:\n\n\`\`\`sh\ninstall -d -o root -g root -m 0700 '${secretDir}'\ninstall -o root -g root -m 0600 platform.env '${envPath}'\ninstall -o root -g root -m 0600 bootstrap.json '/root/${instanceId}-bootstrap.json'\n# Copy ${keyringPath} to separate off-host recovery storage before continuing.\n\`\`\`\n\n## 5. Migrate, install the stopped service, and bootstrap\n\n\`\`\`sh\nset -a; . '${envPath}'; set +a\nnode '${releaseHub}/node_modules/prisma/build/index.js' migrate deploy --schema '${releaseHub}/prisma/schema.prisma'\nBASE='${base}' HUB_DIR='${releaseHub}' INSTANCE_ID='${instanceId}' WORKSPACE_ROOT='${workspaceRoot}' PORT='${port}' MEMORY_MAX='${required(answers, "MEMORY_MAX")}' CPU_QUOTA='${required(answers, "CPU_QUOTA")}' TASKS_MAX='${required(answers, "TASKS_MAX")}' bash '${releaseHub}/deploy/install_service.sh'\nnode '${releaseHub}/dist/deployment/bootstrap-silo-cli.js' --config-file '/root/${instanceId}-bootstrap.json' --keyring-file '${keyringPath}'\nrm -f '/root/${instanceId}-bootstrap.json'\n\`\`\`\n\n## 6. Runtime role and skills\n\nRuntime role/skill configuration is intentionally separate from the release. Follow NEW_SILO_RUNBOOK.md, staging each approved skill directory outside the immutable release, then use \`agent_config.sh\`. Do not silently copy a local personal skill collection.\n\n## 7. Nginx, start, and acceptance\n\nInstall \`nginx.conf\`, run \`nginx -t\`, reload Nginx, then:\n\n\`\`\`sh\nsystemctl start '${unit}'\nsystemctl is-active '${unit}'\ncurl --fail --silent --show-error 'http://127.0.0.1:${port}/api/healthz'\ncurl --fail --silent --show-error '${publicBaseUrl}/api/healthz'\njournalctl -u '${unit}' --since '-10 min' --no-pager\n\`\`\`\n\nAcceptance requires: Feishu OAuth completes, OWNER and a non-OWNER can @bot, a second group creates a separate project/session, Typst works when that skill is enabled, and restart preserves session cursor. Then take the first off-host backup.\n\n## 8. Later releases\n\nAfter the instance exists, source \`deploy.env\` with the new release id and run \`deploy_platform.sh\`. Never reuse another org's database, secret directory, workspace root, port, service identity, domain, Feishu app, or provider credential.\n`;
+
+const selectedSkills = optional(answers, "APPROVED_SKILLS");
+const roleInstructions = `
+## Appendix: default runtime role
+
+Copy \`default-role-prompt.md\` to \`/root/${instanceId}-default-role-prompt.md\`.
+After installing each reviewed skill with \`agent_config.sh install-skill\`, run:
+
+\`\`\`sh
+INSTANCE_ID=${JSON.stringify(instanceId)} ENV_FILE=${JSON.stringify(envPath)} HUB_DIR=${JSON.stringify(releaseHub)} bash ${JSON.stringify(`${releaseHub}/deploy/agent_config.sh`)} upsert-role --organization ${JSON.stringify(orgId)} --role ${JSON.stringify(required(answers, "DEFAULT_ROLE_ID"))} --label ${JSON.stringify(required(answers, "DEFAULT_ROLE_LABEL"))} --model ${JSON.stringify(required(answers, "DEFAULT_MODEL"))} --system-prompt-file ${JSON.stringify(`/root/${instanceId}-default-role-prompt.md`)} --tools-json null
+${selectedSkills ? `INSTANCE_ID=${JSON.stringify(instanceId)} ENV_FILE=${JSON.stringify(envPath)} HUB_DIR=${JSON.stringify(releaseHub)} bash ${JSON.stringify(`${releaseHub}/deploy/agent_config.sh`)} set-role-skills --organization ${JSON.stringify(orgId)} --role ${JSON.stringify(required(answers, "DEFAULT_ROLE_ID"))} --skills ${JSON.stringify(selectedSkills)}
+` : "# No skills selected in the wizard; set-role-skills remains an explicit operator step.\n"}INSTANCE_ID=${JSON.stringify(instanceId)} ENV_FILE=${JSON.stringify(envPath)} HUB_DIR=${JSON.stringify(releaseHub)} bash ${JSON.stringify(`${releaseHub}/deploy/agent_config.sh`)} list --organization ${JSON.stringify(orgId)}
+\`\`\`
+
+Changing a role's model, system prompt, tools or skill selection archives that
+role's existing sessions by design. Finish this setup before inviting Alpha users.
+`;
+
+async function privateWrite(name, contents) {
+  const path = resolve(outputDir, name);
+  await writeFile(path, contents, { encoding: "utf8", mode: 0o600 });
+  await chmod(path, 0o600);
+}
+
+await privateWrite("platform.env", platformEnv);
+await privateWrite("bootstrap.json", `${JSON.stringify(bootstrap, null, 2)}\n`);
+await privateWrite("deploy.env", deployEnv);
+await privateWrite("nginx.conf", nginx);
+await privateWrite("default-role-prompt.md", defaultRolePrompt);
+await privateWrite("OPERATE.md", `${operate}${roleInstructions}`);
+
+console.log(`Rendered private Silo bundle: ${outputDir}`);

hub/deploy/resolve_feishu_bot.mjs

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+
+const chunks = [];
+for await (const chunk of process.stdin) chunks.push(chunk);
+const [appId, appSecret] = Buffer.concat(chunks).toString("utf8").split("\0");
+if (!appId || !appSecret) throw new Error("Feishu App ID and App Secret are required on stdin");
+
+const tokenResponse = await fetch("https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal", {
+  method: "POST",
+  headers: { "content-type": "application/json; charset=utf-8" },
+  body: JSON.stringify({ app_id: appId, app_secret: appSecret }),
+});
+if (!tokenResponse.ok) throw new Error(`Feishu token request failed: HTTP ${tokenResponse.status}`);
+const tokenPayload = await tokenResponse.json();
+if (tokenPayload.code !== 0 || typeof tokenPayload.tenant_access_token !== "string") {
+  throw new Error(`Feishu token request failed: ${JSON.stringify(tokenPayload)}`);
+}
+
+const botResponse = await fetch("https://open.feishu.cn/open-apis/bot/v3/info", {
+  headers: { authorization: `Bearer ${tokenPayload.tenant_access_token}` },
+});
+if (!botResponse.ok) throw new Error(`Feishu bot info request failed: HTTP ${botResponse.status}`);
+const botPayload = await botResponse.json();
+if (botPayload.code !== 0 || typeof botPayload.bot?.open_id !== "string" || !botPayload.bot.open_id) {
+  throw new Error(`Feishu bot info request failed: ${JSON.stringify(botPayload)}`);
+}
+process.stdout.write(botPayload.bot.open_id);