feat(hub): add SearchableSelectField component and update RoleCard to use it

Add a shared transparent OrganizationAgentConfigFolder tree for grouping
agent roles and skills in the admin UI without affecting identity, bindings,
run loading, or slash commands.

.gitea/workflows/checker-check.yml

@@ -19,8 +19,10 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: Install Rust toolchain (stable + clippy + rustfmt)
-        uses: dtolnay/rust-toolchain@stable
+      # Keep local rustup, rustfmt output, clippy, and CI on one reproducible
+      # compiler release. Update this together with rust-toolchain.toml.
+      - name: Install Rust toolchain (1.92.0 + clippy + rustfmt)
+        uses: dtolnay/rust-toolchain@1.92.0
         with:
           components: clippy, rustfmt
 
@@ -35,6 +37,12 @@ jobs:
           restore-keys: |
             cargo-${{ runner.os }}-
 
+      - name: Install cargo-audit 0.22.2
+        run: cargo install cargo-audit --version 0.22.2 --locked
+
+      - name: Audit Rust vulnerabilities and unsoundness
+        run: cargo audit --deny unsound
+
       # Fonts for typst compilation (WU-4 onward): the engineering files are
       # CJK-heavy, so the compile-check / build tests need a CJK font present.
       # Harmless before those crates exist.

.gitea/workflows/deploy-admin.yml

@@ -0,0 +1,126 @@
+name: deploy admin (hub fleet)
+
+# Rolls Hub releases that include the org-admin SPA (`admin-web` → registerStaticSpa)
+# onto the managed Alpha host. Two fleets share one machine and are separated by
+# the middle DNS label of each Silo's public URL:
+#
+#   dev  → https://<slug>.educraft-dev.paradigm-edu.net   (every push + PR)
+#   prod → https://<slug>.educraft.paradigm-edu.net       (main + tags)
+#
+# Example prod tenant:
+#   https://para-26071100.educraft.paradigm-edu.net/auth/feishu/para-26071100
+#
+# Required Gitea secret:
+#   DEPLOY_SSH_KEY  — private key for root@39.107.254.4
+#
+# Optional secrets / vars (defaults match NEW_SILO_RUNBOOK):
+#   DEPLOY_HOST, DEPLOY_USER, DEPLOY_SSH_PORT, DEPLOY_BASE
+#   vars.ALLOW_EMPTY_FLEET=1  — green when the fleet has no silos yet
+
+on:
+  push:
+    paths:
+      - "hub/**"
+      - ".gitea/workflows/deploy-admin.yml"
+  pull_request:
+    paths:
+      - "hub/**"
+      - ".gitea/workflows/deploy-admin.yml"
+  workflow_dispatch:
+    inputs:
+      fleet:
+        description: "Which fleet to deploy (dev | prod | both)"
+        required: true
+        default: both
+      allow_empty_fleet:
+        description: "Succeed if no silo matches (1/0)"
+        required: false
+        default: "0"
+
+concurrency:
+  group: deploy-admin-host
+  cancel-in-progress: false
+
+jobs:
+  deploy-dev:
+    name: deploy fleet dev (educraft-dev)
+    runs-on: ubuntu-latest
+    if: >
+      github.event_name == 'pull_request' ||
+      (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) ||
+      (github.event_name == 'workflow_dispatch' &&
+        (github.event.inputs.fleet == 'dev' || github.event.inputs.fleet == 'both'))
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install rsync + ssh
+        run: sudo apt-get update && sudo apt-get install -y rsync openssh-client
+
+      - name: Deploy Hub + admin SPA to educraft-dev fleet
+        env:
+          CPH_FLEET: dev
+          PLATFORM_DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+          PLATFORM_DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
+          PLATFORM_DEPLOY_PORT: ${{ secrets.DEPLOY_SSH_PORT }}
+          PLATFORM_DEPLOY_BASE: ${{ secrets.DEPLOY_BASE }}
+          PLATFORM_DEPLOY_RELEASE: ${{ github.sha }}
+          ALLOW_EMPTY_FLEET: ${{ github.event.inputs.allow_empty_fleet || vars.ALLOW_EMPTY_FLEET || '0' }}
+          DEPLOY_SSH_KEY_BODY: ${{ secrets.DEPLOY_SSH_KEY }}
+        run: |
+          set -euo pipefail
+          if [ -z "${DEPLOY_SSH_KEY_BODY:-}" ]; then
+            echo "missing secret DEPLOY_SSH_KEY" >&2
+            exit 1
+          fi
+          key="$(mktemp)"
+          trap 'rm -f "$key"' EXIT
+          printf '%s\n' "$DEPLOY_SSH_KEY_BODY" >"$key"
+          chmod 600 "$key"
+          export PLATFORM_DEPLOY_SSH_KEY="$key"
+          # Apply managed-host defaults when secrets are unset.
+          export PLATFORM_DEPLOY_HOST="${PLATFORM_DEPLOY_HOST:-39.107.254.4}"
+          export PLATFORM_DEPLOY_USER="${PLATFORM_DEPLOY_USER:-root}"
+          export PLATFORM_DEPLOY_PORT="${PLATFORM_DEPLOY_PORT:-22}"
+          export PLATFORM_DEPLOY_BASE="${PLATFORM_DEPLOY_BASE:-/srv/curriculum-project-hub}"
+          bash hub/deploy/deploy_fleet_release.sh
+
+  deploy-prod:
+    name: deploy fleet prod (educraft)
+    runs-on: ubuntu-latest
+    if: >
+      (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
+      (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) ||
+      (github.event_name == 'workflow_dispatch' &&
+        (github.event.inputs.fleet == 'prod' || github.event.inputs.fleet == 'both'))
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install rsync + ssh
+        run: sudo apt-get update && sudo apt-get install -y rsync openssh-client
+
+      - name: Deploy Hub + admin SPA to educraft fleet
+        env:
+          CPH_FLEET: prod
+          PLATFORM_DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+          PLATFORM_DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
+          PLATFORM_DEPLOY_PORT: ${{ secrets.DEPLOY_SSH_PORT }}
+          PLATFORM_DEPLOY_BASE: ${{ secrets.DEPLOY_BASE }}
+          PLATFORM_DEPLOY_RELEASE: ${{ github.sha }}
+          ALLOW_EMPTY_FLEET: ${{ github.event.inputs.allow_empty_fleet || vars.ALLOW_EMPTY_FLEET || '0' }}
+          DEPLOY_SSH_KEY_BODY: ${{ secrets.DEPLOY_SSH_KEY }}
+        run: |
+          set -euo pipefail
+          if [ -z "${DEPLOY_SSH_KEY_BODY:-}" ]; then
+            echo "missing secret DEPLOY_SSH_KEY" >&2
+            exit 1
+          fi
+          key="$(mktemp)"
+          trap 'rm -f "$key"' EXIT
+          printf '%s\n' "$DEPLOY_SSH_KEY_BODY" >"$key"
+          chmod 600 "$key"
+          export PLATFORM_DEPLOY_SSH_KEY="$key"
+          export PLATFORM_DEPLOY_HOST="${PLATFORM_DEPLOY_HOST:-39.107.254.4}"
+          export PLATFORM_DEPLOY_USER="${PLATFORM_DEPLOY_USER:-root}"
+          export PLATFORM_DEPLOY_PORT="${PLATFORM_DEPLOY_PORT:-22}"
+          export PLATFORM_DEPLOY_BASE="${PLATFORM_DEPLOY_BASE:-/srv/curriculum-project-hub}"
+          bash hub/deploy/deploy_fleet_release.sh

.gitea/workflows/hub-check.yml

@@ -0,0 +1,121 @@
+name: hub check
+
+# Builds, type-checks, and tests the Hub TS package under hub/.
+# The Hub is the Feishu-group collaboration + agent runtime half
+# (spec/System implementation). This is an INTERNAL gate on the Hub's own
+# health, like checker-check is for the Rust half.
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  hub-check:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: paradigm
+          POSTGRES_PASSWORD: paradigm
+          POSTGRES_DB: cph_hub_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U paradigm -d cph_hub_test"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    defaults:
+      run:
+        working-directory: hub
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+          cache: npm
+          cache-dependency-path: hub/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Audit production Node dependencies
+        run: npm run audit:production
+
+      - name: Install Linux sandbox dependency
+        run: sudo apt-get update && sudo apt-get install --yes bubblewrap socat
+
+      - name: Wait for Postgres
+        run: |
+          node <<'NODE'
+          const net = require("node:net");
+          const deadline = Date.now() + 60000;
+          function tryConnect() {
+            const socket = net.createConnection({ host: "127.0.0.1", port: 5432 });
+            socket.once("connect", () => {
+              socket.end();
+              process.exit(0);
+            });
+            socket.once("error", () => {
+              socket.destroy();
+              if (Date.now() > deadline) {
+                console.error("Postgres did not become reachable at 127.0.0.1:5432");
+                process.exit(1);
+              }
+              setTimeout(tryConnect, 1000);
+            });
+          }
+          tryConnect();
+          NODE
+
+      # Prisma client must be generated before tsc can check imports from
+      # @prisma/client. Stub DATABASE_URL so prisma generate works.
+      - name: Generate Prisma client
+        run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma generate --schema prisma/schema.prisma
+
+      - name: Type-check
+        run: npx tsc -p tsconfig.json --noEmit
+
+      - name: Build
+        run: npm run build
+
+      - name: Validate Prisma schema
+        run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma validate --schema prisma/schema.prisma
+
+      - name: Install cph binary
+        run: |
+          cd ..
+          cargo install --path crates/cph-cli --locked
+
+      - name: Prove real Claude SDK Bash sandbox boundary
+        run: |
+          sudo install -d -o "$(id -u)" -g "$(id -g)" -m 0700 /w/t
+          CPH_SANDBOX_TEST_ROOT=/w/t \
+            /usr/bin/setpriv --no-new-privs \
+            npx vitest run test/integration/agent-sandbox-linux.test.ts
+
+      - name: Run unit tests
+        run: npx vitest run test/unit
+
+      # Integration tests need PostgreSQL + cph. cph is installed above.
+      # PostgreSQL is set up as a service container below.
+      - name: Run integration tests (mock provider, real prisma + cph)
+        run: |
+          npx prisma migrate deploy --schema prisma/schema.prisma
+          npx vitest run test/integration \
+            --exclude test/integration/real-model.test.ts \
+            --exclude test/integration/agent-sandbox-linux.test.ts
+        env:
+          DATABASE_URL: postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test
+
+      # Real-model tests are opt-in: set RUN_REAL_MODEL_TESTS=true and provide
+      # OPENROUTER_API_KEY when a branch should hit live OpenRouter.
+      - name: Run real-model integration tests (optional, needs secret)
+        run: npx vitest run test/integration/real-model.test.ts
+        env:
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          RUN_REAL_MODEL_TESTS: ${{ vars.RUN_REAL_MODEL_TESTS }}

.gitignore

@@ -11,6 +11,9 @@
 # regenerable, not for VCS. The embedded engine mounts cph-render directly.
 render/vendor/local-packages/
 
+# Environment
+.env
+
 # Node (hub/ TS workspace and any future JS package)
 node_modules/
 

.scratch/saas-production-readiness/assets/backup-migration-disaster-recovery-audit.md

@@ -0,0 +1,325 @@
+# Backup, migration, and disaster-recovery audit
+
+## Verdict
+
+The current Hub has no production backup or restore control plane. PostgreSQL
+can be dumped and restored manually, but the service's authoritative state is
+split between PostgreSQL and mutable Project workspaces, with no shared
+consistency point, backup catalog, off-host retention, restore workflow,
+recovery objectives, or recovery drill.
+
+The current deployment path can also delete customer workspaces when the
+documented relative workspace root is used: `deploy_platform.sh` applies
+`rsync --delete` to the live Hub directory while `.env.example` places
+workspaces below `./data/project-workspaces`. Project creation has the same
+cross-store weakness at a smaller scale: it creates the workspace before the
+database transaction and leaves an orphan directory when that transaction
+fails.
+
+Migration safety is incomplete even though the current PostgreSQL migration
+engine rolled back a deliberately failing migration's DDL in the executed
+probe. The failed migration record blocks later migrations, migrations execute
+inside every service start, no compatibility declaration or production-data
+rehearsal exists, and current history contains destructive changes that make an
+old application query fail after schema upgrade. Application rollback is
+therefore not database rollback.
+
+## Contract and persistent-state baseline
+
+The audit preserves the accepted contracts rather than inventing missing
+product policy:
+
+- `Organization` is the tenant root, but `User` is global and can belong to
+  multiple Organizations. A selective Organization restore is not equivalent
+  to restoring a self-contained database schema.
+- Project workspaces are authoritative curriculum-engineering files bounded by
+  ADR-0018. PostgreSQL stores their absolute `Project.workspaceDir` locations;
+  it does not store their file content.
+- Run, provider cursor, ingress, delivery, audit, and workspace commit/recovery
+  semantics remain split across the unresolved lifecycle tickets. A backup
+  design cannot silently choose their crash behavior.
+- ADR-0022 requires storage ceilings and emergency `DRAIN`/`STOP_NOW`, but does
+  not decide backup frequency, retention, recovery objectives, or tenant-level
+  restore scope.
+- Audit retention is still `OPEN`; regulatory backup retention and deletion
+  obligations require product/legal input already recorded in the map's Fog.
+
+The recovery set has at least four classes:
+
+| State | Current location | Recovery significance |
+| --- | --- | --- |
+| Tenant and workflow state | PostgreSQL | Organizations, identities, membership, permissions, Feishu bindings, sessions, runs, locks, receipts, usage facts, and audit records |
+| Customer artifacts | `HUB_PROJECT_WORKSPACE_ROOT` | Project source files, generated artifacts, inbound attachments, and any workspace-local session/history files |
+| Credentials and configuration | `/srv/.../.secrets/platform.env` today; future org-scoped encrypted connections plus external key material | Database/Feishu/provider access and ability to decrypt restored Organization credentials |
+| Release identity | application release, Prisma migrations, `cph`, runtime/toolchain manifest | Required to start a schema-compatible application and reproduce outputs after restore |
+
+Caches, build output, and reinstallable dependencies should be classified as
+regenerable rather than silently mixed into the protected recovery set.
+
+## Executed evidence
+
+All probes used isolated temporary paths or databases. No production or shared
+test data was modified.
+
+### 1. The deployment rsync pattern deletes an in-tree workspace
+
+A temporary live Hub directory was given the documented workspace shape and a
+sentinel customer file, then the exact deployment rsync include/exclude pattern
+was applied:
+
+```text
+rsync -a --delete \
+  --exclude node_modules --exclude dist --exclude .env \
+  hub/ <temporary-live-hub>/
+
+workspace_survived=false
+reproduced: deploy rsync pattern deletes an in-tree workspace root
+```
+
+This is deterministic: `data/` is absent from the release source and is not an
+exclude, so `--delete` removes it from the destination. The root cause is that
+the documented persistent path is nested inside the mutable release tree and
+the deploy contract does not distinguish release artifacts from state.
+
+### 2. A rejected Project transaction leaves an orphan workspace
+
+A minimal Node harness called the real built `createProjectFromOrgAdmin` with a
+valid Organization admin and a missing Folder. The fake Prisma boundary rejected
+inside the same transaction seam used by production. The harness asserted that
+the Organization workspace directory must remain empty and was run twice:
+
+```text
+node .scratch/saas-production-readiness/probes/orphan-workspace.mjs
+
+orphan_workspace_entries=project_4aa0b6792c4547059d2d92b505b9299e
+exit_code=1
+
+orphan_workspace_entries=project_190e21afb8854244872b1241a600aa58
+exit_code=1
+```
+
+`createManagedProject` calls `mkdir` before `prisma.$transaction`; the failure
+path has no compensating removal, staged rename, or orphan reconciler. The
+throwaway harness was removed after reproduction.
+
+### 3. PostgreSQL rolled back failed migration DDL but retained a blocking failure
+
+An isolated database and throwaway Prisma project applied one migration that
+created a table, divided by zero, then would have created another table. The
+real repository Prisma CLI returned `P3018`:
+
+```text
+migration_exit=1
+before_failure=missing
+after_failure=missing
+20260710000000_partial_failure|finished=false|rolled_back=false|steps=0|logs=true
+```
+
+This specific PostgreSQL/Prisma 6.19.3 path did **not** leave partial DDL, so the
+audit does not claim that it did. It did leave an unfinished
+`_prisma_migrations` row, and Prisma's error explicitly refused new migrations
+until operator recovery. Because systemd runs `migrate deploy` as
+`ExecStartPre`, the application cannot start past that state. The throwaway
+database and migration were removed. This one multi-statement result is not a
+general atomicity guarantee: Prisma's first-party guidance says PostgreSQL
+migrations are not transaction-wrapped by default, so each migration must make
+its transaction and partial-failure behavior explicit.
+
+### 4. The current schema is not backward-compatible with an older application
+
+The application immediately before the typed-principal migration queried
+`PermissionGrant.principal`. That migration drops the column in favor of
+`principalType` plus `principalId`. Running the old read against the current
+test database produced:
+
+```text
+ERROR: column "principal" does not exist
+HINT: Perhaps you meant to reference the column "PermissionGrant.principalId".
+old_query_exit=1
+```
+
+This is direct evidence that switching the application symlink back after that
+migration cannot restore service. The migration history also deletes duplicate
+grant rows and later drops `ExternalPrincipalMembership.source`; there are no
+downgrade migrations.
+
+### 5. A database-only custom dump restores cleanly
+
+As a positive baseline, `pg_dump --format=custom` captured the current
+`cph_hub_test` database and `pg_restore --exit-on-error --no-owner` restored it
+into a new isolated database:
+
+```text
+restored_tables=24
+finished_migrations=9
+failed_migrations=0
+dump_bytes=76113
+```
+
+All nine finished migration checksums in the source database also matched the
+checked-in migration files. This proves a manual logical database round trip on
+the current PostgreSQL 14.20 workstation. It does not prove workspace,
+credentials, global database roles, production volume, cross-version restore,
+application readiness, or an operational RPO/RTO.
+
+## Critical findings
+
+### 1. No backup, retention, or restore mechanism exists
+
+The repository has no backup/restore command, systemd timer, off-host target,
+encryption/immutability policy, retention policy, backup manifest/checksum,
+success/failure alert, restore command, or scheduled recovery drill. The
+installer assumes PostgreSQL and its role/database already exist. A database
+dump alone does not reproduce those global prerequisites or the filesystem and
+secret state above.
+
+There is also no decided RPO, RTO, backup frequency, retention duration,
+geographic copy rule, deletion-in-backups rule, or ownership/escalation path.
+Those values are product and operations decisions, not safe constants to infer
+from the single-host topology.
+
+### 2. Persistent workspaces are inside the documented release tree
+
+`.env.example` recommends `./data/project-workspaces`; systemd's working
+directory is the live Hub directory; deployment mutates that same directory
+with `rsync --delete`. No preflight forbids an in-release workspace root and no
+exclude preserves it. This can turn a routine deploy into irreversible customer
+data loss before build, migration, restart, or readiness runs.
+
+The persistent workspace root must live outside every release directory, have
+explicit ownership/mode and capacity, and be impossible for release retention
+or `rsync --delete` to traverse. Immutable release work already exists as a
+separate ticket, but its acceptance test must include this sentinel case.
+
+### 3. PostgreSQL and workspaces have no consistency boundary
+
+PostgreSQL can provide a transactionally consistent database snapshot, but the
+Hub mutates workspace files independently and has no generation, journal,
+commit marker, snapshot ID, or shared checkpoint that binds filesystem content
+to the corresponding database state. A database dump and filesystem copy taken
+at different instants can disagree about Project existence, active runs,
+provider cursors, delivered messages, and file effects.
+
+Project creation already demonstrates the same missing compensation boundary.
+A replacement host has an additional problem: `Project.workspaceDir` is an
+absolute path and runtime paths from the database are used directly; restore
+has no same-mount validation, safe rebasing operation, or missing/orphan scan.
+
+For the initial single-host topology, a quiesced backup checkpoint is the
+smallest credible route: stop new admission, drain/cancel according to the
+accepted run contract, reach durable zero in-flight work, capture database and
+workspace artifacts under one manifest, then resume. An online scheme would
+need a newly designed journal/generation protocol; it must not be implied by
+two independent copy commands.
+
+### 4. Schema mutation is coupled to service start
+
+Both `npm start` and the systemd unit execute `prisma migrate deploy`; every
+ordinary restart can therefore become a schema rollout. The release path has no
+separate migration lock/gate, `migrate status` check, production-data rehearsal,
+duration/lock budget, pre-migration recovery checkpoint, or compatibility
+manifest. CI only applies the full migration history to a fresh empty
+PostgreSQL database, so it does not exercise real upgrade volume, legacy values,
+old/new application overlap, failure recovery, or rollback compatibility.
+
+The current failed-migration behavior is fail-closed, which is better than
+starting on an unknown schema, but there is no operator procedure for inspecting
+the failure, deciding roll-forward versus mark-rolled-back, restoring the
+checkpoint, or preventing a restart loop. Existing destructive migrations prove
+that application rollback cannot be assumed safe.
+
+### 5. Restore has no isolated verification or cutover gate
+
+There is no procedure that restores into a non-production environment with
+outbound Feishu/provider traffic disabled, verifies backup checksums and
+migration state, validates workspace paths/content/permissions, checks tenant
+relationships, decrypts a sample org connection, reconciles non-terminal work,
+runs application readiness and representative journeys, and measures recovery
+time/point before controlled cutover.
+
+Starting the current server is not a neutral verifier: startup deletes all
+Project locks and independently marks only `ACTIVE` runs failed. Restored data
+must first pass the durable lifecycle reconciliation designed by the run-state
+tickets; otherwise validation itself mutates the evidence and can leave
+`WAITING_FOR_USER` inconsistent.
+
+### 6. Full-service and Organization-selective recovery are different products
+
+A full-service restore can recreate the shared database and all workspaces.
+Restoring one Organization is much harder: users are global and may belong to
+multiple Organizations, permission/audit/history rows span many related tables,
+workspace paths are external, and Feishu/provider connections have external
+state. The repository has no export/import identity mapping or conflict policy.
+
+The initial production gate must at least prove full-service disaster recovery.
+Whether Organization-selective restore/export is promised, and how retained
+backups interact with customer deletion, remains an explicit product/legal
+decision rather than an accidental property of `pg_restore` filters.
+
+## Required recovery contract
+
+Production readiness requires the following guarantees, with exact numerical
+objectives supplied by the pending policy decision:
+
+1. **Declared objectives and ownership.** RPO/RTO, backup frequency, retention,
+   copy locations, encryption/key custody, immutability, deletion handling,
+   full-service versus Organization restore scope, operator ownership, and
+   escalation are documented and testable.
+2. **Complete backup set.** One cataloged manifest identifies release and
+   migration checksums, PostgreSQL artifact and global prerequisites, workspace
+   snapshot, configuration inventory, encrypted org-secret data, external key
+   recovery material, timestamps, sizes, and cryptographic checksums. Backup
+   data and decryption material are not kept only on the protected host or in
+   the same failure domain.
+3. **One consistency point.** A backup is taken only after an observable
+   quiesce/checkpoint protocol, or by a proven online generation protocol that
+   binds database state and workspace effects. Accepted input, active runs,
+   queues, deliveries, locks, and provider cursors have explicit recovery
+   semantics.
+4. **Safe migration rollout.** Migration is a release phase, not an incidental
+   service start. The gate checks status/checksums, rehearses on representative
+   production-shaped data, records old/new application compatibility, uses
+   expand-and-contract for destructive changes, requires a recent verified
+   recovery checkpoint, and provides a tested failed-migration procedure.
+5. **Isolated restore verification.** Every backup class is periodically
+   restored to a clean isolated host/database with outbound integrations off.
+   Verification checks integrity, schema/migration state, global database
+   prerequisites, workspace completeness/path ownership, secret decryption,
+   tenant isolation, lifecycle reconciliation, application readiness, and
+   representative admin/Feishu/agent journeys.
+6. **Observable operation.** Backup age, duration, artifact/checksum status,
+   off-host replication, retention deletion, WAL/archive continuity if used,
+   restore-drill result, measured RPO/RTO, and failures are logged/metricized
+   and alerted with bounded cardinality.
+7. **Controlled cutover and fallback.** Disaster recovery has an explicit
+   authority, traffic/workload brake, DNS/proxy/Feishu reconnect sequence,
+   duplicate/gap checks, rollback/fallback point, verification checklist, and
+   incident evidence record.
+
+## Required release evidence
+
+- A deploy test proves an in-tree or release-overlapping workspace root is
+  rejected before any rsync/switch, while persistent sentinel files survive
+  build failure, successful release, rollback, and release pruning.
+- Project creation, archive/deletion, failed transaction, crash, and restore
+  tests prove database/workspace allocation has no silent missing or orphan
+  state; the reconciler reports rather than hides inconsistencies.
+- Every checked-in migration is rehearsed from the previous production-shaped
+  snapshot, with old/new compatibility and lock/runtime evidence. A deliberately
+  failed migration exercises the documented recovery route.
+- The backup implementation proves completeness, checksum verification,
+  encryption, off-host copy, retention, monitoring, and fail-closed partial
+  artifact cleanup.
+- A clean isolated host restores a representative database, workspaces,
+  secrets, and compatible release; lifecycle reconciliation and critical tenant
+  journeys pass with outbound integrations controlled.
+- Scheduled drills publish achieved recovery point and recovery time against the
+  accepted objectives, and alert when no recent successful drill exists.
+
+## Primary-source interpretation
+
+The official PostgreSQL, Prisma, and systemd behavior supporting this audit is
+collected with claim-level links in
+[Backup and migration primary sources](./backup-migration-primary-sources.md).
+That note distinguishes what a PostgreSQL-consistent dump proves from the
+cross-store guarantees the Hub must build itself.

.scratch/saas-production-readiness/assets/backup-migration-primary-sources.md

@@ -0,0 +1,574 @@
+# Backup, migration, and disaster-recovery primary-source research
+
+This note is the primary-source evidence pack for the Wayfinder ticket
+“Audit backup, migration, and disaster recovery safety.” External claims use
+only PostgreSQL, Prisma, systemd, and rsync first-party documentation. Project
+recommendations are explicitly labelled as derived implications rather than
+upstream guarantees.
+
+PostgreSQL links below use `docs/current`, which was PostgreSQL 18 when this
+research was performed on 2026-07-10. A production implementation must pin the
+deployed PostgreSQL major and verify the same pages for that major. The Hub
+currently depends on Prisma `^6.19.3`, so Prisma 6-specific command examples
+remain relevant even where the live documentation also describes Prisma 7.
+
+## Executive conclusions
+
+1. `pg_dump` can take a consistent **single-database** snapshot while ordinary
+   reads and writes continue, but it does not include the Hub's workspace tree,
+   cluster-global roles/tablespaces, or a WAL chain. PostgreSQL now explicitly
+   says it is generally not the right regular production-backup mechanism
+   outside simple cases. It is a useful logical/export layer, not proof of
+   whole-service disaster recovery.
+   ([`pg_dump` description](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION),
+   [continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
+2. Database PITR requires a physical base backup plus a continuous, unbroken
+   WAL sequence from the backup start through the recovery target. A successful
+   base-backup command alone does not prove that chain exists.
+   ([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING),
+   [making a base backup](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-BASE-BACKUP))
+3. `pg_verifybackup` is necessary but not sufficient: PostgreSQL explicitly
+   requires test restores followed by checks that the restored database works
+   and contains the expected data.
+   ([`pg_verifybackup`](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
+4. Prisma `migrate deploy` applies pending migrations, but it does not detect
+   schema drift and, on PostgreSQL, Prisma does not wrap a migration in a
+   transaction by default. A failed migration can therefore leave partial
+   schema changes requiring explicit operator recovery.
+   ([Prisma production workflow](https://www.prisma.io/docs/orm/prisma-migrate/workflows/development-and-production#production-and-testing-environments),
+   [Prisma transactional behaviour](https://www.prisma.io/blog/prisma-migrate-dx-primitives#what-if-schema-migrations-were-atomic),
+   [failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration))
+5. systemd supplies process lifecycle orchestration, not an application-level
+   consistency barrier. It sends the configured termination signal, waits up
+   to `TimeoutStopSec=`, then can forcibly kill remaining processes; a
+   successful `systemctl stop` does not prove that an `ExecStop=` command or
+   application drain succeeded.
+   ([`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=),
+   [`systemctl stop`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#stop%20PATTERN%E2%80%A6))
+6. The Hub persists one logical service state across two stores: PostgreSQL
+   records `Project.workspaceDir`, runs, messages, locks, and file-change audit
+   rows, while workspaces contain project files. A workspace transcript helper
+   exists but is not wired into the production runner. PostgreSQL's snapshot/
+   PITR guarantees cover only PostgreSQL. A whole-service recovery point
+   therefore requires an application-defined database/workspace checkpoint or
+   a journaled/versioned reconciliation protocol; neither PostgreSQL nor
+   systemd creates one automatically.
+   ([Hub schema](../../../hub/prisma/schema.prisma),
+   [workspace transcript helper](../../../hub/src/agent/transcript.ts),
+   [`pg_dump` scope](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION))
+
+## Current repository facts that make the boundaries material
+
+- The systemd unit runs `prisma migrate deploy` as `ExecStartPre=` before every
+  service start and uses `Restart=on-failure`.
+  ([`cph-hub.service`](../../../hub/deploy/cph-hub.service))
+- The deploy script updates the live Hub directory using
+  `rsync -az --delete`, builds in that same directory, restarts the unit, and
+  accepts a constant liveness endpoint as deployment success.
+  ([`deploy_platform.sh`](../../../hub/deploy/deploy_platform.sh))
+- `Project.workspaceDir` is database state, but the referenced directory holds
+  independently mutable tenant content. A workspace JSONL transcript helper
+  exists but is not wired into the production runner; file contents still live
+  in the workspace while the database separately carries run/message/file-change
+  facts.
+  ([Hub schema](../../../hub/prisma/schema.prisma),
+  [workspace transcript helper](../../../hub/src/agent/transcript.ts))
+
+These are project facts, not upstream guarantees. The sections below establish
+what the upstream tools do and, critically, what they do not do.
+
+## PostgreSQL logical dump and restore
+
+### Snapshot and concurrent-write semantics
+
+- `pg_dump` makes a consistent export even while the database is being used;
+  it does not block ordinary readers or writers. It dumps only one database.
+  Cluster-global objects such as roles and tablespaces require `pg_dumpall` or
+  `pg_dumpall --globals-only`.
+  ([`pg_dump` description](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION),
+  [whole-cluster logical dumps](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-ALL))
+- `pg_dumpall` invokes `pg_dump` once per database. Each database is internally
+  consistent, but the snapshots are not synchronized across databases.
+  ([whole-cluster logical dumps](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-ALL))
+- A parallel dump uses directory format, opens `njobs + 1` connections, and
+  uses synchronized snapshots so every worker sees the same data set. The
+  connection count and additional server load are part of its capacity cost.
+  ([`pg_dump --jobs`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+- Concurrent DDL can still interfere. The parallel-dump leader holds
+  `ACCESS SHARE` locks; if a later exclusive-lock request queues ahead of a
+  worker, that worker uses `NOWAIT` and aborts the dump rather than deadlocking.
+  `--lock-wait-timeout` can bound the initial lock wait.
+  ([`pg_dump --jobs` and lock behaviour](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+- The ordinary snapshot is suitable for disaster-recovery dump semantics, but
+  it need not correspond to a serial execution of all transactions that later
+  commit. `--serializable-deferrable` waits for an anomaly-free snapshot;
+  PostgreSQL says that option is not beneficial for a dump intended only for
+  disaster recovery, but it can matter for a continuously updated reporting
+  copy.
+  ([`--serializable-deferrable`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+- The default waits for dump files to be written safely to disk. Production
+  backups must not use `--no-sync`, because an immediate OS crash can otherwise
+  leave a corrupt dump.
+  ([`pg_dump --no-sync`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
+
+Derived Hub implications:
+
+- An online database dump may be internally valid while referring to a
+  `workspaceDir` whose files are from another instant. “`pg_dump` succeeded” is
+  therefore not a whole-Hub consistency claim.
+- Backup scheduling must either serialize against schema migration/DDL or
+  treat any lock wait, abort, non-zero exit, and stderr warning as a failed
+  backup. A partially written archive must never be published as restorable.
+- If the deployed cluster contains more than the Hub database, a single Hub
+  dump is not a cluster backup. Even `pg_dumpall` does not create a single
+  cross-database snapshot.
+
+### Restore error and atomicity semantics
+
+- `pg_restore` defaults to continuing after SQL errors and reports an error
+  count at the end. `--exit-on-error` is required for fail-fast behaviour.
+  `--single-transaction` wraps all emitted commands in `BEGIN`/`COMMIT`, makes
+  the restore all-or-nothing, and implies `--exit-on-error`.
+  ([`pg_restore` error and transaction options](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
+- Parallel restore uses separate connections for data loading, index creation,
+  and constraint creation. It supports only custom/directory archives and
+  cannot be combined with `--single-transaction`; it is not a whole-restore
+  atomic operation.
+  ([`pg_restore --jobs`](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
+- A single giant restore transaction can be impractical because it locks every
+  restored object and may exhaust the lock table. PostgreSQL 18 offers
+  `--transaction-size=N` as a bounded intermediate option; it implies
+  `--exit-on-error` but is not all-or-nothing.
+  ([`pg_restore --transaction-size`](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
+- Plain SQL restored by `psql` also continues after errors unless
+  `ON_ERROR_STOP` is enabled; a single transaction is separately required for
+  all-or-nothing behaviour.
+  ([restoring an SQL dump](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-RESTORE))
+- PostgreSQL recommends restoring into a truly empty database based on
+  `template0`; otherwise local additions in `template1` can cause duplicate
+  definition errors. Required roles must exist before owner/ACL restoration.
+  ([restoring an SQL dump](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-RESTORE))
+- A dump can execute arbitrary code selected by superusers of the source
+  database. Partial dump/restore does not remove that trust boundary.
+  ([`pg_dump` security warning](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION))
+- A `pg_dump` binary can dump older servers but refuses a server newer than its
+  own major version. Output is intended to load into newer PostgreSQL versions,
+  but loading into an older major is not guaranteed.
+  ([`pg_dump` cross-version notes](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-NOTES))
+
+Derived Hub implications:
+
+- Restore should target a new isolated database/cluster. If a non-atomic or
+  parallel restore reports any error, discard that target and start again;
+  never promote a partially restored database.
+- The runbook must record the restore atomicity strategy and exact PostgreSQL
+  client/server versions. A green process exit without explicit fail-fast
+  options is insufficient evidence.
+- A restored database must remain disconnected from real Feishu/provider
+  side-effect endpoints until application-level validation completes.
+
+## Physical backup, WAL, and PITR
+
+### Base backups
+
+- `pg_basebackup` backs up a running PostgreSQL cluster without blocking other
+  clients. A full backup is an exact copy of the entire cluster and can seed
+  PITR or a standby; it cannot select one database or object.
+  ([`pg_basebackup` description](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- It uses the replication protocol. The account needs `REPLICATION` or
+  superuser privilege, `pg_hba.conf` must permit it, and `max_wal_senders`
+  needs capacity for the backup and, when streaming WAL, another connection.
+  ([`pg_basebackup` requirements](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- Default `-X stream` obtains WAL over a second connection while copying. With
+  `-X fetch`, WAL is collected only at the end; if required WAL was recycled,
+  the backup fails and is unusable.
+  ([`pg_basebackup --wal-method`](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- The default waits for backup files to be durable. `--no-sync` is explicitly
+  unsuitable for production because an OS crash can corrupt the backup.
+  ([`pg_basebackup --no-sync`](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+- A manifest is generated by default, with per-file checksums. A SHA manifest
+  can detect malicious modification only if the manifest itself is held in a
+  separate trusted location or otherwise authenticated.
+  ([`pg_basebackup` manifest options](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
+
+Derived minimum backup record:
+
+- source cluster/system identifier and PostgreSQL version;
+- backup tool version, start/end time, start/end LSN and WAL range;
+- manifest digest and its independently protected location;
+- destination object/version, byte count, exit status, and verification state;
+- an explicit `VALID`, `INVALID`, or `QUARANTINED` state so a failed leftover
+  directory cannot enter the retention set merely because files exist.
+
+### WAL archival and recovery
+
+- PITR combines a physical backup with WAL replay. It can stop at a chosen time
+  and reach a consistent database state. Logical dumps do not contain enough
+  information for WAL replay.
+  ([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
+- Successful recovery requires a continuous sequence of archived WAL extending
+  back at least to the base-backup start. PostgreSQL tells operators to set up
+  and test WAL archiving **before** taking the first base backup.
+  ([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
+- The archive command must return zero if and only if durable archival
+  succeeded. Non-zero causes retry. Archive implementations should refuse to
+  overwrite an existing file; a repeated name may report success only when the
+  existing durable content is identical, and differing content must fail.
+  ([WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- Repeated archive failure accumulates WAL in `pg_wal`; filling that filesystem
+  causes PostgreSQL to panic and remain offline until space is freed. PostgreSQL
+  explicitly advises monitoring archival progress.
+  ([WAL archiving failure behaviour](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- WAL is archived after a segment is completed, so low traffic can make the
+  last committed change wait a long time for archival. `archive_timeout` can
+  bound that age by forcing segment switches, at the cost of archive growth.
+  ([WAL archiving latency](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- WAL archival does not restore manually edited PostgreSQL configuration files
+  (`postgresql.conf`, `pg_hba.conf`, `pg_ident.conf`); those need a separate
+  file backup.
+  ([WAL archiving caveat](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+- PostgreSQL's recovery procedure keeps clients out, restores the base backup,
+  configures `restore_command` and `recovery.signal`, replays WAL, and requires
+  inspection of database contents before restoring ordinary access.
+  ([PITR recovery procedure](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-PITR-RECOVERY))
+- Recovery can target a timestamp, LSN, transaction ID, named restore point, or
+  the earliest consistent point. Only one target may be set. Timeline history
+  is required to navigate recovery branches, and a configured target not
+  reached before WAL exhaustion is a fatal recovery failure.
+  ([recovery target settings](https://www.postgresql.org/docs/current/runtime-config-wal.html#RUNTIME-CONFIG-WAL-RECOVERY-TARGET),
+  [timelines](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-TIMELINES))
+
+Derived Hub implications:
+
+- The database RPO is the age of the newest **durably archived, continuous**
+  WAL position, not the age of the newest base backup and not merely the name
+  in `last_archived_wal`.
+- Archive keys need a collision-proof cluster/system-identifier namespace and
+  create-if-absent semantics; unrelated clusters must never overwrite a shared
+  WAL name.
+- Monitoring must cover archive failures, time since durable success, archive
+  lag, WAL continuity, `pg_wal` growth, free space, and base-backup age.
+- PITR restores PostgreSQL only. It does not rewind workspace files. A database
+  recovery target has no whole-Hub meaning until paired with a workspace
+  recovery point and reconciliation rule.
+
+## Verification is a ladder, not a checksum
+
+- `pg_verifybackup` validates the backup manifest and system identifier,
+  detects missing/extra/changed files, checks file checksums, and parses the
+  required WAL with `pg_waldump`. WAL verification is currently supported only
+  for plain-format base backups.
+  ([`pg_verifybackup` checks](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
+- PostgreSQL explicitly says these checks cannot cover everything a running
+  server will encounter and still requires test restores plus checks that the
+  database works and contains correct data.
+  ([`pg_verifybackup` limitation](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
+- `pg_amcheck` can check supported heap/index structures, but currently only
+  ordinary/TOAST tables, materialized views, sequences, and B-tree indexes are
+  supported; other relation types are silently skipped. Stronger B-tree checks
+  can block concurrent modifications.
+  ([`pg_amcheck`](https://www.postgresql.org/docs/current/app-pgamcheck.html#APP-PGAMCHECK))
+
+Derived recovery-evidence ladder:
+
+1. **Artifact verification:** backup command succeeded; manifest and all
+   required WAL validate; any failure marks the artifact unusable.
+2. **Isolated technical restore:** restore to a clean non-production target,
+   reach the intended LSN/time/timeline, and prove PostgreSQL starts without
+   exposing it to users.
+3. **Database validation:** verify migration history/schema identity, tenant
+   counts and relationships, critical constraints/sequences, and appropriate
+   corruption checks.
+4. **Workspace validation:** restore the paired snapshot/version, validate file
+   hashes/ownership/containment, and reconcile database file-change/run state
+   against actual files.
+5. **Application validation:** run read-only tenant-isolation and core workflow
+   smoke tests with external side effects disabled.
+6. **Recovery report:** persist the exact backup, WAL range, workspace point,
+   release/schema versions, checks, observed RPO/RTO, operator, and failures.
+   Only a fully passing report may authorize promotion.
+
+## Prisma Migrate production semantics
+
+### What `migrate deploy` proves and does not prove
+
+- In production, `migrate deploy` compares applied migration checksums with the
+  migration directory, warns when an applied migration was modified, and
+  applies pending migrations.
+  ([Prisma 6 production workflow](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#production-and-testing-environments))
+- It does **not** warn when an already-applied migration is missing, detect
+  production schema drift, reset the database, generate Prisma Client, or use a
+  shadow database.
+  ([Prisma 6 production workflow](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#production-and-testing-environments))
+- The migration directory is the deployable history and must be committed in
+  full, including `migration_lock.toml`. Prisma warns against editing or
+  deleting migrations already applied in production; `migrate deploy` executes
+  migration SQL rather than deriving a new migration from the Prisma schema.
+  ([Prisma migration histories](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/migration-histories))
+- Prisma uses a database advisory lock for production migration commands so
+  multiple migration commands do not run simultaneously. The documented lock
+  timeout is ten seconds and not configurable; after timeout, the caller must
+  retry. The lock can be disabled by environment variable.
+  ([Prisma 6 advisory locking](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#advisory-locking))
+
+Derived Hub implication: `migrate deploy` exit zero proves that this invocation
+applied or found no pending migration. It does not prove the live schema equals
+the expected migration end state, that the new and previous application builds
+are both compatible with it, or that the migration is operationally safe under
+production data volume and traffic.
+
+The advisory lock is a migration-command concurrency guard, not a substitute
+for the repository's deployment lock or application/schema compatibility. As a
+derived safety rule, production preflight should reject
+`PRISMA_SCHEMA_DISABLE_ADVISORY_LOCK` unless an explicit reviewed mechanism
+provides the same serialization guarantee.
+
+### Transactions and failed migrations
+
+- Prisma's first-party migration guidance states that PostgreSQL migrations
+  are not wrapped in a transaction by default. Operators may opt in by adding
+  `BEGIN;`/`COMMIT;` to a migration, but large transactional migrations can hold
+  locks longer and consume more database resources.
+  ([Prisma transactional behaviour](https://www.prisma.io/blog/prisma-migrate-dx-primitives#what-if-schema-migrations-were-atomic))
+- A failed migration records its error in `_prisma_migrations.logs`; further
+  migrations cannot deploy until the failed state is explicitly recovered.
+  Recovery is either (a) revert/repair partial steps, mark the migration rolled
+  back, and redeploy, or (b) manually complete the exact steps and mark it
+  applied.
+  ([Prisma failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration))
+- Prisma error codes `P3009` and `P3018` explicitly report that failed
+  migrations block new migration application until recovery.
+  ([Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference#p3009))
+- `migrate resolve --rolled-back` changes migration-history state so Prisma may
+  apply the migration again; it does not itself undo SQL already executed.
+  `--applied` records success without executing the migration SQL.
+  ([failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration),
+  [`migrate resolve`](https://www.prisma.io/docs/cli/migrate/resolve))
+- Prisma can generate a down SQL script, but its documentation limits the
+  simple rollback workflow to a **failed** up migration. To reverse a successful
+  migration, the supported history is a new forward migration generated from a
+  reverted Prisma schema. A schema down script also cannot restore lost data,
+  application changes, or manually authored SQL automatically.
+  ([generating down migrations](https://www.prisma.io/docs/orm/prisma-migrate/workflows/generating-down-migrations#considerations-when-generating-down-migrations))
+
+Derived Hub implications:
+
+- Every migration needs a declared atomicity choice. If it is intentionally
+  non-transactional, the migration must be resumable/idempotent or ship an
+  exact, tested partial-failure procedure. Blindly rerunning a partially applied
+  script is unsafe.
+- A database backup checkpoint before a destructive or difficult-to-reverse
+  migration is necessary, but restoring that checkpoint is a whole-database
+  recovery event—not a substitute for a compatible rollout design.
+- Service restart cannot be treated as application rollback after a successful
+  schema migration. The previous binary may no longer understand the new
+  schema, and a down migration cannot recreate deleted data.
+
+### Compatibility and rollout ordering
+
+- Prisma documents expand-and-contract as the way to avoid downtime for fields
+  in active use: add the new representation, deploy code that writes both while
+  reading the old, migrate and verify data, switch reads, stop old writes, then
+  remove the old representation.
+  ([Prisma expand-and-contract workflow](https://www.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations#example-use-the-expand-and-contract-pattern-to-evolve-the-schema-without-downtime))
+- Prisma also warns that generated “renames” may be expressed as add/drop and
+  lose data unless the SQL is reviewed and customized before application.
+  ([customizing migrations](https://www.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations#example-rename-a-field))
+
+Derived deployment contract:
+
+1. Migration SQL is immutable after production application and reviewed for
+   destructive operations, lock duration, data volume, and transaction choice.
+2. It is rehearsed against a recent, anonymized production-scale restore and
+   its runtime/locks/disk growth are recorded.
+3. Each release declares a schema compatibility window. Expand releases remain
+   compatible with both old and new application versions; contract/destructive
+   releases occur only after old binaries can no longer run.
+4. Migration status/drift is checked before changing the live release. A
+   failed migration blocks deployment and enters an operator runbook; it must
+   never be hidden by repeated restart attempts.
+5. Rollback means switching to a binary proven compatible with the current
+   schema or deploying a new forward migration. Restoring a backup is the last
+   resort and must also restore/reconcile the workspace recovery point.
+
+## systemd stop, restart, and dependency semantics
+
+- When `ExecStartPre=`, `ExecStart=`, or `ExecStartPost=` fails or times out,
+  service startup does not continue to the normal running process. `ExecStop=`
+  is skipped because startup never succeeded; `ExecStopPost=` is the hook that
+  still runs after a failed startup.
+  ([systemd start/stop command semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStartPre=),
+  [`ExecStopPost=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStopPost=),
+  [permanent upstream man-page source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemd.service.xml#L525-L587))
+- `Restart=on-failure` covers non-zero exits and failures/timeouts in service
+  control processes, including `ExecStartPre=`. An explicit systemd stop does
+  not trigger automatic restart. Restarts are subject to unit start-rate
+  limiting.
+  ([`Restart=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#Restart=),
+  [permanent upstream source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemd.service.xml#L856-L995))
+- With no `ExecStop=`, systemd sends `KillSignal=` (default `SIGTERM`).
+  `TimeoutStopSec=` bounds the wait; after it expires, remaining processes can
+  be forcibly killed with `SIGKILL`/`FinalKillSignal=`.
+  ([`ExecStop=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStop=),
+  [`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=))
+- Default `KillMode=control-group` applies termination to all remaining
+  processes in the unit cgroup. systemd explicitly discourages `process` or
+  `none` because children can escape the service lifecycle and remain running
+  after the service is considered stopped.
+  ([`KillMode=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.kill.html#KillMode=))
+- `systemctl stop` waits for the job unless `--no-block` is used, but the command
+  does not fail merely because an `ExecStop=` command failed: the manager still
+  forcibly terminates the unit.
+  ([`systemctl stop`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#stop%20PATTERN%E2%80%A6),
+  [permanent upstream source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemctl.xml#L444-L459))
+- `systemctl restart` is a stop followed by a start; `ExecStop=` and
+  `ExecStopPost=` participate. It does not necessarily flush every unit resource
+  in the same way as separate completed `stop` then `start` operations.
+  ([restart semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStop=),
+  [`systemctl restart`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#restart%20PATTERN%E2%80%A6))
+- `After=` is only an ordering relationship; it does not pull in another unit.
+  Requirement and ordering dependencies are independent. During shutdown the
+  start order is reversed, so a Hub ordered `After=postgresql.service` stops
+  before PostgreSQL when both are in the shutdown transaction.
+  ([`Requires=` and ordering](https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Requires=),
+  [`Before=`/`After=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Before=))
+- `systemctl freeze` only uses the cgroup freezer to suspend process execution;
+  its documented contract contains no filesystem flush or stable-image
+  guarantee. It is not a substitute for an application drain or a filesystem
+  snapshot barrier.
+  ([permanent `systemctl freeze` man-page source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemctl.xml#L653-L675))
+
+Derived assessment of the current unit:
+
+- A Prisma migration failure in the current `ExecStartPre=` prevents the Hub
+  from starting, then `Restart=on-failure` can retry startup until rate-limited.
+  That is fail-closed, but it is not controlled migration recovery and can
+  repeatedly hit a database already left in Prisma's failed state.
+- `KillSignal=SIGTERM` plus `TimeoutStopSec=30` is only a deadline. Until the Hub
+  installs a real termination handler that makes readiness false, stops ingress,
+  drains/checkpoints active work, closes Feishu and database clients, and reports
+  success, the unit cannot claim a graceful consistency barrier.
+- A backup runbook must issue a controlled drain, verify durable work reached a
+  reconciled terminal/checkpoint state, stop the Hub, verify the unit and its
+  cgroup are inactive, and only then snapshot mutable workspace state. A zero
+  `systemctl stop` exit by itself is insufficient evidence.
+- Recovery should not start the normal unit until the restored database and
+  workspace have been validated. The current unit would run pending migrations
+  immediately on first start, mutating the restored database before validation
+  and possibly destroying the ability to inspect the original recovered state.
+
+## Filesystem and rsync boundaries
+
+- PostgreSQL says a plain filesystem copy of a live data directory is unusable:
+  the database must be shut down, because tools such as `tar` do not create an
+  atomic snapshot and server buffering remains. A correctly implemented frozen
+  snapshot can work while PostgreSQL runs, but all volumes containing data,
+  WAL, and tablespaces must be captured simultaneously; otherwise shut down or
+  use the continuous-archive base-backup protocol.
+  ([PostgreSQL filesystem backups](https://www.postgresql.org/docs/current/backup-file.html))
+- For a PostgreSQL rsync backup, the documented safe pattern is a first pass
+  while running, followed by shutdown and a second `rsync --checksum`; the
+  second pass is what makes the result consistent.
+  ([PostgreSQL filesystem backups](https://www.postgresql.org/docs/current/backup-file.html))
+- At the filesystem layer, `fsfreeze` is documented to let in-progress
+  filesystem operations complete, halt new modifications, and flush dirty data,
+  metadata, and logs before returning. That is still only a filesystem-volume
+  guarantee; it does not replace PostgreSQL's backup protocol and does not make
+  separately frozen volumes simultaneous.
+  ([util-linux `fsfreeze` manual source](https://github.com/util-linux/util-linux/blob/master/sys-utils/fsfreeze.8.adoc))
+- rsync's `--delete` defaults to deletion during transfer on modern peers, so
+  the receiver tree changes incrementally. `--delay-updates` merely holds each
+  updated file and renames files into place rapidly at the end; its own manual
+  says this only attempts to make updates “a little more atomic” and points to
+  a separate parallel-tree/switch approach for stronger atomicity.
+  ([rsync `--delete`](https://rsync.samba.org/ftp/rsync/rsync.1.html#opt--delete),
+  [rsync `--delay-updates`](https://rsync.samba.org/ftp/rsync/rsync.1.html#opt--delay-updates))
+
+Derived Hub implications:
+
+- The current in-place `rsync --delete` deployment is neither an immutable
+  release switch nor a backup snapshot. A failed/interrupted transfer can leave
+  a mixed live tree; service stop/restart cannot reconstruct the deleted prior
+  artifact.
+- The workspace snapshot mechanism remains an explicit design choice. Whichever
+  storage technology is selected must prove atomicity over all workspace
+  volumes or use a quiesced/no-writer interval, then publish an immutable
+  version/checksum manifest. Merely copying a changing directory is not enough.
+
+## Derived minimum whole-Hub recovery contract
+
+The primary sources support the following minimum contract; exact RPO, RTO,
+retention, storage vendor, and numeric schedules remain product/operations
+decisions and must not be invented by implementation.
+
+### Backup path
+
+1. Maintain verified PostgreSQL base backups and a monitored continuous WAL
+   archive, or obtain and test an explicitly equivalent managed-PostgreSQL
+   guarantee. Keep logical dumps as a separate portability/selective-restore
+   layer, not the sole PITR mechanism.
+2. Keep PostgreSQL configuration, deployment/release manifests, migration
+   history, and workspace snapshots in the backup inventory; WAL alone does not
+   cover them.
+3. Create a whole-Hub checkpoint only after ingress is drained and active
+   workspace writers are stopped/reconciled, unless the eventual storage layer
+   supplies a proven atomic snapshot/journal spanning the necessary stores.
+4. Persist one signed/immutable checkpoint manifest joining PostgreSQL
+   backup/WAL position, workspace snapshot/version, application revision,
+   migration digest, tool versions, checksums, timestamps, and validation state.
+5. Treat database/WAL/workspace backups as cross-tenant sensitive data. Apply
+   least-privilege access, encryption, audit, deletion/retention, and an
+   independently protected integrity manifest; WAL contains effectively all
+   database changes and PostgreSQL advises protecting it from unauthorized
+   readers.
+   ([WAL archive confidentiality](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
+
+### Migration path
+
+1. Serialize deploys and backup checkpoints; run a drift/status gate before
+   migration, not just `migrate deploy`.
+2. Rehearse each migration on a recent production-scale restore and record
+   runtime, locks, disk/WAL growth, transaction choice, and failure recovery.
+3. Require expand/contract compatibility for online changes. A release must
+   declare which old/new binaries can run against each schema phase.
+4. Separate migration orchestration from ordinary service restart so a failed
+   migration produces one durable incident state and an operator decision—not
+   an opaque restart loop.
+5. Verify a restorable checkpoint before destructive change. Do not describe
+   backup restore as a normal “rollback”; it is whole-service disaster recovery.
+
+### Restore and promotion path
+
+1. Restore into an isolated environment with real external side effects
+   disabled.
+2. Restore the selected PostgreSQL base backup/WAL target and the checkpoint's
+   exact workspace version; do not run pending Prisma migrations implicitly.
+3. Prove artifact integrity, database startup, recovery target/timeline,
+   migration/schema identity, tenant invariants, workspace hashes/containment,
+   and read-only application flows.
+4. Record observed recovery point and recovery time. A restore that starts but
+   misses its target, loses a tenant workspace, has partial migration state, or
+   fails any invariant is not promotable.
+5. Promote explicitly, then enable ingress and external integrations. Preserve
+   the pre-recovery cluster/WAL and recovery report until the incident is closed.
+
+## Decisions and evidence still required
+
+These values are not fixed by the upstream tools and remain `OPEN` for the SaaS
+production contract:
+
+- whole-Hub RPO and RTO, separately for ordinary deletion, database corruption,
+  host loss, region/provider loss, and migration failure;
+- base-backup, logical-dump, workspace-snapshot, WAL, and restore-drill cadence;
+- immutable/off-host storage, encryption/key ownership, retention and verified
+  deletion policy, including org-deletion and legal-retention interactions;
+- exact workspace checkpoint/journal technology and database/workspace
+  reconciliation semantics for in-flight or partially written runs;
+- whether restore promotion is whole-platform only or supports a safe
+  organization/project export-import path (PostgreSQL physical PITR is
+  cluster-wide);
+- migration transaction policy, maximum lock/downtime budget, compatibility
+  window, and who can authorize contract/destructive phases;
+- evidence from at least one production-like full restore and one PITR drill,
+  including deliberate missing-WAL, corrupted-backup, failed-migration,
+  interrupted-workspace-write, and restart-during-recovery probes.

.scratch/saas-production-readiness/assets/clean-host-deployment-audit.md

@@ -0,0 +1,200 @@
+# Clean-host deployment and rollback audit
+
+## Verdict
+
+The current `hub/deploy` path is **not a production deployment mechanism**.
+It can update a manually-prepared host in the happy path, but a clean supported
+host cannot become a runnable Hub from the documented inputs, and an upgrade
+failure can leave the live application tree partially replaced with no
+application rollback path.
+
+The accepted initial topology remains viable: one Linux/systemd host,
+PostgreSQL, Feishu WebSocket ingress, and a non-root Hub service. The blocker is
+the release contract and its verification, not a need to replace that topology.
+
+## Executed evidence
+
+### Static gates
+
+- `bash -n hub/deploy/install_service.sh` and
+  `bash -n hub/deploy/deploy_platform.sh` pass.
+- ShellCheck reports only informational SC2029 findings for intentional
+  client-side expansion of remote paths. It does not find a shell syntax defect.
+- A source-to-installer comparison finds six `requireEnv` settings in
+  `hub/src/server.ts`, but the generated production environment omits
+  `HUB_PROJECT_WORKSPACE_ROOT` and `HUB_SESSION_SECRET`.
+
+### Seeded configuration cannot boot the built server
+
+The installer was run against a temporary `BASE`, with the real built Hub as
+`HUB_DIR`. It exited successfully after creating `platform.env`. The generated
+file was then used as the production configuration while supplying the model and
+Feishu values that its message asks the operator to fill. The built server
+exited with status 1:
+
+```text
+[hub] missing required env: HUB_PROJECT_WORKSPACE_ROOT
+```
+
+After that is supplied, `HUB_SESSION_SECRET` is the next required omission.
+`HUB_PUBLIC_BASE_URL` is also absent; its code fallback is localhost, which is
+not a valid public OAuth callback for a production deployment.
+
+### The service identity is referenced but not provisioned
+
+The installer was executed with `SERVICE_USER=definitely-missing-cph-user` and
+mocked `install`/`systemctl` boundaries. It returned success and rendered:
+
+```text
+User=definitely-missing-cph-user
+WorkingDirectory=.../hub
+EnvironmentFile=.../.secrets/platform.env
+```
+
+There is no `getent`, `id`, `useradd`, or equivalent validation/provisioning
+path. It also creates no service home, cache, state, or project-workspace
+directory. This contradicts ADR-0018's implemented deployment invariant that
+the Hub runs as a dedicated unprivileged service user. A genuinely clean host
+will fail at systemd user resolution; a manually-created user can still fail
+later when the agent SDK, `cph` cache, or project workspace needs a writable
+path.
+
+### A failed build mutates the live tree before failure
+
+`deploy_platform.sh` was run with mocked SSH and rsync boundaries. The first
+remote build was forced to exit 42. The observed order was:
+
+```text
+ssh:mkdir -p '/srv/curriculum-project-hub/hub'
+rsync:live-tree-mutated
+ssh:cd '/srv/curriculum-project-hub/hub' && npm ci && npm run build
+```
+
+The deploy command correctly propagated status 42, but it had already applied
+`rsync --delete` to the one live directory. There is no staged release,
+immutable release identity, current/previous pointer, deployment lock, or
+rollback command. `npm ci` also replaces that live directory's dependency tree
+before the new build has passed.
+
+## Blocking gaps
+
+### 1. Incomplete clean-host bootstrap
+
+`install_service.sh` writes a unit containing `User=cph-hub` without creating or
+validating that account. It does not create owned state/cache/workspace paths,
+and its generated environment omits required settings. On its first invocation
+it deliberately exits after seeding the environment, after which
+`deploy_platform.sh` proceeds to restart a unit that has not yet been installed.
+The documented workflow is therefore neither one-shot nor complete.
+
+### 2. The runtime product is not fully delivered
+
+The deploy uploads only `hub/`. The target prerequisite list does not include a
+preinstalled `cph`, and no release step builds or installs `crates/cph-cli`.
+The Hub starts without checking it: the first agent `cph check` or `cph build`
+returns exit 127 while `/api/healthz` remains green. Bubblewrap is asserted by
+the unit, but Node/npm, curl, PostgreSQL connectivity, service-user namespace
+support, writable storage, and `cph --version` are not checked as one preflight.
+
+### 3. Releases are in-place and have no application rollback
+
+Source, dependencies, build output, and the running service all share one
+directory. A build failure, interrupted rsync, or unsuccessful restart can
+leave a mixed tree. No release manifest records the git revision, Node version,
+dependency lock digest, migration set, or `cph` version that was deployed.
+Concurrent deploys are not serialized.
+
+### 4. The deployment health check is only liveness
+
+`/api/healthz` returns a constant success response after the HTTP server starts.
+It does not verify database access, migration state, workspace writability,
+`cph`, the agent sandbox, or Feishu listener startup. The listener's start
+promise is not awaited or reflected in readiness. A release can therefore be
+declared successful while its defining workflows are unavailable.
+
+The service unit also claims graceful shutdown, but `server.ts` installs no
+`SIGTERM`/`SIGINT` handler and never calls `app.close()` or closes the Feishu
+listener. Node's default SIGTERM handling terminates the process; Fastify's
+documented graceful path begins only when `fastify.close()` is invoked. The
+full data-integrity consequence belongs to the run-lifecycle audit, but the
+current deployment contract must not claim draining behavior it does not have.
+
+References:
+
+- [Fastify shutdown hooks](https://fastify.dev/docs/latest/Reference/Hooks/)
+- [Node.js signal events](https://nodejs.org/api/process.html#signal-events)
+
+### 5. Application rollback and schema migration are not coordinated
+
+The unit runs `prisma migrate deploy` immediately before every start. Current
+migration history includes column drops, index replacement, and data
+deduplication; there is no declaration that the previous application remains
+compatible after each migration. Switching application code back after a
+successful migration is therefore not generally safe.
+
+Prisma documents that `migrate deploy` applies pending migrations but does not
+detect schema drift, and a partially failed migration requires explicit
+operator recovery. Its recommended zero-downtime direction is an
+expand-and-contract sequence. The repository has no pre-deploy migration
+status gate, production-data rehearsal, backup checkpoint, expand/contract
+rule, or failed-migration runbook.
+
+References:
+
+- [Prisma production migration behavior](https://www.prisma.io/docs/orm/prisma-migrate/workflows/development-and-production)
+- [Prisma failed-migration recovery](https://docs.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing)
+- [Prisma expand-and-contract migrations](https://docs.prisma.io/docs/guides/database/data-migration)
+
+### 6. Network binding configuration is misleading
+
+The installer writes `HOST=127.0.0.1`, but `server.ts` ignores it and binds
+`0.0.0.0`. An operator following the generated configuration can unintentionally
+expose the Hub directly instead of binding it behind the intended TLS reverse
+proxy. Public URL validation and proxy topology are not part of deployment
+preflight.
+
+## Required release contract
+
+A production-capable implementation on the accepted topology should establish
+these invariants:
+
+1. **Provisioned identity and storage.** Installation creates or validates a
+   non-login `cph-hub` user and explicit persistent state, cache, home, and
+   workspace paths with tested ownership.
+2. **Complete validated configuration.** A preflight validates every required
+   secret/value, the public HTTPS URL, bind address, PostgreSQL connectivity,
+   writable storage, `bwrap`, and the exact compatible `cph` binary before the
+   live service is touched.
+3. **Immutable staged releases.** Each revision is built in a new release
+   directory, identified by revision and manifest. The live service points to a
+   `current` symlink only after build and preflight pass. Deploys are locked.
+4. **Complete product artifact.** The release carries both the Hub and the
+   matching `cph` executable; readiness checks the binary/version rather than
+   deferring failure to an agent run.
+5. **Real readiness.** Liveness remains cheap, while readiness proves the
+   dependencies needed to accept work. Deployment success waits for readiness,
+   not merely an open HTTP port.
+6. **Application rollback.** A failed switch returns `current` to the previous
+   release and verifies readiness. Database changes are never silently reversed.
+7. **Migration compatibility.** Every release states whether its migration set
+   is backward-compatible with the previous application. Future destructive
+   changes use expand/contract releases; failed migrations follow an explicit
+   recovery runbook tied to backup evidence.
+8. **Observable lifecycle.** Release revision, migration status, readiness
+   failures, restart/drain outcomes, and rollback results are present in command
+   output and service logs.
+
+## Verification route
+
+The implementation is not complete until an automated Linux test drives these
+scenarios with the real scripts and a production-like PostgreSQL instance:
+
+- first install on a host without the service user or directories;
+- incomplete configuration fails before unit installation/restart;
+- staged build failure leaves the current release byte-for-byte unchanged;
+- successful deploy reports revision and passes readiness;
+- failed new release switches back to the previous application;
+- missing/wrong `cph`, unwritable workspace, unavailable database, and failed
+  migration each produce a distinct non-zero deployment failure;
+- repeated installation/deployment is idempotent;
+- concurrent deployments serialize or one fails clearly.

.scratch/saas-production-readiness/assets/initial-abuse-capacity-controls.md

@@ -0,0 +1,125 @@
+# Initial abuse and capacity controls
+
+## Verdict
+
+The initial production boundary requires hard safety controls before customer
+traffic. The current project lock and small in-memory trigger queue are useful
+local protections, but they do not provide multi-tenant admission, durable
+backpressure, storage safety, request isolation, bounded execution, or an
+operator stop mechanism.
+
+The accepted contract is recorded in
+[ADR-0022](../../../docs/adr/0022-layered-capacity-admission.md) and
+[`Spec.System.Capacity`](../../../spec/Spec/System/Capacity.lean). Exact values
+remain deliberately open until production-like calibration; the required
+dimensions and failure behaviors do not.
+
+## Current implementation inventory
+
+| Surface | Existing protection | Production gap |
+| --- | --- | --- |
+| HTTP | Fastify's default 1 MiB body limit | No route-class, user, Org, IP, or administrative-write rate limits; request/connection/handler timeouts are unset |
+| Project execution | Database lock permits one active run per Project | No platform or Org concurrency ceiling and no fair scheduler |
+| Busy Project | Process-local FIFO, default five items and five-minute wait | Restart loses accepted work; expiry is logged and dropped without a durable state or user notification |
+| Agent execution | Global max-turn default of 25 | No wall-clock, tool-call, output, memory, CPU, process, or Org-specific budget |
+| Feishu messages | Message batching limits text count/characters | File/post attachment count and size are unbounded; downloads may buffer whole files |
+| Storage | Workspace path boundary | No Project or Org byte quota, archive-expansion limit, or entity-count ceiling |
+| Usage | Provider-reported token/cost fields and Org aggregation | Missing cost can disappear from totals; no soft-threshold notification or provider-mode attribution contract |
+| Host | systemd lifecycle | No service CPU, memory, or task ceiling and no audited workload brake |
+
+Most Organization member/team/project lists are also unpaginated, and usage
+aggregation can load all matching runs. Those query shapes must be bounded as
+part of enforcement rather than hidden behind larger process memory.
+
+## Accepted contract
+
+### Layered limits
+
+- Every mandatory dimension has a versioned, non-overridable platform ceiling.
+- An Organization OWNER/ADMIN may configure a lower policy limit.
+- Effective limit is `min(platform, organization)`; absent Organization policy
+  means the platform ceiling, never unlimited.
+- Missing or invalid mandatory production ceilings fail startup.
+
+### Admission and fairness
+
+- Project exclusivity remains one active run.
+- Platform and Organization concurrency limits apply before starting work.
+- Available slots are scheduled fairly across Organizations, FIFO within an
+  Organization, so one tenant cannot monopolize a global FIFO.
+- Accepted work is durable and queryable. Queue length and wait are bounded.
+- Overflow is explicit `capacity_exhausted` (`429` plus `Retry-After` for HTTP;
+  a clear busy response for Feishu). Nothing is silently dropped.
+- Wait timeout produces terminal `EXPIRED`, not a later automatic start.
+- The initiator may withdraw their queued request. Project `MANAGE` or an
+  Organization admin may cancel queued requests in their scope. Actor, reason,
+  and notification are recorded.
+
+### Retry and idempotency
+
+- Feishu `event_id` and API idempotency identify one admission across retries.
+- Once a run starts, the whole run is never automatically replayed because tool
+  side effects may already exist.
+- Only bounded transport retries before model output or tool side effects are
+  automatic. Later failure/timeouts require an explicit new request.
+
+### Request, file, storage, and entity safety
+
+- Authenticated Web requests are limited by user, Organization, and endpoint
+  class; signed Feishu events by Organization app, chat, and sender;
+  administrative writes use stricter independent buckets.
+- Admin surfaces require both a valid user session and the corresponding
+  platform/Organization administrator authorization. OAuth callbacks and
+  signed Feishu ingress remain necessary non-session entry points.
+- Hard limits cover request body, single-file bytes, attachment count, archive
+  expanded bytes/file count/depth, Project/Organization storage, and per-type
+  Organization entity counts.
+- Transfer is streamed and counted; partial files are removed after failure.
+- Crossing a ceiling blocks new growth without auto-deleting or hiding existing
+  resources. Read, export, delete, and administrator remediation stay usable.
+
+### Run budget
+
+Every run is bounded by wall time, model turns, tool-call count, per-tool time,
+output/log/event bytes, and process memory/CPU/child-process count. A breached
+budget ends explicitly as `TIMED_OUT` or `LIMIT_EXCEEDED`, records the dimension
+and observed usage, releases capacity, and never masquerades as success.
+
+### Usage and cost
+
+- Token usage, provider-reported cost, run count, and duration are attributed by
+  Organization, Project, Run, model, and Provider Connection.
+- Missing provider cost is `unknown`, never zero.
+- Thresholds are soft alerts in the pilot; they do not automatically stop runs
+  and are not payment settlement.
+- Org admins receive Organization alerts. Platform admins additionally receive
+  anomalous-cost alerts for platform-managed connections; BYOK does not trigger
+  platform financial alerts.
+
+### Emergency control
+
+An audited Platform Administrator control supports Organization or global
+`DRAIN` (reject new work, pause queue starts, finish active runs) and `STOP_NOW`
+(also cancel active runs). Both require an actor, reason, timestamp, explicit
+recovery, and user-visible outcomes. This is workload control, not Organization
+deletion or account suspension.
+
+## Required production evidence
+
+1. Production configuration validation fails closed for every missing or
+   invalid platform ceiling.
+2. Concurrent noisy-neighbor tests prove platform/Org/Project concurrency and
+   Organization-fair scheduling without starvation.
+3. Restart and crash tests prove accepted queue entries, idempotency, expiry,
+   cancellation, and notifications survive process loss.
+4. HTTP and Feishu load tests prove every rate-limit identity and typed overload
+   response, including retry guidance.
+5. File/archive/storage probes prove streaming enforcement, cleanup, and
+   continued read/delete access while over limit.
+6. Hung provider/tool and resource-exhaustion probes prove every run/process
+   budget terminates, records cause, and releases locks/capacity.
+7. Usage tests prove provider-mode attribution, unknown-cost handling, alert
+   routing, and absence of hard monetary blocking.
+8. `DRAIN` and `STOP_NOW` drills prove scoped, audited, reversible behavior.
+9. Production-like calibration publishes the tested numerical defaults and
+   capacity headroom used by the release gate.

.scratch/saas-production-readiness/assets/product-surface-code-inventory.md

@@ -0,0 +1,191 @@
+# Accepted product-surface code inventory
+
+Audit date: 2026-07-10
+
+Scope: source-level inventory for the accepted SaaS product boundary in ADR-0020,
+ADR-0021 and ADR-0022. This is not a claim that tests were executed during this
+inventory. “Test evidence” below means a repository test exercises the named
+path; “no direct test found” means a search of `hub/test/**/*.ts` found no request
+to the named route or corresponding service call.
+
+## Contract baseline
+
+The destination is not a shared-bot, single-credential Hub. The repository's
+current shared language says:
+
+- an Organization owns its memberships, projects, teams, external connections
+  and settings (`CONTEXT.md:7-9`);
+- a Platform Administrator creates and operates Organizations without becoming
+  their member, while an Organization Administrator is only an OWNER/ADMIN of
+  that Organization (`CONTEXT.md:11-17`);
+- the Admin Surface is a private web control plane requiring the corresponding
+  authenticated administrator authority (`CONTEXT.md:19-21`);
+- each Organization has its own Feishu application for login, bot messaging and
+  directory integration (`CONTEXT.md:23-25`);
+- provider connections are Organization-scoped BYOK or an Organization-distinct
+  platform-managed connection (`CONTEXT.md:27-37`);
+- capacity uses non-optional platform ceilings, optional lower Organization
+  limits, durable Organization-fair admission and audited emergency brakes
+  (`CONTEXT.md:39-77`).
+
+ADR-0021 makes `/admin/platform` and `/admin/org/:orgSlug` separate guarded web
+areas (`docs/adr/0021-org-admin-project-onboarding.md:23-40`), requires a
+customer-owned Feishu app with encrypted Organization-scoped secrets
+(`docs/adr/0021-org-admin-project-onboarding.md:42-47`), and pins both provider
+credential modes as Organization-scoped (`docs/adr/0021-org-admin-project-onboarding.md:49-60`).
+ADR-0022 makes its safety controls release requirements, not optional future UI
+(`docs/adr/0022-layered-capacity-admission.md:22-77`).
+
+Status vocabulary in this inventory:
+
+- **HTTP-backed** / **Feishu-backed**: a production route/event handler is wired.
+- **Service/test only**: code can be invoked directly, but there is no production
+  product entry point.
+- **Partial**: part of the journey exists but the accepted end-to-end product
+  surface cannot complete it.
+- **Missing**: no production entry and no suitable persistence/control model.
+- **Contract divergence**: reachable behavior actively uses the wrong tenancy or
+  credential shape.
+
+## Journey matrix
+
+| Target journey | Reachable entry | Authorization | Persistence | Test evidence | Verdict and exact gap |
+|---|---|---|---|---|---|
+| Platform Administrator signs in | None. The admin plugin registers only Feishu auth and Organization routes (`hub/src/admin/plugin.ts:7-8`, `hub/src/admin/plugin.ts:31-46`). The only HTML is the Org Admin login page (`hub/src/admin/routes/authRoutes.ts:171-207`). | `PlatformRoleAssignment` and `PlatformRole.ADMIN` exist in the schema (`hub/prisma/schema.prisma:111-129`), but the HTTP guard explicitly says the platform control plane is not modeled (`hub/src/admin/auth/guards.ts:1-5`). | Role rows can be stored, but there is no platform session/identity binding, invite/bootstrap flow or platform-admin audit surface. | No platform-auth route test exists. | **Missing.** `/admin/platform` from ADR-0021 is not mounted, so no authenticated operator can reach any platform workflow. |
+| Platform Administrator creates/lists/updates/operates Organizations | None. No `/api/platform/*` or Organization CRUD route is registered; the route composition is auth + `/api/org/:orgSlug/*` only (`hub/src/admin/plugin.ts:31-46`, `hub/src/admin/routes/orgRoutes.ts:24-103`). | No `requirePlatformAdmin` implementation; only `requireSession` and `requireOrgRole` exist (`hub/src/admin/auth/guards.ts:43-113`). | `Organization` has slug, name and ACTIVE/SUSPENDED/ARCHIVED status (`hub/prisma/schema.prisma:28-52`). | Organizations are created directly only by test setup, e.g. `prisma.organization.upsert` (`hub/test/integration/helpers.ts:60-77`) and a direct cross-org fixture create (`hub/test/integration/admin-explorer.test.ts:198-210`). | **Service/test only at the database layer.** Manual pilot onboarding has no safe production command, HTTP API or UI; status changes are also unreachable. |
+| Platform Administrator configures an Organization's customer-owned Feishu app and checks readiness | None. Runtime accepts exactly one `feishuAppId`/`feishuAppSecret` pair in `AdminPluginConfig` (`hub/src/admin/plugin.ts:10-19`), and server startup reads one process-global app id, secret and bot open id (`hub/src/server.ts:47-52`). | No platform guard or Organization connection ownership check. | There is no Feishu application/secret model. `ExternalDirectoryConnection` stores provider/tenant/source metadata but no app id, bot id or secret reference (`hub/prisma/schema.prisma:196-214`). | No provisioning/readiness test exists. | **Missing + contract divergence.** There is no Organization-scoped encrypted Feishu connection or readiness workflow; production can operate only the one env-configured app. |
+| Platform Administrator configures an Organization-distinct platform-managed provider key/base URL | None. No platform provider route or UI is registered. | No platform guard. | The accepted Lean type exists only in the semantic contract (`spec/Spec/System/Organization.lean:52-66`); Prisma has no provider-connection or provider-mode model (`hub/prisma/schema.prisma:26-214`). | Runtime tests prove env values are returned, not Organization values (`hub/test/unit/runtime-settings.test.ts:18-55`). | **Missing + contract divergence.** `EnvRuntimeSettings.provider` ignores its project scope and reads one process-global `ANTHROPIC_BASE_URL`/`ANTHROPIC_AUTH_TOKEN` (`hub/src/settings/runtime.ts:20-21`, `hub/src/settings/runtime.ts:46-67`). |
+| Platform Administrator applies Organization/platform `DRAIN` or `STOP_NOW` and later resumes | None. | No platform guard, reason capture or scope check. | No workload-brake model exists; only the Lean contract defines `open`/`drain`/`stopNow` semantics (`spec/Spec/System/Capacity.lean:71-87`). | No workload-brake test exists. | **Missing.** The required audited, reason-bearing, reversible emergency control from ADR-0022 (`docs/adr/0022-layered-capacity-admission.md:69-72`) has no product or persistence surface. |
+| Organization user logs in with that Organization's Feishu app | `GET /auth/feishu` and callback are HTTP-backed (`hub/src/admin/routes/authRoutes.ts:49-128`). The callback upserts a user and signs a cookie (`hub/src/admin/routes/authRoutes.ts:99-123`). | Login itself is public OAuth; Organization APIs later enforce session + active OWNER/ADMIN membership (`hub/src/admin/auth/guards.ts:43-113`). | `User.feishuOpenId` is one process-global unique identifier (`hub/prisma/schema.prisma:89-109`); the signed cookie carries user id/open id but no Organization or Feishu-connection identity (`hub/src/admin/auth/session.ts:18-23`). | Redirect and mocked callback are covered (`hub/test/integration/admin-auth.test.ts:164-230`); OAuth request construction/exchange is covered (`hub/test/unit/feishu-oauth.test.ts:8-81`). | **HTTP-backed but contract-divergent.** OAuth is built once from the process-global app credentials (`hub/src/admin/routes/authRoutes.ts:38-47`), and the implementation explicitly documents this deferral (`hub/src/admin/auth/feishuOAuth.ts:1-10`). There is no org-selecting login route, connection resolver or issuer/app identity in the user key. |
+| Each Organization operates its own Feishu Bot | One WebSocket listener is production-wired (`hub/src/server.ts:76-90`). | Message actions are gated by project grants once a binding resolves (`hub/src/feishu/trigger.ts:715-755`), but there is no initial Organization/connection ownership gate for the inbound app. | Binding stores only `chatId -> projectId`, with no Feishu connection/application id (`hub/prisma/schema.prisma:287-304`). Event receipts store globally unique `eventId`, likewise without connection id (`hub/src/feishu/trigger.ts:687-707`). | Trigger, onboarding and event dedup are integration-tested (`hub/test/integration/trigger.test.ts:211-330`, `hub/test/integration/trigger.test.ts:878-928`). | **Feishu-backed for one bot, contract-divergent for SaaS.** Startup constructs one client and one WS client from one config (`hub/src/feishu/client.ts:854-901`). It cannot operate or route multiple Organization-owned apps. |
+| Suspended/archived Organization is prevented from bot work | No platform status-change entry exists. Org HTTP APIs do reject non-ACTIVE Organizations (`hub/src/admin/auth/guards.ts:79-88`). | The Feishu trigger resolves an active chat binding directly and then project permissions; it does not load or check `Project.organization.status` (`hub/src/feishu/trigger.ts:715-755`). | Organization status exists (`hub/prisma/schema.prisma:30-52`). | No suspended-Organization trigger test exists. | **Partial and unsafe.** Even a database-side status change only blocks web admin; an already-bound chat remains triggerable through the bot path. |
+| Logged-in Organization OWNER/ADMIN reaches the management panel | OAuth success redirects to `/admin/org/:slug` (`hub/src/admin/routes/authRoutes.ts:237-267`). | All Organization APIs default to OWNER/ADMIN through `requireOrgRole` (`hub/src/admin/auth/guards.ts:66-113`), and tests prove MEMBER is denied while ADMIN succeeds (`hub/test/integration/admin-auth.test.ts:95-131`). | Session is an HMAC cookie; user/membership is re-read for each guard (`hub/src/admin/auth/session.ts:31-57`, `hub/src/admin/auth/guards.ts:43-63`). | Guard behavior is integration-tested (`hub/test/integration/admin-auth.test.ts:56-131`). | **API-backed, UI missing.** The plugin says the static SPA is “later” (`hub/src/admin/plugin.ts:1-3`); no route/static handler serves `/admin/org/:slug`. OAuth therefore redirects to an unmounted page. |
+| Organization OWNER/ADMIN manages members and roles | REST endpoints list/add/change/revoke members (`hub/src/admin/routes/membersRoutes.ts:20-98`). | OWNER/ADMIN route guard plus service rules: only OWNER may affect OWNER and the last OWNER cannot be removed (`hub/src/org/members.ts:1-8`, `hub/src/org/members.ts:97-205`). | `OrganizationMembership` persists org, user, role and revocation (`hub/prisma/schema.prisma:54-76`). | Add/list/last-OWNER protection is covered through HTTP (`hub/test/integration/admin-members-teams.test.ts:49-83`). | **HTTP-backed and persisted; management UI missing.** Addition requires a raw `feishuOpenId` and may create the `User` directly (`hub/src/org/members.ts:46-85`); there is no invite/accept or org-connection directory picker. Role-change/revoke variants have less direct HTTP coverage than add/list. |
+| Organization OWNER/ADMIN manages teams and team membership | REST endpoints cover list/create/update/archive and membership add/revoke (`hub/src/admin/routes/teamsRoutes.ts:21-139`). | All routes require Organization OWNER/ADMIN; service verifies the team and added user belong to the same Organization (`hub/src/org/teams.ts:177-215`, `hub/src/org/teams.ts:263-275`). | `Team` and `TeamMembership` are Organization/project-principal persistence (`hub/prisma/schema.prisma:143-177`). | Create/add-member/archive is covered through HTTP (`hub/test/integration/admin-members-teams.test.ts:85-160`). | **HTTP-backed and persisted; management UI missing.** No production route exposes `ExternalDirectoryConnection`/external team binding sync, so the Feishu directory portion remains service-only. |
+| Organization OWNER/ADMIN creates and manages Folders/Projects | REST endpoints cover explorer, folder create/update/archive, project create/detail/rename/move/archive (`hub/src/admin/routes/explorerRoutes.ts:33-212`). | All routes require Organization OWNER/ADMIN. Service operations scope folder/project ids to the Organization (`hub/src/org/explorer.ts:301-341`); HTTP cross-org lookup is tested. | Folder and Project carry `organizationId`; Folder is transparent and Project owns workspace/session/run relations (`hub/prisma/schema.prisma:238-285`). Project creation also creates a real workspace and default permission rows (`hub/src/projectOnboarding.ts:275-348`). | Create/list and move/archive are HTTP-tested (`hub/test/integration/admin-explorer.test.ts:66-196`); cross-org id access is tested (`hub/test/integration/admin-explorer.test.ts:198-223`). | **HTTP-backed and substantially tested; management UI missing.** This is the strongest accepted product slice. |
+| Organization OWNER/ADMIN manages Feishu chat binding | HTTP can archive a binding (`hub/src/admin/routes/explorerRoutes.ts:214-227`). New binding is exposed through the Feishu onboarding card, not web. | Archive requires OWNER/ADMIN at HTTP and then project manager authorization in the shared service (`hub/src/projectOnboarding.ts:231-272`, `hub/src/org/explorer.ts:301-314`). Bind existing requires project MANAGE (`hub/src/projectOnboarding.ts:179-229`). | `ProjectGroupBinding` and the FEISHU_CHAT EDIT grant are transactionally stored (`hub/src/projectOnboarding.ts:190-215`, `hub/prisma/schema.prisma:287-304`). | Bind/one-active/archive/rebind is service-tested (`hub/test/integration/project-onboarding.test.ts:114-160`); card bind is integration-tested (`hub/test/integration/trigger.test.ts:271-330`). No direct HTTP archive test was found. | **Feishu + partial HTTP-backed; UI missing.** Binding identity is not scoped by Feishu application, so the 1:1 uniqueness and lookup are process-global rather than Organization-connection-scoped. |
+| Organization OWNER/ADMIN manages project team access | REST list/grant/revoke endpoints exist (`hub/src/admin/routes/accessRoutes.ts:19-79`). | OWNER/ADMIN route guard; service refuses a team from another Organization (`hub/src/permissions/projectTeamAccess.ts:168-195`). | TEAM -> PROJECT `PermissionGrant` persists the role (`hub/prisma/schema.prisma:428-447`). | Grant/list/archive-induced revoke is HTTP-tested (`hub/test/integration/admin-members-teams.test.ts:123-156`). | **HTTP-backed and tested; UI missing.** Only team grants are surfaced. There is no HTTP management of direct USER/chat grants, `PermissionSettings`, or per-role `RoleTriggerGrant`. |
+| Organization OWNER/ADMIN manages general project settings, roles/models and agent policies | Only `membersCanCreateProjects` is reachable through GET/PATCH `/settings` (`hub/src/admin/routes/orgRoutes.ts:46-81`). | OWNER/ADMIN route guard. | `OrganizationProjectSettings` stores only that Boolean (`hub/prisma/schema.prisma:78-87`). `PermissionSettings` exists, but no admin route references it (`hub/prisma/schema.prisma:449-467`). Model/role registry is explicitly an in-memory skeleton awaiting DB/admin wiring (`hub/src/agent/models.ts:65-79`). | The Boolean setting is HTTP-tested (`hub/test/integration/admin-auth.test.ts:133-162`); model env behavior is unit-tested (`hub/test/unit/runtime-settings.test.ts:4-64`). | **Partial.** The creation policy works, but the accepted “projects and configuration” control plane lacks editable permission settings, model/role settings and run policies. |
+| Organization OWNER/ADMIN chooses BYOK vs platform-managed Provider mode; BYOK admin writes key/base URL | None. | No Organization provider guard because no route/service exists. | No Prisma provider connection, credential mode, encrypted secret handle or rotation metadata. The only persistent `provider` fields are descriptive strings on sessions/runs (`hub/prisma/schema.prisma:308-330`, `hub/prisma/schema.prisma:352-387`). | No provider-mode test. Existing tests assert the global env resolver (`hub/test/unit/runtime-settings.test.ts:18-55`). | **Missing + contract divergence.** Lean/ADR have the mode but production always calls `settings.provider("openrouter", { projectId })` (`hub/src/feishu/trigger.ts:195-199`), whose implementation discards the scope (`hub/src/settings/runtime.ts:46-67`). |
+| Organization OWNER/ADMIN views sessions and runs | HTTP list and detail endpoints exist (`hub/src/admin/routes/sessionsRoutes.ts:14-45`). | OWNER/ADMIN route guard; service scopes project/session through Organization id (`hub/src/org/sessions.ts:6-23`, `hub/src/org/sessions.ts:49-87`). | `AgentSession`/`AgentRun` persist provider/model/status/tokens/cost and relations (`hub/prisma/schema.prisma:308-387`). | No direct test for these HTTP routes or `org/sessions.ts` was found. Trigger tests do prove sessions/runs are written, e.g. separate role sessions (`hub/test/integration/trigger.test.ts:850-876`). | **HTTP-backed and persisted, but untested at the product boundary and UI missing.** |
+| Organization OWNER/ADMIN views Organization/Project/Folder usage | HTTP Organization and Project usage endpoints exist, with optional folder/date filtering (`hub/src/admin/routes/sessionsRoutes.ts:47-79`). | OWNER/ADMIN route guard. | Rollup derives Organization through Project and uses persisted token/cost facts (`hub/src/org/usage.ts:35-77`, `hub/src/org/usage.ts:106-151`). Missing cost increments `runsWithoutCost`, not known cost (`hub/src/org/usage.ts:106-118`). | No direct usage route/service test was found. `/cost` tests cover the Feishu session command, not these admin endpoints (`hub/test/integration/trigger.test.ts:423-480`). | **HTTP-backed and persisted, but incomplete relative to ADR-0022.** It does not attribute by Provider Connection because none exists, has no alert configuration/delivery, no UI, and no product-boundary tests. |
+| Ordinary MEMBER creates a Project from an unbound Feishu group | Mentioning the bot in an unbound chat sends an onboarding card (`hub/src/feishu/trigger.ts:850-887`); card action calls shared project creation (`hub/src/feishu/trigger.ts:554-614`). | User must already exist and have an active Organization membership; ordinary creation is gated by `membersCanCreateProjects` (`hub/src/projectOnboarding.ts:149-177`, `hub/src/projectOnboarding.ts:380-403`). Creator receives USER MANAGE and the chat receives EDIT (`hub/src/projectOnboarding.ts:302-340`). | Project, Folder, workspace, binding, default settings and grants are persisted (`hub/src/projectOnboarding.ts:275-348`). | Card display/create/bind is integration-tested (`hub/test/integration/trigger.test.ts:211-330`); policy denial and grants are service-tested (`hub/test/integration/project-onboarding.test.ts:70-112`). | **Feishu-backed and well-tested for the single-app case.** For a user with multiple active memberships, onboarding refuses and says the org selector does not exist (`hub/src/feishu/trigger.ts:889-929`). With a per-Org app, the inbound connection should supply tenant context; current global bot cannot. |
+| Ordinary MEMBER binds an existing manageable Project | Onboarding card lists unbound projects and card action binds one (`hub/src/feishu/trigger.ts:868-885`, `hub/src/feishu/trigger.ts:932-969`). | Non-admin candidate projects require `collaborator.manage`; bind service rechecks project MANAGE (`hub/src/feishu/trigger.ts:951-960`, `hub/src/projectOnboarding.ts:179-215`). | Binding + FEISHU_CHAT EDIT grant persist transactionally. | Card journey is integration-tested (`hub/test/integration/trigger.test.ts:271-330`). | **Feishu-backed for the single-app case; per-Organization connection routing is absent.** |
+| Ordinary authorized member triggers an Agent Run from Feishu | Bound message event resolves Project, evaluates `agent.trigger`, resolves role/model/provider, creates session/run and starts the runner (`hub/src/feishu/trigger.ts:684-848`, especially `hub/src/feishu/trigger.ts:715-755` and `hub/src/feishu/trigger.ts:140-256`). | Missing sender id is fail-closed; project EDIT+ and optional role grant are enforced (`hub/src/feishu/trigger.ts:171-194`, `hub/src/feishu/trigger.ts:726-755`). | Session/run, messages, file changes, event receipt and audit fragments persist in Prisma (`hub/prisma/schema.prisma:306-405`, `hub/prisma/schema.prisma:496-525`). | Lifecycle, permission denial, dedup, audit, messages and interrupt have integration coverage (`hub/test/integration/trigger.test.ts:116-154`, `hub/test/integration/trigger.test.ts:597-607`, `hub/test/integration/trigger.test.ts:878-928`, `hub/test/integration/trigger.test.ts:983-1100`). | **Feishu-backed and tested for one process-global provider/Feishu connection.** It is not an Organization-resolved run: provider scope is ignored and bot/identity connection is absent. |
+| Accepted work waits under Project/Organization/platform capacity and survives restart | Only a process-local queue per Project exists (`hub/src/feishu/triggerQueue.ts:18-45`). It is drained after the active Project run (`hub/src/feishu/trigger.ts:430-495`). | No Organization admission identity or capacity policy is evaluated. | Queue is a `Map` in memory (`hub/src/feishu/triggerQueue.ts:21-26`), not a Prisma model; accepted requests vanish on process exit. Expired entries are dropped and only logged (`hub/src/feishu/trigger.ts:475-483`). | FIFO/full/expiry behavior is unit-tested (`hub/test/unit/trigger-queue.test.ts:10-78`); locked-trigger enqueue/drain is integration-tested (`hub/test/integration/trigger.test.ts:609-673`). | **Contract divergence.** ADR-0022 requires durable bounded admission, Organization-fair scheduling, explicit EXPIRED/CANCELED states and notifications (`docs/adr/0022-layered-capacity-admission.md:33-46`); none are present. |
+| Platform ceiling + optional lower Organization policy protect request/run/file/storage/entity capacity | None beyond Fastify defaults, a five-item Project queue and global max turns. `Fastify({ logger: true })` has no capacity options/plugins (`hub/src/server.ts:55-55`); dependencies include no rate-limit plugin (`hub/package.json:9-17`). | No admin policy route. | No capacity/limit model. `OrganizationProjectSettings` has only `membersCanCreateProjects` (`hub/prisma/schema.prisma:78-87`). Run policy contains only global `maxTurns` (`hub/src/settings/runtime.ts:24-37`, `hub/src/settings/runtime.ts:74-78`). The systemd unit has no CPU/memory/process controls (`hub/deploy/cph-hub.service:6-26`). | Unit tests cover only global max turns and the local queue (`hub/test/unit/runtime-settings.test.ts:58-64`, `hub/test/unit/trigger-queue.test.ts:5-113`). | **Missing.** None of ADR-0022's mandatory platform/org request rate, body, concurrency, file/archive, storage, entity-count, wall-time/tool/output or process limits are product-reachable or persistently enforced (`spec/Spec/System/Capacity.lean:13-60`). |
+
+## Reachability summary
+
+### Production-wired HTTP surface
+
+The actual production plugin mounts these capability groups:
+
+- global Feishu OAuth/session: `/auth/feishu`, callback, logout, `/api/me`, and
+  the minimal `/admin/login` HTML (`hub/src/admin/routes/authRoutes.ts:49-207`);
+- Organization summary and one onboarding Boolean
+  (`hub/src/admin/routes/orgRoutes.ts:27-81`);
+- Folder/Project explorer and binding archive
+  (`hub/src/admin/routes/explorerRoutes.ts:33-227`);
+- member roles (`hub/src/admin/routes/membersRoutes.ts:20-98`);
+- teams/team membership (`hub/src/admin/routes/teamsRoutes.ts:21-139`);
+- TEAM -> PROJECT grants (`hub/src/admin/routes/accessRoutes.ts:19-79`);
+- sessions and usage (`hub/src/admin/routes/sessionsRoutes.ts:14-79`).
+
+Every `/api/org/:orgSlug/*` route uses the same OWNER/ADMIN guard. This satisfies
+the clarified requirement that a management API cannot be used by an anonymous
+or ordinary Organization member (`hub/src/admin/auth/guards.ts:43-113`). It does
+not provide the accepted management **panel**: only `/admin/login` is rendered,
+and the post-login `/admin/org/:slug` destination is not served.
+
+### Production-wired Feishu surface
+
+The trigger handler is a real end-to-end application path for one Feishu app:
+event receipt/dedup, chat binding, permission checks, unbound-chat onboarding,
+Project creation/binding, sessions/runs, streaming and interrupt. The service
+wires the handler to a single SDK client and WebSocket listener
+(`hub/src/server.ts:76-90`, `hub/src/feishu/client.ts:854-901`). The code and tests
+therefore establish a useful single-app vertical slice, not the required
+per-Organization connection architecture.
+
+### Only database/service/test reachable
+
+- Organization creation is direct Prisma fixture work
+  (`hub/test/integration/helpers.ts:60-77`).
+- `PlatformRoleAssignment` is a stored type with no platform auth/HTTP surface
+  (`hub/prisma/schema.prisma:111-129`).
+- external-directory sync has service code and can create membership rows
+  (`hub/src/permissions/externalSync.ts:40-90`), but no registered production
+  admin route.
+- project `PermissionSettings` and `RoleTriggerGrant` are stored/enforced in
+  parts of the runtime, but have no Organization management API
+  (`hub/prisma/schema.prisma:449-494`).
+
+### Entirely absent from runtime persistence/control plane
+
+- Organization-scoped Feishu application connection and encrypted secret
+  handle;
+- Organization-scoped Provider Connection, BYOK/platform-managed mode, encrypted
+  secret handle, rotation and runtime resolver;
+- platform administrator session/guard, platform Organization CRUD, platform
+  admin invitation, platform-operation audit;
+- versioned platform capacity ceiling loader/validation;
+- Organization policy-limit rows and Organization admin endpoints;
+- durable run request/admission state, fair Organization scheduler, queue cancel
+  and expiry notification;
+- Organization/platform workload brake state and audited actions;
+- usage/cost alert configuration and delivery;
+- the Organization and platform management UI.
+
+## Critical production blockers
+
+1. **No Organization connection control plane.** The running service has one
+   Feishu app/bot and one provider token/base URL. This contradicts the defining
+   SaaS requirement even though Project/Team data is otherwise Organization-
+   scoped (`hub/src/server.ts:44-52`, `hub/src/settings/runtime.ts:46-67`).
+
+2. **No platform control plane.** A Platform Administrator cannot authenticate,
+   create or operate an Organization, provision its Feishu app, configure its
+   platform-managed provider connection, or apply a workload brake. The only
+   apparent Organization creation path is direct database/test code.
+
+3. **No usable management panel.** The backend Organization APIs are meaningful
+   and guarded, but OAuth redirects to an unmounted `/admin/org/:slug`; platform
+   UI does not exist at all (`hub/src/admin/plugin.ts:1-8`,
+   `hub/src/admin/routes/authRoutes.ts:237-267`).
+
+4. **ADR-0022 is almost wholly unimplemented.** The in-memory Project queue is
+   specifically the implementation divergence ADR-0022 rejects; there are no
+   Organization-fair durable admission, mandatory layered limits, hard run/file/
+   storage/process budgets or emergency brakes
+   (`docs/adr/0022-layered-capacity-admission.md:79-88`).
+
+5. **Tenant context is inferred too late and from global identifiers.** OAuth
+   creates a globally keyed `User`, bot events bind globally by `chatId`, and an
+   unbound-chat user with more than one Organization is rejected rather than
+   routed by the customer-owned Feishu connection (`hub/prisma/schema.prisma:89-109`,
+   `hub/src/feishu/trigger.ts:715-723`, `hub/src/feishu/trigger.ts:889-929`). This
+   blocks a correct multi-app implementation and also leaves Organization status
+   unenforced on the bot path.
+
+## What is already worth retaining
+
+The audit does not imply rewriting the working core. The following seams are
+concrete and have useful test evidence:
+
+- Organization ownership on Project/Team/Folder and same-Organization access
+  checks (`hub/prisma/schema.prisma:143-177`, `hub/prisma/schema.prisma:238-285`);
+- OWNER/ADMIN guards and last-OWNER protection
+  (`hub/src/admin/auth/guards.ts:66-113`, `hub/src/org/members.ts:97-205`);
+- reusable service operations for project creation, chat binding and grants
+  (`hub/src/projectOnboarding.ts:128-229`, `hub/src/projectOnboarding.ts:275-348`);
+- Organization-scoped explorer/member/team/team-access HTTP APIs and their
+  integration tests;
+- Feishu onboarding, project permission checks, session/run persistence and
+  dedup in the single-app vertical slice.
+
+The shortest path to the accepted destination is therefore to add the missing
+platform/connection/capacity control planes and real admin UI around these seams,
+then change ingress and runtime resolution to carry an explicit Organization
+connection identity end to end. Treating the existing env credentials as an
+acceptable pilot fallback would preserve the wrong tenant boundary.

.scratch/saas-production-readiness/assets/product-surface-completeness-audit.md

@@ -0,0 +1,259 @@
+# Accepted SaaS product-surface completeness audit
+
+Audit date: 2026-07-10
+
+## Verdict
+
+The current Hub is **not a usable or production-complete SaaS product surface**
+for the accepted destination. It has a worthwhile Organization-scoped backend
+foundation and a well-tested single-Feishu-application Agent Run slice, but the
+running product still has one process-global Feishu app/bot and one global model
+provider credential. It has no platform-administrator control plane, no
+Organization connection/secret plane, and no Organization-scoped Provider
+Credential Mode. Its only HTML product page is a login form; successful OAuth
+redirects an administrator to an unmounted `/admin/org/:orgSlug` route.
+
+ADR-0022's required production safety surface is also almost wholly absent:
+there is no durable Organization-fair admission, layered platform/Organization
+limits, hard file/storage/run/process budgets, usage-alert control plane, or
+audited workload brake. These are release blockers, not optional polish.
+
+The strongest code should be retained rather than rewritten: all current
+`/api/org/:orgSlug/*` routes reauthorize an active OWNER/ADMIN membership;
+Folder/Project, member, Team, Team-to-Project access, session, and usage services
+are Organization-scoped; and the shared project-onboarding services correctly
+create Project/chat/grant relationships for the single-app case. The shortest
+route is to add explicit Organization connection identity and the two private
+admin control planes around those seams.
+
+The claim-by-claim source inventory, including route, schema and test locations,
+is in [accepted product-surface code inventory](product-surface-code-inventory.md).
+Official Feishu identifier, OAuth, event and long-connection semantics are in
+[Feishu product-surface primary sources](product-surface-feishu-primary-sources.md).
+
+## Accepted boundary
+
+The audit treats these as required because they are already accepted by the
+repository's `CONTEXT.md`, ADR-0020, ADR-0021, ADR-0022, and Lean contract:
+
+- a Platform Administrator is separate from Organization membership and uses a
+  private `/admin/platform` area to create and operate Organizations;
+- an Organization OWNER/ADMIN uses a private `/admin/org/:orgSlug` area to
+  manage members, projects, Teams, connections, settings, sessions and usage;
+- every Organization authenticates and exchanges bot/directory traffic through
+  its own customer-owned Feishu Application;
+- every Organization chooses BYOK or an Organization-distinct platform-managed
+  Provider Connection, with different administrators owning the respective
+  plaintext inputs;
+- ordinary members can create/bind a Project from an unbound Feishu group only
+  under the Organization policy, while all Project actions retain Project-level
+  authorization;
+- production has mandatory platform ceilings, optional lower Organization
+  limits, durable fair admission, bounded work and audited emergency brakes.
+
+The audit does not add deferred self-service signup, payment collection,
+folder ACL inheritance, commercial billing, or a live editor for platform
+capacity ceilings. Guided Feishu app setup and manually initiated Organization
+creation by an authenticated Platform Administrator remain valid for the pilot.
+
+## Journey result
+
+| Accepted journey | Current product entry | Result | Owning frontier |
+| --- | --- | --- | --- |
+| Platform Administrator signs in | No platform login/session/guard; only a stored `PlatformRoleAssignment` type | **Missing** | tickets 42, 45 |
+| Platform Administrator creates and operates Organizations | Direct Prisma calls in test fixtures only; no `/api/platform/*` or UI | **Test/database only** | ticket 45 |
+| Platform Administrator configures each customer Feishu app | One env `FEISHU_APP_ID/SECRET/BOT_OPEN_ID`; no connection or secret model | **Missing and contract-divergent** | tickets 43–45 |
+| Platform Administrator supplies a distinct platform-managed provider key/base URL | One process-global `ANTHROPIC_AUTH_TOKEN/BASE_URL`; resolver ignores Project scope | **Missing and contract-divergent** | tickets 43, 45 |
+| Platform Administrator applies and reverses `DRAIN`/`STOP_NOW` | No persistence, guard, API or UI | **Missing** | tickets 37, 45 |
+| Organization user logs in through that Organization's Feishu app | OAuth exists, but is built once from the global app and has no Organization/application issuer | **Single-app only** | tickets 43, 44 |
+| Logged-in Organization OWNER/ADMIN opens the management panel | OAuth redirects to `/admin/org/:slug`; no handler/static application serves it | **Stable 404** | ticket 46 |
+| Organization OWNER/ADMIN manages members, Teams, Projects/Folders and Team access | Guarded Organization JSON APIs and useful integration coverage exist | **Backend present; browser product absent** | ticket 46 |
+| Organization OWNER/ADMIN manages permission/model/role/run policy | Only `membersCanCreateProjects` is exposed; other stored/runtime settings have no control plane | **Partial** | ticket 46 |
+| Organization selects Provider Credential Mode and manages BYOK | No mode/connection model, API or UI | **Missing** | tickets 43, 46 |
+| Organization views sessions and usage | Guarded JSON APIs exist, but have no direct product-boundary tests or UI and cannot attribute Provider Connection | **Partial** | tickets 36, 46 |
+| Ordinary member creates or binds a Project in Feishu | Card/service path is present and well tested | **Works for the one-global-app case** | ticket 44 for SaaS routing |
+| Authorized member runs the agent through its Organization bot | Full single-app trigger path exists; provider/app resolution is global | **Works only as a single-app deployment** | tickets 43, 44 plus lifecycle tickets |
+| Accepted work queues fairly and durably across Organizations | Five-item process-memory Project queue | **Contract-divergent** | tickets 23, 32–35 |
+| Organization sets lower policy limits and receives soft alerts | No policy/alert persistence or product surface | **Missing** | tickets 32, 34–36, 46 |
+
+## Executed defect evidence
+
+The probes used the real Fastify plugin and test PostgreSQL database through a
+temporary Vitest integration file. The file was removed after capture and was
+run twice to separate deterministic behavior from a transient test failure.
+
+### 1. The successful-login destination is not served
+
+The probe seeded an active Organization ADMIN, minted the normal signed session
+cookie, and requested the exact destination selected by
+`resolvePostLoginRedirect`:
+
+```text
+GET /admin/org/test-default
+expected_status=200
+actual_status=404
+```
+
+Both runs returned 404. The root cause is structural: `registerAdminPlugin`
+mounts cookie, auth and Organization API routes only; its own comment says the
+static SPA is “later”. No other repository HTML/TSX/Vue/Svelte/CSS product
+application or deployment-level static handler exists. Existing OAuth tests
+assert only the redirect `Location`, so they stop immediately before the broken
+journey.
+
+This is not an authorization defect in the JSON APIs. Those APIs correctly
+return 401 without a session and 403 to a MEMBER. It is a reachability defect:
+there is no management panel for either an authorized or unauthorized request
+to reach. Ticket 46 must guard both the page shell and its data, so an anonymous
+user may reach only the login entry and never management content.
+
+### 2. Different Organizations start OAuth with the same application
+
+The probe seeded two Organizations and requested the current OAuth entry with
+two different Organization return paths:
+
+```text
+GET /auth/feishu?returnTo=/admin/org/test-default
+client_id=cli_process_global
+
+GET /auth/feishu?returnTo=/admin/org/second
+client_id=cli_process_global
+```
+
+Both runs produced the same result. The root cause is that
+`registerAuthRoutes` constructs one immutable OAuth configuration from plugin
+startup arguments. The `returnTo` path is signed only as a destination; it does
+not resolve or bind an Organization Feishu Application. `server.ts` obtains the
+same app credentials from one environment tuple and also uses them for the only
+WebSocket listener.
+
+Changing only the redirect page cannot fix this. The composition root and every
+identifier-bearing record/action need an explicit Organization connection
+identity, backed by the encrypted resolver in ticket 43.
+
+## Feishu identifier and transport consequence
+
+Feishu's official identifier contract makes the current global identifier
+shape unsafe for customer-owned applications:
+
+- `open_id` identifies a user inside one application and differs across apps;
+- `user_id` is stable inside one tenant but differs across tenants;
+- `union_id` can link apps from the same application service provider, but the
+  service cannot assume independently customer-owned apps share that provider;
+- OAuth authorization identifies the app with `client_id`; the callback brings
+  `code` and `state`, so state and token exchange must remain bound to the
+  selected Organization/application;
+- v2 HTTP event/card headers expose application and tenant identity, while an
+  encrypted outer request must be routed to the correct connection before its
+  verification/decryption key is known; a long connection is itself created by
+  one app credential pair, so the local client/listener identity is the issuer;
+- Feishu rejects cross-tenant chat/operator combinations and does not provide a
+  contract that permits `chat_id` to be treated as a global SaaS key.
+
+The current `User.feishuOpenId @unique`, session `{userId, feishuOpenId}`,
+`ProjectGroupBinding.chatId`, `FeishuEventReceipt.eventId`, principal IDs,
+sender caches, and batching keys omit this issuer namespace. An `open_id`
+collision across customer apps could merge unrelated external identities;
+different `open_id` values for one person across apps cannot be resolved as one
+global identity automatically. A globally looked-up `chatId` or event ID has
+the analogous ambiguity.
+
+The implementation must therefore persist provider-local external identities
+under a Feishu Application Connection, for example conceptually
+`(connectionId, externalIdType, externalId)`, and carry that connection through
+OAuth state/session, ingress, deduplication, chat binding, authorization,
+onboarding/card actions, rate-limit keys, directory sync, outbound delivery and
+audit. Whether two external identities represent the same global `User` is a
+separate explicit linking decision; it must not be inferred from a provider ID
+outside its documented scope.
+
+## Root causes rather than symptoms
+
+### 1. Tenant data was added without tenant-scoping external connections
+
+Organization ownership is correctly present on Projects, Teams, Folders and
+directory-connection metadata, but the process composition still comes from
+the former single-deployment shape. The app/bot/provider credentials and the
+identifiers they issue never crossed an Organization connection boundary.
+`RuntimeScope.projectId` exists but is discarded by the env resolver, making
+the missing seam especially explicit.
+
+### 2. The control-plane work stopped at guarded Organization APIs
+
+The Organization routes provide a credible backend slice, but there is no web
+application, platform route family, platform guard, Organization provisioning
+service, connection configuration, or browser journey test. The comment
+“until admin SPA lands” and redirect-only OAuth assertion document the precise
+unfinished handoff. A small catch-all page would only hide the gap; the fix must
+deliver the authenticated, authorized workflows and their safe states.
+
+### 3. Production capacity semantics exist only in the contract
+
+ADR-0022 intentionally declares the in-memory queue and global-only settings to
+be implementation divergences. There is no common admission identity that can
+join Organization, Feishu connection, Project, user and request, so fair
+scheduling, workload brakes, rate limits, alerts and control-plane operations
+cannot currently compose. The previously created capacity tickets own this
+implementation; the admin tickets must expose their safe operator/customer
+surfaces rather than invent parallel controls.
+
+## What should remain
+
+- Keep `requireOrgRole`'s per-request session, active Organization, active
+  membership and OWNER/ADMIN checks; extend the same principle to page data and
+  add a separate `requirePlatformAdmin` rather than an Organization override.
+- Keep the Organization-scoped explorer/member/Team/access/session/usage APIs
+  as the initial backend for the org panel, while filling their missing tests,
+  pagination/limits, audit and typed-error contracts through existing tickets.
+- Keep shared Project creation and chat-binding services so web and Feishu
+  entries do not drift into two rule sets.
+- Keep the single-app Feishu handler's authorization, onboarding, run/session
+  and dedup behavior, but call it from a connection-bound listener context and
+  move accepted work/delivery into the durable lifecycle tickets.
+- Keep missing provider cost unknown rather than fabricating zero; add Provider
+  Connection attribution before presenting SaaS usage claims.
+
+## Newly clear execution frontier
+
+The product audit creates four implementation/decision slices after the
+existing secret and browser-boundary design work:
+
+1. [Decide the platform-administrator identity and audit boundary](../issues/42-decide-platform-admin-identity-audit.md).
+2. [Implement the Organization connection and secret plane](../issues/43-implement-org-connection-secret-plane.md).
+3. [Route Feishu identity and traffic by Organization connection](../issues/44-route-feishu-by-org-connection.md).
+4. [Build the private platform-administrator control plane](../issues/45-build-private-platform-admin-control-plane.md).
+5. [Build the private Organization-administrator control plane](../issues/46-build-private-org-admin-control-plane.md).
+
+The Feishu routing slice precedes durable inbound/action/outbound and signed
+traffic limiting so those mechanisms use the right issuer namespace from their
+first durable schema. The platform panel precedes the emergency-brake product
+entry so ticket 37 can reuse one authenticated/audited operator boundary.
+Existing tickets 32–38 continue to own capacity enforcement, attribution,
+alerts, brakes and calibration; the new UI tickets do not duplicate them.
+
+## Required production journey evidence
+
+The final release gate must run against at least two configured customer-owned
+Feishu applications and two Organizations, using deliberately colliding fake
+provider-local IDs where the test harness permits. It must prove:
+
+1. an unauthenticated request and an authenticated wrong-role user cannot load
+   either management page data or perform mutations;
+2. a Platform Administrator can create an Organization, bootstrap its OWNER,
+   configure/test its Feishu app and distinct platform-managed Provider
+   Connection, with redacted responses and complete audit history;
+3. an Organization OWNER/ADMIN logs in through that Organization's app, reaches
+   the real org panel, manages the accepted settings/resources, configures BYOK
+   when selected, and cannot read another Organization or platform secret;
+4. each bot receives, binds, authorizes, queues and replies only inside its
+   connection's Organization even when `open_id`, `chat_id`, event or callback
+   values collide; SUSPENDED/ARCHIVED Organizations fail closed;
+5. provider resolution for every run returns that Organization's selected
+   connection and never a process-global fallback;
+6. current member onboarding, Project authorization and Agent Run behavior
+   remain intact after connection namespacing;
+7. layered limits, fair admission, usage alerts and `DRAIN`/`STOP_NOW` are
+   visible only to the corresponding authorized control plane and behave as
+   proven by their dedicated fault/capacity suites;
+8. the complete browser + Feishu journey survives the production-shaped deploy,
+   restart, backup/restore and rollback evidence owned by ticket 08.

.scratch/saas-production-readiness/assets/product-surface-feishu-primary-sources.md

@@ -0,0 +1,333 @@
+# Feishu multi-app identity and event-routing primary sources
+
+This note answers one narrow design question for the production SaaS audit:
+what identity and routing boundaries apply when **every Hub Organization uses
+its own Feishu App and bot**. External claims below use only first-party Feishu
+Open Platform documentation. Statements labelled **Derived requirement** are
+Hub design implications, not guarantees made by Feishu.
+
+Research date: 2026-07-10.
+
+## Executive answer
+
+1. A customer-owned enterprise custom app is a single-tenant app: Feishu says
+   it is developed, reviewed, published, and used inside one tenant and cannot
+   be used by another organization. That fits “one Organization, one Feishu
+   App,” but it means the Hub is operating many independent app security and
+   identity domains, not one shared Feishu identity domain.
+   ([custom apps versus store apps](https://open.feishu.cn/document/home/app-types-introduction/self-built-apps-and-store-apps))
+2. A bare `open_id` is **not** a SaaS-global user key. It identifies a user
+   inside one app, and the same user has different Open IDs in different apps.
+   `user_id` is tenant-scoped; it is stable across apps in one tenant but
+   differs across tenants. `union_id` crosses apps only for apps from the same
+   application provider/developer; it differs between providers.
+   ([user resource overview](https://open.feishu.cn/document/server-docs/contact-v3/user/field-overview?lang=zh-CN),
+   [official ID parameter definitions](https://open.feishu.cn/document/server-docs/contact-v3/department/search?lang=zh-CN),
+   [Union ID guide](https://open.feishu.cn/document/uAjLw4CM/ugTN1YjL4UTN24CO1UjN/trouble-shooting/how-to-obtain-union-id))
+3. OAuth must be routed to the Organization's app **before** authorization.
+   The authorization request contains that app's App ID as `client_id`; the
+   callback contains `code` and the echoed `state`, not an independently
+   trustworthy Organization choice. Exchanging the code requires the matching
+   app's `client_id` and `client_secret`, and the same `redirect_uri`.
+   ([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code),
+   [obtain user access token](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
+4. A version 2 event or callback carries `header.app_id` and
+   `header.tenant_key` after parsing/decryption. An encrypted HTTP delivery,
+   however, exposes only an `encrypt` envelope before decryption, so a shared
+   endpoint cannot safely choose among many apps' Encrypt Keys from the body.
+   ([callback structure](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/callback-overview),
+   [encrypted Webhook delivery](https://open.feishu.cn/document/event-subscription-guide/event-subscriptions/event-subscription-configure-/choose-a-subscription-mode/send-notifications-to-developers-server?lang=zh-CN))
+5. A WebSocket/long-connection client is authenticated with one app's App ID
+   and App Secret. Therefore its local client instance is the pre-authenticated
+   source-app route. It is not a process-global “all customer apps” listener.
+   ([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
+6. Feishu documents `chat_id` as the conversation identifier, but the cited
+   documentation does not promise that a raw `chat_id` is globally unique
+   across every tenant and app. Chat APIs explicitly reject an operator and
+   chat from different tenants and often require the app bot to be in the
+   chat. A bot is represented by its application's `app_id`.
+   ([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology),
+   [get chat members](https://open.feishu.cn/document/server-docs/group/chat-member/get?lang=zh-CN),
+   [message resource](https://open.feishu.cn/document/server-docs/im-v1/message/intro))
+
+## 1. Application and tenant boundary
+
+### Official facts
+
+- `app_id` is the immutable identifier generated by Feishu for an application.
+  `tenant_key` is the enterprise identifier; one `tenant_key` corresponds to
+  one enterprise in actual use.
+  ([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology))
+- An enterprise custom app is restricted to one tenant. Its developers,
+  administrators, and users belong to that tenant, and another organization
+  cannot use it. A store app is the contrasting model that can be installed by
+  multiple tenants.
+  ([custom apps versus store apps](https://open.feishu.cn/document/home/app-types-introduction/self-built-apps-and-store-apps))
+- Access tokens tell Feishu which app/user is calling and which tenant's data
+  is being accessed. A `tenant_access_token` calls as the application and its
+  data range is determined by that application's permissions.
+  ([access-token overview](https://open.feishu.cn/document/server-docs/api-call-guide/calling-process/get-access-token))
+- A custom app obtains its `tenant_access_token` using that app's `app_id` and
+  `app_secret`.
+  ([custom-app tenant access token](https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal?lang=zh-CN))
+
+### Derived requirements
+
+- Model a durable `FeishuAppConnection` (or equivalent) owned by exactly one
+  Hub Organization. At minimum it binds the internal connection ID,
+  `organization_id`, `app_id`, expected `tenant_key`, encrypted `app_secret`,
+  event/callback credentials, lifecycle state, and credential version.
+- Do not select Feishu credentials from a process-global default. Every OAuth
+  exchange, token cache entry, OpenAPI call, Webhook verification, and
+  long-connection client must start from the Organization's connection.
+- Pin the first verified `tenant_key` to the connection. Later login, event,
+  and callback data must match both the expected `app_id` and pinned
+  `tenant_key`; unknown or mismatched pairs must be rejected rather than
+  auto-attached to another Organization.
+
+## 2. User identifier scopes
+
+### Official scope matrix
+
+| Identifier | Feishu's documented scope | Cross-app/tenant behavior | Availability caveat |
+| --- | --- | --- | --- |
+| `open_id` | User identity inside one application | The same user has a different `open_id` in different apps | The normal app-relative user identifier |
+| `user_id` | User identity inside one tenant | Stable across all apps in the same tenant; different in tenant A and tenant B | Reading it is a sensitive field permission and is limited to custom apps in the cited user API |
+| `union_id` | User identity across apps from the same application provider/developer | Same across that provider's apps; different across different providers | It does not make unrelated customer-owned apps one identity domain |
+| `tenant_key` | Enterprise identity | One key corresponds to one enterprise | Returned by login and carried in events/callbacks |
+
+Sources:
+
+- Feishu's current user resource overview states that `user_id` is tenant
+  identity, `open_id` is app identity, and `union_id` is identity across apps
+  developed by the same application provider. It expressly says the same
+  user's Open ID differs between apps.
+  ([user resource overview](https://open.feishu.cn/document/server-docs/contact-v3/user/field-overview?lang=zh-CN))
+- Feishu's API parameter text is more explicit: the same user's User ID differs
+  between tenants but remains the same across apps within one tenant; the same
+  user's Union ID is the same under one developer and different under another.
+  ([official ID parameter definitions](https://open.feishu.cn/document/server-docs/contact-v3/department/search?lang=zh-CN))
+- Feishu describes Union ID as the identity linking one user across multiple
+  applications supplied by the same application service provider.
+  ([Union ID guide](https://open.feishu.cn/document/uAjLw4CM/ugTN1YjL4UTN24CO1UjN/trouble-shooting/how-to-obtain-union-id))
+- Feishu's common-error guidance says not to use an Open ID obtained from app A
+  in app B, and even a test app and its production app produce different Open
+  IDs for the same user. It also reports cross-tenant errors for both User ID
+  and Union ID use.
+  ([common errors](https://open.feishu.cn/document/ukTMukTMukTM/ugjM14COyUjL4ITN?op_tracking=hc))
+- The current login user-info API returns `open_id`, `union_id`, optional
+  sensitive `user_id`, and `tenant_key` from a `user_access_token`.
+  ([get login user information](https://open.feishu.cn/document/server-docs/authentication-management/login-state-management/get?lang=zh-CN))
+
+### Answer: can different customer Apps share one global `open_id` key?
+
+**No.** This directly contradicts Feishu's documented Open ID semantics. Two
+customer-owned custom apps are different applications, so the same person has
+different Open IDs in them. Because those apps are independently owned custom
+apps, the Hub also has no documented basis to assume they are applications of
+one application provider and therefore no basis to auto-merge on `union_id`.
+`user_id` cannot solve cross-customer identity either because it changes across
+tenants.
+
+Derived identity model:
+
+- The safe external login key is `(feishu_app_connection_id, open_id)`; retain
+  `tenant_key`, `user_id`, and `union_id` as scoped attributes, not as an
+  unqualified SaaS-global primary key.
+- If the product needs one Hub human account to participate in multiple
+  Organizations, represent each Feishu login as a separate external identity
+  attached through an explicit, separately verified account-linking flow. Do
+  not silently merge on Open ID, Union ID, email, phone, or display name.
+- Do not make `user_id` mandatory for login unless every customer app is
+  required to obtain its sensitive field permission. `open_id` and
+  `tenant_key` are returned by the user-info flow without using `user_id` as
+  the login key.
+- Replacing an Organization's Feishu App changes the application identity
+  domain and therefore its users' Open IDs. Treat app replacement as an
+  explicit connection migration/re-link operation, not an in-place secret
+  edit.
+
+## 3. OAuth login must be app-routed
+
+### Official facts
+
+- The authorization URL requires `client_id`, which is the selected
+  application's App ID. `redirect_uri` must already appear in that
+  application's security settings. On success Feishu redirects with `code`
+  and, when supplied, the original `state`.
+  ([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code))
+- Feishu says `state` maintains request/callback context and must be compared
+  to prevent CSRF. Its official sample rejects a callback whose state differs
+  from the stored state.
+  ([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code),
+  [user-access-token sample](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
+- The code lasts five minutes and can be used once. Token exchange requires
+  `client_id`, `client_secret`, `code`, and, when used during authorization,
+  the same `redirect_uri`. Feishu defines error 20024 for a code/refresh token
+  that does not match the supplied client ID and error 20071 for a mismatched
+  redirect URI.
+  ([obtain user access token](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
+- Only redirect URLs configured for the selected application pass Feishu's
+  security check; an app may configure multiple URLs.
+  ([access-token acquisition flow](https://open.feishu.cn/document/server-docs/api-call-guide/calling-process/get-access-token))
+
+### Derived request flow
+
+1. Begin at an Organization-qualified route or invitation, resolve exactly one
+   active `FeishuAppConnection`, and create a short-lived, one-time server-side
+   login transaction bound to that Organization, connection, redirect URI,
+   desired return path, and PKCE verifier if used.
+2. Put only a high-entropy opaque transaction nonce in OAuth `state`; generate
+   the authorization URL with that connection's `app_id` as `client_id`.
+3. On callback, atomically consume and validate `state` before selecting
+   credentials. The callback's untrusted query parameters must never be
+   allowed to choose another Organization or App.
+4. Exchange `code` with the connection's `app_id`, decrypted `app_secret`, and
+   the exact stored redirect URI. Then call user-info and require its
+   `tenant_key` to equal the connection's pinned tenant.
+5. Resolve or create the membership through
+   `(feishu_app_connection_id, open_id)`. Never search all Organizations for a
+   bare Open ID and accept the first match.
+
+A single physical callback URL may be configured independently in multiple
+apps, but a shared URL does not remove the need for state-bound app routing.
+
+## 4. HTTP event and callback routing
+
+### Official facts
+
+- A version 2 callback header contains the application's Verification Token,
+  `event_type`, `tenant_key` (the callback application's tenant), and `app_id`
+  (the callback application's App ID).
+  ([callback overview and structure](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/callback-overview))
+- Version 2 event examples likewise place `app_id` and `tenant_key` in the
+  event header.
+  ([user-leaves-chat event](https://open.feishu.cn/document/server-docs/group/chat-member/event/deleted-2))
+- Verification Token is an application validation identifier. Feishu says the
+  server can compare the event's token with that application's token to check
+  that a request is for the specified app. With an Encrypt Key, the request
+  signature is calculated from request timestamp, nonce, that app's Encrypt
+  Key, and the original request body.
+  ([receive and verify events](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/encrypt-key-encryption-configuration-case))
+- When an Encrypt Key is configured, the HTTP request body visible before
+  decryption is an envelope containing only `{"encrypt":"..."}`; `app_id`
+  and `tenant_key` are inside the ciphertext.
+  ([send events to developer server](https://open.feishu.cn/document/event-subscription-guide/event-subscriptions/event-subscription-configure-/choose-a-subscription-mode/send-notifications-to-developers-server?lang=zh-CN))
+
+### Derived routing and verification requirements
+
+- Give every Feishu connection a stable, unguessable callback route such as a
+  connection-specific path token. Resolve the connection from that route,
+  load exactly its Encrypt Key/Verification Token, verify and decrypt, then
+  require the authenticated payload's `app_id` and `tenant_key` to match the
+  route's connection.
+- For an unencrypted common endpoint, `header.app_id` may identify a candidate
+  connection, but it is untrusted until that connection's signature/token has
+  been verified. Never route the side effect solely from an unverified body.
+- A single encrypted URL shared by every customer app has no documented
+  pre-decryption app selector. Trying every tenant's secret is not an identity
+  protocol. Use connection-specific routing instead.
+- Namespace delivery deduplication by the authenticated connection (for
+  example `(connection_id, event_id)`), and dispatch business work only after
+  app/tenant verification. Unknown, disabled, or mismatched connections fail
+  closed and produce an auditable security log.
+
+## 5. Long-connection routing
+
+### Official facts
+
+- Feishu's SDK creates a WebSocket client with one app's required `APP_ID` and
+  `APP_SECRET`. Events subscribed by that application are then delivered over
+  that channel. Authentication is performed when the connection is
+  established.
+  ([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
+- Long-connection event subscription supports enterprise custom apps, not
+  store apps. Each app may establish at most 50 connections. If one app has
+  multiple clients, delivery is cluster-mode rather than broadcast: one random
+  client receives a message.
+  ([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
+- Long-connection callbacks use the same per-app App ID/App Secret client
+  construction and have the same cluster-mode single-recipient behavior.
+  ([long-connection callback subscription](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/step-1-choose-a-subscription-mode/configure-callback-request-address?lang=zh-CN))
+
+### Derived routing requirements
+
+- Create one logical client binding per enabled `FeishuAppConnection`, even if
+  an in-process supervisor manages many clients. Attach immutable
+  `organization_id` and `connection_id` context to each handler; the client
+  that received the frame is the pre-authenticated application route.
+- Never infer the receiving Organization by looking up a bare actor Open ID or
+  chat ID. First use the authenticated client binding; then cross-check any
+  delivered `app_id`/`tenant_key` fields available in the event structure.
+- Scaling several replicas for the same app does not create duplicate
+  broadcast processing according to Feishu's contract, but delivery still
+  requires application-level idempotency and health ownership. Capacity must
+  count clients per app against the documented 50-connection ceiling.
+
+## 6. Bot, chat, and message identifiers
+
+### Official facts
+
+- In the message resource, a sender can be a user or an application. For an
+  application sender, `id_type` is `app_id`; the sender also carries a
+  `tenant_key`. Thus the Feishu bot identity used by messaging is its
+  application identity, not a Hub-global bot ID.
+  ([message management overview](https://open.feishu.cn/document/server-docs/im-v1/message/intro))
+- `chat_id` identifies a conversation, including direct and group chats.
+  Feishu's group resource also exposes the chat's `tenant_key`.
+  ([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology),
+  [group management overview](https://open.feishu.cn/document/server-docs/group/chat/intro?lang=zh-CN))
+- Chat APIs require the operator to be in the same tenant as an internal chat;
+  common errors include `Operator and chat can NOT be in different tenants`.
+  Many operations additionally require the calling app's bot to be in the
+  group and reject an app that is unavailable in that tenant.
+  ([get chat members](https://open.feishu.cn/document/server-docs/group/chat-member/get?lang=zh-CN))
+- The first bot direct-chat event calls `chat_id` the conversation ID between
+  **that robot and user** and includes `app_id`, `tenant_key`, and app-scoped
+  user Open IDs in the same event.
+  ([user and bot first conversation](https://open.feishu.cn/document/server-docs/group/chat-member/event/bot-events?lang=zh-CN))
+
+### What the official sources do not establish
+
+The cited documentation calls `chat_id` a conversation/group identifier but
+does not state that its value is globally unique across every Feishu tenant and
+every independently created app. It does establish tenant and bot-membership
+authorization boundaries. Under the repository rule that unspecified scope
+must not be assumed, raw global uniqueness would be an unsupported inference.
+
+Derived storage/call rules:
+
+- Store chats as `(feishu_app_connection_id, chat_id)` and retain observed
+  `tenant_key` and chat type. Do the same for other resource identifiers unless
+  Feishu explicitly contracts a wider scope.
+- Always call chat/message APIs with the token obtained from the chat's bound
+  connection. Never retry an authorization failure with another Organization's
+  token.
+- Treat bot direct chats as connection-specific. A `chat_id` for user X's
+  conversation with Organization A's bot is not evidence of a conversation
+  with Organization B's bot.
+- For group chats, two customer apps may both be members of a real Feishu
+  group, but that does not authorize the Hub to move configuration, Projects,
+  or memberships between Organizations. The authenticated connection remains
+  the tenant route and Hub Organization remains the authorization root.
+
+## Minimum invariants for implementation review
+
+1. `FeishuIdentity` uniqueness is `(connection_id, open_id)`, never bare
+   `open_id`.
+2. OAuth `state` resolves one server-side transaction bound to one Organization
+   and connection; code exchange uses only that connection's credentials and
+   stored redirect URI.
+3. Login accepts the returned identity only when `tenant_key` matches the
+   connection's pinned tenant.
+4. HTTP Webhooks choose a connection-specific verification/decryption context
+   before trusting payload routing fields; authenticated `app_id` and
+   `tenant_key` must match it.
+5. Every long-connection handler carries the local connection/Organization
+   binding of the App ID/App Secret client that received the frame.
+6. Bot, chat, event-deduplication, token-cache, and user identifiers are all
+   namespaced by the Feishu connection unless Feishu explicitly guarantees a
+   wider scope.
+7. A credential, app, or tenant mismatch is a hard failure with traceable
+   logging; there is no fallback to a default app or search across tenant
+   secrets.

.scratch/saas-production-readiness/assets/production-observability-recovery.md

@@ -0,0 +1,245 @@
+# Production observability and incident-recovery audit
+
+Date: 2026-07-10
+
+Scope: the `hub` HTTP service, Feishu WebSocket ingress and outbound calls, PostgreSQL/Prisma, agent execution, the `cph` subprocess, project workspaces, and the packaged deployment path. This is a release-gate audit, not a proposal to change product semantics. Where the Lean contract is open, this report asks for an operational decision rather than inventing one.
+
+## Verdict
+
+**No-go for unattended SaaS production deployment.** The service exposes a liveness-shaped endpoint, emits some structured component logs, and persists useful run/audit fields, but it cannot currently answer the minimum production questions:
+
+1. Can this instance accept a real request *now* (database reachable, Feishu listener authenticated, required runtime executable and workspace usable)?
+2. Which tenant, request, Feishu event, run, subprocess, and outbound message participated in one failed user action?
+3. How many requests/runs/messages are failing, stuck, delayed, retried, or duplicated?
+4. Which condition should page an operator, and which documented recovery action is safe?
+5. After a restart, deploy, rollback, or dependency outage, what evidence proves that work was neither lost nor silently acknowledged?
+
+Three fault injections demonstrate that the current `/api/healthz` and logging surface can be green while the service is already unable to serve its core paths:
+
+- After startup, cutting PostgreSQL sockets left two consecutive `/api/healthz` requests at HTTP 200.
+- An invalid Feishu app ID was rejected by the installed Feishu SDK while `/api/healthz` remained HTTP 200.
+- With PostgreSQL cut, `/api/me` returned HTTP 400 `bad_request`, exposed a raw Prisma invocation in the response, produced no request-scoped application error log, and only emitted two unscoped Prisma error lines.
+
+Together with the already-confirmed lifecycle and deployment failures, this means green process state, green health checks, and even a visible user error do not prove that the service or its durable workflows are healthy.
+
+## Evidence and positive signals already present
+
+The starting point is not empty. These are useful foundations to retain:
+
+- Fastify's built-in HTTP logging records request/response facts with an internal request ID through the server logger ([`hub/src/server.ts`](../../../hub/src/server.ts)).
+- The same Fastify logger is threaded into the Feishu trigger/runtime, while Prisma emits its configured error/warn events (and optional query events) ([`hub/src/server.ts`](../../../hub/src/server.ts), [`hub/src/db.ts`](../../../hub/src/db.ts)).
+- `AgentRun` persists status, timestamps, error/result fields, token/cost fields, and links to project/session; `ProjectAgentLock` persists a heartbeat; `AuditEntry` and `AgentMessage` provide potential durable trace anchors ([`hub/prisma/schema.prisma`](../../../hub/prisma/schema.prisma)).
+- Agent execution emits intermediate activity to the streaming Feishu card, and lock heartbeat activity is attempted during execution ([`hub/src/agent/runner.ts`](../../../hub/src/agent/runner.ts), [`hub/src/feishu/trigger.ts`](../../../hub/src/feishu/trigger.ts)).
+- Deployment is managed by systemd, so restart policy, exit status, and journal collection have an existing operational host surface ([`hub/deploy/cph-hub.service`](../../../hub/deploy/cph-hub.service)).
+- Database migrations are explicit Prisma artifacts rather than ad-hoc startup DDL ([`hub/prisma/migrations`](../../../hub/prisma/migrations)).
+- Existing tests cover many individual authorization, trigger, lock, queue, runner, and API behaviors. They are valuable regression evidence, although they are not yet a deploy/recovery acceptance suite.
+
+These signals are fragmented rather than joined into an operational contract. Several writes are best-effort or swallowed, identifiers are not propagated end-to-end, there is no metrics/alert surface, and the health endpoint does not test dependencies.
+
+## Failure-to-signal matrix
+
+| Surface | Failure or degraded condition | Current externally visible signal | What is missing / why the signal is unsafe | Minimum production signal and recovery evidence |
+| --- | --- | --- | --- | --- |
+| HTTP | PostgreSQL becomes unreachable after startup | Affected request fails; `/api/healthz` stays 200 | Health is process liveness only. `/api/me` misclassifies the dependency failure as client `400 bad_request`, leaks raw Prisma call text, and creates no request-scoped error record. An LB/operator cannot distinguish bad input from platform outage. | Separate liveness and readiness. Readiness performs a bounded DB probe and returns non-2xx on failure. Unexpected exceptions map to sanitized 5xx with a stable error code and `request_id`; structured log contains the same ID and normalized Prisma category. Alert on readiness failure and 5xx/error-code rate. |
+| HTTP | Handler latency, event-loop stall, exhausted connections, overload | Access log only after a response is completed | No in-flight count, latency histogram, timeout counter, saturation signal, or request correlation. A hanging request may emit no timely evidence. | Request count, duration, in-flight requests, timeout/cancellation count, response class, and event-loop/runtime/process saturation. Bound request duration and log start/end/error with one `request_id`. |
+| Feishu ingress | Invalid/revoked app credentials; WebSocket connect/auth failure | SDK rejects/logs asynchronously; process and `/api/healthz` remain green | Startup success is reported before listener readiness. No listener state, last-connected/last-event timestamp, reconnect counter, or readiness dependency. The installed SDK already exposes `onReady`, `onError`, `onReconnecting`, `onReconnected`, `getConnectionStatus`, and `close`, but the Hub does not retain or wire the `WSClient`. systemd can perpetually supervise a process that cannot receive work. | Listener state machine (`starting/connected/degraded/stopped`), last successful connection/event, reconnect/auth-failure counters, bounded startup readiness gate, and alert on auth failure or prolonged disconnect. Credential contents must never be logged. |
+| Feishu ingress | Duplicate delivery, handler failure after receipt, process restart with queued/batched work | Event receipt/dedup row and selected warning logs | The run-lifecycle audit proved receipt-before-success and in-memory batching/queue loss. There is no durable processing state/lease, retry age, dead-letter view, or alert for accepted-but-unstarted work. | Durable ingress states and timestamps, attempts, last normalized error, event/chat/project correlation, queue depth/oldest age, dead-letter count. Recovery drill proves restart resumes or deliberately redrives exactly the intended work. See [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
+| Feishu outbound | API timeout/ambiguous success, rate limit, invalid credentials, permanent card patch/send failure | Some helpers log a warning; several message helpers catch failure and return `null` | No idempotent delivery record, delivery state, attempt count, provider request/message ID, retry delay, terminal failure metric, or user-visible reconciliation path. Fault injection proved a run can be marked completed although nothing was delivered. | Durable outbox/delivery state; idempotency key; provider message/request ID; attempts, latency and normalized provider error; queue depth/oldest age/dead-letter signals; recovery command to retry or reconcile safely. |
+| PostgreSQL | Startup unavailable | Process startup rejects | This is a useful fail-fast signal, but it is not joined to a deploy gate with bounded logs and rollback criteria. | Installer/supervisor waits for explicit readiness, captures failure reason, and aborts/rolls back without replacing last-known-good service. |
+| PostgreSQL | Runtime outage, pool saturation, slow queries, migration incompatibility | Raw Prisma errors; request symptoms | No readiness failure, pool/saturation metric, query latency/timeout signal, migration compatibility assertion, or tenant/request/run correlation. Raw client internals can cross the HTTP boundary. | DB probe latency/result, Prisma error code/category counters, bounded query timing for critical operations, connection/pool saturation if available, schema migration identifier, sanitized API mapping, and DB outage/runbook drill. Prisma documents structured client error types/codes and configurable event-based logging; use those instead of returning raw exception strings ([Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference), [Prisma logging](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging)). |
+| Agent run | SDK/model error, credentials invalid, provider throttling, malformed stream | Run may become `FAILED`; the card may show an error and trigger finalization may warn | The installed SDK emits API retry, rate-limit, tool-progress, runtime-init/MCP-status, provider request ID, duration, and error-category messages, but `runner.ts` handles only stream/assistant/user/result messages and drops the other lifecycle signals. Provider/model/attempt/latency/normalized error are not consistently available as metrics or correlated logs. Some persistence and audit operations are best-effort. | Run start/finish transition event with `run_id`, `session_id`, tenant/project, provider/model, status, duration, turns, tokens/cost, normalized error and retryability; capture the SDK's bounded retry/rate-limit/tool/runtime signals; aggregate run outcome/latency/stuck gauges without high-cardinality IDs as metric labels. |
+| Agent run | No result/hang, missed heartbeats, process death, terminal DB write failure | Lock heartbeat may stop; restart sweep mutates some state | No wall-clock deadline/watchdog, stale-lock/stale-active-run metric, readiness impact, page, or documented reconciliation. Lifecycle fault injections proved active run without lock, terminal run with lock, and timeout misclassification are possible. | Deadline and durable lease semantics, gauges for active/waiting/stale runs and locks, oldest age, transition-failure counter, reconciliation command and drill. Startup recovery must be transactional and evidence-producing. See [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
+| `cph` subprocess | Missing binary, permission failure, timeout, non-zero exit, malformed output | In production, `cph_check`/`cph_build` grant the Claude SDK's Bash tool; tool results are forwarded only to the in-memory streaming card. The direct `agent/cph.ts` wrapper captures exit/stdout/stderr but is used only by its integration tests, not by the production runner. | There is no production-owned `cph` invocation boundary, startup/readiness validation of the exact deployed artifact, version/build identity, persisted operation result, duration metric, bounded/redacted stderr contract, or reliable distinction between spawn failure and domain diagnostics. | Readiness checks executable path/version and required filesystem permissions without mutating domain state. Introduce an observable production invocation boundary or capture the SDK Bash/tool lifecycle with operation, run/project correlation, duration, exit category and bounded redacted diagnostic; metrics count timeout/spawn/non-zero/protocol errors. Deployment evidence pins and smoke-tests the artifact. |
+| Workspace | Missing/unwritable root, wrong ownership, disk/inode exhaustion, symlink/root escape, partial agent writes | Error appears only when a command/file operation happens | No readiness check, free-space/inode monitoring, workspace mutation journal/commit marker, or recovery procedure. Security audit already proved containment failures; lifecycle audit proved a run can fail after workspace side effects. | Bounded non-destructive readiness check for configured root, owner/mode, read/write/rename/fsync as required, free space and inode signals, per-project mutation correlation, containment validation, backup/restore and partial-write reconciliation drill. See [tenant/auth security audit](./tenant-auth-security-audit.md) and [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
+| Deploy | Bad env/service user/artifact, failed migration/build, unhealthy new process | Shell/systemd exit in some cases; process may restart | Clean-host audit found missing required env/service identity/artifact and in-place, non-atomic release replacement. Health is liveness-only and there is no last-known-good rollback unit. | Immutable versioned release, preflight, migration compatibility gate, explicit readiness deadline, release/build/schema identity endpoint/log, automated rollback criteria, last-known-good artifact and tested rollback. See [clean-host deployment audit](./clean-host-deployment-audit.md). |
+| Deploy | SIGTERM/restart during active work | systemd stops/restarts the process | No graceful drain/stop readiness, bounded shutdown, durable queue resume, or post-restart reconciliation proof. In-memory queues are lost. | On termination: readiness false, stop ingress, drain or checkpoint durable work, enforce timeout, emit shutdown summary. Recovery drill sends work across restart and verifies a single durable outcome and delivery. systemd supports explicit notification/readiness/watchdog protocols, but the current unit does not implement one ([systemd service types](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html), [`sd_notify`](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html)). |
+
+## Logs and end-to-end traceability
+
+### Current gaps
+
+- Fastify generates an internal request ID for its access logs, but it is not returned to the client as a support handle or propagated into application errors, Prisma diagnostics, Feishu work, runs, or subprocess calls.
+- Feishu `event_id`, provider request/message IDs, `chat_id`, `project_id`, `organization_id`, `run_id`, `session_id`, and subprocess invocation are not propagated as one causally connected context.
+- The DB-cut experiment produced raw Prisma diagnostics without an application error record that identified route, request, response class, or tenant context.
+- Several critical failure paths catch and ignore persistence errors or only warn, including audit/message persistence and some card/lock finalization paths. This creates absence of evidence precisely when durable state is least trustworthy.
+- The production runner discards SDK `api_retry`, `rate_limit_event`, `tool_progress`, runtime initialization/MCP status, and related system messages even though the installed SDK exposes the fields needed to diagnose provider and tool degradation.
+- Exception strings can cross the API boundary. Besides leaking implementation details, this makes the client-visible error vocabulary unstable and impossible to alert on reliably.
+- Logs have no documented retention, rotation, ingestion, access-control, redaction, or tenant-support lookup contract. Journald availability alone is not a log strategy.
+
+### Minimum structured event envelope
+
+Every boundary and durable state transition should emit one structured event with a stable `event_name`, `timestamp`, `severity`, `service`, `environment`, `release`, and the applicable correlation fields:
+
+`request_id`, `feishu_event_id`, `organization_id`, `project_id`, `chat_id`, `actor_user_id`, `run_id`, `session_id`, `delivery_id`, `provider_request_id`, and `cph_invocation_id`.
+
+Raw prompt/message/file contents, credentials, cookies, authorization headers, provider tokens, full Prisma invocation text, and unrestricted stdout/stderr must not be logged. Tenant/project/user identifiers are operationally necessary but access-controlled personal/customer metadata; metric labels must not use high-cardinality IDs. OpenTelemetry's HTTP semantic conventions specify the stable `http.server.request.duration` instrument, an opt-in `http.server.active_requests` instrument, and low-cardinality route/error guidance ([HTTP metrics semantic conventions](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/)).
+
+At minimum, events are required for:
+
+- request accepted/completed/failed/canceled;
+- Feishu listener connected/disconnected/auth-failed/reconnecting;
+- ingress received/deduplicated/queued/claimed/started/retried/dead-lettered;
+- run and lock state transitions, transition-write failures, heartbeat stale detection, and recovery decisions;
+- provider/agent start/result/error with normalized category;
+- `cph` spawn/result/timeout/protocol failure;
+- workspace preflight and mutation/commit/reconciliation decisions;
+- outbound delivery queued/attempted/confirmed/retried/dead-lettered;
+- deploy, schema migration, readiness transition, drain, shutdown, startup recovery, and rollback.
+
+Errors must be allowed to fail the affected operation rather than disappear. If a non-critical telemetry sink is unavailable, that telemetry failure itself needs a bounded local signal/counter; durable business audit records are not optional telemetry and must follow their explicit transactional contract.
+
+## Health, readiness, and startup contract
+
+The service needs three distinct concepts:
+
+1. **Liveness**: the process/event loop is responsive. This endpoint must be cheap and must not depend on remote services. The present `/api/healthz` can serve this purpose if it is named/documented as liveness only.
+2. **Readiness**: the instance can accept the traffic assigned to it. A bounded probe must include at least PostgreSQL connectivity/schema compatibility, Feishu listener authenticated/connected state for ingress-serving instances, required runtime configuration, exact `cph` artifact availability/version, and workspace root access/capacity. A failed dependency makes readiness non-2xx without killing the process and losing diagnostics.
+3. **Startup/deploy acceptance**: migrations/preflight complete, listener reaches ready state within a deadline, one authenticated HTTP/database smoke path and one controlled runtime/workspace smoke path pass, and the deployed release/schema identity matches the intended version. Only then is traffic enabled and the previous release eligible for cleanup.
+
+Readiness responses should expose only component names, stable state/error codes, and timestamps—not secrets or raw exceptions. Full diagnostic context belongs in correlated restricted logs. The deployment audit must decide whether Feishu and HTTP run in one instance role; if roles are split later, readiness dependencies become role-specific rather than globally optional.
+
+## Minimum metrics and alerts
+
+Exact SLO values are product/operations decisions and are currently unspecified. The release nevertheless needs the following bounded-cardinality instruments and a reviewed initial threshold for each page/ticket alert.
+
+### Traffic and failures
+
+- HTTP request count, duration, active requests, status/error-code class, timeouts/cancellations.
+- Feishu listener connected state, reconnect/auth failures, time since last successful connection/event.
+- Ingress queue/receipt counts by durable state, processing attempts, oldest pending age, dead-letter count.
+- Outbound delivery attempts/results/latency, retry/dead-letter count, oldest pending age, provider rate-limit responses.
+- Prisma operation failures by normalized code/category and bounded critical-query duration; DB readiness result/latency.
+
+### Workflow integrity and capacity
+
+- Runs started and terminal outcomes, run duration, active/waiting/stale count and oldest age.
+- Locks held/stale/orphaned and transition/reconciliation failures.
+- Agent/provider failures by provider/model/error category; turns/tokens/cost as operational/business counters with tenant detail kept out of metric labels.
+- `cph` invocation count/duration/result category/timeouts.
+- Workspace free bytes/inodes, preflight failures, write/commit/reconciliation failures.
+- Process uptime/restarts, event-loop delay, memory, CPU, file descriptors, unhandled rejection/uncaught exception, graceful shutdown outcome.
+- Release/build/schema identity as an info metric or startup event, never as an unbounded label set.
+
+### Required release-time alerts
+
+Before production traffic, alerts and routing must exist for:
+
+- instance readiness false beyond the rollout/startup grace period;
+- Feishu auth failure or prolonged listener disconnect;
+- sustained server-side HTTP errors or request timeouts;
+- database readiness failure / critical Prisma failure spike;
+- oldest ingress/outbound item, active run, or lock exceeds its decided deadline;
+- dead-letter count non-zero or durable state-transition/reconciliation failure;
+- agent/provider or `cph` failure/timeout spike;
+- workspace disk/inodes below headroom threshold or workspace preflight fails;
+- restart loop, failed deployment, failed migration, or rollback;
+- telemetry pipeline blind (expected heartbeat/scrape/log ingestion absent).
+
+Each alert needs an owner, severity, user impact statement, dashboard/log query, first safe diagnostic command, mitigation, escalation criterion, and recovery verification. A page without a runbook is not a production control.
+
+## Incident recovery runbooks required before release
+
+At minimum, checked-in runbooks must cover these scenarios. Commands must default to read-only inspection; destructive/redrive/force-release actions require explicit confirmation, identify tenant/project/run scope, emit an audit record, and be idempotent or document why they are not.
+
+1. **PostgreSQL unavailable/degraded** — identify connection vs credential vs migration vs capacity failure; stop accepting work through readiness; restore service; reconcile active runs, locks, ingress receipts, and outbox; verify no raw errors escape HTTP.
+2. **Feishu listener disconnected/credentials revoked** — distinguish auth from transient network/reconnect; restore credentials safely; verify last event and controlled inbound round trip; inspect ingress gap and redrive rules.
+3. **Ingress backlog/dead letter** — list oldest/attempts/error by tenant-safe scope; decide retry vs reject; prove dedup/idempotency; verify one resulting run.
+4. **Stuck/orphaned run or lock** — inspect run/session/lock/provider/workspace/audit/outbox together; use lease/reconciliation operation rather than direct row deletion; prove terminal state and next queued work behavior.
+5. **Outbound delivery failure** — reconcile provider message ID and durable outbox; retry with idempotency key; confirm user-visible delivery without duplicating a card/message.
+6. **Agent/provider or `cph` outage** — identify provider credential/rate-limit/runtime/protocol failure; halt or queue new runs; test a canary; resume and reconcile affected work.
+7. **Workspace capacity/corruption/containment event** — quarantine affected project, snapshot evidence, restore/rollback documented unit, verify ownership/containment and repository state before resuming.
+8. **Failed migration/deployment** — preserve old release, collect preflight/migration/readiness diagnostics, follow compatibility-aware application/database rollback, and verify release/schema identity plus canaries.
+9. **Process/host restart during work** — drain when planned; after unplanned restart run durable reconciliation; verify receipts, runs, locks, workspace effects and deliveries converge exactly once at the business level.
+10. **Suspected tenant data exposure** — disable affected path/tenant access without deleting evidence, preserve correlated audit/log records, rotate exposed credentials, determine affected org/project/users, and follow an explicitly owned incident-notification decision process.
+
+## Minimum release evidence
+
+Production readiness is not proved by a green unit suite alone. A release candidate needs an archived, attributable evidence bundle containing all of the following:
+
+### Build and supply identity
+
+- immutable application and `cph` artifact digests, source commit, build timestamp/toolchain, dependency lockfiles and vulnerability policy result;
+- deployed configuration manifest listing required keys and instance role, with secret values redacted;
+- migration set/schema identity and a compatibility statement for both forward deploy and rollback;
+- systemd unit and installer/preflight version used on a clean host.
+
+### Automated gates
+
+- typecheck, complete unit/integration suite, migration-from-supported-version tests, and Lean `lake build` when `spec/` changes;
+- clean-host install test with the real packaged artifacts and declared service user/directories;
+- negative startup tests for missing/invalid env, DB unavailable, invalid Feishu credentials, missing/wrong `cph`, and unwritable/full workspace;
+- readiness tests proving each dependency failure independently makes readiness fail while liveness remains diagnostically available;
+- API error-mapping tests proving unexpected dependency exceptions return sanitized 5xx/stable codes and create a correlated error log;
+- lifecycle fault tests from the run-lifecycle audit, including process kill/restart, DB transition failures, duplicate ingress, ambiguous outbound completion, timeout, and concurrent trigger/session creation;
+- tenant isolation/security regression suite, symlink/root containment tests, and dependency audit policy gate.
+
+### Staging drills against production-shaped dependencies
+
+- authenticated Feishu inbound → durable receipt → one run → workspace effect → durable terminal state → one outbound confirmation, searchable end-to-end by a correlation ID;
+- PostgreSQL socket cut/restoration while serving: readiness changes, alert fires, requests return sanitized server errors, state reconciles, alert clears;
+- Feishu credential rejection/disconnect/reconnect: readiness/alert and controlled recovery are visible;
+- SIGTERM and forced process kill during queued, active, and outbound-pending work: restart reconciliation proves no silent loss and no unintended duplicate user effect;
+- `cph` timeout/non-zero/malformed response, provider throttle/error, workspace disk/permission failure, and outbound rate-limit/permanent failure drills;
+- failed deploy/readiness and rollback drill on a clean host, followed by migration/data compatibility and canary verification;
+- backup restore of representative database and workspace data, with measured recovery point/time and tenant isolation checks.
+
+### Operations acceptance
+
+- dashboards expose all release metrics above and show release/schema identity;
+- every required alert is injected once in staging, reaches the named owner, links to its checked-in runbook, and clears only after verified recovery;
+- log search can pivot from request/event to org/project/run/session/delivery without message content or secrets; retention and access controls are documented and tested;
+- on-call ownership, escalation, incident severity, maintenance/rollback authority, backup ownership, and recovery objectives are explicitly decided;
+- a signed release checklist records who reviewed each piece of evidence and where immutable test/drill outputs are stored.
+
+## Prioritized remediation / release gates
+
+### P0 — before any external production traffic
+
+1. Implement distinct bounded liveness, readiness, and startup/deploy acceptance contracts; make deployment traffic activation depend on readiness.
+2. Install centralized unexpected-error handling that returns sanitized stable 5xx responses and emits request-correlated structured errors. Never return raw Prisma/SDK/process exception text.
+3. Make Feishu connection/auth state observable and readiness-relevant; make listener startup rejection fail the startup gate.
+4. Implement the durable ingress, transactional run/lock/session, durable outbox, workspace commit/reconciliation, and durable audit/history work already opened by the lifecycle audit. Observability cannot compensate for acknowledged work that has no recoverable state.
+5. Add end-to-end correlation and stable structured transition events; remove silent catches from critical paths.
+6. Add the minimum metrics, alerts, dashboards, and checked-in recovery runbooks, then execute the fault/restart/rollback drills.
+7. Replace in-place deployment with the clean-host audit's immutable release/readiness/rollback contract and prove it on a clean host.
+8. Close the critical tenant/agent/workspace/Feishu authorization and dependency findings from the security audit before declaring readiness.
+
+### P1 — required for routine operations immediately after the P0 architecture is in place
+
+- Define initial SLOs/error budgets and tune alert thresholds using staging/load evidence.
+- Add capacity/load/soak tests for DB connections, event-loop/process resources, ingress/outbound backlog, agent concurrency, provider limits, workspace disk/inodes, and log/metric pipelines.
+- Exercise backup/restore, credential rotation, tenant-support trace lookup, and incident evidence preservation.
+- Define data retention/deletion/access-control policy for logs, metrics, traces, audits, messages, receipts, deliveries, runs, and workspaces.
+
+## Decision points that remain OPEN
+
+The code/spec does not currently authorize choosing these product/operations semantics implicitly:
+
+- readiness dependency roles (combined HTTP+Feishu process versus independently deployable roles);
+- maximum acceptable ingress, run, lock, and outbound ages; retry budgets; terminal/dead-letter semantics;
+- workspace mutation commit/rollback unit and what a user should observe after partial agent work;
+- audit durability/availability trade-off and exact event taxonomy;
+- SLOs, recovery point objective, recovery time objective, retention, regional/data-residency constraints, and incident-notification ownership;
+- whether deploy migration compatibility must support application rollback across one or more releases.
+
+These decisions should be recorded in the upstream contract/ADR layer before implementation when they affect domain-visible behavior. Operational mechanisms that only expose already-defined behavior can be implemented and tested without inventing domain semantics.
+
+## Source notes
+
+Primary local evidence:
+
+- [`hub/src/server.ts`](../../../hub/src/server.ts), [`hub/src/admin/errors.ts`](../../../hub/src/admin/errors.ts) — process startup, health/listener/restart behavior, and HTTP exception-to-response mapping.
+- [`hub/src/db.ts`](../../../hub/src/db.ts) — current Prisma event/log configuration.
+- [`hub/src/feishu/client.ts`](../../../hub/src/feishu/client.ts), [`hub/src/feishu/trigger.ts`](../../../hub/src/feishu/trigger.ts), [`hub/src/feishu/messageBatcher.ts`](../../../hub/src/feishu/messageBatcher.ts), [`hub/src/feishu/triggerQueue.ts`](../../../hub/src/feishu/triggerQueue.ts) — ingress/listener, batching, dedup, queue, run and outbound behavior.
+- [`hub/src/agent/runner.ts`](../../../hub/src/agent/runner.ts), [`hub/src/agent/roleTools.ts`](../../../hub/src/agent/roleTools.ts), [`hub/src/agent/cph.ts`](../../../hub/src/agent/cph.ts), [`hub/src/lock.ts`](../../../hub/src/lock.ts), [`hub/src/audit.ts`](../../../hub/src/audit.ts) — production SDK/Bash tool routing, the legacy direct cph wrapper, persistence, heartbeat/lock, and audit behavior.
+- [`hub/prisma/schema.prisma`](../../../hub/prisma/schema.prisma) — durable state and available correlation fields.
+- [`hub/deploy`](../../../hub/deploy) — installer/systemd/deploy behavior.
+- [Clean-host deployment audit](./clean-host-deployment-audit.md), [tenant/auth security audit](./tenant-auth-security-audit.md), and [run-lifecycle integrity audit](./run-lifecycle-integrity-audit.md) — prior deterministic audits and their recorded commands/results.
+
+Official references used only for interface semantics and implementation guidance:
+
+- [Prisma Client error reference](https://www.prisma.io/docs/orm/reference/error-reference) and [Prisma logging](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging).
+- [OpenTelemetry HTTP metric semantic conventions](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/).
+- [systemd service types/readiness](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html) and [`sd_notify`](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html).
+
+No secondary production-readiness checklist was treated as authority. Recommendations are derived from the local failure evidence and the primary interfaces the service actually uses.

.scratch/saas-production-readiness/assets/run-lifecycle-integrity-audit.md

@@ -0,0 +1,272 @@
+# Run lifecycle and data-integrity audit
+
+## Verdict
+
+The current Hub does **not** have a crash-safe or database-failure-safe run
+lifecycle. The normal happy path is covered by tests and the database correctly
+enforces one lock per project, but the surrounding workflow is a chain of
+independent effects: inbound receipt, in-memory batching/queueing, session/run
+creation, lock acquisition, workspace mutation, Feishu delivery, session cursor
+update, run terminal update, audit write, and lock release.
+
+There is no durable coordinator or reconciliation loop across those effects.
+Deterministic fault injection confirmed that the current code can permanently
+lose accepted work, leave a terminal run holding a lock, leave an `ACTIVE` run
+without a lock, split one logical session into duplicate active rows, declare a
+run complete without delivering its result, and create an audit entry referring
+to no run. A hung provider call also has no wall-clock deadline or usable lease,
+so one run can block a project until an operator restarts the service or edits
+the database.
+
+These are production blockers for the accepted single-host topology; they do
+not require HA to reproduce.
+
+## Contract baseline
+
+The audit used the following upstream decisions and intentionally did not
+invent semantics where the contract is open:
+
+- `Spec.System.Run`: `active` and `waitingForUser` are non-terminal;
+  `completed`, `failed`, `timedOut`, and `canceled` are terminal.
+- `Spec.System.Lock` and ADR-0002: each project has at most one lock, its owner
+  is an `AgentRun`, every lock holder is non-terminal, and the lock is released
+  when the run completes, fails, times out, or is canceled.
+- ADR-0002: many runs reuse the same long-lived `AgentSession`.
+- ADR-0017: the active session identity is project + provider + role + model,
+  and its provider cursor is persisted for the next run.
+- `Spec.System.Audit`: every `AuditEntry` is associated with a run. Audit event
+  content, retention, and query dimensions remain `OPEN`.
+- The contract does **not** decide whether filesystem changes made by a failed,
+  canceled, timed-out, or process-killed run should commit, remain as partial
+  work, or roll back. That choice is surfaced below rather than assumed.
+
+## Positive evidence
+
+The core happy path is real:
+
+- PostgreSQL uniqueness enforces at most one `ProjectAgentLock` per project and
+  at most one project lock per run.
+- `currentLockRunId` rejects a terminal run that still holds a lock instead of
+  silently treating it as healthy.
+- Normal trigger completion updates the provider cursor, writes a terminal run
+  state, and releases the lock. Interrupt uses an `AbortController` and maps to
+  `CANCELED` on the normal path.
+- A sequential Feishu redelivery with the same event ID is skipped, so the
+  ordinary duplicate case does not start a second run.
+- The targeted lifecycle baseline passed before fault injection:
+
+```text
+npm run check
+6 test files passed
+71 tests passed
+```
+
+Those protections are necessary, but they only cover operations in which every
+dependency succeeds in the expected order.
+
+## Executed fault-injection evidence
+
+A temporary Vitest harness exercised the real trigger, Prisma test database,
+message batcher, and runner seams. It asserted the production invariant rather
+than the current behavior. The harness was run repeatedly and produced the same
+result each time; the final run was:
+
+```text
+npm test -- --run test/run-lifecycle-audit.repro.test.ts
+
+Test Files  1 failed (1)
+Tests       8 failed (8)
+Duration    918ms
+```
+
+The throwaway harness was removed after diagnosis. Its eight minimal probes and
+observed values were:
+
+| Injected condition | Required invariant | Observed current result |
+| --- | --- | --- |
+| Authorization database error after Feishu receipt creation, followed by the same event redelivery | The failed event is retried | Authorizer calls remained `1`; redelivery was skipped |
+| Batched-run setup callback throws, then `flushAll` retries | The pending batch remains recoverable | Callback attempts remained `1`; batch had already been deleted |
+| Two active sessions with the same project/provider/role/model tuple | Database rejects the duplicate | Second insert succeeded |
+| Audit insert uses a nonexistent run ID | Database enforces `AuditEntry → AgentRun` | Insert succeeded with arbitrary `runId` |
+| Every Feishu message create/reply fails | Completion is not reported as delivered | Run became `COMPLETED`; delivered cards were `0` |
+| First lock release delete fails | Terminal run is eventually unlocked | Lock count remained `1` after run became `COMPLETED` |
+| Every terminal `AgentRun.update` fails | A non-terminal run retains/reacquires its lock | Run remained `ACTIVE`; lock count became `0` |
+| Claude SDK returns `error_max_turns` | Runner returns `length`, trigger stores `TIMED_OUT` | Runner returned `failed` |
+
+This is a tight reproduction at the actual call seams; none of the failures
+depends on timing or external network access.
+
+## Critical findings
+
+### 1. Accepted Feishu work can be permanently lost
+
+`trigger.ts` commits `FeishuEventReceipt` before message validation, binding
+lookup, authorization, file download, command execution, batching, or run
+creation. The receipt has no processing state, lease, attempt count, error, or
+completion marker. Any later exception leaves a permanent row, so Feishu's
+at-least-once redelivery is treated as already completed.
+
+The two in-process buffers compound that behavior:
+
+- `MessageBatcher.flushPendingBatch` deletes the batch before invoking its
+  callback. Timer-triggered callback failures are swallowed with
+  `.catch(() => undefined)`.
+- `TriggerQueue` and `batchContexts` are process-local maps. A restart loses all
+  queued prompts and pending debounce batches. Queue draining removes an item
+  before run setup is durable, so a setup error also loses it.
+
+There is no shutdown hook that flushes or rejects these buffers visibly. The
+service can therefore acknowledge and forget user work even when PostgreSQL is
+healthy again milliseconds later.
+
+### 2. Run state and lock ownership can contradict each other
+
+Run creation, lock acquisition, terminal update, and lock release are separate
+Prisma calls. Their failure handling is asymmetric:
+
+- a lock-acquisition race creates a session and an `AgentRun` first, then marks
+  the losing run failed;
+- terminal update failure is caught, a second best-effort failure update is
+  attempted, and `finally` releases the lock regardless;
+- lock release failure is only logged once, with no retry or reconciler.
+
+Fault injection reached both forbidden outcomes: `COMPLETED + lock` and
+`ACTIVE + no lock`. The first makes `currentLockRunId` throw the explicit
+ADR-0002 invariant error on all later triggers; the second permits another run
+to start while the database still claims the earlier run is live. In the
+injected terminal-write case the earlier execution had already returned, but
+the same unlocked/non-terminal shape created by an overlapping startup can
+coexist with a still-running process.
+
+Startup does not repair these states safely. `server.ts` first deletes **all**
+locks and then, in a separate statement, marks only `ACTIVE` runs failed.
+`WAITING_FOR_USER` is left non-terminal and unlocked, contrary to the contract.
+The two statements are not transactional, and running a second Hub process
+would destructively clear the live first process's locks and fail its runs.
+
+### 3. A provider hang can block a project indefinitely
+
+`maxTurns` is passed to the SDK, but there is no wall-clock run deadline. The
+SDK max-turn result is currently mapped to runner status `failed`, making the
+trigger's `length → TIMED_OUT` branch unreachable for the actual max-turn case.
+
+`ProjectAgentLock.expiresAt` is never populated or consumed. `heartbeatAt` is a
+best-effort update only after an assistant message, not a periodic heartbeat,
+and there is no stale-lock/run sweeper. A query that stalls before its first
+assistant message remains `ACTIVE` while holding the project lock forever.
+`WAITING_FOR_USER` is declared but never written; approval waits remain
+`ACTIVE` in an in-memory `ApprovalManager`.
+
+Systemd sends `SIGTERM`, but the application has no signal handler, listener
+close, in-flight drain, run cancellation, Prisma disconnect, or final recovery
+transaction. A subsequent successful restart happens to fail `ACTIVE` runs,
+but that is not a graceful lifecycle and does not cover all non-terminal
+states.
+
+### 4. Workspace effects have no defined commit or recovery boundary
+
+The Claude SDK's built-in Write/Bash tools mutate the project workspace while
+the run is still `ACTIVE`. A process kill, timeout, cancellation, database
+failure, or later delivery failure can therefore leave real filesystem changes
+behind while the database says the run failed or never completed.
+
+No runtime code writes `AgentFileChange`: the only capture callback is in the
+legacy AI-SDK `writeFileTool`, while the production runner uses Claude SDK
+built-ins and never injects that collector. The schema comment claiming these
+changes are captured for review is not true. The transcript comments are also
+stale: the production runner neither loads nor appends the JSONL transcript;
+continuity comes from the provider cursor.
+
+`AgentMessage` writes are best-effort and silently swallowed. The session
+cursor, message projection, workspace state, run terminal state, and audit
+history can consequently describe different versions of the same execution.
+Before implementing a mechanism, the product/contract must decide whether
+partial workspace changes are committed, quarantined, or rolled back.
+
+## High findings
+
+### 5. Session identity and cursor continuity are not concurrency-safe
+
+`startAgentRun` performs `findFirst` then `create` for the active
+project/provider/role/model session before acquiring the project lock. The
+schema has only a non-unique index for that tuple. The database probe inserted
+two active rows with the same identity.
+
+Concurrent triggers can therefore fork a logical conversation before one loses
+the project-lock race. Future `findFirst` selection is unspecified and may
+resume the wrong provider cursor. On the completion path, the session cursor is
+updated separately and before the run terminal update, so a partial database
+failure can advance the conversation while leaving the originating run active
+or failed.
+
+### 6. Outbound delivery is silent and not recoverable
+
+The Feishu send helpers catch message errors and return `null`; `sendText`
+returns `void`. `StreamingAgentCard.finish` treats a failed create/patch as a
+boolean and normal run completion proceeds. The fault probe recorded a
+`COMPLETED` run with zero delivered cards.
+
+There is no durable outbox, provider message ID record, delivery state,
+idempotency key, retry schedule, or reconciliation job. An operator cannot tell
+whether the user saw the result, and retrying a larger workflow later would
+have no durable key with which to prevent duplicate messages.
+
+### 7. Audit rows violate the pinned relation and audit failures are invisible
+
+`AuditEntry.runId` is nullable, has no foreign key, and accepts arbitrary
+strings. This directly diverges from the pinned `Spec.System.Audit` relation
+that every audit entry belongs to a run. Pre-run permission/security events are
+valid product needs, but they require an explicit event model rather than
+weakening the one relation the contract fixes.
+
+`writeAudit` catches every Prisma error without logging, metric, retry, or
+caller signal, and existing unit tests encode that swallowing as desired.
+Lifecycle history can be absent exactly when PostgreSQL is unhealthy, with no
+evidence that evidence was lost. The content and failure policy are `OPEN` and
+need a decision, but silent unobservable loss is incompatible with operating
+and debugging the service.
+
+## Failure-mode summary
+
+- **Process termination/restart:** loses batches, queued prompts, approvals,
+  active controller handles, and unrecorded message/file history; may leave
+  partial workspace changes; recovery ignores `WAITING_FOR_USER`.
+- **Timeout/hang:** no wall deadline, periodic heartbeat, lease, or sweeper;
+  max-turn termination is mislabeled `FAILED` rather than `TIMED_OUT`.
+- **Duplicate Feishu delivery:** sequential success is deduplicated, but any
+  failure after receipt becomes permanent loss; receipt check/create is also
+  not one transactional claim operation.
+- **Database failure:** can produce either terminal-with-lock or
+  active-without-lock, lose session/message/audit facts, or advance only the
+  provider cursor.
+- **Concurrent requests/processes:** the project-lock unique key prevents two
+  normal lock rows, but active sessions can fork; a second process's startup
+  reset can invalidate the first process's live lifecycle.
+- **Outbound failure:** run completion and user-visible delivery are unrelated,
+  and no durable recovery/idempotency record exists.
+
+## Required release evidence
+
+Production readiness requires automated proof of all of the following:
+
+1. Each accepted inbound event has a durable state machine and can be retried
+   after process/DB failure without duplicate run creation.
+2. Queueing and batching either survive restart or have an explicit, observable
+   rejection/acknowledgement contract that cannot silently lose work.
+3. Every database fault point preserves `lock exists ⇒ holder non-terminal`
+   and prevents a live non-terminal run from continuing without its lock.
+4. A wall deadline, lease/heartbeat, watchdog, and startup reconciler converge
+   every non-terminal state, including human approval waits.
+5. One active session identity is enforced in PostgreSQL and cursor advancement
+   is serialized with the run transition that produced it.
+6. Feishu delivery uses a durable, idempotent outbox/reconciliation path and
+   exposes delivered, pending, and failed delivery states separately from model
+   completion.
+7. The decided failed-run workspace policy is implemented and tested across
+   kill, cancel, timeout, and database failure; file/message history reflects
+   the real effects or explicitly reports missing evidence.
+8. The audit/event model conforms to the pinned run relation, records pre-run
+   security events in an explicitly modeled place, and makes write loss
+   observable and recoverable according to the accepted policy.
+9. Graceful shutdown and two-process/startup-race tests prove that one process
+   cannot clear another live process's locks or accept new work while draining.

.scratch/saas-production-readiness/assets/tenant-auth-security-audit.md

@@ -0,0 +1,262 @@
+# Tenant, authentication, and request security audit
+
+## Verdict
+
+The current Hub has a sound application-level foundation for ordinary org-admin
+CRUD isolation, but it is **not safe to deploy as a multi-tenant SaaS**. Several
+agent and asynchronous-action paths can cross the Organization/Project boundary,
+and the current process-global credential design directly contradicts the
+org-scoped encrypted-secret decision in ADR-0021.
+
+The answer to the audit question is therefore **yes**: current filesystem,
+Feishu-context, deferred authorization, card-action, and credential paths can
+cross or bypass the intended boundary. These are implementation divergences,
+not unspecified product behavior.
+
+## Contract baseline
+
+The audit used these upstream requirements:
+
+- `Spec.System.Organization`: PROJECT and TEAM each have one Organization;
+  TEAM→PROJECT grants are same-org only.
+- `Spec.System.ProjectWorkspace`: folders are transparent; folder/project
+  placement must remain inside one Organization.
+- `Spec.System.Permission`: agent trigger requires EDIT, normal cancel requires
+  MANAGE, and force release is outside the project role lattice.
+- `Spec.System.Memory`: a run may read Feishu context only from its project's
+  bound chat.
+- `Spec.System.AgentSurface`: every agent file operation must stay inside that
+  run's project workspace.
+- ADR-0019: authorization is evaluated against principals resolved at request
+  time so membership changes take effect immediately.
+- ADR-0021: Feishu and model-provider credentials are org-scoped secrets,
+  encrypted at rest, and accessed through a resolver. Provider ownership may be
+  Organization-managed BYOK or a distinct platform-managed connection for that
+  Organization; neither mode permits a process-global shared key.
+
+## Confirmed protections
+
+The following paths are correctly fail-closed today:
+
+- Every `/api/org/:orgSlug/*` route calls `requireOrgRole`; the guard reloads
+  the user, active membership, allowed OWNER/ADMIN role, and ACTIVE org for each
+  request.
+- Project/session/folder/team service functions compare the requested object's
+  Organization before returning or mutating it. Cross-org lookups return not
+  found instead of revealing the foreign object.
+- `PermissionAuthorizer` derives the Organization from the resource before
+  expanding team and external principals. `grantTeamProjectAccess` refuses a
+  TEAM whose Organization differs from the Project and detects corrupt active
+  grants when listing.
+- Feishu messages with no sender `open_id` are denied. Bound-chat triggers run
+  through `agent.trigger` and `role.trigger` gates before creating an AgentRun.
+- OAuth state is signed, expiry-checked, and paired with an HttpOnly nonce
+  cookie. `returnTo` is constrained to same-origin `/admin` paths. Session
+  signatures use HMAC-SHA256 with constant-time comparison, and session users
+  are reloaded on every request.
+- Fastify's current default body limit is 1 MiB, so HTTP bodies are not
+  completely unbounded even though field-level and rate limits are absent.
+
+Executed evidence:
+
+```text
+5 integration files passed
+28 tests passed
+```
+
+Those tests include unauthenticated access, member-vs-admin roles, cross-org
+project IDs, Organization-scoped team slugs, and cross-org TEAM grant refusal.
+
+## Critical findings
+
+### 1. The agent receives service secrets and can read other tenant data
+
+`hub/src/agent/runner.ts` builds the SDK environment as:
+
+```text
+{ ...process.env, ...providerEnv }
+```
+
+The sandboxed agent and its Bash subprocess therefore inherit `DATABASE_URL`,
+`FEISHU_APP_SECRET`, `HUB_SESSION_SECRET`, the model-provider token, and every
+other service variable. A model or prompt injection can run `env`; network
+egress is intentionally open, so exposed values can leave the host.
+
+The same sandbox config has only `allowWrite: [workspaceDir]`. Reads outside
+the workspace are allowed except for a small deny list. All organizations run
+as the same service user, and sibling project workspaces are not denied. This
+violates `AgentFileOp.Authorized`, which covers reads as well as writes.
+
+The installed Claude Agent SDK already exposes two mechanisms that are not
+used: sandbox credential rules can `deny` or proxy-`mask` environment variables,
+and filesystem rules can deny a workspace root while re-allowing the current
+workspace. Production still needs an actual Linux test proving that SDK/Bash
+cannot read another project or service credential.
+
+### 2. MCP file and Feishu-context tools escape their run objects
+
+`resolveDeliverableFile` intentionally adds the repository Git root to allowed
+roots, even when it is above the project workspace. It also checks lexical paths
+with `stat`, which follows symlinks. A deterministic reproduction created
+`workspace/leak.txt` as a symlink to another tenant's file:
+
+```json
+{"symlinkEscapeAccepted":true,"outsideContentReadable":true}
+```
+
+The checked-in unit test currently treats sending a repo-root `README.md` from
+a nested project as desired behavior. That expectation is a direct divergence
+from the AgentSurface contract and must be reversed.
+
+`readFeishuContext` ignores its `ToolContext`. The MCP wrapper closes over the
+bound `chatId`, but the model supplies an arbitrary message/thread ID and the
+reader never verifies the returned message's `chat_id`. A stub Feishu API
+returning a message from `chat-other` produced:
+
+```json
+{"returnedChatId":"chat-other","crossChatContentReturned":true}
+```
+
+The same validation is required for thread parents and every returned item.
+
+Inbound downloads have the inverse symlink problem: the unsandboxed Hub writes
+to `workspace/.cph/inbox`. A prior agent command can replace a path component
+with a symlink, turning the Hub into a confused deputy unless writes use a
+verified real path/no-follow boundary.
+
+### 3. SaaS credentials are process-global instead of org-scoped
+
+`server.ts` creates one process-global Feishu client/listener and passes one
+global app ID/secret to OAuth. `feishuOAuth.ts` explicitly calls per-org
+encrypted credentials "deferred", although ADR-0021 says they **must** be
+org-scoped, encrypted at rest, and accessed through a resolver.
+
+`EnvRuntimeSettings.provider` similarly discards its `scope` argument and uses
+one global OpenRouter token for every project. This erases the accepted
+per-Organization provider mode and makes both BYOK and platform-managed cost
+attribution false. The schema has no encrypted credential store, key version,
+rotation state, or resolver seam.
+
+This is also a product reachability blocker: one customer-owned Feishu app
+cannot receive bot/OAuth traffic for unrelated customer tenants under the
+ADR-0021 model.
+
+## High findings
+
+### 4. Delayed and card actions are not bound to the authorized object
+
+- `onMessage` checks `agent.trigger` before enqueueing/batching, but
+  `startAgentRun` checks only `role.trigger`. If a grant, team membership, or
+  tenant status changes while a request waits, it still starts. This contradicts
+  ADR-0019's request-time effectiveness rule.
+- Interrupt handling authorizes `agent.cancel` against the project bound to the
+  card action's current chat, then calls `activeRuns.get(runId)` without proving
+  that `runId` belongs to that project. A stale, forwarded, or otherwise
+  mismatched action can abort another project's active controller.
+- Approval handling stores the expected `chatId` but never compares it to the
+  action context before resolving the pending approval.
+
+Execution-time authorization must resolve one immutable tuple
+`(organization, project, run/action, actor, chat)` and reject any mismatch
+before mutation.
+
+### 5. SUSPENDED organizations can still execute bound projects
+
+HTTP org-admin guards and unbound-chat onboarding reject non-ACTIVE orgs, but
+`PermissionAuthorizer.organizationForResource` returns only `organizationId`
+and never checks status. A real database reproduction created a SUSPENDED org,
+project, and EDIT grant:
+
+```json
+{"organizationStatus":"SUSPENDED","agentTriggerAllowed":true}
+```
+
+Project onboarding helpers also check membership existence without checking
+Organization status. The same status must be enforced at the shared resource
+authorization/onboarding seam, not independently in selected transports.
+
+### 6. Production dependency graphs contain known vulnerabilities
+
+`npm audit --omit=dev` exits 1 with two high findings:
+
+- `@larksuiteoapi/node-sdk@1.68.0`
+- transitive `axios@1.13.6`
+
+The current latest Feishu SDK (`1.70.0` at audit time) still pins Axios
+`~1.13.3`, while the audited fixed Axios line is newer. The relevant advisories
+include request/credential manipulation, proxy credential leakage, ReDoS, and
+unbounded resource allocation; for example
+[GHSA-3g43-6gmg-66jw](https://github.com/advisories/GHSA-3g43-6gmg-66jw),
+[GHSA-hfxv-24rg-xrqf](https://github.com/advisories/GHSA-hfxv-24rg-xrqf), and
+[GHSA-777c-7fjr-54vf](https://github.com/advisories/GHSA-777c-7fjr-54vf).
+An override is outside the SDK's declared semver range and therefore requires
+integration testing rather than a blind lockfile edit.
+
+`cargo audit` exits 1 with five vulnerability records:
+
+- `crossbeam-epoch@0.9.18`, patched in `>=0.9.20`;
+- `quick-xml@0.38.4` and `0.39.4`, each affected by namespace-allocation and
+  quadratic-attribute DoS advisories, patched in `>=0.41.0`.
+
+It also reports `memmap2@0.9.10` as unsound (patched in `>=0.9.11`) and four
+unmaintained transitive crates. All enter through the Typst 0.15 dependency
+graph. References:
+[RUSTSEC-2026-0204](https://rustsec.org/advisories/RUSTSEC-2026-0204.html),
+[RUSTSEC-2026-0194](https://rustsec.org/advisories/RUSTSEC-2026-0194.html),
+[RUSTSEC-2026-0195](https://rustsec.org/advisories/RUSTSEC-2026-0195.html), and
+[RUSTSEC-2026-0186](https://rustsec.org/advisories/RUSTSEC-2026-0186.html).
+
+No `npm audit` or `cargo audit` gate exists in CI.
+
+## Browser/session and availability gaps
+
+These do not currently prove a cross-org data read, but they are required before
+exposing the service publicly:
+
+- state-changing cookie-auth routes have no CSRF token or Origin/Sec-Fetch-Site
+  enforcement; SameSite=Lax is the only browser-side barrier;
+- there is no session revocation/version store, so a stolen cookie remains valid
+  for up to seven days unless the global secret rotates;
+- startup accepts any non-empty session secret rather than enforcing generated
+  entropy, and there is no key rotation scheme;
+- no security-header policy is registered, and Fastify request timeout is zero;
+- OAuth, login, and org APIs have no per-IP/user/tenant rate limit;
+- Feishu file/post download count, response size, MIME, archive shape, and total
+  workspace quota are unbounded; uploads read entire files into memory.
+
+The exact abuse quotas and file limits remain with
+`Define initial abuse and capacity controls`; they must not be guessed here.
+
+## Secret scan
+
+All 377 tracked files were scanned for private-key blocks, common cloud/API-key
+shapes, and non-empty assignments to the Hub's principal secret variables. No
+private key or real credential was found. The only assignment hits were the
+documented `<OpenRouter API key>` placeholder and local/test PostgreSQL URLs.
+`hub/.env` is ignored and untracked.
+
+This is positive repository hygiene, but it does not mitigate the runtime
+environment leak into the agent process.
+
+## Required security release evidence
+
+Production readiness requires automated proof of all of the following:
+
+1. An agent/Bash process cannot read service credentials, another project
+   workspace, repo-root files, or a symlink target outside its workspace.
+2. Feishu context and file delivery validate the real target after resolution,
+   not only a caller-supplied lexical ID/path.
+3. Every delayed action reauthorizes at execution and binds actor, chat,
+   organization, project, run, and pending action consistently.
+4. ACTIVE Organization status is enforced in the shared authorization seam.
+5. Feishu and provider credentials resolve by Organization, are encrypted at
+   rest with versioned keys, never enter logs/agent tools, and can rotate;
+   provider resolution honors either Organization-managed BYOK or a distinct
+   platform-managed connection for that Organization.
+6. Cross-org HTTP and Prisma service tests remain green, while new negative
+   tests cover every finding above.
+7. Production Node and Rust dependency audits pass with no ignored
+   vulnerability/unsound advisory unless a time-bounded, reviewed exception is
+   explicitly recorded.
+8. Browser/session, request, and file-ingestion limits are verified against the
+   accepted production threat and capacity policy.

.scratch/saas-production-readiness/issues/01-audit-clean-host-deployment.md

@@ -0,0 +1,33 @@
+# Audit the clean-host deployment and rollback contract
+
+Type: research
+Status: resolved
+
+## Question
+
+What concrete defects prevent a clean, supported Linux host from installing,
+starting, health-checking, upgrading, and rolling back the current Hub without
+manual repository knowledge or unsafe partial deployment?
+
+## Answer
+
+The current path is a happy-path updater for a manually prepared host, not a
+production deployment mechanism. The executed audit found an unrunnable seeded
+configuration, an unprovisioned service identity and storage, a missing `cph`
+artifact, in-place mutation before build success, liveness-only verification,
+no coordinated application/schema rollback, and a configured bind host that the
+server ignores.
+
+The complete evidence, operational consequences, required release invariants,
+and verification scenarios are recorded in the
+[clean-host deployment and rollback audit](../assets/clean-host-deployment-audit.md).
+
+The accepted single-host topology remains suitable. The route forward is now
+explicit in these implementation tickets:
+
+- [Provision a runnable non-root Hub service](10-provision-nonroot-service.md)
+- [Package and verify cph with every Hub release](11-package-cph-with-release.md)
+- [Make application releases atomic and rollbackable](12-atomic-releases-and-rollback.md)
+- [Add real readiness and an accurate service lifecycle](13-real-readiness-and-lifecycle.md)
+- [Establish migration compatibility and failed-migration recovery](14-migration-compatibility-and-recovery.md)
+- [Test clean-host deployment and rollback on Linux](15-test-clean-host-deployments-on-linux.md)

.scratch/saas-production-readiness/issues/02-audit-tenant-auth-security.md

@@ -0,0 +1,40 @@
+# Audit tenant, authentication, and request security boundaries
+
+Type: research
+Status: resolved
+
+## Question
+
+Can any current HTTP, Feishu, filesystem, agent, or database path cross an
+Organization boundary, bypass the intended platform/org/project authorization
+seams, expose secrets, or accept unbounded hostile input in the initial
+production topology?
+
+## Answer
+
+Yes. Basic org-admin HTTP/Prisma scoping is correctly enforced and its 28
+targeted integration tests pass, but the agent inherits service secrets and can
+read sibling files, MCP file/context tools accept real targets outside the run
+objects, queued/card actions do not consistently bind or reauthorize their
+target, non-ACTIVE tenants can still trigger agents, and Feishu/provider
+credentials remain global instead of ADR-0021 org-scoped encrypted secrets.
+Current Node and Rust dependency audits also fail on high/vulnerability/unsound
+advisories.
+
+The complete contract mapping, reproductions, positive evidence, advisory
+inventory, and required release proofs are in the
+[tenant, authentication, and request security audit](../assets/tenant-auth-security-audit.md).
+
+The newly-clear implementation and decision frontier is:
+
+- [Confine the agent runtime and protect service credentials](16-confine-agent-and-protect-credentials.md)
+- [Close MCP context and file-delivery escape paths](17-close-mcp-data-egress-escapes.md)
+- [Bind and reauthorize deferred and card actions](18-bind-and-reauthorize-deferred-actions.md)
+- [Design the org-scoped secret and connection control plane](19-design-org-scoped-secret-control-plane.md)
+- [Design the production browser and session boundary](20-design-production-web-session-boundary.md)
+- [Clear dependency advisories and gate releases](21-clear-dependency-security-advisories.md)
+- [Enforce Organization status at the shared authorization seam](22-enforce-organization-status.md)
+
+Inbound Feishu file count/size/MIME and request-rate policy remain owned by
+[Define initial abuse and capacity controls](05-define-abuse-capacity-controls.md)
+rather than being guessed in this audit.

.scratch/saas-production-readiness/issues/03-audit-run-lifecycle-integrity.md

@@ -0,0 +1,38 @@
+# Audit run lifecycle and data integrity under failure
+
+Type: research
+Status: resolved
+
+## Question
+
+Under process termination, restart, timeout, duplicate Feishu delivery,
+database failure, or concurrent requests, which AgentRun, lock, session,
+workspace, audit, and outbound-message invariants can be violated or leave work
+lost, duplicated, or permanently stuck?
+
+## Answer
+
+The happy path and database lock uniqueness work, but the lifecycle is not
+crash-safe. Deterministic fault injection confirmed permanent inbound loss,
+terminal runs retaining locks, active runs losing locks, duplicate active
+sessions, completion without outbound delivery, unconstrained audit run IDs,
+and max-turn termination being stored as failure. Process-local queues,
+non-transactional startup recovery, unused leases, silent audit/history writes,
+and direct workspace mutation add restart and indefinite-hang failure modes.
+
+The complete contract mapping, eight fault probes, recovery analysis, and
+required release evidence are in the
+[run lifecycle and data-integrity audit](../assets/run-lifecycle-integrity-audit.md).
+
+The newly-clear implementation and decision frontier is:
+
+- [Make Feishu intake and queued work durable](23-durable-feishu-inbox-and-queue.md)
+- [Make run, session, lock, and recovery transitions safe](24-transactional-run-lock-recovery.md)
+- [Make outbound Feishu delivery durable and idempotent](25-durable-outbound-delivery.md)
+- [Decide workspace commit semantics for failed runs](26-decide-workspace-run-commit-semantics.md)
+- [Reconcile the audit and run-history contract](27-reconcile-audit-history-contract.md)
+
+Execution-time authorization remains owned by
+[Bind and reauthorize deferred and card actions](18-bind-and-reauthorize-deferred-actions.md),
+while cross-component metrics, alerts, and operator runbooks remain owned by
+[Define production observability and incident recovery](04-define-observability-recovery.md).

.scratch/saas-production-readiness/issues/04-define-observability-recovery.md

@@ -0,0 +1,47 @@
+# Define production observability and incident recovery
+
+Type: research
+Status: resolved
+Blocked by: 01, 02, 03
+
+## Question
+
+Which logs, metrics, readiness signals, audit correlations, alerts, and operator
+recovery actions are missing for a production operator to detect and diagnose
+failures across HTTP, Feishu, PostgreSQL, agent execution, cph builds, and
+workspace storage?
+
+## Answer
+
+The Hub has useful fragments—Fastify access logs, Prisma error/warn output,
+durable AgentRun facts, selected Feishu warnings, explicit migrations, and a
+systemd journal—but no joined operational contract. It has no metrics or alert
+surface, no end-to-end request/event/run/delivery/release correlation, no
+checked-in recovery runbooks, and many critical writes or sends are silently
+best-effort.
+
+Three process-level fault probes confirmed the release gate is actively
+misleading: cutting every PostgreSQL socket left `/api/healthz` at 200 twice;
+an SDK-rejected Feishu app ID also left health at 200; and a runtime database
+outage made `/api/me` return a raw Prisma invocation as `400 bad_request` with
+no request-scoped exception log. The current deploy check can therefore accept
+an instance unable to serve either database or Feishu work, while dependency
+failures evade normal 5xx alerting.
+
+The full failure-to-signal matrix, liveness/readiness/startup contract,
+correlation envelope, metric and alert inventory, required runbooks, executed
+probe evidence, primary sources, and minimum release evidence are in the
+[production observability and incident-recovery audit](../assets/production-observability-recovery.md).
+
+The newly-clear implementation and decision frontier is:
+
+- [Make HTTP dependency failures typed and observable](28-typed-observable-http-errors.md)
+- [Decide the telemetry, SLO, retention, and alerting control plane](29-decide-telemetry-slos-alert-routing.md)
+- [Instrument correlated production runtime telemetry](30-instrument-correlated-runtime-telemetry.md)
+- [Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md)
+
+[Add real readiness and an accurate service lifecycle](13-real-readiness-and-lifecycle.md)
+owns component probes, Feishu connection state and deploy gating. Durable
+ingress/run/outbound recovery stays with tickets 23–25, audit semantics with
+ticket 27, and backup/restore guarantees with ticket 06; this audit does not
+duplicate those implementations.

.scratch/saas-production-readiness/issues/05-define-abuse-capacity-controls.md

@@ -0,0 +1,43 @@
+# Define initial abuse and capacity controls
+
+Type: grilling
+Status: resolved
+Blocked by: 02, 03
+
+## Question
+
+Which request limits, file limits, concurrency limits, agent budgets, tenant
+quotas, and backpressure policies are mandatory for the accepted initial
+production boundary, and which values or behaviors remain product decisions?
+
+## Answer
+
+The initial production service requires layered, fail-closed capacity controls:
+every mandatory dimension has a versioned non-overridable platform ceiling,
+while an Organization may configure only a lower policy limit. Agent work uses
+a durable bounded queue, fair scheduling across Organizations, FIFO within an
+Organization, explicit overload/expiry/cancellation states, and no automatic
+whole-run replay after execution starts.
+
+Hard controls cover request rate/body size, concurrency and queueing, files and
+archive expansion, Project/Organization storage and entity counts, run/tool
+budgets, and process resources. Reaching a limit blocks new growth without
+deleting or hiding existing data. Token and provider cost remain attributed
+soft-alert signals in the pilot, not hard spend controls; unknown cost is not
+zero. Platform administrators also receive audited `DRAIN` and `STOP_NOW`
+workload brakes.
+
+The full current-state inventory, accepted behavior, and release evidence are
+recorded in [Initial abuse and capacity controls](../assets/initial-abuse-capacity-controls.md),
+with the durable decision in ADR-0022 and `Spec.System.Capacity`. Numerical
+ceilings remain open until production-like calibration.
+
+The implementation frontier is:
+
+- [Implement layered limits and multi-dimensional request rate limiting](32-layered-limits-rate-limiting.md)
+- [Build the durable Organization-fair admission scheduler](33-durable-fair-admission-scheduler.md)
+- [Enforce file, archive, storage, and entity budgets](34-file-storage-budgets.md)
+- [Enforce Agent Run and process resource budgets](35-run-runtime-budgets.md)
+- [Implement provider-mode usage attribution and soft alerts](36-usage-soft-alerts.md)
+- [Add audited emergency workload brakes](37-emergency-workload-brakes.md)
+- [Calibrate and prove production capacity ceilings](38-calibrate-production-capacity.md)

.scratch/saas-production-readiness/issues/06-audit-backup-migration-recovery.md

@@ -0,0 +1,48 @@
+# Audit backup, migration, and disaster recovery safety
+
+Type: research
+Status: resolved
+
+## Question
+
+What backup, restore, workspace/database consistency, migration compatibility,
+and disaster-recovery guarantees are needed to deploy current persistent data
+safely, and what evidence or tooling is missing to prove them?
+
+## Answer
+
+The Hub has no production backup/restore control plane. PostgreSQL can be
+logically dumped and restored, but authoritative state is split across the
+database, mutable Project workspaces, credentials/key material, and a
+schema-compatible release with no shared checkpoint or recovery manifest.
+RPO/RTO, retention, off-host/immutable storage, encryption/key custody,
+full-service versus Organization-selective restore, monitoring, and recovery
+drills are all undefined.
+
+Executed probes proved that the current deploy `rsync --delete` removes the
+documented in-tree workspace root, a failed Project transaction leaves an
+orphan workspace, and an older application query fails after a destructive
+migration. A deliberately failed Prisma migration rolled back that probe's DDL
+but left a blocking failed migration record; the official contract does not
+make PostgreSQL migrations transactional by default. A positive custom-format
+round trip restored 24 tables and all nine current migration records, proving
+only database-level portability on the local PostgreSQL 14.20 setup.
+
+The complete state map, probe commands/results, root causes, required recovery
+contract, and release evidence are in the
+[backup, migration, and disaster-recovery audit](../assets/backup-migration-disaster-recovery-audit.md).
+Official PostgreSQL, Prisma, systemd, and rsync semantics with claim-level links
+are in [backup and migration primary sources](../assets/backup-migration-primary-sources.md).
+
+The newly-clear decision and implementation frontier is:
+
+- [Decide recovery objectives, backup retention, and restore scope](39-decide-recovery-objectives.md)
+- [Implement coherent backup sets and verified restore](40-verified-backup-restore.md)
+- [Make the Project workspace lifecycle recoverable](41-recoverable-project-workspace-lifecycle.md)
+- [Establish migration compatibility and failed-migration recovery](14-migration-compatibility-and-recovery.md)
+
+Persistent-path provisioning and release-tree isolation remain owned by
+[Provision a runnable non-root Hub service](10-provision-nonroot-service.md)
+and [Make application releases atomic and rollbackable](12-atomic-releases-and-rollback.md).
+Full recovery/cutover procedures and drill evidence remain owned by
+[Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md).

.scratch/saas-production-readiness/issues/07-audit-product-surface-completeness.md

@@ -0,0 +1,58 @@
+# Audit the accepted product surface for production completeness
+
+Type: research
+Status: resolved
+
+## Question
+
+Against ADR-0020 and ADR-0021, which required platform-admin, org-admin,
+project-onboarding, team/access, session, usage, provider-mode, Organization
+policy-limit, emergency workload-control, and Feishu flows are absent,
+unreachable, or only test-facing rather than usable in the initial production
+service? Treat ADR-0022's capacity control surfaces as part of the accepted
+product boundary.
+
+## Answer
+
+The current Hub is not a usable or production-complete SaaS product surface.
+Its Organization-scoped JSON APIs and single-Feishu-app Agent Run path are
+worth retaining, but the running service has one process-global Feishu
+app/bot, one global provider key/base URL, no Organization connection/secret
+plane, no platform-administrator control plane, and no real Organization or
+platform management UI. OAuth redirects a successful login to an unmounted
+`/admin/org/:orgSlug` route.
+
+Two real Fastify integration probes, each run twice, proved that an authenticated
+Organization ADMIN receives 404 at the post-login destination and two distinct
+Organization return paths produce the same OAuth `client_id`. Official Feishu
+documentation also proves that `open_id` is application-scoped, `user_id` is
+tenant-scoped, and `union_id` only links applications from the same application
+provider. The current globally unique `User.feishuOpenId`, raw `chatId`, event
+identity, OAuth state and one-listener composition therefore cannot safely
+route independently customer-owned applications.
+
+ADR-0022 remains an additional release blocker: beyond a process-memory
+Project queue and global max-turn setting, layered limits, durable
+Organization-fair admission, hard storage/run/process budgets, alerts and
+audited emergency brakes have no complete persistence or product surface.
+
+The full verdict, deterministic probe results, root causes, retained seams and
+required end-to-end release evidence are in the
+[accepted SaaS product-surface completeness audit](../assets/product-surface-completeness-audit.md).
+The detailed route/schema/test matrix is in the
+[accepted product-surface code inventory](../assets/product-surface-code-inventory.md),
+and the Feishu identifier/OAuth/event/long-connection facts are in the
+[Feishu product-surface primary sources](../assets/product-surface-feishu-primary-sources.md).
+
+The newly-clear execution frontier is:
+
+- [Decide the platform-administrator identity and audit boundary](42-decide-platform-admin-identity-audit.md)
+- [Implement the Organization connection and secret plane](43-implement-org-connection-secret-plane.md)
+- [Route Feishu identity and traffic by Organization connection](44-route-feishu-by-org-connection.md)
+- [Build the private platform-administrator control plane](45-build-private-platform-admin-control-plane.md)
+- [Build the private Organization-administrator control plane](46-build-private-org-admin-control-plane.md)
+
+Existing tickets 23 and 32–38 remain the owners of durable ingress/admission,
+capacity enforcement, usage alerts, workload brakes and calibration. Ticket 08
+now requires all of these product slices before claiming the production release
+gate.

.scratch/saas-production-readiness/issues/08-prove-production-release-gate.md

@@ -0,0 +1,12 @@
+# Prove the production release gate
+
+Type: task
+Status: open
+Blocked by: 01, 02, 03, 04, 05, 06, 07, 09, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46
+
+## Question
+
+After the readiness investigations and resulting fixes are resolved, can one
+repeatable release procedure prove build/test/spec health, deploy a clean
+production-like environment, exercise critical tenant and agent journeys,
+verify observability and recovery, and either roll forward or roll back safely?

.scratch/saas-production-readiness/issues/09-restore-checker-format-gate.md

@@ -0,0 +1,40 @@
+# Restore the checker formatting gate
+
+Type: task
+Status: resolved
+
+## Question
+
+Apply the repository's configured rustfmt output to the currently reported
+Rust files, verify that no semantic changes are introduced, and restore
+`cargo fmt --all --check` so the checker CI gate is green.
+
+## Answer
+
+The exact CI command failed deterministically on seven Rust files and an
+isolated single-file rustfmt check reproduced the same output. The source
+changes were formatting-only, but the underlying gate was not reproducible:
+the repository had no `rust-toolchain.toml` and checker CI installed floating
+`stable`, so local rustfmt, clippy and CI could change independently.
+
+The checker toolchain is now pinned to Rust 1.92.0 with `rustfmt` and `clippy`
+in both `rust-toolchain.toml` and the Gitea workflow. The seven files were
+formatted with that toolchain. Running the complete CI path then surfaced three
+test helpers accepting `&PathBuf` instead of `&Path` and a Clippy name collision
+between `Shell` and `PowerShell`; those were corrected without changing CLI
+values or behavior.
+
+Verification:
+
+```text
+cargo fmt --all --check                         PASS (twice)
+cargo clippy --all-targets --all-features -- -D warnings
+                                                  PASS
+cargo test --all-features --workspace            PASS (53 tests)
+cph completions zsh / power-shell                 PASS
+```
+
+The final Rust diff consists of rustfmt layout plus four borrowed-Path
+signature generalizations and the internal `Shell` → `CompletionTarget` type
+rename. No expression, string value, command behavior, or public Rust API was
+changed.

.scratch/saas-production-readiness/issues/10-provision-nonroot-service.md

@@ -0,0 +1,40 @@
+# Provision a runnable non-root Hub service
+
+Type: task
+Status: resolved
+
+## Question
+
+Make clean-host installation create or validate the dedicated service identity,
+home/cache/state/workspace paths and ownership, generate a complete production
+configuration, honor the configured bind host, and fail preflight before unit
+installation when any required value or prerequisite is invalid. The persistent
+workspace root must be outside all release trees, and preflight must reject any
+path that overlaps rsync, release switching, rollback, or release pruning.
+
+## Answer
+
+The installer now creates or strictly validates a dedicated non-root,
+non-login service identity with no supplementary groups; provisions its
+home/state/cache/workspace directories at mode `0750`; and refuses persistent
+paths that overlap the deployment tree after both lexical and physical path
+resolution. A missing production environment file is seeded completely and
+exits with status 78; invalid configuration or prerequisites fail before the
+unit is written or enabled.
+
+Preflight validates the configured bind host/port, authenticated PostgreSQL
+access, Node/cph/Prisma, configured and canonical path traversal, and a real
+bubblewrap namespace. The post-provision phase runs the built Hub dependency
+graph, a database query, cph, socat and bubblewrap as the service user under
+`no_new_privs`, matching the systemd security context. HTTP binding is awaited
+before the Feishu listener starts, so bind failure rejects startup instead of
+leaving a listener-only process alive.
+
+Verification evidence:
+
+- `npm run check`, `npm run build`, and `npm run prisma:validate`
+- 166 unit tests and 86 integration tests passed; one external-model test skipped
+- `bash -n deploy/install_service.sh` and `shellcheck deploy/install_service.sh`
+- `prisma migrate deploy`: 9 migrations found, none pending
+- production-mode HTTP smoke test on `127.0.0.1:8878` returned `/api/healthz`
+- final Standards and Spec reviews found no remaining Issue 10 gap

.scratch/saas-production-readiness/issues/11-package-cph-with-release.md

@@ -0,0 +1,10 @@
+# Package and verify cph with every Hub release
+
+Type: task
+Status: open
+
+## Question
+
+Deliver the compatible `cph` executable as part of the same immutable release
+as the Hub, configure its exact path, and make deployment preflight reject a
+missing, non-executable, or version-incompatible binary before restart.

.scratch/saas-production-readiness/issues/12-atomic-releases-and-rollback.md

@@ -0,0 +1,13 @@
+# Make application releases atomic and rollbackable
+
+Type: task
+Status: open
+Blocked by: 10, 11
+
+## Question
+
+Replace in-place rsync/build with deployment locking, immutable staged release
+directories, a revision manifest, atomic current/previous switching, bounded
+retention, and an explicit application rollback that preserves the live release
+on build, preflight, switch, or readiness failure. Prove deploy, rollback, and
+release pruning cannot traverse or delete the persistent workspace root.

.scratch/saas-production-readiness/issues/13-real-readiness-and-lifecycle.md

@@ -0,0 +1,12 @@
+# Add real readiness and an accurate service lifecycle
+
+Type: task
+Status: open
+Blocked by: 03, 04, 10, 11, 12, 23, 24, 25
+
+## Question
+
+Separate liveness from readiness, verify every dependency needed to accept Hub
+work, make deployment wait on readiness, and align systemd shutdown/restart
+claims with an implemented and observable Fastify, Feishu, Prisma, and in-flight
+run lifecycle.

.scratch/saas-production-readiness/issues/14-migration-compatibility-and-recovery.md

@@ -0,0 +1,15 @@
+# Establish migration compatibility and failed-migration recovery
+
+Type: task
+Status: open
+Blocked by: 06
+
+## Question
+
+Define and enforce the application/schema compatibility rule for each release,
+require expand-and-contract sequencing for destructive changes, add migration
+status/checksum and production-like rehearsal gates, and move schema mutation
+out of ordinary service start into an explicit release phase. Require a recent
+verified recovery checkpoint, bounded lock/runtime evidence, and an operator
+procedure for resolving or restoring a failed Prisma migration without
+pretending that an application rollback reverses the database.

.scratch/saas-production-readiness/issues/15-test-clean-host-deployments-on-linux.md

@@ -0,0 +1,12 @@
+# Test clean-host deployment and rollback on Linux
+
+Type: task
+Status: open
+Blocked by: 10, 11, 12, 13, 14
+
+## Question
+
+Add an automated Linux production-like harness that executes the real install,
+deploy and rollback paths against PostgreSQL and proves first install,
+idempotence, failure atomicity, readiness failures, application rollback, and
+deployment serialization.

.scratch/saas-production-readiness/issues/16-confine-agent-and-protect-credentials.md

@@ -0,0 +1,38 @@
+# Confine the agent runtime and protect service credentials
+
+Type: task
+Status: resolved
+
+## Question
+
+Make the real Claude SDK/Bash execution surface uphold `AgentFileOp.Authorized`:
+deny sibling tenant workspaces and service files, expose only the minimum safe
+process environment, use SDK credential protection for secrets, and add an
+actual Linux sandbox test proving another workspace and every non-provider
+credential are unreadable while required tools still work.
+
+## Answer
+
+The Agent subprocess now receives a replacement environment containing only
+named runtime values and the selected provider protocol values; PostgreSQL,
+Feishu and Hub session credentials never cross the process boundary. Provider
+credentials are additionally denied inside Bash. The SDK must sandbox every
+command (`failIfUnavailable=true`, `allowUnsandboxedCommands=false`), denies
+host reads and writes from root, and re-opens only the canonical project
+workspace plus the reviewed read-only system runtime needed by `cph`.
+
+SDK config/cache/home and general temp storage live inside the workspace. A
+short relative `CLAUDE_CODE_TMPDIR` prevents the pinned SDK from falling back
+to host `/tmp`, while absolute workspace-local `TMPDIR`/`TMP`/`TEMP` remain
+stable after Bash changes directory. Linux production and CI now require both
+bubblewrap and `socat`; preflight probes both as the service user. Bounded,
+credential-redacted SDK stderr is logged with run and project correlation.
+
+The real Linux proof runs the actual Claude SDK, Bash, bubblewrap, `socat` and
+release `cph` as UID 1000 with no capabilities and `no_new_privs`. It proves
+parent, sibling-project, `/tmp` and `/var/tmp` writes are rejected and have no
+host effect; sibling/service-secret reads fail; platform and provider secrets
+are absent from Bash; workspace temp remains anchored after `cd`; and `cph`
+still executes. Final local gates passed: TypeScript check/build, Prisma schema,
+173 unit tests, 85 mock-provider integration tests, deployment shell syntax,
+and the dedicated Linux sandbox test.

.scratch/saas-production-readiness/issues/17-close-mcp-data-egress-escapes.md

@@ -0,0 +1,28 @@
+# Close MCP context and file-delivery escape paths
+
+Type: task
+Status: resolved
+
+## Question
+
+Restrict file delivery to a realpath/no-follow target inside the current project
+workspace, remove Git-root access, make inbound Hub writes symlink-safe, and
+validate every Feishu message/thread result against the run's bound chat before
+returning data to the model.
+
+## Answer
+
+All Agent-facing workspace file reads and writes are anchored to the configured
+workspace root and current project. Linux uses descriptor-relative, no-follow
+traversal so a symlink swap cannot escape after validation; unsupported unsafe
+paths fail closed. Inbound Feishu files use the same boundary. Outbound file
+delivery no longer accepts the repository Git root or arbitrary absolute
+paths: it reads a no-follow snapshot from the current project and uploads that
+snapshot, so the checked object cannot be swapped before delivery.
+
+The Feishu MCP read surface carries the run's project and bound-chat context.
+Message and thread results are rejected unless their returned chat matches that
+binding, and file-delivery MCP calls remain scoped to the same run/project/chat
+tuple. Unit and integration regressions cover symlink paths, sibling projects,
+Git-root delivery attempts, swapped files, and cross-chat results; they are
+included in the final 173-unit/85-integration green gate reported by Issue 16.

.scratch/saas-production-readiness/issues/18-bind-and-reauthorize-deferred-actions.md

@@ -0,0 +1,12 @@
+# Bind and reauthorize deferred and card actions
+
+Type: task
+Status: open
+Blocked by: 44
+
+## Question
+
+At execution time, reauthorize queued/batched triggers and validate one
+actor/chat/organization/project/run-or-pending-action tuple for interrupt,
+approval, and onboarding card actions so revoked permissions and mismatched
+object IDs cannot affect another project.

.scratch/saas-production-readiness/issues/19-design-org-scoped-secret-control-plane.md

@@ -0,0 +1,24 @@
+# Design the org-scoped secret and connection control plane
+
+Type: grilling
+Status: resolved
+
+## Question
+
+Which key-management, envelope-encryption, rotation, connection identity, and
+runtime resolver design will implement ADR-0021's org-scoped Feishu and model
+provider credentials; support one customer-owned Feishu app per Organization;
+support Organization-managed BYOK and a distinct platform-managed provider
+connection per Organization; and ensure plaintext credentials never reach
+business records, logs, or agent tools?
+
+## Answer
+
+Use the local master-key envelope, immutable connection-secret versions,
+writer-authority split, explicit Organization/Project resolver, staged KEK
+rotation, and separately protected keyring recovery contract accepted in
+ADR-0024. Production receives the root-owned keyring only through systemd
+`LoadCredential`; it has no process-global credential fallback. Agent runs use
+a loopback proxy capability rather than the Organization Provider credential;
+the offline rotation command atomically retains old/new KEKs, rewraps stored
+DEKs, audits the changes, and authenticates the complete envelope set.

.scratch/saas-production-readiness/issues/20-design-production-web-session-boundary.md

@@ -0,0 +1,14 @@
+# Design the production browser and session boundary
+
+Type: grilling
+Status: open
+
+## Question
+
+What same-origin/CSRF, proxy/TLS, security-header, session lifetime,
+revocation, secret rotation, and request-timeout contract should the initial
+org-admin web surface enforce, and which parts require server-side session state
+rather than the current seven-day stateless cookie? Apply the same browser
+boundary to ADR-0023's separate server-side Platform Session while preserving
+its distinct cookie/token namespace, immediate revocation, per-request grant
+reload, and bounded recent-authentication requirement.

.scratch/saas-production-readiness/issues/21-clear-dependency-security-advisories.md

@@ -0,0 +1,44 @@
+# Clear dependency advisories and gate releases
+
+Type: task
+Status: resolved
+
+## Question
+
+Remove every current `npm audit --omit=dev` high finding and every `cargo audit`
+vulnerability/unsound finding through tested upgrades, compatible overrides or
+upstream fixes—not ignores—and add reproducible Node and Rust security-audit
+gates to CI and the production release evidence.
+
+## Answer
+
+All current production-high Node advisories and all Rust vulnerability/unsound
+advisories are removed without advisory ignores:
+
+- `@larksuiteoapi/node-sdk` 1.70.0 declares Axios `~1.13.3`, which prevented npm
+  from selecting a patched release. The Hub now has a targeted npm override to
+  Axios 1.18.1. The direct SDK stays current; the override is locked and the
+  full Hub type/build/test gate validates the resolved tree. A deterministic
+  local-HTTP smoke also drives the real SDK/Axios stack through tenant-token,
+  authenticated success and HTTP-error paths without external credentials.
+- `memmap2` is upgraded to 0.9.11 and `crossbeam-epoch` to 0.9.20.
+- `plist` is upgraded to 1.10.0, which moves its XML parser to `quick-xml`
+  0.41.0.
+- crates.io `citationberg` 0.7.0 still pins vulnerable `quick-xml` 0.38.4.
+  Cargo now patches it to the exact upstream citationberg commit that adopts
+  `quick-xml` 0.41.0; the git revision is immutable in both manifest and lock.
+
+`hub-check` runs the checked-in `npm run audit:production` command immediately
+after `npm ci`. `checker-check` installs the pinned cargo-audit 0.22.2 release
+with `--locked`, then runs `cargo audit --deny unsound`; vulnerabilities fail by
+default and unsound warnings are promoted to failures.
+
+Current evidence:
+
+- `npm audit --omit=dev --audit-level=high`: 0 vulnerabilities.
+- `cargo audit --deny unsound`: 0 vulnerabilities and 0 unsound warnings.
+- Four Rust `unmaintained` informational warnings remain visible and are not
+  ignored; they are outside this issue's vulnerability/unsound acceptance
+  criterion and will continue to appear in CI output.
+- Hub: type-check, build and 280 tests pass (10 environment-specific skips).
+- Rust: fmt, clippy with warnings denied, and all 53 workspace tests pass.

.scratch/saas-production-readiness/issues/22-enforce-organization-status.md

@@ -0,0 +1,50 @@
+# Enforce Organization status at the shared authorization seam
+
+Type: task
+Status: resolved
+
+## Question
+
+Make non-ACTIVE Organizations fail closed for project authorization,
+onboarding, queued execution, and Feishu triggers through one shared seam, with
+negative tests proving a SUSPENDED or ARCHIVED tenant cannot create, bind,
+trigger, resume, or mutate work while operator recovery remains explicit.
+
+## Answer
+
+Organization lifecycle is now enforced at both authorization and mutation
+linearization points:
+
+- Project and project-group authorization resolves the owning Organization and
+  denies every customer action before principal resolution when status is not
+  `ACTIVE`. The legacy role helper follows the same rule.
+- Every customer mutation service (project/folder/binding/policy, explorer,
+  members, teams, team grants, external-directory sync and slash-session
+  mutation) verifies `ACTIVE` while holding a PostgreSQL `FOR SHARE` lock on
+  the Organization row. A concurrent status `UPDATE` therefore orders before
+  the operation (which fails closed) or after its transaction commits.
+- Agent session/run/project-lock admission is one transaction behind that same
+  lifecycle lock. Queued triggers reauthorize when drained. Slash-command races
+  return an explicit user-facing rejection instead of becoming deduplicated,
+  silent failures.
+- Feishu attachments download into a private
+  `HUB_PROJECT_WORKSPACE_ROOT/.cph-staging` batch outside every project. A
+  durably committed ACTIVE run/project-lock admission is the linearization
+  point; publication then uses exclusive same-filesystem hard links rather than
+  copying bytes under a database lock. Failure compensates inode-tracked links,
+  fails the admitted run and releases its lock. Startup removes abandoned
+  staging batches and refuses staging symlinks.
+- Project creation holds the lifecycle lock across workspace allocation and DB
+  creation. A later transaction failure removes the new leaf workspace; cleanup
+  failure is surfaced together with the original error as `AggregateError`.
+- External-directory sync uses short lifecycle-serialized transactions per
+  unit rather than holding the Organization row lock across an unbounded sync.
+
+No customer API can reactivate or otherwise change Organization lifecycle
+state. Operator recovery remains a separate platform-control-plane task.
+
+Evidence includes deterministic status/update lock ordering, direct-service
+negative tests for both `SUSPENDED` and `ARCHIVED`, queued-trigger and
+post-authorization races, DB-triggered workspace compensation (including
+cleanup failure), full local Hub tests, and the real Linux attachment trigger
+path.

.scratch/saas-production-readiness/issues/23-durable-feishu-inbox-and-queue.md

@@ -0,0 +1,16 @@
+# Make Feishu intake and queued work durable
+
+Type: task
+Status: open
+Blocked by: 18, 44
+
+## Question
+
+Replace receipt-before-success and process-local batching/queueing with one
+durable inbound work lifecycle that atomically claims duplicate events, records
+processing/completion/failure, survives restart, retries recoverable failures,
+and proves concurrent redelivery cannot duplicate a run or silently lose an
+accepted prompt. Preserve execution-time authorization from ticket 18 when
+deferred work is finally started, and expose the durable identity/state needed
+by ADR-0022's Organization-fair admission scheduler rather than recreating a
+second queue lifecycle.

.scratch/saas-production-readiness/issues/24-transactional-run-lock-recovery.md

@@ -0,0 +1,12 @@
+# Make run, session, lock, and recovery transitions safe
+
+Type: task
+Status: open
+
+## Question
+
+Implement and fault-test a centralized AgentRun lifecycle that enforces one
+active project/provider/role/model session, serializes provider-cursor updates,
+preserves the ADR-0002 lock invariant across every database failure, applies a
+real wall deadline and renewable lease, reconciles every non-terminal state on
+startup/watchdog, and cannot let a second Hub process invalidate live work.

.scratch/saas-production-readiness/issues/25-durable-outbound-delivery.md

@@ -0,0 +1,13 @@
+# Make outbound Feishu delivery durable and idempotent
+
+Type: task
+Status: open
+Blocked by: 44
+
+## Question
+
+Give status cards, final responses, files, and recovery notices a durable
+outbox/delivery lifecycle with stable idempotency keys, provider message IDs,
+bounded retry and reconciliation, so model completion is distinguishable from
+pending/failed/delivered user-visible output and an operator can safely replay
+ambiguous failures without duplicates.

.scratch/saas-production-readiness/issues/26-decide-workspace-run-commit-semantics.md

@@ -0,0 +1,13 @@
+# Decide workspace commit semantics for failed runs
+
+Type: grilling
+Status: open
+
+## Question
+
+When an agent run fails, is canceled, times out, loses the database, or is
+killed mid-tool, should its workspace changes remain as partial collaborative
+work, be quarantined for review, or roll back atomically? Decide the contract,
+including the relationship among provider cursor, AgentMessage,
+AgentFileChange, git/filesystem state, and the next run, then create the
+smallest implementation tickets and crash tests that enforce it.

.scratch/saas-production-readiness/issues/27-reconcile-audit-history-contract.md

@@ -0,0 +1,16 @@
+# Reconcile the audit and run-history contract
+
+Type: grilling
+Status: open
+Blocked by: 04
+
+## Question
+
+Separate or unify run-bound audit entries, pre-run security/permission events,
+structured messages, and operational recovery events without weakening
+`Spec.System.Audit`'s pinned AuditEntry-to-run relation. Decide durability,
+failure, retention, and query semantics; then enforce referential integrity and
+observable/recoverable writes instead of silently swallowing lost evidence.
+Do not merge these customer Project/Run records with ADR-0023's already-decided
+separate, append-only, fail-closed Platform Audit; decide only any shared
+retention/export infrastructure without weakening either semantic boundary.

.scratch/saas-production-readiness/issues/28-typed-observable-http-errors.md

@@ -0,0 +1,13 @@
+# Make HTTP dependency failures typed and observable
+
+Type: task
+Status: open
+
+## Question
+
+Replace message-substring HTTP error classification with typed validation,
+domain, authorization, Prisma/dependency, and unexpected-error boundaries.
+Return only sanitized stable client codes, preserve true 5xx/availability
+semantics, emit one request-scoped structured error with a support-visible
+request ID, redact secrets and raw internals, and add a real database-outage
+regression proving Prisma failures can never become client 4xx responses.

.scratch/saas-production-readiness/issues/29-decide-telemetry-slos-alert-routing.md

@@ -0,0 +1,16 @@
+# Decide the telemetry, SLO, retention, and alerting control plane
+
+Type: grilling
+Status: open
+
+## Question
+
+For the accepted initial single-host service, decide the log/metric/trace
+collection and storage path, instance roles, release and correlation identity,
+redaction/access/retention policy, initial service-level indicators and
+objectives, error budgets, alert thresholds, on-call destinations, escalation
+ownership, and telemetry-blind detection. Prefer the smallest operable control
+plane, but record every domain-visible availability or retry assumption in the
+appropriate ADR/spec rather than hiding it in dashboard configuration. Consume
+ADR-0022's already-decided usage/cost soft-alert ownership as input; do not
+reopen its hard-limit or provider-mode financial-alert semantics here.

.scratch/saas-production-readiness/issues/30-instrument-correlated-runtime-telemetry.md

@@ -0,0 +1,15 @@
+# Instrument correlated production runtime telemetry
+
+Type: task
+Status: open
+Blocked by: 13, 23, 24, 25, 27, 28, 29
+
+## Question
+
+Implement the decided telemetry control plane with one redacted structured
+context from HTTP/Feishu ingress through durable work, run/session/lock,
+provider and SDK retry/rate-limit/tool events, cph/workspace operations,
+outbound delivery, audit, startup/drain/recovery, and release/schema identity.
+Expose bounded-cardinality metrics, dashboards and alert rules; eliminate
+silent critical-path failures; and prove logs can pivot end-to-end while metric
+labels contain no tenant/user/run cardinality or secret/message content.

.scratch/saas-production-readiness/issues/31-write-and-drill-incident-runbooks.md

@@ -0,0 +1,22 @@
+# Write and drill production incident recovery runbooks
+
+Type: task
+Status: open
+Blocked by: 06, 12, 13, 14, 23, 24, 25, 26, 27, 28, 29, 30, 39, 40, 41
+
+## Question
+
+Check in operator runbooks for PostgreSQL outage/migration failure, Feishu
+disconnect and credential rejection, ingress/outbound backlog or dead letter,
+stuck run/lock, provider or cph failure, workspace capacity/corruption,
+deploy/rollback, host restart during work, and suspected tenant exposure. Give
+each alert an owner and safe read-first commands, require confirmed/audited
+scope for destructive or replay actions, execute every runbook against
+production-shaped staging, and archive evidence that detection, mitigation,
+reconciliation, verification, escalation, and alert clearing all work. Include
+database/workspace backup failure, failed migration, full-host restore and
+cutover, duplicate/gap review, fallback, and achieved RPO/RTO evidence.
+Also drill ADR-0023 platform-administrator lockout and Platform-owned Feishu
+Application credential loss through the dual-factor offline recovery command,
+expiring Emergency Platform Grant, session revocation, immutable Platform Audit,
+and explicit recovery closeout; never introduce a standing break-glass account.

.scratch/saas-production-readiness/issues/32-layered-limits-rate-limiting.md

@@ -0,0 +1,13 @@
+# Implement layered limits and multi-dimensional request rate limiting
+
+Type: task
+Status: open
+Blocked by: 20, 22, 44
+
+## Question
+
+Implement ADR-0022's fail-closed platform ceilings, Organization-lowerable
+policy limits, bounded list/query behavior, and typed multi-dimensional rate
+limits for authenticated Web, signed Feishu, OAuth, and administrative writes.
+Prove effective limits never exceed platform ceilings and every overload result
+is observable, scoped, and carries retry guidance where applicable.

.scratch/saas-production-readiness/issues/33-durable-fair-admission-scheduler.md

@@ -0,0 +1,13 @@
+# Build the durable Organization-fair admission scheduler
+
+Type: task
+Status: open
+Blocked by: 18, 23, 24, 32
+
+## Question
+
+Extend the durable inbound lifecycle into ADR-0022's bounded Agent Admission
+Queue with platform/Organization/Project concurrency, fair scheduling across
+Organizations, FIFO within each Organization, durable position/state,
+expiration notification, scoped cancellation, idempotent admission, explicit
+capacity rejection, and no automatic replay after a run has started.

.scratch/saas-production-readiness/issues/34-file-storage-budgets.md

@@ -0,0 +1,13 @@
+# Enforce file, archive, storage, and entity budgets
+
+Type: task
+Status: open
+Blocked by: 17, 23, 32
+
+## Question
+
+Enforce ADR-0022's request/file/attachment/archive-expansion,
+Project/Organization storage, and Organization entity-count limits with
+streaming accounting, atomic reservation, cleanup of incomplete artifacts,
+bounded queries, explicit errors, and continued read/export/delete/admin
+remediation while an Organization is at or above a ceiling.

.scratch/saas-production-readiness/issues/35-run-runtime-budgets.md

@@ -0,0 +1,13 @@
+# Enforce Agent Run and process resource budgets
+
+Type: task
+Status: open
+Blocked by: 16, 24, 32
+
+## Question
+
+Implement ADR-0022's wall-time, turn, tool-count, per-tool-time,
+output/log/event-size, memory, CPU, and child-process budgets. Termination must
+be fail-fast and observable, persist `TIMED_OUT` or `LIMIT_EXCEEDED` with the
+triggered dimension and actual usage, cancel the provider/process safely, and
+release run locks and admission capacity under every injected failure.

.scratch/saas-production-readiness/issues/36-usage-soft-alerts.md

@@ -0,0 +1,13 @@
+# Implement provider-mode usage attribution and soft alerts
+
+Type: task
+Status: open
+Blocked by: 19, 24, 29, 32, 43
+
+## Question
+
+Attribute token usage, provider-reported cost, run count, and duration by
+Organization, Project, Run, model, and Provider Connection; preserve missing
+cost as unknown; expose Organization-configurable soft thresholds; and route
+Organization alerts plus platform-managed anomalous-cost alerts without hard
+spend blocking, BYOK financial alerts, or payment-settlement semantics.

.scratch/saas-production-readiness/issues/37-emergency-workload-brakes.md

@@ -0,0 +1,13 @@
+# Add audited emergency workload brakes
+
+Type: task
+Status: open
+Blocked by: 22, 23, 24, 27, 30, 32, 33, 45
+
+## Question
+
+Implement reversible, reason-bearing, audited `DRAIN` and `STOP_NOW` controls at
+Organization and platform scope. Prove new admissions, queued work, active runs,
+locks, user notifications, authorization, telemetry, restart persistence, and
+explicit recovery follow ADR-0022 without deleting or silently suspending
+Organization data.

.scratch/saas-production-readiness/issues/38-calibrate-production-capacity.md

@@ -0,0 +1,14 @@
+# Calibrate and prove production capacity ceilings
+
+Type: research
+Status: open
+Blocked by: 30, 32, 33, 34, 35, 36, 37
+
+## Question
+
+On the accepted production-like host and provider path, measure safe numerical
+defaults and headroom for every ADR-0022 platform ceiling. Exercise normal and
+noisy-neighbor traffic, provider stalls and rate limits, queue saturation,
+large files/archives, storage pressure, process budgets, and both workload
+brakes; publish reproducible evidence and the checked-in configuration values
+required by the production release gate.

.scratch/saas-production-readiness/issues/39-decide-recovery-objectives.md

@@ -0,0 +1,14 @@
+# Decide recovery objectives, backup retention, and restore scope
+
+Type: grilling
+Status: open
+
+## Question
+
+For the initial SaaS service, decide measurable RPO/RTO targets, backup and
+restore-drill frequency, retention and deletion-in-backups policy, off-host
+copy/failure-domain requirements, logical export versus physical base-backup
+and continuous-WAL/PITR coverage, encryption and key custody, operator ownership,
+and whether the supported promise is full-service disaster recovery only or
+also Organization-selective export/restore. Record compliance-dependent items
+as explicit OPEN inputs rather than inventing defaults.

.scratch/saas-production-readiness/issues/40-verified-backup-restore.md

@@ -0,0 +1,16 @@
+# Implement coherent backup sets and verified restore
+
+Type: task
+Status: open
+Blocked by: 10, 12, 14, 19, 23, 24, 25, 26, 29, 39, 41, 43
+
+## Question
+
+Implement the decided recovery policy as cataloged, checksummed, encrypted,
+off-host backup sets covering PostgreSQL logical exports plus physical base
+backup/continuous WAL or a proven managed equivalent, Project workspaces,
+compatible release/migration identity, configuration inventory, org-secret
+ciphertext, and separately recoverable key material. Establish a proven
+cross-store checkpoint, retention and alerting, partial-artifact cleanup,
+isolated restore with outbound integrations disabled, lifecycle reconciliation,
+critical SaaS journey verification, and measured RPO/RTO drills.

.scratch/saas-production-readiness/issues/41-recoverable-project-workspace-lifecycle.md

@@ -0,0 +1,14 @@
+# Make the Project workspace lifecycle recoverable
+
+Type: task
+Status: open
+Blocked by: 10, 26, 34
+
+## Question
+
+Move persistent workspaces outside every mutable release tree and make Project
+workspace allocation, database commit, archive/deletion, relocation, restore,
+and orphan/missing-path reconciliation explicit and crash-safe. Prove failed
+Project creation leaves no silent orphan, deployment/rollback/pruning cannot
+delete customer files, restored absolute paths are validated or safely rebased,
+and every inconsistency is observable rather than auto-hidden.

.scratch/saas-production-readiness/issues/42-decide-platform-admin-identity-audit.md

@@ -0,0 +1,54 @@
+# Decide the platform-administrator identity and audit boundary
+
+Type: grilling
+Status: resolved
+
+## Question
+
+Which identity issuer, bootstrap and recovery procedure, invite/allowlist model,
+session claims, role lifecycle, separation from Organization membership, and
+append-only audit contract will authenticate Platform Administrators for the
+initial service? The decision must keep `/admin/platform` private, make every
+Organization/connection/workload-control mutation attributable, and define a
+reviewable break-glass path without treating platform staff as customer members.
+
+## Answer
+
+Platform Administrators authenticate only through a dedicated Platform-owned
+Feishu Application and a separate, server-side Platform Session. Platform
+Identity, standing grant, invitation, session, audit and emergency grant are
+independent of customer `User`, `OrganizationMembership`, Project permission
+and Project/Run audit. The initial control plane has one peer role only.
+
+The first administrator is created by a versioned, idempotent, transactional
+database bootstrap procedure that runs only when no standing administrator is
+active and atomically records the Platform Audit Entry. Later administrators
+receive short-lived, single-use invitations bound to a specific platform
+Feishu account. The last active standing administrator cannot be revoked;
+revocation immediately invalidates that identity's sessions and invitations.
+
+Every platform request reloads the revocable session and current standing or
+unexpired emergency grant. Administrator, connection/credential,
+Organization-lifecycle, workload-brake and emergency-grant actions require
+recent Feishu OAuth step-up.
+
+Every successful platform mutation commits with a separate append-only
+Platform Audit Entry or fails. Organization Administrators receive a redacted
+projection of platform actions affecting their own Organization. No standing
+break-glass account exists: offline recovery requires controlled host access,
+an off-host recovery key, an incident and reason, and issues only an expiring
+Emergency Platform Grant.
+
+The complete accepted decision and implementation divergences are in
+[ADR-0023](../../../docs/adr/0023-platform-administrator-identity-and-audit.md).
+The pinned semantic invariants are in
+[`Spec.System.PlatformAdministration`](../../../spec/Spec/System/PlatformAdministration.lean),
+and the canonical terms are in [`CONTEXT.md`](../../../CONTEXT.md).
+
+Exact numeric session/invitation/step-up limits and browser mechanics remain
+with [Design the production browser and session boundary](20-design-production-web-session-boundary.md).
+Customer Project/Run audit retention remains with
+[Reconcile the audit and run-history contract](27-reconcile-audit-history-contract.md),
+without reopening Platform Audit's separate fail-closed boundary. Implementation
+is owned by [Build the private platform-administrator control plane](45-build-private-platform-admin-control-plane.md),
+and recovery rehearsal by [Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md).

.scratch/saas-production-readiness/issues/43-implement-org-connection-secret-plane.md

@@ -0,0 +1,16 @@
+# Implement the Organization connection and secret plane
+
+Type: task
+Status: open
+Blocked by: none
+
+## Question
+
+Implement encrypted, versioned, Organization-scoped Feishu Application and
+model Provider Connection records plus one resolver boundary. Support one
+customer-owned Feishu app per Organization, Organization-admin-managed BYOK,
+and one distinct platform-managed key/base URL per Organization; enforce which
+administrator may write each mode, readiness and rotation lifecycle, runtime
+resolution from Organization/Project, redacted observability, backup/key
+recovery hooks, and no process-global credential fallback or plaintext leakage
+into business rows, logs, HTTP responses, or agent environments.

.scratch/saas-production-readiness/issues/44-route-feishu-by-org-connection.md

@@ -0,0 +1,17 @@
+# Route Feishu identity and traffic by Organization connection
+
+Type: task
+Status: open
+Blocked by: 20, 22, 43
+
+## Question
+
+Make every Feishu OAuth, WebSocket/event, card action, directory sync, chat
+binding, API call, and outbound message start from an explicit active
+Organization Feishu Application Connection. Model external user, chat, event,
+and callback identifiers in that connection's namespace; bind OAuth state to
+the intended Organization/application; run and observe one listener/client
+lifecycle per configured app; and prove two customer-owned apps with colliding
+provider-local identifiers cannot authenticate, bind, authorize, deduplicate,
+rate-limit, or deliver across Organizations. Do not infer tenant from a global
+`open_id`, `chat_id`, or from the user's membership count.

.scratch/saas-production-readiness/issues/45-build-private-platform-admin-control-plane.md

@@ -0,0 +1,25 @@
+# Build the private platform-administrator control plane
+
+Type: task
+Status: open
+Blocked by: 20, 27, 28, 42, 43, 44
+
+## Question
+
+Implement authenticated `requirePlatformAdmin` APIs and a private
+`/admin/platform` web area for Platform Administrators to create/list/update
+Organizations, bootstrap their OWNER, view/change lifecycle state, configure
+and test each customer-owned Feishu Application, configure the Organization's
+distinct platform-managed Provider Connection, and manage platform-admin
+invitations and revocations. Implement ADR-0023's separate Platform Identity,
+single standing role, guarded bootstrap procedure, identity-bound single-use
+invitations, revocable server-side Platform Session, recent-authentication
+step-up, last-administrator protection, fail-closed append-only Platform Audit,
+Organization-safe audit projection, and dual-factor offline recovery with
+expiring Emergency Platform Grants. Replace rather than reuse the legacy
+customer-`User`-bound `PlatformRoleAssignment`/`TEACHER` model. Prove anonymous
+users, ordinary members, and Organization admins cannot load either page data
+or mutations; every sensitive action is validated, redacted, audited,
+observable, and covered by browser-level critical-journey tests. Emergency
+workload controls remain integrated through
+[Add audited emergency workload brakes](37-emergency-workload-brakes.md).

.scratch/saas-production-readiness/issues/46-build-private-org-admin-control-plane.md

@@ -0,0 +1,17 @@
+# Build the private Organization-administrator control plane
+
+Type: task
+Status: open
+Blocked by: 20, 27, 28, 32, 36, 43, 44
+
+## Question
+
+Turn the guarded Organization JSON APIs into a usable private
+`/admin/org/:orgSlug` web control plane for OWNER/ADMIN users. Cover members and
+roles, teams and Project access, Folders/Projects/chat unbinding, sessions and
+usage, model/role/run policy, Organization policy limits and alerts, Feishu
+connection readiness, Provider Credential Mode selection, and BYOK key/base URL
+management without ever displaying stored plaintext. Preserve server-side
+authorization on every request, define safe loading/error/empty states, and
+prove login redirect plus browser journeys, cross-Organization denial, MEMBER
+denial, secret redaction, and logout/session expiry end to end.

.scratch/saas-production-readiness/map.md

@@ -0,0 +1,62 @@
+# Initial Production SaaS Readiness
+
+Label: wayfinder:map
+
+## Destination
+
+The repository can be deployed from a clean checkout as the initial production
+multi-tenant Curriculum Project Hub: platform staff manage Organizations; every
+Organization authenticates through and operates its own customer-owned Feishu
+app/bot, and selects either an Organization-managed BYOK provider connection or
+a distinct platform-managed provider connection; Organization owners/admins
+manage their own members, projects and settings. The ADR-0018/0020/0021
+single-host Linux topology is operated with repeatable release evidence,
+rollback and recovery, and no known critical security or data-integrity gaps.
+
+## Notes
+
+- `spec/` remains the semantic source of truth; implementation drift must be
+  surfaced rather than silently resolved.
+- The initial product boundary follows the accepted ADRs: Organization is the
+  tenant root and customer onboarding may be manual. Full billing, tenant
+  suspension, and self-service platform administration remain deferred;
+  ADR-0022's safety limits and workload brakes are required for production.
+- Manual guided setup may provision each Organization's customer-owned Feishu
+  app during the pilot; one process-global Feishu/OAuth credential shared by
+  unrelated Organizations is not an acceptable destination state.
+- Use the `diagnosing-bugs` loop for every reproduced defect. Prefer explicit
+  failures and observable critical paths over fallback behavior.
+- Work directly on `main` per developer authorization; make small conventional
+  commits after verified, independently reversible milestones. Do not push
+  without explicit authorization.
+- Local `lake build` currently requires a configured elan toolchain; CI remains
+  the authoritative configured Lean build environment until that local toolchain
+  is available.
+
+## Decisions so far
+
+- [Audit the clean-host deployment and rollback contract](issues/01-audit-clean-host-deployment.md) — keep the accepted single-host topology, but replace the incomplete in-place updater with a provisioned, immutable, readiness-gated and rollbackable release contract.
+- [Audit tenant, authentication, and request security boundaries](issues/02-audit-tenant-auth-security.md) — retain the working org-admin/application authorization core, but production is blocked on agent/MCP isolation, execution-time object binding, org-scoped encrypted credentials, tenant status, browser sessions, and clean dependency audits.
+- [Audit run lifecycle and data integrity under failure](issues/03-audit-run-lifecycle-integrity.md) — retain the database-enforced project-lock uniqueness and normal happy path, but replace receipt-before-success, process-local queues, independent run/session/lock effects, silent delivery/audit loss, and partial startup reset with durable, recoverable state machines; explicitly decide failed-run workspace semantics.
+- [Define production observability and incident recovery](issues/04-define-observability-recovery.md) — keep the existing Fastify/Prisma/systemd signal fragments, but production requires dependency-aware readiness, typed and correlated errors, durable-work telemetry, an explicit SLO/retention/alerting control plane, and recovery runbooks proven by fault drills.
+- [Define initial abuse and capacity controls](issues/05-define-abuse-capacity-controls.md) — require layered fail-closed limits, durable Organization-fair admission, explicit backpressure, bounded files/storage/runs, soft usage alerts, and audited emergency workload brakes; calibrate numerical ceilings under production-like load.
+- [Audit backup, migration, and disaster recovery safety](issues/06-audit-backup-migration-recovery.md) — separate persistent state from releases, define a database/workspace checkpoint and recovery objectives, make migrations compatibility-gated, and prove complete off-host backup sets through isolated restore and cutover drills.
+- [Audit the accepted product surface for production completeness](issues/07-audit-product-surface-completeness.md) — retain the guarded Organization APIs and single-app Feishu vertical slice, but require explicit Organization-scoped Feishu/provider connections and external-identity namespaces, private platform and Organization admin control planes, and the full ADR-0022 capacity surface before release.
+- [Decide the platform-administrator identity and audit boundary](issues/42-decide-platform-admin-identity-audit.md) — use a dedicated platform Feishu issuer, one peer administrator role, guarded bootstrap and identity-bound invitations, revocable server-side sessions, atomic append-only Platform Audit, and dual-factor offline recovery without a standing break-glass account.
+- [Restore the checker formatting gate](issues/09-restore-checker-format-gate.md) — pin local and CI Rust/rustfmt/clippy to 1.92.0, apply the canonical workspace format, and keep the full fmt + clippy + 53-test checker gate green.
+- [Provision a runnable non-root Hub service](issues/10-provision-nonroot-service.md) — install a strictly validated service identity and persistent layout, reject release/workspace overlap, validate complete production configuration and real bind settings, and execute Hub/Prisma/cph/bubblewrap probes as the service user under `no_new_privs` before writing the unit.
+- [Confine the agent runtime and protect service credentials](issues/16-confine-agent-and-protect-credentials.md) — replace the Agent environment, deny host reads/writes from root, keep every writable SDK path inside the canonical project, require bubblewrap plus socat, emit bounded correlated diagnostics, and prove the boundary with the real Linux SDK under a capability-free non-root identity.
+- [Close MCP context and file-delivery escape paths](issues/17-close-mcp-data-egress-escapes.md) — use no-follow project-rooted file operations and delivery snapshots, remove Git-root delivery, and bind every Feishu MCP read to the run's project and chat.
+- [Enforce Organization status at the shared authorization seam](issues/22-enforce-organization-status.md) — deny non-ACTIVE tenants before principal resolution, serialize every customer mutation and run admission against Organization lifecycle updates, reauthorize queued work, and stage/publish attachments with crash recovery and explicit filesystem compensation.
+- [Clear dependency advisories and gate releases](issues/21-clear-dependency-security-advisories.md) — resolve Node production-high and Rust vulnerability/unsound advisories with locked compatible upgrades plus an exact upstream citationberg fix, and make both audits mandatory CI gates without ignores.
+
+## Fog
+
+- The production topology after the initial single-host pilot (HA, horizontal
+  workers, multi-region, or managed container orchestration) depends on measured
+  demand and failure experience.
+- Commercial self-service concerns such as billing plans and automated tenant
+  provisioning remain beyond the accepted ADR boundary; revisit after the
+  initial safety and operability route is clear.
+- Regulatory retention, residency, and customer-specific compliance controls
+  require product/legal inputs that the repository does not currently contain.

AGENTS.md

@@ -0,0 +1,59 @@
+# AGENTS.md —— agent 操作手册(全 repo)
+
+本 repo 是 monorepo。先读根 `README.md` 的"宪法"5 条,那是一切工作的前提。本文件是给在这里干活的 coding agent 的纪律。
+
+## 这个 repo 是什么
+
+- `spec/` 是一份**人机共识的契约**(Lean 语义母本),是产品语义的上游参照。
+- 其余部件(将来的 `spec/` 外文件夹)是**向 `spec/` 对齐的实现**。
+- `hub/` 的平台层按 SaaS 形态演进:`Organization` 是 tenant root;`Project`/`Team`
+  必须归属 org,TEAM→PROJECT 授权不得跨 org(见 ADR-0020 / `Spec.System.Organization`)。
+- org 后台 project explorer 里 `Folder` 是透明组织节点,不是权限资源;project 仍是权限边界。
+  普通老师可在飞书群自助建 project 但受 org policy 控制(见 ADR-0021 /
+  `Spec.System.ProjectWorkspace`)。
+- 每个 org 自选 BYOK 或平台托管 model provider connection;平台托管也必须是该 org
+  独享的 key/base URL,不得让无关 org 共用 process-global provider key(见 ADR-0021 /
+  `Spec.System.Organization`)。
+- Feishu/provider secret 使用本地版本化 master-key keyring 的信封加密;生产由 systemd
+  credential 注入,运行时只允许显式 org/project scope 的 fail-closed resolver,不得回退
+  process-global credential;Agent child 只接收 run-scoped loopback proxy capability,
+  不接收 org provider credential(见 ADR-0024 / `Spec.System.Organization`)。
+- 生产容量按不可突破的 platform ceiling 与 org 可下调 policy 分层;有效限制取两者较低值。
+  Agent admission 必须持久、有界、跨 org 公平且显式背压(见 ADR-0022 /
+  `Spec.System.Capacity`)。
+- 平台管理员只通过独立的 platform-owned 飞书应用与可撤销 Platform Session 认证,不复用
+  客户 `User`/org membership;平台写操作与 append-only audit 同事务,break-glass 只走
+  双因子的离线恢复流程(见 ADR-0023 / `Spec.System.PlatformAdministration`)。
+- 受控 alpha 暂采用一 Organization 一具名 systemd Silo:独立 database role/database、
+  service identity、workspace、keyring 与 Feishu/provider connection；进程必须由
+  `HUB_SILO_ORGANIZATION_ID` fail-closed 绑定唯一 org，平台后台不开放。共享 SaaS
+  控制面与 Docker adapter 后置(见 ADR-0025)。
+- Agent role 与 skill 是 Organization-scoped 动态运行配置：role 组合 model、system prompt、
+  tools 与已安装 skill；skill 版本进入 content-addressed 持久存储，run 只读加载所选快照。
+  每个 Organization 必须且只能有一个启用中的默认 role；新建群绑定从该默认值初始化，之后
+  `ProjectGroupBinding` 持久化群内当前 role，run 在接纳时冻结该 role。飞书公开 slash 协议只含
+  `/project`、`/usage`、`/help`；role、会话、目录操作走 `/project` 卡片，Claude 原生
+  `/compact` 只能由卡片动作以未经包装的精确 prompt 转发。
+  `settingSources: []` 继续禁用项目/用户配置加载，不得把任意 workspace `.claude` 配置变成
+  运行时能力(见 ADR-0018)。
+  skill/role 的管理面分组由一棵 org 内共用、可嵌套的 folder 树承载：透明组织节点，
+  不进入身份、解析与授权——name/roleId 仍 org 内唯一，role→skill 绑定、run 加载与
+  slash 命令均不引用 folder;folder 归属变更是 label 类变更,不归档会话;仅空 folder
+  可删(见 ADR-0028 / `Spec.System.AgentRole`)。
+- 项目发现由 `ProjectDiscovery` 模块统一承载：PostgreSQL `pg_trgm` 搜索派生文档、项目编号
+  归一化、完整 Folder breadcrumb、MANAGE 授权过滤与分页都在该模块内；飞书卡片只是 adapter。
+  `Project`/`Folder` 仍是事实来源，搜索文档必须可重建且由数据库触发器同步，禁止调用方双写。
+  系统 `Inbox` 只作为未分类项目的内部落点，不作为业务 folder 暴露；已绑定群通过
+  `@bot /project` 随时打开项目管理卡片，重命名仍走 org-scoped MANAGE 授权与审计。
+
+## 纪律
+
+1. **不得用预训练先验脑补本领域。** 这个领域很新,你没有相关先验。契约里 prose doc 注释是语义的唯一权威来源;契约没写的,就是没定的。
+
+2. **凡契约未写明者,不得假设。** 遇到标了 `OPEN` 的地方,或契约根本没覆盖的地方,**显式 surface 出来**让开发者决定,绝不擅自替它选一个解。
+
+3. **改 `spec/` 必须保持其 `lake build` 通过。** 在 `spec/` 目录下跑 `lake build`。新增声明必须带 `/-- … -/` doc 注释和恰当标签(`PINNED` / `OPEN` / `ADR-NNNN`)。规范见 `spec/README.md`。不准用 `sorry` 把 build 糊绿。
+
+4. **实现向契约对齐;偏离必须 surface。** 没有 CI gate 替你把关 spec↔实现的一致性(见宪法第 2 条)——这道对齐靠 review 和你巡逻 diff。发现实现与契约不一致时,报告它,不要默默让其中一边将就另一边。
+
+5. **写操作谨慎。** 线上操作、git 写操作前与开发者确认(这是开发者的全局偏好)。

CLAUDE.md

@@ -6,6 +6,11 @@
 
 - `spec/` 是一份**人机共识的契约**(Lean 语义母本),是产品语义的上游参照。
 - 其余部件(将来的 `spec/` 外文件夹)是**向 `spec/` 对齐的实现**。
+- `hub/` 的平台层按 SaaS 形态演进:`Organization` 是 tenant root;`Project`/`Team`
+  必须归属 org,TEAM→PROJECT 授权不得跨 org(见 ADR-0020 / `Spec.System.Organization`)。
+- org 后台 project explorer 里 `Folder` 是透明组织节点,不是权限资源;project 仍是权限边界。
+  普通老师可在飞书群自助建 project 但受 org policy 控制(见 ADR-0021 /
+  `Spec.System.ProjectWorkspace`)。
 
 ## 纪律
 

CONTEXT.md

@@ -0,0 +1,101 @@
+# Curriculum Project Hub
+
+The shared language for the multi-tenant service that lets customer organizations manage curriculum projects and operate agents through their own Feishu connection.
+
+## Language
+
+**Organization**:
+The customer tenant root that owns its memberships, projects, teams, external connections, and settings.
+_Avoid_: Account, workspace
+
+**Platform Administrator**:
+An internal operator who creates and operates Organizations without becoming a member of them.
+_Avoid_: Organization admin, super member
+
+**Organization Administrator**:
+An Organization OWNER or ADMIN who manages only that Organization's members, projects, connections, and settings.
+_Avoid_: Platform admin
+
+**Admin Surface**:
+The private web control plane available only to an authenticated user with the corresponding Platform Administrator or Organization Administrator authority.
+_Avoid_: Public management page, member portal
+
+**Customer-owned Feishu Application**:
+The Organization-scoped Feishu application used for that Organization's login, bot messages, and directory integration; unrelated Organizations never share it.
+_Avoid_: Global Feishu app, platform bot
+
+**Platform-owned Feishu Application**:
+The dedicated Feishu application used only to authenticate Platform Administrators; it is separate from every Customer-owned Feishu Application and grants no Organization or Project authority.
+_Avoid_: Customer-owned Feishu Application, Organization Feishu app
+
+**Platform Administrator Bootstrap**:
+The one-time, guarded grant of Platform Administrator authority to a verified Platform Identity when no active Platform Administrator exists.
+_Avoid_: Seed administrator, standing bootstrap account
+
+**Platform Administrator Invitation**:
+A short-lived, single-use invitation issued by a Platform Administrator and bound to one expected identity from the Platform-owned Feishu Application.
+_Avoid_: Open administrator enrollment, unbound administrator invitation
+
+**Platform Session**:
+A revocable authentication session bound only to a Platform Identity and its active Platform Administrator authority; it carries no Organization membership or Project authority.
+_Avoid_: Organization admin session, customer session
+
+**Platform Audit Entry**:
+An immutable, durable record of a platform control-plane decision or action, separate from customer Project and Agent Run audit and mandatory for every platform mutation. Organization Administrators receive only a redacted projection of entries affecting their Organization.
+_Avoid_: Best-effort log, Project audit entry
+
+**Emergency Platform Grant**:
+Short-lived Platform Administrator authority issued only by the offline recovery procedure for a declared incident and automatically expired afterward.
+_Avoid_: Standing break-glass account, permanent emergency administrator
+
+**Provider Credential Mode**:
+The Organization's choice between a BYOK Provider Connection and a Platform-managed Provider Connection.
+_Avoid_: Global provider setting
+
+**BYOK Provider Connection**:
+An Organization-scoped model-provider connection whose key and base URL are supplied and managed by an Organization Administrator.
+_Avoid_: Platform key
+
+**Platform-managed Provider Connection**:
+An Organization-scoped model-provider connection whose distinct key and base URL are supplied and managed by a Platform Administrator for that Organization.
+_Avoid_: Shared platform key, BYOK
+
+**Platform Capacity Ceiling**:
+A non-overridable upper bound imposed by the platform to protect shared service capacity.
+_Avoid_: Organization quota
+
+**Organization Policy Limit**:
+An optional lower bound configured by an Organization Administrator to govern that Organization's own usage.
+_Avoid_: Platform limit
+
+**Effective Limit**:
+The lower of the Platform Capacity Ceiling and the Organization Policy Limit; when no Organization Policy Limit exists, the Platform Capacity Ceiling applies.
+_Avoid_: Unlimited default
+
+**Organization-fair Scheduling**:
+The policy that allocates available agent execution capacity fairly across Organizations, while preserving first-in-first-out order within each Organization.
+_Avoid_: Global FIFO
+
+**Agent Admission Queue**:
+The durable, bounded set of accepted agent run requests waiting for execution capacity.
+_Avoid_: In-memory trigger queue, unlimited backlog
+
+**Expired Run Request**:
+An accepted agent run request that reached its maximum queue wait before execution; it is terminal and will never execute automatically.
+_Avoid_: Delayed request, automatic retry
+
+**Run Request Initiator**:
+The Organization member whose action created an agent run request.
+_Avoid_: Queue owner, Project owner
+
+**Explicit Run Retry**:
+A new agent run request deliberately created after an earlier run failed or timed out.
+_Avoid_: Automatic replay, implicit rerun
+
+**Run Budget**:
+The deterministic resource boundary for one agent run, independent of monetary or token usage alerts.
+_Avoid_: Cost budget, unlimited run
+
+**Emergency Workload Brake**:
+An audited Platform Administrator control that prevents new agent work for one Organization or the whole platform and may explicitly stop active work during an incident.
+_Avoid_: Organization deletion, service restart

Cargo.lock

@@ -264,10 +264,9 @@ dependencies = [
 [[package]]
 name = "citationberg"
 version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "756ff1e3d43a9ecc8183932fb4d9fd3971236f3ce4acb62fe51d1cd43297547d"
+source = "git+https://github.com/typst/citationberg?rev=06a591e2f237d25e1dfdedac3f3d1494c496c52d#06a591e2f237d25e1dfdedac3f3d1494c496c52d"
 dependencies = [
- "quick-xml 0.38.4",
+ "quick-xml",
  "serde",
  "serde_path_to_error",
 ]
@@ -427,6 +426,7 @@ version = "0.0.2"
 dependencies = [
  "cph-diag",
  "serde",
+ "tempfile",
  "toml",
 ]
 
@@ -480,9 +480,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1402,9 +1402,9 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "memmap2"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
 dependencies = [
  "libc",
 ]
@@ -1641,13 +1641,13 @@ checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315"
 
 [[package]]
 name = "plist"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1"
+checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85"
 dependencies = [
  "base64",
  "indexmap",
- "quick-xml 0.39.4",
+ "quick-xml",
  "serde",
  "time",
 ]
@@ -1755,23 +1755,14 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
 
 [[package]]
 name = "quick-xml"
-version = "0.38.4"
+version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
 dependencies = [
  "memchr",
  "serde",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.39.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.45"

Cargo.toml

@@ -21,3 +21,8 @@ serde = { version = "1", features = ["derive"] }
 toml = "0.8"
 cph-diag = { path = "crates/cph-diag" }
 cph-model = { path = "crates/cph-model" }
+
+# citationberg 0.7.0 on crates.io pins vulnerable quick-xml 0.38. Upstream has
+# adopted quick-xml 0.41 but has not published a replacement release yet.
+[patch.crates-io]
+citationberg = { git = "https://github.com/typst/citationberg", rev = "06a591e2f237d25e1dfdedac3f3d1494c496c52d" }

README.md

@@ -41,7 +41,8 @@ crates/            ← 实现:rule-based checker(向 spec 对齐)。见 crates/R
   cph-check / cph-cli                             ← checker 本体 + `cph` 命令行
 render/            ← typst 渲染包 cph-render(母本的渲染后端之一,ADR-0005)
 examples/          ← 样例工程文件(如 TH-141),流水线的真实输入
-(hub/ exporter/ …) ← 将来的其他部件,平级于 spec/。尚未创建
+hub/              ← SaaS Hub:飞书协作、org 管理、agent runtime 与生产部署
+(exporter/ …)     ← 将来的其他部件,平级于 spec/
 ```
 
 `spec/` 与实现部件**物理分离、平级共存**:谁是上游、谁向谁对齐,一眼可见。
@@ -72,3 +73,7 @@ examples/          ← 样例工程文件(如 TH-141),流水线的真实输入
 ## CI
 
 `.gitea/workflows/spec-check.yml` 在每次 push / PR 时于 `spec/` 下跑 `lake build`,确保契约始终 type-check 通过(从第一天起就是"绿"的)。这是良构 gate,见宪法第 2 条。
+
+Rust checker 的本地与 CI 工具链由根 `rust-toolchain.toml` 固定;`.gitea/workflows/checker-check.yml`
+必须安装同一精确版本并执行 `cargo fmt --all --check`、Clippy `-D warnings` 与 workspace
+全测试。升级 Rust 时这两处必须在同一提交更新并通过完整 checker gate。

crates/cph-check/src/lib.rs

@@ -360,8 +360,13 @@ pub fn target_is_shell(root: &Path, target: &str) -> bool {
         .iter()
         .find(|t| t.name == target)
         .is_some_and(|t| {
-            t.steps.iter().any(|s| matches!(s, cph_model::Step::Shell { .. }))
-                && !t.steps.iter().any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
+            t.steps
+                .iter()
+                .any(|s| matches!(s, cph_model::Step::Shell { .. }))
+                && !t
+                    .steps
+                    .iter()
+                    .any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
         })
 }
 
@@ -370,9 +375,17 @@ fn run_one_shell(cwd: &Path, run: &str) -> ShellStepOutcome {
     use std::process::Command;
 
     let output = if cfg!(target_os = "windows") {
-        Command::new("cmd").arg("/C").arg(run).current_dir(cwd).output()
+        Command::new("cmd")
+            .arg("/C")
+            .arg(run)
+            .current_dir(cwd)
+            .output()
     } else {
-        Command::new("sh").arg("-c").arg(run).current_dir(cwd).output()
+        Command::new("sh")
+            .arg("-c")
+            .arg(run)
+            .current_dir(cwd)
+            .output()
     };
 
     match output {
@@ -704,7 +717,11 @@ fn collect_image_assets(eng_root: &Path, build_root: &Path, body: &str) -> Resul
             }
         }
         if let Err(e) = std::fs::copy(&src, &dst) {
-            return Err(format!("failed to copy '{}' → '{}': {e}", src.display(), dst.display()));
+            return Err(format!(
+                "failed to copy '{}' → '{}': {e}",
+                src.display(),
+                dst.display()
+            ));
         }
     }
     Ok(())
@@ -763,7 +780,8 @@ pub fn target_is_markdown_assemble(root: &Path, target: &str) -> bool {
             t.steps
                 .iter()
                 .any(|s| matches!(s, cph_model::Step::AssembleMarkdown { .. }))
-                && !t.steps
+                && !t
+                    .steps
                     .iter()
                     .any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
         })
@@ -833,7 +851,11 @@ fn compile_targets(lesson: &cph_model::Lesson) -> Vec<&str> {
     lesson
         .targets
         .iter()
-        .filter(|t| t.steps.iter().any(|s| matches!(s, cph_model::Step::TypstCompile { .. })))
+        .filter(|t| {
+            t.steps
+                .iter()
+                .any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
+        })
         .map(|t| t.name.as_str())
         .collect()
 }

crates/cph-check/tests/pipeline.rs

@@ -5,7 +5,7 @@
 //! fixtures built in a temp dir. The typst Engine is pointed at the repo's real
 //! `render/` package via `Engine::with_render_dir`.
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use cph_diag::DiagCode;
 use cph_typst::Engine;
@@ -138,7 +138,7 @@ path = "segments/does-not-exist"
 
 /// Write a throwaway one-segment lesson into `tmp`, with the given `[targets.x]`
 /// body spliced in. Returns nothing; the caller runs `check`.
-fn write_segment_lesson_with_target(tmp: &PathBuf, target_body: &str) {
+fn write_segment_lesson_with_target(tmp: &Path, target_body: &str) {
     std::fs::write(
         tmp.join("manifest.toml"),
         format!(
@@ -264,7 +264,7 @@ template = "exports/handout.typ"
 
 /// Write a one-segment lesson with a `file-tree` shell target whose `run` is the
 /// given command. Returns nothing; the caller runs `run_shell_target`.
-fn write_shell_target_lesson(tmp: &PathBuf, run: &str) {
+fn write_shell_target_lesson(tmp: &Path, run: &str) {
     std::fs::write(
         tmp.join("manifest.toml"),
         format!(
@@ -365,7 +365,10 @@ run = "printf SHOULD_NOT_RUN > build/ran.txt"
 
     let report = cph_check::run_shell_target(&tmp, &engine(), "assets");
     assert!(!report.ok);
-    assert!(report.outcomes.is_empty(), "no command should run on a broken lesson");
+    assert!(
+        report.outcomes.is_empty(),
+        "no command should run on a broken lesson"
+    );
     assert!(report.check.has_errors());
     assert!(
         !tmp.join("build").join("ran.txt").exists(),
@@ -380,7 +383,7 @@ run = "printf SHOULD_NOT_RUN > build/ran.txt"
 /// content file, plus a `slides` target that assembles them. The `which` slice
 /// controls which segments get a `slides.md` (so the "skip absent" path can be
 /// exercised). Returns nothing; the caller runs `run_markdown_assemble_target`.
-fn write_markdown_assemble_target_lesson(tmp: &PathBuf, slides: &[(&str, &str)]) {
+fn write_markdown_assemble_target_lesson(tmp: &Path, slides: &[(&str, &str)]) {
     let mut parts = String::new();
     for (i, (name, _)) in slides.iter().enumerate() {
         if i > 0 {
@@ -424,10 +427,7 @@ field = "slides"
 #[test]
 fn target_is_markdown_assemble_detects_shape() {
     let tmp = tempdir();
-    write_markdown_assemble_target_lesson(
-        &tmp,
-        &[("a", "# A"), ("b", "# B")],
-    );
+    write_markdown_assemble_target_lesson(&tmp, &[("a", "# A"), ("b", "# B")]);
     assert!(cph_check::target_is_markdown_assemble(&tmp, "slides"));
     // The student target doesn't exist here → falls through to typst path → false.
     assert!(!cph_check::target_is_markdown_assemble(&tmp, "student"));
@@ -439,7 +439,10 @@ fn run_markdown_assemble_target_concatenates_in_parts_order() {
     let tmp = tempdir();
     write_markdown_assemble_target_lesson(
         &tmp,
-        &[("intro", "# 规则一\n- 范围"), ("rule2", "# 规则二\n$8\\times$")],
+        &[
+            ("intro", "# 规则一\n- 范围"),
+            ("rule2", "# 规则二\n$8\\times$"),
+        ],
     );
 
     let report = cph_check::run_markdown_assemble_target(&tmp, &engine(), "slides");
@@ -667,6 +670,6 @@ fn tempdir() -> PathBuf {
     p
 }
 
-fn cleanup(p: &PathBuf) {
+fn cleanup(p: &Path) {
     let _ = std::fs::remove_dir_all(p);
 }

crates/cph-cli/src/main.rs

@@ -51,12 +51,12 @@ enum Command {
     /// completion file, e.g. `cph completions zsh > ~/.zfunc/_cph`.
     Completions {
         /// Which shell to generate completions for.
-        shell: Shell,
+        shell: CompletionTarget,
     },
 }
 
 #[derive(Debug, Clone, Copy, clap::ValueEnum)]
-enum Shell {
+enum CompletionTarget {
     Bash,
     Zsh,
     Fish,
@@ -82,16 +82,16 @@ fn main() -> ExitCode {
 /// Emit a shell-completion script for `shell` to stdout. The script is built
 /// from the same `Cli` clap definition above, so it tracks subcommands/flags as
 /// they evolve.
-fn run_completions(shell: Shell) -> ExitCode {
+fn run_completions(shell: CompletionTarget) -> ExitCode {
     use clap::CommandFactory;
     use clap_complete::Shell as CompShell;
 
     let sh = match shell {
-        Shell::Bash => CompShell::Bash,
-        Shell::Zsh => CompShell::Zsh,
-        Shell::Fish => CompShell::Fish,
-        Shell::PowerShell => CompShell::PowerShell,
-        Shell::Elvish => CompShell::Elvish,
+        CompletionTarget::Bash => CompShell::Bash,
+        CompletionTarget::Zsh => CompShell::Zsh,
+        CompletionTarget::Fish => CompShell::Fish,
+        CompletionTarget::PowerShell => CompShell::PowerShell,
+        CompletionTarget::Elvish => CompShell::Elvish,
     };
     let mut cmd = Cli::command();
     clap_complete::generate(sh, &mut cmd, "cph", &mut std::io::stdout());
@@ -188,7 +188,10 @@ fn run_build(
 /// explicit `cph build --target <name>`, and each command is printed before it
 /// runs so the user sees exactly what is executed.
 fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> ExitCode {
-    eprintln!("running shell-step target '{target}' (commands execute in {})", path.display());
+    eprintln!(
+        "running shell-step target '{target}' (commands execute in {})",
+        path.display()
+    );
     let report = cph_check::run_shell_target(path, engine, target);
     print_diagnostics(&report.check);
 
@@ -196,7 +199,10 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
         // Refused before running anything (check errors / unknown target / no
         // shell steps): the diagnostics above explain why.
         if report.check.has_errors() {
-            eprintln!("build refused: {} errors (fix the lesson first)", report.check.error_count());
+            eprintln!(
+                "build refused: {} errors (fix the lesson first)",
+                report.check.error_count()
+            );
         } else {
             eprintln!("build failed: target '{target}' has no shell steps to run");
         }
@@ -212,13 +218,19 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
             eprint!("{}", outcome.stderr);
             eprintln!(
                 "step failed (exit {})",
-                outcome.status.map(|c| c.to_string()).unwrap_or_else(|| "signal".into())
+                outcome
+                    .status
+                    .map(|c| c.to_string())
+                    .unwrap_or_else(|| "signal".into())
             );
         }
     }
 
     if report.ok {
-        println!("shell-step target '{target}' completed: {} step(s) ok", report.outcomes.len());
+        println!(
+            "shell-step target '{target}' completed: {} step(s) ok",
+            report.outcomes.len()
+        );
         ExitCode::SUCCESS
     } else {
         eprintln!("shell-step target '{target}' failed");
@@ -231,13 +243,19 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
 /// shell path, this is opt-in — only on an explicit `cph build --target <name>`,
 /// and never in `check`.
 fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target: &str) -> ExitCode {
-    eprintln!("assembling markdown target '{target}' (reading parts under {})", path.display());
+    eprintln!(
+        "assembling markdown target '{target}' (reading parts under {})",
+        path.display()
+    );
     let report = cph_check::run_markdown_assemble_target(path, engine, target);
     print_diagnostics(&report.check);
 
     if report.outcomes.is_empty() && !report.ok {
         if report.check.has_errors() {
-            eprintln!("build refused: {} errors (fix the lesson first)", report.check.error_count());
+            eprintln!(
+                "build refused: {} errors (fix the lesson first)",
+                report.check.error_count()
+            );
         } else {
             eprintln!("build failed: target '{target}' has no assemble-markdown steps to run");
         }
@@ -250,7 +268,11 @@ fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target:
             outcome.field, outcome.parts_read
         );
         if let Some(out_rel) = &outcome.written {
-            println!("wrote {} ({} bytes)", path.join(out_rel).display(), outcome.body.len());
+            println!(
+                "wrote {} ({} bytes)",
+                path.join(out_rel).display(),
+                outcome.body.len()
+            );
         }
         if let Some(err) = &outcome.error {
             eprintln!("assemble failed: {err}");
@@ -258,7 +280,10 @@ fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target:
     }
 
     if report.ok {
-        println!("assemble-markdown target '{target}' completed: {} step(s) ok", report.outcomes.len());
+        println!(
+            "assemble-markdown target '{target}' completed: {} step(s) ok",
+            report.outcomes.len()
+        );
         ExitCode::SUCCESS
     } else {
         eprintln!("assemble-markdown target '{target}' failed");

crates/cph-model/Cargo.toml

@@ -10,3 +10,6 @@ serde = { workspace = true }
 # `preserve_order` keeps `[targets.*]` tables in TOML document order, which is
 # the declared order this crate exposes via `Lesson.targets` (ADR-0009).
 toml = { workspace = true, features = ["preserve_order"] }
+
+[dev-dependencies]
+tempfile = "3"

crates/cph-model/src/lib.rs

@@ -737,7 +737,11 @@ fn parse_target(name: String, value: toml::Value, diags: &mut Vec<Diagnostic>) -
 /// A well-formed but empty `covers = []` yields `Some(vec![])`: a target that
 /// explicitly renders no kind (every used kind then draws a `renderIgnored`
 /// warning) — distinct from an absent key.
-fn parse_covers(name: &str, value: toml::Value, diags: &mut Vec<Diagnostic>) -> Option<Vec<String>> {
+fn parse_covers(
+    name: &str,
+    value: toml::Value,
+    diags: &mut Vec<Diagnostic>,
+) -> Option<Vec<String>> {
     let items = match value {
         toml::Value::Array(items) => items,
         _ => {

crates/cph-model/tests/load.rs

@@ -279,16 +279,15 @@ fn malformed_target_config_is_non_fatal_with_schema_violations() {
 /// match exactly (ADR-0016's MVP rule).
 const CPH_VERSION: &str = env!("CARGO_PKG_VERSION");
 
-/// Write a one-part lesson into a temp dir, optionally with a `.cph-version`
-/// file. Returns the temp dir so the caller runs `load`.
-fn tmp_lesson_with_version(version: Option<&str>) -> PathBuf {
-    let mut p = std::env::temp_dir();
-    let nanos = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap()
-        .as_nanos();
-    p.push(format!("cph-version-test-{nanos}"));
-    std::fs::create_dir_all(&p).unwrap();
+/// Write a one-part lesson into a uniquely-created temp dir, optionally with a
+/// `.cph-version` file. `TempDir` owns cleanup so parallel tests cannot delete
+/// one another's fixture after a timestamp collision.
+fn tmp_lesson_with_version(version: Option<&str>) -> tempfile::TempDir {
+    let tmp = tempfile::Builder::new()
+        .prefix("cph-version-test-")
+        .tempdir()
+        .unwrap();
+    let p = tmp.path();
     std::fs::write(
         p.join("manifest.toml"),
         "[project]\nid = \"v\"\nname = \"v\"\n[info]\ntitle = \"v\"\n[[parts]]\nkind = \"segment\"\npath = \"segments/a\"\n",
@@ -301,14 +300,13 @@ fn tmp_lesson_with_version(version: Option<&str>) -> PathBuf {
     if let Some(v) = version {
         std::fs::write(p.join(".cph-version"), v).unwrap();
     }
-    p
+    tmp
 }
 
 #[test]
 fn cph_version_matching_is_not_a_diagnostic() {
     let tmp = tmp_lesson_with_version(Some(CPH_VERSION));
-    let (_lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (_lesson, diags) = load(tmp.path());
     assert!(
         diags.iter().all(|d| d.code != DiagCode::CphVersionMismatch),
         "a matching .cph-version must not warn, got {diags:?}"
@@ -318,8 +316,7 @@ fn cph_version_matching_is_not_a_diagnostic() {
 #[test]
 fn cph_version_mismatch_is_an_error_diagnostic() {
     let tmp = tmp_lesson_with_version(Some("99.99.99"));
-    let (lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (lesson, diags) = load(tmp.path());
     // The lesson still loads (so other defects could surface), but the
     // mismatch is an error diagnostic — which alone makes it illegal.
     assert!(lesson.is_some(), "a mismatch should not halt loading");
@@ -327,7 +324,11 @@ fn cph_version_mismatch_is_an_error_diagnostic() {
         .iter()
         .filter(|d| d.code == DiagCode::CphVersionMismatch)
         .collect();
-    assert_eq!(mm.len(), 1, "expected one CphVersionMismatch, got {diags:?}");
+    assert_eq!(
+        mm.len(),
+        1,
+        "expected one CphVersionMismatch, got {diags:?}"
+    );
     assert!(
         mm[0].message.contains("99.99.99") && mm[0].message.contains(CPH_VERSION),
         "diagnostic should name both versions, got: {}",
@@ -339,8 +340,7 @@ fn cph_version_mismatch_is_an_error_diagnostic() {
 fn cph_version_missing_is_skipped() {
     // No `.cph-version` file at all → skipped (ADR-0016 migration period; OPEN).
     let tmp = tmp_lesson_with_version(None);
-    let (_lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (_lesson, diags) = load(tmp.path());
     assert!(
         diags.iter().all(|d| d.code != DiagCode::CphVersionMismatch),
         "a missing .cph-version must be skipped, got {diags:?}"
@@ -350,8 +350,7 @@ fn cph_version_missing_is_skipped() {
 #[test]
 fn cph_version_empty_is_an_error() {
     let tmp = tmp_lesson_with_version(Some("   \n"));
-    let (_lesson, diags) = load(&tmp);
-    let _ = std::fs::remove_dir_all(&tmp);
+    let (_lesson, diags) = load(tmp.path());
     assert!(
         diags
             .iter()

crates/cph-schema/src/lib.rs

@@ -526,7 +526,11 @@ mod tests {
         // ADR-0015: `slides`/`transcript` are markdown content fields (`x-cph-content: "md"`),
         // distinct from the `.typ` content fields.
         let s = schema_for("segment").unwrap();
-        let slides = s.content_fields.iter().find(|f| f.name == "slides").unwrap();
+        let slides = s
+            .content_fields
+            .iter()
+            .find(|f| f.name == "slides")
+            .unwrap();
         assert_eq!(slides.ext, "md");
         assert!(!slides.required);
         let transcript = s

crates/cph-typst/src/embedded.rs

@@ -48,7 +48,8 @@ pub fn resolve_render_dir() -> PathBuf {
     ensure_extracted().unwrap_or_else(|_| {
         // Last-resort: extract under the OS temp dir. Still correct, just not
         // cached across processes.
-        let fallback = std::env::temp_dir().join(format!("cph-render-{}", env!("CARGO_PKG_VERSION")));
+        let fallback =
+            std::env::temp_dir().join(format!("cph-render-{}", env!("CARGO_PKG_VERSION")));
         let _ = extract_to(&fallback);
         fallback
     })

docs/adr/0001-project-group-is-collaboration-space.md

@@ -26,4 +26,6 @@ The group can stay open while no Claude processing is active. A teacher leaving
 - Multi-open is natural: a teacher participates in multiple project groups.
 - Normal group discussion does not occupy the project.
 - "Exit project" should not be the normal action in the group workflow.
-- Project rebinding or group archival needs explicit product rules, separate from run cancellation.
+- ADR-0021 adds those explicit product rules: active project↔Feishu chat binding
+  is strict 1:1, while archived historical bindings are retained so org admins
+  can correct pilot setup mistakes without losing audit/session history.

docs/adr/0017-agent-session-is-provider-bound.md

@@ -1,4 +1,4 @@
-# ADR 0017: AgentSession Is Provider-Bound
+# ADR 0017: Agent Runtime — Claude Code SDK via OpenRouter
 
 ## Status
 
@@ -6,42 +6,90 @@ Accepted.
 
 ## Context
 
-ADR-0002 says a long-lived `AgentSession` can be reused by many runs, but never
-addressed the provider/model dimension. The agent layer is provider-agnostic
-(ADR-0001..0003 never required Claude specifically; the `@Claude` trigger name
-is a product brand, not a provider commitment). In practice the Hub routes
-model calls through an OpenAI-compatible client (e.g. OpenRouter), so a run may
-target different models — and the motivation for multi-model is **role-based
-routing**: drafting, reviewing, and triage suit different models.
+The original ADR-0017 mandated a provider-agnostic agent layer: any
+OpenAI-compatible model via OpenRouter, custom agent loop built on Vercel AI
+SDK's `streamText`. The motivation was to avoid vendor lock-in.
 
-That raises the question ADR-0002 left open: when a teacher switches model
-mid-project, does the `AgentSession` carry live context across the switch, or
-is it bound to a single provider/model?
+In practice, the hand-rolled agent loop lacked capabilities that a production
+agent needs: context compaction (long conversations), tool approval flows,
+partial-message streaming, and a battle-tested multi-turn loop. Building these
+ourselves meant re-implementing what the Claude Code SDK already provides.
+
+The Feishu integration also proved costly to hand-roll — card schema quirks,
+patch limitations, file support — each API quirk cost a round-trip.
 
 ## Decision
 
-An `AgentSession` is **provider/model-bound**. A model or provider switch
-starts a new `AgentSession`. The `AgentSession` does not carry live context
-across a switch.
+Adopt `@anthropic-ai/claude-agent-sdk` as the agent runtime, routed through
+OpenRouter's **"Anthropic Skin"** (`https://openrouter.ai/api`).
 
-Cross-session continuity is not the session's job — it is carried by ADR-0003's
-project memory and anchors, which the Hub reads on demand when seeding a new
-run. This keeps B consistent with ADR-0003 ("the Hub avoids becoming a full
-chat history store"): session continuity and history continuity are the same
-mechanism, both via memory/anchors, never via a live cross-provider session.
+OpenRouter exposes an Anthropic Messages API-compatible endpoint. Claude Code
+SDK speaks its native protocol directly to OpenRouter — no proxy, no format
+conversion. The SDK's `query()` owns the agent loop (tool dispatch, compaction,
+streaming). Built-in tools (Read, Write, Bash, Glob, Grep) cover the entire
+tool surface — no custom tools needed. `cph check` / `cph build` run via Bash.
 
-Per-run model selection (the run carries its model) is the switching point.
+The Hub `AgentSession` is provider/role/model-bound, and it must persist the
+provider runtime cursor needed to continue a conversation. For Claude Code SDK,
+that cursor is the `result.session_id`; store it in `AgentSession.metadata` as
+`claudeSessionId` and pass it back to the next `query()` call as
+`options.resume`. Role is part of the session binding because role prompts and
+tool surfaces can differ even when the underlying model is the same. A Feishu
+project group's active binding selects one Organization role for ordinary
+messages. Switching that selection routes future messages to the selected
+role's own session; it never mutates or merges provider cursors across roles.
+Work freezes the selected role when accepted so queued requests cannot drift
+after a later switch.
+
+Role definitions are Organization-scoped runtime data. A role bundle selects
+its default model, system prompt, tool allowlist and installed Agent skill
+versions. PostgreSQL is authoritative for role composition and skill metadata;
+skill bytes live in a content-addressed persistent store selected only by the
+recorded SHA-256 digest. Updating a role or binding skills takes effect without
+a Hub release or process restart. A change to the role's execution surface
+(model, prompt, tools, selected skill content) archives its active sessions so
+the next run cannot resume a provider context created under stale instructions;
+label and ordering-only changes preserve conversational continuity.
+
+Exactly one active role per Organization is the default used when a project
+group is first bound. The default is configuration data, not a hard-coded role
+name. Roles are selected through the Hub project console; role ids are not
+public slash commands. A project participant may change the group's shared
+selection only when both `agent.trigger` for the project and `role.trigger` for
+the target role authorize that actor. The selection affects every participant's
+future messages, while already accepted work keeps its frozen role.
+
+Hub slash commands are a closed control-plane protocol. Unknown commands fail
+explicitly and are never downgraded to Agent text. Claude SDK session commands
+are invoked only through typed Hub actions. In particular, compaction resumes
+the selected role session and sends the exact `/compact` prompt without
+prepending Feishu context.
+
+Environment variables:
+```
+ANTHROPIC_BASE_URL=https://openrouter.ai/api
+ANTHROPIC_AUTH_TOKEN=<OpenRouter API key>
+ANTHROPIC_API_KEY=""  # must be explicitly empty
+```
+
+Model routing: `ANTHROPIC_DEFAULT_SONNET_MODEL` etc. accept OpenRouter model
+IDs (e.g. `z-ai/glm-4.7`, `anthropic/claude-sonnet-4-20250514`). This gives
+limited OpenRouter-level model routing, but the runtime is still Claude Agent
+SDK-shaped: non-Claude models are best-effort and may not support every
+Claude Code feature.
 
 ## Consequences
 
-- Switching model mid-project = new session; the new run seeds from project
-  memory/anchors (ADR-0003), not the prior session's live context.
-- The agent layer is provider-agnostic: the agent loop is ours; model calls go
-  through an OpenAI-compatible client. No hard dependency on a single-vendor
-  agent SDK.
-- Role-based model routing (different run kinds default to different models) is
-  a product/admin configuration concern, not a spec invariant.
-- "AgentSession reused by many runs" (ADR-0002) holds *within* one
-  provider/model; it does not span a switch.
-- The `@Claude` trigger name remains the product mention brand regardless of the
-  backing model.
+- Provider-agnosticism is not fully preserved. OpenRouter can route to GLM,
+  Claude, GPT, etc., but the runtime protocol and agent loop remain Claude
+  Agent SDK-bound.
+- The agent loop is Claude Code SDK's (compaction, tool approval, partial
+  message streaming) — not hand-rolled.
+- Custom tools (read_file, write_file, cph_check, cph_build) are replaced by
+  the SDK's built-in Read/Write/Bash/Glob/Grep.
+- The SSE event format matches Claude-to-IM's expected protocol natively.
+- Per-run model selection works via role→model mapping; models are OpenRouter
+  IDs, not Anthropic-only aliases.
+- OpenRouter recommends setting Anthropic as the top-priority provider for
+  Claude Code; non-Anthropic models may have reduced compatibility with some
+  SDK features (e.g. thinking blocks).

docs/adr/0018-agent-execution-surface-bounded-by-workspace.md

@@ -0,0 +1,211 @@
+# ADR 0018: The Agent Execution Surface Is Bounded By The Project Workspace
+
+## Status
+
+Accepted, and implemented. The mechanism is settled: the Claude Code SDK's
+built-in sandbox (bubblewrap on Linux, seatbelt on macOS) confines the agent
+process and its Bash subprocesses at the OS level. `bypassPermissions`
+remains (headless server — no interactive prompts), but the sandbox is the
+hard boundary, not the permission-prompt layer. Constitution rule 4
+divergence (ADR-0017's unbounded tools) is resolved.
+
+Builds on **ADR-0001** (project group as collaboration space), **ADR-0002**
+(project lock scoped to `run_id`), **ADR-0004** (permission grants govern
+`triggerAgent`), **ADR-0007** (the engineering file is a directory tree on
+disk), and **ADR-0017** (Claude Code SDK as the agent runtime).
+
+## Context
+
+ADR-0001/0002/0004 together cover *collaboration governance* up to the
+moment a teacher `@Claude`s: who may trigger an `AgentRun`, that the run
+holds a project-scoped lock, that the group is a long-lived space. They stop
+at `triggerAgent`. What the agent may do *after* the trigger — which files it
+may read or write, what commands it may run — is not named in any ADR, not
+modeled in `spec/`, and not enforced by any contract-level invariant.
+
+The gap is not theoretical. ADR-0017 adopted the Claude Code SDK with
+`allowedTools: ["Read", "Write", "Bash", "Glob", "Grep"]` and
+`permissionMode: "bypassPermissions"`. Under that configuration the SDK's
+built-in tools operate over the whole host filesystem with no confinement, and
+`Bash` executes arbitrary shell with no approval gate. The Hub's
+`hub/src/agent/workspace.ts` was *intended* to be "the agent's only file
+surface" (its module doc says so) and ships a `confine()` path validator that
+rejects `..` and absolute escapes. But ADR-0017's cutover left those custom
+tools unwired — the SDK's built-in tools replaced them — so `confine()` is
+dead code and the boundary it was meant to enforce does not exist at runtime.
+Compounding this, the systemd unit's default `SERVICE_USER` is `root`
+(`hub/deploy/install_service.sh`), so the unbounded agent runs as root.
+
+This is a real divergence point by the constitution's test: "if unwritten,
+will the developer and the agent make different assumptions?" They already
+have. ADR-0017 assumed the boundary was an implementation concern; the
+`workspace.ts` author assumed it was a contract. The next person to touch the
+runner has nothing telling them the boundary must be enforced, and ADR-0017
+silently overrode the one that existed.
+
+## Decision
+
+### The agent execution surface is bounded by the run's project workspace
+
+During an `AgentRun`, every file operation the agent performs — read, write,
+glob, grep target — must fall **within the run's project workspace directory**
+(the on-disk engineering-file tree of ADR-0007). Operations whose target path
+escapes that tree are refused. This is a platform core invariant, pinned in
+`Spec.System.AgentSurface`.
+
+The boundary is **per-run, anchored to the run's project**, the same scope as
+the Lock (ADR-0002). This keeps the two invariants composable:
+
+- `Lock` (ADR-0002) bounds **concurrency** — who may be modifying the project.
+- `AgentSurface` (this ADR) bounds **blast radius** — what the agent may touch.
+
+Both scope to `run × project`. Acquiring the lock and entering the surface
+happen together at run start; releasing the lock and leaving the surface
+happen together at run termination.
+
+### Shell execution is subject to the same boundary, by effect
+
+A shell command the agent runs executes with the workspace as its working
+directory (already true for the `cph` subprocess, ADR-0016) and its **file
+effects** — files read, written, created, or deleted — are subject to the same
+workspace boundary. The agent may invoke **named external tools** whose
+binaries live outside the workspace (notably `cph` via `CPH_BIN`); the
+boundary governs *file effects*, not *binary location*. Arbitrary
+`cat /etc/shadow` or `curl … > /etc/...` is out of bounds regardless of how it
+is invoked.
+
+### The Hub process runs as a non-privileged service user
+
+The systemd unit must not run as `root`. The default in
+`hub/deploy/install_service.sh` changes from `root` to a dedicated unprivileged
+user. This is a deployment invariant, recorded here because the unbounded-agent
+risk is compounded by root; it is enforced in the unit, not in `spec/` (it is
+not a semantic divergence between developer and agent — it is operational
+discipline).
+
+### The mechanism: SDK sandbox (OS-level, per agent process)
+
+The boundary is enforced by the Claude Code SDK's built-in sandbox
+(bubblewrap plus `socat` on Linux, seatbelt on macOS), configured via `query()` options in
+`hub/src/agent/runner.ts`:
+
+- `sandbox.enabled: true` + `failIfUnavailable: true` — the agent process and
+  its Bash subprocesses run sandboxed; if the sandbox can't start, `query()`
+  emits an error and exits rather than running unsandboxed.
+- `sandbox.allowUnsandboxedCommands: false` — a tool cannot opt out with the
+  SDK's `dangerouslyDisableSandbox` input. Claude Code 2.1.202 does not enforce
+  that option reliably, so a host-side `PreToolUse` hook also denies every
+  Bash request whose input explicitly sets `dangerouslyDisableSandbox: true`
+  before a process can start.
+- `sandbox.filesystem.denyRead: ["/"]` with `allowRead` for the canonical
+  current workspace and a small named system-runtime set — normal reads stay
+  in the run's workspace while `/bin`, shared libraries, CA certificates,
+  fonts and the configured `cph` executable remain available as the external
+  tool exception described above.
+- `sandbox.filesystem.allowWrite: [workspaceDir]` confines persistent host
+  effects to the canonical ADR-0007 workspace. Config/cache/home stay beneath
+  `.cph/agent-runtime/`; `TMPDIR`, `TMP`, `TEMP`, and `CLAUDE_CODE_TMPDIR` all
+  point at the workspace-local `.cph/t`. The workspace allocator uses stable
+  compact Organization/Project path segments, deployment requires a short
+  workspace root, and the canonical temp prefix fails fast above 56 bytes so
+  the SDK can append randomized `socat` bridge socket names without exceeding
+  Linux `sockaddr_un.sun_path`. Bubblewrap shadows non-allowlisted host trees
+  with disposable tmpfs mounts: a shell write there may succeed inside that
+  private namespace, but it cannot mutate the corresponding host path. The
+  Linux proof checks host state after the sandbox exits.
+- The SDK subprocess environment replaces rather than spreads `process.env`.
+  Only provider protocol variables and non-secret runtime variables cross the
+  boundary; database, Feishu and Hub session credentials never enter it.
+  `sandbox.credentials` denies provider token variables inside Bash and denies
+  configured credential files. `CPH_SANDBOX_EXTRA_DENY_READ` names additional
+  deployment-specific secret paths on the Hub side only.
+- `settingSources: []` and strict MCP configuration prevent an untrusted
+  workspace or service-user config from widening tools, hooks, MCP servers, or
+  sandbox paths.
+- Agent skills are Organization-scoped runtime configuration, not Hub release
+  assets. Skill content is imported into a content-addressed persistent store
+  through a shared ingestion pipeline (`importSkillDirectory` for host-console
+  CLI, `importSkillFromFiles` for org-admin web surface) that enforces the same
+  safety checks: `SKILL.md` manifest required, 512-file / 16-byte limits,
+  symlink rejection, SHA-256 content addressing. The web surface
+  (`OrganizationAgentConfiguration.installSkillFromFiles`) writes to the same
+  store as the host-console CLI (`installSkill`); both flow through
+  `commitSkillContent` for deduplication and atomic rename into
+  `versions/<digest>/`. Org-admin authentication gates the web surface; the
+  content-addressed store remains platform-controlled. A role selects enabled
+  Organization skills alongside its model, system prompt and tool allowlist.
+  Each run copies only those selected immutable versions into a run-scoped
+  plugin outside the project workspace; the sandbox exposes that snapshot
+  read-only and deletes it after the run. SDK-bundled skills and filesystem
+  setting sources remain disabled, so project `.claude` content cannot register
+  skills or widen tools. Requested skill versions are recorded on
+  `run.created`; SDK initialization remains authoritative loading evidence.
+- Network: open (see Open Questions).
+
+`bypassPermissions` is kept (headless server — no interactive prompts); the
+sandbox is the hard boundary, not the permission-prompt layer. This is
+subprocess-level, not process-level: the Hub host process is not sandboxed
+(it manages DB, Feishu ws, logs), only the Claude Code agent process and its
+children. This keeps the sandbox config surface small and lets future
+external CLIs (Gitea, Lark) run inside the sandbox by giving their binaries
+read access, without widening the Hub's own filesystem surface.
+
+The contract pins the **invariant** (operations stay in the workspace), not
+the **mechanism**. The SDK sandbox is the current mechanism; switching to a
+different one (tool wrappers, container) would not be a spec change as long
+as it upholds `AgentFileOp.Authorized`.
+
+## Consequences
+
+- `Spec.System.AgentSurface` gains an `AgentFileOp` structure and an
+  `Authorized` predicate mirroring `Memory.McpReadRequest.Authorized`: the op
+  carries a `run` and a `path`; `Authorized` holds iff the path falls within
+  the run's workspace, via platform-provided `runWorkspace` and `pathWithin`
+  (representations `OPEN`).
+- `hub/src/agent/workspace.ts`'s `confine()` intent is now contract-backed.
+  Re-wiring it (or replacing it with an OS sandbox) is **mandatory**, not a
+  preference. Its current dead-code status under ADR-0017 is a divergence to
+  be addressed, not silently retained.
+- ADR-0017's `bypassPermissions` + unrestricted built-in tools is a
+  **divergence from this ADR** and is surfaced as such (constitution rule 4).
+  The reconciliation — restrict the SDK tool set, wrap Bash, or sandbox at the
+  OS level — is the open mechanism question above. Whichever is chosen must
+  uphold the `AgentSurface` invariant.
+- Inbound file/image delivery (`hub/src/feishu/trigger.ts`) lands files under
+  `${workspaceDir}/.cph/inbox/`, which is inside the boundary — consistent, no
+  change to the delivery path.
+- The `cph` subprocess (ADR-0016) already runs with `cwd = workspaceDir`
+  (`hub/src/agent/cph.ts`), so it is inside the boundary by construction.
+
+## Open Questions / Deferred
+
+- **Network egress.** Decided: network is **open** in the initial
+  production deployment. The sandbox confines filesystem writes to the
+  workspace and denies reads of sensitive host paths, but does not restrict
+  outbound network. Rationale: future Gitea/Lark CLI integration needs
+  unconstrained network egress; `cph build` may fetch typst packages. The
+  exfiltration risk (agent `curl`s workspace content out) is accepted as a
+  tradeoff for CLI integration simplicity. This is revisitable — a network
+  allowlist (`sandbox.network.allowedDomains`) can be added without a spec
+  change if the threat model tightens.
+- **Named system-runtime read set.** The implementation allows only the
+  platform runtime directories/files needed for shell, dynamic libraries,
+  fonts, TLS/DNS and `cph` execution in addition to the project workspace.
+  Expanding that set requires an explicit reviewed code change. Deployments
+  name credential files with `CPH_SANDBOX_EXTRA_DENY_READ`; these paths are
+  protected through the SDK credential layer and are not passed to the Agent
+  environment. The exact runtime set remains plumbing and is not spec-pinned.
+- **Shell command vocabulary.** Whether the agent's Bash is restricted to a
+  whitelist (e.g. `cph`, `ls`, `grep`) or allowed arbitrary commands whose
+  *effects* are then confined, is a policy choice left to implementation. The
+  invariant governs effects either way; current choice is arbitrary commands
+  (sandbox confines effects).
+
+## Related Decisions
+
+- **Rate limiting and token usage.** ADR-0022 requires multi-dimensional hard
+  request limits while keeping token thresholds as attributed soft alerts in
+  the initial release.
+- **Inbound file limits.** ADR-0022 requires hard file, attachment, archive,
+  Project-storage, and Organization-storage limits with explicit failure and
+  partial-file cleanup.

docs/adr/0019-principal-set-team-permissions.md

@@ -0,0 +1,129 @@
+# ADR 0019: Resolve Actors to Principal Sets for Team Permissions
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0004 deliberately chose a Feishu Docs-like permission model:
+`grant(resource, principal, role)` plus separate resource settings. The first
+Hub implementation stored `principal` as an opaque string and the Feishu trigger
+used the sender's `open_id` directly. That was enough for individual teacher
+grants, but not for production collaboration:
+
+- a curriculum team needs one grant to apply to many teachers;
+- Feishu departments, user groups, and project chats should be grantable
+  principals;
+- role-specific agent grants (`/review`, `/draft`) must compose with project
+  permissions consistently;
+- audit/debug output must explain which principal and grant allowed or denied a
+  run.
+
+## Decision
+
+Introduce typed principals and resolve every actor to a principal set before
+authorization.
+
+### Principal types
+
+`PrincipalType` is:
+
+- `USER`: a Feishu user, identified by `feishuOpenId`.
+- `TEAM`: a Hub-managed teacher team.
+- `FEISHU_CHAT`: a Feishu chat/group.
+- `FEISHU_DEPARTMENT`: a Feishu department.
+- `FEISHU_USER_GROUP`: a Feishu user group.
+- `APP`: an integration or bot principal.
+
+`PermissionGrant` and `RoleTriggerGrant` store `principalType` and
+`principalId`. Legacy opaque principal strings are backfilled as
+`USER/<old principal>`.
+
+### Team membership
+
+Hub teams are first-class. A user can be a direct active member of a team. A
+team can also be bound to external Feishu principals: chat, department, or user
+group. If an actor resolves to any bound external principal, the actor also
+resolves to that Hub team.
+
+Membership changes are effective immediately because authorization resolves the
+principal set at request time.
+
+### External Feishu principal sync
+
+Feishu department, user group, and chat memberships are not scattered through
+callers. They are synchronized into `ExternalPrincipalMembership` rows and then
+read by `PrincipalResolver`. The Feishu message context can also contribute the
+current `FEISHU_CHAT` principal directly because receiving a group event is
+already contextual proof that the actor is speaking in that group.
+
+### Grant merging
+
+Authorization evaluates all active grants matching any resolved principal for
+the requested resource. There is no explicit deny in this ADR. Among matching
+grants, the highest role wins:
+
+```text
+READ < EDIT < MANAGE
+```
+
+Action thresholds:
+
+- project read requires `READ`.
+- project edit requires `EDIT`.
+- collaborator management requires `MANAGE`.
+- agent trigger requires `EDIT`.
+- normal agent cancel requires `MANAGE`.
+
+`PermissionSettings` never grants access beyond grants. Settings only constrain
+an already-granted capability. For `agentTrigger`:
+
+- missing setting or `ROLE` means role-derived grants decide;
+- `MANAGE_ONLY` raises the threshold to `MANAGE`;
+- `DISABLED` denies the action.
+
+Other settings keep their existing string storage, but the same rule applies:
+settings are policy constraints, not positive grants.
+
+### Role-trigger composition
+
+Role-trigger grants are a second gate after the project-level `agent.trigger`
+gate:
+
+1. The actor must pass project `agent.trigger` for the project.
+2. If no `RoleTriggerGrant` row has ever existed for `(projectId, roleId)`,
+   the role is open for backward compatibility.
+3. Once any row exists for `(projectId, roleId)`, including a revoked row, the
+   role is configured. The actor must match an active role grant by any resolved
+   principal.
+
+This keeps `/review` restrictions orthogonal to ordinary edit access while
+preventing a fully revoked role from silently reopening.
+
+### Module seam
+
+Callers use a single authorization module:
+
+```ts
+authorizer.can({
+  actor,
+  action,
+  resource,
+  roleId,
+})
+```
+
+The caller does not know how users, teams, Feishu departments, user groups, or
+chats expand. `PrincipalResolver` owns that implementation, and
+`PermissionAuthorizer` owns grant/settings/role-trigger composition.
+
+## Consequences
+
+- Teacher teams become production-grade principals rather than UI sugar.
+- Feishu external organization concepts can be synchronized and granted without
+  leaking Feishu-specific checks into the trigger path.
+- A single audit decision can report actor, principal set, matched grant, and
+  matched role grant.
+- Backward compatibility is explicit: old principal strings become user
+  principals, and unconfigured role grants remain open.

docs/adr/0020-organization-is-saas-tenant-root.md

@@ -0,0 +1,94 @@
+# ADR 0020: Organization Is The SaaS Tenant Root
+
+## Status
+
+Accepted.
+
+## Context
+
+The Hub is moving from a single-organization deployment assumption toward a
+SaaS shape. The existing permission model already supports team principals via
+`PermissionGrant(PROJECT, TEAM, role)`, but without a tenant root the data model
+cannot safely answer:
+
+- which customer owns a project;
+- which organization a team belongs to;
+- whether a team grant crosses customer boundaries;
+- which external directory sync produced a Feishu department/user-group
+  membership;
+- where future org admin, billing, audit, and platform break-glass controls
+  attach.
+
+Retrofitting this after projects, teams, runs, and audit records grow would
+force tenant assumptions into every caller.
+
+## Decision
+
+Introduce `Organization` as the SaaS tenant root.
+
+Customer-side collaboration data is organization-scoped:
+
+```text
+Organization
+  OrganizationMembership
+  Team
+  TeamMembership
+  Project
+  ExternalDirectoryConnection
+  PermissionGrant
+```
+
+`User` remains global. A user can belong to multiple organizations through
+`OrganizationMembership`. Project-level authorization still uses
+`PermissionGrant` and `PermissionAuthorizer`; the authorizer derives the
+organization from the requested resource, then resolves teams and external
+principals in that organization scope.
+
+`Team` belongs to exactly one organization. `Project` belongs to exactly one
+organization. A `PermissionGrant` that grants a `TEAM` principal on a `PROJECT`
+resource is valid only when the team and project are in the same organization.
+This invariant is pinned in `Spec.System.Organization`.
+
+External directory membership is scoped through `ExternalDirectoryConnection`:
+
+```text
+ExternalDirectoryConnection(
+  organizationId,
+  provider,
+  providerTenantId,
+  source
+)
+```
+
+`ExternalPrincipalMembership` points at a connection rather than carrying a
+free-floating `source` string. `source` names a sync pipeline; it is not the
+tenant identity.
+
+The SaaS platform control plane is separate from customer project permissions.
+`READ/EDIT/MANAGE` remains the customer-side project permission lattice.
+Platform staff access, tenant suspension, billing, and break-glass access are a
+separate control plane and must not be modeled as project `MANAGE`.
+
+## Consequences
+
+- Existing single-org data is backfilled into a default organization during
+  migration.
+- `Team.slug` becomes organization-scoped instead of globally unique.
+- Actor resolution is organization-aware: direct teams and synchronized external
+  principals only expand inside the resource's organization.
+- Direct `USER` grants remain possible because a global user can collaborate
+  across organizations when explicitly granted.
+- Future org admin UI can be built on this model without changing the
+  authorization core.
+
+## Open Questions / Deferred
+
+- Full platform admin UI, billing, and tenant suspension workflows are
+  deferred. Production safety limits and workload brakes are no longer
+  deferred; ADR-0022 defines them without treating a workload brake as tenant
+  suspension.
+- Whether direct `USER` project grants should require active
+  `OrganizationMembership` is deferred; explicit direct grants remain allowed
+  for now.
+- Provider-specific directory semantics beyond Feishu are deferred, but the
+  connection model leaves room for them.

docs/adr/0021-org-admin-project-onboarding.md

@@ -0,0 +1,117 @@
+# ADR 0021: Org Admin Project Onboarding
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0020 introduced `Organization` as the SaaS tenant root. The next product
+surface is not only the Feishu bot trigger path, but also the control planes
+around it:
+
+- platform staff need a private platform admin area to create and operate orgs;
+- customer org owners/admins need an org admin area for teams, roles, provider
+  mode and BYOK configuration, projects, folders, sessions, and usage accounting;
+- ordinary teachers should be able to start work from the natural Feishu group
+  flow without entering the web backend.
+
+Without a crisp project onboarding model, project creation, Feishu chat binding,
+permissions, session history, and usage accounting would drift into separate
+ad-hoc rules.
+
+## Decision
+
+Use one web app with two separate admin areas:
+
+```text
+/admin/platform       internal platform admins only
+/admin/org/:orgSlug   customer org OWNER/ADMIN only
+```
+
+The guards are intentionally separate:
+
+- `requirePlatformAdmin` for internal operators;
+- `requireOrgRole` for org owner/admin backend access;
+- `requireProjectPermission` for project-level actions.
+
+Platform admin is not an `OrganizationMembership` and is not represented by
+project `PermissionGrant`. Platform admin audit remains separate from customer
+project audit.
+
+Customer orgs are manually created by platform staff for the pilot. Platform
+admins invite/allowlist other platform admins. Customer users authenticate with
+the customer's Feishu app. The same customer-owned Feishu app may be used for
+OAuth login, bot messages, and directory sync. App secrets are org-scoped
+secrets and must be encrypted at rest; business code should access them through
+a secret/connection resolver, not raw plaintext columns.
+
+Each Organization chooses one model-provider credential mode:
+
+- `BYOK`: an Organization OWNER/ADMIN supplies and manages that Organization's
+  provider key and base URL in the org admin area;
+- `PLATFORM_MANAGED`: a platform admin supplies and manages one distinct key
+  and base URL for that Organization in the platform admin area. Organization
+  users cannot read the plaintext credential.
+
+Provider connections remain Organization-scoped in both modes. Unrelated
+Organizations never share a process-global provider key. Provider secrets must
+be encrypted at rest and resolved at runtime through the same connection/secret
+boundary as Feishu credentials.
+
+Project management has two creation paths:
+
+- org owner/admin creates projects in the org web backend;
+- ordinary org members may create a project from an unbound Feishu group when
+  `membersCanCreateProjects` is enabled for the org.
+
+In both paths the creator gets project `MANAGE`. A Feishu-created project is
+immediately bound to the source chat, and that chat receives project `EDIT`.
+
+Feishu chat binding is strict 1:1:
+
+- one Feishu chat binds to at most one active project;
+- one project binds to at most one Feishu chat;
+- binding an existing unbound project from Feishu requires the clicking user to
+  have `MANAGE` on that project;
+- binding creates an active `FEISHU_CHAT -> PROJECT EDIT` grant;
+- binding mistakes are corrected by org owner/admin unbinding or archiving the
+  binding in the backend; historical sessions stay on their original project.
+
+Folders are transparent organization nodes for project navigation and usage
+aggregation:
+
+- folders belong to one organization;
+- projects may sit in folders;
+- folders can be nested;
+- folders are not permission resources;
+- folder visibility, team policies, and inherited grants are deferred.
+
+Project permissions remain on `PROJECT` resources. Moving a project between
+folders does not change grants.
+
+Usage accounting is org-wise and project/folder aggregatable for both provider
+modes. It is not payment collection in the pilot: BYOK spend belongs to the
+customer's provider account, while platform-managed spend is attributed to the
+Organization for operational reporting; commercial billing remains deferred.
+
+## Consequences
+
+- `OrganizationProjectSettings.membersCanCreateProjects` gates Feishu group
+  project creation for ordinary members.
+- `Folder` and `Project.folderId` support the file-manager-like project
+  explorer without creating a second ACL system.
+- Service code should expose project creation and chat binding as reusable
+  backend operations so Feishu cards and future web APIs call the same rules.
+- Org role/model/provider-mode panels and the platform-managed connection panel
+  can be added on top of the org tenant root without changing project authorization.
+
+## Open Questions / Deferred
+
+- True one-click Feishu app provisioning is deferred; pilot uses guided setup
+  and readiness checks.
+- Folder-level permissions are deferred until there is a concrete customer need.
+- Self-serve org signup and payment collection are deferred beyond pilot.
+- The platform admin identity, session, bootstrap, invitation, audit, and
+  offline recovery boundary is decided by ADR-0023 and must be implemented
+  before exposing the platform admin panel.

docs/adr/0022-layered-capacity-admission.md

@@ -0,0 +1,88 @@
+# ADR 0022: Layered Capacity Admission And Workload Safety
+
+## Status
+
+Accepted.
+
+## Context
+
+The initial SaaS deployment shares one production host across Organizations.
+The current Hub only has a one-run-per-project lock, a process-local five-item
+trigger queue, a five-minute queue wait, a global max-turn setting, and
+Fastify's default request-body limit. It has no global or Organization agent
+capacity, durable backpressure, fair scheduling, storage quotas, run wall-clock
+budget, multi-dimensional rate limiting, or emergency workload control.
+
+Leaving these limits implicit lets one Organization monopolize the host, lets
+accepted work disappear on restart, and makes overload look like success. A
+single global FIFO would preserve arrival order while still permitting one
+tenant to occupy every execution slot. Unlimited queues or automatic whole-run
+replay would hide overload and can duplicate tool side effects.
+
+## Decision
+
+Every mandatory capacity dimension has a non-optional **Platform Capacity
+Ceiling**. An Organization OWNER/ADMIN may configure a lower **Organization
+Policy Limit**; the **Effective Limit** is the lower value, or the platform
+ceiling when the Organization value is absent. There is no unlimited fallback.
+Production refuses to start when a mandatory platform ceiling is absent or
+invalid. Platform ceilings are changed through versioned deployment
+configuration in the initial release; live platform-admin editing is deferred
+until audited approval and rollback exist.
+
+Agent work uses a durable, bounded admission queue. Scheduling is fair across
+Organizations and FIFO within an Organization, while the existing one-active-
+run-per-Project rule remains. Capacity saturation queues only within effective
+length and wait limits. Queue overflow is an explicit `capacity_exhausted`
+response with retry guidance; an accepted request that waits too long becomes
+`EXPIRED`, is never started later, and notifies its initiator. Initiators may
+withdraw their own queued requests; Project `MANAGE` and Organization admins
+may cancel queued requests in their scope. Every transition is auditable.
+
+The platform never automatically replays a whole Agent Run after execution has
+started. Limited transport retries are allowed only before model output or tool
+side effects. Otherwise the run ends explicitly and a retry is a new user-
+initiated request. Feishu event identity and API idempotency prevent delivery
+retries from creating duplicate admissions.
+
+The first production gate requires hard limits for request rate/body size,
+agent concurrency, queue length/wait, files and attachment counts, archive
+expansion, Project and Organization storage, Organization entity counts, run
+wall time/turns/tool calls/output, individual tool duration, and execution
+process memory/CPU/process count. Reaching an entity or storage ceiling blocks
+new growth but does not delete, hide, or lock existing data; read, export,
+delete, and administrator remediation remain available.
+
+Admin surfaces require an authenticated user plus the corresponding platform
+or Organization administrator authorization. Authenticated Web traffic is
+limited by user, Organization, and endpoint class; signed Feishu traffic is
+limited by Organization application, chat, and sender; administrative writes
+use separate stricter buckets. Rejections and truncation are never silent.
+
+Token, provider-reported cost, run count, and duration are attributed by
+Organization, Project, Run, model, and Provider Connection. They are statistics
+and soft alerts in the initial release, not hard spend controls or settlement.
+Missing provider cost remains unknown rather than zero. Organization admins
+receive their alerts; platform admins additionally receive anomalous-cost
+alerts for platform-managed Provider Connections, not BYOK financial alerts.
+
+Platform administrators have audited, reason-bearing, reversible workload
+brakes at Organization and platform scope. `DRAIN` rejects new work and pauses
+queue starts while allowing active runs to finish. `STOP_NOW` also cancels
+active runs. These controls do not delete or suspend the Organization.
+
+Exact numerical ceilings remain open until production-like capacity tests
+measure the supported host and provider behavior. Defaults must be checked in,
+validated, observable, and proven at the release gate rather than guessed as
+domain constants.
+
+## Consequences
+
+- The current in-memory `TriggerQueue` and global-only agent settings are
+  implementation divergences, not acceptable production fallbacks.
+- Durable ingress, run/lock recovery, authorization, telemetry, and capacity
+  enforcement must share one Organization-scoped admission identity.
+- HTTP overload uses typed `429` responses with `Retry-After`; Feishu reports a
+  clear busy/expired/canceled outcome to the initiating user.
+- Capacity calibration is a release prerequisite and must include noisy-neighbor,
+  restart, queue expiry, file/archive, provider-stall, and workload-brake tests.

docs/adr/0023-platform-administrator-identity-and-audit.md

@@ -0,0 +1,136 @@
+# ADR 0023: Platform Administrator Identity And Audit Boundary
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0021 separated `/admin/platform` from Organization administration but left
+the exact platform identity store and audit schema open. That choice must be
+settled before exposing the panel: customer Feishu identities are scoped by
+their customer-owned applications, Organization membership must not confer
+platform authority, and a platform operator can create tenants, change
+connections, rotate credentials, or stop workloads across tenant boundaries.
+
+The current implementation is not the accepted platform model.
+`PlatformRoleAssignment` points at the customer-side `User`, includes a legacy
+`TEACHER` value, and has no platform login or guard. The current browser session
+is a seven-day stateless customer cookie, while `AuditEntry` is Project/Run
+oriented and deliberately best-effort. Reusing any of these would couple the
+two control planes and make immediate revocation or fail-closed privileged
+audit impossible.
+
+## Decision
+
+### Identity issuer and authority
+
+Platform Administrators authenticate only through one dedicated
+**Platform-owned Feishu Application**. It is separate from every Organization's
+Customer-owned Feishu Application and cannot grant Organization membership or
+Project permission.
+
+A `PlatformIdentity` is scoped by that platform application and its verified
+external Feishu user identity. It is not a customer `User`. The same person may
+also have customer identities, but there is no implicit link, shared session,
+or authority propagation between them.
+
+The initial platform control plane has exactly one standing role:
+`Platform Administrator`. There is no Platform Owner, Super Admin, or Teacher
+hierarchy. The first administrator has no permanent special authority after
+another administrator is active.
+
+### Bootstrap, invitation, and revocation
+
+The first Platform Administrator is granted through a checked-in, versioned,
+idempotent and transactional bootstrap database procedure. It accepts only a
+verified Platform Identity, refuses to run while any standing Platform
+Administrator is active, and atomically writes the grant plus its Platform
+Audit Entry. Ad-hoc manual `INSERT` statements and permanent environment
+allowlists are not supported bootstrap paths.
+
+Later administrators join through short-lived, single-use invitation links
+created by an active Platform Administrator. An invitation is bound to one
+specific Feishu account in the Platform-owned Feishu Application; possessing a
+link without authenticating as that account is insufficient. The invitation
+records its inviter and reason and becomes terminal after acceptance, expiry,
+or revocation. One active Platform Administrator approval is sufficient for the
+initial service.
+
+All standing Platform Administrators are peers. An administrator may invite or
+revoke another administrator, but the last active standing administrator cannot
+be revoked. Revocation immediately invalidates all Platform Sessions and all
+unclaimed invitations belonging to the revoked identity.
+
+### Platform Session and step-up authentication
+
+`Platform Session` is a separate, server-side, revocable session with a distinct
+cookie/token namespace. The browser receives an opaque random token; persistence
+stores only its hash and binds the session to a Platform Identity. Every request
+reloads the session and active grant. The session carries no Organization,
+membership, Project, or customer Feishu authority.
+
+Administrator lifecycle changes, platform or customer Feishu connection
+changes, platform-managed Provider credential changes, Organization lifecycle
+changes, workload-brake changes, and Emergency Platform Grant changes require
+recent Feishu OAuth reauthentication. Exact session/idle/step-up durations,
+CSRF, cookie and proxy/TLS mechanics belong to the production browser/session
+boundary decision; they must be configurable under checked maximums rather than
+silently inheriting the current seven-day customer cookie.
+
+### Platform audit
+
+`Platform Audit Entry` is a separate append-only record from customer Project
+and Agent Run audit. Every successful platform mutation and its audit entry
+commit in the same database transaction. If the audit entry cannot be written,
+the mutation fails. Platform audit is durable business evidence, not
+best-effort telemetry.
+
+An entry identifies the Platform Identity and Platform Session, action, target
+kind and identifier, affected Organization when applicable, mandatory reason
+for security-sensitive actions, request/correlation identity, outcome, time,
+and redacted before/after facts. It never stores credential, token, recovery-key
+or other plaintext secret material. The normal service path can append and read
+but cannot update or delete history.
+
+Platform Administrators can read the complete redacted platform audit.
+Organization Administrators can read a customer-safe projection of entries
+affecting their Organization, including Organization lifecycle, Feishu/provider
+connection, and workload-brake changes; they cannot read other Organizations,
+platform administrator lifecycle, recovery material, or internal security
+events.
+
+### Offline recovery and break-glass
+
+There is no standing break-glass web account or shared emergency password.
+Recovery uses a checked-in offline command from a controlled production-host
+console. The supported procedure requires both privileged host access and a
+recovery key held outside the host, environment file, and ordinary backup set.
+It also requires an incident identifier, reason, and one explicit recovery
+action.
+
+Recovery may revoke Platform Sessions, repair the Platform-owned Feishu
+connection, or issue an `Emergency Platform Grant` to a verified Platform
+Identity. An emergency grant is time-bounded, automatically expires, is not a
+standing role, and must be explicitly closed when recovery finishes. Every
+recovery action is represented in the append-only Platform Audit. Two factors
+are mandatory for the initial service; two separate human approvers are not.
+
+## Consequences
+
+- Platform identity, invitation, standing grant, server-side session, platform
+  audit, and emergency-grant persistence must be distinct from customer
+  `User`, `OrganizationMembership`, Project permission, and Project/Run audit.
+- `requirePlatformAdmin` must resolve the dedicated session and an active
+  standing or unexpired emergency grant on every request. It is never an
+  Organization-role override.
+- The legacy `PlatformRoleAssignment`/`TEACHER` schema is not a compatible
+  shortcut and must be migrated or replaced before the platform panel ships.
+- Bootstrap and recovery remain direct database/control-plane operations, but
+  only through versioned, asserted, auditable procedures rather than arbitrary
+  SQL.
+- Exact numeric invitation/session/step-up/emergency durations, audit retention
+  and export policy, recovery-key custody rotation, and future multi-party
+  approval remain follow-on operational decisions. They may not weaken the
+  identity separation, last-administrator protection, fail-closed audit, or
+  two-factor recovery invariants fixed here.

docs/adr/0024-local-secret-envelope-and-connection-resolution.md

@@ -0,0 +1,143 @@
+# ADR 0024: Local Secret Envelope And Connection Resolution
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0021 requires every customer Feishu application and model provider
+credential to belong to one Organization. It forbids a process-global provider
+credential and requires business code to obtain plaintext only through a
+resolver. Before the pilot can store real customer credentials, the deployment
+also needs a concrete encryption, rotation, recovery, writer-authority, and
+connection-identity contract.
+
+The initial production shape is one Hub service and one PostgreSQL database on
+a controlled host. An external KMS would add another remote availability and
+bootstrap dependency before the pilot has operational support for it. Keeping
+plaintext credentials in environment variables, configuration files readable
+by the service account, database columns, logs, or Agent environments is not an
+acceptable substitute.
+
+## Decision
+
+### Local master-key keyring
+
+The pilot uses a versioned **local master-key keyring**. Each key is a random
+256-bit key-encryption key (KEK) with a stable, non-secret key identifier. The
+keyring names exactly one active KEK and may retain previous KEKs while rotation
+is in progress. It is not stored in PostgreSQL, a release directory, an
+environment variable, or the ordinary database/workspace backup set.
+
+In production, the source keyring is owned by root, mode `0600`, and delivered
+to the unprivileged Hub service by systemd `LoadCredential`. Hub reads the
+read-only file under `CREDENTIALS_DIRECTORY`; production startup rejects a
+direct arbitrary file path. Development and tests may use an explicit local
+keyring file. Startup fails when the keyring is absent, malformed, contains an
+invalid key, has no active key, or names an active key that is not present.
+
+### Envelope format and binding
+
+Every immutable secret version receives a new random 256-bit data-encryption
+key (DEK). AES-256-GCM encrypts the secret payload with that DEK. AES-256-GCM
+also wraps the DEK with the keyring's active KEK. Nonces are random and never
+reused with the same key.
+
+The versioned envelope persists only algorithm/version metadata, the KEK
+identifier, nonces, authentication tags, the wrapped DEK, and ciphertext.
+Authenticated additional data binds both layers to the secret purpose,
+Organization identifier, Connection identifier, immutable secret-version
+identifier, and envelope version. Copying ciphertext between rows,
+Organizations, connections, purposes, or versions therefore fails
+authentication. Unknown algorithms or versions, missing KEKs, malformed
+envelopes, and failed authentication are hard errors; there is no fallback to
+another credential source.
+
+The encrypted payload is a versioned object specific to its connection type.
+For a Provider Connection, provider base URL and authentication material are
+inside the payload. For a Feishu Application Connection, application ID,
+application secret, verification/encryption material, and bot identity are
+inside the payload. HTTP responses, business records, logs, metrics, traces,
+audit facts, exceptions, and Agent process environments receive only redacted
+metadata, never those plaintext fields or a decrypted DEK.
+
+### Connection identity, lifecycle, and writer authority
+
+A connection has a stable opaque identifier and belongs to exactly one
+Organization. Secret versions are immutable; the connection points to one
+active version. Rotation creates and validates a new version, atomically moves
+the active pointer, and retains the previous encrypted version for rollback and
+audit until retention policy removes it. A connection is `DRAFT`, `ACTIVE`, or
+`DISABLED`; runtime resolution accepts only an active connection with a valid
+active secret version.
+
+There is one customer-owned Feishu Application Connection per Organization.
+Its provider-local user, chat, event, and callback identifiers are meaningful
+only together with that connection identity. Organization OWNER/ADMIN may
+create and rotate that customer-owned connection. Platform repair remains a
+separate stepped-up, reason-bearing, fail-closed-audited operation under
+ADR-0023, not an Organization API bypass.
+
+A model Provider Connection is unique by Organization and provider identity.
+In `BYOK` mode an Organization OWNER/ADMIN creates and rotates it. In
+`PLATFORM_MANAGED` mode only a stepped-up Platform Administrator may create or
+rotate it, with the platform mutation and redacted Platform Audit Entry in one
+transaction. Switching modes creates or activates the correctly governed
+connection version; it never re-labels plaintext or grants the other control
+plane write authority.
+
+### Resolver boundary
+
+Business code asks one connection resolver for an active connection using an
+explicit Organization, or using a Project that is first resolved to its active
+Organization. Provider resolution also requires the provider identity; Feishu
+resolution requires the intended Feishu Connection identity whenever external
+traffic enters the system. Resolution fails for missing, draft, disabled,
+cross-Organization, corrupt, or undecryptable records.
+
+Decrypted values exist only in the resolver result for the duration of the
+outbound provider/Feishu operation. They are not cached across connections,
+returned through administrative reads, passed to Agent tools, or written back
+to business rows. Production has no process-global Feishu or model-provider
+credential fallback.
+
+The Claude Agent child never receives an Organization's Provider credential.
+For each run, Hub opens a loopback-only proxy with a random, short-lived run
+capability. The child receives only that local endpoint and capability; Hub
+validates the capability, replaces it with the Organization credential at the
+upstream hop, refuses automatic redirects, and destroys the listener when the
+run ends. Agent sandbox credential rules hide even the run capability from
+tool subprocesses.
+
+### KEK rotation, backup, and recovery
+
+KEK rotation is staged: install a new KEK alongside the old keys, mark it
+active, rewrap each stored DEK under the new KEK without decrypting its payload,
+verify every envelope, then retire the old KEK only after the separately
+protected recovery copy and restore drill are current. Secret rotation and KEK
+rotation are distinct operations.
+
+A recoverable deployment requires both the ordinary database/workspace backup
+and a separately protected backup of the local keyring. Neither copy alone can
+recover plaintext credentials. Restore preflight verifies key identifiers and
+authenticates sample/all envelopes before traffic is enabled. A missing or
+incorrect keyring leaves the service unavailable and produces redacted
+diagnostics; operators must restore the correct keyring rather than reset or
+silently discard credentials.
+
+## Consequences
+
+- The database may be restored or copied without exposing plaintext secrets,
+  but losing the separately held keyring makes encrypted credentials
+  unrecoverable.
+- The service unit, installer, preflight, backup, restore, and rotation tooling
+  must treat the keyring as an explicit production dependency.
+- Administrative writes must accept secret material as write-only input and
+  return only connection state, version, timestamps, and redacted fingerprints.
+- Tests must prove cross-Organization/row substitution failure, wrong/missing
+  key failure, version rotation, absence of global fallback, and absence of
+  plaintext from persistence, responses, logs, and Agent environments.
+- External KMS/HSM adoption is deferred. It may replace the local KEK provider
+  without changing connection identity, immutable secret versions, AAD
+  binding, writer authority, or resolver behavior.

docs/adr/0025-alpha-silo-deployment.md

@@ -0,0 +1,57 @@
+# ADR 0025: Alpha Uses Managed Single-Organization Silos
+
+## Status
+
+Accepted for the supervised alpha.
+
+## Context
+
+The shared SaaS control plane, durable cross-Organization admission scheduler,
+and customer administration SPA are not complete. Real customer data must not
+depend on partially migrated application-level tenant filtering merely to start
+the supervised alpha.
+
+## Decision
+
+Each alpha Organization runs as a separately named Silo deployment. A Silo has
+one Hub systemd unit and service identity, one logical PostgreSQL database and
+database role, one workspace root, one environment/session secret, one local
+master-key keyring, one Feishu Application Connection, and Organization-local
+Provider Connections. The process is pinned by `HUB_SILO_ORGANIZATION_ID` and
+refuses startup unless its database contains exactly that one active
+Organization.
+
+Application releases remain shared and versioned; customer state does not live
+inside a release directory. Host-console bootstrap, upgrade, backup, restore,
+and KEK rotation always name a Silo instance explicitly. The alpha exposes no
+platform administration surface. Operators provision and support Silos through
+the controlled host console and retain an external change record.
+
+The process-local trigger queue is not an accepted-work queue in the Silo
+alpha. A busy project or instance rejects new work explicitly. Each instance
+has checked-in application limits and explicit systemd CPU, memory, and process
+ceilings. Workspace filesystem quotas are a host provisioning prerequisite.
+
+This topology changes deployment, not the semantic tenant root:
+`Organization` remains the tenant root and every authorization, secret, audit,
+Project, Team, and Run remains Organization-scoped. A later pooled or bridge
+adapter may host multiple Organizations only after ADR-0022's durable admission
+and platform control requirements are implemented.
+
+## Consequences
+
+- Cross-Organization data access is prevented by database credentials and
+  process/filesystem identity in addition to application authorization.
+- Provisioning, migrations, backups, health, and upgrades are repeated per
+  Silo and therefore must be automated before tenant count grows materially.
+- Cross-Organization fairness is not applicable inside one Silo; host cgroup
+  ceilings prevent one Silo from exhausting the machine.
+- Database/workspace backups and the keyring/environment recovery copy are
+  produced to distinct protected destinations and must pass a restore drill.
+
+## Deferred
+
+- Shared platform identity and administration from ADR-0023.
+- Pooled durable fair scheduling from ADR-0022.
+- Customer org administration SPA and self-service onboarding.
+- Docker or Kubernetes as an alternative deployment adapter.

docs/adr/0026-usage-fact-ledger.md

@@ -0,0 +1,146 @@
+# ADR 0026: Usage Fact Ledger
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0022 pins the cost-attribution contract for the platform: token,
+provider-reported cost, run count, and duration are attributed by Organization,
+Project, Run, model, and Provider Connection; "missing provider cost remains
+unknown rather than zero." ADR-0021 scopes usage accounting as operational
+reporting, not payment collection (commercial billing stays deferred).
+
+The implementation today stores this as **one scalar per `AgentRun`**:
+`inputTokens`, `outputTokens`, `costUsd?`, `costSource?`, written once from the
+Claude Agent SDK's `result.total_cost_usd` when the run finishes. This is
+adequate for a single provider-reported model loop, but it cannot represent:
+
+- **External capability consumption** inside a run. PDF→Markdown bundle
+  conversion, audio/video transcription, OCR and similar media transforms are
+  not the main agent loop. They run as side effects of a run, may use a
+  different provider/model, may bill in non-token units (pages, seconds,
+  invocations), and may report cost through a different channel than the
+  OpenRouter gateway. Today there is no row to write that cost to — it would
+  either disappear or silently corrupt the run's single scalar.
+- **Multiple model calls within one run** (e.g. a sub-model invoked by a
+  tool, a gateway-side reroute). The scalar collapses them into one number.
+- **Pricebook derivation** after the fact. With only a final USD figure and no
+  `(provider, model, occurredAt, tokens)` fact, an operator cannot re-derive
+  cost from a price table when the provider did not report it.
+
+Treating each external call as a nested `AgentRun` was considered and
+rejected: `AgentRun` carries lock ownership (ADR-0002), session/provider/role
+binding (ADR-0017), admission/capacity semantics (ADR-0022), and the
+user-visible task boundary. External calls hold none of those. Making them
+`AgentRun`s would pollute run counts, admission, lock semantics, and session
+continuity, and would still not solve non-token metering.
+
+## Decision
+
+Introduce **`UsageFact`** as the single source of truth for billable
+consumption inside an `AgentRun`. An `AgentRun` owns zero or more
+append-only `UsageFact` rows; each row records one billable consumption event:
+
+- `kind` — `model_completion` (the main agent loop) | `external_capability`
+  | `tool_proxy`. The kind set is `OPEN`; new kinds must be surfaced, not
+  silently folded into an existing one.
+- `provider` — e.g. `openrouter`, `mineru`, `openai_whisper`.
+- `model?`, `inputTokens?`, `outputTokens?` — token metering, optional
+  because non-token capabilities have none.
+- `quantity?` + `unit?` — non-token metering (pages, audio_seconds,
+  invocations), coexisting with tokens rather than replacing them.
+- `costUsd?` + `costSource` — `provider_reported` | `pricebook_derived` |
+  `unknown`. `costUsd = null` means **unknown, not zero** (ADR-0022). When the
+  provider reported a cost, `costSource = provider_reported` and that value
+  wins. When only tokens are known, a later pricebook pass may derive
+  `costUsd` with `costSource = pricebook_derived`. When neither is possible,
+  `costSource = unknown` and `costUsd` stays null.
+- `occurredAt` — when the consumption happened; the pricebook derivation
+  depends on this, not on `AgentRun.finishedAt`, because an external
+  capability may complete before the run finishes.
+- `capabilityId?` — for `external_capability` facts, the registered capability
+  id (e.g. `pdf_to_md_bundle`, `audio_video_to_text`).
+- `correlationId?` — external request id for reconciliation / idempotency;
+  not part of the aggregation key.
+
+### Invariants
+
+1. **Append-only.** A `UsageFact` row is never updated or deleted. A cost
+   correction is a new row; the old row stays. `onDelete: Cascade` exists only
+   so a hard run delete (itself not a normal path) cleans up its facts.
+2. **Belongs to exactly one Run; never holds a lock.** A fact is a side-effect
+   ledger of one run, not a sub-run. External capability calls obey the same
+   boundary: their identity is `capabilityId + correlationId`, not `RunId`.
+3. **Missing cost ≠ zero.** Aggregation MUST NOT sum `null` `costUsd` as 0.
+   A run whose facts all have `costUsd = null` is "cost unknown" — reported as
+   `runsWithoutCost`, exactly as the pre-migration `costUsd = null` runs are
+   today. A run with at least one `costUsd`-bearing fact contributes its sum.
+
+### Rollup cache
+
+`AgentRun.costUsd / inputTokens / outputTokens / costSource` columns are kept
+as a **derived rollup cache**, not dropped:
+
+- Existing non-`/usage` readers (slash `/usage`, session detail, integration
+  tests asserting `run.costUsd`) continue to work without code changes for
+  historical runs.
+- On run finish, the writer writes the `UsageFact` row first and then mirrors
+  it onto `AgentRun` as two separate statements, not one transaction. The
+  fact is the truth, so it is written first; the cache is derived, so it is
+  written second. A crash between the two leaves the cache stale, but the
+  usage service re-reads `UsageFact` directly, so staleness is recoverable
+  (and the reverse order would lose the truth, which is not). Keeping the
+  fact insert and the run update in separate statements also avoids holding
+  the `AgentRun` row lock across the FK ShareLock taken by the insert, which
+  deadlocks against concurrent workspace teardown under the cascade path
+  `Organization → Project → AgentRun → UsageFact`.
+- The org/project usage service and the slash `/usage` command read from
+  `UsageFact` directly — they are the canonical aggregation path.
+
+### Migration
+
+A new `UsageFact` table is added. For every existing `AgentRun` with a
+non-null `costUsd` or non-null `inputTokens`/`outputTokens`, the migration
+inserts **one synthetic `model_completion` fact** carrying the run's
+provider, model, tokens, cost, and `costSource`. Its `correlationId` is the
+run id, so the row is identifiable as a backfill artefact. This keeps
+historical reporting correct under the new aggregation path. Pre-migration
+runs with no recorded cost remain `runsWithoutCost` by design, matching
+ADR-0022's "missing cost ≠ zero" rule and the existing migration
+`20260709143000_agent_run_cost_tracking`'s "no backfill" stance for the
+truly unrecorded.
+
+## Consequences
+
+- The billing model now supports external capabilities (PDF→MD, ASR, OCR, …)
+  without per-capability schema changes: a new capability is one new
+  `capabilityId` value and one or more `external_capability` facts.
+- `AgentRun.costUsd` is no longer the truth; it is a convenience cache.
+  Readers that need the truth (cost breakdown, per-capability attribution)
+  must read `UsageFact`. The cache MUST be kept consistent on the write path.
+- The capability registry, pricebook, and any commercial settlement remain
+  `OPEN` and out of pilot scope (ADR-0021). This ADR only pins the ledger
+  shape and invariants.
+- `usage.ts` and `/usage` now perform a join against `UsageFact` rather than a
+  single-table scan of `AgentRun`; the index on `(runId, occurredAt)` and
+  `(provider, model, occurredAt)` keeps the existing org/project/report
+  queries bounded.
+- Cost corrections (e.g. a provider rebills a run) produce a new fact row; the
+  rollup cache must be recomputed. The initial release writes facts once at
+  run finish and does not support post-hoc correction flows — that remains
+  `OPEN`.
+
+## Deferred
+
+- **Capability registry** as a first-class spec entity (`Spec.System.Capability`)
+  with org-scoped enable/disable, input/output contracts, metering schemas,
+  and org-exclusive credentials (ADR-0024 alignment). This ADR only reserves
+  the `capabilityId` field and the `external_capability` fact kind.
+- **Pricebook** — a versioned price table keyed by `(provider, model, unit)`
+  with time validity. Required to actually produce `pricebook_derived` costs;
+  until then, facts without provider-reported cost stay `unknown`.
+- **Post-hoc cost correction flow** — append-only today; correction UI and
+  rollup recompute are `OPEN`.
+- **Commercial billing, invoicing, settlement** — still deferred per ADR-0021.

docs/adr/0027-external-capability-registry.md

@@ -0,0 +1,152 @@
+# ADR 0027: External Capability Registry
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0026 introduced the `UsageFact` ledger with a `kind = external_capability`
+fact and a `capabilityId` field, but deferred the capability registry itself.
+Two concrete needs now force the issue:
+
+- **PDF→Markdown bundle** conversion (and, imminently, audio/video→text)
+  must run as a side effect of an `AgentRun`, bill in non-token units
+  (pages, seconds), use a different provider than the model loop, and report
+  cost through a different channel. It is not a sub-run (ADR-0026 rejected
+  that) and not a model-provider call (it does not speak the Anthropic/
+  OpenRouter protocol).
+- The Agent already has a Bash tool. Without a first-class capability seam,
+  the path of least resistance is for the agent to shell out to ad-hoc
+  scripts that embed API keys, write to arbitrary paths, and report nothing
+  to the ledger. That is exactly the unattributed, uncontained external
+  consumption ADR-0022/0026 exist to prevent.
+
+The model-provider connection (`OrganizationProviderConnection`, ADR-0024)
+is the wrong seam for these services:
+
+- Its payload schema (`baseUrl` + `authToken` + `anthropicApiKey`) and
+  readiness probe (`/v1/models?supported_parameters=tools`) are specific to
+  OpenRouter/Anthropic. MinerU, Whisper, and future OCR/ASR services have
+  different auth shapes (an API token, optionally a project id) and no
+  `/v1/models` endpoint.
+- Its uniqueness key is `(organizationId, providerId)` where `providerId`
+  is an OpenRouter-style model-routing id. A capability provider id
+  (`mineru`) names a *service*, not a model.
+- Coupling capability credentials into the model-provider table would force
+  every capability's auth shape through `ProviderSecretPayloadV1` and every
+  readiness probe through `probeOpenRouterCredential`.
+
+The Feishu Application Connection (`OrganizationFeishuApplicationConnection`)
+is the right structural precedent: it reuses the ADR-0024 envelope (KEK →
+DEK → AES-256-GCM, AAD-bound to purpose/org/connection/version) but has its
+own connection table, its own payload schema, its own readiness probe, and
+its own per-org uniqueness. Capability connections follow the same pattern.
+
+## Decision
+
+### External Capability
+
+An **External Capability** is a platform-registered, org-enabled document or
+media transform service invoked as a side effect of an `AgentRun`. It is
+identified by a stable `capabilityId` (e.g. `pdf_to_md_bundle`,
+`audio_video_to_text`). A capability:
+
+- Has an **input kind** (PDF, image, audio, video, …) and an **output
+  contract** (markdown bundle with extracted images, transcript text, …).
+- Bills in **non-token units** (pages, audio-seconds) recorded on a
+  `UsageFact` with `kind = external_capability`, or in tokens when the
+  backing service reports them.
+- Writes its output **into the invoking run's workspace** (ADR-0018
+  `AgentSurface` — no escapes).
+- Is invoked through a **capability adapter** in Hub, never by the Agent
+  shelling out with embedded credentials.
+
+### Capability Connection
+
+Credentials for a capability live in an **`OrganizationCapabilityConnection`**,
+structurally identical to the Feishu Application Connection:
+
+- Belongs to exactly one Organization.
+- Unique by `(organizationId, capabilityId)`.
+- `DRAFT` / `ACTIVE` / `DISABLED`; resolution accepts only `ACTIVE` with a
+  valid active secret version.
+- Secret material is an immutable, AAD-bound, KEK-wrapped envelope version
+  (`CapabilityCredentialVersion`), reusing the ADR-0024 encryption
+  machinery with `purpose = "capability"`.
+- Its payload schema is capability-specific (`CapabilitySecretPayloadV1`:
+  `baseUrl`, `apiToken`, optional `projectId`). New capability types extend
+  the payload, not the connection table.
+- A capability-specific **readiness probe** validates the credential before
+  activation (e.g. MinerU: a trivial authenticated GET). The probe is
+  injectable, matching the Feishu/provider pattern, so tests never hit the
+  network.
+
+### Capability Adapter
+
+The adapter is the seam between the Agent and the external service. It:
+
+- Resolves the org's active capability connection (fail-closed, no
+  process-global fallback — ADR-0024).
+- Accepts a workspace-relative input path and an output directory.
+- Calls the backing service (MinerU, Whisper, …) via an injectable
+  `Client` interface so the real HTTP client is swappable and mockable.
+- Writes the produced markdown + image assets into the run's workspace.
+- Writes one `UsageFact` (or more, if the service reports per-stage
+  consumption) with `kind = external_capability`, `capabilityId`,
+  `provider` (the service id), `quantity + unit` (pages / seconds), and
+  `costUsd + costSource = provider_reported` when the service reports cost.
+
+### Registry
+
+The platform maintains a **registry** of known capabilities: their id,
+input kind, output contract, metering unit, and adapter. This is
+code-level registration (like `ToolRegistry`), not a database table — a
+capability is available to an Organization only when (a) the platform
+knows the adapter and (b) the Organization has an `ACTIVE` connection for
+it. Both gates are required.
+
+### What is NOT in this ADR
+
+- The capability invocation is **not** a first-class persisted record
+  (`CapabilityInvocation` table) in this ADR. The `UsageFact` row with
+  `capabilityId` + `correlationId` is the durable trace. If we later need
+  a richer invocation log (retries, partial output, multi-stage status),
+  that is a follow-up; for now the fact is enough.
+- **Pricebook** remains deferred (ADR-0026). Capability facts use
+  `provider_reported` when the service returns cost; otherwise `unknown`.
+- **Org-scoped enable/disable policy** beyond connection status is
+  deferred. An org with an `ACTIVE` connection has the capability; one
+  without does not. A finer "enabled but no credential" toggle is not
+  needed yet.
+- **Agent-facing tool exposure** (how the Agent discovers and calls the
+  capability — MCP tool, Bash wrapper, or built-in) is an implementation
+  detail of the adapter wiring, not a contract concern. The contract pins
+  that the Agent never receives the capability credential.
+
+## Consequences
+
+- Adding a new external capability (e.g. `image_ocr`) is: register an
+  adapter, add a `capabilityId` constant, optionally extend the secret
+  payload — no schema change to `UsageFact` or `AgentRun`.
+- The model-provider connection table stays focused on model routing;
+  capability credentials do not pollute its payload or readiness probe.
+- Three connection types now share the ADR-0024 envelope: model-provider,
+  Feishu application, and capability. Each has its own table, payload
+  schema, and probe, but the same encryption, rotation, and resolver
+  boundary.
+- The Agent's Bash tool remains available, but the intended path for
+  document/media transforms is the capability adapter. Whether to narrow
+  Bash for capability-shaped tasks is an operational policy decision,
+  not a contract one.
+- Tests prove: workspace containment of capability output, fail-closed
+  credential resolution, `UsageFact` attribution with non-token metering,
+  and that the Agent process never receives the capability credential.
+
+## Deferred
+
+- `CapabilityInvocation` as a first-class durable record (status, retries,
+  partial output) — currently the `UsageFact` row is the only trace.
+- Pricebook derivation for capability costs (ADR-0026 deferred).
+- Org-scoped capability enable/disable policy finer than connection status.
+- Agent-facing tool discovery (MCP vs built-in) for capabilities.

docs/adr/0028-agent-config-folder-tree.md

@@ -0,0 +1,81 @@
+# ADR 0028: Agent Configuration Folder Tree
+
+## Status
+
+Accepted.
+
+## Context
+
+ADR-0017/0018 made Agent roles and skills Organization-scoped dynamic runtime
+configuration, managed without process restarts. The org admin surfaces for
+them (`/admin/roles` and `/admin/skills`) render every
+role/skill as one large editor card in a single flat list. As an Organization
+accumulates roles and skills, the management pages degrade into an endless
+scroll with no grouping affordance.
+
+The project explorer already solves the analogous problem for projects with
+transparent folders (ADR-0021): org-scoped navigation nodes that are not
+permission resources. Roles and skills need the same affordance, but their
+semantics differ from projects in one crucial way: skill names and role IDs
+are referenced by role→skill bindings, run-time skill snapshot loading, and
+Feishu slash commands. Any grouping mechanism must not leak into those
+resolution paths.
+
+## Decision
+
+Introduce an Organization-scoped folder tree shared by Agent roles and Agent
+skills:
+
+- One folder tree per Organization is shared by both roles and skills (e.g. a
+  "高三化学组" folder groups that team's roles and its skills together). It is
+  a distinct entity from the project explorer `Folder` of ADR-0021 — the two
+  trees are managed independently and never reference each other.
+- Folders are **transparent organization nodes**, following the ADR-0021
+  project-folder precedent: they exist for management-surface navigation and
+  grouping only, are not permission resources, and hold no grants.
+- Folder membership is **not part of role/skill identity or resolution**:
+  - skill `name` and role `roleId` remain unique per Organization regardless
+    of folder membership;
+  - role→skill bindings, run admission's frozen role snapshot, run-scoped
+    skill loading, and Feishu slash commands never reference folders.
+- Each role/skill sits in at most one folder; membership is optional
+  (unfiled items remain first-class). Folders nest arbitrarily.
+- Folder assignment is a label-class change in the ADR-0017 sense: it never
+  archives Agent sessions, because the execution surface (model, prompt,
+  tools, skill content) is untouched.
+- A folder can be deleted only when empty — no child folders, no roles, no
+  skills. Relocating items out of a folder is an explicit user action, so no
+  orphan-placement rule is needed yet.
+- Admin web renders both pages as a left folder tree plus the item list of
+  the selected folder ("all" and "unfiled" included). The host-console CLI is
+  unchanged: folder management lives in the web surface, and CLI
+  `upsert-role`/`install-skill` never touch folder assignment.
+
+## Consequences
+
+- New DB entity `OrganizationAgentConfigFolder` (org-scoped, self-nesting via
+  `parentId`, delete restricted while referenced) plus nullable `folderId` on
+  `OrganizationAgentRole` and `OrganizationAgentSkill` (SetNull on folder
+  delete, though the service refuses to delete non-empty folders).
+- The spec pins the transparency and single-membership semantics in
+  `Spec.System.AgentRole` (`AgentConfigFolder`), so future implementors do
+  not re-derive them differently (e.g. path-style names or per-folder
+  uniqueness).
+- Org admin APIs gain folder CRUD plus role/skill folder-assignment endpoints
+  that skip session archival by construction.
+- Moving a role/skill between folders changes nothing about authorization,
+  run resolution, or audit-visible configuration lineage beyond the folder
+  assignment event itself.
+
+## Open Questions / Deferred
+
+- Drag-and-drop assignment and bulk moves are deferred; assignment is a
+  per-item select for now.
+- Folder-level usage aggregation for roles/skills is deferred (project
+  folders already aggregate usage under ADR-0021; agent configuration has no
+  usage dimension yet).
+- CLI flags for folder assignment are deferred until a console workflow asks
+  for them.
+- Folder-scoped default-role policies (e.g. per-folder defaults) are rejected
+  for now: the Organization keeps exactly one active default role
+  (ADR-0017/0018 invariant) regardless of folder structure.

docs/para-26071100-feishu-setup.md

@@ -0,0 +1,221 @@
+# Educraft 组织接入与飞书应用配置指南
+
+本文供准备接入 Educraft Alpha Silo 的学校、教培机构和组织管理员使用。完成本文后，请把末尾的“部署信息交付单”交给 Educraft 部署人员；我们会为组织创建独立的服务账号、数据库、运行目录和域名入口。
+
+> **安全提醒：** App Secret、模型 Provider Token 属于密钥，禁止粘贴到飞书群、普通云文档、工单正文或截图中。请只通过双方约定的安全渠道传递。
+
+## 1. 双方分别负责什么
+
+| 角色 | 负责事项 |
+| --- | --- |
+| 组织管理员 | 创建企业自建应用、启用机器人、开通最小权限、配置事件、回调和 OAuth 重定向 URL、发布应用、提供 OWNER 身份 |
+| Educraft 部署人员 | 分配组织 slug 和域名、部署独立 Silo、加密保存应用及模型密钥、初始化 OWNER、联调和验收 |
+| 试点 OWNER | 把机器人加入试点群、创建或绑定项目、组织首轮验收 |
+
+## 2. 创建企业自建应用
+
+1. 打开[飞书开放平台开发者后台](https://open.feishu.cn/app)。
+2. 在目标企业下点击“创建企业自建应用”。应用名称建议填写“Educraft + 组织简称”。
+3. 进入“凭证与基础信息”，记录 App ID 和 App Secret。
+4. 进入“添加应用能力”，添加并启用“机器人”。
+
+![凭证与基础信息页面；App Secret 默认以星号隐藏](assets/feishu-setup/01-credentials.png)
+
+App ID 通常以 `cli_` 开头，可以写入交付单。App Secret 必须通过安全渠道单独发送。Bot Open ID 不需要管理员手工查找；部署程序会用 App ID/App Secret 调用 Bot Info API 获取并校验归属。
+
+## 3. 开通最小权限
+
+进入“权限管理”，点击“开通权限”，搜索并申请以下应用身份权限。控制台中文名称可能调整，请优先核对 scope。
+
+| 用途 | Scope |
+| --- | --- |
+| 接收群聊中 @ 机器人的消息 | `im:message.group_at_msg:readonly` |
+| 以应用身份发送消息 | `im:message:send_as_bot` |
+| 读取触发消息和线程上下文 | `im:message:readonly` |
+| 获取与上传图片或文件 | `im:resource` |
+| 添加、删除消息表情回复 | `im:message.reactions:write_only` |
+| 获取用户基本信息 | `contact:user.base:readonly` |
+| 获取用户基本资料 | `contact:user.basic_profile:readonly` |
+| 通过手机号或邮箱查询 OWNER Open ID | `contact:user.id:readonly` |
+
+### 批量导入权限（推荐）
+
+在“权限管理”页面点击“批量处理 → 导入”，粘贴以下 JSON 后确认。导入只会新增本次列出的权限，不会删除或影响应用已经申请、开通的其他权限。
+
+```json
+{
+  "scopes": {
+    "tenant": [
+      "im:message.group_at_msg:readonly",
+      "im:message:send_as_bot",
+      "im:message:readonly",
+      "im:resource",
+      "im:message.reactions:write_only",
+      "contact:user.base:readonly",
+      "contact:user.basic_profile:readonly",
+      "contact:user.id:readonly"
+    ],
+    "user": []
+  }
+}
+```
+
+Educraft 机器人以应用身份调用上述 API，因此这些 scope 全部放在 `tenant`，不要为了省事把相同权限重复放进 `user`。
+
+![权限管理入口与已开通权限列表](assets/feishu-setup/02-permissions.png)
+
+如果 API 调试台提示缺少更细粒度权限，请把错误提示和发生时间截图给部署人员。不要自行开通通讯录全量读取等超出本表的权限。
+
+## 4. 配置事件与卡片回调
+
+进入“事件与回调”。
+
+1. 在“事件配置”中将订阅方式设为“使用长连接接收事件”。
+2. 添加事件“接收消息” `im.message.receive_v1`。
+3. 添加事件“解散群” `im.chat.disbanded_v1`，用于立即归档该群的项目绑定。
+4. 添加事件“机器人被移出群” `im.chat.member.bot.deleted_v1`，用于立即归档该群的项目绑定。
+5. 在“回调配置”中同样选择长连接。
+6. 添加回调“卡片回传交互” `card.action.trigger`，用于审批、运行中断和项目创建/绑定按钮。
+
+![长连接与消息事件配置](assets/feishu-setup/03-events.png)
+
+![卡片交互回调配置](assets/feishu-setup/04-callbacks.png)
+
+这里不需要填写公网 Event Callback URL。Educraft Hub 使用飞书官方 SDK 的长连接模式。
+
+## 5. 配置用户 OAuth 重定向 URL（必需）
+
+普通群成员首次使用前，需要通过飞书 OAuth 建立其在本应用下的用户身份。进入“安全设置 → 重定向 URL”，添加组织专属 callback：
+
+```text
+https://<organization-slug>.educraft.paradigm-edu.net/auth/feishu/callback
+```
+
+例如组织 slug 为 `example-school`：
+
+```text
+https://example-school.educraft.paradigm-edu.net/auth/feishu/callback
+```
+
+![在安全设置中添加组织专属 OAuth 重定向 URL](assets/feishu-setup/05-security.png)
+
+必须使用 Educraft 部署人员最终确认的 slug；不要直接照抄示例。该 URL 用于 OAuth 返回并创建应用作用域下的飞书用户身份，不代表当前已经开放组织管理台。
+
+组织专属 OAuth 同时完成身份建立和入组：首次成功登录的用户会自动成为当前 Organization 的 `MEMBER`，回到群聊即可使用。`OWNER` 和 `ADMIN` 仍只能由部署人员或管理员显式授予；曾被移除的成员重新登录不会自动恢复资格。
+
+## 6. 发布并安装应用
+
+1. 进入“版本管理与发布”，点击“创建版本”。
+2. 将应用可用范围至少覆盖试点 OWNER 和试点群成员。
+3. 提交企业管理员审核并发布。
+4. 发布成功后，将机器人加入准备试用的群。
+
+![版本管理与发布页面](assets/feishu-setup/06-publish.png)
+
+仅保存开发配置但未发布时，新增权限、事件和可用范围通常不会对试点用户生效。
+
+## 7. 获取首位 OWNER 身份
+
+首次部署必须指定一位组织 OWNER。OWNER 是 Educraft 组织内的初始管理员，不等同于飞书应用所有者；两者可以是同一个人，也可以不同。部署所需的 Open ID 必须由本次创建的企业自建应用查询，因为同一用户在不同应用下的 Open ID 不同，不能复用其他应用查到的值。
+
+### 7.1 确认 OWNER
+
+先确认哪一位企业成员将担任 OWNER。记录其飞书显示名称，并准备在飞书的成员选择器中按姓名找到本人。若企业内有同名成员，选择前须通过部门等信息核对身份。
+
+### 7.2 开通查询权限和数据范围
+
+确认应用已开通上文列出的用户基本信息和用户 ID 权限。如果使用上文的批量导入 JSON，这些权限已包含在内。
+
+应用的通讯录数据范围还必须覆盖这位 OWNER。最小做法是把 OWNER 加入应用可用范围；不需要为此开放全企业通讯录。
+
+### 7.3 在官方接口页面获取 Open ID
+
+1. 打开飞书开放平台的[“获取单个用户信息”接口页面](https://open.feishu.cn/document/server-docs/contact-v3/user/get)。如果使用带 `appId` 参数的页面链接，可以直接进入对应应用；本文不提供固定 App ID，请在页面顶部选择本组织刚创建的企业自建应用，并核对 App ID 与交付单一致。
+2. 找到路径参数 `user_id`，点击参数输入框旁的“获取”。
+3. 在成员选择器中找到并选择 OWNER；如有同名成员，依据部门等信息确认本人。
+4. ID 类型选择 `open_id`。将选择器返回的值填入 `user_id`，并保持查询参数 `user_id_type=open_id`。
+5. 以应用身份（`tenant_access_token`）调用接口，核对成功响应中 `data.user.name` 与 OWNER 本人一致。
+6. 复制完整的 `data.user.open_id` 交给 Educraft 部署人员。Open ID 通常以 `ou_` 开头。
+
+参数旁的“获取”是飞书文档调试台提供的成员选择功能，不是要求管理员预先知道 Open ID。不要复用其他应用查到的 Open ID；同一用户在不同应用下的 Open ID 不同。若无法选择成员或接口调用失败，依次检查：页面当前选择的 App ID、应用可用范围和通讯录数据范围是否覆盖 OWNER、用户基本信息与用户 ID 权限是否已开通并随应用版本发布。
+
+### 7.4 核对并交付
+
+交付前完成以下检查：
+
+- 返回用户的姓名与 OWNER 本人一致；
+- Open ID 来自本次组织的这一个 App ID；
+- Open ID 完整复制，没有空格或省略号；
+- 显示名称使用组织希望在 Educraft 中展示的姓名；
+- Union ID 不是必填项，查不到可以留空。
+
+最终向部署人员提供：
+
+```text
+OWNER Open ID：ou_...
+OWNER 显示名称：
+OWNER Union ID：（可选）
+用于查询的 App ID：cli_...
+```
+
+Open ID 和显示名称可以放在普通交付单中；不要把 App Secret 一起粘贴进去。
+
+## 8. 部署信息交付单
+
+请复制下面的模板填写。标注“安全渠道”的字段不要与普通字段放在同一条群消息或云文档中。
+
+```text
+【组织信息】
+组织正式名称：
+组织简称：
+期望 organization slug：（小写字母、数字和连字符，例如 example-school）
+期望机器人显示名称：
+
+【飞书应用】
+App ID：cli_...
+App Secret：（通过安全渠道单独发送）
+应用已发布：是 / 否
+机器人能力已启用：是 / 否
+消息事件（含解散群、机器人被移出群）和卡片回调已配置：是 / 否
+OAuth 重定向 URL 已配置：是 / 否
+
+【首位 OWNER】
+OWNER Open ID：ou_...
+OWNER 显示名称：
+OWNER Union ID：（可选）
+
+【试点范围】
+试点群名称：（可选，用于验收定位）
+初始 Team 名称：（可选；没有 Team 不影响首次部署）
+预计试用人数：
+
+【模型配置】
+Provider 名称：（例如 OpenRouter）
+Provider Base URL：
+Provider Token：（通过安全渠道单独发送）
+启用的模型 ID：
+```
+
+Educraft 部署人员收到信息后，会回传最终 organization slug、访问域名、部署窗口和验收时间。若期望 slug 已被占用或不符合命名规则，会在部署前协调调整。
+
+## 9. 上线验收
+
+部署人员通知服务就绪后，由 OWNER 完成：
+
+1. OWNER 在试点群中 @机器人发送一条纯文本消息。
+2. 如果群尚未绑定项目，确认机器人返回项目创建/绑定卡片。
+3. 创建项目后再次 @机器人，确认出现处理状态、流式卡片和最终回答。
+4. 选择一位非 OWNER 试点成员完成 OAuth 登录，确认其自动以 MEMBER 身份加入组织。
+5. 该成员在同一群中 @机器人，确认能够进入已绑定项目。
+6. 测试一个小文件附件、一次运行中断，以及一个需要生成文档的任务。
+
+出现问题时，请保留发生时间、群名、消息截图和飞书 request/log ID。截图前确认其中不包含 App Secret、Provider Token 或其他密钥。
+
+## 10. Alpha 阶段边界
+
+- 每个组织运行在独立的系统用户、服务实例、数据库和持久化目录中。
+- 组织的 role、system prompt、tools 和 skills 是运行时配置，不需要跟随版本发布。
+- 同一项目同一时间只执行一个任务，避免并发修改同一个 workspace；组织级并发上限由部署配置决定。
+- 当前由 Educraft 人工创建组织、OWNER、Provider Connection 和初始 Team，并通过服务器上的受控管理命令运维；组织管理台尚未开放。
+- 非 OWNER 试点成员通过组织专属 OAuth 首次登录后自动成为 MEMBER；OWNER/ADMIN 提权和被移除成员的恢复仍需人工操作。
+- Alpha 不提供开放注册、自助密钥管理或跨组织资源共享。

examples/trial/hello_world.txt

@@ -0,0 +1 @@
+Hello, World!

hub/.env.example

@@ -1,23 +1,72 @@
 # Hub runtime configuration. Copy to .env and fill in.
 
-# PostgreSQL connection string. Used by Prisma and the Hub server.
+# Production preflight requires the explicit production runtime mode.
+NODE_ENV="production"
+
+# PostgreSQL connection string.
 DATABASE_URL="postgresql://paradigm:paradigm@127.0.0.1:5432/paradigm"
 
-# OpenRouter (or any OpenAI-compatible) API key + base URL.
-# The Hub's agent layer is provider-agnostic (ADR-0017); this points the
-# OpenAI-compatible client at OpenRouter by default.
-OPENROUTER_API_KEY=""
-# Optional: override the base URL (e.g. direct vendor API, local gateway).
-# OPENROUTER_BASE_URL="https://openrouter.ai/api/v1"
+# Provider URL and credentials are write-only Organization Provider Connection
+# data in PostgreSQL, encrypted by ADR-0024. Process-global ANTHROPIC_* provider
+# settings are rejected in production and must not be added here.
 
-# Hub server port. Defaults to 8788.
+# Model override (optional). Defaults to anthropic/claude-sonnet-5.
+# Use an OpenRouter model slug, for example:
+# ANTHROPIC_DEFAULT_SONNET_MODEL="~anthropic/claude-sonnet-latest"
+# To use GLM:
+# ANTHROPIC_DEFAULT_SONNET_MODEL="z-ai/glm-4.7"
+# ANTHROPIC_DEFAULT_OPUS_MODEL="z-ai/glm-4.7"
+# ANTHROPIC_DEFAULT_HAIKU_MODEL="z-ai/glm-4.6"
+
+# Alpha Silo safety limits. max turns may use its default; every other value is
+# mandatory in production and should be calibrated on the target host.
+# HUB_AGENT_MAX_TURNS=25
+HUB_AGENT_MAX_CONCURRENT_RUNS="1"
+HUB_AGENT_MAX_RUN_SECONDS="900"
+HUB_HTTP_BODY_LIMIT_BYTES="1048576"
+HUB_MAX_FILES_PER_MESSAGE="8"
+HUB_MAX_FILE_BYTES="26214400"
+HUB_HTTP_REQUESTS_PER_MINUTE="120"
+HUB_FEISHU_EVENTS_PER_MINUTE="120"
+
+# Persistent system-managed root for project workspaces. Production must use an
+# absolute path outside the deployment/release tree; install_service.sh defaults
+# to this path and rejects any overlap before installing the unit.
+HUB_PROJECT_WORKSPACE_ROOT="/var/lib/cph-hub/workspaces"
+
+# Persistent root for the content-addressed agent skill store. Required at hub
+# startup unless XDG_STATE_HOME is set (then defaults to $XDG_STATE_HOME/skills).
+HUB_SKILL_STORE_ROOT="/var/lib/cph-hub/state/skills"
+
+# This process is pinned to exactly one Organization. Feishu credentials are
+# resolved from that Organization's encrypted ACTIVE connection.
+HUB_SILO_ORGANIZATION_ID=""
+HUB_SYSTEMD_UNIT="cph-hub-example.service"
+
+# Absolute path to the `cph` binary (ADR-0016). Production preflight requires
+# the file to be executable and `cph --version` to succeed.
+CPH_BIN="/usr/local/bin/cph"
+
+# Hub bind address and port. Production defaults to loopback for a local TLS
+# reverse proxy; both values are validated and honored by the HTTP server.
+HOST="127.0.0.1"
 PORT=8788
 
-# Feishu (lark) bot credentials. The bot receives @mentions in project groups.
-FEISHU_APP_ID=""
-FEISHU_APP_SECRET=""
-FEISHU_BOT_OPEN_ID=""
+# --- Org admin web (ADR-0021) ---------------------------------------------
+# Public base URL of this Hub (no trailing slash). Used for Feishu OAuth
+# redirect_uri = ${HUB_PUBLIC_BASE_URL}/auth/feishu/callback
+# Configure the same callback URL in the Feishu developer console under
+# Security Settings → Redirect URLs.
+HUB_PUBLIC_BASE_URL="https://hub.example.com"
 
-# Path to the `cph` binary (the Courseware-side checker, ADR-0016).
-# Defaults to `cph` on PATH if unset.
-# CPH_BIN="/usr/local/bin/cph"
+# HMAC secret for signed session + OAuth state cookies. Generate with:
+#   openssl rand -base64 32
+HUB_SESSION_SECRET=""
+
+# Optional OAuth scope (space-separated). Default: contact:user.base:readonly
+# Apply matching scopes in the Feishu developer console.
+# HUB_OAUTH_SCOPE="contact:user.base:readonly"
+
+# Local development/tests only. Production must not set this: systemd injects
+# the fixed cph-secret-keyring credential through CREDENTIALS_DIRECTORY.
+# HUB_SECRET_KEYRING_FILE="/absolute/path/to/dev-keyring.json"

hub/.gitignore

@@ -4,3 +4,7 @@ dist/
 .env
 .env.*
 !.env.example
+.secrets/
+admin-web/node_modules/
+admin-web/build/
+admin-web/.svelte-kit/

hub/admin-web/.gitignore

@@ -0,0 +1,23 @@
+node_modules
+
+# Output
+.output
+.vercel
+.netlify
+.wrangler
+/.svelte-kit
+/build
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Env
+.env
+.env.*
+!.env.example
+!.env.test
+
+# Vite
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*

hub/admin-web/.npmrc

@@ -0,0 +1 @@
+engine-strict=true