forked from EduCraft/curriculum-project-hub
Compare commits
95 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d36b00bbec | |||
| 17c0536958 | |||
| 96e120e02c | |||
| 1a892ccb54 | |||
| 9d494f446f | |||
| 5a65188b5d | |||
| 7fcb57013e | |||
| 035c264179 | |||
| 38d9a9c6cb | |||
| cb1a9e823a | |||
| 9e954790dc | |||
| 44557da499 | |||
| 848f913597 | |||
| e049cb6880 | |||
| d730e51c3d | |||
| f07f280b8f | |||
| f4968d6657 | |||
| 73fa28a178 | |||
| 5f791c5ce4 | |||
| 420c1c40bc | |||
| 913a7ae5c0 | |||
| 599a2f6e66 | |||
| 1094ebd651 | |||
| 4ca4afb141 | |||
| 16329df357 | |||
| ec630b9151 | |||
| 682bc70563 | |||
| 5e412761d2 | |||
| 90cbd74562 | |||
| 2dc125aec1 | |||
| b0eedfab05 | |||
| 008c8bcd09 | |||
| 683d5674d1 | |||
| 1f8510a47f | |||
| c4f052efa2 | |||
| 87e3d3f990 | |||
| 519ea8144f | |||
| 34e07e229f | |||
| 2b7aa3294f | |||
| 5d315fff21 | |||
| 716101eed0 | |||
| fc5a5365d8 | |||
| fc0df7b823 | |||
| f1d29176d3 | |||
| f260195439 | |||
| d01ea0e1cb | |||
| d6724e5a83 | |||
| 346aee5a68 | |||
| 491fd13ca8 | |||
| 79505e6d44 | |||
| a0df8b6bcf | |||
| 36c0801e6f | |||
| 4479f88f4f | |||
| 2736006262 | |||
| 294d51dbfe | |||
| bef10149a9 | |||
| 4736610cf3 | |||
| ad65d93b2e | |||
| 6227e1931b | |||
| a00e4f9871 | |||
| a025d23af8 | |||
| dfcc2d70f8 | |||
| 3f486232a8 | |||
| a766d99573 | |||
| 1cfda3ccd2 | |||
| ead5cf04d6 | |||
| ecbc2c8666 | |||
| 5b7fd70124 | |||
| 6178664696 | |||
| c73b41e1da | |||
| 012c8a7d89 | |||
| 250f918346 | |||
| 30d20421b3 | |||
| 20bf7e271f | |||
| eaa7fd4fd8 | |||
| 1ad78dbcce | |||
| 085f7c7186 | |||
| b9f50407be | |||
| 082562ca08 | |||
| 4bd5b03bb1 | |||
| 1abcc495f4 | |||
| e7924f78d5 | |||
| f8c3e16a52 | |||
| a2c8fa8eaf | |||
| afaf5bee09 | |||
| 0fe6cabc68 | |||
| e60aa43731 | |||
| 09cadd0148 | |||
| ce767e9cab | |||
| a4d52e1850 | |||
| 313e890b6c | |||
| b4dd8f09e1 | |||
| a08c33205b | |||
| 3bca137b48 | |||
| 18aac1ff16 |
@@ -19,8 +19,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust toolchain (stable + clippy + rustfmt)
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
# Keep local rustup, rustfmt output, clippy, and CI on one reproducible
|
||||
# compiler release. Update this together with rust-toolchain.toml.
|
||||
- name: Install Rust toolchain (1.92.0 + clippy + rustfmt)
|
||||
uses: dtolnay/rust-toolchain@1.92.0
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
|
||||
@@ -35,6 +37,12 @@ jobs:
|
||||
restore-keys: |
|
||||
cargo-${{ runner.os }}-
|
||||
|
||||
- name: Install cargo-audit 0.22.2
|
||||
run: cargo install cargo-audit --version 0.22.2 --locked
|
||||
|
||||
- name: Audit Rust vulnerabilities and unsoundness
|
||||
run: cargo audit --deny unsound
|
||||
|
||||
# Fonts for typst compilation (WU-4 onward): the engineering files are
|
||||
# CJK-heavy, so the compile-check / build tests need a CJK font present.
|
||||
# Harmless before those crates exist.
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
name: hub check
|
||||
|
||||
# Builds, type-checks, and tests the Hub TS package under hub/.
|
||||
# The Hub is the Feishu-group collaboration + agent runtime half
|
||||
# (spec/System implementation). This is an INTERNAL gate on the Hub's own
|
||||
# health, like checker-check is for the Rust half.
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
hub-check:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_USER: paradigm
|
||||
POSTGRES_PASSWORD: paradigm
|
||||
POSTGRES_DB: cph_hub_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U paradigm -d cph_hub_test"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
defaults:
|
||||
run:
|
||||
working-directory: hub
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
cache-dependency-path: hub/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Audit production Node dependencies
|
||||
run: npm run audit:production
|
||||
|
||||
- name: Install Linux sandbox dependency
|
||||
run: sudo apt-get update && sudo apt-get install --yes bubblewrap socat
|
||||
|
||||
- name: Wait for Postgres
|
||||
run: |
|
||||
node <<'NODE'
|
||||
const net = require("node:net");
|
||||
const deadline = Date.now() + 60000;
|
||||
function tryConnect() {
|
||||
const socket = net.createConnection({ host: "127.0.0.1", port: 5432 });
|
||||
socket.once("connect", () => {
|
||||
socket.end();
|
||||
process.exit(0);
|
||||
});
|
||||
socket.once("error", () => {
|
||||
socket.destroy();
|
||||
if (Date.now() > deadline) {
|
||||
console.error("Postgres did not become reachable at 127.0.0.1:5432");
|
||||
process.exit(1);
|
||||
}
|
||||
setTimeout(tryConnect, 1000);
|
||||
});
|
||||
}
|
||||
tryConnect();
|
||||
NODE
|
||||
|
||||
# Prisma client must be generated before tsc can check imports from
|
||||
# @prisma/client. Stub DATABASE_URL so prisma generate works.
|
||||
- name: Generate Prisma client
|
||||
run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma generate --schema prisma/schema.prisma
|
||||
|
||||
- name: Type-check
|
||||
run: npx tsc -p tsconfig.json --noEmit
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Validate Prisma schema
|
||||
run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma validate --schema prisma/schema.prisma
|
||||
|
||||
- name: Install cph binary
|
||||
run: |
|
||||
cd ..
|
||||
cargo install --path crates/cph-cli --locked
|
||||
|
||||
- name: Prove real Claude SDK Bash sandbox boundary
|
||||
run: |
|
||||
sudo install -d -o "$(id -u)" -g "$(id -g)" -m 0700 /w/t
|
||||
CPH_SANDBOX_TEST_ROOT=/w/t \
|
||||
/usr/bin/setpriv --no-new-privs \
|
||||
npx vitest run test/integration/agent-sandbox-linux.test.ts
|
||||
|
||||
- name: Run unit tests
|
||||
run: npx vitest run test/unit
|
||||
|
||||
# Integration tests need PostgreSQL + cph. cph is installed above.
|
||||
# PostgreSQL is set up as a service container below.
|
||||
- name: Run integration tests (mock provider, real prisma + cph)
|
||||
run: |
|
||||
npx prisma migrate deploy --schema prisma/schema.prisma
|
||||
npx vitest run test/integration \
|
||||
--exclude test/integration/real-model.test.ts \
|
||||
--exclude test/integration/agent-sandbox-linux.test.ts
|
||||
env:
|
||||
DATABASE_URL: postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test
|
||||
|
||||
# Real-model tests are opt-in: set RUN_REAL_MODEL_TESTS=true and provide
|
||||
# OPENROUTER_API_KEY when a branch should hit live OpenRouter.
|
||||
- name: Run real-model integration tests (optional, needs secret)
|
||||
run: npx vitest run test/integration/real-model.test.ts
|
||||
env:
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
RUN_REAL_MODEL_TESTS: ${{ vars.RUN_REAL_MODEL_TESTS }}
|
||||
@@ -11,5 +11,8 @@
|
||||
# regenerable, not for VCS. The embedded engine mounts cph-render directly.
|
||||
render/vendor/local-packages/
|
||||
|
||||
# Node (hub/ TS workspace and any future JS package)
|
||||
node_modules/
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
|
||||
+325
@@ -0,0 +1,325 @@
|
||||
# Backup, migration, and disaster-recovery audit
|
||||
|
||||
## Verdict
|
||||
|
||||
The current Hub has no production backup or restore control plane. PostgreSQL
|
||||
can be dumped and restored manually, but the service's authoritative state is
|
||||
split between PostgreSQL and mutable Project workspaces, with no shared
|
||||
consistency point, backup catalog, off-host retention, restore workflow,
|
||||
recovery objectives, or recovery drill.
|
||||
|
||||
The current deployment path can also delete customer workspaces when the
|
||||
documented relative workspace root is used: `deploy_platform.sh` applies
|
||||
`rsync --delete` to the live Hub directory while `.env.example` places
|
||||
workspaces below `./data/project-workspaces`. Project creation has the same
|
||||
cross-store weakness at a smaller scale: it creates the workspace before the
|
||||
database transaction and leaves an orphan directory when that transaction
|
||||
fails.
|
||||
|
||||
Migration safety is incomplete even though the current PostgreSQL migration
|
||||
engine rolled back a deliberately failing migration's DDL in the executed
|
||||
probe. The failed migration record blocks later migrations, migrations execute
|
||||
inside every service start, no compatibility declaration or production-data
|
||||
rehearsal exists, and current history contains destructive changes that make an
|
||||
old application query fail after schema upgrade. Application rollback is
|
||||
therefore not database rollback.
|
||||
|
||||
## Contract and persistent-state baseline
|
||||
|
||||
The audit preserves the accepted contracts rather than inventing missing
|
||||
product policy:
|
||||
|
||||
- `Organization` is the tenant root, but `User` is global and can belong to
|
||||
multiple Organizations. A selective Organization restore is not equivalent
|
||||
to restoring a self-contained database schema.
|
||||
- Project workspaces are authoritative curriculum-engineering files bounded by
|
||||
ADR-0018. PostgreSQL stores their absolute `Project.workspaceDir` locations;
|
||||
it does not store their file content.
|
||||
- Run, provider cursor, ingress, delivery, audit, and workspace commit/recovery
|
||||
semantics remain split across the unresolved lifecycle tickets. A backup
|
||||
design cannot silently choose their crash behavior.
|
||||
- ADR-0022 requires storage ceilings and emergency `DRAIN`/`STOP_NOW`, but does
|
||||
not decide backup frequency, retention, recovery objectives, or tenant-level
|
||||
restore scope.
|
||||
- Audit retention is still `OPEN`; regulatory backup retention and deletion
|
||||
obligations require product/legal input already recorded in the map's Fog.
|
||||
|
||||
The recovery set has at least four classes:
|
||||
|
||||
| State | Current location | Recovery significance |
|
||||
| --- | --- | --- |
|
||||
| Tenant and workflow state | PostgreSQL | Organizations, identities, membership, permissions, Feishu bindings, sessions, runs, locks, receipts, usage facts, and audit records |
|
||||
| Customer artifacts | `HUB_PROJECT_WORKSPACE_ROOT` | Project source files, generated artifacts, inbound attachments, and any workspace-local session/history files |
|
||||
| Credentials and configuration | `/srv/.../.secrets/platform.env` today; future org-scoped encrypted connections plus external key material | Database/Feishu/provider access and ability to decrypt restored Organization credentials |
|
||||
| Release identity | application release, Prisma migrations, `cph`, runtime/toolchain manifest | Required to start a schema-compatible application and reproduce outputs after restore |
|
||||
|
||||
Caches, build output, and reinstallable dependencies should be classified as
|
||||
regenerable rather than silently mixed into the protected recovery set.
|
||||
|
||||
## Executed evidence
|
||||
|
||||
All probes used isolated temporary paths or databases. No production or shared
|
||||
test data was modified.
|
||||
|
||||
### 1. The deployment rsync pattern deletes an in-tree workspace
|
||||
|
||||
A temporary live Hub directory was given the documented workspace shape and a
|
||||
sentinel customer file, then the exact deployment rsync include/exclude pattern
|
||||
was applied:
|
||||
|
||||
```text
|
||||
rsync -a --delete \
|
||||
--exclude node_modules --exclude dist --exclude .env \
|
||||
hub/ <temporary-live-hub>/
|
||||
|
||||
workspace_survived=false
|
||||
reproduced: deploy rsync pattern deletes an in-tree workspace root
|
||||
```
|
||||
|
||||
This is deterministic: `data/` is absent from the release source and is not an
|
||||
exclude, so `--delete` removes it from the destination. The root cause is that
|
||||
the documented persistent path is nested inside the mutable release tree and
|
||||
the deploy contract does not distinguish release artifacts from state.
|
||||
|
||||
### 2. A rejected Project transaction leaves an orphan workspace
|
||||
|
||||
A minimal Node harness called the real built `createProjectFromOrgAdmin` with a
|
||||
valid Organization admin and a missing Folder. The fake Prisma boundary rejected
|
||||
inside the same transaction seam used by production. The harness asserted that
|
||||
the Organization workspace directory must remain empty and was run twice:
|
||||
|
||||
```text
|
||||
node .scratch/saas-production-readiness/probes/orphan-workspace.mjs
|
||||
|
||||
orphan_workspace_entries=project_4aa0b6792c4547059d2d92b505b9299e
|
||||
exit_code=1
|
||||
|
||||
orphan_workspace_entries=project_190e21afb8854244872b1241a600aa58
|
||||
exit_code=1
|
||||
```
|
||||
|
||||
`createManagedProject` calls `mkdir` before `prisma.$transaction`; the failure
|
||||
path has no compensating removal, staged rename, or orphan reconciler. The
|
||||
throwaway harness was removed after reproduction.
|
||||
|
||||
### 3. PostgreSQL rolled back failed migration DDL but retained a blocking failure
|
||||
|
||||
An isolated database and throwaway Prisma project applied one migration that
|
||||
created a table, divided by zero, then would have created another table. The
|
||||
real repository Prisma CLI returned `P3018`:
|
||||
|
||||
```text
|
||||
migration_exit=1
|
||||
before_failure=missing
|
||||
after_failure=missing
|
||||
20260710000000_partial_failure|finished=false|rolled_back=false|steps=0|logs=true
|
||||
```
|
||||
|
||||
This specific PostgreSQL/Prisma 6.19.3 path did **not** leave partial DDL, so the
|
||||
audit does not claim that it did. It did leave an unfinished
|
||||
`_prisma_migrations` row, and Prisma's error explicitly refused new migrations
|
||||
until operator recovery. Because systemd runs `migrate deploy` as
|
||||
`ExecStartPre`, the application cannot start past that state. The throwaway
|
||||
database and migration were removed. This one multi-statement result is not a
|
||||
general atomicity guarantee: Prisma's first-party guidance says PostgreSQL
|
||||
migrations are not transaction-wrapped by default, so each migration must make
|
||||
its transaction and partial-failure behavior explicit.
|
||||
|
||||
### 4. The current schema is not backward-compatible with an older application
|
||||
|
||||
The application immediately before the typed-principal migration queried
|
||||
`PermissionGrant.principal`. That migration drops the column in favor of
|
||||
`principalType` plus `principalId`. Running the old read against the current
|
||||
test database produced:
|
||||
|
||||
```text
|
||||
ERROR: column "principal" does not exist
|
||||
HINT: Perhaps you meant to reference the column "PermissionGrant.principalId".
|
||||
old_query_exit=1
|
||||
```
|
||||
|
||||
This is direct evidence that switching the application symlink back after that
|
||||
migration cannot restore service. The migration history also deletes duplicate
|
||||
grant rows and later drops `ExternalPrincipalMembership.source`; there are no
|
||||
downgrade migrations.
|
||||
|
||||
### 5. A database-only custom dump restores cleanly
|
||||
|
||||
As a positive baseline, `pg_dump --format=custom` captured the current
|
||||
`cph_hub_test` database and `pg_restore --exit-on-error --no-owner` restored it
|
||||
into a new isolated database:
|
||||
|
||||
```text
|
||||
restored_tables=24
|
||||
finished_migrations=9
|
||||
failed_migrations=0
|
||||
dump_bytes=76113
|
||||
```
|
||||
|
||||
All nine finished migration checksums in the source database also matched the
|
||||
checked-in migration files. This proves a manual logical database round trip on
|
||||
the current PostgreSQL 14.20 workstation. It does not prove workspace,
|
||||
credentials, global database roles, production volume, cross-version restore,
|
||||
application readiness, or an operational RPO/RTO.
|
||||
|
||||
## Critical findings
|
||||
|
||||
### 1. No backup, retention, or restore mechanism exists
|
||||
|
||||
The repository has no backup/restore command, systemd timer, off-host target,
|
||||
encryption/immutability policy, retention policy, backup manifest/checksum,
|
||||
success/failure alert, restore command, or scheduled recovery drill. The
|
||||
installer assumes PostgreSQL and its role/database already exist. A database
|
||||
dump alone does not reproduce those global prerequisites or the filesystem and
|
||||
secret state above.
|
||||
|
||||
There is also no decided RPO, RTO, backup frequency, retention duration,
|
||||
geographic copy rule, deletion-in-backups rule, or ownership/escalation path.
|
||||
Those values are product and operations decisions, not safe constants to infer
|
||||
from the single-host topology.
|
||||
|
||||
### 2. Persistent workspaces are inside the documented release tree
|
||||
|
||||
`.env.example` recommends `./data/project-workspaces`; systemd's working
|
||||
directory is the live Hub directory; deployment mutates that same directory
|
||||
with `rsync --delete`. No preflight forbids an in-release workspace root and no
|
||||
exclude preserves it. This can turn a routine deploy into irreversible customer
|
||||
data loss before build, migration, restart, or readiness runs.
|
||||
|
||||
The persistent workspace root must live outside every release directory, have
|
||||
explicit ownership/mode and capacity, and be impossible for release retention
|
||||
or `rsync --delete` to traverse. Immutable release work already exists as a
|
||||
separate ticket, but its acceptance test must include this sentinel case.
|
||||
|
||||
### 3. PostgreSQL and workspaces have no consistency boundary
|
||||
|
||||
PostgreSQL can provide a transactionally consistent database snapshot, but the
|
||||
Hub mutates workspace files independently and has no generation, journal,
|
||||
commit marker, snapshot ID, or shared checkpoint that binds filesystem content
|
||||
to the corresponding database state. A database dump and filesystem copy taken
|
||||
at different instants can disagree about Project existence, active runs,
|
||||
provider cursors, delivered messages, and file effects.
|
||||
|
||||
Project creation already demonstrates the same missing compensation boundary.
|
||||
A replacement host has an additional problem: `Project.workspaceDir` is an
|
||||
absolute path and runtime paths from the database are used directly; restore
|
||||
has no same-mount validation, safe rebasing operation, or missing/orphan scan.
|
||||
|
||||
For the initial single-host topology, a quiesced backup checkpoint is the
|
||||
smallest credible route: stop new admission, drain/cancel according to the
|
||||
accepted run contract, reach durable zero in-flight work, capture database and
|
||||
workspace artifacts under one manifest, then resume. An online scheme would
|
||||
need a newly designed journal/generation protocol; it must not be implied by
|
||||
two independent copy commands.
|
||||
|
||||
### 4. Schema mutation is coupled to service start
|
||||
|
||||
Both `npm start` and the systemd unit execute `prisma migrate deploy`; every
|
||||
ordinary restart can therefore become a schema rollout. The release path has no
|
||||
separate migration lock/gate, `migrate status` check, production-data rehearsal,
|
||||
duration/lock budget, pre-migration recovery checkpoint, or compatibility
|
||||
manifest. CI only applies the full migration history to a fresh empty
|
||||
PostgreSQL database, so it does not exercise real upgrade volume, legacy values,
|
||||
old/new application overlap, failure recovery, or rollback compatibility.
|
||||
|
||||
The current failed-migration behavior is fail-closed, which is better than
|
||||
starting on an unknown schema, but there is no operator procedure for inspecting
|
||||
the failure, deciding roll-forward versus mark-rolled-back, restoring the
|
||||
checkpoint, or preventing a restart loop. Existing destructive migrations prove
|
||||
that application rollback cannot be assumed safe.
|
||||
|
||||
### 5. Restore has no isolated verification or cutover gate
|
||||
|
||||
There is no procedure that restores into a non-production environment with
|
||||
outbound Feishu/provider traffic disabled, verifies backup checksums and
|
||||
migration state, validates workspace paths/content/permissions, checks tenant
|
||||
relationships, decrypts a sample org connection, reconciles non-terminal work,
|
||||
runs application readiness and representative journeys, and measures recovery
|
||||
time/point before controlled cutover.
|
||||
|
||||
Starting the current server is not a neutral verifier: startup deletes all
|
||||
Project locks and independently marks only `ACTIVE` runs failed. Restored data
|
||||
must first pass the durable lifecycle reconciliation designed by the run-state
|
||||
tickets; otherwise validation itself mutates the evidence and can leave
|
||||
`WAITING_FOR_USER` inconsistent.
|
||||
|
||||
### 6. Full-service and Organization-selective recovery are different products
|
||||
|
||||
A full-service restore can recreate the shared database and all workspaces.
|
||||
Restoring one Organization is much harder: users are global and may belong to
|
||||
multiple Organizations, permission/audit/history rows span many related tables,
|
||||
workspace paths are external, and Feishu/provider connections have external
|
||||
state. The repository has no export/import identity mapping or conflict policy.
|
||||
|
||||
The initial production gate must at least prove full-service disaster recovery.
|
||||
Whether Organization-selective restore/export is promised, and how retained
|
||||
backups interact with customer deletion, remains an explicit product/legal
|
||||
decision rather than an accidental property of `pg_restore` filters.
|
||||
|
||||
## Required recovery contract
|
||||
|
||||
Production readiness requires the following guarantees, with exact numerical
|
||||
objectives supplied by the pending policy decision:
|
||||
|
||||
1. **Declared objectives and ownership.** RPO/RTO, backup frequency, retention,
|
||||
copy locations, encryption/key custody, immutability, deletion handling,
|
||||
full-service versus Organization restore scope, operator ownership, and
|
||||
escalation are documented and testable.
|
||||
2. **Complete backup set.** One cataloged manifest identifies release and
|
||||
migration checksums, PostgreSQL artifact and global prerequisites, workspace
|
||||
snapshot, configuration inventory, encrypted org-secret data, external key
|
||||
recovery material, timestamps, sizes, and cryptographic checksums. Backup
|
||||
data and decryption material are not kept only on the protected host or in
|
||||
the same failure domain.
|
||||
3. **One consistency point.** A backup is taken only after an observable
|
||||
quiesce/checkpoint protocol, or by a proven online generation protocol that
|
||||
binds database state and workspace effects. Accepted input, active runs,
|
||||
queues, deliveries, locks, and provider cursors have explicit recovery
|
||||
semantics.
|
||||
4. **Safe migration rollout.** Migration is a release phase, not an incidental
|
||||
service start. The gate checks status/checksums, rehearses on representative
|
||||
production-shaped data, records old/new application compatibility, uses
|
||||
expand-and-contract for destructive changes, requires a recent verified
|
||||
recovery checkpoint, and provides a tested failed-migration procedure.
|
||||
5. **Isolated restore verification.** Every backup class is periodically
|
||||
restored to a clean isolated host/database with outbound integrations off.
|
||||
Verification checks integrity, schema/migration state, global database
|
||||
prerequisites, workspace completeness/path ownership, secret decryption,
|
||||
tenant isolation, lifecycle reconciliation, application readiness, and
|
||||
representative admin/Feishu/agent journeys.
|
||||
6. **Observable operation.** Backup age, duration, artifact/checksum status,
|
||||
off-host replication, retention deletion, WAL/archive continuity if used,
|
||||
restore-drill result, measured RPO/RTO, and failures are logged/metricized
|
||||
and alerted with bounded cardinality.
|
||||
7. **Controlled cutover and fallback.** Disaster recovery has an explicit
|
||||
authority, traffic/workload brake, DNS/proxy/Feishu reconnect sequence,
|
||||
duplicate/gap checks, rollback/fallback point, verification checklist, and
|
||||
incident evidence record.
|
||||
|
||||
## Required release evidence
|
||||
|
||||
- A deploy test proves an in-tree or release-overlapping workspace root is
|
||||
rejected before any rsync/switch, while persistent sentinel files survive
|
||||
build failure, successful release, rollback, and release pruning.
|
||||
- Project creation, archive/deletion, failed transaction, crash, and restore
|
||||
tests prove database/workspace allocation has no silent missing or orphan
|
||||
state; the reconciler reports rather than hides inconsistencies.
|
||||
- Every checked-in migration is rehearsed from the previous production-shaped
|
||||
snapshot, with old/new compatibility and lock/runtime evidence. A deliberately
|
||||
failed migration exercises the documented recovery route.
|
||||
- The backup implementation proves completeness, checksum verification,
|
||||
encryption, off-host copy, retention, monitoring, and fail-closed partial
|
||||
artifact cleanup.
|
||||
- A clean isolated host restores a representative database, workspaces,
|
||||
secrets, and compatible release; lifecycle reconciliation and critical tenant
|
||||
journeys pass with outbound integrations controlled.
|
||||
- Scheduled drills publish achieved recovery point and recovery time against the
|
||||
accepted objectives, and alert when no recent successful drill exists.
|
||||
|
||||
## Primary-source interpretation
|
||||
|
||||
The official PostgreSQL, Prisma, and systemd behavior supporting this audit is
|
||||
collected with claim-level links in
|
||||
[Backup and migration primary sources](./backup-migration-primary-sources.md).
|
||||
That note distinguishes what a PostgreSQL-consistent dump proves from the
|
||||
cross-store guarantees the Hub must build itself.
|
||||
@@ -0,0 +1,574 @@
|
||||
# Backup, migration, and disaster-recovery primary-source research
|
||||
|
||||
This note is the primary-source evidence pack for the Wayfinder ticket
|
||||
“Audit backup, migration, and disaster recovery safety.” External claims use
|
||||
only PostgreSQL, Prisma, systemd, and rsync first-party documentation. Project
|
||||
recommendations are explicitly labelled as derived implications rather than
|
||||
upstream guarantees.
|
||||
|
||||
PostgreSQL links below use `docs/current`, which was PostgreSQL 18 when this
|
||||
research was performed on 2026-07-10. A production implementation must pin the
|
||||
deployed PostgreSQL major and verify the same pages for that major. The Hub
|
||||
currently depends on Prisma `^6.19.3`, so Prisma 6-specific command examples
|
||||
remain relevant even where the live documentation also describes Prisma 7.
|
||||
|
||||
## Executive conclusions
|
||||
|
||||
1. `pg_dump` can take a consistent **single-database** snapshot while ordinary
|
||||
reads and writes continue, but it does not include the Hub's workspace tree,
|
||||
cluster-global roles/tablespaces, or a WAL chain. PostgreSQL now explicitly
|
||||
says it is generally not the right regular production-backup mechanism
|
||||
outside simple cases. It is a useful logical/export layer, not proof of
|
||||
whole-service disaster recovery.
|
||||
([`pg_dump` description](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION),
|
||||
[continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
|
||||
2. Database PITR requires a physical base backup plus a continuous, unbroken
|
||||
WAL sequence from the backup start through the recovery target. A successful
|
||||
base-backup command alone does not prove that chain exists.
|
||||
([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING),
|
||||
[making a base backup](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-BASE-BACKUP))
|
||||
3. `pg_verifybackup` is necessary but not sufficient: PostgreSQL explicitly
|
||||
requires test restores followed by checks that the restored database works
|
||||
and contains the expected data.
|
||||
([`pg_verifybackup`](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
|
||||
4. Prisma `migrate deploy` applies pending migrations, but it does not detect
|
||||
schema drift and, on PostgreSQL, Prisma does not wrap a migration in a
|
||||
transaction by default. A failed migration can therefore leave partial
|
||||
schema changes requiring explicit operator recovery.
|
||||
([Prisma production workflow](https://www.prisma.io/docs/orm/prisma-migrate/workflows/development-and-production#production-and-testing-environments),
|
||||
[Prisma transactional behaviour](https://www.prisma.io/blog/prisma-migrate-dx-primitives#what-if-schema-migrations-were-atomic),
|
||||
[failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration))
|
||||
5. systemd supplies process lifecycle orchestration, not an application-level
|
||||
consistency barrier. It sends the configured termination signal, waits up
|
||||
to `TimeoutStopSec=`, then can forcibly kill remaining processes; a
|
||||
successful `systemctl stop` does not prove that an `ExecStop=` command or
|
||||
application drain succeeded.
|
||||
([`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=),
|
||||
[`systemctl stop`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#stop%20PATTERN%E2%80%A6))
|
||||
6. The Hub persists one logical service state across two stores: PostgreSQL
|
||||
records `Project.workspaceDir`, runs, messages, locks, and file-change audit
|
||||
rows, while workspaces contain project files. A workspace transcript helper
|
||||
exists but is not wired into the production runner. PostgreSQL's snapshot/
|
||||
PITR guarantees cover only PostgreSQL. A whole-service recovery point
|
||||
therefore requires an application-defined database/workspace checkpoint or
|
||||
a journaled/versioned reconciliation protocol; neither PostgreSQL nor
|
||||
systemd creates one automatically.
|
||||
([Hub schema](../../../hub/prisma/schema.prisma),
|
||||
[workspace transcript helper](../../../hub/src/agent/transcript.ts),
|
||||
[`pg_dump` scope](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION))
|
||||
|
||||
## Current repository facts that make the boundaries material
|
||||
|
||||
- The systemd unit runs `prisma migrate deploy` as `ExecStartPre=` before every
|
||||
service start and uses `Restart=on-failure`.
|
||||
([`cph-hub.service`](../../../hub/deploy/cph-hub.service))
|
||||
- The deploy script updates the live Hub directory using
|
||||
`rsync -az --delete`, builds in that same directory, restarts the unit, and
|
||||
accepts a constant liveness endpoint as deployment success.
|
||||
([`deploy_platform.sh`](../../../hub/deploy/deploy_platform.sh))
|
||||
- `Project.workspaceDir` is database state, but the referenced directory holds
|
||||
independently mutable tenant content. A workspace JSONL transcript helper
|
||||
exists but is not wired into the production runner; file contents still live
|
||||
in the workspace while the database separately carries run/message/file-change
|
||||
facts.
|
||||
([Hub schema](../../../hub/prisma/schema.prisma),
|
||||
[workspace transcript helper](../../../hub/src/agent/transcript.ts))
|
||||
|
||||
These are project facts, not upstream guarantees. The sections below establish
|
||||
what the upstream tools do and, critically, what they do not do.
|
||||
|
||||
## PostgreSQL logical dump and restore
|
||||
|
||||
### Snapshot and concurrent-write semantics
|
||||
|
||||
- `pg_dump` makes a consistent export even while the database is being used;
|
||||
it does not block ordinary readers or writers. It dumps only one database.
|
||||
Cluster-global objects such as roles and tablespaces require `pg_dumpall` or
|
||||
`pg_dumpall --globals-only`.
|
||||
([`pg_dump` description](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION),
|
||||
[whole-cluster logical dumps](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-ALL))
|
||||
- `pg_dumpall` invokes `pg_dump` once per database. Each database is internally
|
||||
consistent, but the snapshots are not synchronized across databases.
|
||||
([whole-cluster logical dumps](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-ALL))
|
||||
- A parallel dump uses directory format, opens `njobs + 1` connections, and
|
||||
uses synchronized snapshots so every worker sees the same data set. The
|
||||
connection count and additional server load are part of its capacity cost.
|
||||
([`pg_dump --jobs`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
|
||||
- Concurrent DDL can still interfere. The parallel-dump leader holds
|
||||
`ACCESS SHARE` locks; if a later exclusive-lock request queues ahead of a
|
||||
worker, that worker uses `NOWAIT` and aborts the dump rather than deadlocking.
|
||||
`--lock-wait-timeout` can bound the initial lock wait.
|
||||
([`pg_dump --jobs` and lock behaviour](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
|
||||
- The ordinary snapshot is suitable for disaster-recovery dump semantics, but
|
||||
it need not correspond to a serial execution of all transactions that later
|
||||
commit. `--serializable-deferrable` waits for an anomaly-free snapshot;
|
||||
PostgreSQL says that option is not beneficial for a dump intended only for
|
||||
disaster recovery, but it can matter for a continuously updated reporting
|
||||
copy.
|
||||
([`--serializable-deferrable`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
|
||||
- The default waits for dump files to be written safely to disk. Production
|
||||
backups must not use `--no-sync`, because an immediate OS crash can otherwise
|
||||
leave a corrupt dump.
|
||||
([`pg_dump --no-sync`](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-OPTIONS))
|
||||
|
||||
Derived Hub implications:
|
||||
|
||||
- An online database dump may be internally valid while referring to a
|
||||
`workspaceDir` whose files are from another instant. “`pg_dump` succeeded” is
|
||||
therefore not a whole-Hub consistency claim.
|
||||
- Backup scheduling must either serialize against schema migration/DDL or
|
||||
treat any lock wait, abort, non-zero exit, and stderr warning as a failed
|
||||
backup. A partially written archive must never be published as restorable.
|
||||
- If the deployed cluster contains more than the Hub database, a single Hub
|
||||
dump is not a cluster backup. Even `pg_dumpall` does not create a single
|
||||
cross-database snapshot.
|
||||
|
||||
### Restore error and atomicity semantics
|
||||
|
||||
- `pg_restore` defaults to continuing after SQL errors and reports an error
|
||||
count at the end. `--exit-on-error` is required for fail-fast behaviour.
|
||||
`--single-transaction` wraps all emitted commands in `BEGIN`/`COMMIT`, makes
|
||||
the restore all-or-nothing, and implies `--exit-on-error`.
|
||||
([`pg_restore` error and transaction options](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
|
||||
- Parallel restore uses separate connections for data loading, index creation,
|
||||
and constraint creation. It supports only custom/directory archives and
|
||||
cannot be combined with `--single-transaction`; it is not a whole-restore
|
||||
atomic operation.
|
||||
([`pg_restore --jobs`](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
|
||||
- A single giant restore transaction can be impractical because it locks every
|
||||
restored object and may exhaust the lock table. PostgreSQL 18 offers
|
||||
`--transaction-size=N` as a bounded intermediate option; it implies
|
||||
`--exit-on-error` but is not all-or-nothing.
|
||||
([`pg_restore --transaction-size`](https://www.postgresql.org/docs/current/app-pgrestore.html#APP-PGRESTORE-OPTIONS))
|
||||
- Plain SQL restored by `psql` also continues after errors unless
|
||||
`ON_ERROR_STOP` is enabled; a single transaction is separately required for
|
||||
all-or-nothing behaviour.
|
||||
([restoring an SQL dump](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-RESTORE))
|
||||
- PostgreSQL recommends restoring into a truly empty database based on
|
||||
`template0`; otherwise local additions in `template1` can cause duplicate
|
||||
definition errors. Required roles must exist before owner/ACL restoration.
|
||||
([restoring an SQL dump](https://www.postgresql.org/docs/current/backup-dump.html#BACKUP-DUMP-RESTORE))
|
||||
- A dump can execute arbitrary code selected by superusers of the source
|
||||
database. Partial dump/restore does not remove that trust boundary.
|
||||
([`pg_dump` security warning](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-DESCRIPTION))
|
||||
- A `pg_dump` binary can dump older servers but refuses a server newer than its
|
||||
own major version. Output is intended to load into newer PostgreSQL versions,
|
||||
but loading into an older major is not guaranteed.
|
||||
([`pg_dump` cross-version notes](https://www.postgresql.org/docs/current/app-pgdump.html#PG-DUMP-NOTES))
|
||||
|
||||
Derived Hub implications:
|
||||
|
||||
- Restore should target a new isolated database/cluster. If a non-atomic or
|
||||
parallel restore reports any error, discard that target and start again;
|
||||
never promote a partially restored database.
|
||||
- The runbook must record the restore atomicity strategy and exact PostgreSQL
|
||||
client/server versions. A green process exit without explicit fail-fast
|
||||
options is insufficient evidence.
|
||||
- A restored database must remain disconnected from real Feishu/provider
|
||||
side-effect endpoints until application-level validation completes.
|
||||
|
||||
## Physical backup, WAL, and PITR
|
||||
|
||||
### Base backups
|
||||
|
||||
- `pg_basebackup` backs up a running PostgreSQL cluster without blocking other
|
||||
clients. A full backup is an exact copy of the entire cluster and can seed
|
||||
PITR or a standby; it cannot select one database or object.
|
||||
([`pg_basebackup` description](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
|
||||
- It uses the replication protocol. The account needs `REPLICATION` or
|
||||
superuser privilege, `pg_hba.conf` must permit it, and `max_wal_senders`
|
||||
needs capacity for the backup and, when streaming WAL, another connection.
|
||||
([`pg_basebackup` requirements](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
|
||||
- Default `-X stream` obtains WAL over a second connection while copying. With
|
||||
`-X fetch`, WAL is collected only at the end; if required WAL was recycled,
|
||||
the backup fails and is unusable.
|
||||
([`pg_basebackup --wal-method`](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
|
||||
- The default waits for backup files to be durable. `--no-sync` is explicitly
|
||||
unsuitable for production because an OS crash can corrupt the backup.
|
||||
([`pg_basebackup --no-sync`](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
|
||||
- A manifest is generated by default, with per-file checksums. A SHA manifest
|
||||
can detect malicious modification only if the manifest itself is held in a
|
||||
separate trusted location or otherwise authenticated.
|
||||
([`pg_basebackup` manifest options](https://www.postgresql.org/docs/current/app-pgbasebackup.html#APP-PGBASEBACKUP))
|
||||
|
||||
Derived minimum backup record:
|
||||
|
||||
- source cluster/system identifier and PostgreSQL version;
|
||||
- backup tool version, start/end time, start/end LSN and WAL range;
|
||||
- manifest digest and its independently protected location;
|
||||
- destination object/version, byte count, exit status, and verification state;
|
||||
- an explicit `VALID`, `INVALID`, or `QUARANTINED` state so a failed leftover
|
||||
directory cannot enter the retention set merely because files exist.
|
||||
|
||||
### WAL archival and recovery
|
||||
|
||||
- PITR combines a physical backup with WAL replay. It can stop at a chosen time
|
||||
and reach a consistent database state. Logical dumps do not contain enough
|
||||
information for WAL replay.
|
||||
([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
|
||||
- Successful recovery requires a continuous sequence of archived WAL extending
|
||||
back at least to the base-backup start. PostgreSQL tells operators to set up
|
||||
and test WAL archiving **before** taking the first base backup.
|
||||
([continuous archiving overview](https://www.postgresql.org/docs/current/continuous-archiving.html#CONTINUOUS-ARCHIVING))
|
||||
- The archive command must return zero if and only if durable archival
|
||||
succeeded. Non-zero causes retry. Archive implementations should refuse to
|
||||
overwrite an existing file; a repeated name may report success only when the
|
||||
existing durable content is identical, and differing content must fail.
|
||||
([WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
|
||||
- Repeated archive failure accumulates WAL in `pg_wal`; filling that filesystem
|
||||
causes PostgreSQL to panic and remain offline until space is freed. PostgreSQL
|
||||
explicitly advises monitoring archival progress.
|
||||
([WAL archiving failure behaviour](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
|
||||
- WAL is archived after a segment is completed, so low traffic can make the
|
||||
last committed change wait a long time for archival. `archive_timeout` can
|
||||
bound that age by forcing segment switches, at the cost of archive growth.
|
||||
([WAL archiving latency](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
|
||||
- WAL archival does not restore manually edited PostgreSQL configuration files
|
||||
(`postgresql.conf`, `pg_hba.conf`, `pg_ident.conf`); those need a separate
|
||||
file backup.
|
||||
([WAL archiving caveat](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
|
||||
- PostgreSQL's recovery procedure keeps clients out, restores the base backup,
|
||||
configures `restore_command` and `recovery.signal`, replays WAL, and requires
|
||||
inspection of database contents before restoring ordinary access.
|
||||
([PITR recovery procedure](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-PITR-RECOVERY))
|
||||
- Recovery can target a timestamp, LSN, transaction ID, named restore point, or
|
||||
the earliest consistent point. Only one target may be set. Timeline history
|
||||
is required to navigate recovery branches, and a configured target not
|
||||
reached before WAL exhaustion is a fatal recovery failure.
|
||||
([recovery target settings](https://www.postgresql.org/docs/current/runtime-config-wal.html#RUNTIME-CONFIG-WAL-RECOVERY-TARGET),
|
||||
[timelines](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-TIMELINES))
|
||||
|
||||
Derived Hub implications:
|
||||
|
||||
- The database RPO is the age of the newest **durably archived, continuous**
|
||||
WAL position, not the age of the newest base backup and not merely the name
|
||||
in `last_archived_wal`.
|
||||
- Archive keys need a collision-proof cluster/system-identifier namespace and
|
||||
create-if-absent semantics; unrelated clusters must never overwrite a shared
|
||||
WAL name.
|
||||
- Monitoring must cover archive failures, time since durable success, archive
|
||||
lag, WAL continuity, `pg_wal` growth, free space, and base-backup age.
|
||||
- PITR restores PostgreSQL only. It does not rewind workspace files. A database
|
||||
recovery target has no whole-Hub meaning until paired with a workspace
|
||||
recovery point and reconciliation rule.
|
||||
|
||||
## Verification is a ladder, not a checksum
|
||||
|
||||
- `pg_verifybackup` validates the backup manifest and system identifier,
|
||||
detects missing/extra/changed files, checks file checksums, and parses the
|
||||
required WAL with `pg_waldump`. WAL verification is currently supported only
|
||||
for plain-format base backups.
|
||||
([`pg_verifybackup` checks](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
|
||||
- PostgreSQL explicitly says these checks cannot cover everything a running
|
||||
server will encounter and still requires test restores plus checks that the
|
||||
database works and contains correct data.
|
||||
([`pg_verifybackup` limitation](https://www.postgresql.org/docs/current/app-pgverifybackup.html#APP-PGVERIFYBACKUP))
|
||||
- `pg_amcheck` can check supported heap/index structures, but currently only
|
||||
ordinary/TOAST tables, materialized views, sequences, and B-tree indexes are
|
||||
supported; other relation types are silently skipped. Stronger B-tree checks
|
||||
can block concurrent modifications.
|
||||
([`pg_amcheck`](https://www.postgresql.org/docs/current/app-pgamcheck.html#APP-PGAMCHECK))
|
||||
|
||||
Derived recovery-evidence ladder:
|
||||
|
||||
1. **Artifact verification:** backup command succeeded; manifest and all
|
||||
required WAL validate; any failure marks the artifact unusable.
|
||||
2. **Isolated technical restore:** restore to a clean non-production target,
|
||||
reach the intended LSN/time/timeline, and prove PostgreSQL starts without
|
||||
exposing it to users.
|
||||
3. **Database validation:** verify migration history/schema identity, tenant
|
||||
counts and relationships, critical constraints/sequences, and appropriate
|
||||
corruption checks.
|
||||
4. **Workspace validation:** restore the paired snapshot/version, validate file
|
||||
hashes/ownership/containment, and reconcile database file-change/run state
|
||||
against actual files.
|
||||
5. **Application validation:** run read-only tenant-isolation and core workflow
|
||||
smoke tests with external side effects disabled.
|
||||
6. **Recovery report:** persist the exact backup, WAL range, workspace point,
|
||||
release/schema versions, checks, observed RPO/RTO, operator, and failures.
|
||||
Only a fully passing report may authorize promotion.
|
||||
|
||||
## Prisma Migrate production semantics
|
||||
|
||||
### What `migrate deploy` proves and does not prove
|
||||
|
||||
- In production, `migrate deploy` compares applied migration checksums with the
|
||||
migration directory, warns when an applied migration was modified, and
|
||||
applies pending migrations.
|
||||
([Prisma 6 production workflow](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#production-and-testing-environments))
|
||||
- It does **not** warn when an already-applied migration is missing, detect
|
||||
production schema drift, reset the database, generate Prisma Client, or use a
|
||||
shadow database.
|
||||
([Prisma 6 production workflow](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#production-and-testing-environments))
|
||||
- The migration directory is the deployable history and must be committed in
|
||||
full, including `migration_lock.toml`. Prisma warns against editing or
|
||||
deleting migrations already applied in production; `migrate deploy` executes
|
||||
migration SQL rather than deriving a new migration from the Prisma schema.
|
||||
([Prisma migration histories](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/migration-histories))
|
||||
- Prisma uses a database advisory lock for production migration commands so
|
||||
multiple migration commands do not run simultaneously. The documented lock
|
||||
timeout is ten seconds and not configurable; after timeout, the caller must
|
||||
retry. The lock can be disabled by environment variable.
|
||||
([Prisma 6 advisory locking](https://www.prisma.io/docs/orm/v6/prisma-migrate/workflows/development-and-production#advisory-locking))
|
||||
|
||||
Derived Hub implication: `migrate deploy` exit zero proves that this invocation
|
||||
applied or found no pending migration. It does not prove the live schema equals
|
||||
the expected migration end state, that the new and previous application builds
|
||||
are both compatible with it, or that the migration is operationally safe under
|
||||
production data volume and traffic.
|
||||
|
||||
The advisory lock is a migration-command concurrency guard, not a substitute
|
||||
for the repository's deployment lock or application/schema compatibility. As a
|
||||
derived safety rule, production preflight should reject
|
||||
`PRISMA_SCHEMA_DISABLE_ADVISORY_LOCK` unless an explicit reviewed mechanism
|
||||
provides the same serialization guarantee.
|
||||
|
||||
### Transactions and failed migrations
|
||||
|
||||
- Prisma's first-party migration guidance states that PostgreSQL migrations
|
||||
are not wrapped in a transaction by default. Operators may opt in by adding
|
||||
`BEGIN;`/`COMMIT;` to a migration, but large transactional migrations can hold
|
||||
locks longer and consume more database resources.
|
||||
([Prisma transactional behaviour](https://www.prisma.io/blog/prisma-migrate-dx-primitives#what-if-schema-migrations-were-atomic))
|
||||
- A failed migration records its error in `_prisma_migrations.logs`; further
|
||||
migrations cannot deploy until the failed state is explicitly recovered.
|
||||
Recovery is either (a) revert/repair partial steps, mark the migration rolled
|
||||
back, and redeploy, or (b) manually complete the exact steps and mark it
|
||||
applied.
|
||||
([Prisma failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration))
|
||||
- Prisma error codes `P3009` and `P3018` explicitly report that failed
|
||||
migrations block new migration application until recovery.
|
||||
([Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference#p3009))
|
||||
- `migrate resolve --rolled-back` changes migration-history state so Prisma may
|
||||
apply the migration again; it does not itself undo SQL already executed.
|
||||
`--applied` records success without executing the migration SQL.
|
||||
([failed-migration recovery](https://www.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing#failed-migration),
|
||||
[`migrate resolve`](https://www.prisma.io/docs/cli/migrate/resolve))
|
||||
- Prisma can generate a down SQL script, but its documentation limits the
|
||||
simple rollback workflow to a **failed** up migration. To reverse a successful
|
||||
migration, the supported history is a new forward migration generated from a
|
||||
reverted Prisma schema. A schema down script also cannot restore lost data,
|
||||
application changes, or manually authored SQL automatically.
|
||||
([generating down migrations](https://www.prisma.io/docs/orm/prisma-migrate/workflows/generating-down-migrations#considerations-when-generating-down-migrations))
|
||||
|
||||
Derived Hub implications:
|
||||
|
||||
- Every migration needs a declared atomicity choice. If it is intentionally
|
||||
non-transactional, the migration must be resumable/idempotent or ship an
|
||||
exact, tested partial-failure procedure. Blindly rerunning a partially applied
|
||||
script is unsafe.
|
||||
- A database backup checkpoint before a destructive or difficult-to-reverse
|
||||
migration is necessary, but restoring that checkpoint is a whole-database
|
||||
recovery event—not a substitute for a compatible rollout design.
|
||||
- Service restart cannot be treated as application rollback after a successful
|
||||
schema migration. The previous binary may no longer understand the new
|
||||
schema, and a down migration cannot recreate deleted data.
|
||||
|
||||
### Compatibility and rollout ordering
|
||||
|
||||
- Prisma documents expand-and-contract as the way to avoid downtime for fields
|
||||
in active use: add the new representation, deploy code that writes both while
|
||||
reading the old, migrate and verify data, switch reads, stop old writes, then
|
||||
remove the old representation.
|
||||
([Prisma expand-and-contract workflow](https://www.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations#example-use-the-expand-and-contract-pattern-to-evolve-the-schema-without-downtime))
|
||||
- Prisma also warns that generated “renames” may be expressed as add/drop and
|
||||
lose data unless the SQL is reviewed and customized before application.
|
||||
([customizing migrations](https://www.prisma.io/docs/orm/prisma-migrate/workflows/customizing-migrations#example-rename-a-field))
|
||||
|
||||
Derived deployment contract:
|
||||
|
||||
1. Migration SQL is immutable after production application and reviewed for
|
||||
destructive operations, lock duration, data volume, and transaction choice.
|
||||
2. It is rehearsed against a recent, anonymized production-scale restore and
|
||||
its runtime/locks/disk growth are recorded.
|
||||
3. Each release declares a schema compatibility window. Expand releases remain
|
||||
compatible with both old and new application versions; contract/destructive
|
||||
releases occur only after old binaries can no longer run.
|
||||
4. Migration status/drift is checked before changing the live release. A
|
||||
failed migration blocks deployment and enters an operator runbook; it must
|
||||
never be hidden by repeated restart attempts.
|
||||
5. Rollback means switching to a binary proven compatible with the current
|
||||
schema or deploying a new forward migration. Restoring a backup is the last
|
||||
resort and must also restore/reconcile the workspace recovery point.
|
||||
|
||||
## systemd stop, restart, and dependency semantics
|
||||
|
||||
- When `ExecStartPre=`, `ExecStart=`, or `ExecStartPost=` fails or times out,
|
||||
service startup does not continue to the normal running process. `ExecStop=`
|
||||
is skipped because startup never succeeded; `ExecStopPost=` is the hook that
|
||||
still runs after a failed startup.
|
||||
([systemd start/stop command semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStartPre=),
|
||||
[`ExecStopPost=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStopPost=),
|
||||
[permanent upstream man-page source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemd.service.xml#L525-L587))
|
||||
- `Restart=on-failure` covers non-zero exits and failures/timeouts in service
|
||||
control processes, including `ExecStartPre=`. An explicit systemd stop does
|
||||
not trigger automatic restart. Restarts are subject to unit start-rate
|
||||
limiting.
|
||||
([`Restart=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#Restart=),
|
||||
[permanent upstream source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemd.service.xml#L856-L995))
|
||||
- With no `ExecStop=`, systemd sends `KillSignal=` (default `SIGTERM`).
|
||||
`TimeoutStopSec=` bounds the wait; after it expires, remaining processes can
|
||||
be forcibly killed with `SIGKILL`/`FinalKillSignal=`.
|
||||
([`ExecStop=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStop=),
|
||||
[`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=))
|
||||
- Default `KillMode=control-group` applies termination to all remaining
|
||||
processes in the unit cgroup. systemd explicitly discourages `process` or
|
||||
`none` because children can escape the service lifecycle and remain running
|
||||
after the service is considered stopped.
|
||||
([`KillMode=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.kill.html#KillMode=))
|
||||
- `systemctl stop` waits for the job unless `--no-block` is used, but the command
|
||||
does not fail merely because an `ExecStop=` command failed: the manager still
|
||||
forcibly terminates the unit.
|
||||
([`systemctl stop`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#stop%20PATTERN%E2%80%A6),
|
||||
[permanent upstream source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemctl.xml#L444-L459))
|
||||
- `systemctl restart` is a stop followed by a start; `ExecStop=` and
|
||||
`ExecStopPost=` participate. It does not necessarily flush every unit resource
|
||||
in the same way as separate completed `stop` then `start` operations.
|
||||
([restart semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#ExecStop=),
|
||||
[`systemctl restart`](https://www.freedesktop.org/software/systemd/man/latest/systemctl.html#restart%20PATTERN%E2%80%A6))
|
||||
- `After=` is only an ordering relationship; it does not pull in another unit.
|
||||
Requirement and ordering dependencies are independent. During shutdown the
|
||||
start order is reversed, so a Hub ordered `After=postgresql.service` stops
|
||||
before PostgreSQL when both are in the shutdown transaction.
|
||||
([`Requires=` and ordering](https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Requires=),
|
||||
[`Before=`/`After=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.unit.html#Before=))
|
||||
- `systemctl freeze` only uses the cgroup freezer to suspend process execution;
|
||||
its documented contract contains no filesystem flush or stable-image
|
||||
guarantee. It is not a substitute for an application drain or a filesystem
|
||||
snapshot barrier.
|
||||
([permanent `systemctl freeze` man-page source](https://github.com/systemd/systemd/blob/924552cfbacd51be0229ac6ce95fe3b69c7b7923/man/systemctl.xml#L653-L675))
|
||||
|
||||
Derived assessment of the current unit:
|
||||
|
||||
- A Prisma migration failure in the current `ExecStartPre=` prevents the Hub
|
||||
from starting, then `Restart=on-failure` can retry startup until rate-limited.
|
||||
That is fail-closed, but it is not controlled migration recovery and can
|
||||
repeatedly hit a database already left in Prisma's failed state.
|
||||
- `KillSignal=SIGTERM` plus `TimeoutStopSec=30` is only a deadline. Until the Hub
|
||||
installs a real termination handler that makes readiness false, stops ingress,
|
||||
drains/checkpoints active work, closes Feishu and database clients, and reports
|
||||
success, the unit cannot claim a graceful consistency barrier.
|
||||
- A backup runbook must issue a controlled drain, verify durable work reached a
|
||||
reconciled terminal/checkpoint state, stop the Hub, verify the unit and its
|
||||
cgroup are inactive, and only then snapshot mutable workspace state. A zero
|
||||
`systemctl stop` exit by itself is insufficient evidence.
|
||||
- Recovery should not start the normal unit until the restored database and
|
||||
workspace have been validated. The current unit would run pending migrations
|
||||
immediately on first start, mutating the restored database before validation
|
||||
and possibly destroying the ability to inspect the original recovered state.
|
||||
|
||||
## Filesystem and rsync boundaries
|
||||
|
||||
- PostgreSQL says a plain filesystem copy of a live data directory is unusable:
|
||||
the database must be shut down, because tools such as `tar` do not create an
|
||||
atomic snapshot and server buffering remains. A correctly implemented frozen
|
||||
snapshot can work while PostgreSQL runs, but all volumes containing data,
|
||||
WAL, and tablespaces must be captured simultaneously; otherwise shut down or
|
||||
use the continuous-archive base-backup protocol.
|
||||
([PostgreSQL filesystem backups](https://www.postgresql.org/docs/current/backup-file.html))
|
||||
- For a PostgreSQL rsync backup, the documented safe pattern is a first pass
|
||||
while running, followed by shutdown and a second `rsync --checksum`; the
|
||||
second pass is what makes the result consistent.
|
||||
([PostgreSQL filesystem backups](https://www.postgresql.org/docs/current/backup-file.html))
|
||||
- At the filesystem layer, `fsfreeze` is documented to let in-progress
|
||||
filesystem operations complete, halt new modifications, and flush dirty data,
|
||||
metadata, and logs before returning. That is still only a filesystem-volume
|
||||
guarantee; it does not replace PostgreSQL's backup protocol and does not make
|
||||
separately frozen volumes simultaneous.
|
||||
([util-linux `fsfreeze` manual source](https://github.com/util-linux/util-linux/blob/master/sys-utils/fsfreeze.8.adoc))
|
||||
- rsync's `--delete` defaults to deletion during transfer on modern peers, so
|
||||
the receiver tree changes incrementally. `--delay-updates` merely holds each
|
||||
updated file and renames files into place rapidly at the end; its own manual
|
||||
says this only attempts to make updates “a little more atomic” and points to
|
||||
a separate parallel-tree/switch approach for stronger atomicity.
|
||||
([rsync `--delete`](https://rsync.samba.org/ftp/rsync/rsync.1.html#opt--delete),
|
||||
[rsync `--delay-updates`](https://rsync.samba.org/ftp/rsync/rsync.1.html#opt--delay-updates))
|
||||
|
||||
Derived Hub implications:
|
||||
|
||||
- The current in-place `rsync --delete` deployment is neither an immutable
|
||||
release switch nor a backup snapshot. A failed/interrupted transfer can leave
|
||||
a mixed live tree; service stop/restart cannot reconstruct the deleted prior
|
||||
artifact.
|
||||
- The workspace snapshot mechanism remains an explicit design choice. Whichever
|
||||
storage technology is selected must prove atomicity over all workspace
|
||||
volumes or use a quiesced/no-writer interval, then publish an immutable
|
||||
version/checksum manifest. Merely copying a changing directory is not enough.
|
||||
|
||||
## Derived minimum whole-Hub recovery contract
|
||||
|
||||
The primary sources support the following minimum contract; exact RPO, RTO,
|
||||
retention, storage vendor, and numeric schedules remain product/operations
|
||||
decisions and must not be invented by implementation.
|
||||
|
||||
### Backup path
|
||||
|
||||
1. Maintain verified PostgreSQL base backups and a monitored continuous WAL
|
||||
archive, or obtain and test an explicitly equivalent managed-PostgreSQL
|
||||
guarantee. Keep logical dumps as a separate portability/selective-restore
|
||||
layer, not the sole PITR mechanism.
|
||||
2. Keep PostgreSQL configuration, deployment/release manifests, migration
|
||||
history, and workspace snapshots in the backup inventory; WAL alone does not
|
||||
cover them.
|
||||
3. Create a whole-Hub checkpoint only after ingress is drained and active
|
||||
workspace writers are stopped/reconciled, unless the eventual storage layer
|
||||
supplies a proven atomic snapshot/journal spanning the necessary stores.
|
||||
4. Persist one signed/immutable checkpoint manifest joining PostgreSQL
|
||||
backup/WAL position, workspace snapshot/version, application revision,
|
||||
migration digest, tool versions, checksums, timestamps, and validation state.
|
||||
5. Treat database/WAL/workspace backups as cross-tenant sensitive data. Apply
|
||||
least-privilege access, encryption, audit, deletion/retention, and an
|
||||
independently protected integrity manifest; WAL contains effectively all
|
||||
database changes and PostgreSQL advises protecting it from unauthorized
|
||||
readers.
|
||||
([WAL archive confidentiality](https://www.postgresql.org/docs/current/continuous-archiving.html#BACKUP-ARCHIVING-WAL))
|
||||
|
||||
### Migration path
|
||||
|
||||
1. Serialize deploys and backup checkpoints; run a drift/status gate before
|
||||
migration, not just `migrate deploy`.
|
||||
2. Rehearse each migration on a recent production-scale restore and record
|
||||
runtime, locks, disk/WAL growth, transaction choice, and failure recovery.
|
||||
3. Require expand/contract compatibility for online changes. A release must
|
||||
declare which old/new binaries can run against each schema phase.
|
||||
4. Separate migration orchestration from ordinary service restart so a failed
|
||||
migration produces one durable incident state and an operator decision—not
|
||||
an opaque restart loop.
|
||||
5. Verify a restorable checkpoint before destructive change. Do not describe
|
||||
backup restore as a normal “rollback”; it is whole-service disaster recovery.
|
||||
|
||||
### Restore and promotion path
|
||||
|
||||
1. Restore into an isolated environment with real external side effects
|
||||
disabled.
|
||||
2. Restore the selected PostgreSQL base backup/WAL target and the checkpoint's
|
||||
exact workspace version; do not run pending Prisma migrations implicitly.
|
||||
3. Prove artifact integrity, database startup, recovery target/timeline,
|
||||
migration/schema identity, tenant invariants, workspace hashes/containment,
|
||||
and read-only application flows.
|
||||
4. Record observed recovery point and recovery time. A restore that starts but
|
||||
misses its target, loses a tenant workspace, has partial migration state, or
|
||||
fails any invariant is not promotable.
|
||||
5. Promote explicitly, then enable ingress and external integrations. Preserve
|
||||
the pre-recovery cluster/WAL and recovery report until the incident is closed.
|
||||
|
||||
## Decisions and evidence still required
|
||||
|
||||
These values are not fixed by the upstream tools and remain `OPEN` for the SaaS
|
||||
production contract:
|
||||
|
||||
- whole-Hub RPO and RTO, separately for ordinary deletion, database corruption,
|
||||
host loss, region/provider loss, and migration failure;
|
||||
- base-backup, logical-dump, workspace-snapshot, WAL, and restore-drill cadence;
|
||||
- immutable/off-host storage, encryption/key ownership, retention and verified
|
||||
deletion policy, including org-deletion and legal-retention interactions;
|
||||
- exact workspace checkpoint/journal technology and database/workspace
|
||||
reconciliation semantics for in-flight or partially written runs;
|
||||
- whether restore promotion is whole-platform only or supports a safe
|
||||
organization/project export-import path (PostgreSQL physical PITR is
|
||||
cluster-wide);
|
||||
- migration transaction policy, maximum lock/downtime budget, compatibility
|
||||
window, and who can authorize contract/destructive phases;
|
||||
- evidence from at least one production-like full restore and one PITR drill,
|
||||
including deliberate missing-WAL, corrupted-backup, failed-migration,
|
||||
interrupted-workspace-write, and restart-during-recovery probes.
|
||||
@@ -0,0 +1,200 @@
|
||||
# Clean-host deployment and rollback audit
|
||||
|
||||
## Verdict
|
||||
|
||||
The current `hub/deploy` path is **not a production deployment mechanism**.
|
||||
It can update a manually-prepared host in the happy path, but a clean supported
|
||||
host cannot become a runnable Hub from the documented inputs, and an upgrade
|
||||
failure can leave the live application tree partially replaced with no
|
||||
application rollback path.
|
||||
|
||||
The accepted initial topology remains viable: one Linux/systemd host,
|
||||
PostgreSQL, Feishu WebSocket ingress, and a non-root Hub service. The blocker is
|
||||
the release contract and its verification, not a need to replace that topology.
|
||||
|
||||
## Executed evidence
|
||||
|
||||
### Static gates
|
||||
|
||||
- `bash -n hub/deploy/install_service.sh` and
|
||||
`bash -n hub/deploy/deploy_platform.sh` pass.
|
||||
- ShellCheck reports only informational SC2029 findings for intentional
|
||||
client-side expansion of remote paths. It does not find a shell syntax defect.
|
||||
- A source-to-installer comparison finds six `requireEnv` settings in
|
||||
`hub/src/server.ts`, but the generated production environment omits
|
||||
`HUB_PROJECT_WORKSPACE_ROOT` and `HUB_SESSION_SECRET`.
|
||||
|
||||
### Seeded configuration cannot boot the built server
|
||||
|
||||
The installer was run against a temporary `BASE`, with the real built Hub as
|
||||
`HUB_DIR`. It exited successfully after creating `platform.env`. The generated
|
||||
file was then used as the production configuration while supplying the model and
|
||||
Feishu values that its message asks the operator to fill. The built server
|
||||
exited with status 1:
|
||||
|
||||
```text
|
||||
[hub] missing required env: HUB_PROJECT_WORKSPACE_ROOT
|
||||
```
|
||||
|
||||
After that is supplied, `HUB_SESSION_SECRET` is the next required omission.
|
||||
`HUB_PUBLIC_BASE_URL` is also absent; its code fallback is localhost, which is
|
||||
not a valid public OAuth callback for a production deployment.
|
||||
|
||||
### The service identity is referenced but not provisioned
|
||||
|
||||
The installer was executed with `SERVICE_USER=definitely-missing-cph-user` and
|
||||
mocked `install`/`systemctl` boundaries. It returned success and rendered:
|
||||
|
||||
```text
|
||||
User=definitely-missing-cph-user
|
||||
WorkingDirectory=.../hub
|
||||
EnvironmentFile=.../.secrets/platform.env
|
||||
```
|
||||
|
||||
There is no `getent`, `id`, `useradd`, or equivalent validation/provisioning
|
||||
path. It also creates no service home, cache, state, or project-workspace
|
||||
directory. This contradicts ADR-0018's implemented deployment invariant that
|
||||
the Hub runs as a dedicated unprivileged service user. A genuinely clean host
|
||||
will fail at systemd user resolution; a manually-created user can still fail
|
||||
later when the agent SDK, `cph` cache, or project workspace needs a writable
|
||||
path.
|
||||
|
||||
### A failed build mutates the live tree before failure
|
||||
|
||||
`deploy_platform.sh` was run with mocked SSH and rsync boundaries. The first
|
||||
remote build was forced to exit 42. The observed order was:
|
||||
|
||||
```text
|
||||
ssh:mkdir -p '/srv/curriculum-project-hub/hub'
|
||||
rsync:live-tree-mutated
|
||||
ssh:cd '/srv/curriculum-project-hub/hub' && npm ci && npm run build
|
||||
```
|
||||
|
||||
The deploy command correctly propagated status 42, but it had already applied
|
||||
`rsync --delete` to the one live directory. There is no staged release,
|
||||
immutable release identity, current/previous pointer, deployment lock, or
|
||||
rollback command. `npm ci` also replaces that live directory's dependency tree
|
||||
before the new build has passed.
|
||||
|
||||
## Blocking gaps
|
||||
|
||||
### 1. Incomplete clean-host bootstrap
|
||||
|
||||
`install_service.sh` writes a unit containing `User=cph-hub` without creating or
|
||||
validating that account. It does not create owned state/cache/workspace paths,
|
||||
and its generated environment omits required settings. On its first invocation
|
||||
it deliberately exits after seeding the environment, after which
|
||||
`deploy_platform.sh` proceeds to restart a unit that has not yet been installed.
|
||||
The documented workflow is therefore neither one-shot nor complete.
|
||||
|
||||
### 2. The runtime product is not fully delivered
|
||||
|
||||
The deploy uploads only `hub/`. The target prerequisite list does not include a
|
||||
preinstalled `cph`, and no release step builds or installs `crates/cph-cli`.
|
||||
The Hub starts without checking it: the first agent `cph check` or `cph build`
|
||||
returns exit 127 while `/api/healthz` remains green. Bubblewrap is asserted by
|
||||
the unit, but Node/npm, curl, PostgreSQL connectivity, service-user namespace
|
||||
support, writable storage, and `cph --version` are not checked as one preflight.
|
||||
|
||||
### 3. Releases are in-place and have no application rollback
|
||||
|
||||
Source, dependencies, build output, and the running service all share one
|
||||
directory. A build failure, interrupted rsync, or unsuccessful restart can
|
||||
leave a mixed tree. No release manifest records the git revision, Node version,
|
||||
dependency lock digest, migration set, or `cph` version that was deployed.
|
||||
Concurrent deploys are not serialized.
|
||||
|
||||
### 4. The deployment health check is only liveness
|
||||
|
||||
`/api/healthz` returns a constant success response after the HTTP server starts.
|
||||
It does not verify database access, migration state, workspace writability,
|
||||
`cph`, the agent sandbox, or Feishu listener startup. The listener's start
|
||||
promise is not awaited or reflected in readiness. A release can therefore be
|
||||
declared successful while its defining workflows are unavailable.
|
||||
|
||||
The service unit also claims graceful shutdown, but `server.ts` installs no
|
||||
`SIGTERM`/`SIGINT` handler and never calls `app.close()` or closes the Feishu
|
||||
listener. Node's default SIGTERM handling terminates the process; Fastify's
|
||||
documented graceful path begins only when `fastify.close()` is invoked. The
|
||||
full data-integrity consequence belongs to the run-lifecycle audit, but the
|
||||
current deployment contract must not claim draining behavior it does not have.
|
||||
|
||||
References:
|
||||
|
||||
- [Fastify shutdown hooks](https://fastify.dev/docs/latest/Reference/Hooks/)
|
||||
- [Node.js signal events](https://nodejs.org/api/process.html#signal-events)
|
||||
|
||||
### 5. Application rollback and schema migration are not coordinated
|
||||
|
||||
The unit runs `prisma migrate deploy` immediately before every start. Current
|
||||
migration history includes column drops, index replacement, and data
|
||||
deduplication; there is no declaration that the previous application remains
|
||||
compatible after each migration. Switching application code back after a
|
||||
successful migration is therefore not generally safe.
|
||||
|
||||
Prisma documents that `migrate deploy` applies pending migrations but does not
|
||||
detect schema drift, and a partially failed migration requires explicit
|
||||
operator recovery. Its recommended zero-downtime direction is an
|
||||
expand-and-contract sequence. The repository has no pre-deploy migration
|
||||
status gate, production-data rehearsal, backup checkpoint, expand/contract
|
||||
rule, or failed-migration runbook.
|
||||
|
||||
References:
|
||||
|
||||
- [Prisma production migration behavior](https://www.prisma.io/docs/orm/prisma-migrate/workflows/development-and-production)
|
||||
- [Prisma failed-migration recovery](https://docs.prisma.io/docs/orm/prisma-migrate/workflows/patching-and-hotfixing)
|
||||
- [Prisma expand-and-contract migrations](https://docs.prisma.io/docs/guides/database/data-migration)
|
||||
|
||||
### 6. Network binding configuration is misleading
|
||||
|
||||
The installer writes `HOST=127.0.0.1`, but `server.ts` ignores it and binds
|
||||
`0.0.0.0`. An operator following the generated configuration can unintentionally
|
||||
expose the Hub directly instead of binding it behind the intended TLS reverse
|
||||
proxy. Public URL validation and proxy topology are not part of deployment
|
||||
preflight.
|
||||
|
||||
## Required release contract
|
||||
|
||||
A production-capable implementation on the accepted topology should establish
|
||||
these invariants:
|
||||
|
||||
1. **Provisioned identity and storage.** Installation creates or validates a
|
||||
non-login `cph-hub` user and explicit persistent state, cache, home, and
|
||||
workspace paths with tested ownership.
|
||||
2. **Complete validated configuration.** A preflight validates every required
|
||||
secret/value, the public HTTPS URL, bind address, PostgreSQL connectivity,
|
||||
writable storage, `bwrap`, and the exact compatible `cph` binary before the
|
||||
live service is touched.
|
||||
3. **Immutable staged releases.** Each revision is built in a new release
|
||||
directory, identified by revision and manifest. The live service points to a
|
||||
`current` symlink only after build and preflight pass. Deploys are locked.
|
||||
4. **Complete product artifact.** The release carries both the Hub and the
|
||||
matching `cph` executable; readiness checks the binary/version rather than
|
||||
deferring failure to an agent run.
|
||||
5. **Real readiness.** Liveness remains cheap, while readiness proves the
|
||||
dependencies needed to accept work. Deployment success waits for readiness,
|
||||
not merely an open HTTP port.
|
||||
6. **Application rollback.** A failed switch returns `current` to the previous
|
||||
release and verifies readiness. Database changes are never silently reversed.
|
||||
7. **Migration compatibility.** Every release states whether its migration set
|
||||
is backward-compatible with the previous application. Future destructive
|
||||
changes use expand/contract releases; failed migrations follow an explicit
|
||||
recovery runbook tied to backup evidence.
|
||||
8. **Observable lifecycle.** Release revision, migration status, readiness
|
||||
failures, restart/drain outcomes, and rollback results are present in command
|
||||
output and service logs.
|
||||
|
||||
## Verification route
|
||||
|
||||
The implementation is not complete until an automated Linux test drives these
|
||||
scenarios with the real scripts and a production-like PostgreSQL instance:
|
||||
|
||||
- first install on a host without the service user or directories;
|
||||
- incomplete configuration fails before unit installation/restart;
|
||||
- staged build failure leaves the current release byte-for-byte unchanged;
|
||||
- successful deploy reports revision and passes readiness;
|
||||
- failed new release switches back to the previous application;
|
||||
- missing/wrong `cph`, unwritable workspace, unavailable database, and failed
|
||||
migration each produce a distinct non-zero deployment failure;
|
||||
- repeated installation/deployment is idempotent;
|
||||
- concurrent deployments serialize or one fails clearly.
|
||||
@@ -0,0 +1,125 @@
|
||||
# Initial abuse and capacity controls
|
||||
|
||||
## Verdict
|
||||
|
||||
The initial production boundary requires hard safety controls before customer
|
||||
traffic. The current project lock and small in-memory trigger queue are useful
|
||||
local protections, but they do not provide multi-tenant admission, durable
|
||||
backpressure, storage safety, request isolation, bounded execution, or an
|
||||
operator stop mechanism.
|
||||
|
||||
The accepted contract is recorded in
|
||||
[ADR-0022](../../../docs/adr/0022-layered-capacity-admission.md) and
|
||||
[`Spec.System.Capacity`](../../../spec/Spec/System/Capacity.lean). Exact values
|
||||
remain deliberately open until production-like calibration; the required
|
||||
dimensions and failure behaviors do not.
|
||||
|
||||
## Current implementation inventory
|
||||
|
||||
| Surface | Existing protection | Production gap |
|
||||
| --- | --- | --- |
|
||||
| HTTP | Fastify's default 1 MiB body limit | No route-class, user, Org, IP, or administrative-write rate limits; request/connection/handler timeouts are unset |
|
||||
| Project execution | Database lock permits one active run per Project | No platform or Org concurrency ceiling and no fair scheduler |
|
||||
| Busy Project | Process-local FIFO, default five items and five-minute wait | Restart loses accepted work; expiry is logged and dropped without a durable state or user notification |
|
||||
| Agent execution | Global max-turn default of 25 | No wall-clock, tool-call, output, memory, CPU, process, or Org-specific budget |
|
||||
| Feishu messages | Message batching limits text count/characters | File/post attachment count and size are unbounded; downloads may buffer whole files |
|
||||
| Storage | Workspace path boundary | No Project or Org byte quota, archive-expansion limit, or entity-count ceiling |
|
||||
| Usage | Provider-reported token/cost fields and Org aggregation | Missing cost can disappear from totals; no soft-threshold notification or provider-mode attribution contract |
|
||||
| Host | systemd lifecycle | No service CPU, memory, or task ceiling and no audited workload brake |
|
||||
|
||||
Most Organization member/team/project lists are also unpaginated, and usage
|
||||
aggregation can load all matching runs. Those query shapes must be bounded as
|
||||
part of enforcement rather than hidden behind larger process memory.
|
||||
|
||||
## Accepted contract
|
||||
|
||||
### Layered limits
|
||||
|
||||
- Every mandatory dimension has a versioned, non-overridable platform ceiling.
|
||||
- An Organization OWNER/ADMIN may configure a lower policy limit.
|
||||
- Effective limit is `min(platform, organization)`; absent Organization policy
|
||||
means the platform ceiling, never unlimited.
|
||||
- Missing or invalid mandatory production ceilings fail startup.
|
||||
|
||||
### Admission and fairness
|
||||
|
||||
- Project exclusivity remains one active run.
|
||||
- Platform and Organization concurrency limits apply before starting work.
|
||||
- Available slots are scheduled fairly across Organizations, FIFO within an
|
||||
Organization, so one tenant cannot monopolize a global FIFO.
|
||||
- Accepted work is durable and queryable. Queue length and wait are bounded.
|
||||
- Overflow is explicit `capacity_exhausted` (`429` plus `Retry-After` for HTTP;
|
||||
a clear busy response for Feishu). Nothing is silently dropped.
|
||||
- Wait timeout produces terminal `EXPIRED`, not a later automatic start.
|
||||
- The initiator may withdraw their queued request. Project `MANAGE` or an
|
||||
Organization admin may cancel queued requests in their scope. Actor, reason,
|
||||
and notification are recorded.
|
||||
|
||||
### Retry and idempotency
|
||||
|
||||
- Feishu `event_id` and API idempotency identify one admission across retries.
|
||||
- Once a run starts, the whole run is never automatically replayed because tool
|
||||
side effects may already exist.
|
||||
- Only bounded transport retries before model output or tool side effects are
|
||||
automatic. Later failure/timeouts require an explicit new request.
|
||||
|
||||
### Request, file, storage, and entity safety
|
||||
|
||||
- Authenticated Web requests are limited by user, Organization, and endpoint
|
||||
class; signed Feishu events by Organization app, chat, and sender;
|
||||
administrative writes use stricter independent buckets.
|
||||
- Admin surfaces require both a valid user session and the corresponding
|
||||
platform/Organization administrator authorization. OAuth callbacks and
|
||||
signed Feishu ingress remain necessary non-session entry points.
|
||||
- Hard limits cover request body, single-file bytes, attachment count, archive
|
||||
expanded bytes/file count/depth, Project/Organization storage, and per-type
|
||||
Organization entity counts.
|
||||
- Transfer is streamed and counted; partial files are removed after failure.
|
||||
- Crossing a ceiling blocks new growth without auto-deleting or hiding existing
|
||||
resources. Read, export, delete, and administrator remediation stay usable.
|
||||
|
||||
### Run budget
|
||||
|
||||
Every run is bounded by wall time, model turns, tool-call count, per-tool time,
|
||||
output/log/event bytes, and process memory/CPU/child-process count. A breached
|
||||
budget ends explicitly as `TIMED_OUT` or `LIMIT_EXCEEDED`, records the dimension
|
||||
and observed usage, releases capacity, and never masquerades as success.
|
||||
|
||||
### Usage and cost
|
||||
|
||||
- Token usage, provider-reported cost, run count, and duration are attributed by
|
||||
Organization, Project, Run, model, and Provider Connection.
|
||||
- Missing provider cost is `unknown`, never zero.
|
||||
- Thresholds are soft alerts in the pilot; they do not automatically stop runs
|
||||
and are not payment settlement.
|
||||
- Org admins receive Organization alerts. Platform admins additionally receive
|
||||
anomalous-cost alerts for platform-managed connections; BYOK does not trigger
|
||||
platform financial alerts.
|
||||
|
||||
### Emergency control
|
||||
|
||||
An audited Platform Administrator control supports Organization or global
|
||||
`DRAIN` (reject new work, pause queue starts, finish active runs) and `STOP_NOW`
|
||||
(also cancel active runs). Both require an actor, reason, timestamp, explicit
|
||||
recovery, and user-visible outcomes. This is workload control, not Organization
|
||||
deletion or account suspension.
|
||||
|
||||
## Required production evidence
|
||||
|
||||
1. Production configuration validation fails closed for every missing or
|
||||
invalid platform ceiling.
|
||||
2. Concurrent noisy-neighbor tests prove platform/Org/Project concurrency and
|
||||
Organization-fair scheduling without starvation.
|
||||
3. Restart and crash tests prove accepted queue entries, idempotency, expiry,
|
||||
cancellation, and notifications survive process loss.
|
||||
4. HTTP and Feishu load tests prove every rate-limit identity and typed overload
|
||||
response, including retry guidance.
|
||||
5. File/archive/storage probes prove streaming enforcement, cleanup, and
|
||||
continued read/delete access while over limit.
|
||||
6. Hung provider/tool and resource-exhaustion probes prove every run/process
|
||||
budget terminates, records cause, and releases locks/capacity.
|
||||
7. Usage tests prove provider-mode attribution, unknown-cost handling, alert
|
||||
routing, and absence of hard monetary blocking.
|
||||
8. `DRAIN` and `STOP_NOW` drills prove scoped, audited, reversible behavior.
|
||||
9. Production-like calibration publishes the tested numerical defaults and
|
||||
capacity headroom used by the release gate.
|
||||
@@ -0,0 +1,191 @@
|
||||
# Accepted product-surface code inventory
|
||||
|
||||
Audit date: 2026-07-10
|
||||
|
||||
Scope: source-level inventory for the accepted SaaS product boundary in ADR-0020,
|
||||
ADR-0021 and ADR-0022. This is not a claim that tests were executed during this
|
||||
inventory. “Test evidence” below means a repository test exercises the named
|
||||
path; “no direct test found” means a search of `hub/test/**/*.ts` found no request
|
||||
to the named route or corresponding service call.
|
||||
|
||||
## Contract baseline
|
||||
|
||||
The destination is not a shared-bot, single-credential Hub. The repository's
|
||||
current shared language says:
|
||||
|
||||
- an Organization owns its memberships, projects, teams, external connections
|
||||
and settings (`CONTEXT.md:7-9`);
|
||||
- a Platform Administrator creates and operates Organizations without becoming
|
||||
their member, while an Organization Administrator is only an OWNER/ADMIN of
|
||||
that Organization (`CONTEXT.md:11-17`);
|
||||
- the Admin Surface is a private web control plane requiring the corresponding
|
||||
authenticated administrator authority (`CONTEXT.md:19-21`);
|
||||
- each Organization has its own Feishu application for login, bot messaging and
|
||||
directory integration (`CONTEXT.md:23-25`);
|
||||
- provider connections are Organization-scoped BYOK or an Organization-distinct
|
||||
platform-managed connection (`CONTEXT.md:27-37`);
|
||||
- capacity uses non-optional platform ceilings, optional lower Organization
|
||||
limits, durable Organization-fair admission and audited emergency brakes
|
||||
(`CONTEXT.md:39-77`).
|
||||
|
||||
ADR-0021 makes `/admin/platform` and `/admin/org/:orgSlug` separate guarded web
|
||||
areas (`docs/adr/0021-org-admin-project-onboarding.md:23-40`), requires a
|
||||
customer-owned Feishu app with encrypted Organization-scoped secrets
|
||||
(`docs/adr/0021-org-admin-project-onboarding.md:42-47`), and pins both provider
|
||||
credential modes as Organization-scoped (`docs/adr/0021-org-admin-project-onboarding.md:49-60`).
|
||||
ADR-0022 makes its safety controls release requirements, not optional future UI
|
||||
(`docs/adr/0022-layered-capacity-admission.md:22-77`).
|
||||
|
||||
Status vocabulary in this inventory:
|
||||
|
||||
- **HTTP-backed** / **Feishu-backed**: a production route/event handler is wired.
|
||||
- **Service/test only**: code can be invoked directly, but there is no production
|
||||
product entry point.
|
||||
- **Partial**: part of the journey exists but the accepted end-to-end product
|
||||
surface cannot complete it.
|
||||
- **Missing**: no production entry and no suitable persistence/control model.
|
||||
- **Contract divergence**: reachable behavior actively uses the wrong tenancy or
|
||||
credential shape.
|
||||
|
||||
## Journey matrix
|
||||
|
||||
| Target journey | Reachable entry | Authorization | Persistence | Test evidence | Verdict and exact gap |
|
||||
|---|---|---|---|---|---|
|
||||
| Platform Administrator signs in | None. The admin plugin registers only Feishu auth and Organization routes (`hub/src/admin/plugin.ts:7-8`, `hub/src/admin/plugin.ts:31-46`). The only HTML is the Org Admin login page (`hub/src/admin/routes/authRoutes.ts:171-207`). | `PlatformRoleAssignment` and `PlatformRole.ADMIN` exist in the schema (`hub/prisma/schema.prisma:111-129`), but the HTTP guard explicitly says the platform control plane is not modeled (`hub/src/admin/auth/guards.ts:1-5`). | Role rows can be stored, but there is no platform session/identity binding, invite/bootstrap flow or platform-admin audit surface. | No platform-auth route test exists. | **Missing.** `/admin/platform` from ADR-0021 is not mounted, so no authenticated operator can reach any platform workflow. |
|
||||
| Platform Administrator creates/lists/updates/operates Organizations | None. No `/api/platform/*` or Organization CRUD route is registered; the route composition is auth + `/api/org/:orgSlug/*` only (`hub/src/admin/plugin.ts:31-46`, `hub/src/admin/routes/orgRoutes.ts:24-103`). | No `requirePlatformAdmin` implementation; only `requireSession` and `requireOrgRole` exist (`hub/src/admin/auth/guards.ts:43-113`). | `Organization` has slug, name and ACTIVE/SUSPENDED/ARCHIVED status (`hub/prisma/schema.prisma:28-52`). | Organizations are created directly only by test setup, e.g. `prisma.organization.upsert` (`hub/test/integration/helpers.ts:60-77`) and a direct cross-org fixture create (`hub/test/integration/admin-explorer.test.ts:198-210`). | **Service/test only at the database layer.** Manual pilot onboarding has no safe production command, HTTP API or UI; status changes are also unreachable. |
|
||||
| Platform Administrator configures an Organization's customer-owned Feishu app and checks readiness | None. Runtime accepts exactly one `feishuAppId`/`feishuAppSecret` pair in `AdminPluginConfig` (`hub/src/admin/plugin.ts:10-19`), and server startup reads one process-global app id, secret and bot open id (`hub/src/server.ts:47-52`). | No platform guard or Organization connection ownership check. | There is no Feishu application/secret model. `ExternalDirectoryConnection` stores provider/tenant/source metadata but no app id, bot id or secret reference (`hub/prisma/schema.prisma:196-214`). | No provisioning/readiness test exists. | **Missing + contract divergence.** There is no Organization-scoped encrypted Feishu connection or readiness workflow; production can operate only the one env-configured app. |
|
||||
| Platform Administrator configures an Organization-distinct platform-managed provider key/base URL | None. No platform provider route or UI is registered. | No platform guard. | The accepted Lean type exists only in the semantic contract (`spec/Spec/System/Organization.lean:52-66`); Prisma has no provider-connection or provider-mode model (`hub/prisma/schema.prisma:26-214`). | Runtime tests prove env values are returned, not Organization values (`hub/test/unit/runtime-settings.test.ts:18-55`). | **Missing + contract divergence.** `EnvRuntimeSettings.provider` ignores its project scope and reads one process-global `ANTHROPIC_BASE_URL`/`ANTHROPIC_AUTH_TOKEN` (`hub/src/settings/runtime.ts:20-21`, `hub/src/settings/runtime.ts:46-67`). |
|
||||
| Platform Administrator applies Organization/platform `DRAIN` or `STOP_NOW` and later resumes | None. | No platform guard, reason capture or scope check. | No workload-brake model exists; only the Lean contract defines `open`/`drain`/`stopNow` semantics (`spec/Spec/System/Capacity.lean:71-87`). | No workload-brake test exists. | **Missing.** The required audited, reason-bearing, reversible emergency control from ADR-0022 (`docs/adr/0022-layered-capacity-admission.md:69-72`) has no product or persistence surface. |
|
||||
| Organization user logs in with that Organization's Feishu app | `GET /auth/feishu` and callback are HTTP-backed (`hub/src/admin/routes/authRoutes.ts:49-128`). The callback upserts a user and signs a cookie (`hub/src/admin/routes/authRoutes.ts:99-123`). | Login itself is public OAuth; Organization APIs later enforce session + active OWNER/ADMIN membership (`hub/src/admin/auth/guards.ts:43-113`). | `User.feishuOpenId` is one process-global unique identifier (`hub/prisma/schema.prisma:89-109`); the signed cookie carries user id/open id but no Organization or Feishu-connection identity (`hub/src/admin/auth/session.ts:18-23`). | Redirect and mocked callback are covered (`hub/test/integration/admin-auth.test.ts:164-230`); OAuth request construction/exchange is covered (`hub/test/unit/feishu-oauth.test.ts:8-81`). | **HTTP-backed but contract-divergent.** OAuth is built once from the process-global app credentials (`hub/src/admin/routes/authRoutes.ts:38-47`), and the implementation explicitly documents this deferral (`hub/src/admin/auth/feishuOAuth.ts:1-10`). There is no org-selecting login route, connection resolver or issuer/app identity in the user key. |
|
||||
| Each Organization operates its own Feishu Bot | One WebSocket listener is production-wired (`hub/src/server.ts:76-90`). | Message actions are gated by project grants once a binding resolves (`hub/src/feishu/trigger.ts:715-755`), but there is no initial Organization/connection ownership gate for the inbound app. | Binding stores only `chatId -> projectId`, with no Feishu connection/application id (`hub/prisma/schema.prisma:287-304`). Event receipts store globally unique `eventId`, likewise without connection id (`hub/src/feishu/trigger.ts:687-707`). | Trigger, onboarding and event dedup are integration-tested (`hub/test/integration/trigger.test.ts:211-330`, `hub/test/integration/trigger.test.ts:878-928`). | **Feishu-backed for one bot, contract-divergent for SaaS.** Startup constructs one client and one WS client from one config (`hub/src/feishu/client.ts:854-901`). It cannot operate or route multiple Organization-owned apps. |
|
||||
| Suspended/archived Organization is prevented from bot work | No platform status-change entry exists. Org HTTP APIs do reject non-ACTIVE Organizations (`hub/src/admin/auth/guards.ts:79-88`). | The Feishu trigger resolves an active chat binding directly and then project permissions; it does not load or check `Project.organization.status` (`hub/src/feishu/trigger.ts:715-755`). | Organization status exists (`hub/prisma/schema.prisma:30-52`). | No suspended-Organization trigger test exists. | **Partial and unsafe.** Even a database-side status change only blocks web admin; an already-bound chat remains triggerable through the bot path. |
|
||||
| Logged-in Organization OWNER/ADMIN reaches the management panel | OAuth success redirects to `/admin/org/:slug` (`hub/src/admin/routes/authRoutes.ts:237-267`). | All Organization APIs default to OWNER/ADMIN through `requireOrgRole` (`hub/src/admin/auth/guards.ts:66-113`), and tests prove MEMBER is denied while ADMIN succeeds (`hub/test/integration/admin-auth.test.ts:95-131`). | Session is an HMAC cookie; user/membership is re-read for each guard (`hub/src/admin/auth/session.ts:31-57`, `hub/src/admin/auth/guards.ts:43-63`). | Guard behavior is integration-tested (`hub/test/integration/admin-auth.test.ts:56-131`). | **API-backed, UI missing.** The plugin says the static SPA is “later” (`hub/src/admin/plugin.ts:1-3`); no route/static handler serves `/admin/org/:slug`. OAuth therefore redirects to an unmounted page. |
|
||||
| Organization OWNER/ADMIN manages members and roles | REST endpoints list/add/change/revoke members (`hub/src/admin/routes/membersRoutes.ts:20-98`). | OWNER/ADMIN route guard plus service rules: only OWNER may affect OWNER and the last OWNER cannot be removed (`hub/src/org/members.ts:1-8`, `hub/src/org/members.ts:97-205`). | `OrganizationMembership` persists org, user, role and revocation (`hub/prisma/schema.prisma:54-76`). | Add/list/last-OWNER protection is covered through HTTP (`hub/test/integration/admin-members-teams.test.ts:49-83`). | **HTTP-backed and persisted; management UI missing.** Addition requires a raw `feishuOpenId` and may create the `User` directly (`hub/src/org/members.ts:46-85`); there is no invite/accept or org-connection directory picker. Role-change/revoke variants have less direct HTTP coverage than add/list. |
|
||||
| Organization OWNER/ADMIN manages teams and team membership | REST endpoints cover list/create/update/archive and membership add/revoke (`hub/src/admin/routes/teamsRoutes.ts:21-139`). | All routes require Organization OWNER/ADMIN; service verifies the team and added user belong to the same Organization (`hub/src/org/teams.ts:177-215`, `hub/src/org/teams.ts:263-275`). | `Team` and `TeamMembership` are Organization/project-principal persistence (`hub/prisma/schema.prisma:143-177`). | Create/add-member/archive is covered through HTTP (`hub/test/integration/admin-members-teams.test.ts:85-160`). | **HTTP-backed and persisted; management UI missing.** No production route exposes `ExternalDirectoryConnection`/external team binding sync, so the Feishu directory portion remains service-only. |
|
||||
| Organization OWNER/ADMIN creates and manages Folders/Projects | REST endpoints cover explorer, folder create/update/archive, project create/detail/rename/move/archive (`hub/src/admin/routes/explorerRoutes.ts:33-212`). | All routes require Organization OWNER/ADMIN. Service operations scope folder/project ids to the Organization (`hub/src/org/explorer.ts:301-341`); HTTP cross-org lookup is tested. | Folder and Project carry `organizationId`; Folder is transparent and Project owns workspace/session/run relations (`hub/prisma/schema.prisma:238-285`). Project creation also creates a real workspace and default permission rows (`hub/src/projectOnboarding.ts:275-348`). | Create/list and move/archive are HTTP-tested (`hub/test/integration/admin-explorer.test.ts:66-196`); cross-org id access is tested (`hub/test/integration/admin-explorer.test.ts:198-223`). | **HTTP-backed and substantially tested; management UI missing.** This is the strongest accepted product slice. |
|
||||
| Organization OWNER/ADMIN manages Feishu chat binding | HTTP can archive a binding (`hub/src/admin/routes/explorerRoutes.ts:214-227`). New binding is exposed through the Feishu onboarding card, not web. | Archive requires OWNER/ADMIN at HTTP and then project manager authorization in the shared service (`hub/src/projectOnboarding.ts:231-272`, `hub/src/org/explorer.ts:301-314`). Bind existing requires project MANAGE (`hub/src/projectOnboarding.ts:179-229`). | `ProjectGroupBinding` and the FEISHU_CHAT EDIT grant are transactionally stored (`hub/src/projectOnboarding.ts:190-215`, `hub/prisma/schema.prisma:287-304`). | Bind/one-active/archive/rebind is service-tested (`hub/test/integration/project-onboarding.test.ts:114-160`); card bind is integration-tested (`hub/test/integration/trigger.test.ts:271-330`). No direct HTTP archive test was found. | **Feishu + partial HTTP-backed; UI missing.** Binding identity is not scoped by Feishu application, so the 1:1 uniqueness and lookup are process-global rather than Organization-connection-scoped. |
|
||||
| Organization OWNER/ADMIN manages project team access | REST list/grant/revoke endpoints exist (`hub/src/admin/routes/accessRoutes.ts:19-79`). | OWNER/ADMIN route guard; service refuses a team from another Organization (`hub/src/permissions/projectTeamAccess.ts:168-195`). | TEAM -> PROJECT `PermissionGrant` persists the role (`hub/prisma/schema.prisma:428-447`). | Grant/list/archive-induced revoke is HTTP-tested (`hub/test/integration/admin-members-teams.test.ts:123-156`). | **HTTP-backed and tested; UI missing.** Only team grants are surfaced. There is no HTTP management of direct USER/chat grants, `PermissionSettings`, or per-role `RoleTriggerGrant`. |
|
||||
| Organization OWNER/ADMIN manages general project settings, roles/models and agent policies | Only `membersCanCreateProjects` is reachable through GET/PATCH `/settings` (`hub/src/admin/routes/orgRoutes.ts:46-81`). | OWNER/ADMIN route guard. | `OrganizationProjectSettings` stores only that Boolean (`hub/prisma/schema.prisma:78-87`). `PermissionSettings` exists, but no admin route references it (`hub/prisma/schema.prisma:449-467`). Model/role registry is explicitly an in-memory skeleton awaiting DB/admin wiring (`hub/src/agent/models.ts:65-79`). | The Boolean setting is HTTP-tested (`hub/test/integration/admin-auth.test.ts:133-162`); model env behavior is unit-tested (`hub/test/unit/runtime-settings.test.ts:4-64`). | **Partial.** The creation policy works, but the accepted “projects and configuration” control plane lacks editable permission settings, model/role settings and run policies. |
|
||||
| Organization OWNER/ADMIN chooses BYOK vs platform-managed Provider mode; BYOK admin writes key/base URL | None. | No Organization provider guard because no route/service exists. | No Prisma provider connection, credential mode, encrypted secret handle or rotation metadata. The only persistent `provider` fields are descriptive strings on sessions/runs (`hub/prisma/schema.prisma:308-330`, `hub/prisma/schema.prisma:352-387`). | No provider-mode test. Existing tests assert the global env resolver (`hub/test/unit/runtime-settings.test.ts:18-55`). | **Missing + contract divergence.** Lean/ADR have the mode but production always calls `settings.provider("openrouter", { projectId })` (`hub/src/feishu/trigger.ts:195-199`), whose implementation discards the scope (`hub/src/settings/runtime.ts:46-67`). |
|
||||
| Organization OWNER/ADMIN views sessions and runs | HTTP list and detail endpoints exist (`hub/src/admin/routes/sessionsRoutes.ts:14-45`). | OWNER/ADMIN route guard; service scopes project/session through Organization id (`hub/src/org/sessions.ts:6-23`, `hub/src/org/sessions.ts:49-87`). | `AgentSession`/`AgentRun` persist provider/model/status/tokens/cost and relations (`hub/prisma/schema.prisma:308-387`). | No direct test for these HTTP routes or `org/sessions.ts` was found. Trigger tests do prove sessions/runs are written, e.g. separate role sessions (`hub/test/integration/trigger.test.ts:850-876`). | **HTTP-backed and persisted, but untested at the product boundary and UI missing.** |
|
||||
| Organization OWNER/ADMIN views Organization/Project/Folder usage | HTTP Organization and Project usage endpoints exist, with optional folder/date filtering (`hub/src/admin/routes/sessionsRoutes.ts:47-79`). | OWNER/ADMIN route guard. | Rollup derives Organization through Project and uses persisted token/cost facts (`hub/src/org/usage.ts:35-77`, `hub/src/org/usage.ts:106-151`). Missing cost increments `runsWithoutCost`, not known cost (`hub/src/org/usage.ts:106-118`). | No direct usage route/service test was found. `/cost` tests cover the Feishu session command, not these admin endpoints (`hub/test/integration/trigger.test.ts:423-480`). | **HTTP-backed and persisted, but incomplete relative to ADR-0022.** It does not attribute by Provider Connection because none exists, has no alert configuration/delivery, no UI, and no product-boundary tests. |
|
||||
| Ordinary MEMBER creates a Project from an unbound Feishu group | Mentioning the bot in an unbound chat sends an onboarding card (`hub/src/feishu/trigger.ts:850-887`); card action calls shared project creation (`hub/src/feishu/trigger.ts:554-614`). | User must already exist and have an active Organization membership; ordinary creation is gated by `membersCanCreateProjects` (`hub/src/projectOnboarding.ts:149-177`, `hub/src/projectOnboarding.ts:380-403`). Creator receives USER MANAGE and the chat receives EDIT (`hub/src/projectOnboarding.ts:302-340`). | Project, Folder, workspace, binding, default settings and grants are persisted (`hub/src/projectOnboarding.ts:275-348`). | Card display/create/bind is integration-tested (`hub/test/integration/trigger.test.ts:211-330`); policy denial and grants are service-tested (`hub/test/integration/project-onboarding.test.ts:70-112`). | **Feishu-backed and well-tested for the single-app case.** For a user with multiple active memberships, onboarding refuses and says the org selector does not exist (`hub/src/feishu/trigger.ts:889-929`). With a per-Org app, the inbound connection should supply tenant context; current global bot cannot. |
|
||||
| Ordinary MEMBER binds an existing manageable Project | Onboarding card lists unbound projects and card action binds one (`hub/src/feishu/trigger.ts:868-885`, `hub/src/feishu/trigger.ts:932-969`). | Non-admin candidate projects require `collaborator.manage`; bind service rechecks project MANAGE (`hub/src/feishu/trigger.ts:951-960`, `hub/src/projectOnboarding.ts:179-215`). | Binding + FEISHU_CHAT EDIT grant persist transactionally. | Card journey is integration-tested (`hub/test/integration/trigger.test.ts:271-330`). | **Feishu-backed for the single-app case; per-Organization connection routing is absent.** |
|
||||
| Ordinary authorized member triggers an Agent Run from Feishu | Bound message event resolves Project, evaluates `agent.trigger`, resolves role/model/provider, creates session/run and starts the runner (`hub/src/feishu/trigger.ts:684-848`, especially `hub/src/feishu/trigger.ts:715-755` and `hub/src/feishu/trigger.ts:140-256`). | Missing sender id is fail-closed; project EDIT+ and optional role grant are enforced (`hub/src/feishu/trigger.ts:171-194`, `hub/src/feishu/trigger.ts:726-755`). | Session/run, messages, file changes, event receipt and audit fragments persist in Prisma (`hub/prisma/schema.prisma:306-405`, `hub/prisma/schema.prisma:496-525`). | Lifecycle, permission denial, dedup, audit, messages and interrupt have integration coverage (`hub/test/integration/trigger.test.ts:116-154`, `hub/test/integration/trigger.test.ts:597-607`, `hub/test/integration/trigger.test.ts:878-928`, `hub/test/integration/trigger.test.ts:983-1100`). | **Feishu-backed and tested for one process-global provider/Feishu connection.** It is not an Organization-resolved run: provider scope is ignored and bot/identity connection is absent. |
|
||||
| Accepted work waits under Project/Organization/platform capacity and survives restart | Only a process-local queue per Project exists (`hub/src/feishu/triggerQueue.ts:18-45`). It is drained after the active Project run (`hub/src/feishu/trigger.ts:430-495`). | No Organization admission identity or capacity policy is evaluated. | Queue is a `Map` in memory (`hub/src/feishu/triggerQueue.ts:21-26`), not a Prisma model; accepted requests vanish on process exit. Expired entries are dropped and only logged (`hub/src/feishu/trigger.ts:475-483`). | FIFO/full/expiry behavior is unit-tested (`hub/test/unit/trigger-queue.test.ts:10-78`); locked-trigger enqueue/drain is integration-tested (`hub/test/integration/trigger.test.ts:609-673`). | **Contract divergence.** ADR-0022 requires durable bounded admission, Organization-fair scheduling, explicit EXPIRED/CANCELED states and notifications (`docs/adr/0022-layered-capacity-admission.md:33-46`); none are present. |
|
||||
| Platform ceiling + optional lower Organization policy protect request/run/file/storage/entity capacity | None beyond Fastify defaults, a five-item Project queue and global max turns. `Fastify({ logger: true })` has no capacity options/plugins (`hub/src/server.ts:55-55`); dependencies include no rate-limit plugin (`hub/package.json:9-17`). | No admin policy route. | No capacity/limit model. `OrganizationProjectSettings` has only `membersCanCreateProjects` (`hub/prisma/schema.prisma:78-87`). Run policy contains only global `maxTurns` (`hub/src/settings/runtime.ts:24-37`, `hub/src/settings/runtime.ts:74-78`). The systemd unit has no CPU/memory/process controls (`hub/deploy/cph-hub.service:6-26`). | Unit tests cover only global max turns and the local queue (`hub/test/unit/runtime-settings.test.ts:58-64`, `hub/test/unit/trigger-queue.test.ts:5-113`). | **Missing.** None of ADR-0022's mandatory platform/org request rate, body, concurrency, file/archive, storage, entity-count, wall-time/tool/output or process limits are product-reachable or persistently enforced (`spec/Spec/System/Capacity.lean:13-60`). |
|
||||
|
||||
## Reachability summary
|
||||
|
||||
### Production-wired HTTP surface
|
||||
|
||||
The actual production plugin mounts these capability groups:
|
||||
|
||||
- global Feishu OAuth/session: `/auth/feishu`, callback, logout, `/api/me`, and
|
||||
the minimal `/admin/login` HTML (`hub/src/admin/routes/authRoutes.ts:49-207`);
|
||||
- Organization summary and one onboarding Boolean
|
||||
(`hub/src/admin/routes/orgRoutes.ts:27-81`);
|
||||
- Folder/Project explorer and binding archive
|
||||
(`hub/src/admin/routes/explorerRoutes.ts:33-227`);
|
||||
- member roles (`hub/src/admin/routes/membersRoutes.ts:20-98`);
|
||||
- teams/team membership (`hub/src/admin/routes/teamsRoutes.ts:21-139`);
|
||||
- TEAM -> PROJECT grants (`hub/src/admin/routes/accessRoutes.ts:19-79`);
|
||||
- sessions and usage (`hub/src/admin/routes/sessionsRoutes.ts:14-79`).
|
||||
|
||||
Every `/api/org/:orgSlug/*` route uses the same OWNER/ADMIN guard. This satisfies
|
||||
the clarified requirement that a management API cannot be used by an anonymous
|
||||
or ordinary Organization member (`hub/src/admin/auth/guards.ts:43-113`). It does
|
||||
not provide the accepted management **panel**: only `/admin/login` is rendered,
|
||||
and the post-login `/admin/org/:slug` destination is not served.
|
||||
|
||||
### Production-wired Feishu surface
|
||||
|
||||
The trigger handler is a real end-to-end application path for one Feishu app:
|
||||
event receipt/dedup, chat binding, permission checks, unbound-chat onboarding,
|
||||
Project creation/binding, sessions/runs, streaming and interrupt. The service
|
||||
wires the handler to a single SDK client and WebSocket listener
|
||||
(`hub/src/server.ts:76-90`, `hub/src/feishu/client.ts:854-901`). The code and tests
|
||||
therefore establish a useful single-app vertical slice, not the required
|
||||
per-Organization connection architecture.
|
||||
|
||||
### Only database/service/test reachable
|
||||
|
||||
- Organization creation is direct Prisma fixture work
|
||||
(`hub/test/integration/helpers.ts:60-77`).
|
||||
- `PlatformRoleAssignment` is a stored type with no platform auth/HTTP surface
|
||||
(`hub/prisma/schema.prisma:111-129`).
|
||||
- external-directory sync has service code and can create membership rows
|
||||
(`hub/src/permissions/externalSync.ts:40-90`), but no registered production
|
||||
admin route.
|
||||
- project `PermissionSettings` and `RoleTriggerGrant` are stored/enforced in
|
||||
parts of the runtime, but have no Organization management API
|
||||
(`hub/prisma/schema.prisma:449-494`).
|
||||
|
||||
### Entirely absent from runtime persistence/control plane
|
||||
|
||||
- Organization-scoped Feishu application connection and encrypted secret
|
||||
handle;
|
||||
- Organization-scoped Provider Connection, BYOK/platform-managed mode, encrypted
|
||||
secret handle, rotation and runtime resolver;
|
||||
- platform administrator session/guard, platform Organization CRUD, platform
|
||||
admin invitation, platform-operation audit;
|
||||
- versioned platform capacity ceiling loader/validation;
|
||||
- Organization policy-limit rows and Organization admin endpoints;
|
||||
- durable run request/admission state, fair Organization scheduler, queue cancel
|
||||
and expiry notification;
|
||||
- Organization/platform workload brake state and audited actions;
|
||||
- usage/cost alert configuration and delivery;
|
||||
- the Organization and platform management UI.
|
||||
|
||||
## Critical production blockers
|
||||
|
||||
1. **No Organization connection control plane.** The running service has one
|
||||
Feishu app/bot and one provider token/base URL. This contradicts the defining
|
||||
SaaS requirement even though Project/Team data is otherwise Organization-
|
||||
scoped (`hub/src/server.ts:44-52`, `hub/src/settings/runtime.ts:46-67`).
|
||||
|
||||
2. **No platform control plane.** A Platform Administrator cannot authenticate,
|
||||
create or operate an Organization, provision its Feishu app, configure its
|
||||
platform-managed provider connection, or apply a workload brake. The only
|
||||
apparent Organization creation path is direct database/test code.
|
||||
|
||||
3. **No usable management panel.** The backend Organization APIs are meaningful
|
||||
and guarded, but OAuth redirects to an unmounted `/admin/org/:slug`; platform
|
||||
UI does not exist at all (`hub/src/admin/plugin.ts:1-8`,
|
||||
`hub/src/admin/routes/authRoutes.ts:237-267`).
|
||||
|
||||
4. **ADR-0022 is almost wholly unimplemented.** The in-memory Project queue is
|
||||
specifically the implementation divergence ADR-0022 rejects; there are no
|
||||
Organization-fair durable admission, mandatory layered limits, hard run/file/
|
||||
storage/process budgets or emergency brakes
|
||||
(`docs/adr/0022-layered-capacity-admission.md:79-88`).
|
||||
|
||||
5. **Tenant context is inferred too late and from global identifiers.** OAuth
|
||||
creates a globally keyed `User`, bot events bind globally by `chatId`, and an
|
||||
unbound-chat user with more than one Organization is rejected rather than
|
||||
routed by the customer-owned Feishu connection (`hub/prisma/schema.prisma:89-109`,
|
||||
`hub/src/feishu/trigger.ts:715-723`, `hub/src/feishu/trigger.ts:889-929`). This
|
||||
blocks a correct multi-app implementation and also leaves Organization status
|
||||
unenforced on the bot path.
|
||||
|
||||
## What is already worth retaining
|
||||
|
||||
The audit does not imply rewriting the working core. The following seams are
|
||||
concrete and have useful test evidence:
|
||||
|
||||
- Organization ownership on Project/Team/Folder and same-Organization access
|
||||
checks (`hub/prisma/schema.prisma:143-177`, `hub/prisma/schema.prisma:238-285`);
|
||||
- OWNER/ADMIN guards and last-OWNER protection
|
||||
(`hub/src/admin/auth/guards.ts:66-113`, `hub/src/org/members.ts:97-205`);
|
||||
- reusable service operations for project creation, chat binding and grants
|
||||
(`hub/src/projectOnboarding.ts:128-229`, `hub/src/projectOnboarding.ts:275-348`);
|
||||
- Organization-scoped explorer/member/team/team-access HTTP APIs and their
|
||||
integration tests;
|
||||
- Feishu onboarding, project permission checks, session/run persistence and
|
||||
dedup in the single-app vertical slice.
|
||||
|
||||
The shortest path to the accepted destination is therefore to add the missing
|
||||
platform/connection/capacity control planes and real admin UI around these seams,
|
||||
then change ingress and runtime resolution to carry an explicit Organization
|
||||
connection identity end to end. Treating the existing env credentials as an
|
||||
acceptable pilot fallback would preserve the wrong tenant boundary.
|
||||
@@ -0,0 +1,259 @@
|
||||
# Accepted SaaS product-surface completeness audit
|
||||
|
||||
Audit date: 2026-07-10
|
||||
|
||||
## Verdict
|
||||
|
||||
The current Hub is **not a usable or production-complete SaaS product surface**
|
||||
for the accepted destination. It has a worthwhile Organization-scoped backend
|
||||
foundation and a well-tested single-Feishu-application Agent Run slice, but the
|
||||
running product still has one process-global Feishu app/bot and one global model
|
||||
provider credential. It has no platform-administrator control plane, no
|
||||
Organization connection/secret plane, and no Organization-scoped Provider
|
||||
Credential Mode. Its only HTML product page is a login form; successful OAuth
|
||||
redirects an administrator to an unmounted `/admin/org/:orgSlug` route.
|
||||
|
||||
ADR-0022's required production safety surface is also almost wholly absent:
|
||||
there is no durable Organization-fair admission, layered platform/Organization
|
||||
limits, hard file/storage/run/process budgets, usage-alert control plane, or
|
||||
audited workload brake. These are release blockers, not optional polish.
|
||||
|
||||
The strongest code should be retained rather than rewritten: all current
|
||||
`/api/org/:orgSlug/*` routes reauthorize an active OWNER/ADMIN membership;
|
||||
Folder/Project, member, Team, Team-to-Project access, session, and usage services
|
||||
are Organization-scoped; and the shared project-onboarding services correctly
|
||||
create Project/chat/grant relationships for the single-app case. The shortest
|
||||
route is to add explicit Organization connection identity and the two private
|
||||
admin control planes around those seams.
|
||||
|
||||
The claim-by-claim source inventory, including route, schema and test locations,
|
||||
is in [accepted product-surface code inventory](product-surface-code-inventory.md).
|
||||
Official Feishu identifier, OAuth, event and long-connection semantics are in
|
||||
[Feishu product-surface primary sources](product-surface-feishu-primary-sources.md).
|
||||
|
||||
## Accepted boundary
|
||||
|
||||
The audit treats these as required because they are already accepted by the
|
||||
repository's `CONTEXT.md`, ADR-0020, ADR-0021, ADR-0022, and Lean contract:
|
||||
|
||||
- a Platform Administrator is separate from Organization membership and uses a
|
||||
private `/admin/platform` area to create and operate Organizations;
|
||||
- an Organization OWNER/ADMIN uses a private `/admin/org/:orgSlug` area to
|
||||
manage members, projects, Teams, connections, settings, sessions and usage;
|
||||
- every Organization authenticates and exchanges bot/directory traffic through
|
||||
its own customer-owned Feishu Application;
|
||||
- every Organization chooses BYOK or an Organization-distinct platform-managed
|
||||
Provider Connection, with different administrators owning the respective
|
||||
plaintext inputs;
|
||||
- ordinary members can create/bind a Project from an unbound Feishu group only
|
||||
under the Organization policy, while all Project actions retain Project-level
|
||||
authorization;
|
||||
- production has mandatory platform ceilings, optional lower Organization
|
||||
limits, durable fair admission, bounded work and audited emergency brakes.
|
||||
|
||||
The audit does not add deferred self-service signup, payment collection,
|
||||
folder ACL inheritance, commercial billing, or a live editor for platform
|
||||
capacity ceilings. Guided Feishu app setup and manually initiated Organization
|
||||
creation by an authenticated Platform Administrator remain valid for the pilot.
|
||||
|
||||
## Journey result
|
||||
|
||||
| Accepted journey | Current product entry | Result | Owning frontier |
|
||||
| --- | --- | --- | --- |
|
||||
| Platform Administrator signs in | No platform login/session/guard; only a stored `PlatformRoleAssignment` type | **Missing** | tickets 42, 45 |
|
||||
| Platform Administrator creates and operates Organizations | Direct Prisma calls in test fixtures only; no `/api/platform/*` or UI | **Test/database only** | ticket 45 |
|
||||
| Platform Administrator configures each customer Feishu app | One env `FEISHU_APP_ID/SECRET/BOT_OPEN_ID`; no connection or secret model | **Missing and contract-divergent** | tickets 43–45 |
|
||||
| Platform Administrator supplies a distinct platform-managed provider key/base URL | One process-global `ANTHROPIC_AUTH_TOKEN/BASE_URL`; resolver ignores Project scope | **Missing and contract-divergent** | tickets 43, 45 |
|
||||
| Platform Administrator applies and reverses `DRAIN`/`STOP_NOW` | No persistence, guard, API or UI | **Missing** | tickets 37, 45 |
|
||||
| Organization user logs in through that Organization's Feishu app | OAuth exists, but is built once from the global app and has no Organization/application issuer | **Single-app only** | tickets 43, 44 |
|
||||
| Logged-in Organization OWNER/ADMIN opens the management panel | OAuth redirects to `/admin/org/:slug`; no handler/static application serves it | **Stable 404** | ticket 46 |
|
||||
| Organization OWNER/ADMIN manages members, Teams, Projects/Folders and Team access | Guarded Organization JSON APIs and useful integration coverage exist | **Backend present; browser product absent** | ticket 46 |
|
||||
| Organization OWNER/ADMIN manages permission/model/role/run policy | Only `membersCanCreateProjects` is exposed; other stored/runtime settings have no control plane | **Partial** | ticket 46 |
|
||||
| Organization selects Provider Credential Mode and manages BYOK | No mode/connection model, API or UI | **Missing** | tickets 43, 46 |
|
||||
| Organization views sessions and usage | Guarded JSON APIs exist, but have no direct product-boundary tests or UI and cannot attribute Provider Connection | **Partial** | tickets 36, 46 |
|
||||
| Ordinary member creates or binds a Project in Feishu | Card/service path is present and well tested | **Works for the one-global-app case** | ticket 44 for SaaS routing |
|
||||
| Authorized member runs the agent through its Organization bot | Full single-app trigger path exists; provider/app resolution is global | **Works only as a single-app deployment** | tickets 43, 44 plus lifecycle tickets |
|
||||
| Accepted work queues fairly and durably across Organizations | Five-item process-memory Project queue | **Contract-divergent** | tickets 23, 32–35 |
|
||||
| Organization sets lower policy limits and receives soft alerts | No policy/alert persistence or product surface | **Missing** | tickets 32, 34–36, 46 |
|
||||
|
||||
## Executed defect evidence
|
||||
|
||||
The probes used the real Fastify plugin and test PostgreSQL database through a
|
||||
temporary Vitest integration file. The file was removed after capture and was
|
||||
run twice to separate deterministic behavior from a transient test failure.
|
||||
|
||||
### 1. The successful-login destination is not served
|
||||
|
||||
The probe seeded an active Organization ADMIN, minted the normal signed session
|
||||
cookie, and requested the exact destination selected by
|
||||
`resolvePostLoginRedirect`:
|
||||
|
||||
```text
|
||||
GET /admin/org/test-default
|
||||
expected_status=200
|
||||
actual_status=404
|
||||
```
|
||||
|
||||
Both runs returned 404. The root cause is structural: `registerAdminPlugin`
|
||||
mounts cookie, auth and Organization API routes only; its own comment says the
|
||||
static SPA is “later”. No other repository HTML/TSX/Vue/Svelte/CSS product
|
||||
application or deployment-level static handler exists. Existing OAuth tests
|
||||
assert only the redirect `Location`, so they stop immediately before the broken
|
||||
journey.
|
||||
|
||||
This is not an authorization defect in the JSON APIs. Those APIs correctly
|
||||
return 401 without a session and 403 to a MEMBER. It is a reachability defect:
|
||||
there is no management panel for either an authorized or unauthorized request
|
||||
to reach. Ticket 46 must guard both the page shell and its data, so an anonymous
|
||||
user may reach only the login entry and never management content.
|
||||
|
||||
### 2. Different Organizations start OAuth with the same application
|
||||
|
||||
The probe seeded two Organizations and requested the current OAuth entry with
|
||||
two different Organization return paths:
|
||||
|
||||
```text
|
||||
GET /auth/feishu?returnTo=/admin/org/test-default
|
||||
client_id=cli_process_global
|
||||
|
||||
GET /auth/feishu?returnTo=/admin/org/second
|
||||
client_id=cli_process_global
|
||||
```
|
||||
|
||||
Both runs produced the same result. The root cause is that
|
||||
`registerAuthRoutes` constructs one immutable OAuth configuration from plugin
|
||||
startup arguments. The `returnTo` path is signed only as a destination; it does
|
||||
not resolve or bind an Organization Feishu Application. `server.ts` obtains the
|
||||
same app credentials from one environment tuple and also uses them for the only
|
||||
WebSocket listener.
|
||||
|
||||
Changing only the redirect page cannot fix this. The composition root and every
|
||||
identifier-bearing record/action need an explicit Organization connection
|
||||
identity, backed by the encrypted resolver in ticket 43.
|
||||
|
||||
## Feishu identifier and transport consequence
|
||||
|
||||
Feishu's official identifier contract makes the current global identifier
|
||||
shape unsafe for customer-owned applications:
|
||||
|
||||
- `open_id` identifies a user inside one application and differs across apps;
|
||||
- `user_id` is stable inside one tenant but differs across tenants;
|
||||
- `union_id` can link apps from the same application service provider, but the
|
||||
service cannot assume independently customer-owned apps share that provider;
|
||||
- OAuth authorization identifies the app with `client_id`; the callback brings
|
||||
`code` and `state`, so state and token exchange must remain bound to the
|
||||
selected Organization/application;
|
||||
- v2 HTTP event/card headers expose application and tenant identity, while an
|
||||
encrypted outer request must be routed to the correct connection before its
|
||||
verification/decryption key is known; a long connection is itself created by
|
||||
one app credential pair, so the local client/listener identity is the issuer;
|
||||
- Feishu rejects cross-tenant chat/operator combinations and does not provide a
|
||||
contract that permits `chat_id` to be treated as a global SaaS key.
|
||||
|
||||
The current `User.feishuOpenId @unique`, session `{userId, feishuOpenId}`,
|
||||
`ProjectGroupBinding.chatId`, `FeishuEventReceipt.eventId`, principal IDs,
|
||||
sender caches, and batching keys omit this issuer namespace. An `open_id`
|
||||
collision across customer apps could merge unrelated external identities;
|
||||
different `open_id` values for one person across apps cannot be resolved as one
|
||||
global identity automatically. A globally looked-up `chatId` or event ID has
|
||||
the analogous ambiguity.
|
||||
|
||||
The implementation must therefore persist provider-local external identities
|
||||
under a Feishu Application Connection, for example conceptually
|
||||
`(connectionId, externalIdType, externalId)`, and carry that connection through
|
||||
OAuth state/session, ingress, deduplication, chat binding, authorization,
|
||||
onboarding/card actions, rate-limit keys, directory sync, outbound delivery and
|
||||
audit. Whether two external identities represent the same global `User` is a
|
||||
separate explicit linking decision; it must not be inferred from a provider ID
|
||||
outside its documented scope.
|
||||
|
||||
## Root causes rather than symptoms
|
||||
|
||||
### 1. Tenant data was added without tenant-scoping external connections
|
||||
|
||||
Organization ownership is correctly present on Projects, Teams, Folders and
|
||||
directory-connection metadata, but the process composition still comes from
|
||||
the former single-deployment shape. The app/bot/provider credentials and the
|
||||
identifiers they issue never crossed an Organization connection boundary.
|
||||
`RuntimeScope.projectId` exists but is discarded by the env resolver, making
|
||||
the missing seam especially explicit.
|
||||
|
||||
### 2. The control-plane work stopped at guarded Organization APIs
|
||||
|
||||
The Organization routes provide a credible backend slice, but there is no web
|
||||
application, platform route family, platform guard, Organization provisioning
|
||||
service, connection configuration, or browser journey test. The comment
|
||||
“until admin SPA lands” and redirect-only OAuth assertion document the precise
|
||||
unfinished handoff. A small catch-all page would only hide the gap; the fix must
|
||||
deliver the authenticated, authorized workflows and their safe states.
|
||||
|
||||
### 3. Production capacity semantics exist only in the contract
|
||||
|
||||
ADR-0022 intentionally declares the in-memory queue and global-only settings to
|
||||
be implementation divergences. There is no common admission identity that can
|
||||
join Organization, Feishu connection, Project, user and request, so fair
|
||||
scheduling, workload brakes, rate limits, alerts and control-plane operations
|
||||
cannot currently compose. The previously created capacity tickets own this
|
||||
implementation; the admin tickets must expose their safe operator/customer
|
||||
surfaces rather than invent parallel controls.
|
||||
|
||||
## What should remain
|
||||
|
||||
- Keep `requireOrgRole`'s per-request session, active Organization, active
|
||||
membership and OWNER/ADMIN checks; extend the same principle to page data and
|
||||
add a separate `requirePlatformAdmin` rather than an Organization override.
|
||||
- Keep the Organization-scoped explorer/member/Team/access/session/usage APIs
|
||||
as the initial backend for the org panel, while filling their missing tests,
|
||||
pagination/limits, audit and typed-error contracts through existing tickets.
|
||||
- Keep shared Project creation and chat-binding services so web and Feishu
|
||||
entries do not drift into two rule sets.
|
||||
- Keep the single-app Feishu handler's authorization, onboarding, run/session
|
||||
and dedup behavior, but call it from a connection-bound listener context and
|
||||
move accepted work/delivery into the durable lifecycle tickets.
|
||||
- Keep missing provider cost unknown rather than fabricating zero; add Provider
|
||||
Connection attribution before presenting SaaS usage claims.
|
||||
|
||||
## Newly clear execution frontier
|
||||
|
||||
The product audit creates four implementation/decision slices after the
|
||||
existing secret and browser-boundary design work:
|
||||
|
||||
1. [Decide the platform-administrator identity and audit boundary](../issues/42-decide-platform-admin-identity-audit.md).
|
||||
2. [Implement the Organization connection and secret plane](../issues/43-implement-org-connection-secret-plane.md).
|
||||
3. [Route Feishu identity and traffic by Organization connection](../issues/44-route-feishu-by-org-connection.md).
|
||||
4. [Build the private platform-administrator control plane](../issues/45-build-private-platform-admin-control-plane.md).
|
||||
5. [Build the private Organization-administrator control plane](../issues/46-build-private-org-admin-control-plane.md).
|
||||
|
||||
The Feishu routing slice precedes durable inbound/action/outbound and signed
|
||||
traffic limiting so those mechanisms use the right issuer namespace from their
|
||||
first durable schema. The platform panel precedes the emergency-brake product
|
||||
entry so ticket 37 can reuse one authenticated/audited operator boundary.
|
||||
Existing tickets 32–38 continue to own capacity enforcement, attribution,
|
||||
alerts, brakes and calibration; the new UI tickets do not duplicate them.
|
||||
|
||||
## Required production journey evidence
|
||||
|
||||
The final release gate must run against at least two configured customer-owned
|
||||
Feishu applications and two Organizations, using deliberately colliding fake
|
||||
provider-local IDs where the test harness permits. It must prove:
|
||||
|
||||
1. an unauthenticated request and an authenticated wrong-role user cannot load
|
||||
either management page data or perform mutations;
|
||||
2. a Platform Administrator can create an Organization, bootstrap its OWNER,
|
||||
configure/test its Feishu app and distinct platform-managed Provider
|
||||
Connection, with redacted responses and complete audit history;
|
||||
3. an Organization OWNER/ADMIN logs in through that Organization's app, reaches
|
||||
the real org panel, manages the accepted settings/resources, configures BYOK
|
||||
when selected, and cannot read another Organization or platform secret;
|
||||
4. each bot receives, binds, authorizes, queues and replies only inside its
|
||||
connection's Organization even when `open_id`, `chat_id`, event or callback
|
||||
values collide; SUSPENDED/ARCHIVED Organizations fail closed;
|
||||
5. provider resolution for every run returns that Organization's selected
|
||||
connection and never a process-global fallback;
|
||||
6. current member onboarding, Project authorization and Agent Run behavior
|
||||
remain intact after connection namespacing;
|
||||
7. layered limits, fair admission, usage alerts and `DRAIN`/`STOP_NOW` are
|
||||
visible only to the corresponding authorized control plane and behave as
|
||||
proven by their dedicated fault/capacity suites;
|
||||
8. the complete browser + Feishu journey survives the production-shaped deploy,
|
||||
restart, backup/restore and rollback evidence owned by ticket 08.
|
||||
@@ -0,0 +1,333 @@
|
||||
# Feishu multi-app identity and event-routing primary sources
|
||||
|
||||
This note answers one narrow design question for the production SaaS audit:
|
||||
what identity and routing boundaries apply when **every Hub Organization uses
|
||||
its own Feishu App and bot**. External claims below use only first-party Feishu
|
||||
Open Platform documentation. Statements labelled **Derived requirement** are
|
||||
Hub design implications, not guarantees made by Feishu.
|
||||
|
||||
Research date: 2026-07-10.
|
||||
|
||||
## Executive answer
|
||||
|
||||
1. A customer-owned enterprise custom app is a single-tenant app: Feishu says
|
||||
it is developed, reviewed, published, and used inside one tenant and cannot
|
||||
be used by another organization. That fits “one Organization, one Feishu
|
||||
App,” but it means the Hub is operating many independent app security and
|
||||
identity domains, not one shared Feishu identity domain.
|
||||
([custom apps versus store apps](https://open.feishu.cn/document/home/app-types-introduction/self-built-apps-and-store-apps))
|
||||
2. A bare `open_id` is **not** a SaaS-global user key. It identifies a user
|
||||
inside one app, and the same user has different Open IDs in different apps.
|
||||
`user_id` is tenant-scoped; it is stable across apps in one tenant but
|
||||
differs across tenants. `union_id` crosses apps only for apps from the same
|
||||
application provider/developer; it differs between providers.
|
||||
([user resource overview](https://open.feishu.cn/document/server-docs/contact-v3/user/field-overview?lang=zh-CN),
|
||||
[official ID parameter definitions](https://open.feishu.cn/document/server-docs/contact-v3/department/search?lang=zh-CN),
|
||||
[Union ID guide](https://open.feishu.cn/document/uAjLw4CM/ugTN1YjL4UTN24CO1UjN/trouble-shooting/how-to-obtain-union-id))
|
||||
3. OAuth must be routed to the Organization's app **before** authorization.
|
||||
The authorization request contains that app's App ID as `client_id`; the
|
||||
callback contains `code` and the echoed `state`, not an independently
|
||||
trustworthy Organization choice. Exchanging the code requires the matching
|
||||
app's `client_id` and `client_secret`, and the same `redirect_uri`.
|
||||
([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code),
|
||||
[obtain user access token](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
|
||||
4. A version 2 event or callback carries `header.app_id` and
|
||||
`header.tenant_key` after parsing/decryption. An encrypted HTTP delivery,
|
||||
however, exposes only an `encrypt` envelope before decryption, so a shared
|
||||
endpoint cannot safely choose among many apps' Encrypt Keys from the body.
|
||||
([callback structure](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/callback-overview),
|
||||
[encrypted Webhook delivery](https://open.feishu.cn/document/event-subscription-guide/event-subscriptions/event-subscription-configure-/choose-a-subscription-mode/send-notifications-to-developers-server?lang=zh-CN))
|
||||
5. A WebSocket/long-connection client is authenticated with one app's App ID
|
||||
and App Secret. Therefore its local client instance is the pre-authenticated
|
||||
source-app route. It is not a process-global “all customer apps” listener.
|
||||
([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
|
||||
6. Feishu documents `chat_id` as the conversation identifier, but the cited
|
||||
documentation does not promise that a raw `chat_id` is globally unique
|
||||
across every tenant and app. Chat APIs explicitly reject an operator and
|
||||
chat from different tenants and often require the app bot to be in the
|
||||
chat. A bot is represented by its application's `app_id`.
|
||||
([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology),
|
||||
[get chat members](https://open.feishu.cn/document/server-docs/group/chat-member/get?lang=zh-CN),
|
||||
[message resource](https://open.feishu.cn/document/server-docs/im-v1/message/intro))
|
||||
|
||||
## 1. Application and tenant boundary
|
||||
|
||||
### Official facts
|
||||
|
||||
- `app_id` is the immutable identifier generated by Feishu for an application.
|
||||
`tenant_key` is the enterprise identifier; one `tenant_key` corresponds to
|
||||
one enterprise in actual use.
|
||||
([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology))
|
||||
- An enterprise custom app is restricted to one tenant. Its developers,
|
||||
administrators, and users belong to that tenant, and another organization
|
||||
cannot use it. A store app is the contrasting model that can be installed by
|
||||
multiple tenants.
|
||||
([custom apps versus store apps](https://open.feishu.cn/document/home/app-types-introduction/self-built-apps-and-store-apps))
|
||||
- Access tokens tell Feishu which app/user is calling and which tenant's data
|
||||
is being accessed. A `tenant_access_token` calls as the application and its
|
||||
data range is determined by that application's permissions.
|
||||
([access-token overview](https://open.feishu.cn/document/server-docs/api-call-guide/calling-process/get-access-token))
|
||||
- A custom app obtains its `tenant_access_token` using that app's `app_id` and
|
||||
`app_secret`.
|
||||
([custom-app tenant access token](https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal?lang=zh-CN))
|
||||
|
||||
### Derived requirements
|
||||
|
||||
- Model a durable `FeishuAppConnection` (or equivalent) owned by exactly one
|
||||
Hub Organization. At minimum it binds the internal connection ID,
|
||||
`organization_id`, `app_id`, expected `tenant_key`, encrypted `app_secret`,
|
||||
event/callback credentials, lifecycle state, and credential version.
|
||||
- Do not select Feishu credentials from a process-global default. Every OAuth
|
||||
exchange, token cache entry, OpenAPI call, Webhook verification, and
|
||||
long-connection client must start from the Organization's connection.
|
||||
- Pin the first verified `tenant_key` to the connection. Later login, event,
|
||||
and callback data must match both the expected `app_id` and pinned
|
||||
`tenant_key`; unknown or mismatched pairs must be rejected rather than
|
||||
auto-attached to another Organization.
|
||||
|
||||
## 2. User identifier scopes
|
||||
|
||||
### Official scope matrix
|
||||
|
||||
| Identifier | Feishu's documented scope | Cross-app/tenant behavior | Availability caveat |
|
||||
| --- | --- | --- | --- |
|
||||
| `open_id` | User identity inside one application | The same user has a different `open_id` in different apps | The normal app-relative user identifier |
|
||||
| `user_id` | User identity inside one tenant | Stable across all apps in the same tenant; different in tenant A and tenant B | Reading it is a sensitive field permission and is limited to custom apps in the cited user API |
|
||||
| `union_id` | User identity across apps from the same application provider/developer | Same across that provider's apps; different across different providers | It does not make unrelated customer-owned apps one identity domain |
|
||||
| `tenant_key` | Enterprise identity | One key corresponds to one enterprise | Returned by login and carried in events/callbacks |
|
||||
|
||||
Sources:
|
||||
|
||||
- Feishu's current user resource overview states that `user_id` is tenant
|
||||
identity, `open_id` is app identity, and `union_id` is identity across apps
|
||||
developed by the same application provider. It expressly says the same
|
||||
user's Open ID differs between apps.
|
||||
([user resource overview](https://open.feishu.cn/document/server-docs/contact-v3/user/field-overview?lang=zh-CN))
|
||||
- Feishu's API parameter text is more explicit: the same user's User ID differs
|
||||
between tenants but remains the same across apps within one tenant; the same
|
||||
user's Union ID is the same under one developer and different under another.
|
||||
([official ID parameter definitions](https://open.feishu.cn/document/server-docs/contact-v3/department/search?lang=zh-CN))
|
||||
- Feishu describes Union ID as the identity linking one user across multiple
|
||||
applications supplied by the same application service provider.
|
||||
([Union ID guide](https://open.feishu.cn/document/uAjLw4CM/ugTN1YjL4UTN24CO1UjN/trouble-shooting/how-to-obtain-union-id))
|
||||
- Feishu's common-error guidance says not to use an Open ID obtained from app A
|
||||
in app B, and even a test app and its production app produce different Open
|
||||
IDs for the same user. It also reports cross-tenant errors for both User ID
|
||||
and Union ID use.
|
||||
([common errors](https://open.feishu.cn/document/ukTMukTMukTM/ugjM14COyUjL4ITN?op_tracking=hc))
|
||||
- The current login user-info API returns `open_id`, `union_id`, optional
|
||||
sensitive `user_id`, and `tenant_key` from a `user_access_token`.
|
||||
([get login user information](https://open.feishu.cn/document/server-docs/authentication-management/login-state-management/get?lang=zh-CN))
|
||||
|
||||
### Answer: can different customer Apps share one global `open_id` key?
|
||||
|
||||
**No.** This directly contradicts Feishu's documented Open ID semantics. Two
|
||||
customer-owned custom apps are different applications, so the same person has
|
||||
different Open IDs in them. Because those apps are independently owned custom
|
||||
apps, the Hub also has no documented basis to assume they are applications of
|
||||
one application provider and therefore no basis to auto-merge on `union_id`.
|
||||
`user_id` cannot solve cross-customer identity either because it changes across
|
||||
tenants.
|
||||
|
||||
Derived identity model:
|
||||
|
||||
- The safe external login key is `(feishu_app_connection_id, open_id)`; retain
|
||||
`tenant_key`, `user_id`, and `union_id` as scoped attributes, not as an
|
||||
unqualified SaaS-global primary key.
|
||||
- If the product needs one Hub human account to participate in multiple
|
||||
Organizations, represent each Feishu login as a separate external identity
|
||||
attached through an explicit, separately verified account-linking flow. Do
|
||||
not silently merge on Open ID, Union ID, email, phone, or display name.
|
||||
- Do not make `user_id` mandatory for login unless every customer app is
|
||||
required to obtain its sensitive field permission. `open_id` and
|
||||
`tenant_key` are returned by the user-info flow without using `user_id` as
|
||||
the login key.
|
||||
- Replacing an Organization's Feishu App changes the application identity
|
||||
domain and therefore its users' Open IDs. Treat app replacement as an
|
||||
explicit connection migration/re-link operation, not an in-place secret
|
||||
edit.
|
||||
|
||||
## 3. OAuth login must be app-routed
|
||||
|
||||
### Official facts
|
||||
|
||||
- The authorization URL requires `client_id`, which is the selected
|
||||
application's App ID. `redirect_uri` must already appear in that
|
||||
application's security settings. On success Feishu redirects with `code`
|
||||
and, when supplied, the original `state`.
|
||||
([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code))
|
||||
- Feishu says `state` maintains request/callback context and must be compared
|
||||
to prevent CSRF. Its official sample rejects a callback whose state differs
|
||||
from the stored state.
|
||||
([obtain authorization code](https://open.feishu.cn/document/common-capabilities/sso/api/obtain-oauth-code),
|
||||
[user-access-token sample](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
|
||||
- The code lasts five minutes and can be used once. Token exchange requires
|
||||
`client_id`, `client_secret`, `code`, and, when used during authorization,
|
||||
the same `redirect_uri`. Feishu defines error 20024 for a code/refresh token
|
||||
that does not match the supplied client ID and error 20071 for a mismatched
|
||||
redirect URI.
|
||||
([obtain user access token](https://open.feishu.cn/document/authentication-management/access-token/get-user-access-token))
|
||||
- Only redirect URLs configured for the selected application pass Feishu's
|
||||
security check; an app may configure multiple URLs.
|
||||
([access-token acquisition flow](https://open.feishu.cn/document/server-docs/api-call-guide/calling-process/get-access-token))
|
||||
|
||||
### Derived request flow
|
||||
|
||||
1. Begin at an Organization-qualified route or invitation, resolve exactly one
|
||||
active `FeishuAppConnection`, and create a short-lived, one-time server-side
|
||||
login transaction bound to that Organization, connection, redirect URI,
|
||||
desired return path, and PKCE verifier if used.
|
||||
2. Put only a high-entropy opaque transaction nonce in OAuth `state`; generate
|
||||
the authorization URL with that connection's `app_id` as `client_id`.
|
||||
3. On callback, atomically consume and validate `state` before selecting
|
||||
credentials. The callback's untrusted query parameters must never be
|
||||
allowed to choose another Organization or App.
|
||||
4. Exchange `code` with the connection's `app_id`, decrypted `app_secret`, and
|
||||
the exact stored redirect URI. Then call user-info and require its
|
||||
`tenant_key` to equal the connection's pinned tenant.
|
||||
5. Resolve or create the membership through
|
||||
`(feishu_app_connection_id, open_id)`. Never search all Organizations for a
|
||||
bare Open ID and accept the first match.
|
||||
|
||||
A single physical callback URL may be configured independently in multiple
|
||||
apps, but a shared URL does not remove the need for state-bound app routing.
|
||||
|
||||
## 4. HTTP event and callback routing
|
||||
|
||||
### Official facts
|
||||
|
||||
- A version 2 callback header contains the application's Verification Token,
|
||||
`event_type`, `tenant_key` (the callback application's tenant), and `app_id`
|
||||
(the callback application's App ID).
|
||||
([callback overview and structure](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/callback-overview))
|
||||
- Version 2 event examples likewise place `app_id` and `tenant_key` in the
|
||||
event header.
|
||||
([user-leaves-chat event](https://open.feishu.cn/document/server-docs/group/chat-member/event/deleted-2))
|
||||
- Verification Token is an application validation identifier. Feishu says the
|
||||
server can compare the event's token with that application's token to check
|
||||
that a request is for the specified app. With an Encrypt Key, the request
|
||||
signature is calculated from request timestamp, nonce, that app's Encrypt
|
||||
Key, and the original request body.
|
||||
([receive and verify events](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/encrypt-key-encryption-configuration-case))
|
||||
- When an Encrypt Key is configured, the HTTP request body visible before
|
||||
decryption is an envelope containing only `{"encrypt":"..."}`; `app_id`
|
||||
and `tenant_key` are inside the ciphertext.
|
||||
([send events to developer server](https://open.feishu.cn/document/event-subscription-guide/event-subscriptions/event-subscription-configure-/choose-a-subscription-mode/send-notifications-to-developers-server?lang=zh-CN))
|
||||
|
||||
### Derived routing and verification requirements
|
||||
|
||||
- Give every Feishu connection a stable, unguessable callback route such as a
|
||||
connection-specific path token. Resolve the connection from that route,
|
||||
load exactly its Encrypt Key/Verification Token, verify and decrypt, then
|
||||
require the authenticated payload's `app_id` and `tenant_key` to match the
|
||||
route's connection.
|
||||
- For an unencrypted common endpoint, `header.app_id` may identify a candidate
|
||||
connection, but it is untrusted until that connection's signature/token has
|
||||
been verified. Never route the side effect solely from an unverified body.
|
||||
- A single encrypted URL shared by every customer app has no documented
|
||||
pre-decryption app selector. Trying every tenant's secret is not an identity
|
||||
protocol. Use connection-specific routing instead.
|
||||
- Namespace delivery deduplication by the authenticated connection (for
|
||||
example `(connection_id, event_id)`), and dispatch business work only after
|
||||
app/tenant verification. Unknown, disabled, or mismatched connections fail
|
||||
closed and produce an auditable security log.
|
||||
|
||||
## 5. Long-connection routing
|
||||
|
||||
### Official facts
|
||||
|
||||
- Feishu's SDK creates a WebSocket client with one app's required `APP_ID` and
|
||||
`APP_SECRET`. Events subscribed by that application are then delivered over
|
||||
that channel. Authentication is performed when the connection is
|
||||
established.
|
||||
([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
|
||||
- Long-connection event subscription supports enterprise custom apps, not
|
||||
store apps. Each app may establish at most 50 connections. If one app has
|
||||
multiple clients, delivery is cluster-mode rather than broadcast: one random
|
||||
client receives a message.
|
||||
([long-connection event subscription](https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/request-url-configuration-case?lang=zh-CN))
|
||||
- Long-connection callbacks use the same per-app App ID/App Secret client
|
||||
construction and have the same cluster-mode single-recipient behavior.
|
||||
([long-connection callback subscription](https://open.feishu.cn/document/event-subscription-guide/callback-subscription/step-1-choose-a-subscription-mode/configure-callback-request-address?lang=zh-CN))
|
||||
|
||||
### Derived routing requirements
|
||||
|
||||
- Create one logical client binding per enabled `FeishuAppConnection`, even if
|
||||
an in-process supervisor manages many clients. Attach immutable
|
||||
`organization_id` and `connection_id` context to each handler; the client
|
||||
that received the frame is the pre-authenticated application route.
|
||||
- Never infer the receiving Organization by looking up a bare actor Open ID or
|
||||
chat ID. First use the authenticated client binding; then cross-check any
|
||||
delivered `app_id`/`tenant_key` fields available in the event structure.
|
||||
- Scaling several replicas for the same app does not create duplicate
|
||||
broadcast processing according to Feishu's contract, but delivery still
|
||||
requires application-level idempotency and health ownership. Capacity must
|
||||
count clients per app against the documented 50-connection ceiling.
|
||||
|
||||
## 6. Bot, chat, and message identifiers
|
||||
|
||||
### Official facts
|
||||
|
||||
- In the message resource, a sender can be a user or an application. For an
|
||||
application sender, `id_type` is `app_id`; the sender also carries a
|
||||
`tenant_key`. Thus the Feishu bot identity used by messaging is its
|
||||
application identity, not a Hub-global bot ID.
|
||||
([message management overview](https://open.feishu.cn/document/server-docs/im-v1/message/intro))
|
||||
- `chat_id` identifies a conversation, including direct and group chats.
|
||||
Feishu's group resource also exposes the chat's `tenant_key`.
|
||||
([common parameters](https://open.feishu.cn/document/ukTMukTMukTM/uYTM5UjL2ETO14iNxkTN/terminology),
|
||||
[group management overview](https://open.feishu.cn/document/server-docs/group/chat/intro?lang=zh-CN))
|
||||
- Chat APIs require the operator to be in the same tenant as an internal chat;
|
||||
common errors include `Operator and chat can NOT be in different tenants`.
|
||||
Many operations additionally require the calling app's bot to be in the
|
||||
group and reject an app that is unavailable in that tenant.
|
||||
([get chat members](https://open.feishu.cn/document/server-docs/group/chat-member/get?lang=zh-CN))
|
||||
- The first bot direct-chat event calls `chat_id` the conversation ID between
|
||||
**that robot and user** and includes `app_id`, `tenant_key`, and app-scoped
|
||||
user Open IDs in the same event.
|
||||
([user and bot first conversation](https://open.feishu.cn/document/server-docs/group/chat-member/event/bot-events?lang=zh-CN))
|
||||
|
||||
### What the official sources do not establish
|
||||
|
||||
The cited documentation calls `chat_id` a conversation/group identifier but
|
||||
does not state that its value is globally unique across every Feishu tenant and
|
||||
every independently created app. It does establish tenant and bot-membership
|
||||
authorization boundaries. Under the repository rule that unspecified scope
|
||||
must not be assumed, raw global uniqueness would be an unsupported inference.
|
||||
|
||||
Derived storage/call rules:
|
||||
|
||||
- Store chats as `(feishu_app_connection_id, chat_id)` and retain observed
|
||||
`tenant_key` and chat type. Do the same for other resource identifiers unless
|
||||
Feishu explicitly contracts a wider scope.
|
||||
- Always call chat/message APIs with the token obtained from the chat's bound
|
||||
connection. Never retry an authorization failure with another Organization's
|
||||
token.
|
||||
- Treat bot direct chats as connection-specific. A `chat_id` for user X's
|
||||
conversation with Organization A's bot is not evidence of a conversation
|
||||
with Organization B's bot.
|
||||
- For group chats, two customer apps may both be members of a real Feishu
|
||||
group, but that does not authorize the Hub to move configuration, Projects,
|
||||
or memberships between Organizations. The authenticated connection remains
|
||||
the tenant route and Hub Organization remains the authorization root.
|
||||
|
||||
## Minimum invariants for implementation review
|
||||
|
||||
1. `FeishuIdentity` uniqueness is `(connection_id, open_id)`, never bare
|
||||
`open_id`.
|
||||
2. OAuth `state` resolves one server-side transaction bound to one Organization
|
||||
and connection; code exchange uses only that connection's credentials and
|
||||
stored redirect URI.
|
||||
3. Login accepts the returned identity only when `tenant_key` matches the
|
||||
connection's pinned tenant.
|
||||
4. HTTP Webhooks choose a connection-specific verification/decryption context
|
||||
before trusting payload routing fields; authenticated `app_id` and
|
||||
`tenant_key` must match it.
|
||||
5. Every long-connection handler carries the local connection/Organization
|
||||
binding of the App ID/App Secret client that received the frame.
|
||||
6. Bot, chat, event-deduplication, token-cache, and user identifiers are all
|
||||
namespaced by the Feishu connection unless Feishu explicitly guarantees a
|
||||
wider scope.
|
||||
7. A credential, app, or tenant mismatch is a hard failure with traceable
|
||||
logging; there is no fallback to a default app or search across tenant
|
||||
secrets.
|
||||
@@ -0,0 +1,245 @@
|
||||
# Production observability and incident-recovery audit
|
||||
|
||||
Date: 2026-07-10
|
||||
|
||||
Scope: the `hub` HTTP service, Feishu WebSocket ingress and outbound calls, PostgreSQL/Prisma, agent execution, the `cph` subprocess, project workspaces, and the packaged deployment path. This is a release-gate audit, not a proposal to change product semantics. Where the Lean contract is open, this report asks for an operational decision rather than inventing one.
|
||||
|
||||
## Verdict
|
||||
|
||||
**No-go for unattended SaaS production deployment.** The service exposes a liveness-shaped endpoint, emits some structured component logs, and persists useful run/audit fields, but it cannot currently answer the minimum production questions:
|
||||
|
||||
1. Can this instance accept a real request *now* (database reachable, Feishu listener authenticated, required runtime executable and workspace usable)?
|
||||
2. Which tenant, request, Feishu event, run, subprocess, and outbound message participated in one failed user action?
|
||||
3. How many requests/runs/messages are failing, stuck, delayed, retried, or duplicated?
|
||||
4. Which condition should page an operator, and which documented recovery action is safe?
|
||||
5. After a restart, deploy, rollback, or dependency outage, what evidence proves that work was neither lost nor silently acknowledged?
|
||||
|
||||
Three fault injections demonstrate that the current `/api/healthz` and logging surface can be green while the service is already unable to serve its core paths:
|
||||
|
||||
- After startup, cutting PostgreSQL sockets left two consecutive `/api/healthz` requests at HTTP 200.
|
||||
- An invalid Feishu app ID was rejected by the installed Feishu SDK while `/api/healthz` remained HTTP 200.
|
||||
- With PostgreSQL cut, `/api/me` returned HTTP 400 `bad_request`, exposed a raw Prisma invocation in the response, produced no request-scoped application error log, and only emitted two unscoped Prisma error lines.
|
||||
|
||||
Together with the already-confirmed lifecycle and deployment failures, this means green process state, green health checks, and even a visible user error do not prove that the service or its durable workflows are healthy.
|
||||
|
||||
## Evidence and positive signals already present
|
||||
|
||||
The starting point is not empty. These are useful foundations to retain:
|
||||
|
||||
- Fastify's built-in HTTP logging records request/response facts with an internal request ID through the server logger ([`hub/src/server.ts`](../../../hub/src/server.ts)).
|
||||
- The same Fastify logger is threaded into the Feishu trigger/runtime, while Prisma emits its configured error/warn events (and optional query events) ([`hub/src/server.ts`](../../../hub/src/server.ts), [`hub/src/db.ts`](../../../hub/src/db.ts)).
|
||||
- `AgentRun` persists status, timestamps, error/result fields, token/cost fields, and links to project/session; `ProjectAgentLock` persists a heartbeat; `AuditEntry` and `AgentMessage` provide potential durable trace anchors ([`hub/prisma/schema.prisma`](../../../hub/prisma/schema.prisma)).
|
||||
- Agent execution emits intermediate activity to the streaming Feishu card, and lock heartbeat activity is attempted during execution ([`hub/src/agent/runner.ts`](../../../hub/src/agent/runner.ts), [`hub/src/feishu/trigger.ts`](../../../hub/src/feishu/trigger.ts)).
|
||||
- Deployment is managed by systemd, so restart policy, exit status, and journal collection have an existing operational host surface ([`hub/deploy/cph-hub.service`](../../../hub/deploy/cph-hub.service)).
|
||||
- Database migrations are explicit Prisma artifacts rather than ad-hoc startup DDL ([`hub/prisma/migrations`](../../../hub/prisma/migrations)).
|
||||
- Existing tests cover many individual authorization, trigger, lock, queue, runner, and API behaviors. They are valuable regression evidence, although they are not yet a deploy/recovery acceptance suite.
|
||||
|
||||
These signals are fragmented rather than joined into an operational contract. Several writes are best-effort or swallowed, identifiers are not propagated end-to-end, there is no metrics/alert surface, and the health endpoint does not test dependencies.
|
||||
|
||||
## Failure-to-signal matrix
|
||||
|
||||
| Surface | Failure or degraded condition | Current externally visible signal | What is missing / why the signal is unsafe | Minimum production signal and recovery evidence |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| HTTP | PostgreSQL becomes unreachable after startup | Affected request fails; `/api/healthz` stays 200 | Health is process liveness only. `/api/me` misclassifies the dependency failure as client `400 bad_request`, leaks raw Prisma call text, and creates no request-scoped error record. An LB/operator cannot distinguish bad input from platform outage. | Separate liveness and readiness. Readiness performs a bounded DB probe and returns non-2xx on failure. Unexpected exceptions map to sanitized 5xx with a stable error code and `request_id`; structured log contains the same ID and normalized Prisma category. Alert on readiness failure and 5xx/error-code rate. |
|
||||
| HTTP | Handler latency, event-loop stall, exhausted connections, overload | Access log only after a response is completed | No in-flight count, latency histogram, timeout counter, saturation signal, or request correlation. A hanging request may emit no timely evidence. | Request count, duration, in-flight requests, timeout/cancellation count, response class, and event-loop/runtime/process saturation. Bound request duration and log start/end/error with one `request_id`. |
|
||||
| Feishu ingress | Invalid/revoked app credentials; WebSocket connect/auth failure | SDK rejects/logs asynchronously; process and `/api/healthz` remain green | Startup success is reported before listener readiness. No listener state, last-connected/last-event timestamp, reconnect counter, or readiness dependency. The installed SDK already exposes `onReady`, `onError`, `onReconnecting`, `onReconnected`, `getConnectionStatus`, and `close`, but the Hub does not retain or wire the `WSClient`. systemd can perpetually supervise a process that cannot receive work. | Listener state machine (`starting/connected/degraded/stopped`), last successful connection/event, reconnect/auth-failure counters, bounded startup readiness gate, and alert on auth failure or prolonged disconnect. Credential contents must never be logged. |
|
||||
| Feishu ingress | Duplicate delivery, handler failure after receipt, process restart with queued/batched work | Event receipt/dedup row and selected warning logs | The run-lifecycle audit proved receipt-before-success and in-memory batching/queue loss. There is no durable processing state/lease, retry age, dead-letter view, or alert for accepted-but-unstarted work. | Durable ingress states and timestamps, attempts, last normalized error, event/chat/project correlation, queue depth/oldest age, dead-letter count. Recovery drill proves restart resumes or deliberately redrives exactly the intended work. See [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
|
||||
| Feishu outbound | API timeout/ambiguous success, rate limit, invalid credentials, permanent card patch/send failure | Some helpers log a warning; several message helpers catch failure and return `null` | No idempotent delivery record, delivery state, attempt count, provider request/message ID, retry delay, terminal failure metric, or user-visible reconciliation path. Fault injection proved a run can be marked completed although nothing was delivered. | Durable outbox/delivery state; idempotency key; provider message/request ID; attempts, latency and normalized provider error; queue depth/oldest age/dead-letter signals; recovery command to retry or reconcile safely. |
|
||||
| PostgreSQL | Startup unavailable | Process startup rejects | This is a useful fail-fast signal, but it is not joined to a deploy gate with bounded logs and rollback criteria. | Installer/supervisor waits for explicit readiness, captures failure reason, and aborts/rolls back without replacing last-known-good service. |
|
||||
| PostgreSQL | Runtime outage, pool saturation, slow queries, migration incompatibility | Raw Prisma errors; request symptoms | No readiness failure, pool/saturation metric, query latency/timeout signal, migration compatibility assertion, or tenant/request/run correlation. Raw client internals can cross the HTTP boundary. | DB probe latency/result, Prisma error code/category counters, bounded query timing for critical operations, connection/pool saturation if available, schema migration identifier, sanitized API mapping, and DB outage/runbook drill. Prisma documents structured client error types/codes and configurable event-based logging; use those instead of returning raw exception strings ([Prisma error reference](https://www.prisma.io/docs/orm/reference/error-reference), [Prisma logging](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging)). |
|
||||
| Agent run | SDK/model error, credentials invalid, provider throttling, malformed stream | Run may become `FAILED`; the card may show an error and trigger finalization may warn | The installed SDK emits API retry, rate-limit, tool-progress, runtime-init/MCP-status, provider request ID, duration, and error-category messages, but `runner.ts` handles only stream/assistant/user/result messages and drops the other lifecycle signals. Provider/model/attempt/latency/normalized error are not consistently available as metrics or correlated logs. Some persistence and audit operations are best-effort. | Run start/finish transition event with `run_id`, `session_id`, tenant/project, provider/model, status, duration, turns, tokens/cost, normalized error and retryability; capture the SDK's bounded retry/rate-limit/tool/runtime signals; aggregate run outcome/latency/stuck gauges without high-cardinality IDs as metric labels. |
|
||||
| Agent run | No result/hang, missed heartbeats, process death, terminal DB write failure | Lock heartbeat may stop; restart sweep mutates some state | No wall-clock deadline/watchdog, stale-lock/stale-active-run metric, readiness impact, page, or documented reconciliation. Lifecycle fault injections proved active run without lock, terminal run with lock, and timeout misclassification are possible. | Deadline and durable lease semantics, gauges for active/waiting/stale runs and locks, oldest age, transition-failure counter, reconciliation command and drill. Startup recovery must be transactional and evidence-producing. See [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
|
||||
| `cph` subprocess | Missing binary, permission failure, timeout, non-zero exit, malformed output | In production, `cph_check`/`cph_build` grant the Claude SDK's Bash tool; tool results are forwarded only to the in-memory streaming card. The direct `agent/cph.ts` wrapper captures exit/stdout/stderr but is used only by its integration tests, not by the production runner. | There is no production-owned `cph` invocation boundary, startup/readiness validation of the exact deployed artifact, version/build identity, persisted operation result, duration metric, bounded/redacted stderr contract, or reliable distinction between spawn failure and domain diagnostics. | Readiness checks executable path/version and required filesystem permissions without mutating domain state. Introduce an observable production invocation boundary or capture the SDK Bash/tool lifecycle with operation, run/project correlation, duration, exit category and bounded redacted diagnostic; metrics count timeout/spawn/non-zero/protocol errors. Deployment evidence pins and smoke-tests the artifact. |
|
||||
| Workspace | Missing/unwritable root, wrong ownership, disk/inode exhaustion, symlink/root escape, partial agent writes | Error appears only when a command/file operation happens | No readiness check, free-space/inode monitoring, workspace mutation journal/commit marker, or recovery procedure. Security audit already proved containment failures; lifecycle audit proved a run can fail after workspace side effects. | Bounded non-destructive readiness check for configured root, owner/mode, read/write/rename/fsync as required, free space and inode signals, per-project mutation correlation, containment validation, backup/restore and partial-write reconciliation drill. See [tenant/auth security audit](./tenant-auth-security-audit.md) and [run lifecycle audit](./run-lifecycle-integrity-audit.md). |
|
||||
| Deploy | Bad env/service user/artifact, failed migration/build, unhealthy new process | Shell/systemd exit in some cases; process may restart | Clean-host audit found missing required env/service identity/artifact and in-place, non-atomic release replacement. Health is liveness-only and there is no last-known-good rollback unit. | Immutable versioned release, preflight, migration compatibility gate, explicit readiness deadline, release/build/schema identity endpoint/log, automated rollback criteria, last-known-good artifact and tested rollback. See [clean-host deployment audit](./clean-host-deployment-audit.md). |
|
||||
| Deploy | SIGTERM/restart during active work | systemd stops/restarts the process | No graceful drain/stop readiness, bounded shutdown, durable queue resume, or post-restart reconciliation proof. In-memory queues are lost. | On termination: readiness false, stop ingress, drain or checkpoint durable work, enforce timeout, emit shutdown summary. Recovery drill sends work across restart and verifies a single durable outcome and delivery. systemd supports explicit notification/readiness/watchdog protocols, but the current unit does not implement one ([systemd service types](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html), [`sd_notify`](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html)). |
|
||||
|
||||
## Logs and end-to-end traceability
|
||||
|
||||
### Current gaps
|
||||
|
||||
- Fastify generates an internal request ID for its access logs, but it is not returned to the client as a support handle or propagated into application errors, Prisma diagnostics, Feishu work, runs, or subprocess calls.
|
||||
- Feishu `event_id`, provider request/message IDs, `chat_id`, `project_id`, `organization_id`, `run_id`, `session_id`, and subprocess invocation are not propagated as one causally connected context.
|
||||
- The DB-cut experiment produced raw Prisma diagnostics without an application error record that identified route, request, response class, or tenant context.
|
||||
- Several critical failure paths catch and ignore persistence errors or only warn, including audit/message persistence and some card/lock finalization paths. This creates absence of evidence precisely when durable state is least trustworthy.
|
||||
- The production runner discards SDK `api_retry`, `rate_limit_event`, `tool_progress`, runtime initialization/MCP status, and related system messages even though the installed SDK exposes the fields needed to diagnose provider and tool degradation.
|
||||
- Exception strings can cross the API boundary. Besides leaking implementation details, this makes the client-visible error vocabulary unstable and impossible to alert on reliably.
|
||||
- Logs have no documented retention, rotation, ingestion, access-control, redaction, or tenant-support lookup contract. Journald availability alone is not a log strategy.
|
||||
|
||||
### Minimum structured event envelope
|
||||
|
||||
Every boundary and durable state transition should emit one structured event with a stable `event_name`, `timestamp`, `severity`, `service`, `environment`, `release`, and the applicable correlation fields:
|
||||
|
||||
`request_id`, `feishu_event_id`, `organization_id`, `project_id`, `chat_id`, `actor_user_id`, `run_id`, `session_id`, `delivery_id`, `provider_request_id`, and `cph_invocation_id`.
|
||||
|
||||
Raw prompt/message/file contents, credentials, cookies, authorization headers, provider tokens, full Prisma invocation text, and unrestricted stdout/stderr must not be logged. Tenant/project/user identifiers are operationally necessary but access-controlled personal/customer metadata; metric labels must not use high-cardinality IDs. OpenTelemetry's HTTP semantic conventions specify the stable `http.server.request.duration` instrument, an opt-in `http.server.active_requests` instrument, and low-cardinality route/error guidance ([HTTP metrics semantic conventions](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/)).
|
||||
|
||||
At minimum, events are required for:
|
||||
|
||||
- request accepted/completed/failed/canceled;
|
||||
- Feishu listener connected/disconnected/auth-failed/reconnecting;
|
||||
- ingress received/deduplicated/queued/claimed/started/retried/dead-lettered;
|
||||
- run and lock state transitions, transition-write failures, heartbeat stale detection, and recovery decisions;
|
||||
- provider/agent start/result/error with normalized category;
|
||||
- `cph` spawn/result/timeout/protocol failure;
|
||||
- workspace preflight and mutation/commit/reconciliation decisions;
|
||||
- outbound delivery queued/attempted/confirmed/retried/dead-lettered;
|
||||
- deploy, schema migration, readiness transition, drain, shutdown, startup recovery, and rollback.
|
||||
|
||||
Errors must be allowed to fail the affected operation rather than disappear. If a non-critical telemetry sink is unavailable, that telemetry failure itself needs a bounded local signal/counter; durable business audit records are not optional telemetry and must follow their explicit transactional contract.
|
||||
|
||||
## Health, readiness, and startup contract
|
||||
|
||||
The service needs three distinct concepts:
|
||||
|
||||
1. **Liveness**: the process/event loop is responsive. This endpoint must be cheap and must not depend on remote services. The present `/api/healthz` can serve this purpose if it is named/documented as liveness only.
|
||||
2. **Readiness**: the instance can accept the traffic assigned to it. A bounded probe must include at least PostgreSQL connectivity/schema compatibility, Feishu listener authenticated/connected state for ingress-serving instances, required runtime configuration, exact `cph` artifact availability/version, and workspace root access/capacity. A failed dependency makes readiness non-2xx without killing the process and losing diagnostics.
|
||||
3. **Startup/deploy acceptance**: migrations/preflight complete, listener reaches ready state within a deadline, one authenticated HTTP/database smoke path and one controlled runtime/workspace smoke path pass, and the deployed release/schema identity matches the intended version. Only then is traffic enabled and the previous release eligible for cleanup.
|
||||
|
||||
Readiness responses should expose only component names, stable state/error codes, and timestamps—not secrets or raw exceptions. Full diagnostic context belongs in correlated restricted logs. The deployment audit must decide whether Feishu and HTTP run in one instance role; if roles are split later, readiness dependencies become role-specific rather than globally optional.
|
||||
|
||||
## Minimum metrics and alerts
|
||||
|
||||
Exact SLO values are product/operations decisions and are currently unspecified. The release nevertheless needs the following bounded-cardinality instruments and a reviewed initial threshold for each page/ticket alert.
|
||||
|
||||
### Traffic and failures
|
||||
|
||||
- HTTP request count, duration, active requests, status/error-code class, timeouts/cancellations.
|
||||
- Feishu listener connected state, reconnect/auth failures, time since last successful connection/event.
|
||||
- Ingress queue/receipt counts by durable state, processing attempts, oldest pending age, dead-letter count.
|
||||
- Outbound delivery attempts/results/latency, retry/dead-letter count, oldest pending age, provider rate-limit responses.
|
||||
- Prisma operation failures by normalized code/category and bounded critical-query duration; DB readiness result/latency.
|
||||
|
||||
### Workflow integrity and capacity
|
||||
|
||||
- Runs started and terminal outcomes, run duration, active/waiting/stale count and oldest age.
|
||||
- Locks held/stale/orphaned and transition/reconciliation failures.
|
||||
- Agent/provider failures by provider/model/error category; turns/tokens/cost as operational/business counters with tenant detail kept out of metric labels.
|
||||
- `cph` invocation count/duration/result category/timeouts.
|
||||
- Workspace free bytes/inodes, preflight failures, write/commit/reconciliation failures.
|
||||
- Process uptime/restarts, event-loop delay, memory, CPU, file descriptors, unhandled rejection/uncaught exception, graceful shutdown outcome.
|
||||
- Release/build/schema identity as an info metric or startup event, never as an unbounded label set.
|
||||
|
||||
### Required release-time alerts
|
||||
|
||||
Before production traffic, alerts and routing must exist for:
|
||||
|
||||
- instance readiness false beyond the rollout/startup grace period;
|
||||
- Feishu auth failure or prolonged listener disconnect;
|
||||
- sustained server-side HTTP errors or request timeouts;
|
||||
- database readiness failure / critical Prisma failure spike;
|
||||
- oldest ingress/outbound item, active run, or lock exceeds its decided deadline;
|
||||
- dead-letter count non-zero or durable state-transition/reconciliation failure;
|
||||
- agent/provider or `cph` failure/timeout spike;
|
||||
- workspace disk/inodes below headroom threshold or workspace preflight fails;
|
||||
- restart loop, failed deployment, failed migration, or rollback;
|
||||
- telemetry pipeline blind (expected heartbeat/scrape/log ingestion absent).
|
||||
|
||||
Each alert needs an owner, severity, user impact statement, dashboard/log query, first safe diagnostic command, mitigation, escalation criterion, and recovery verification. A page without a runbook is not a production control.
|
||||
|
||||
## Incident recovery runbooks required before release
|
||||
|
||||
At minimum, checked-in runbooks must cover these scenarios. Commands must default to read-only inspection; destructive/redrive/force-release actions require explicit confirmation, identify tenant/project/run scope, emit an audit record, and be idempotent or document why they are not.
|
||||
|
||||
1. **PostgreSQL unavailable/degraded** — identify connection vs credential vs migration vs capacity failure; stop accepting work through readiness; restore service; reconcile active runs, locks, ingress receipts, and outbox; verify no raw errors escape HTTP.
|
||||
2. **Feishu listener disconnected/credentials revoked** — distinguish auth from transient network/reconnect; restore credentials safely; verify last event and controlled inbound round trip; inspect ingress gap and redrive rules.
|
||||
3. **Ingress backlog/dead letter** — list oldest/attempts/error by tenant-safe scope; decide retry vs reject; prove dedup/idempotency; verify one resulting run.
|
||||
4. **Stuck/orphaned run or lock** — inspect run/session/lock/provider/workspace/audit/outbox together; use lease/reconciliation operation rather than direct row deletion; prove terminal state and next queued work behavior.
|
||||
5. **Outbound delivery failure** — reconcile provider message ID and durable outbox; retry with idempotency key; confirm user-visible delivery without duplicating a card/message.
|
||||
6. **Agent/provider or `cph` outage** — identify provider credential/rate-limit/runtime/protocol failure; halt or queue new runs; test a canary; resume and reconcile affected work.
|
||||
7. **Workspace capacity/corruption/containment event** — quarantine affected project, snapshot evidence, restore/rollback documented unit, verify ownership/containment and repository state before resuming.
|
||||
8. **Failed migration/deployment** — preserve old release, collect preflight/migration/readiness diagnostics, follow compatibility-aware application/database rollback, and verify release/schema identity plus canaries.
|
||||
9. **Process/host restart during work** — drain when planned; after unplanned restart run durable reconciliation; verify receipts, runs, locks, workspace effects and deliveries converge exactly once at the business level.
|
||||
10. **Suspected tenant data exposure** — disable affected path/tenant access without deleting evidence, preserve correlated audit/log records, rotate exposed credentials, determine affected org/project/users, and follow an explicitly owned incident-notification decision process.
|
||||
|
||||
## Minimum release evidence
|
||||
|
||||
Production readiness is not proved by a green unit suite alone. A release candidate needs an archived, attributable evidence bundle containing all of the following:
|
||||
|
||||
### Build and supply identity
|
||||
|
||||
- immutable application and `cph` artifact digests, source commit, build timestamp/toolchain, dependency lockfiles and vulnerability policy result;
|
||||
- deployed configuration manifest listing required keys and instance role, with secret values redacted;
|
||||
- migration set/schema identity and a compatibility statement for both forward deploy and rollback;
|
||||
- systemd unit and installer/preflight version used on a clean host.
|
||||
|
||||
### Automated gates
|
||||
|
||||
- typecheck, complete unit/integration suite, migration-from-supported-version tests, and Lean `lake build` when `spec/` changes;
|
||||
- clean-host install test with the real packaged artifacts and declared service user/directories;
|
||||
- negative startup tests for missing/invalid env, DB unavailable, invalid Feishu credentials, missing/wrong `cph`, and unwritable/full workspace;
|
||||
- readiness tests proving each dependency failure independently makes readiness fail while liveness remains diagnostically available;
|
||||
- API error-mapping tests proving unexpected dependency exceptions return sanitized 5xx/stable codes and create a correlated error log;
|
||||
- lifecycle fault tests from the run-lifecycle audit, including process kill/restart, DB transition failures, duplicate ingress, ambiguous outbound completion, timeout, and concurrent trigger/session creation;
|
||||
- tenant isolation/security regression suite, symlink/root containment tests, and dependency audit policy gate.
|
||||
|
||||
### Staging drills against production-shaped dependencies
|
||||
|
||||
- authenticated Feishu inbound → durable receipt → one run → workspace effect → durable terminal state → one outbound confirmation, searchable end-to-end by a correlation ID;
|
||||
- PostgreSQL socket cut/restoration while serving: readiness changes, alert fires, requests return sanitized server errors, state reconciles, alert clears;
|
||||
- Feishu credential rejection/disconnect/reconnect: readiness/alert and controlled recovery are visible;
|
||||
- SIGTERM and forced process kill during queued, active, and outbound-pending work: restart reconciliation proves no silent loss and no unintended duplicate user effect;
|
||||
- `cph` timeout/non-zero/malformed response, provider throttle/error, workspace disk/permission failure, and outbound rate-limit/permanent failure drills;
|
||||
- failed deploy/readiness and rollback drill on a clean host, followed by migration/data compatibility and canary verification;
|
||||
- backup restore of representative database and workspace data, with measured recovery point/time and tenant isolation checks.
|
||||
|
||||
### Operations acceptance
|
||||
|
||||
- dashboards expose all release metrics above and show release/schema identity;
|
||||
- every required alert is injected once in staging, reaches the named owner, links to its checked-in runbook, and clears only after verified recovery;
|
||||
- log search can pivot from request/event to org/project/run/session/delivery without message content or secrets; retention and access controls are documented and tested;
|
||||
- on-call ownership, escalation, incident severity, maintenance/rollback authority, backup ownership, and recovery objectives are explicitly decided;
|
||||
- a signed release checklist records who reviewed each piece of evidence and where immutable test/drill outputs are stored.
|
||||
|
||||
## Prioritized remediation / release gates
|
||||
|
||||
### P0 — before any external production traffic
|
||||
|
||||
1. Implement distinct bounded liveness, readiness, and startup/deploy acceptance contracts; make deployment traffic activation depend on readiness.
|
||||
2. Install centralized unexpected-error handling that returns sanitized stable 5xx responses and emits request-correlated structured errors. Never return raw Prisma/SDK/process exception text.
|
||||
3. Make Feishu connection/auth state observable and readiness-relevant; make listener startup rejection fail the startup gate.
|
||||
4. Implement the durable ingress, transactional run/lock/session, durable outbox, workspace commit/reconciliation, and durable audit/history work already opened by the lifecycle audit. Observability cannot compensate for acknowledged work that has no recoverable state.
|
||||
5. Add end-to-end correlation and stable structured transition events; remove silent catches from critical paths.
|
||||
6. Add the minimum metrics, alerts, dashboards, and checked-in recovery runbooks, then execute the fault/restart/rollback drills.
|
||||
7. Replace in-place deployment with the clean-host audit's immutable release/readiness/rollback contract and prove it on a clean host.
|
||||
8. Close the critical tenant/agent/workspace/Feishu authorization and dependency findings from the security audit before declaring readiness.
|
||||
|
||||
### P1 — required for routine operations immediately after the P0 architecture is in place
|
||||
|
||||
- Define initial SLOs/error budgets and tune alert thresholds using staging/load evidence.
|
||||
- Add capacity/load/soak tests for DB connections, event-loop/process resources, ingress/outbound backlog, agent concurrency, provider limits, workspace disk/inodes, and log/metric pipelines.
|
||||
- Exercise backup/restore, credential rotation, tenant-support trace lookup, and incident evidence preservation.
|
||||
- Define data retention/deletion/access-control policy for logs, metrics, traces, audits, messages, receipts, deliveries, runs, and workspaces.
|
||||
|
||||
## Decision points that remain OPEN
|
||||
|
||||
The code/spec does not currently authorize choosing these product/operations semantics implicitly:
|
||||
|
||||
- readiness dependency roles (combined HTTP+Feishu process versus independently deployable roles);
|
||||
- maximum acceptable ingress, run, lock, and outbound ages; retry budgets; terminal/dead-letter semantics;
|
||||
- workspace mutation commit/rollback unit and what a user should observe after partial agent work;
|
||||
- audit durability/availability trade-off and exact event taxonomy;
|
||||
- SLOs, recovery point objective, recovery time objective, retention, regional/data-residency constraints, and incident-notification ownership;
|
||||
- whether deploy migration compatibility must support application rollback across one or more releases.
|
||||
|
||||
These decisions should be recorded in the upstream contract/ADR layer before implementation when they affect domain-visible behavior. Operational mechanisms that only expose already-defined behavior can be implemented and tested without inventing domain semantics.
|
||||
|
||||
## Source notes
|
||||
|
||||
Primary local evidence:
|
||||
|
||||
- [`hub/src/server.ts`](../../../hub/src/server.ts), [`hub/src/admin/errors.ts`](../../../hub/src/admin/errors.ts) — process startup, health/listener/restart behavior, and HTTP exception-to-response mapping.
|
||||
- [`hub/src/db.ts`](../../../hub/src/db.ts) — current Prisma event/log configuration.
|
||||
- [`hub/src/feishu/client.ts`](../../../hub/src/feishu/client.ts), [`hub/src/feishu/trigger.ts`](../../../hub/src/feishu/trigger.ts), [`hub/src/feishu/messageBatcher.ts`](../../../hub/src/feishu/messageBatcher.ts), [`hub/src/feishu/triggerQueue.ts`](../../../hub/src/feishu/triggerQueue.ts) — ingress/listener, batching, dedup, queue, run and outbound behavior.
|
||||
- [`hub/src/agent/runner.ts`](../../../hub/src/agent/runner.ts), [`hub/src/agent/roleTools.ts`](../../../hub/src/agent/roleTools.ts), [`hub/src/agent/cph.ts`](../../../hub/src/agent/cph.ts), [`hub/src/lock.ts`](../../../hub/src/lock.ts), [`hub/src/audit.ts`](../../../hub/src/audit.ts) — production SDK/Bash tool routing, the legacy direct cph wrapper, persistence, heartbeat/lock, and audit behavior.
|
||||
- [`hub/prisma/schema.prisma`](../../../hub/prisma/schema.prisma) — durable state and available correlation fields.
|
||||
- [`hub/deploy`](../../../hub/deploy) — installer/systemd/deploy behavior.
|
||||
- [Clean-host deployment audit](./clean-host-deployment-audit.md), [tenant/auth security audit](./tenant-auth-security-audit.md), and [run-lifecycle integrity audit](./run-lifecycle-integrity-audit.md) — prior deterministic audits and their recorded commands/results.
|
||||
|
||||
Official references used only for interface semantics and implementation guidance:
|
||||
|
||||
- [Prisma Client error reference](https://www.prisma.io/docs/orm/reference/error-reference) and [Prisma logging](https://www.prisma.io/docs/orm/prisma-client/observability-and-logging/logging).
|
||||
- [OpenTelemetry HTTP metric semantic conventions](https://opentelemetry.io/docs/specs/semconv/http/http-metrics/).
|
||||
- [systemd service types/readiness](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html) and [`sd_notify`](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html).
|
||||
|
||||
No secondary production-readiness checklist was treated as authority. Recommendations are derived from the local failure evidence and the primary interfaces the service actually uses.
|
||||
@@ -0,0 +1,272 @@
|
||||
# Run lifecycle and data-integrity audit
|
||||
|
||||
## Verdict
|
||||
|
||||
The current Hub does **not** have a crash-safe or database-failure-safe run
|
||||
lifecycle. The normal happy path is covered by tests and the database correctly
|
||||
enforces one lock per project, but the surrounding workflow is a chain of
|
||||
independent effects: inbound receipt, in-memory batching/queueing, session/run
|
||||
creation, lock acquisition, workspace mutation, Feishu delivery, session cursor
|
||||
update, run terminal update, audit write, and lock release.
|
||||
|
||||
There is no durable coordinator or reconciliation loop across those effects.
|
||||
Deterministic fault injection confirmed that the current code can permanently
|
||||
lose accepted work, leave a terminal run holding a lock, leave an `ACTIVE` run
|
||||
without a lock, split one logical session into duplicate active rows, declare a
|
||||
run complete without delivering its result, and create an audit entry referring
|
||||
to no run. A hung provider call also has no wall-clock deadline or usable lease,
|
||||
so one run can block a project until an operator restarts the service or edits
|
||||
the database.
|
||||
|
||||
These are production blockers for the accepted single-host topology; they do
|
||||
not require HA to reproduce.
|
||||
|
||||
## Contract baseline
|
||||
|
||||
The audit used the following upstream decisions and intentionally did not
|
||||
invent semantics where the contract is open:
|
||||
|
||||
- `Spec.System.Run`: `active` and `waitingForUser` are non-terminal;
|
||||
`completed`, `failed`, `timedOut`, and `canceled` are terminal.
|
||||
- `Spec.System.Lock` and ADR-0002: each project has at most one lock, its owner
|
||||
is an `AgentRun`, every lock holder is non-terminal, and the lock is released
|
||||
when the run completes, fails, times out, or is canceled.
|
||||
- ADR-0002: many runs reuse the same long-lived `AgentSession`.
|
||||
- ADR-0017: the active session identity is project + provider + role + model,
|
||||
and its provider cursor is persisted for the next run.
|
||||
- `Spec.System.Audit`: every `AuditEntry` is associated with a run. Audit event
|
||||
content, retention, and query dimensions remain `OPEN`.
|
||||
- The contract does **not** decide whether filesystem changes made by a failed,
|
||||
canceled, timed-out, or process-killed run should commit, remain as partial
|
||||
work, or roll back. That choice is surfaced below rather than assumed.
|
||||
|
||||
## Positive evidence
|
||||
|
||||
The core happy path is real:
|
||||
|
||||
- PostgreSQL uniqueness enforces at most one `ProjectAgentLock` per project and
|
||||
at most one project lock per run.
|
||||
- `currentLockRunId` rejects a terminal run that still holds a lock instead of
|
||||
silently treating it as healthy.
|
||||
- Normal trigger completion updates the provider cursor, writes a terminal run
|
||||
state, and releases the lock. Interrupt uses an `AbortController` and maps to
|
||||
`CANCELED` on the normal path.
|
||||
- A sequential Feishu redelivery with the same event ID is skipped, so the
|
||||
ordinary duplicate case does not start a second run.
|
||||
- The targeted lifecycle baseline passed before fault injection:
|
||||
|
||||
```text
|
||||
npm run check
|
||||
6 test files passed
|
||||
71 tests passed
|
||||
```
|
||||
|
||||
Those protections are necessary, but they only cover operations in which every
|
||||
dependency succeeds in the expected order.
|
||||
|
||||
## Executed fault-injection evidence
|
||||
|
||||
A temporary Vitest harness exercised the real trigger, Prisma test database,
|
||||
message batcher, and runner seams. It asserted the production invariant rather
|
||||
than the current behavior. The harness was run repeatedly and produced the same
|
||||
result each time; the final run was:
|
||||
|
||||
```text
|
||||
npm test -- --run test/run-lifecycle-audit.repro.test.ts
|
||||
|
||||
Test Files 1 failed (1)
|
||||
Tests 8 failed (8)
|
||||
Duration 918ms
|
||||
```
|
||||
|
||||
The throwaway harness was removed after diagnosis. Its eight minimal probes and
|
||||
observed values were:
|
||||
|
||||
| Injected condition | Required invariant | Observed current result |
|
||||
| --- | --- | --- |
|
||||
| Authorization database error after Feishu receipt creation, followed by the same event redelivery | The failed event is retried | Authorizer calls remained `1`; redelivery was skipped |
|
||||
| Batched-run setup callback throws, then `flushAll` retries | The pending batch remains recoverable | Callback attempts remained `1`; batch had already been deleted |
|
||||
| Two active sessions with the same project/provider/role/model tuple | Database rejects the duplicate | Second insert succeeded |
|
||||
| Audit insert uses a nonexistent run ID | Database enforces `AuditEntry → AgentRun` | Insert succeeded with arbitrary `runId` |
|
||||
| Every Feishu message create/reply fails | Completion is not reported as delivered | Run became `COMPLETED`; delivered cards were `0` |
|
||||
| First lock release delete fails | Terminal run is eventually unlocked | Lock count remained `1` after run became `COMPLETED` |
|
||||
| Every terminal `AgentRun.update` fails | A non-terminal run retains/reacquires its lock | Run remained `ACTIVE`; lock count became `0` |
|
||||
| Claude SDK returns `error_max_turns` | Runner returns `length`, trigger stores `TIMED_OUT` | Runner returned `failed` |
|
||||
|
||||
This is a tight reproduction at the actual call seams; none of the failures
|
||||
depends on timing or external network access.
|
||||
|
||||
## Critical findings
|
||||
|
||||
### 1. Accepted Feishu work can be permanently lost
|
||||
|
||||
`trigger.ts` commits `FeishuEventReceipt` before message validation, binding
|
||||
lookup, authorization, file download, command execution, batching, or run
|
||||
creation. The receipt has no processing state, lease, attempt count, error, or
|
||||
completion marker. Any later exception leaves a permanent row, so Feishu's
|
||||
at-least-once redelivery is treated as already completed.
|
||||
|
||||
The two in-process buffers compound that behavior:
|
||||
|
||||
- `MessageBatcher.flushPendingBatch` deletes the batch before invoking its
|
||||
callback. Timer-triggered callback failures are swallowed with
|
||||
`.catch(() => undefined)`.
|
||||
- `TriggerQueue` and `batchContexts` are process-local maps. A restart loses all
|
||||
queued prompts and pending debounce batches. Queue draining removes an item
|
||||
before run setup is durable, so a setup error also loses it.
|
||||
|
||||
There is no shutdown hook that flushes or rejects these buffers visibly. The
|
||||
service can therefore acknowledge and forget user work even when PostgreSQL is
|
||||
healthy again milliseconds later.
|
||||
|
||||
### 2. Run state and lock ownership can contradict each other
|
||||
|
||||
Run creation, lock acquisition, terminal update, and lock release are separate
|
||||
Prisma calls. Their failure handling is asymmetric:
|
||||
|
||||
- a lock-acquisition race creates a session and an `AgentRun` first, then marks
|
||||
the losing run failed;
|
||||
- terminal update failure is caught, a second best-effort failure update is
|
||||
attempted, and `finally` releases the lock regardless;
|
||||
- lock release failure is only logged once, with no retry or reconciler.
|
||||
|
||||
Fault injection reached both forbidden outcomes: `COMPLETED + lock` and
|
||||
`ACTIVE + no lock`. The first makes `currentLockRunId` throw the explicit
|
||||
ADR-0002 invariant error on all later triggers; the second permits another run
|
||||
to start while the database still claims the earlier run is live. In the
|
||||
injected terminal-write case the earlier execution had already returned, but
|
||||
the same unlocked/non-terminal shape created by an overlapping startup can
|
||||
coexist with a still-running process.
|
||||
|
||||
Startup does not repair these states safely. `server.ts` first deletes **all**
|
||||
locks and then, in a separate statement, marks only `ACTIVE` runs failed.
|
||||
`WAITING_FOR_USER` is left non-terminal and unlocked, contrary to the contract.
|
||||
The two statements are not transactional, and running a second Hub process
|
||||
would destructively clear the live first process's locks and fail its runs.
|
||||
|
||||
### 3. A provider hang can block a project indefinitely
|
||||
|
||||
`maxTurns` is passed to the SDK, but there is no wall-clock run deadline. The
|
||||
SDK max-turn result is currently mapped to runner status `failed`, making the
|
||||
trigger's `length → TIMED_OUT` branch unreachable for the actual max-turn case.
|
||||
|
||||
`ProjectAgentLock.expiresAt` is never populated or consumed. `heartbeatAt` is a
|
||||
best-effort update only after an assistant message, not a periodic heartbeat,
|
||||
and there is no stale-lock/run sweeper. A query that stalls before its first
|
||||
assistant message remains `ACTIVE` while holding the project lock forever.
|
||||
`WAITING_FOR_USER` is declared but never written; approval waits remain
|
||||
`ACTIVE` in an in-memory `ApprovalManager`.
|
||||
|
||||
Systemd sends `SIGTERM`, but the application has no signal handler, listener
|
||||
close, in-flight drain, run cancellation, Prisma disconnect, or final recovery
|
||||
transaction. A subsequent successful restart happens to fail `ACTIVE` runs,
|
||||
but that is not a graceful lifecycle and does not cover all non-terminal
|
||||
states.
|
||||
|
||||
### 4. Workspace effects have no defined commit or recovery boundary
|
||||
|
||||
The Claude SDK's built-in Write/Bash tools mutate the project workspace while
|
||||
the run is still `ACTIVE`. A process kill, timeout, cancellation, database
|
||||
failure, or later delivery failure can therefore leave real filesystem changes
|
||||
behind while the database says the run failed or never completed.
|
||||
|
||||
No runtime code writes `AgentFileChange`: the only capture callback is in the
|
||||
legacy AI-SDK `writeFileTool`, while the production runner uses Claude SDK
|
||||
built-ins and never injects that collector. The schema comment claiming these
|
||||
changes are captured for review is not true. The transcript comments are also
|
||||
stale: the production runner neither loads nor appends the JSONL transcript;
|
||||
continuity comes from the provider cursor.
|
||||
|
||||
`AgentMessage` writes are best-effort and silently swallowed. The session
|
||||
cursor, message projection, workspace state, run terminal state, and audit
|
||||
history can consequently describe different versions of the same execution.
|
||||
Before implementing a mechanism, the product/contract must decide whether
|
||||
partial workspace changes are committed, quarantined, or rolled back.
|
||||
|
||||
## High findings
|
||||
|
||||
### 5. Session identity and cursor continuity are not concurrency-safe
|
||||
|
||||
`startAgentRun` performs `findFirst` then `create` for the active
|
||||
project/provider/role/model session before acquiring the project lock. The
|
||||
schema has only a non-unique index for that tuple. The database probe inserted
|
||||
two active rows with the same identity.
|
||||
|
||||
Concurrent triggers can therefore fork a logical conversation before one loses
|
||||
the project-lock race. Future `findFirst` selection is unspecified and may
|
||||
resume the wrong provider cursor. On the completion path, the session cursor is
|
||||
updated separately and before the run terminal update, so a partial database
|
||||
failure can advance the conversation while leaving the originating run active
|
||||
or failed.
|
||||
|
||||
### 6. Outbound delivery is silent and not recoverable
|
||||
|
||||
The Feishu send helpers catch message errors and return `null`; `sendText`
|
||||
returns `void`. `StreamingAgentCard.finish` treats a failed create/patch as a
|
||||
boolean and normal run completion proceeds. The fault probe recorded a
|
||||
`COMPLETED` run with zero delivered cards.
|
||||
|
||||
There is no durable outbox, provider message ID record, delivery state,
|
||||
idempotency key, retry schedule, or reconciliation job. An operator cannot tell
|
||||
whether the user saw the result, and retrying a larger workflow later would
|
||||
have no durable key with which to prevent duplicate messages.
|
||||
|
||||
### 7. Audit rows violate the pinned relation and audit failures are invisible
|
||||
|
||||
`AuditEntry.runId` is nullable, has no foreign key, and accepts arbitrary
|
||||
strings. This directly diverges from the pinned `Spec.System.Audit` relation
|
||||
that every audit entry belongs to a run. Pre-run permission/security events are
|
||||
valid product needs, but they require an explicit event model rather than
|
||||
weakening the one relation the contract fixes.
|
||||
|
||||
`writeAudit` catches every Prisma error without logging, metric, retry, or
|
||||
caller signal, and existing unit tests encode that swallowing as desired.
|
||||
Lifecycle history can be absent exactly when PostgreSQL is unhealthy, with no
|
||||
evidence that evidence was lost. The content and failure policy are `OPEN` and
|
||||
need a decision, but silent unobservable loss is incompatible with operating
|
||||
and debugging the service.
|
||||
|
||||
## Failure-mode summary
|
||||
|
||||
- **Process termination/restart:** loses batches, queued prompts, approvals,
|
||||
active controller handles, and unrecorded message/file history; may leave
|
||||
partial workspace changes; recovery ignores `WAITING_FOR_USER`.
|
||||
- **Timeout/hang:** no wall deadline, periodic heartbeat, lease, or sweeper;
|
||||
max-turn termination is mislabeled `FAILED` rather than `TIMED_OUT`.
|
||||
- **Duplicate Feishu delivery:** sequential success is deduplicated, but any
|
||||
failure after receipt becomes permanent loss; receipt check/create is also
|
||||
not one transactional claim operation.
|
||||
- **Database failure:** can produce either terminal-with-lock or
|
||||
active-without-lock, lose session/message/audit facts, or advance only the
|
||||
provider cursor.
|
||||
- **Concurrent requests/processes:** the project-lock unique key prevents two
|
||||
normal lock rows, but active sessions can fork; a second process's startup
|
||||
reset can invalidate the first process's live lifecycle.
|
||||
- **Outbound failure:** run completion and user-visible delivery are unrelated,
|
||||
and no durable recovery/idempotency record exists.
|
||||
|
||||
## Required release evidence
|
||||
|
||||
Production readiness requires automated proof of all of the following:
|
||||
|
||||
1. Each accepted inbound event has a durable state machine and can be retried
|
||||
after process/DB failure without duplicate run creation.
|
||||
2. Queueing and batching either survive restart or have an explicit, observable
|
||||
rejection/acknowledgement contract that cannot silently lose work.
|
||||
3. Every database fault point preserves `lock exists ⇒ holder non-terminal`
|
||||
and prevents a live non-terminal run from continuing without its lock.
|
||||
4. A wall deadline, lease/heartbeat, watchdog, and startup reconciler converge
|
||||
every non-terminal state, including human approval waits.
|
||||
5. One active session identity is enforced in PostgreSQL and cursor advancement
|
||||
is serialized with the run transition that produced it.
|
||||
6. Feishu delivery uses a durable, idempotent outbox/reconciliation path and
|
||||
exposes delivered, pending, and failed delivery states separately from model
|
||||
completion.
|
||||
7. The decided failed-run workspace policy is implemented and tested across
|
||||
kill, cancel, timeout, and database failure; file/message history reflects
|
||||
the real effects or explicitly reports missing evidence.
|
||||
8. The audit/event model conforms to the pinned run relation, records pre-run
|
||||
security events in an explicitly modeled place, and makes write loss
|
||||
observable and recoverable according to the accepted policy.
|
||||
9. Graceful shutdown and two-process/startup-race tests prove that one process
|
||||
cannot clear another live process's locks or accept new work while draining.
|
||||
@@ -0,0 +1,262 @@
|
||||
# Tenant, authentication, and request security audit
|
||||
|
||||
## Verdict
|
||||
|
||||
The current Hub has a sound application-level foundation for ordinary org-admin
|
||||
CRUD isolation, but it is **not safe to deploy as a multi-tenant SaaS**. Several
|
||||
agent and asynchronous-action paths can cross the Organization/Project boundary,
|
||||
and the current process-global credential design directly contradicts the
|
||||
org-scoped encrypted-secret decision in ADR-0021.
|
||||
|
||||
The answer to the audit question is therefore **yes**: current filesystem,
|
||||
Feishu-context, deferred authorization, card-action, and credential paths can
|
||||
cross or bypass the intended boundary. These are implementation divergences,
|
||||
not unspecified product behavior.
|
||||
|
||||
## Contract baseline
|
||||
|
||||
The audit used these upstream requirements:
|
||||
|
||||
- `Spec.System.Organization`: PROJECT and TEAM each have one Organization;
|
||||
TEAM→PROJECT grants are same-org only.
|
||||
- `Spec.System.ProjectWorkspace`: folders are transparent; folder/project
|
||||
placement must remain inside one Organization.
|
||||
- `Spec.System.Permission`: agent trigger requires EDIT, normal cancel requires
|
||||
MANAGE, and force release is outside the project role lattice.
|
||||
- `Spec.System.Memory`: a run may read Feishu context only from its project's
|
||||
bound chat.
|
||||
- `Spec.System.AgentSurface`: every agent file operation must stay inside that
|
||||
run's project workspace.
|
||||
- ADR-0019: authorization is evaluated against principals resolved at request
|
||||
time so membership changes take effect immediately.
|
||||
- ADR-0021: Feishu and model-provider credentials are org-scoped secrets,
|
||||
encrypted at rest, and accessed through a resolver. Provider ownership may be
|
||||
Organization-managed BYOK or a distinct platform-managed connection for that
|
||||
Organization; neither mode permits a process-global shared key.
|
||||
|
||||
## Confirmed protections
|
||||
|
||||
The following paths are correctly fail-closed today:
|
||||
|
||||
- Every `/api/org/:orgSlug/*` route calls `requireOrgRole`; the guard reloads
|
||||
the user, active membership, allowed OWNER/ADMIN role, and ACTIVE org for each
|
||||
request.
|
||||
- Project/session/folder/team service functions compare the requested object's
|
||||
Organization before returning or mutating it. Cross-org lookups return not
|
||||
found instead of revealing the foreign object.
|
||||
- `PermissionAuthorizer` derives the Organization from the resource before
|
||||
expanding team and external principals. `grantTeamProjectAccess` refuses a
|
||||
TEAM whose Organization differs from the Project and detects corrupt active
|
||||
grants when listing.
|
||||
- Feishu messages with no sender `open_id` are denied. Bound-chat triggers run
|
||||
through `agent.trigger` and `role.trigger` gates before creating an AgentRun.
|
||||
- OAuth state is signed, expiry-checked, and paired with an HttpOnly nonce
|
||||
cookie. `returnTo` is constrained to same-origin `/admin` paths. Session
|
||||
signatures use HMAC-SHA256 with constant-time comparison, and session users
|
||||
are reloaded on every request.
|
||||
- Fastify's current default body limit is 1 MiB, so HTTP bodies are not
|
||||
completely unbounded even though field-level and rate limits are absent.
|
||||
|
||||
Executed evidence:
|
||||
|
||||
```text
|
||||
5 integration files passed
|
||||
28 tests passed
|
||||
```
|
||||
|
||||
Those tests include unauthenticated access, member-vs-admin roles, cross-org
|
||||
project IDs, Organization-scoped team slugs, and cross-org TEAM grant refusal.
|
||||
|
||||
## Critical findings
|
||||
|
||||
### 1. The agent receives service secrets and can read other tenant data
|
||||
|
||||
`hub/src/agent/runner.ts` builds the SDK environment as:
|
||||
|
||||
```text
|
||||
{ ...process.env, ...providerEnv }
|
||||
```
|
||||
|
||||
The sandboxed agent and its Bash subprocess therefore inherit `DATABASE_URL`,
|
||||
`FEISHU_APP_SECRET`, `HUB_SESSION_SECRET`, the model-provider token, and every
|
||||
other service variable. A model or prompt injection can run `env`; network
|
||||
egress is intentionally open, so exposed values can leave the host.
|
||||
|
||||
The same sandbox config has only `allowWrite: [workspaceDir]`. Reads outside
|
||||
the workspace are allowed except for a small deny list. All organizations run
|
||||
as the same service user, and sibling project workspaces are not denied. This
|
||||
violates `AgentFileOp.Authorized`, which covers reads as well as writes.
|
||||
|
||||
The installed Claude Agent SDK already exposes two mechanisms that are not
|
||||
used: sandbox credential rules can `deny` or proxy-`mask` environment variables,
|
||||
and filesystem rules can deny a workspace root while re-allowing the current
|
||||
workspace. Production still needs an actual Linux test proving that SDK/Bash
|
||||
cannot read another project or service credential.
|
||||
|
||||
### 2. MCP file and Feishu-context tools escape their run objects
|
||||
|
||||
`resolveDeliverableFile` intentionally adds the repository Git root to allowed
|
||||
roots, even when it is above the project workspace. It also checks lexical paths
|
||||
with `stat`, which follows symlinks. A deterministic reproduction created
|
||||
`workspace/leak.txt` as a symlink to another tenant's file:
|
||||
|
||||
```json
|
||||
{"symlinkEscapeAccepted":true,"outsideContentReadable":true}
|
||||
```
|
||||
|
||||
The checked-in unit test currently treats sending a repo-root `README.md` from
|
||||
a nested project as desired behavior. That expectation is a direct divergence
|
||||
from the AgentSurface contract and must be reversed.
|
||||
|
||||
`readFeishuContext` ignores its `ToolContext`. The MCP wrapper closes over the
|
||||
bound `chatId`, but the model supplies an arbitrary message/thread ID and the
|
||||
reader never verifies the returned message's `chat_id`. A stub Feishu API
|
||||
returning a message from `chat-other` produced:
|
||||
|
||||
```json
|
||||
{"returnedChatId":"chat-other","crossChatContentReturned":true}
|
||||
```
|
||||
|
||||
The same validation is required for thread parents and every returned item.
|
||||
|
||||
Inbound downloads have the inverse symlink problem: the unsandboxed Hub writes
|
||||
to `workspace/.cph/inbox`. A prior agent command can replace a path component
|
||||
with a symlink, turning the Hub into a confused deputy unless writes use a
|
||||
verified real path/no-follow boundary.
|
||||
|
||||
### 3. SaaS credentials are process-global instead of org-scoped
|
||||
|
||||
`server.ts` creates one process-global Feishu client/listener and passes one
|
||||
global app ID/secret to OAuth. `feishuOAuth.ts` explicitly calls per-org
|
||||
encrypted credentials "deferred", although ADR-0021 says they **must** be
|
||||
org-scoped, encrypted at rest, and accessed through a resolver.
|
||||
|
||||
`EnvRuntimeSettings.provider` similarly discards its `scope` argument and uses
|
||||
one global OpenRouter token for every project. This erases the accepted
|
||||
per-Organization provider mode and makes both BYOK and platform-managed cost
|
||||
attribution false. The schema has no encrypted credential store, key version,
|
||||
rotation state, or resolver seam.
|
||||
|
||||
This is also a product reachability blocker: one customer-owned Feishu app
|
||||
cannot receive bot/OAuth traffic for unrelated customer tenants under the
|
||||
ADR-0021 model.
|
||||
|
||||
## High findings
|
||||
|
||||
### 4. Delayed and card actions are not bound to the authorized object
|
||||
|
||||
- `onMessage` checks `agent.trigger` before enqueueing/batching, but
|
||||
`startAgentRun` checks only `role.trigger`. If a grant, team membership, or
|
||||
tenant status changes while a request waits, it still starts. This contradicts
|
||||
ADR-0019's request-time effectiveness rule.
|
||||
- Interrupt handling authorizes `agent.cancel` against the project bound to the
|
||||
card action's current chat, then calls `activeRuns.get(runId)` without proving
|
||||
that `runId` belongs to that project. A stale, forwarded, or otherwise
|
||||
mismatched action can abort another project's active controller.
|
||||
- Approval handling stores the expected `chatId` but never compares it to the
|
||||
action context before resolving the pending approval.
|
||||
|
||||
Execution-time authorization must resolve one immutable tuple
|
||||
`(organization, project, run/action, actor, chat)` and reject any mismatch
|
||||
before mutation.
|
||||
|
||||
### 5. SUSPENDED organizations can still execute bound projects
|
||||
|
||||
HTTP org-admin guards and unbound-chat onboarding reject non-ACTIVE orgs, but
|
||||
`PermissionAuthorizer.organizationForResource` returns only `organizationId`
|
||||
and never checks status. A real database reproduction created a SUSPENDED org,
|
||||
project, and EDIT grant:
|
||||
|
||||
```json
|
||||
{"organizationStatus":"SUSPENDED","agentTriggerAllowed":true}
|
||||
```
|
||||
|
||||
Project onboarding helpers also check membership existence without checking
|
||||
Organization status. The same status must be enforced at the shared resource
|
||||
authorization/onboarding seam, not independently in selected transports.
|
||||
|
||||
### 6. Production dependency graphs contain known vulnerabilities
|
||||
|
||||
`npm audit --omit=dev` exits 1 with two high findings:
|
||||
|
||||
- `@larksuiteoapi/node-sdk@1.68.0`
|
||||
- transitive `axios@1.13.6`
|
||||
|
||||
The current latest Feishu SDK (`1.70.0` at audit time) still pins Axios
|
||||
`~1.13.3`, while the audited fixed Axios line is newer. The relevant advisories
|
||||
include request/credential manipulation, proxy credential leakage, ReDoS, and
|
||||
unbounded resource allocation; for example
|
||||
[GHSA-3g43-6gmg-66jw](https://github.com/advisories/GHSA-3g43-6gmg-66jw),
|
||||
[GHSA-hfxv-24rg-xrqf](https://github.com/advisories/GHSA-hfxv-24rg-xrqf), and
|
||||
[GHSA-777c-7fjr-54vf](https://github.com/advisories/GHSA-777c-7fjr-54vf).
|
||||
An override is outside the SDK's declared semver range and therefore requires
|
||||
integration testing rather than a blind lockfile edit.
|
||||
|
||||
`cargo audit` exits 1 with five vulnerability records:
|
||||
|
||||
- `crossbeam-epoch@0.9.18`, patched in `>=0.9.20`;
|
||||
- `quick-xml@0.38.4` and `0.39.4`, each affected by namespace-allocation and
|
||||
quadratic-attribute DoS advisories, patched in `>=0.41.0`.
|
||||
|
||||
It also reports `memmap2@0.9.10` as unsound (patched in `>=0.9.11`) and four
|
||||
unmaintained transitive crates. All enter through the Typst 0.15 dependency
|
||||
graph. References:
|
||||
[RUSTSEC-2026-0204](https://rustsec.org/advisories/RUSTSEC-2026-0204.html),
|
||||
[RUSTSEC-2026-0194](https://rustsec.org/advisories/RUSTSEC-2026-0194.html),
|
||||
[RUSTSEC-2026-0195](https://rustsec.org/advisories/RUSTSEC-2026-0195.html), and
|
||||
[RUSTSEC-2026-0186](https://rustsec.org/advisories/RUSTSEC-2026-0186.html).
|
||||
|
||||
No `npm audit` or `cargo audit` gate exists in CI.
|
||||
|
||||
## Browser/session and availability gaps
|
||||
|
||||
These do not currently prove a cross-org data read, but they are required before
|
||||
exposing the service publicly:
|
||||
|
||||
- state-changing cookie-auth routes have no CSRF token or Origin/Sec-Fetch-Site
|
||||
enforcement; SameSite=Lax is the only browser-side barrier;
|
||||
- there is no session revocation/version store, so a stolen cookie remains valid
|
||||
for up to seven days unless the global secret rotates;
|
||||
- startup accepts any non-empty session secret rather than enforcing generated
|
||||
entropy, and there is no key rotation scheme;
|
||||
- no security-header policy is registered, and Fastify request timeout is zero;
|
||||
- OAuth, login, and org APIs have no per-IP/user/tenant rate limit;
|
||||
- Feishu file/post download count, response size, MIME, archive shape, and total
|
||||
workspace quota are unbounded; uploads read entire files into memory.
|
||||
|
||||
The exact abuse quotas and file limits remain with
|
||||
`Define initial abuse and capacity controls`; they must not be guessed here.
|
||||
|
||||
## Secret scan
|
||||
|
||||
All 377 tracked files were scanned for private-key blocks, common cloud/API-key
|
||||
shapes, and non-empty assignments to the Hub's principal secret variables. No
|
||||
private key or real credential was found. The only assignment hits were the
|
||||
documented `<OpenRouter API key>` placeholder and local/test PostgreSQL URLs.
|
||||
`hub/.env` is ignored and untracked.
|
||||
|
||||
This is positive repository hygiene, but it does not mitigate the runtime
|
||||
environment leak into the agent process.
|
||||
|
||||
## Required security release evidence
|
||||
|
||||
Production readiness requires automated proof of all of the following:
|
||||
|
||||
1. An agent/Bash process cannot read service credentials, another project
|
||||
workspace, repo-root files, or a symlink target outside its workspace.
|
||||
2. Feishu context and file delivery validate the real target after resolution,
|
||||
not only a caller-supplied lexical ID/path.
|
||||
3. Every delayed action reauthorizes at execution and binds actor, chat,
|
||||
organization, project, run, and pending action consistently.
|
||||
4. ACTIVE Organization status is enforced in the shared authorization seam.
|
||||
5. Feishu and provider credentials resolve by Organization, are encrypted at
|
||||
rest with versioned keys, never enter logs/agent tools, and can rotate;
|
||||
provider resolution honors either Organization-managed BYOK or a distinct
|
||||
platform-managed connection for that Organization.
|
||||
6. Cross-org HTTP and Prisma service tests remain green, while new negative
|
||||
tests cover every finding above.
|
||||
7. Production Node and Rust dependency audits pass with no ignored
|
||||
vulnerability/unsound advisory unless a time-bounded, reviewed exception is
|
||||
explicitly recorded.
|
||||
8. Browser/session, request, and file-ingestion limits are verified against the
|
||||
accepted production threat and capacity policy.
|
||||
@@ -0,0 +1,33 @@
|
||||
# Audit the clean-host deployment and rollback contract
|
||||
|
||||
Type: research
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
What concrete defects prevent a clean, supported Linux host from installing,
|
||||
starting, health-checking, upgrading, and rolling back the current Hub without
|
||||
manual repository knowledge or unsafe partial deployment?
|
||||
|
||||
## Answer
|
||||
|
||||
The current path is a happy-path updater for a manually prepared host, not a
|
||||
production deployment mechanism. The executed audit found an unrunnable seeded
|
||||
configuration, an unprovisioned service identity and storage, a missing `cph`
|
||||
artifact, in-place mutation before build success, liveness-only verification,
|
||||
no coordinated application/schema rollback, and a configured bind host that the
|
||||
server ignores.
|
||||
|
||||
The complete evidence, operational consequences, required release invariants,
|
||||
and verification scenarios are recorded in the
|
||||
[clean-host deployment and rollback audit](../assets/clean-host-deployment-audit.md).
|
||||
|
||||
The accepted single-host topology remains suitable. The route forward is now
|
||||
explicit in these implementation tickets:
|
||||
|
||||
- [Provision a runnable non-root Hub service](10-provision-nonroot-service.md)
|
||||
- [Package and verify cph with every Hub release](11-package-cph-with-release.md)
|
||||
- [Make application releases atomic and rollbackable](12-atomic-releases-and-rollback.md)
|
||||
- [Add real readiness and an accurate service lifecycle](13-real-readiness-and-lifecycle.md)
|
||||
- [Establish migration compatibility and failed-migration recovery](14-migration-compatibility-and-recovery.md)
|
||||
- [Test clean-host deployment and rollback on Linux](15-test-clean-host-deployments-on-linux.md)
|
||||
@@ -0,0 +1,40 @@
|
||||
# Audit tenant, authentication, and request security boundaries
|
||||
|
||||
Type: research
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Can any current HTTP, Feishu, filesystem, agent, or database path cross an
|
||||
Organization boundary, bypass the intended platform/org/project authorization
|
||||
seams, expose secrets, or accept unbounded hostile input in the initial
|
||||
production topology?
|
||||
|
||||
## Answer
|
||||
|
||||
Yes. Basic org-admin HTTP/Prisma scoping is correctly enforced and its 28
|
||||
targeted integration tests pass, but the agent inherits service secrets and can
|
||||
read sibling files, MCP file/context tools accept real targets outside the run
|
||||
objects, queued/card actions do not consistently bind or reauthorize their
|
||||
target, non-ACTIVE tenants can still trigger agents, and Feishu/provider
|
||||
credentials remain global instead of ADR-0021 org-scoped encrypted secrets.
|
||||
Current Node and Rust dependency audits also fail on high/vulnerability/unsound
|
||||
advisories.
|
||||
|
||||
The complete contract mapping, reproductions, positive evidence, advisory
|
||||
inventory, and required release proofs are in the
|
||||
[tenant, authentication, and request security audit](../assets/tenant-auth-security-audit.md).
|
||||
|
||||
The newly-clear implementation and decision frontier is:
|
||||
|
||||
- [Confine the agent runtime and protect service credentials](16-confine-agent-and-protect-credentials.md)
|
||||
- [Close MCP context and file-delivery escape paths](17-close-mcp-data-egress-escapes.md)
|
||||
- [Bind and reauthorize deferred and card actions](18-bind-and-reauthorize-deferred-actions.md)
|
||||
- [Design the org-scoped secret and connection control plane](19-design-org-scoped-secret-control-plane.md)
|
||||
- [Design the production browser and session boundary](20-design-production-web-session-boundary.md)
|
||||
- [Clear dependency advisories and gate releases](21-clear-dependency-security-advisories.md)
|
||||
- [Enforce Organization status at the shared authorization seam](22-enforce-organization-status.md)
|
||||
|
||||
Inbound Feishu file count/size/MIME and request-rate policy remain owned by
|
||||
[Define initial abuse and capacity controls](05-define-abuse-capacity-controls.md)
|
||||
rather than being guessed in this audit.
|
||||
@@ -0,0 +1,38 @@
|
||||
# Audit run lifecycle and data integrity under failure
|
||||
|
||||
Type: research
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Under process termination, restart, timeout, duplicate Feishu delivery,
|
||||
database failure, or concurrent requests, which AgentRun, lock, session,
|
||||
workspace, audit, and outbound-message invariants can be violated or leave work
|
||||
lost, duplicated, or permanently stuck?
|
||||
|
||||
## Answer
|
||||
|
||||
The happy path and database lock uniqueness work, but the lifecycle is not
|
||||
crash-safe. Deterministic fault injection confirmed permanent inbound loss,
|
||||
terminal runs retaining locks, active runs losing locks, duplicate active
|
||||
sessions, completion without outbound delivery, unconstrained audit run IDs,
|
||||
and max-turn termination being stored as failure. Process-local queues,
|
||||
non-transactional startup recovery, unused leases, silent audit/history writes,
|
||||
and direct workspace mutation add restart and indefinite-hang failure modes.
|
||||
|
||||
The complete contract mapping, eight fault probes, recovery analysis, and
|
||||
required release evidence are in the
|
||||
[run lifecycle and data-integrity audit](../assets/run-lifecycle-integrity-audit.md).
|
||||
|
||||
The newly-clear implementation and decision frontier is:
|
||||
|
||||
- [Make Feishu intake and queued work durable](23-durable-feishu-inbox-and-queue.md)
|
||||
- [Make run, session, lock, and recovery transitions safe](24-transactional-run-lock-recovery.md)
|
||||
- [Make outbound Feishu delivery durable and idempotent](25-durable-outbound-delivery.md)
|
||||
- [Decide workspace commit semantics for failed runs](26-decide-workspace-run-commit-semantics.md)
|
||||
- [Reconcile the audit and run-history contract](27-reconcile-audit-history-contract.md)
|
||||
|
||||
Execution-time authorization remains owned by
|
||||
[Bind and reauthorize deferred and card actions](18-bind-and-reauthorize-deferred-actions.md),
|
||||
while cross-component metrics, alerts, and operator runbooks remain owned by
|
||||
[Define production observability and incident recovery](04-define-observability-recovery.md).
|
||||
@@ -0,0 +1,47 @@
|
||||
# Define production observability and incident recovery
|
||||
|
||||
Type: research
|
||||
Status: resolved
|
||||
Blocked by: 01, 02, 03
|
||||
|
||||
## Question
|
||||
|
||||
Which logs, metrics, readiness signals, audit correlations, alerts, and operator
|
||||
recovery actions are missing for a production operator to detect and diagnose
|
||||
failures across HTTP, Feishu, PostgreSQL, agent execution, cph builds, and
|
||||
workspace storage?
|
||||
|
||||
## Answer
|
||||
|
||||
The Hub has useful fragments—Fastify access logs, Prisma error/warn output,
|
||||
durable AgentRun facts, selected Feishu warnings, explicit migrations, and a
|
||||
systemd journal—but no joined operational contract. It has no metrics or alert
|
||||
surface, no end-to-end request/event/run/delivery/release correlation, no
|
||||
checked-in recovery runbooks, and many critical writes or sends are silently
|
||||
best-effort.
|
||||
|
||||
Three process-level fault probes confirmed the release gate is actively
|
||||
misleading: cutting every PostgreSQL socket left `/api/healthz` at 200 twice;
|
||||
an SDK-rejected Feishu app ID also left health at 200; and a runtime database
|
||||
outage made `/api/me` return a raw Prisma invocation as `400 bad_request` with
|
||||
no request-scoped exception log. The current deploy check can therefore accept
|
||||
an instance unable to serve either database or Feishu work, while dependency
|
||||
failures evade normal 5xx alerting.
|
||||
|
||||
The full failure-to-signal matrix, liveness/readiness/startup contract,
|
||||
correlation envelope, metric and alert inventory, required runbooks, executed
|
||||
probe evidence, primary sources, and minimum release evidence are in the
|
||||
[production observability and incident-recovery audit](../assets/production-observability-recovery.md).
|
||||
|
||||
The newly-clear implementation and decision frontier is:
|
||||
|
||||
- [Make HTTP dependency failures typed and observable](28-typed-observable-http-errors.md)
|
||||
- [Decide the telemetry, SLO, retention, and alerting control plane](29-decide-telemetry-slos-alert-routing.md)
|
||||
- [Instrument correlated production runtime telemetry](30-instrument-correlated-runtime-telemetry.md)
|
||||
- [Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md)
|
||||
|
||||
[Add real readiness and an accurate service lifecycle](13-real-readiness-and-lifecycle.md)
|
||||
owns component probes, Feishu connection state and deploy gating. Durable
|
||||
ingress/run/outbound recovery stays with tickets 23–25, audit semantics with
|
||||
ticket 27, and backup/restore guarantees with ticket 06; this audit does not
|
||||
duplicate those implementations.
|
||||
@@ -0,0 +1,43 @@
|
||||
# Define initial abuse and capacity controls
|
||||
|
||||
Type: grilling
|
||||
Status: resolved
|
||||
Blocked by: 02, 03
|
||||
|
||||
## Question
|
||||
|
||||
Which request limits, file limits, concurrency limits, agent budgets, tenant
|
||||
quotas, and backpressure policies are mandatory for the accepted initial
|
||||
production boundary, and which values or behaviors remain product decisions?
|
||||
|
||||
## Answer
|
||||
|
||||
The initial production service requires layered, fail-closed capacity controls:
|
||||
every mandatory dimension has a versioned non-overridable platform ceiling,
|
||||
while an Organization may configure only a lower policy limit. Agent work uses
|
||||
a durable bounded queue, fair scheduling across Organizations, FIFO within an
|
||||
Organization, explicit overload/expiry/cancellation states, and no automatic
|
||||
whole-run replay after execution starts.
|
||||
|
||||
Hard controls cover request rate/body size, concurrency and queueing, files and
|
||||
archive expansion, Project/Organization storage and entity counts, run/tool
|
||||
budgets, and process resources. Reaching a limit blocks new growth without
|
||||
deleting or hiding existing data. Token and provider cost remain attributed
|
||||
soft-alert signals in the pilot, not hard spend controls; unknown cost is not
|
||||
zero. Platform administrators also receive audited `DRAIN` and `STOP_NOW`
|
||||
workload brakes.
|
||||
|
||||
The full current-state inventory, accepted behavior, and release evidence are
|
||||
recorded in [Initial abuse and capacity controls](../assets/initial-abuse-capacity-controls.md),
|
||||
with the durable decision in ADR-0022 and `Spec.System.Capacity`. Numerical
|
||||
ceilings remain open until production-like calibration.
|
||||
|
||||
The implementation frontier is:
|
||||
|
||||
- [Implement layered limits and multi-dimensional request rate limiting](32-layered-limits-rate-limiting.md)
|
||||
- [Build the durable Organization-fair admission scheduler](33-durable-fair-admission-scheduler.md)
|
||||
- [Enforce file, archive, storage, and entity budgets](34-file-storage-budgets.md)
|
||||
- [Enforce Agent Run and process resource budgets](35-run-runtime-budgets.md)
|
||||
- [Implement provider-mode usage attribution and soft alerts](36-usage-soft-alerts.md)
|
||||
- [Add audited emergency workload brakes](37-emergency-workload-brakes.md)
|
||||
- [Calibrate and prove production capacity ceilings](38-calibrate-production-capacity.md)
|
||||
@@ -0,0 +1,48 @@
|
||||
# Audit backup, migration, and disaster recovery safety
|
||||
|
||||
Type: research
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
What backup, restore, workspace/database consistency, migration compatibility,
|
||||
and disaster-recovery guarantees are needed to deploy current persistent data
|
||||
safely, and what evidence or tooling is missing to prove them?
|
||||
|
||||
## Answer
|
||||
|
||||
The Hub has no production backup/restore control plane. PostgreSQL can be
|
||||
logically dumped and restored, but authoritative state is split across the
|
||||
database, mutable Project workspaces, credentials/key material, and a
|
||||
schema-compatible release with no shared checkpoint or recovery manifest.
|
||||
RPO/RTO, retention, off-host/immutable storage, encryption/key custody,
|
||||
full-service versus Organization-selective restore, monitoring, and recovery
|
||||
drills are all undefined.
|
||||
|
||||
Executed probes proved that the current deploy `rsync --delete` removes the
|
||||
documented in-tree workspace root, a failed Project transaction leaves an
|
||||
orphan workspace, and an older application query fails after a destructive
|
||||
migration. A deliberately failed Prisma migration rolled back that probe's DDL
|
||||
but left a blocking failed migration record; the official contract does not
|
||||
make PostgreSQL migrations transactional by default. A positive custom-format
|
||||
round trip restored 24 tables and all nine current migration records, proving
|
||||
only database-level portability on the local PostgreSQL 14.20 setup.
|
||||
|
||||
The complete state map, probe commands/results, root causes, required recovery
|
||||
contract, and release evidence are in the
|
||||
[backup, migration, and disaster-recovery audit](../assets/backup-migration-disaster-recovery-audit.md).
|
||||
Official PostgreSQL, Prisma, systemd, and rsync semantics with claim-level links
|
||||
are in [backup and migration primary sources](../assets/backup-migration-primary-sources.md).
|
||||
|
||||
The newly-clear decision and implementation frontier is:
|
||||
|
||||
- [Decide recovery objectives, backup retention, and restore scope](39-decide-recovery-objectives.md)
|
||||
- [Implement coherent backup sets and verified restore](40-verified-backup-restore.md)
|
||||
- [Make the Project workspace lifecycle recoverable](41-recoverable-project-workspace-lifecycle.md)
|
||||
- [Establish migration compatibility and failed-migration recovery](14-migration-compatibility-and-recovery.md)
|
||||
|
||||
Persistent-path provisioning and release-tree isolation remain owned by
|
||||
[Provision a runnable non-root Hub service](10-provision-nonroot-service.md)
|
||||
and [Make application releases atomic and rollbackable](12-atomic-releases-and-rollback.md).
|
||||
Full recovery/cutover procedures and drill evidence remain owned by
|
||||
[Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md).
|
||||
@@ -0,0 +1,58 @@
|
||||
# Audit the accepted product surface for production completeness
|
||||
|
||||
Type: research
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Against ADR-0020 and ADR-0021, which required platform-admin, org-admin,
|
||||
project-onboarding, team/access, session, usage, provider-mode, Organization
|
||||
policy-limit, emergency workload-control, and Feishu flows are absent,
|
||||
unreachable, or only test-facing rather than usable in the initial production
|
||||
service? Treat ADR-0022's capacity control surfaces as part of the accepted
|
||||
product boundary.
|
||||
|
||||
## Answer
|
||||
|
||||
The current Hub is not a usable or production-complete SaaS product surface.
|
||||
Its Organization-scoped JSON APIs and single-Feishu-app Agent Run path are
|
||||
worth retaining, but the running service has one process-global Feishu
|
||||
app/bot, one global provider key/base URL, no Organization connection/secret
|
||||
plane, no platform-administrator control plane, and no real Organization or
|
||||
platform management UI. OAuth redirects a successful login to an unmounted
|
||||
`/admin/org/:orgSlug` route.
|
||||
|
||||
Two real Fastify integration probes, each run twice, proved that an authenticated
|
||||
Organization ADMIN receives 404 at the post-login destination and two distinct
|
||||
Organization return paths produce the same OAuth `client_id`. Official Feishu
|
||||
documentation also proves that `open_id` is application-scoped, `user_id` is
|
||||
tenant-scoped, and `union_id` only links applications from the same application
|
||||
provider. The current globally unique `User.feishuOpenId`, raw `chatId`, event
|
||||
identity, OAuth state and one-listener composition therefore cannot safely
|
||||
route independently customer-owned applications.
|
||||
|
||||
ADR-0022 remains an additional release blocker: beyond a process-memory
|
||||
Project queue and global max-turn setting, layered limits, durable
|
||||
Organization-fair admission, hard storage/run/process budgets, alerts and
|
||||
audited emergency brakes have no complete persistence or product surface.
|
||||
|
||||
The full verdict, deterministic probe results, root causes, retained seams and
|
||||
required end-to-end release evidence are in the
|
||||
[accepted SaaS product-surface completeness audit](../assets/product-surface-completeness-audit.md).
|
||||
The detailed route/schema/test matrix is in the
|
||||
[accepted product-surface code inventory](../assets/product-surface-code-inventory.md),
|
||||
and the Feishu identifier/OAuth/event/long-connection facts are in the
|
||||
[Feishu product-surface primary sources](../assets/product-surface-feishu-primary-sources.md).
|
||||
|
||||
The newly-clear execution frontier is:
|
||||
|
||||
- [Decide the platform-administrator identity and audit boundary](42-decide-platform-admin-identity-audit.md)
|
||||
- [Implement the Organization connection and secret plane](43-implement-org-connection-secret-plane.md)
|
||||
- [Route Feishu identity and traffic by Organization connection](44-route-feishu-by-org-connection.md)
|
||||
- [Build the private platform-administrator control plane](45-build-private-platform-admin-control-plane.md)
|
||||
- [Build the private Organization-administrator control plane](46-build-private-org-admin-control-plane.md)
|
||||
|
||||
Existing tickets 23 and 32–38 remain the owners of durable ingress/admission,
|
||||
capacity enforcement, usage alerts, workload brakes and calibration. Ticket 08
|
||||
now requires all of these product slices before claiming the production release
|
||||
gate.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Prove the production release gate
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 01, 02, 03, 04, 05, 06, 07, 09, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46
|
||||
|
||||
## Question
|
||||
|
||||
After the readiness investigations and resulting fixes are resolved, can one
|
||||
repeatable release procedure prove build/test/spec health, deploy a clean
|
||||
production-like environment, exercise critical tenant and agent journeys,
|
||||
verify observability and recovery, and either roll forward or roll back safely?
|
||||
@@ -0,0 +1,40 @@
|
||||
# Restore the checker formatting gate
|
||||
|
||||
Type: task
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Apply the repository's configured rustfmt output to the currently reported
|
||||
Rust files, verify that no semantic changes are introduced, and restore
|
||||
`cargo fmt --all --check` so the checker CI gate is green.
|
||||
|
||||
## Answer
|
||||
|
||||
The exact CI command failed deterministically on seven Rust files and an
|
||||
isolated single-file rustfmt check reproduced the same output. The source
|
||||
changes were formatting-only, but the underlying gate was not reproducible:
|
||||
the repository had no `rust-toolchain.toml` and checker CI installed floating
|
||||
`stable`, so local rustfmt, clippy and CI could change independently.
|
||||
|
||||
The checker toolchain is now pinned to Rust 1.92.0 with `rustfmt` and `clippy`
|
||||
in both `rust-toolchain.toml` and the Gitea workflow. The seven files were
|
||||
formatted with that toolchain. Running the complete CI path then surfaced three
|
||||
test helpers accepting `&PathBuf` instead of `&Path` and a Clippy name collision
|
||||
between `Shell` and `PowerShell`; those were corrected without changing CLI
|
||||
values or behavior.
|
||||
|
||||
Verification:
|
||||
|
||||
```text
|
||||
cargo fmt --all --check PASS (twice)
|
||||
cargo clippy --all-targets --all-features -- -D warnings
|
||||
PASS
|
||||
cargo test --all-features --workspace PASS (53 tests)
|
||||
cph completions zsh / power-shell PASS
|
||||
```
|
||||
|
||||
The final Rust diff consists of rustfmt layout plus four borrowed-Path
|
||||
signature generalizations and the internal `Shell` → `CompletionTarget` type
|
||||
rename. No expression, string value, command behavior, or public Rust API was
|
||||
changed.
|
||||
@@ -0,0 +1,40 @@
|
||||
# Provision a runnable non-root Hub service
|
||||
|
||||
Type: task
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Make clean-host installation create or validate the dedicated service identity,
|
||||
home/cache/state/workspace paths and ownership, generate a complete production
|
||||
configuration, honor the configured bind host, and fail preflight before unit
|
||||
installation when any required value or prerequisite is invalid. The persistent
|
||||
workspace root must be outside all release trees, and preflight must reject any
|
||||
path that overlaps rsync, release switching, rollback, or release pruning.
|
||||
|
||||
## Answer
|
||||
|
||||
The installer now creates or strictly validates a dedicated non-root,
|
||||
non-login service identity with no supplementary groups; provisions its
|
||||
home/state/cache/workspace directories at mode `0750`; and refuses persistent
|
||||
paths that overlap the deployment tree after both lexical and physical path
|
||||
resolution. A missing production environment file is seeded completely and
|
||||
exits with status 78; invalid configuration or prerequisites fail before the
|
||||
unit is written or enabled.
|
||||
|
||||
Preflight validates the configured bind host/port, authenticated PostgreSQL
|
||||
access, Node/cph/Prisma, configured and canonical path traversal, and a real
|
||||
bubblewrap namespace. The post-provision phase runs the built Hub dependency
|
||||
graph, a database query, cph, socat and bubblewrap as the service user under
|
||||
`no_new_privs`, matching the systemd security context. HTTP binding is awaited
|
||||
before the Feishu listener starts, so bind failure rejects startup instead of
|
||||
leaving a listener-only process alive.
|
||||
|
||||
Verification evidence:
|
||||
|
||||
- `npm run check`, `npm run build`, and `npm run prisma:validate`
|
||||
- 166 unit tests and 86 integration tests passed; one external-model test skipped
|
||||
- `bash -n deploy/install_service.sh` and `shellcheck deploy/install_service.sh`
|
||||
- `prisma migrate deploy`: 9 migrations found, none pending
|
||||
- production-mode HTTP smoke test on `127.0.0.1:8878` returned `/api/healthz`
|
||||
- final Standards and Spec reviews found no remaining Issue 10 gap
|
||||
@@ -0,0 +1,10 @@
|
||||
# Package and verify cph with every Hub release
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
Deliver the compatible `cph` executable as part of the same immutable release
|
||||
as the Hub, configure its exact path, and make deployment preflight reject a
|
||||
missing, non-executable, or version-incompatible binary before restart.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Make application releases atomic and rollbackable
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 10, 11
|
||||
|
||||
## Question
|
||||
|
||||
Replace in-place rsync/build with deployment locking, immutable staged release
|
||||
directories, a revision manifest, atomic current/previous switching, bounded
|
||||
retention, and an explicit application rollback that preserves the live release
|
||||
on build, preflight, switch, or readiness failure. Prove deploy, rollback, and
|
||||
release pruning cannot traverse or delete the persistent workspace root.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Add real readiness and an accurate service lifecycle
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 03, 04, 10, 11, 12, 23, 24, 25
|
||||
|
||||
## Question
|
||||
|
||||
Separate liveness from readiness, verify every dependency needed to accept Hub
|
||||
work, make deployment wait on readiness, and align systemd shutdown/restart
|
||||
claims with an implemented and observable Fastify, Feishu, Prisma, and in-flight
|
||||
run lifecycle.
|
||||
@@ -0,0 +1,15 @@
|
||||
# Establish migration compatibility and failed-migration recovery
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 06
|
||||
|
||||
## Question
|
||||
|
||||
Define and enforce the application/schema compatibility rule for each release,
|
||||
require expand-and-contract sequencing for destructive changes, add migration
|
||||
status/checksum and production-like rehearsal gates, and move schema mutation
|
||||
out of ordinary service start into an explicit release phase. Require a recent
|
||||
verified recovery checkpoint, bounded lock/runtime evidence, and an operator
|
||||
procedure for resolving or restoring a failed Prisma migration without
|
||||
pretending that an application rollback reverses the database.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Test clean-host deployment and rollback on Linux
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 10, 11, 12, 13, 14
|
||||
|
||||
## Question
|
||||
|
||||
Add an automated Linux production-like harness that executes the real install,
|
||||
deploy and rollback paths against PostgreSQL and proves first install,
|
||||
idempotence, failure atomicity, readiness failures, application rollback, and
|
||||
deployment serialization.
|
||||
@@ -0,0 +1,38 @@
|
||||
# Confine the agent runtime and protect service credentials
|
||||
|
||||
Type: task
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Make the real Claude SDK/Bash execution surface uphold `AgentFileOp.Authorized`:
|
||||
deny sibling tenant workspaces and service files, expose only the minimum safe
|
||||
process environment, use SDK credential protection for secrets, and add an
|
||||
actual Linux sandbox test proving another workspace and every non-provider
|
||||
credential are unreadable while required tools still work.
|
||||
|
||||
## Answer
|
||||
|
||||
The Agent subprocess now receives a replacement environment containing only
|
||||
named runtime values and the selected provider protocol values; PostgreSQL,
|
||||
Feishu and Hub session credentials never cross the process boundary. Provider
|
||||
credentials are additionally denied inside Bash. The SDK must sandbox every
|
||||
command (`failIfUnavailable=true`, `allowUnsandboxedCommands=false`), denies
|
||||
host reads and writes from root, and re-opens only the canonical project
|
||||
workspace plus the reviewed read-only system runtime needed by `cph`.
|
||||
|
||||
SDK config/cache/home and general temp storage live inside the workspace. A
|
||||
short relative `CLAUDE_CODE_TMPDIR` prevents the pinned SDK from falling back
|
||||
to host `/tmp`, while absolute workspace-local `TMPDIR`/`TMP`/`TEMP` remain
|
||||
stable after Bash changes directory. Linux production and CI now require both
|
||||
bubblewrap and `socat`; preflight probes both as the service user. Bounded,
|
||||
credential-redacted SDK stderr is logged with run and project correlation.
|
||||
|
||||
The real Linux proof runs the actual Claude SDK, Bash, bubblewrap, `socat` and
|
||||
release `cph` as UID 1000 with no capabilities and `no_new_privs`. It proves
|
||||
parent, sibling-project, `/tmp` and `/var/tmp` writes are rejected and have no
|
||||
host effect; sibling/service-secret reads fail; platform and provider secrets
|
||||
are absent from Bash; workspace temp remains anchored after `cd`; and `cph`
|
||||
still executes. Final local gates passed: TypeScript check/build, Prisma schema,
|
||||
173 unit tests, 85 mock-provider integration tests, deployment shell syntax,
|
||||
and the dedicated Linux sandbox test.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Close MCP context and file-delivery escape paths
|
||||
|
||||
Type: task
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Restrict file delivery to a realpath/no-follow target inside the current project
|
||||
workspace, remove Git-root access, make inbound Hub writes symlink-safe, and
|
||||
validate every Feishu message/thread result against the run's bound chat before
|
||||
returning data to the model.
|
||||
|
||||
## Answer
|
||||
|
||||
All Agent-facing workspace file reads and writes are anchored to the configured
|
||||
workspace root and current project. Linux uses descriptor-relative, no-follow
|
||||
traversal so a symlink swap cannot escape after validation; unsupported unsafe
|
||||
paths fail closed. Inbound Feishu files use the same boundary. Outbound file
|
||||
delivery no longer accepts the repository Git root or arbitrary absolute
|
||||
paths: it reads a no-follow snapshot from the current project and uploads that
|
||||
snapshot, so the checked object cannot be swapped before delivery.
|
||||
|
||||
The Feishu MCP read surface carries the run's project and bound-chat context.
|
||||
Message and thread results are rejected unless their returned chat matches that
|
||||
binding, and file-delivery MCP calls remain scoped to the same run/project/chat
|
||||
tuple. Unit and integration regressions cover symlink paths, sibling projects,
|
||||
Git-root delivery attempts, swapped files, and cross-chat results; they are
|
||||
included in the final 173-unit/85-integration green gate reported by Issue 16.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Bind and reauthorize deferred and card actions
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 44
|
||||
|
||||
## Question
|
||||
|
||||
At execution time, reauthorize queued/batched triggers and validate one
|
||||
actor/chat/organization/project/run-or-pending-action tuple for interrupt,
|
||||
approval, and onboarding card actions so revoked permissions and mismatched
|
||||
object IDs cannot affect another project.
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
# Design the org-scoped secret and connection control plane
|
||||
|
||||
Type: grilling
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Which key-management, envelope-encryption, rotation, connection identity, and
|
||||
runtime resolver design will implement ADR-0021's org-scoped Feishu and model
|
||||
provider credentials; support one customer-owned Feishu app per Organization;
|
||||
support Organization-managed BYOK and a distinct platform-managed provider
|
||||
connection per Organization; and ensure plaintext credentials never reach
|
||||
business records, logs, or agent tools?
|
||||
|
||||
## Answer
|
||||
|
||||
Use the local master-key envelope, immutable connection-secret versions,
|
||||
writer-authority split, explicit Organization/Project resolver, staged KEK
|
||||
rotation, and separately protected keyring recovery contract accepted in
|
||||
ADR-0024. Production receives the root-owned keyring only through systemd
|
||||
`LoadCredential`; it has no process-global credential fallback. Agent runs use
|
||||
a loopback proxy capability rather than the Organization Provider credential;
|
||||
the offline rotation command atomically retains old/new KEKs, rewraps stored
|
||||
DEKs, audits the changes, and authenticates the complete envelope set.
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Design the production browser and session boundary
|
||||
|
||||
Type: grilling
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
What same-origin/CSRF, proxy/TLS, security-header, session lifetime,
|
||||
revocation, secret rotation, and request-timeout contract should the initial
|
||||
org-admin web surface enforce, and which parts require server-side session state
|
||||
rather than the current seven-day stateless cookie? Apply the same browser
|
||||
boundary to ADR-0023's separate server-side Platform Session while preserving
|
||||
its distinct cookie/token namespace, immediate revocation, per-request grant
|
||||
reload, and bounded recent-authentication requirement.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Clear dependency advisories and gate releases
|
||||
|
||||
Type: task
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Remove every current `npm audit --omit=dev` high finding and every `cargo audit`
|
||||
vulnerability/unsound finding through tested upgrades, compatible overrides or
|
||||
upstream fixes—not ignores—and add reproducible Node and Rust security-audit
|
||||
gates to CI and the production release evidence.
|
||||
|
||||
## Answer
|
||||
|
||||
All current production-high Node advisories and all Rust vulnerability/unsound
|
||||
advisories are removed without advisory ignores:
|
||||
|
||||
- `@larksuiteoapi/node-sdk` 1.70.0 declares Axios `~1.13.3`, which prevented npm
|
||||
from selecting a patched release. The Hub now has a targeted npm override to
|
||||
Axios 1.18.1. The direct SDK stays current; the override is locked and the
|
||||
full Hub type/build/test gate validates the resolved tree. A deterministic
|
||||
local-HTTP smoke also drives the real SDK/Axios stack through tenant-token,
|
||||
authenticated success and HTTP-error paths without external credentials.
|
||||
- `memmap2` is upgraded to 0.9.11 and `crossbeam-epoch` to 0.9.20.
|
||||
- `plist` is upgraded to 1.10.0, which moves its XML parser to `quick-xml`
|
||||
0.41.0.
|
||||
- crates.io `citationberg` 0.7.0 still pins vulnerable `quick-xml` 0.38.4.
|
||||
Cargo now patches it to the exact upstream citationberg commit that adopts
|
||||
`quick-xml` 0.41.0; the git revision is immutable in both manifest and lock.
|
||||
|
||||
`hub-check` runs the checked-in `npm run audit:production` command immediately
|
||||
after `npm ci`. `checker-check` installs the pinned cargo-audit 0.22.2 release
|
||||
with `--locked`, then runs `cargo audit --deny unsound`; vulnerabilities fail by
|
||||
default and unsound warnings are promoted to failures.
|
||||
|
||||
Current evidence:
|
||||
|
||||
- `npm audit --omit=dev --audit-level=high`: 0 vulnerabilities.
|
||||
- `cargo audit --deny unsound`: 0 vulnerabilities and 0 unsound warnings.
|
||||
- Four Rust `unmaintained` informational warnings remain visible and are not
|
||||
ignored; they are outside this issue's vulnerability/unsound acceptance
|
||||
criterion and will continue to appear in CI output.
|
||||
- Hub: type-check, build and 280 tests pass (10 environment-specific skips).
|
||||
- Rust: fmt, clippy with warnings denied, and all 53 workspace tests pass.
|
||||
@@ -0,0 +1,50 @@
|
||||
# Enforce Organization status at the shared authorization seam
|
||||
|
||||
Type: task
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Make non-ACTIVE Organizations fail closed for project authorization,
|
||||
onboarding, queued execution, and Feishu triggers through one shared seam, with
|
||||
negative tests proving a SUSPENDED or ARCHIVED tenant cannot create, bind,
|
||||
trigger, resume, or mutate work while operator recovery remains explicit.
|
||||
|
||||
## Answer
|
||||
|
||||
Organization lifecycle is now enforced at both authorization and mutation
|
||||
linearization points:
|
||||
|
||||
- Project and project-group authorization resolves the owning Organization and
|
||||
denies every customer action before principal resolution when status is not
|
||||
`ACTIVE`. The legacy role helper follows the same rule.
|
||||
- Every customer mutation service (project/folder/binding/policy, explorer,
|
||||
members, teams, team grants, external-directory sync and slash-session
|
||||
mutation) verifies `ACTIVE` while holding a PostgreSQL `FOR SHARE` lock on
|
||||
the Organization row. A concurrent status `UPDATE` therefore orders before
|
||||
the operation (which fails closed) or after its transaction commits.
|
||||
- Agent session/run/project-lock admission is one transaction behind that same
|
||||
lifecycle lock. Queued triggers reauthorize when drained. Slash-command races
|
||||
return an explicit user-facing rejection instead of becoming deduplicated,
|
||||
silent failures.
|
||||
- Feishu attachments download into a private
|
||||
`HUB_PROJECT_WORKSPACE_ROOT/.cph-staging` batch outside every project. A
|
||||
durably committed ACTIVE run/project-lock admission is the linearization
|
||||
point; publication then uses exclusive same-filesystem hard links rather than
|
||||
copying bytes under a database lock. Failure compensates inode-tracked links,
|
||||
fails the admitted run and releases its lock. Startup removes abandoned
|
||||
staging batches and refuses staging symlinks.
|
||||
- Project creation holds the lifecycle lock across workspace allocation and DB
|
||||
creation. A later transaction failure removes the new leaf workspace; cleanup
|
||||
failure is surfaced together with the original error as `AggregateError`.
|
||||
- External-directory sync uses short lifecycle-serialized transactions per
|
||||
unit rather than holding the Organization row lock across an unbounded sync.
|
||||
|
||||
No customer API can reactivate or otherwise change Organization lifecycle
|
||||
state. Operator recovery remains a separate platform-control-plane task.
|
||||
|
||||
Evidence includes deterministic status/update lock ordering, direct-service
|
||||
negative tests for both `SUSPENDED` and `ARCHIVED`, queued-trigger and
|
||||
post-authorization races, DB-triggered workspace compensation (including
|
||||
cleanup failure), full local Hub tests, and the real Linux attachment trigger
|
||||
path.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Make Feishu intake and queued work durable
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 18, 44
|
||||
|
||||
## Question
|
||||
|
||||
Replace receipt-before-success and process-local batching/queueing with one
|
||||
durable inbound work lifecycle that atomically claims duplicate events, records
|
||||
processing/completion/failure, survives restart, retries recoverable failures,
|
||||
and proves concurrent redelivery cannot duplicate a run or silently lose an
|
||||
accepted prompt. Preserve execution-time authorization from ticket 18 when
|
||||
deferred work is finally started, and expose the durable identity/state needed
|
||||
by ADR-0022's Organization-fair admission scheduler rather than recreating a
|
||||
second queue lifecycle.
|
||||
@@ -0,0 +1,12 @@
|
||||
# Make run, session, lock, and recovery transitions safe
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
Implement and fault-test a centralized AgentRun lifecycle that enforces one
|
||||
active project/provider/role/model session, serializes provider-cursor updates,
|
||||
preserves the ADR-0002 lock invariant across every database failure, applies a
|
||||
real wall deadline and renewable lease, reconciles every non-terminal state on
|
||||
startup/watchdog, and cannot let a second Hub process invalidate live work.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Make outbound Feishu delivery durable and idempotent
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 44
|
||||
|
||||
## Question
|
||||
|
||||
Give status cards, final responses, files, and recovery notices a durable
|
||||
outbox/delivery lifecycle with stable idempotency keys, provider message IDs,
|
||||
bounded retry and reconciliation, so model completion is distinguishable from
|
||||
pending/failed/delivered user-visible output and an operator can safely replay
|
||||
ambiguous failures without duplicates.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Decide workspace commit semantics for failed runs
|
||||
|
||||
Type: grilling
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
When an agent run fails, is canceled, times out, loses the database, or is
|
||||
killed mid-tool, should its workspace changes remain as partial collaborative
|
||||
work, be quarantined for review, or roll back atomically? Decide the contract,
|
||||
including the relationship among provider cursor, AgentMessage,
|
||||
AgentFileChange, git/filesystem state, and the next run, then create the
|
||||
smallest implementation tickets and crash tests that enforce it.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Reconcile the audit and run-history contract
|
||||
|
||||
Type: grilling
|
||||
Status: open
|
||||
Blocked by: 04
|
||||
|
||||
## Question
|
||||
|
||||
Separate or unify run-bound audit entries, pre-run security/permission events,
|
||||
structured messages, and operational recovery events without weakening
|
||||
`Spec.System.Audit`'s pinned AuditEntry-to-run relation. Decide durability,
|
||||
failure, retention, and query semantics; then enforce referential integrity and
|
||||
observable/recoverable writes instead of silently swallowing lost evidence.
|
||||
Do not merge these customer Project/Run records with ADR-0023's already-decided
|
||||
separate, append-only, fail-closed Platform Audit; decide only any shared
|
||||
retention/export infrastructure without weakening either semantic boundary.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Make HTTP dependency failures typed and observable
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
Replace message-substring HTTP error classification with typed validation,
|
||||
domain, authorization, Prisma/dependency, and unexpected-error boundaries.
|
||||
Return only sanitized stable client codes, preserve true 5xx/availability
|
||||
semantics, emit one request-scoped structured error with a support-visible
|
||||
request ID, redact secrets and raw internals, and add a real database-outage
|
||||
regression proving Prisma failures can never become client 4xx responses.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Decide the telemetry, SLO, retention, and alerting control plane
|
||||
|
||||
Type: grilling
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
For the accepted initial single-host service, decide the log/metric/trace
|
||||
collection and storage path, instance roles, release and correlation identity,
|
||||
redaction/access/retention policy, initial service-level indicators and
|
||||
objectives, error budgets, alert thresholds, on-call destinations, escalation
|
||||
ownership, and telemetry-blind detection. Prefer the smallest operable control
|
||||
plane, but record every domain-visible availability or retry assumption in the
|
||||
appropriate ADR/spec rather than hiding it in dashboard configuration. Consume
|
||||
ADR-0022's already-decided usage/cost soft-alert ownership as input; do not
|
||||
reopen its hard-limit or provider-mode financial-alert semantics here.
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
# Instrument correlated production runtime telemetry
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 13, 23, 24, 25, 27, 28, 29
|
||||
|
||||
## Question
|
||||
|
||||
Implement the decided telemetry control plane with one redacted structured
|
||||
context from HTTP/Feishu ingress through durable work, run/session/lock,
|
||||
provider and SDK retry/rate-limit/tool events, cph/workspace operations,
|
||||
outbound delivery, audit, startup/drain/recovery, and release/schema identity.
|
||||
Expose bounded-cardinality metrics, dashboards and alert rules; eliminate
|
||||
silent critical-path failures; and prove logs can pivot end-to-end while metric
|
||||
labels contain no tenant/user/run cardinality or secret/message content.
|
||||
@@ -0,0 +1,22 @@
|
||||
# Write and drill production incident recovery runbooks
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 06, 12, 13, 14, 23, 24, 25, 26, 27, 28, 29, 30, 39, 40, 41
|
||||
|
||||
## Question
|
||||
|
||||
Check in operator runbooks for PostgreSQL outage/migration failure, Feishu
|
||||
disconnect and credential rejection, ingress/outbound backlog or dead letter,
|
||||
stuck run/lock, provider or cph failure, workspace capacity/corruption,
|
||||
deploy/rollback, host restart during work, and suspected tenant exposure. Give
|
||||
each alert an owner and safe read-first commands, require confirmed/audited
|
||||
scope for destructive or replay actions, execute every runbook against
|
||||
production-shaped staging, and archive evidence that detection, mitigation,
|
||||
reconciliation, verification, escalation, and alert clearing all work. Include
|
||||
database/workspace backup failure, failed migration, full-host restore and
|
||||
cutover, duplicate/gap review, fallback, and achieved RPO/RTO evidence.
|
||||
Also drill ADR-0023 platform-administrator lockout and Platform-owned Feishu
|
||||
Application credential loss through the dual-factor offline recovery command,
|
||||
expiring Emergency Platform Grant, session revocation, immutable Platform Audit,
|
||||
and explicit recovery closeout; never introduce a standing break-glass account.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Implement layered limits and multi-dimensional request rate limiting
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 20, 22, 44
|
||||
|
||||
## Question
|
||||
|
||||
Implement ADR-0022's fail-closed platform ceilings, Organization-lowerable
|
||||
policy limits, bounded list/query behavior, and typed multi-dimensional rate
|
||||
limits for authenticated Web, signed Feishu, OAuth, and administrative writes.
|
||||
Prove effective limits never exceed platform ceilings and every overload result
|
||||
is observable, scoped, and carries retry guidance where applicable.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Build the durable Organization-fair admission scheduler
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 18, 23, 24, 32
|
||||
|
||||
## Question
|
||||
|
||||
Extend the durable inbound lifecycle into ADR-0022's bounded Agent Admission
|
||||
Queue with platform/Organization/Project concurrency, fair scheduling across
|
||||
Organizations, FIFO within each Organization, durable position/state,
|
||||
expiration notification, scoped cancellation, idempotent admission, explicit
|
||||
capacity rejection, and no automatic replay after a run has started.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Enforce file, archive, storage, and entity budgets
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 17, 23, 32
|
||||
|
||||
## Question
|
||||
|
||||
Enforce ADR-0022's request/file/attachment/archive-expansion,
|
||||
Project/Organization storage, and Organization entity-count limits with
|
||||
streaming accounting, atomic reservation, cleanup of incomplete artifacts,
|
||||
bounded queries, explicit errors, and continued read/export/delete/admin
|
||||
remediation while an Organization is at or above a ceiling.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Enforce Agent Run and process resource budgets
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 16, 24, 32
|
||||
|
||||
## Question
|
||||
|
||||
Implement ADR-0022's wall-time, turn, tool-count, per-tool-time,
|
||||
output/log/event-size, memory, CPU, and child-process budgets. Termination must
|
||||
be fail-fast and observable, persist `TIMED_OUT` or `LIMIT_EXCEEDED` with the
|
||||
triggered dimension and actual usage, cancel the provider/process safely, and
|
||||
release run locks and admission capacity under every injected failure.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Implement provider-mode usage attribution and soft alerts
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 19, 24, 29, 32, 43
|
||||
|
||||
## Question
|
||||
|
||||
Attribute token usage, provider-reported cost, run count, and duration by
|
||||
Organization, Project, Run, model, and Provider Connection; preserve missing
|
||||
cost as unknown; expose Organization-configurable soft thresholds; and route
|
||||
Organization alerts plus platform-managed anomalous-cost alerts without hard
|
||||
spend blocking, BYOK financial alerts, or payment-settlement semantics.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Add audited emergency workload brakes
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 22, 23, 24, 27, 30, 32, 33, 45
|
||||
|
||||
## Question
|
||||
|
||||
Implement reversible, reason-bearing, audited `DRAIN` and `STOP_NOW` controls at
|
||||
Organization and platform scope. Prove new admissions, queued work, active runs,
|
||||
locks, user notifications, authorization, telemetry, restart persistence, and
|
||||
explicit recovery follow ADR-0022 without deleting or silently suspending
|
||||
Organization data.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Calibrate and prove production capacity ceilings
|
||||
|
||||
Type: research
|
||||
Status: open
|
||||
Blocked by: 30, 32, 33, 34, 35, 36, 37
|
||||
|
||||
## Question
|
||||
|
||||
On the accepted production-like host and provider path, measure safe numerical
|
||||
defaults and headroom for every ADR-0022 platform ceiling. Exercise normal and
|
||||
noisy-neighbor traffic, provider stalls and rate limits, queue saturation,
|
||||
large files/archives, storage pressure, process budgets, and both workload
|
||||
brakes; publish reproducible evidence and the checked-in configuration values
|
||||
required by the production release gate.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Decide recovery objectives, backup retention, and restore scope
|
||||
|
||||
Type: grilling
|
||||
Status: open
|
||||
|
||||
## Question
|
||||
|
||||
For the initial SaaS service, decide measurable RPO/RTO targets, backup and
|
||||
restore-drill frequency, retention and deletion-in-backups policy, off-host
|
||||
copy/failure-domain requirements, logical export versus physical base-backup
|
||||
and continuous-WAL/PITR coverage, encryption and key custody, operator ownership,
|
||||
and whether the supported promise is full-service disaster recovery only or
|
||||
also Organization-selective export/restore. Record compliance-dependent items
|
||||
as explicit OPEN inputs rather than inventing defaults.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Implement coherent backup sets and verified restore
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 10, 12, 14, 19, 23, 24, 25, 26, 29, 39, 41, 43
|
||||
|
||||
## Question
|
||||
|
||||
Implement the decided recovery policy as cataloged, checksummed, encrypted,
|
||||
off-host backup sets covering PostgreSQL logical exports plus physical base
|
||||
backup/continuous WAL or a proven managed equivalent, Project workspaces,
|
||||
compatible release/migration identity, configuration inventory, org-secret
|
||||
ciphertext, and separately recoverable key material. Establish a proven
|
||||
cross-store checkpoint, retention and alerting, partial-artifact cleanup,
|
||||
isolated restore with outbound integrations disabled, lifecycle reconciliation,
|
||||
critical SaaS journey verification, and measured RPO/RTO drills.
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Make the Project workspace lifecycle recoverable
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 10, 26, 34
|
||||
|
||||
## Question
|
||||
|
||||
Move persistent workspaces outside every mutable release tree and make Project
|
||||
workspace allocation, database commit, archive/deletion, relocation, restore,
|
||||
and orphan/missing-path reconciliation explicit and crash-safe. Prove failed
|
||||
Project creation leaves no silent orphan, deployment/rollback/pruning cannot
|
||||
delete customer files, restored absolute paths are validated or safely rebased,
|
||||
and every inconsistency is observable rather than auto-hidden.
|
||||
@@ -0,0 +1,54 @@
|
||||
# Decide the platform-administrator identity and audit boundary
|
||||
|
||||
Type: grilling
|
||||
Status: resolved
|
||||
|
||||
## Question
|
||||
|
||||
Which identity issuer, bootstrap and recovery procedure, invite/allowlist model,
|
||||
session claims, role lifecycle, separation from Organization membership, and
|
||||
append-only audit contract will authenticate Platform Administrators for the
|
||||
initial service? The decision must keep `/admin/platform` private, make every
|
||||
Organization/connection/workload-control mutation attributable, and define a
|
||||
reviewable break-glass path without treating platform staff as customer members.
|
||||
|
||||
## Answer
|
||||
|
||||
Platform Administrators authenticate only through a dedicated Platform-owned
|
||||
Feishu Application and a separate, server-side Platform Session. Platform
|
||||
Identity, standing grant, invitation, session, audit and emergency grant are
|
||||
independent of customer `User`, `OrganizationMembership`, Project permission
|
||||
and Project/Run audit. The initial control plane has one peer role only.
|
||||
|
||||
The first administrator is created by a versioned, idempotent, transactional
|
||||
database bootstrap procedure that runs only when no standing administrator is
|
||||
active and atomically records the Platform Audit Entry. Later administrators
|
||||
receive short-lived, single-use invitations bound to a specific platform
|
||||
Feishu account. The last active standing administrator cannot be revoked;
|
||||
revocation immediately invalidates that identity's sessions and invitations.
|
||||
|
||||
Every platform request reloads the revocable session and current standing or
|
||||
unexpired emergency grant. Administrator, connection/credential,
|
||||
Organization-lifecycle, workload-brake and emergency-grant actions require
|
||||
recent Feishu OAuth step-up.
|
||||
|
||||
Every successful platform mutation commits with a separate append-only
|
||||
Platform Audit Entry or fails. Organization Administrators receive a redacted
|
||||
projection of platform actions affecting their own Organization. No standing
|
||||
break-glass account exists: offline recovery requires controlled host access,
|
||||
an off-host recovery key, an incident and reason, and issues only an expiring
|
||||
Emergency Platform Grant.
|
||||
|
||||
The complete accepted decision and implementation divergences are in
|
||||
[ADR-0023](../../../docs/adr/0023-platform-administrator-identity-and-audit.md).
|
||||
The pinned semantic invariants are in
|
||||
[`Spec.System.PlatformAdministration`](../../../spec/Spec/System/PlatformAdministration.lean),
|
||||
and the canonical terms are in [`CONTEXT.md`](../../../CONTEXT.md).
|
||||
|
||||
Exact numeric session/invitation/step-up limits and browser mechanics remain
|
||||
with [Design the production browser and session boundary](20-design-production-web-session-boundary.md).
|
||||
Customer Project/Run audit retention remains with
|
||||
[Reconcile the audit and run-history contract](27-reconcile-audit-history-contract.md),
|
||||
without reopening Platform Audit's separate fail-closed boundary. Implementation
|
||||
is owned by [Build the private platform-administrator control plane](45-build-private-platform-admin-control-plane.md),
|
||||
and recovery rehearsal by [Write and drill production incident recovery runbooks](31-write-and-drill-incident-runbooks.md).
|
||||
@@ -0,0 +1,16 @@
|
||||
# Implement the Organization connection and secret plane
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: none
|
||||
|
||||
## Question
|
||||
|
||||
Implement encrypted, versioned, Organization-scoped Feishu Application and
|
||||
model Provider Connection records plus one resolver boundary. Support one
|
||||
customer-owned Feishu app per Organization, Organization-admin-managed BYOK,
|
||||
and one distinct platform-managed key/base URL per Organization; enforce which
|
||||
administrator may write each mode, readiness and rotation lifecycle, runtime
|
||||
resolution from Organization/Project, redacted observability, backup/key
|
||||
recovery hooks, and no process-global credential fallback or plaintext leakage
|
||||
into business rows, logs, HTTP responses, or agent environments.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Route Feishu identity and traffic by Organization connection
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 20, 22, 43
|
||||
|
||||
## Question
|
||||
|
||||
Make every Feishu OAuth, WebSocket/event, card action, directory sync, chat
|
||||
binding, API call, and outbound message start from an explicit active
|
||||
Organization Feishu Application Connection. Model external user, chat, event,
|
||||
and callback identifiers in that connection's namespace; bind OAuth state to
|
||||
the intended Organization/application; run and observe one listener/client
|
||||
lifecycle per configured app; and prove two customer-owned apps with colliding
|
||||
provider-local identifiers cannot authenticate, bind, authorize, deduplicate,
|
||||
rate-limit, or deliver across Organizations. Do not infer tenant from a global
|
||||
`open_id`, `chat_id`, or from the user's membership count.
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# Build the private platform-administrator control plane
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 20, 27, 28, 42, 43, 44
|
||||
|
||||
## Question
|
||||
|
||||
Implement authenticated `requirePlatformAdmin` APIs and a private
|
||||
`/admin/platform` web area for Platform Administrators to create/list/update
|
||||
Organizations, bootstrap their OWNER, view/change lifecycle state, configure
|
||||
and test each customer-owned Feishu Application, configure the Organization's
|
||||
distinct platform-managed Provider Connection, and manage platform-admin
|
||||
invitations and revocations. Implement ADR-0023's separate Platform Identity,
|
||||
single standing role, guarded bootstrap procedure, identity-bound single-use
|
||||
invitations, revocable server-side Platform Session, recent-authentication
|
||||
step-up, last-administrator protection, fail-closed append-only Platform Audit,
|
||||
Organization-safe audit projection, and dual-factor offline recovery with
|
||||
expiring Emergency Platform Grants. Replace rather than reuse the legacy
|
||||
customer-`User`-bound `PlatformRoleAssignment`/`TEACHER` model. Prove anonymous
|
||||
users, ordinary members, and Organization admins cannot load either page data
|
||||
or mutations; every sensitive action is validated, redacted, audited,
|
||||
observable, and covered by browser-level critical-journey tests. Emergency
|
||||
workload controls remain integrated through
|
||||
[Add audited emergency workload brakes](37-emergency-workload-brakes.md).
|
||||
@@ -0,0 +1,17 @@
|
||||
# Build the private Organization-administrator control plane
|
||||
|
||||
Type: task
|
||||
Status: open
|
||||
Blocked by: 20, 27, 28, 32, 36, 43, 44
|
||||
|
||||
## Question
|
||||
|
||||
Turn the guarded Organization JSON APIs into a usable private
|
||||
`/admin/org/:orgSlug` web control plane for OWNER/ADMIN users. Cover members and
|
||||
roles, teams and Project access, Folders/Projects/chat unbinding, sessions and
|
||||
usage, model/role/run policy, Organization policy limits and alerts, Feishu
|
||||
connection readiness, Provider Credential Mode selection, and BYOK key/base URL
|
||||
management without ever displaying stored plaintext. Preserve server-side
|
||||
authorization on every request, define safe loading/error/empty states, and
|
||||
prove login redirect plus browser journeys, cross-Organization denial, MEMBER
|
||||
denial, secret redaction, and logout/session expiry end to end.
|
||||
@@ -0,0 +1,62 @@
|
||||
# Initial Production SaaS Readiness
|
||||
|
||||
Label: wayfinder:map
|
||||
|
||||
## Destination
|
||||
|
||||
The repository can be deployed from a clean checkout as the initial production
|
||||
multi-tenant Curriculum Project Hub: platform staff manage Organizations; every
|
||||
Organization authenticates through and operates its own customer-owned Feishu
|
||||
app/bot, and selects either an Organization-managed BYOK provider connection or
|
||||
a distinct platform-managed provider connection; Organization owners/admins
|
||||
manage their own members, projects and settings. The ADR-0018/0020/0021
|
||||
single-host Linux topology is operated with repeatable release evidence,
|
||||
rollback and recovery, and no known critical security or data-integrity gaps.
|
||||
|
||||
## Notes
|
||||
|
||||
- `spec/` remains the semantic source of truth; implementation drift must be
|
||||
surfaced rather than silently resolved.
|
||||
- The initial product boundary follows the accepted ADRs: Organization is the
|
||||
tenant root and customer onboarding may be manual. Full billing, tenant
|
||||
suspension, and self-service platform administration remain deferred;
|
||||
ADR-0022's safety limits and workload brakes are required for production.
|
||||
- Manual guided setup may provision each Organization's customer-owned Feishu
|
||||
app during the pilot; one process-global Feishu/OAuth credential shared by
|
||||
unrelated Organizations is not an acceptable destination state.
|
||||
- Use the `diagnosing-bugs` loop for every reproduced defect. Prefer explicit
|
||||
failures and observable critical paths over fallback behavior.
|
||||
- Work directly on `main` per developer authorization; make small conventional
|
||||
commits after verified, independently reversible milestones. Do not push
|
||||
without explicit authorization.
|
||||
- Local `lake build` currently requires a configured elan toolchain; CI remains
|
||||
the authoritative configured Lean build environment until that local toolchain
|
||||
is available.
|
||||
|
||||
## Decisions so far
|
||||
|
||||
- [Audit the clean-host deployment and rollback contract](issues/01-audit-clean-host-deployment.md) — keep the accepted single-host topology, but replace the incomplete in-place updater with a provisioned, immutable, readiness-gated and rollbackable release contract.
|
||||
- [Audit tenant, authentication, and request security boundaries](issues/02-audit-tenant-auth-security.md) — retain the working org-admin/application authorization core, but production is blocked on agent/MCP isolation, execution-time object binding, org-scoped encrypted credentials, tenant status, browser sessions, and clean dependency audits.
|
||||
- [Audit run lifecycle and data integrity under failure](issues/03-audit-run-lifecycle-integrity.md) — retain the database-enforced project-lock uniqueness and normal happy path, but replace receipt-before-success, process-local queues, independent run/session/lock effects, silent delivery/audit loss, and partial startup reset with durable, recoverable state machines; explicitly decide failed-run workspace semantics.
|
||||
- [Define production observability and incident recovery](issues/04-define-observability-recovery.md) — keep the existing Fastify/Prisma/systemd signal fragments, but production requires dependency-aware readiness, typed and correlated errors, durable-work telemetry, an explicit SLO/retention/alerting control plane, and recovery runbooks proven by fault drills.
|
||||
- [Define initial abuse and capacity controls](issues/05-define-abuse-capacity-controls.md) — require layered fail-closed limits, durable Organization-fair admission, explicit backpressure, bounded files/storage/runs, soft usage alerts, and audited emergency workload brakes; calibrate numerical ceilings under production-like load.
|
||||
- [Audit backup, migration, and disaster recovery safety](issues/06-audit-backup-migration-recovery.md) — separate persistent state from releases, define a database/workspace checkpoint and recovery objectives, make migrations compatibility-gated, and prove complete off-host backup sets through isolated restore and cutover drills.
|
||||
- [Audit the accepted product surface for production completeness](issues/07-audit-product-surface-completeness.md) — retain the guarded Organization APIs and single-app Feishu vertical slice, but require explicit Organization-scoped Feishu/provider connections and external-identity namespaces, private platform and Organization admin control planes, and the full ADR-0022 capacity surface before release.
|
||||
- [Decide the platform-administrator identity and audit boundary](issues/42-decide-platform-admin-identity-audit.md) — use a dedicated platform Feishu issuer, one peer administrator role, guarded bootstrap and identity-bound invitations, revocable server-side sessions, atomic append-only Platform Audit, and dual-factor offline recovery without a standing break-glass account.
|
||||
- [Restore the checker formatting gate](issues/09-restore-checker-format-gate.md) — pin local and CI Rust/rustfmt/clippy to 1.92.0, apply the canonical workspace format, and keep the full fmt + clippy + 53-test checker gate green.
|
||||
- [Provision a runnable non-root Hub service](issues/10-provision-nonroot-service.md) — install a strictly validated service identity and persistent layout, reject release/workspace overlap, validate complete production configuration and real bind settings, and execute Hub/Prisma/cph/bubblewrap probes as the service user under `no_new_privs` before writing the unit.
|
||||
- [Confine the agent runtime and protect service credentials](issues/16-confine-agent-and-protect-credentials.md) — replace the Agent environment, deny host reads/writes from root, keep every writable SDK path inside the canonical project, require bubblewrap plus socat, emit bounded correlated diagnostics, and prove the boundary with the real Linux SDK under a capability-free non-root identity.
|
||||
- [Close MCP context and file-delivery escape paths](issues/17-close-mcp-data-egress-escapes.md) — use no-follow project-rooted file operations and delivery snapshots, remove Git-root delivery, and bind every Feishu MCP read to the run's project and chat.
|
||||
- [Enforce Organization status at the shared authorization seam](issues/22-enforce-organization-status.md) — deny non-ACTIVE tenants before principal resolution, serialize every customer mutation and run admission against Organization lifecycle updates, reauthorize queued work, and stage/publish attachments with crash recovery and explicit filesystem compensation.
|
||||
- [Clear dependency advisories and gate releases](issues/21-clear-dependency-security-advisories.md) — resolve Node production-high and Rust vulnerability/unsound advisories with locked compatible upgrades plus an exact upstream citationberg fix, and make both audits mandatory CI gates without ignores.
|
||||
|
||||
## Fog
|
||||
|
||||
- The production topology after the initial single-host pilot (HA, horizontal
|
||||
workers, multi-region, or managed container orchestration) depends on measured
|
||||
demand and failure experience.
|
||||
- Commercial self-service concerns such as billing plans and automated tenant
|
||||
provisioning remain beyond the accepted ADR boundary; revisit after the
|
||||
initial safety and operability route is clear.
|
||||
- Regulatory retention, residency, and customer-specific compliance controls
|
||||
require product/legal inputs that the repository does not currently contain.
|
||||
@@ -0,0 +1,46 @@
|
||||
# AGENTS.md —— agent 操作手册(全 repo)
|
||||
|
||||
本 repo 是 monorepo。先读根 `README.md` 的"宪法"5 条,那是一切工作的前提。本文件是给在这里干活的 coding agent 的纪律。
|
||||
|
||||
## 这个 repo 是什么
|
||||
|
||||
- `spec/` 是一份**人机共识的契约**(Lean 语义母本),是产品语义的上游参照。
|
||||
- 其余部件(将来的 `spec/` 外文件夹)是**向 `spec/` 对齐的实现**。
|
||||
- `hub/` 的平台层按 SaaS 形态演进:`Organization` 是 tenant root;`Project`/`Team`
|
||||
必须归属 org,TEAM→PROJECT 授权不得跨 org(见 ADR-0020 / `Spec.System.Organization`)。
|
||||
- org 后台 project explorer 里 `Folder` 是透明组织节点,不是权限资源;project 仍是权限边界。
|
||||
普通老师可在飞书群自助建 project 但受 org policy 控制(见 ADR-0021 /
|
||||
`Spec.System.ProjectWorkspace`)。
|
||||
- 每个 org 自选 BYOK 或平台托管 model provider connection;平台托管也必须是该 org
|
||||
独享的 key/base URL,不得让无关 org 共用 process-global provider key(见 ADR-0021 /
|
||||
`Spec.System.Organization`)。
|
||||
- Feishu/provider secret 使用本地版本化 master-key keyring 的信封加密;生产由 systemd
|
||||
credential 注入,运行时只允许显式 org/project scope 的 fail-closed resolver,不得回退
|
||||
process-global credential;Agent child 只接收 run-scoped loopback proxy capability,
|
||||
不接收 org provider credential(见 ADR-0024 / `Spec.System.Organization`)。
|
||||
- 生产容量按不可突破的 platform ceiling 与 org 可下调 policy 分层;有效限制取两者较低值。
|
||||
Agent admission 必须持久、有界、跨 org 公平且显式背压(见 ADR-0022 /
|
||||
`Spec.System.Capacity`)。
|
||||
- 平台管理员只通过独立的 platform-owned 飞书应用与可撤销 Platform Session 认证,不复用
|
||||
客户 `User`/org membership;平台写操作与 append-only audit 同事务,break-glass 只走
|
||||
双因子的离线恢复流程(见 ADR-0023 / `Spec.System.PlatformAdministration`)。
|
||||
- 受控 alpha 暂采用一 Organization 一具名 systemd Silo:独立 database role/database、
|
||||
service identity、workspace、keyring 与 Feishu/provider connection;进程必须由
|
||||
`HUB_SILO_ORGANIZATION_ID` fail-closed 绑定唯一 org,平台后台不开放。共享 SaaS
|
||||
控制面与 Docker adapter 后置(见 ADR-0025)。
|
||||
- Agent role 与 skill 是 Organization-scoped 动态运行配置:role 组合 model、system prompt、
|
||||
tools 与已安装 skill;skill 版本进入 content-addressed 持久存储,run 只读加载所选快照。
|
||||
`settingSources: []` 继续禁用项目/用户配置加载,不得把任意 workspace `.claude` 配置变成
|
||||
运行时能力(见 ADR-0018)。
|
||||
|
||||
## 纪律
|
||||
|
||||
1. **不得用预训练先验脑补本领域。** 这个领域很新,你没有相关先验。契约里 prose doc 注释是语义的唯一权威来源;契约没写的,就是没定的。
|
||||
|
||||
2. **凡契约未写明者,不得假设。** 遇到标了 `OPEN` 的地方,或契约根本没覆盖的地方,**显式 surface 出来**让开发者决定,绝不擅自替它选一个解。
|
||||
|
||||
3. **改 `spec/` 必须保持其 `lake build` 通过。** 在 `spec/` 目录下跑 `lake build`。新增声明必须带 `/-- … -/` doc 注释和恰当标签(`PINNED` / `OPEN` / `ADR-NNNN`)。规范见 `spec/README.md`。不准用 `sorry` 把 build 糊绿。
|
||||
|
||||
4. **实现向契约对齐;偏离必须 surface。** 没有 CI gate 替你把关 spec↔实现的一致性(见宪法第 2 条)——这道对齐靠 review 和你巡逻 diff。发现实现与契约不一致时,报告它,不要默默让其中一边将就另一边。
|
||||
|
||||
5. **写操作谨慎。** 线上操作、git 写操作前与开发者确认(这是开发者的全局偏好)。
|
||||
@@ -6,6 +6,11 @@
|
||||
|
||||
- `spec/` 是一份**人机共识的契约**(Lean 语义母本),是产品语义的上游参照。
|
||||
- 其余部件(将来的 `spec/` 外文件夹)是**向 `spec/` 对齐的实现**。
|
||||
- `hub/` 的平台层按 SaaS 形态演进:`Organization` 是 tenant root;`Project`/`Team`
|
||||
必须归属 org,TEAM→PROJECT 授权不得跨 org(见 ADR-0020 / `Spec.System.Organization`)。
|
||||
- org 后台 project explorer 里 `Folder` 是透明组织节点,不是权限资源;project 仍是权限边界。
|
||||
普通老师可在飞书群自助建 project 但受 org policy 控制(见 ADR-0021 /
|
||||
`Spec.System.ProjectWorkspace`)。
|
||||
|
||||
## 纪律
|
||||
|
||||
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
# Curriculum Project Hub
|
||||
|
||||
The shared language for the multi-tenant service that lets customer organizations manage curriculum projects and operate agents through their own Feishu connection.
|
||||
|
||||
## Language
|
||||
|
||||
**Organization**:
|
||||
The customer tenant root that owns its memberships, projects, teams, external connections, and settings.
|
||||
_Avoid_: Account, workspace
|
||||
|
||||
**Platform Administrator**:
|
||||
An internal operator who creates and operates Organizations without becoming a member of them.
|
||||
_Avoid_: Organization admin, super member
|
||||
|
||||
**Organization Administrator**:
|
||||
An Organization OWNER or ADMIN who manages only that Organization's members, projects, connections, and settings.
|
||||
_Avoid_: Platform admin
|
||||
|
||||
**Admin Surface**:
|
||||
The private web control plane available only to an authenticated user with the corresponding Platform Administrator or Organization Administrator authority.
|
||||
_Avoid_: Public management page, member portal
|
||||
|
||||
**Customer-owned Feishu Application**:
|
||||
The Organization-scoped Feishu application used for that Organization's login, bot messages, and directory integration; unrelated Organizations never share it.
|
||||
_Avoid_: Global Feishu app, platform bot
|
||||
|
||||
**Platform-owned Feishu Application**:
|
||||
The dedicated Feishu application used only to authenticate Platform Administrators; it is separate from every Customer-owned Feishu Application and grants no Organization or Project authority.
|
||||
_Avoid_: Customer-owned Feishu Application, Organization Feishu app
|
||||
|
||||
**Platform Administrator Bootstrap**:
|
||||
The one-time, guarded grant of Platform Administrator authority to a verified Platform Identity when no active Platform Administrator exists.
|
||||
_Avoid_: Seed administrator, standing bootstrap account
|
||||
|
||||
**Platform Administrator Invitation**:
|
||||
A short-lived, single-use invitation issued by a Platform Administrator and bound to one expected identity from the Platform-owned Feishu Application.
|
||||
_Avoid_: Open administrator enrollment, unbound administrator invitation
|
||||
|
||||
**Platform Session**:
|
||||
A revocable authentication session bound only to a Platform Identity and its active Platform Administrator authority; it carries no Organization membership or Project authority.
|
||||
_Avoid_: Organization admin session, customer session
|
||||
|
||||
**Platform Audit Entry**:
|
||||
An immutable, durable record of a platform control-plane decision or action, separate from customer Project and Agent Run audit and mandatory for every platform mutation. Organization Administrators receive only a redacted projection of entries affecting their Organization.
|
||||
_Avoid_: Best-effort log, Project audit entry
|
||||
|
||||
**Emergency Platform Grant**:
|
||||
Short-lived Platform Administrator authority issued only by the offline recovery procedure for a declared incident and automatically expired afterward.
|
||||
_Avoid_: Standing break-glass account, permanent emergency administrator
|
||||
|
||||
**Provider Credential Mode**:
|
||||
The Organization's choice between a BYOK Provider Connection and a Platform-managed Provider Connection.
|
||||
_Avoid_: Global provider setting
|
||||
|
||||
**BYOK Provider Connection**:
|
||||
An Organization-scoped model-provider connection whose key and base URL are supplied and managed by an Organization Administrator.
|
||||
_Avoid_: Platform key
|
||||
|
||||
**Platform-managed Provider Connection**:
|
||||
An Organization-scoped model-provider connection whose distinct key and base URL are supplied and managed by a Platform Administrator for that Organization.
|
||||
_Avoid_: Shared platform key, BYOK
|
||||
|
||||
**Platform Capacity Ceiling**:
|
||||
A non-overridable upper bound imposed by the platform to protect shared service capacity.
|
||||
_Avoid_: Organization quota
|
||||
|
||||
**Organization Policy Limit**:
|
||||
An optional lower bound configured by an Organization Administrator to govern that Organization's own usage.
|
||||
_Avoid_: Platform limit
|
||||
|
||||
**Effective Limit**:
|
||||
The lower of the Platform Capacity Ceiling and the Organization Policy Limit; when no Organization Policy Limit exists, the Platform Capacity Ceiling applies.
|
||||
_Avoid_: Unlimited default
|
||||
|
||||
**Organization-fair Scheduling**:
|
||||
The policy that allocates available agent execution capacity fairly across Organizations, while preserving first-in-first-out order within each Organization.
|
||||
_Avoid_: Global FIFO
|
||||
|
||||
**Agent Admission Queue**:
|
||||
The durable, bounded set of accepted agent run requests waiting for execution capacity.
|
||||
_Avoid_: In-memory trigger queue, unlimited backlog
|
||||
|
||||
**Expired Run Request**:
|
||||
An accepted agent run request that reached its maximum queue wait before execution; it is terminal and will never execute automatically.
|
||||
_Avoid_: Delayed request, automatic retry
|
||||
|
||||
**Run Request Initiator**:
|
||||
The Organization member whose action created an agent run request.
|
||||
_Avoid_: Queue owner, Project owner
|
||||
|
||||
**Explicit Run Retry**:
|
||||
A new agent run request deliberately created after an earlier run failed or timed out.
|
||||
_Avoid_: Automatic replay, implicit rerun
|
||||
|
||||
**Run Budget**:
|
||||
The deterministic resource boundary for one agent run, independent of monetary or token usage alerts.
|
||||
_Avoid_: Cost budget, unlimited run
|
||||
|
||||
**Emergency Workload Brake**:
|
||||
An audited Platform Administrator control that prevents new agent work for one Organization or the whole platform and may explicitly stop active work during an incident.
|
||||
_Avoid_: Organization deletion, service restart
|
||||
Generated
+12
-21
@@ -264,10 +264,9 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "citationberg"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "756ff1e3d43a9ecc8183932fb4d9fd3971236f3ce4acb62fe51d1cd43297547d"
|
||||
source = "git+https://github.com/typst/citationberg?rev=06a591e2f237d25e1dfdedac3f3d1494c496c52d#06a591e2f237d25e1dfdedac3f3d1494c496c52d"
|
||||
dependencies = [
|
||||
"quick-xml 0.38.4",
|
||||
"quick-xml",
|
||||
"serde",
|
||||
"serde_path_to_error",
|
||||
]
|
||||
@@ -427,6 +426,7 @@ version = "0.0.2"
|
||||
dependencies = [
|
||||
"cph-diag",
|
||||
"serde",
|
||||
"tempfile",
|
||||
"toml",
|
||||
]
|
||||
|
||||
@@ -480,9 +480,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.18"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
@@ -1402,9 +1402,9 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.10"
|
||||
version = "0.9.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
|
||||
checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -1641,13 +1641,13 @@ checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315"
|
||||
|
||||
[[package]]
|
||||
name = "plist"
|
||||
version = "1.9.0"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1"
|
||||
checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"indexmap",
|
||||
"quick-xml 0.39.4",
|
||||
"quick-xml",
|
||||
"serde",
|
||||
"time",
|
||||
]
|
||||
@@ -1755,23 +1755,14 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.38.4"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
|
||||
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.39.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
|
||||
@@ -21,3 +21,8 @@ serde = { version = "1", features = ["derive"] }
|
||||
toml = "0.8"
|
||||
cph-diag = { path = "crates/cph-diag" }
|
||||
cph-model = { path = "crates/cph-model" }
|
||||
|
||||
# citationberg 0.7.0 on crates.io pins vulnerable quick-xml 0.38. Upstream has
|
||||
# adopted quick-xml 0.41 but has not published a replacement release yet.
|
||||
[patch.crates-io]
|
||||
citationberg = { git = "https://github.com/typst/citationberg", rev = "06a591e2f237d25e1dfdedac3f3d1494c496c52d" }
|
||||
|
||||
@@ -41,7 +41,8 @@ crates/ ← 实现:rule-based checker(向 spec 对齐)。见 crates/R
|
||||
cph-check / cph-cli ← checker 本体 + `cph` 命令行
|
||||
render/ ← typst 渲染包 cph-render(母本的渲染后端之一,ADR-0005)
|
||||
examples/ ← 样例工程文件(如 TH-141),流水线的真实输入
|
||||
(hub/ exporter/ …) ← 将来的其他部件,平级于 spec/。尚未创建
|
||||
hub/ ← SaaS Hub:飞书协作、org 管理、agent runtime 与生产部署
|
||||
(exporter/ …) ← 将来的其他部件,平级于 spec/
|
||||
```
|
||||
|
||||
`spec/` 与实现部件**物理分离、平级共存**:谁是上游、谁向谁对齐,一眼可见。
|
||||
@@ -72,3 +73,7 @@ examples/ ← 样例工程文件(如 TH-141),流水线的真实输入
|
||||
## CI
|
||||
|
||||
`.gitea/workflows/spec-check.yml` 在每次 push / PR 时于 `spec/` 下跑 `lake build`,确保契约始终 type-check 通过(从第一天起就是"绿"的)。这是良构 gate,见宪法第 2 条。
|
||||
|
||||
Rust checker 的本地与 CI 工具链由根 `rust-toolchain.toml` 固定;`.gitea/workflows/checker-check.yml`
|
||||
必须安装同一精确版本并执行 `cargo fmt --all --check`、Clippy `-D warnings` 与 workspace
|
||||
全测试。升级 Rust 时这两处必须在同一提交更新并通过完整 checker gate。
|
||||
|
||||
@@ -360,8 +360,13 @@ pub fn target_is_shell(root: &Path, target: &str) -> bool {
|
||||
.iter()
|
||||
.find(|t| t.name == target)
|
||||
.is_some_and(|t| {
|
||||
t.steps.iter().any(|s| matches!(s, cph_model::Step::Shell { .. }))
|
||||
&& !t.steps.iter().any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
|
||||
t.steps
|
||||
.iter()
|
||||
.any(|s| matches!(s, cph_model::Step::Shell { .. }))
|
||||
&& !t
|
||||
.steps
|
||||
.iter()
|
||||
.any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -370,9 +375,17 @@ fn run_one_shell(cwd: &Path, run: &str) -> ShellStepOutcome {
|
||||
use std::process::Command;
|
||||
|
||||
let output = if cfg!(target_os = "windows") {
|
||||
Command::new("cmd").arg("/C").arg(run).current_dir(cwd).output()
|
||||
Command::new("cmd")
|
||||
.arg("/C")
|
||||
.arg(run)
|
||||
.current_dir(cwd)
|
||||
.output()
|
||||
} else {
|
||||
Command::new("sh").arg("-c").arg(run).current_dir(cwd).output()
|
||||
Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(run)
|
||||
.current_dir(cwd)
|
||||
.output()
|
||||
};
|
||||
|
||||
match output {
|
||||
@@ -704,7 +717,11 @@ fn collect_image_assets(eng_root: &Path, build_root: &Path, body: &str) -> Resul
|
||||
}
|
||||
}
|
||||
if let Err(e) = std::fs::copy(&src, &dst) {
|
||||
return Err(format!("failed to copy '{}' → '{}': {e}", src.display(), dst.display()));
|
||||
return Err(format!(
|
||||
"failed to copy '{}' → '{}': {e}",
|
||||
src.display(),
|
||||
dst.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -763,7 +780,8 @@ pub fn target_is_markdown_assemble(root: &Path, target: &str) -> bool {
|
||||
t.steps
|
||||
.iter()
|
||||
.any(|s| matches!(s, cph_model::Step::AssembleMarkdown { .. }))
|
||||
&& !t.steps
|
||||
&& !t
|
||||
.steps
|
||||
.iter()
|
||||
.any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
|
||||
})
|
||||
@@ -833,7 +851,11 @@ fn compile_targets(lesson: &cph_model::Lesson) -> Vec<&str> {
|
||||
lesson
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|t| t.steps.iter().any(|s| matches!(s, cph_model::Step::TypstCompile { .. })))
|
||||
.filter(|t| {
|
||||
t.steps
|
||||
.iter()
|
||||
.any(|s| matches!(s, cph_model::Step::TypstCompile { .. }))
|
||||
})
|
||||
.map(|t| t.name.as_str())
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
//! fixtures built in a temp dir. The typst Engine is pointed at the repo's real
|
||||
//! `render/` package via `Engine::with_render_dir`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cph_diag::DiagCode;
|
||||
use cph_typst::Engine;
|
||||
@@ -138,7 +138,7 @@ path = "segments/does-not-exist"
|
||||
|
||||
/// Write a throwaway one-segment lesson into `tmp`, with the given `[targets.x]`
|
||||
/// body spliced in. Returns nothing; the caller runs `check`.
|
||||
fn write_segment_lesson_with_target(tmp: &PathBuf, target_body: &str) {
|
||||
fn write_segment_lesson_with_target(tmp: &Path, target_body: &str) {
|
||||
std::fs::write(
|
||||
tmp.join("manifest.toml"),
|
||||
format!(
|
||||
@@ -264,7 +264,7 @@ template = "exports/handout.typ"
|
||||
|
||||
/// Write a one-segment lesson with a `file-tree` shell target whose `run` is the
|
||||
/// given command. Returns nothing; the caller runs `run_shell_target`.
|
||||
fn write_shell_target_lesson(tmp: &PathBuf, run: &str) {
|
||||
fn write_shell_target_lesson(tmp: &Path, run: &str) {
|
||||
std::fs::write(
|
||||
tmp.join("manifest.toml"),
|
||||
format!(
|
||||
@@ -365,7 +365,10 @@ run = "printf SHOULD_NOT_RUN > build/ran.txt"
|
||||
|
||||
let report = cph_check::run_shell_target(&tmp, &engine(), "assets");
|
||||
assert!(!report.ok);
|
||||
assert!(report.outcomes.is_empty(), "no command should run on a broken lesson");
|
||||
assert!(
|
||||
report.outcomes.is_empty(),
|
||||
"no command should run on a broken lesson"
|
||||
);
|
||||
assert!(report.check.has_errors());
|
||||
assert!(
|
||||
!tmp.join("build").join("ran.txt").exists(),
|
||||
@@ -380,7 +383,7 @@ run = "printf SHOULD_NOT_RUN > build/ran.txt"
|
||||
/// content file, plus a `slides` target that assembles them. The `which` slice
|
||||
/// controls which segments get a `slides.md` (so the "skip absent" path can be
|
||||
/// exercised). Returns nothing; the caller runs `run_markdown_assemble_target`.
|
||||
fn write_markdown_assemble_target_lesson(tmp: &PathBuf, slides: &[(&str, &str)]) {
|
||||
fn write_markdown_assemble_target_lesson(tmp: &Path, slides: &[(&str, &str)]) {
|
||||
let mut parts = String::new();
|
||||
for (i, (name, _)) in slides.iter().enumerate() {
|
||||
if i > 0 {
|
||||
@@ -424,10 +427,7 @@ field = "slides"
|
||||
#[test]
|
||||
fn target_is_markdown_assemble_detects_shape() {
|
||||
let tmp = tempdir();
|
||||
write_markdown_assemble_target_lesson(
|
||||
&tmp,
|
||||
&[("a", "# A"), ("b", "# B")],
|
||||
);
|
||||
write_markdown_assemble_target_lesson(&tmp, &[("a", "# A"), ("b", "# B")]);
|
||||
assert!(cph_check::target_is_markdown_assemble(&tmp, "slides"));
|
||||
// The student target doesn't exist here → falls through to typst path → false.
|
||||
assert!(!cph_check::target_is_markdown_assemble(&tmp, "student"));
|
||||
@@ -439,7 +439,10 @@ fn run_markdown_assemble_target_concatenates_in_parts_order() {
|
||||
let tmp = tempdir();
|
||||
write_markdown_assemble_target_lesson(
|
||||
&tmp,
|
||||
&[("intro", "# 规则一\n- 范围"), ("rule2", "# 规则二\n$8\\times$")],
|
||||
&[
|
||||
("intro", "# 规则一\n- 范围"),
|
||||
("rule2", "# 规则二\n$8\\times$"),
|
||||
],
|
||||
);
|
||||
|
||||
let report = cph_check::run_markdown_assemble_target(&tmp, &engine(), "slides");
|
||||
@@ -667,6 +670,6 @@ fn tempdir() -> PathBuf {
|
||||
p
|
||||
}
|
||||
|
||||
fn cleanup(p: &PathBuf) {
|
||||
fn cleanup(p: &Path) {
|
||||
let _ = std::fs::remove_dir_all(p);
|
||||
}
|
||||
|
||||
+41
-16
@@ -51,12 +51,12 @@ enum Command {
|
||||
/// completion file, e.g. `cph completions zsh > ~/.zfunc/_cph`.
|
||||
Completions {
|
||||
/// Which shell to generate completions for.
|
||||
shell: Shell,
|
||||
shell: CompletionTarget,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
|
||||
enum Shell {
|
||||
enum CompletionTarget {
|
||||
Bash,
|
||||
Zsh,
|
||||
Fish,
|
||||
@@ -82,16 +82,16 @@ fn main() -> ExitCode {
|
||||
/// Emit a shell-completion script for `shell` to stdout. The script is built
|
||||
/// from the same `Cli` clap definition above, so it tracks subcommands/flags as
|
||||
/// they evolve.
|
||||
fn run_completions(shell: Shell) -> ExitCode {
|
||||
fn run_completions(shell: CompletionTarget) -> ExitCode {
|
||||
use clap::CommandFactory;
|
||||
use clap_complete::Shell as CompShell;
|
||||
|
||||
let sh = match shell {
|
||||
Shell::Bash => CompShell::Bash,
|
||||
Shell::Zsh => CompShell::Zsh,
|
||||
Shell::Fish => CompShell::Fish,
|
||||
Shell::PowerShell => CompShell::PowerShell,
|
||||
Shell::Elvish => CompShell::Elvish,
|
||||
CompletionTarget::Bash => CompShell::Bash,
|
||||
CompletionTarget::Zsh => CompShell::Zsh,
|
||||
CompletionTarget::Fish => CompShell::Fish,
|
||||
CompletionTarget::PowerShell => CompShell::PowerShell,
|
||||
CompletionTarget::Elvish => CompShell::Elvish,
|
||||
};
|
||||
let mut cmd = Cli::command();
|
||||
clap_complete::generate(sh, &mut cmd, "cph", &mut std::io::stdout());
|
||||
@@ -188,7 +188,10 @@ fn run_build(
|
||||
/// explicit `cph build --target <name>`, and each command is printed before it
|
||||
/// runs so the user sees exactly what is executed.
|
||||
fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> ExitCode {
|
||||
eprintln!("running shell-step target '{target}' (commands execute in {})", path.display());
|
||||
eprintln!(
|
||||
"running shell-step target '{target}' (commands execute in {})",
|
||||
path.display()
|
||||
);
|
||||
let report = cph_check::run_shell_target(path, engine, target);
|
||||
print_diagnostics(&report.check);
|
||||
|
||||
@@ -196,7 +199,10 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
|
||||
// Refused before running anything (check errors / unknown target / no
|
||||
// shell steps): the diagnostics above explain why.
|
||||
if report.check.has_errors() {
|
||||
eprintln!("build refused: {} errors (fix the lesson first)", report.check.error_count());
|
||||
eprintln!(
|
||||
"build refused: {} errors (fix the lesson first)",
|
||||
report.check.error_count()
|
||||
);
|
||||
} else {
|
||||
eprintln!("build failed: target '{target}' has no shell steps to run");
|
||||
}
|
||||
@@ -212,13 +218,19 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
|
||||
eprint!("{}", outcome.stderr);
|
||||
eprintln!(
|
||||
"step failed (exit {})",
|
||||
outcome.status.map(|c| c.to_string()).unwrap_or_else(|| "signal".into())
|
||||
outcome
|
||||
.status
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_else(|| "signal".into())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if report.ok {
|
||||
println!("shell-step target '{target}' completed: {} step(s) ok", report.outcomes.len());
|
||||
println!(
|
||||
"shell-step target '{target}' completed: {} step(s) ok",
|
||||
report.outcomes.len()
|
||||
);
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
eprintln!("shell-step target '{target}' failed");
|
||||
@@ -231,13 +243,19 @@ fn run_shell_build(path: &std::path::Path, engine: &Engine, target: &str) -> Exi
|
||||
/// shell path, this is opt-in — only on an explicit `cph build --target <name>`,
|
||||
/// and never in `check`.
|
||||
fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target: &str) -> ExitCode {
|
||||
eprintln!("assembling markdown target '{target}' (reading parts under {})", path.display());
|
||||
eprintln!(
|
||||
"assembling markdown target '{target}' (reading parts under {})",
|
||||
path.display()
|
||||
);
|
||||
let report = cph_check::run_markdown_assemble_target(path, engine, target);
|
||||
print_diagnostics(&report.check);
|
||||
|
||||
if report.outcomes.is_empty() && !report.ok {
|
||||
if report.check.has_errors() {
|
||||
eprintln!("build refused: {} errors (fix the lesson first)", report.check.error_count());
|
||||
eprintln!(
|
||||
"build refused: {} errors (fix the lesson first)",
|
||||
report.check.error_count()
|
||||
);
|
||||
} else {
|
||||
eprintln!("build failed: target '{target}' has no assemble-markdown steps to run");
|
||||
}
|
||||
@@ -250,7 +268,11 @@ fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target:
|
||||
outcome.field, outcome.parts_read
|
||||
);
|
||||
if let Some(out_rel) = &outcome.written {
|
||||
println!("wrote {} ({} bytes)", path.join(out_rel).display(), outcome.body.len());
|
||||
println!(
|
||||
"wrote {} ({} bytes)",
|
||||
path.join(out_rel).display(),
|
||||
outcome.body.len()
|
||||
);
|
||||
}
|
||||
if let Some(err) = &outcome.error {
|
||||
eprintln!("assemble failed: {err}");
|
||||
@@ -258,7 +280,10 @@ fn run_markdown_assemble_build(path: &std::path::Path, engine: &Engine, target:
|
||||
}
|
||||
|
||||
if report.ok {
|
||||
println!("assemble-markdown target '{target}' completed: {} step(s) ok", report.outcomes.len());
|
||||
println!(
|
||||
"assemble-markdown target '{target}' completed: {} step(s) ok",
|
||||
report.outcomes.len()
|
||||
);
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
eprintln!("assemble-markdown target '{target}' failed");
|
||||
|
||||
@@ -10,3 +10,6 @@ serde = { workspace = true }
|
||||
# `preserve_order` keeps `[targets.*]` tables in TOML document order, which is
|
||||
# the declared order this crate exposes via `Lesson.targets` (ADR-0009).
|
||||
toml = { workspace = true, features = ["preserve_order"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
@@ -737,7 +737,11 @@ fn parse_target(name: String, value: toml::Value, diags: &mut Vec<Diagnostic>) -
|
||||
/// A well-formed but empty `covers = []` yields `Some(vec![])`: a target that
|
||||
/// explicitly renders no kind (every used kind then draws a `renderIgnored`
|
||||
/// warning) — distinct from an absent key.
|
||||
fn parse_covers(name: &str, value: toml::Value, diags: &mut Vec<Diagnostic>) -> Option<Vec<String>> {
|
||||
fn parse_covers(
|
||||
name: &str,
|
||||
value: toml::Value,
|
||||
diags: &mut Vec<Diagnostic>,
|
||||
) -> Option<Vec<String>> {
|
||||
let items = match value {
|
||||
toml::Value::Array(items) => items,
|
||||
_ => {
|
||||
|
||||
@@ -279,16 +279,15 @@ fn malformed_target_config_is_non_fatal_with_schema_violations() {
|
||||
/// match exactly (ADR-0016's MVP rule).
|
||||
const CPH_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Write a one-part lesson into a temp dir, optionally with a `.cph-version`
|
||||
/// file. Returns the temp dir so the caller runs `load`.
|
||||
fn tmp_lesson_with_version(version: Option<&str>) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
p.push(format!("cph-version-test-{nanos}"));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
/// Write a one-part lesson into a uniquely-created temp dir, optionally with a
|
||||
/// `.cph-version` file. `TempDir` owns cleanup so parallel tests cannot delete
|
||||
/// one another's fixture after a timestamp collision.
|
||||
fn tmp_lesson_with_version(version: Option<&str>) -> tempfile::TempDir {
|
||||
let tmp = tempfile::Builder::new()
|
||||
.prefix("cph-version-test-")
|
||||
.tempdir()
|
||||
.unwrap();
|
||||
let p = tmp.path();
|
||||
std::fs::write(
|
||||
p.join("manifest.toml"),
|
||||
"[project]\nid = \"v\"\nname = \"v\"\n[info]\ntitle = \"v\"\n[[parts]]\nkind = \"segment\"\npath = \"segments/a\"\n",
|
||||
@@ -301,14 +300,13 @@ fn tmp_lesson_with_version(version: Option<&str>) -> PathBuf {
|
||||
if let Some(v) = version {
|
||||
std::fs::write(p.join(".cph-version"), v).unwrap();
|
||||
}
|
||||
p
|
||||
tmp
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cph_version_matching_is_not_a_diagnostic() {
|
||||
let tmp = tmp_lesson_with_version(Some(CPH_VERSION));
|
||||
let (_lesson, diags) = load(&tmp);
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
let (_lesson, diags) = load(tmp.path());
|
||||
assert!(
|
||||
diags.iter().all(|d| d.code != DiagCode::CphVersionMismatch),
|
||||
"a matching .cph-version must not warn, got {diags:?}"
|
||||
@@ -318,8 +316,7 @@ fn cph_version_matching_is_not_a_diagnostic() {
|
||||
#[test]
|
||||
fn cph_version_mismatch_is_an_error_diagnostic() {
|
||||
let tmp = tmp_lesson_with_version(Some("99.99.99"));
|
||||
let (lesson, diags) = load(&tmp);
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
let (lesson, diags) = load(tmp.path());
|
||||
// The lesson still loads (so other defects could surface), but the
|
||||
// mismatch is an error diagnostic — which alone makes it illegal.
|
||||
assert!(lesson.is_some(), "a mismatch should not halt loading");
|
||||
@@ -327,7 +324,11 @@ fn cph_version_mismatch_is_an_error_diagnostic() {
|
||||
.iter()
|
||||
.filter(|d| d.code == DiagCode::CphVersionMismatch)
|
||||
.collect();
|
||||
assert_eq!(mm.len(), 1, "expected one CphVersionMismatch, got {diags:?}");
|
||||
assert_eq!(
|
||||
mm.len(),
|
||||
1,
|
||||
"expected one CphVersionMismatch, got {diags:?}"
|
||||
);
|
||||
assert!(
|
||||
mm[0].message.contains("99.99.99") && mm[0].message.contains(CPH_VERSION),
|
||||
"diagnostic should name both versions, got: {}",
|
||||
@@ -339,8 +340,7 @@ fn cph_version_mismatch_is_an_error_diagnostic() {
|
||||
fn cph_version_missing_is_skipped() {
|
||||
// No `.cph-version` file at all → skipped (ADR-0016 migration period; OPEN).
|
||||
let tmp = tmp_lesson_with_version(None);
|
||||
let (_lesson, diags) = load(&tmp);
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
let (_lesson, diags) = load(tmp.path());
|
||||
assert!(
|
||||
diags.iter().all(|d| d.code != DiagCode::CphVersionMismatch),
|
||||
"a missing .cph-version must be skipped, got {diags:?}"
|
||||
@@ -350,8 +350,7 @@ fn cph_version_missing_is_skipped() {
|
||||
#[test]
|
||||
fn cph_version_empty_is_an_error() {
|
||||
let tmp = tmp_lesson_with_version(Some(" \n"));
|
||||
let (_lesson, diags) = load(&tmp);
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
let (_lesson, diags) = load(tmp.path());
|
||||
assert!(
|
||||
diags
|
||||
.iter()
|
||||
|
||||
@@ -526,7 +526,11 @@ mod tests {
|
||||
// ADR-0015: `slides`/`transcript` are markdown content fields (`x-cph-content: "md"`),
|
||||
// distinct from the `.typ` content fields.
|
||||
let s = schema_for("segment").unwrap();
|
||||
let slides = s.content_fields.iter().find(|f| f.name == "slides").unwrap();
|
||||
let slides = s
|
||||
.content_fields
|
||||
.iter()
|
||||
.find(|f| f.name == "slides")
|
||||
.unwrap();
|
||||
assert_eq!(slides.ext, "md");
|
||||
assert!(!slides.required);
|
||||
let transcript = s
|
||||
|
||||
@@ -48,7 +48,8 @@ pub fn resolve_render_dir() -> PathBuf {
|
||||
ensure_extracted().unwrap_or_else(|_| {
|
||||
// Last-resort: extract under the OS temp dir. Still correct, just not
|
||||
// cached across processes.
|
||||
let fallback = std::env::temp_dir().join(format!("cph-render-{}", env!("CARGO_PKG_VERSION")));
|
||||
let fallback =
|
||||
std::env::temp_dir().join(format!("cph-render-{}", env!("CARGO_PKG_VERSION")));
|
||||
let _ = extract_to(&fallback);
|
||||
fallback
|
||||
})
|
||||
|
||||
@@ -26,4 +26,6 @@ The group can stay open while no Claude processing is active. A teacher leaving
|
||||
- Multi-open is natural: a teacher participates in multiple project groups.
|
||||
- Normal group discussion does not occupy the project.
|
||||
- "Exit project" should not be the normal action in the group workflow.
|
||||
- Project rebinding or group archival needs explicit product rules, separate from run cancellation.
|
||||
- ADR-0021 adds those explicit product rules: active project↔Feishu chat binding
|
||||
is strict 1:1, while archived historical bindings are retained so org admins
|
||||
can correct pilot setup mistakes without losing audit/session history.
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# ADR 0017: Agent Runtime — Claude Code SDK via OpenRouter
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
The original ADR-0017 mandated a provider-agnostic agent layer: any
|
||||
OpenAI-compatible model via OpenRouter, custom agent loop built on Vercel AI
|
||||
SDK's `streamText`. The motivation was to avoid vendor lock-in.
|
||||
|
||||
In practice, the hand-rolled agent loop lacked capabilities that a production
|
||||
agent needs: context compaction (long conversations), tool approval flows,
|
||||
partial-message streaming, and a battle-tested multi-turn loop. Building these
|
||||
ourselves meant re-implementing what the Claude Code SDK already provides.
|
||||
|
||||
The Feishu integration also proved costly to hand-roll — card schema quirks,
|
||||
patch limitations, file support — each API quirk cost a round-trip.
|
||||
|
||||
## Decision
|
||||
|
||||
Adopt `@anthropic-ai/claude-agent-sdk` as the agent runtime, routed through
|
||||
OpenRouter's **"Anthropic Skin"** (`https://openrouter.ai/api`).
|
||||
|
||||
OpenRouter exposes an Anthropic Messages API-compatible endpoint. Claude Code
|
||||
SDK speaks its native protocol directly to OpenRouter — no proxy, no format
|
||||
conversion. The SDK's `query()` owns the agent loop (tool dispatch, compaction,
|
||||
streaming). Built-in tools (Read, Write, Bash, Glob, Grep) cover the entire
|
||||
tool surface — no custom tools needed. `cph check` / `cph build` run via Bash.
|
||||
|
||||
The Hub `AgentSession` is provider/role/model-bound, and it must persist the
|
||||
provider runtime cursor needed to continue a conversation. For Claude Code SDK,
|
||||
that cursor is the `result.session_id`; store it in `AgentSession.metadata` as
|
||||
`claudeSessionId` and pass it back to the next `query()` call as
|
||||
`options.resume`. Role is part of the session binding because role prompts and
|
||||
tool surfaces can differ even when the underlying model is the same; `/draft`
|
||||
and `/review` must not resume the same Claude runtime cursor by accident.
|
||||
|
||||
Role definitions are Organization-scoped runtime data. A role bundle selects
|
||||
its default model, system prompt, tool allowlist and installed Agent skill
|
||||
versions. PostgreSQL is authoritative for role composition and skill metadata;
|
||||
skill bytes live in a content-addressed persistent store selected only by the
|
||||
recorded SHA-256 digest. Updating a role or binding skills takes effect without
|
||||
a Hub release or process restart. A change to the role's execution surface
|
||||
(model, prompt, tools, selected skill content) archives its active sessions so
|
||||
the next run cannot resume a provider context created under stale instructions;
|
||||
label and ordering-only changes preserve conversational continuity.
|
||||
|
||||
Environment variables:
|
||||
```
|
||||
ANTHROPIC_BASE_URL=https://openrouter.ai/api
|
||||
ANTHROPIC_AUTH_TOKEN=<OpenRouter API key>
|
||||
ANTHROPIC_API_KEY="" # must be explicitly empty
|
||||
```
|
||||
|
||||
Model routing: `ANTHROPIC_DEFAULT_SONNET_MODEL` etc. accept OpenRouter model
|
||||
IDs (e.g. `z-ai/glm-4.7`, `anthropic/claude-sonnet-4-20250514`). This gives
|
||||
limited OpenRouter-level model routing, but the runtime is still Claude Agent
|
||||
SDK-shaped: non-Claude models are best-effort and may not support every
|
||||
Claude Code feature.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Provider-agnosticism is not fully preserved. OpenRouter can route to GLM,
|
||||
Claude, GPT, etc., but the runtime protocol and agent loop remain Claude
|
||||
Agent SDK-bound.
|
||||
- The agent loop is Claude Code SDK's (compaction, tool approval, partial
|
||||
message streaming) — not hand-rolled.
|
||||
- Custom tools (read_file, write_file, cph_check, cph_build) are replaced by
|
||||
the SDK's built-in Read/Write/Bash/Glob/Grep.
|
||||
- The SSE event format matches Claude-to-IM's expected protocol natively.
|
||||
- Per-run model selection works via role→model mapping; models are OpenRouter
|
||||
IDs, not Anthropic-only aliases.
|
||||
- OpenRouter recommends setting Anthropic as the top-priority provider for
|
||||
Claude Code; non-Anthropic models may have reduced compatibility with some
|
||||
SDK features (e.g. thinking blocks).
|
||||
@@ -0,0 +1,203 @@
|
||||
# ADR 0018: The Agent Execution Surface Is Bounded By The Project Workspace
|
||||
|
||||
## Status
|
||||
|
||||
Accepted, and implemented. The mechanism is settled: the Claude Code SDK's
|
||||
built-in sandbox (bubblewrap on Linux, seatbelt on macOS) confines the agent
|
||||
process and its Bash subprocesses at the OS level. `bypassPermissions`
|
||||
remains (headless server — no interactive prompts), but the sandbox is the
|
||||
hard boundary, not the permission-prompt layer. Constitution rule 4
|
||||
divergence (ADR-0017's unbounded tools) is resolved.
|
||||
|
||||
Builds on **ADR-0001** (project group as collaboration space), **ADR-0002**
|
||||
(project lock scoped to `run_id`), **ADR-0004** (permission grants govern
|
||||
`triggerAgent`), **ADR-0007** (the engineering file is a directory tree on
|
||||
disk), and **ADR-0017** (Claude Code SDK as the agent runtime).
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0001/0002/0004 together cover *collaboration governance* up to the
|
||||
moment a teacher `@Claude`s: who may trigger an `AgentRun`, that the run
|
||||
holds a project-scoped lock, that the group is a long-lived space. They stop
|
||||
at `triggerAgent`. What the agent may do *after* the trigger — which files it
|
||||
may read or write, what commands it may run — is not named in any ADR, not
|
||||
modeled in `spec/`, and not enforced by any contract-level invariant.
|
||||
|
||||
The gap is not theoretical. ADR-0017 adopted the Claude Code SDK with
|
||||
`allowedTools: ["Read", "Write", "Bash", "Glob", "Grep"]` and
|
||||
`permissionMode: "bypassPermissions"`. Under that configuration the SDK's
|
||||
built-in tools operate over the whole host filesystem with no confinement, and
|
||||
`Bash` executes arbitrary shell with no approval gate. The Hub's
|
||||
`hub/src/agent/workspace.ts` was *intended* to be "the agent's only file
|
||||
surface" (its module doc says so) and ships a `confine()` path validator that
|
||||
rejects `..` and absolute escapes. But ADR-0017's cutover left those custom
|
||||
tools unwired — the SDK's built-in tools replaced them — so `confine()` is
|
||||
dead code and the boundary it was meant to enforce does not exist at runtime.
|
||||
Compounding this, the systemd unit's default `SERVICE_USER` is `root`
|
||||
(`hub/deploy/install_service.sh`), so the unbounded agent runs as root.
|
||||
|
||||
This is a real divergence point by the constitution's test: "if unwritten,
|
||||
will the developer and the agent make different assumptions?" They already
|
||||
have. ADR-0017 assumed the boundary was an implementation concern; the
|
||||
`workspace.ts` author assumed it was a contract. The next person to touch the
|
||||
runner has nothing telling them the boundary must be enforced, and ADR-0017
|
||||
silently overrode the one that existed.
|
||||
|
||||
## Decision
|
||||
|
||||
### The agent execution surface is bounded by the run's project workspace
|
||||
|
||||
During an `AgentRun`, every file operation the agent performs — read, write,
|
||||
glob, grep target — must fall **within the run's project workspace directory**
|
||||
(the on-disk engineering-file tree of ADR-0007). Operations whose target path
|
||||
escapes that tree are refused. This is a platform core invariant, pinned in
|
||||
`Spec.System.AgentSurface`.
|
||||
|
||||
The boundary is **per-run, anchored to the run's project**, the same scope as
|
||||
the Lock (ADR-0002). This keeps the two invariants composable:
|
||||
|
||||
- `Lock` (ADR-0002) bounds **concurrency** — who may be modifying the project.
|
||||
- `AgentSurface` (this ADR) bounds **blast radius** — what the agent may touch.
|
||||
|
||||
Both scope to `run × project`. Acquiring the lock and entering the surface
|
||||
happen together at run start; releasing the lock and leaving the surface
|
||||
happen together at run termination.
|
||||
|
||||
### Shell execution is subject to the same boundary, by effect
|
||||
|
||||
A shell command the agent runs executes with the workspace as its working
|
||||
directory (already true for the `cph` subprocess, ADR-0016) and its **file
|
||||
effects** — files read, written, created, or deleted — are subject to the same
|
||||
workspace boundary. The agent may invoke **named external tools** whose
|
||||
binaries live outside the workspace (notably `cph` via `CPH_BIN`); the
|
||||
boundary governs *file effects*, not *binary location*. Arbitrary
|
||||
`cat /etc/shadow` or `curl … > /etc/...` is out of bounds regardless of how it
|
||||
is invoked.
|
||||
|
||||
### The Hub process runs as a non-privileged service user
|
||||
|
||||
The systemd unit must not run as `root`. The default in
|
||||
`hub/deploy/install_service.sh` changes from `root` to a dedicated unprivileged
|
||||
user. This is a deployment invariant, recorded here because the unbounded-agent
|
||||
risk is compounded by root; it is enforced in the unit, not in `spec/` (it is
|
||||
not a semantic divergence between developer and agent — it is operational
|
||||
discipline).
|
||||
|
||||
### The mechanism: SDK sandbox (OS-level, per agent process)
|
||||
|
||||
The boundary is enforced by the Claude Code SDK's built-in sandbox
|
||||
(bubblewrap plus `socat` on Linux, seatbelt on macOS), configured via `query()` options in
|
||||
`hub/src/agent/runner.ts`:
|
||||
|
||||
- `sandbox.enabled: true` + `failIfUnavailable: true` — the agent process and
|
||||
its Bash subprocesses run sandboxed; if the sandbox can't start, `query()`
|
||||
emits an error and exits rather than running unsandboxed.
|
||||
- `sandbox.allowUnsandboxedCommands: false` — a tool cannot opt out with the
|
||||
SDK's `dangerouslyDisableSandbox` input. Claude Code 2.1.202 does not enforce
|
||||
that option reliably, so a host-side `PreToolUse` hook also denies every
|
||||
Bash request whose input explicitly sets `dangerouslyDisableSandbox: true`
|
||||
before a process can start.
|
||||
- `sandbox.filesystem.denyRead: ["/"]` with `allowRead` for the canonical
|
||||
current workspace and a small named system-runtime set — normal reads stay
|
||||
in the run's workspace while `/bin`, shared libraries, CA certificates,
|
||||
fonts and the configured `cph` executable remain available as the external
|
||||
tool exception described above.
|
||||
- `sandbox.filesystem.allowWrite: [workspaceDir]` confines persistent host
|
||||
effects to the canonical ADR-0007 workspace. Config/cache/home stay beneath
|
||||
`.cph/agent-runtime/`; `TMPDIR`, `TMP`, `TEMP`, and `CLAUDE_CODE_TMPDIR` all
|
||||
point at the workspace-local `.cph/t`. The workspace allocator uses stable
|
||||
compact Organization/Project path segments, deployment requires a short
|
||||
workspace root, and the canonical temp prefix fails fast above 56 bytes so
|
||||
the SDK can append randomized `socat` bridge socket names without exceeding
|
||||
Linux `sockaddr_un.sun_path`. Bubblewrap shadows non-allowlisted host trees
|
||||
with disposable tmpfs mounts: a shell write there may succeed inside that
|
||||
private namespace, but it cannot mutate the corresponding host path. The
|
||||
Linux proof checks host state after the sandbox exits.
|
||||
- The SDK subprocess environment replaces rather than spreads `process.env`.
|
||||
Only provider protocol variables and non-secret runtime variables cross the
|
||||
boundary; database, Feishu and Hub session credentials never enter it.
|
||||
`sandbox.credentials` denies provider token variables inside Bash and denies
|
||||
configured credential files. `CPH_SANDBOX_EXTRA_DENY_READ` names additional
|
||||
deployment-specific secret paths on the Hub side only.
|
||||
- `settingSources: []` and strict MCP configuration prevent an untrusted
|
||||
workspace or service-user config from widening tools, hooks, MCP servers, or
|
||||
sandbox paths.
|
||||
- Agent skills are Organization-scoped runtime configuration, not Hub release
|
||||
assets. A controlled host-console installer imports each version into a
|
||||
content-addressed persistent store and records its digest in PostgreSQL. A
|
||||
role selects enabled Organization skills alongside its model, system prompt
|
||||
and tool allowlist. Each run copies only those selected immutable versions
|
||||
into a run-scoped plugin outside the project workspace; the sandbox exposes
|
||||
that snapshot read-only and deletes it after the run. SDK-bundled skills and
|
||||
filesystem setting sources remain disabled, so project `.claude` content
|
||||
cannot register skills or widen tools. Requested skill versions are recorded
|
||||
on `run.created`; SDK initialization remains authoritative loading evidence.
|
||||
- Network: open (see Open Questions).
|
||||
|
||||
`bypassPermissions` is kept (headless server — no interactive prompts); the
|
||||
sandbox is the hard boundary, not the permission-prompt layer. This is
|
||||
subprocess-level, not process-level: the Hub host process is not sandboxed
|
||||
(it manages DB, Feishu ws, logs), only the Claude Code agent process and its
|
||||
children. This keeps the sandbox config surface small and lets future
|
||||
external CLIs (Gitea, Lark) run inside the sandbox by giving their binaries
|
||||
read access, without widening the Hub's own filesystem surface.
|
||||
|
||||
The contract pins the **invariant** (operations stay in the workspace), not
|
||||
the **mechanism**. The SDK sandbox is the current mechanism; switching to a
|
||||
different one (tool wrappers, container) would not be a spec change as long
|
||||
as it upholds `AgentFileOp.Authorized`.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `Spec.System.AgentSurface` gains an `AgentFileOp` structure and an
|
||||
`Authorized` predicate mirroring `Memory.McpReadRequest.Authorized`: the op
|
||||
carries a `run` and a `path`; `Authorized` holds iff the path falls within
|
||||
the run's workspace, via platform-provided `runWorkspace` and `pathWithin`
|
||||
(representations `OPEN`).
|
||||
- `hub/src/agent/workspace.ts`'s `confine()` intent is now contract-backed.
|
||||
Re-wiring it (or replacing it with an OS sandbox) is **mandatory**, not a
|
||||
preference. Its current dead-code status under ADR-0017 is a divergence to
|
||||
be addressed, not silently retained.
|
||||
- ADR-0017's `bypassPermissions` + unrestricted built-in tools is a
|
||||
**divergence from this ADR** and is surfaced as such (constitution rule 4).
|
||||
The reconciliation — restrict the SDK tool set, wrap Bash, or sandbox at the
|
||||
OS level — is the open mechanism question above. Whichever is chosen must
|
||||
uphold the `AgentSurface` invariant.
|
||||
- Inbound file/image delivery (`hub/src/feishu/trigger.ts`) lands files under
|
||||
`${workspaceDir}/.cph/inbox/`, which is inside the boundary — consistent, no
|
||||
change to the delivery path.
|
||||
- The `cph` subprocess (ADR-0016) already runs with `cwd = workspaceDir`
|
||||
(`hub/src/agent/cph.ts`), so it is inside the boundary by construction.
|
||||
|
||||
## Open Questions / Deferred
|
||||
|
||||
- **Network egress.** Decided: network is **open** in the initial
|
||||
production deployment. The sandbox confines filesystem writes to the
|
||||
workspace and denies reads of sensitive host paths, but does not restrict
|
||||
outbound network. Rationale: future Gitea/Lark CLI integration needs
|
||||
unconstrained network egress; `cph build` may fetch typst packages. The
|
||||
exfiltration risk (agent `curl`s workspace content out) is accepted as a
|
||||
tradeoff for CLI integration simplicity. This is revisitable — a network
|
||||
allowlist (`sandbox.network.allowedDomains`) can be added without a spec
|
||||
change if the threat model tightens.
|
||||
- **Named system-runtime read set.** The implementation allows only the
|
||||
platform runtime directories/files needed for shell, dynamic libraries,
|
||||
fonts, TLS/DNS and `cph` execution in addition to the project workspace.
|
||||
Expanding that set requires an explicit reviewed code change. Deployments
|
||||
name credential files with `CPH_SANDBOX_EXTRA_DENY_READ`; these paths are
|
||||
protected through the SDK credential layer and are not passed to the Agent
|
||||
environment. The exact runtime set remains plumbing and is not spec-pinned.
|
||||
- **Shell command vocabulary.** Whether the agent's Bash is restricted to a
|
||||
whitelist (e.g. `cph`, `ls`, `grep`) or allowed arbitrary commands whose
|
||||
*effects* are then confined, is a policy choice left to implementation. The
|
||||
invariant governs effects either way; current choice is arbitrary commands
|
||||
(sandbox confines effects).
|
||||
|
||||
## Related Decisions
|
||||
|
||||
- **Rate limiting and token usage.** ADR-0022 requires multi-dimensional hard
|
||||
request limits while keeping token thresholds as attributed soft alerts in
|
||||
the initial release.
|
||||
- **Inbound file limits.** ADR-0022 requires hard file, attachment, archive,
|
||||
Project-storage, and Organization-storage limits with explicit failure and
|
||||
partial-file cleanup.
|
||||
@@ -0,0 +1,129 @@
|
||||
# ADR 0019: Resolve Actors to Principal Sets for Team Permissions
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0004 deliberately chose a Feishu Docs-like permission model:
|
||||
`grant(resource, principal, role)` plus separate resource settings. The first
|
||||
Hub implementation stored `principal` as an opaque string and the Feishu trigger
|
||||
used the sender's `open_id` directly. That was enough for individual teacher
|
||||
grants, but not for production collaboration:
|
||||
|
||||
- a curriculum team needs one grant to apply to many teachers;
|
||||
- Feishu departments, user groups, and project chats should be grantable
|
||||
principals;
|
||||
- role-specific agent grants (`/review`, `/draft`) must compose with project
|
||||
permissions consistently;
|
||||
- audit/debug output must explain which principal and grant allowed or denied a
|
||||
run.
|
||||
|
||||
## Decision
|
||||
|
||||
Introduce typed principals and resolve every actor to a principal set before
|
||||
authorization.
|
||||
|
||||
### Principal types
|
||||
|
||||
`PrincipalType` is:
|
||||
|
||||
- `USER`: a Feishu user, identified by `feishuOpenId`.
|
||||
- `TEAM`: a Hub-managed teacher team.
|
||||
- `FEISHU_CHAT`: a Feishu chat/group.
|
||||
- `FEISHU_DEPARTMENT`: a Feishu department.
|
||||
- `FEISHU_USER_GROUP`: a Feishu user group.
|
||||
- `APP`: an integration or bot principal.
|
||||
|
||||
`PermissionGrant` and `RoleTriggerGrant` store `principalType` and
|
||||
`principalId`. Legacy opaque principal strings are backfilled as
|
||||
`USER/<old principal>`.
|
||||
|
||||
### Team membership
|
||||
|
||||
Hub teams are first-class. A user can be a direct active member of a team. A
|
||||
team can also be bound to external Feishu principals: chat, department, or user
|
||||
group. If an actor resolves to any bound external principal, the actor also
|
||||
resolves to that Hub team.
|
||||
|
||||
Membership changes are effective immediately because authorization resolves the
|
||||
principal set at request time.
|
||||
|
||||
### External Feishu principal sync
|
||||
|
||||
Feishu department, user group, and chat memberships are not scattered through
|
||||
callers. They are synchronized into `ExternalPrincipalMembership` rows and then
|
||||
read by `PrincipalResolver`. The Feishu message context can also contribute the
|
||||
current `FEISHU_CHAT` principal directly because receiving a group event is
|
||||
already contextual proof that the actor is speaking in that group.
|
||||
|
||||
### Grant merging
|
||||
|
||||
Authorization evaluates all active grants matching any resolved principal for
|
||||
the requested resource. There is no explicit deny in this ADR. Among matching
|
||||
grants, the highest role wins:
|
||||
|
||||
```text
|
||||
READ < EDIT < MANAGE
|
||||
```
|
||||
|
||||
Action thresholds:
|
||||
|
||||
- project read requires `READ`.
|
||||
- project edit requires `EDIT`.
|
||||
- collaborator management requires `MANAGE`.
|
||||
- agent trigger requires `EDIT`.
|
||||
- normal agent cancel requires `MANAGE`.
|
||||
|
||||
`PermissionSettings` never grants access beyond grants. Settings only constrain
|
||||
an already-granted capability. For `agentTrigger`:
|
||||
|
||||
- missing setting or `ROLE` means role-derived grants decide;
|
||||
- `MANAGE_ONLY` raises the threshold to `MANAGE`;
|
||||
- `DISABLED` denies the action.
|
||||
|
||||
Other settings keep their existing string storage, but the same rule applies:
|
||||
settings are policy constraints, not positive grants.
|
||||
|
||||
### Role-trigger composition
|
||||
|
||||
Role-trigger grants are a second gate after the project-level `agent.trigger`
|
||||
gate:
|
||||
|
||||
1. The actor must pass project `agent.trigger` for the project.
|
||||
2. If no `RoleTriggerGrant` row has ever existed for `(projectId, roleId)`,
|
||||
the role is open for backward compatibility.
|
||||
3. Once any row exists for `(projectId, roleId)`, including a revoked row, the
|
||||
role is configured. The actor must match an active role grant by any resolved
|
||||
principal.
|
||||
|
||||
This keeps `/review` restrictions orthogonal to ordinary edit access while
|
||||
preventing a fully revoked role from silently reopening.
|
||||
|
||||
### Module seam
|
||||
|
||||
Callers use a single authorization module:
|
||||
|
||||
```ts
|
||||
authorizer.can({
|
||||
actor,
|
||||
action,
|
||||
resource,
|
||||
roleId,
|
||||
})
|
||||
```
|
||||
|
||||
The caller does not know how users, teams, Feishu departments, user groups, or
|
||||
chats expand. `PrincipalResolver` owns that implementation, and
|
||||
`PermissionAuthorizer` owns grant/settings/role-trigger composition.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Teacher teams become production-grade principals rather than UI sugar.
|
||||
- Feishu external organization concepts can be synchronized and granted without
|
||||
leaking Feishu-specific checks into the trigger path.
|
||||
- A single audit decision can report actor, principal set, matched grant, and
|
||||
matched role grant.
|
||||
- Backward compatibility is explicit: old principal strings become user
|
||||
principals, and unconfigured role grants remain open.
|
||||
@@ -0,0 +1,94 @@
|
||||
# ADR 0020: Organization Is The SaaS Tenant Root
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
The Hub is moving from a single-organization deployment assumption toward a
|
||||
SaaS shape. The existing permission model already supports team principals via
|
||||
`PermissionGrant(PROJECT, TEAM, role)`, but without a tenant root the data model
|
||||
cannot safely answer:
|
||||
|
||||
- which customer owns a project;
|
||||
- which organization a team belongs to;
|
||||
- whether a team grant crosses customer boundaries;
|
||||
- which external directory sync produced a Feishu department/user-group
|
||||
membership;
|
||||
- where future org admin, billing, audit, and platform break-glass controls
|
||||
attach.
|
||||
|
||||
Retrofitting this after projects, teams, runs, and audit records grow would
|
||||
force tenant assumptions into every caller.
|
||||
|
||||
## Decision
|
||||
|
||||
Introduce `Organization` as the SaaS tenant root.
|
||||
|
||||
Customer-side collaboration data is organization-scoped:
|
||||
|
||||
```text
|
||||
Organization
|
||||
OrganizationMembership
|
||||
Team
|
||||
TeamMembership
|
||||
Project
|
||||
ExternalDirectoryConnection
|
||||
PermissionGrant
|
||||
```
|
||||
|
||||
`User` remains global. A user can belong to multiple organizations through
|
||||
`OrganizationMembership`. Project-level authorization still uses
|
||||
`PermissionGrant` and `PermissionAuthorizer`; the authorizer derives the
|
||||
organization from the requested resource, then resolves teams and external
|
||||
principals in that organization scope.
|
||||
|
||||
`Team` belongs to exactly one organization. `Project` belongs to exactly one
|
||||
organization. A `PermissionGrant` that grants a `TEAM` principal on a `PROJECT`
|
||||
resource is valid only when the team and project are in the same organization.
|
||||
This invariant is pinned in `Spec.System.Organization`.
|
||||
|
||||
External directory membership is scoped through `ExternalDirectoryConnection`:
|
||||
|
||||
```text
|
||||
ExternalDirectoryConnection(
|
||||
organizationId,
|
||||
provider,
|
||||
providerTenantId,
|
||||
source
|
||||
)
|
||||
```
|
||||
|
||||
`ExternalPrincipalMembership` points at a connection rather than carrying a
|
||||
free-floating `source` string. `source` names a sync pipeline; it is not the
|
||||
tenant identity.
|
||||
|
||||
The SaaS platform control plane is separate from customer project permissions.
|
||||
`READ/EDIT/MANAGE` remains the customer-side project permission lattice.
|
||||
Platform staff access, tenant suspension, billing, and break-glass access are a
|
||||
separate control plane and must not be modeled as project `MANAGE`.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Existing single-org data is backfilled into a default organization during
|
||||
migration.
|
||||
- `Team.slug` becomes organization-scoped instead of globally unique.
|
||||
- Actor resolution is organization-aware: direct teams and synchronized external
|
||||
principals only expand inside the resource's organization.
|
||||
- Direct `USER` grants remain possible because a global user can collaborate
|
||||
across organizations when explicitly granted.
|
||||
- Future org admin UI can be built on this model without changing the
|
||||
authorization core.
|
||||
|
||||
## Open Questions / Deferred
|
||||
|
||||
- Full platform admin UI, billing, and tenant suspension workflows are
|
||||
deferred. Production safety limits and workload brakes are no longer
|
||||
deferred; ADR-0022 defines them without treating a workload brake as tenant
|
||||
suspension.
|
||||
- Whether direct `USER` project grants should require active
|
||||
`OrganizationMembership` is deferred; explicit direct grants remain allowed
|
||||
for now.
|
||||
- Provider-specific directory semantics beyond Feishu are deferred, but the
|
||||
connection model leaves room for them.
|
||||
@@ -0,0 +1,117 @@
|
||||
# ADR 0021: Org Admin Project Onboarding
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0020 introduced `Organization` as the SaaS tenant root. The next product
|
||||
surface is not only the Feishu bot trigger path, but also the control planes
|
||||
around it:
|
||||
|
||||
- platform staff need a private platform admin area to create and operate orgs;
|
||||
- customer org owners/admins need an org admin area for teams, roles, provider
|
||||
mode and BYOK configuration, projects, folders, sessions, and usage accounting;
|
||||
- ordinary teachers should be able to start work from the natural Feishu group
|
||||
flow without entering the web backend.
|
||||
|
||||
Without a crisp project onboarding model, project creation, Feishu chat binding,
|
||||
permissions, session history, and usage accounting would drift into separate
|
||||
ad-hoc rules.
|
||||
|
||||
## Decision
|
||||
|
||||
Use one web app with two separate admin areas:
|
||||
|
||||
```text
|
||||
/admin/platform internal platform admins only
|
||||
/admin/org/:orgSlug customer org OWNER/ADMIN only
|
||||
```
|
||||
|
||||
The guards are intentionally separate:
|
||||
|
||||
- `requirePlatformAdmin` for internal operators;
|
||||
- `requireOrgRole` for org owner/admin backend access;
|
||||
- `requireProjectPermission` for project-level actions.
|
||||
|
||||
Platform admin is not an `OrganizationMembership` and is not represented by
|
||||
project `PermissionGrant`. Platform admin audit remains separate from customer
|
||||
project audit.
|
||||
|
||||
Customer orgs are manually created by platform staff for the pilot. Platform
|
||||
admins invite/allowlist other platform admins. Customer users authenticate with
|
||||
the customer's Feishu app. The same customer-owned Feishu app may be used for
|
||||
OAuth login, bot messages, and directory sync. App secrets are org-scoped
|
||||
secrets and must be encrypted at rest; business code should access them through
|
||||
a secret/connection resolver, not raw plaintext columns.
|
||||
|
||||
Each Organization chooses one model-provider credential mode:
|
||||
|
||||
- `BYOK`: an Organization OWNER/ADMIN supplies and manages that Organization's
|
||||
provider key and base URL in the org admin area;
|
||||
- `PLATFORM_MANAGED`: a platform admin supplies and manages one distinct key
|
||||
and base URL for that Organization in the platform admin area. Organization
|
||||
users cannot read the plaintext credential.
|
||||
|
||||
Provider connections remain Organization-scoped in both modes. Unrelated
|
||||
Organizations never share a process-global provider key. Provider secrets must
|
||||
be encrypted at rest and resolved at runtime through the same connection/secret
|
||||
boundary as Feishu credentials.
|
||||
|
||||
Project management has two creation paths:
|
||||
|
||||
- org owner/admin creates projects in the org web backend;
|
||||
- ordinary org members may create a project from an unbound Feishu group when
|
||||
`membersCanCreateProjects` is enabled for the org.
|
||||
|
||||
In both paths the creator gets project `MANAGE`. A Feishu-created project is
|
||||
immediately bound to the source chat, and that chat receives project `EDIT`.
|
||||
|
||||
Feishu chat binding is strict 1:1:
|
||||
|
||||
- one Feishu chat binds to at most one active project;
|
||||
- one project binds to at most one Feishu chat;
|
||||
- binding an existing unbound project from Feishu requires the clicking user to
|
||||
have `MANAGE` on that project;
|
||||
- binding creates an active `FEISHU_CHAT -> PROJECT EDIT` grant;
|
||||
- binding mistakes are corrected by org owner/admin unbinding or archiving the
|
||||
binding in the backend; historical sessions stay on their original project.
|
||||
|
||||
Folders are transparent organization nodes for project navigation and usage
|
||||
aggregation:
|
||||
|
||||
- folders belong to one organization;
|
||||
- projects may sit in folders;
|
||||
- folders can be nested;
|
||||
- folders are not permission resources;
|
||||
- folder visibility, team policies, and inherited grants are deferred.
|
||||
|
||||
Project permissions remain on `PROJECT` resources. Moving a project between
|
||||
folders does not change grants.
|
||||
|
||||
Usage accounting is org-wise and project/folder aggregatable for both provider
|
||||
modes. It is not payment collection in the pilot: BYOK spend belongs to the
|
||||
customer's provider account, while platform-managed spend is attributed to the
|
||||
Organization for operational reporting; commercial billing remains deferred.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `OrganizationProjectSettings.membersCanCreateProjects` gates Feishu group
|
||||
project creation for ordinary members.
|
||||
- `Folder` and `Project.folderId` support the file-manager-like project
|
||||
explorer without creating a second ACL system.
|
||||
- Service code should expose project creation and chat binding as reusable
|
||||
backend operations so Feishu cards and future web APIs call the same rules.
|
||||
- Org role/model/provider-mode panels and the platform-managed connection panel
|
||||
can be added on top of the org tenant root without changing project authorization.
|
||||
|
||||
## Open Questions / Deferred
|
||||
|
||||
- True one-click Feishu app provisioning is deferred; pilot uses guided setup
|
||||
and readiness checks.
|
||||
- Folder-level permissions are deferred until there is a concrete customer need.
|
||||
- Self-serve org signup and payment collection are deferred beyond pilot.
|
||||
- The platform admin identity, session, bootstrap, invitation, audit, and
|
||||
offline recovery boundary is decided by ADR-0023 and must be implemented
|
||||
before exposing the platform admin panel.
|
||||
@@ -0,0 +1,88 @@
|
||||
# ADR 0022: Layered Capacity Admission And Workload Safety
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
The initial SaaS deployment shares one production host across Organizations.
|
||||
The current Hub only has a one-run-per-project lock, a process-local five-item
|
||||
trigger queue, a five-minute queue wait, a global max-turn setting, and
|
||||
Fastify's default request-body limit. It has no global or Organization agent
|
||||
capacity, durable backpressure, fair scheduling, storage quotas, run wall-clock
|
||||
budget, multi-dimensional rate limiting, or emergency workload control.
|
||||
|
||||
Leaving these limits implicit lets one Organization monopolize the host, lets
|
||||
accepted work disappear on restart, and makes overload look like success. A
|
||||
single global FIFO would preserve arrival order while still permitting one
|
||||
tenant to occupy every execution slot. Unlimited queues or automatic whole-run
|
||||
replay would hide overload and can duplicate tool side effects.
|
||||
|
||||
## Decision
|
||||
|
||||
Every mandatory capacity dimension has a non-optional **Platform Capacity
|
||||
Ceiling**. An Organization OWNER/ADMIN may configure a lower **Organization
|
||||
Policy Limit**; the **Effective Limit** is the lower value, or the platform
|
||||
ceiling when the Organization value is absent. There is no unlimited fallback.
|
||||
Production refuses to start when a mandatory platform ceiling is absent or
|
||||
invalid. Platform ceilings are changed through versioned deployment
|
||||
configuration in the initial release; live platform-admin editing is deferred
|
||||
until audited approval and rollback exist.
|
||||
|
||||
Agent work uses a durable, bounded admission queue. Scheduling is fair across
|
||||
Organizations and FIFO within an Organization, while the existing one-active-
|
||||
run-per-Project rule remains. Capacity saturation queues only within effective
|
||||
length and wait limits. Queue overflow is an explicit `capacity_exhausted`
|
||||
response with retry guidance; an accepted request that waits too long becomes
|
||||
`EXPIRED`, is never started later, and notifies its initiator. Initiators may
|
||||
withdraw their own queued requests; Project `MANAGE` and Organization admins
|
||||
may cancel queued requests in their scope. Every transition is auditable.
|
||||
|
||||
The platform never automatically replays a whole Agent Run after execution has
|
||||
started. Limited transport retries are allowed only before model output or tool
|
||||
side effects. Otherwise the run ends explicitly and a retry is a new user-
|
||||
initiated request. Feishu event identity and API idempotency prevent delivery
|
||||
retries from creating duplicate admissions.
|
||||
|
||||
The first production gate requires hard limits for request rate/body size,
|
||||
agent concurrency, queue length/wait, files and attachment counts, archive
|
||||
expansion, Project and Organization storage, Organization entity counts, run
|
||||
wall time/turns/tool calls/output, individual tool duration, and execution
|
||||
process memory/CPU/process count. Reaching an entity or storage ceiling blocks
|
||||
new growth but does not delete, hide, or lock existing data; read, export,
|
||||
delete, and administrator remediation remain available.
|
||||
|
||||
Admin surfaces require an authenticated user plus the corresponding platform
|
||||
or Organization administrator authorization. Authenticated Web traffic is
|
||||
limited by user, Organization, and endpoint class; signed Feishu traffic is
|
||||
limited by Organization application, chat, and sender; administrative writes
|
||||
use separate stricter buckets. Rejections and truncation are never silent.
|
||||
|
||||
Token, provider-reported cost, run count, and duration are attributed by
|
||||
Organization, Project, Run, model, and Provider Connection. They are statistics
|
||||
and soft alerts in the initial release, not hard spend controls or settlement.
|
||||
Missing provider cost remains unknown rather than zero. Organization admins
|
||||
receive their alerts; platform admins additionally receive anomalous-cost
|
||||
alerts for platform-managed Provider Connections, not BYOK financial alerts.
|
||||
|
||||
Platform administrators have audited, reason-bearing, reversible workload
|
||||
brakes at Organization and platform scope. `DRAIN` rejects new work and pauses
|
||||
queue starts while allowing active runs to finish. `STOP_NOW` also cancels
|
||||
active runs. These controls do not delete or suspend the Organization.
|
||||
|
||||
Exact numerical ceilings remain open until production-like capacity tests
|
||||
measure the supported host and provider behavior. Defaults must be checked in,
|
||||
validated, observable, and proven at the release gate rather than guessed as
|
||||
domain constants.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The current in-memory `TriggerQueue` and global-only agent settings are
|
||||
implementation divergences, not acceptable production fallbacks.
|
||||
- Durable ingress, run/lock recovery, authorization, telemetry, and capacity
|
||||
enforcement must share one Organization-scoped admission identity.
|
||||
- HTTP overload uses typed `429` responses with `Retry-After`; Feishu reports a
|
||||
clear busy/expired/canceled outcome to the initiating user.
|
||||
- Capacity calibration is a release prerequisite and must include noisy-neighbor,
|
||||
restart, queue expiry, file/archive, provider-stall, and workload-brake tests.
|
||||
@@ -0,0 +1,136 @@
|
||||
# ADR 0023: Platform Administrator Identity And Audit Boundary
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0021 separated `/admin/platform` from Organization administration but left
|
||||
the exact platform identity store and audit schema open. That choice must be
|
||||
settled before exposing the panel: customer Feishu identities are scoped by
|
||||
their customer-owned applications, Organization membership must not confer
|
||||
platform authority, and a platform operator can create tenants, change
|
||||
connections, rotate credentials, or stop workloads across tenant boundaries.
|
||||
|
||||
The current implementation is not the accepted platform model.
|
||||
`PlatformRoleAssignment` points at the customer-side `User`, includes a legacy
|
||||
`TEACHER` value, and has no platform login or guard. The current browser session
|
||||
is a seven-day stateless customer cookie, while `AuditEntry` is Project/Run
|
||||
oriented and deliberately best-effort. Reusing any of these would couple the
|
||||
two control planes and make immediate revocation or fail-closed privileged
|
||||
audit impossible.
|
||||
|
||||
## Decision
|
||||
|
||||
### Identity issuer and authority
|
||||
|
||||
Platform Administrators authenticate only through one dedicated
|
||||
**Platform-owned Feishu Application**. It is separate from every Organization's
|
||||
Customer-owned Feishu Application and cannot grant Organization membership or
|
||||
Project permission.
|
||||
|
||||
A `PlatformIdentity` is scoped by that platform application and its verified
|
||||
external Feishu user identity. It is not a customer `User`. The same person may
|
||||
also have customer identities, but there is no implicit link, shared session,
|
||||
or authority propagation between them.
|
||||
|
||||
The initial platform control plane has exactly one standing role:
|
||||
`Platform Administrator`. There is no Platform Owner, Super Admin, or Teacher
|
||||
hierarchy. The first administrator has no permanent special authority after
|
||||
another administrator is active.
|
||||
|
||||
### Bootstrap, invitation, and revocation
|
||||
|
||||
The first Platform Administrator is granted through a checked-in, versioned,
|
||||
idempotent and transactional bootstrap database procedure. It accepts only a
|
||||
verified Platform Identity, refuses to run while any standing Platform
|
||||
Administrator is active, and atomically writes the grant plus its Platform
|
||||
Audit Entry. Ad-hoc manual `INSERT` statements and permanent environment
|
||||
allowlists are not supported bootstrap paths.
|
||||
|
||||
Later administrators join through short-lived, single-use invitation links
|
||||
created by an active Platform Administrator. An invitation is bound to one
|
||||
specific Feishu account in the Platform-owned Feishu Application; possessing a
|
||||
link without authenticating as that account is insufficient. The invitation
|
||||
records its inviter and reason and becomes terminal after acceptance, expiry,
|
||||
or revocation. One active Platform Administrator approval is sufficient for the
|
||||
initial service.
|
||||
|
||||
All standing Platform Administrators are peers. An administrator may invite or
|
||||
revoke another administrator, but the last active standing administrator cannot
|
||||
be revoked. Revocation immediately invalidates all Platform Sessions and all
|
||||
unclaimed invitations belonging to the revoked identity.
|
||||
|
||||
### Platform Session and step-up authentication
|
||||
|
||||
`Platform Session` is a separate, server-side, revocable session with a distinct
|
||||
cookie/token namespace. The browser receives an opaque random token; persistence
|
||||
stores only its hash and binds the session to a Platform Identity. Every request
|
||||
reloads the session and active grant. The session carries no Organization,
|
||||
membership, Project, or customer Feishu authority.
|
||||
|
||||
Administrator lifecycle changes, platform or customer Feishu connection
|
||||
changes, platform-managed Provider credential changes, Organization lifecycle
|
||||
changes, workload-brake changes, and Emergency Platform Grant changes require
|
||||
recent Feishu OAuth reauthentication. Exact session/idle/step-up durations,
|
||||
CSRF, cookie and proxy/TLS mechanics belong to the production browser/session
|
||||
boundary decision; they must be configurable under checked maximums rather than
|
||||
silently inheriting the current seven-day customer cookie.
|
||||
|
||||
### Platform audit
|
||||
|
||||
`Platform Audit Entry` is a separate append-only record from customer Project
|
||||
and Agent Run audit. Every successful platform mutation and its audit entry
|
||||
commit in the same database transaction. If the audit entry cannot be written,
|
||||
the mutation fails. Platform audit is durable business evidence, not
|
||||
best-effort telemetry.
|
||||
|
||||
An entry identifies the Platform Identity and Platform Session, action, target
|
||||
kind and identifier, affected Organization when applicable, mandatory reason
|
||||
for security-sensitive actions, request/correlation identity, outcome, time,
|
||||
and redacted before/after facts. It never stores credential, token, recovery-key
|
||||
or other plaintext secret material. The normal service path can append and read
|
||||
but cannot update or delete history.
|
||||
|
||||
Platform Administrators can read the complete redacted platform audit.
|
||||
Organization Administrators can read a customer-safe projection of entries
|
||||
affecting their Organization, including Organization lifecycle, Feishu/provider
|
||||
connection, and workload-brake changes; they cannot read other Organizations,
|
||||
platform administrator lifecycle, recovery material, or internal security
|
||||
events.
|
||||
|
||||
### Offline recovery and break-glass
|
||||
|
||||
There is no standing break-glass web account or shared emergency password.
|
||||
Recovery uses a checked-in offline command from a controlled production-host
|
||||
console. The supported procedure requires both privileged host access and a
|
||||
recovery key held outside the host, environment file, and ordinary backup set.
|
||||
It also requires an incident identifier, reason, and one explicit recovery
|
||||
action.
|
||||
|
||||
Recovery may revoke Platform Sessions, repair the Platform-owned Feishu
|
||||
connection, or issue an `Emergency Platform Grant` to a verified Platform
|
||||
Identity. An emergency grant is time-bounded, automatically expires, is not a
|
||||
standing role, and must be explicitly closed when recovery finishes. Every
|
||||
recovery action is represented in the append-only Platform Audit. Two factors
|
||||
are mandatory for the initial service; two separate human approvers are not.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Platform identity, invitation, standing grant, server-side session, platform
|
||||
audit, and emergency-grant persistence must be distinct from customer
|
||||
`User`, `OrganizationMembership`, Project permission, and Project/Run audit.
|
||||
- `requirePlatformAdmin` must resolve the dedicated session and an active
|
||||
standing or unexpired emergency grant on every request. It is never an
|
||||
Organization-role override.
|
||||
- The legacy `PlatformRoleAssignment`/`TEACHER` schema is not a compatible
|
||||
shortcut and must be migrated or replaced before the platform panel ships.
|
||||
- Bootstrap and recovery remain direct database/control-plane operations, but
|
||||
only through versioned, asserted, auditable procedures rather than arbitrary
|
||||
SQL.
|
||||
- Exact numeric invitation/session/step-up/emergency durations, audit retention
|
||||
and export policy, recovery-key custody rotation, and future multi-party
|
||||
approval remain follow-on operational decisions. They may not weaken the
|
||||
identity separation, last-administrator protection, fail-closed audit, or
|
||||
two-factor recovery invariants fixed here.
|
||||
@@ -0,0 +1,143 @@
|
||||
# ADR 0024: Local Secret Envelope And Connection Resolution
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0021 requires every customer Feishu application and model provider
|
||||
credential to belong to one Organization. It forbids a process-global provider
|
||||
credential and requires business code to obtain plaintext only through a
|
||||
resolver. Before the pilot can store real customer credentials, the deployment
|
||||
also needs a concrete encryption, rotation, recovery, writer-authority, and
|
||||
connection-identity contract.
|
||||
|
||||
The initial production shape is one Hub service and one PostgreSQL database on
|
||||
a controlled host. An external KMS would add another remote availability and
|
||||
bootstrap dependency before the pilot has operational support for it. Keeping
|
||||
plaintext credentials in environment variables, configuration files readable
|
||||
by the service account, database columns, logs, or Agent environments is not an
|
||||
acceptable substitute.
|
||||
|
||||
## Decision
|
||||
|
||||
### Local master-key keyring
|
||||
|
||||
The pilot uses a versioned **local master-key keyring**. Each key is a random
|
||||
256-bit key-encryption key (KEK) with a stable, non-secret key identifier. The
|
||||
keyring names exactly one active KEK and may retain previous KEKs while rotation
|
||||
is in progress. It is not stored in PostgreSQL, a release directory, an
|
||||
environment variable, or the ordinary database/workspace backup set.
|
||||
|
||||
In production, the source keyring is owned by root, mode `0600`, and delivered
|
||||
to the unprivileged Hub service by systemd `LoadCredential`. Hub reads the
|
||||
read-only file under `CREDENTIALS_DIRECTORY`; production startup rejects a
|
||||
direct arbitrary file path. Development and tests may use an explicit local
|
||||
keyring file. Startup fails when the keyring is absent, malformed, contains an
|
||||
invalid key, has no active key, or names an active key that is not present.
|
||||
|
||||
### Envelope format and binding
|
||||
|
||||
Every immutable secret version receives a new random 256-bit data-encryption
|
||||
key (DEK). AES-256-GCM encrypts the secret payload with that DEK. AES-256-GCM
|
||||
also wraps the DEK with the keyring's active KEK. Nonces are random and never
|
||||
reused with the same key.
|
||||
|
||||
The versioned envelope persists only algorithm/version metadata, the KEK
|
||||
identifier, nonces, authentication tags, the wrapped DEK, and ciphertext.
|
||||
Authenticated additional data binds both layers to the secret purpose,
|
||||
Organization identifier, Connection identifier, immutable secret-version
|
||||
identifier, and envelope version. Copying ciphertext between rows,
|
||||
Organizations, connections, purposes, or versions therefore fails
|
||||
authentication. Unknown algorithms or versions, missing KEKs, malformed
|
||||
envelopes, and failed authentication are hard errors; there is no fallback to
|
||||
another credential source.
|
||||
|
||||
The encrypted payload is a versioned object specific to its connection type.
|
||||
For a Provider Connection, provider base URL and authentication material are
|
||||
inside the payload. For a Feishu Application Connection, application ID,
|
||||
application secret, verification/encryption material, and bot identity are
|
||||
inside the payload. HTTP responses, business records, logs, metrics, traces,
|
||||
audit facts, exceptions, and Agent process environments receive only redacted
|
||||
metadata, never those plaintext fields or a decrypted DEK.
|
||||
|
||||
### Connection identity, lifecycle, and writer authority
|
||||
|
||||
A connection has a stable opaque identifier and belongs to exactly one
|
||||
Organization. Secret versions are immutable; the connection points to one
|
||||
active version. Rotation creates and validates a new version, atomically moves
|
||||
the active pointer, and retains the previous encrypted version for rollback and
|
||||
audit until retention policy removes it. A connection is `DRAFT`, `ACTIVE`, or
|
||||
`DISABLED`; runtime resolution accepts only an active connection with a valid
|
||||
active secret version.
|
||||
|
||||
There is one customer-owned Feishu Application Connection per Organization.
|
||||
Its provider-local user, chat, event, and callback identifiers are meaningful
|
||||
only together with that connection identity. Organization OWNER/ADMIN may
|
||||
create and rotate that customer-owned connection. Platform repair remains a
|
||||
separate stepped-up, reason-bearing, fail-closed-audited operation under
|
||||
ADR-0023, not an Organization API bypass.
|
||||
|
||||
A model Provider Connection is unique by Organization and provider identity.
|
||||
In `BYOK` mode an Organization OWNER/ADMIN creates and rotates it. In
|
||||
`PLATFORM_MANAGED` mode only a stepped-up Platform Administrator may create or
|
||||
rotate it, with the platform mutation and redacted Platform Audit Entry in one
|
||||
transaction. Switching modes creates or activates the correctly governed
|
||||
connection version; it never re-labels plaintext or grants the other control
|
||||
plane write authority.
|
||||
|
||||
### Resolver boundary
|
||||
|
||||
Business code asks one connection resolver for an active connection using an
|
||||
explicit Organization, or using a Project that is first resolved to its active
|
||||
Organization. Provider resolution also requires the provider identity; Feishu
|
||||
resolution requires the intended Feishu Connection identity whenever external
|
||||
traffic enters the system. Resolution fails for missing, draft, disabled,
|
||||
cross-Organization, corrupt, or undecryptable records.
|
||||
|
||||
Decrypted values exist only in the resolver result for the duration of the
|
||||
outbound provider/Feishu operation. They are not cached across connections,
|
||||
returned through administrative reads, passed to Agent tools, or written back
|
||||
to business rows. Production has no process-global Feishu or model-provider
|
||||
credential fallback.
|
||||
|
||||
The Claude Agent child never receives an Organization's Provider credential.
|
||||
For each run, Hub opens a loopback-only proxy with a random, short-lived run
|
||||
capability. The child receives only that local endpoint and capability; Hub
|
||||
validates the capability, replaces it with the Organization credential at the
|
||||
upstream hop, refuses automatic redirects, and destroys the listener when the
|
||||
run ends. Agent sandbox credential rules hide even the run capability from
|
||||
tool subprocesses.
|
||||
|
||||
### KEK rotation, backup, and recovery
|
||||
|
||||
KEK rotation is staged: install a new KEK alongside the old keys, mark it
|
||||
active, rewrap each stored DEK under the new KEK without decrypting its payload,
|
||||
verify every envelope, then retire the old KEK only after the separately
|
||||
protected recovery copy and restore drill are current. Secret rotation and KEK
|
||||
rotation are distinct operations.
|
||||
|
||||
A recoverable deployment requires both the ordinary database/workspace backup
|
||||
and a separately protected backup of the local keyring. Neither copy alone can
|
||||
recover plaintext credentials. Restore preflight verifies key identifiers and
|
||||
authenticates sample/all envelopes before traffic is enabled. A missing or
|
||||
incorrect keyring leaves the service unavailable and produces redacted
|
||||
diagnostics; operators must restore the correct keyring rather than reset or
|
||||
silently discard credentials.
|
||||
|
||||
## Consequences
|
||||
|
||||
- The database may be restored or copied without exposing plaintext secrets,
|
||||
but losing the separately held keyring makes encrypted credentials
|
||||
unrecoverable.
|
||||
- The service unit, installer, preflight, backup, restore, and rotation tooling
|
||||
must treat the keyring as an explicit production dependency.
|
||||
- Administrative writes must accept secret material as write-only input and
|
||||
return only connection state, version, timestamps, and redacted fingerprints.
|
||||
- Tests must prove cross-Organization/row substitution failure, wrong/missing
|
||||
key failure, version rotation, absence of global fallback, and absence of
|
||||
plaintext from persistence, responses, logs, and Agent environments.
|
||||
- External KMS/HSM adoption is deferred. It may replace the local KEK provider
|
||||
without changing connection identity, immutable secret versions, AAD
|
||||
binding, writer authority, or resolver behavior.
|
||||
@@ -0,0 +1,57 @@
|
||||
# ADR 0025: Alpha Uses Managed Single-Organization Silos
|
||||
|
||||
## Status
|
||||
|
||||
Accepted for the supervised alpha.
|
||||
|
||||
## Context
|
||||
|
||||
The shared SaaS control plane, durable cross-Organization admission scheduler,
|
||||
and customer administration SPA are not complete. Real customer data must not
|
||||
depend on partially migrated application-level tenant filtering merely to start
|
||||
the supervised alpha.
|
||||
|
||||
## Decision
|
||||
|
||||
Each alpha Organization runs as a separately named Silo deployment. A Silo has
|
||||
one Hub systemd unit and service identity, one logical PostgreSQL database and
|
||||
database role, one workspace root, one environment/session secret, one local
|
||||
master-key keyring, one Feishu Application Connection, and Organization-local
|
||||
Provider Connections. The process is pinned by `HUB_SILO_ORGANIZATION_ID` and
|
||||
refuses startup unless its database contains exactly that one active
|
||||
Organization.
|
||||
|
||||
Application releases remain shared and versioned; customer state does not live
|
||||
inside a release directory. Host-console bootstrap, upgrade, backup, restore,
|
||||
and KEK rotation always name a Silo instance explicitly. The alpha exposes no
|
||||
platform administration surface. Operators provision and support Silos through
|
||||
the controlled host console and retain an external change record.
|
||||
|
||||
The process-local trigger queue is not an accepted-work queue in the Silo
|
||||
alpha. A busy project or instance rejects new work explicitly. Each instance
|
||||
has checked-in application limits and explicit systemd CPU, memory, and process
|
||||
ceilings. Workspace filesystem quotas are a host provisioning prerequisite.
|
||||
|
||||
This topology changes deployment, not the semantic tenant root:
|
||||
`Organization` remains the tenant root and every authorization, secret, audit,
|
||||
Project, Team, and Run remains Organization-scoped. A later pooled or bridge
|
||||
adapter may host multiple Organizations only after ADR-0022's durable admission
|
||||
and platform control requirements are implemented.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Cross-Organization data access is prevented by database credentials and
|
||||
process/filesystem identity in addition to application authorization.
|
||||
- Provisioning, migrations, backups, health, and upgrades are repeated per
|
||||
Silo and therefore must be automated before tenant count grows materially.
|
||||
- Cross-Organization fairness is not applicable inside one Silo; host cgroup
|
||||
ceilings prevent one Silo from exhausting the machine.
|
||||
- Database/workspace backups and the keyring/environment recovery copy are
|
||||
produced to distinct protected destinations and must pass a restore drill.
|
||||
|
||||
## Deferred
|
||||
|
||||
- Shared platform identity and administration from ADR-0023.
|
||||
- Pooled durable fair scheduling from ADR-0022.
|
||||
- Customer org administration SPA and self-service onboarding.
|
||||
- Docker or Kubernetes as an alternative deployment adapter.
|
||||
@@ -0,0 +1,81 @@
|
||||
# para-26071100 飞书应用配置清单
|
||||
|
||||
本文供 `para-26071100` 的飞书企业管理员操作。不要把 App Secret 粘贴到群聊、工单或本文档中;请通过约定的安全渠道交给平台部署人员。
|
||||
|
||||
## 1. 创建企业自建应用
|
||||
|
||||
1. 打开飞书开放平台开发者后台。
|
||||
2. 在目标企业下创建“企业自建应用”。
|
||||
3. 应用名称可填写 `Educraft para-26071100`。
|
||||
4. 在“凭证与基础信息”记录:
|
||||
- App ID(通常以 `cli_` 开头)
|
||||
- App Secret
|
||||
5. 添加并启用“机器人”能力。
|
||||
|
||||
Bot Open ID 不需要管理员手工寻找。平台部署人员会使用 App ID/App Secret 调用飞书 Bot Info API 获取,并在 bootstrap 时校验它确实属于这一个应用。
|
||||
|
||||
## 2. 开通权限
|
||||
|
||||
在“权限管理”中搜索并申请下列能力。飞书控制台的中文名称可能随版本调整;如控制台同时显示 scope,可优先核对括号中的 scope。
|
||||
|
||||
- 接收群聊中 @ 机器人的消息(`im:message.group_at_msg:readonly`)
|
||||
- 以应用身份发送消息(`im:message:send_as_bot`)
|
||||
- 获取消息内容,用于读取触发消息和线程上下文(`im:message:readonly`)
|
||||
- 获取与上传图片或文件资源(`im:resource`)
|
||||
- 添加、删除消息表情回复(`im:message.reactions:write_only`)
|
||||
- 获取用户基本信息(`contact:user.base:readonly`、`contact:user.basic_profile:readonly`)
|
||||
|
||||
如果飞书 API 调试台提示某个上述操作缺少更细粒度权限,请把提示截图交给平台部署人员,不要直接勾选通讯录全量读取或其他超出清单的权限。
|
||||
|
||||
## 3. 配置事件与卡片回调
|
||||
|
||||
1. 进入“事件与回调”。
|
||||
2. 订阅方式选择“使用长连接接收事件”。
|
||||
3. 添加事件 `im.message.receive_v1`(接收消息)。
|
||||
4. 启用卡片交互回调 `card.action.trigger`,用于审批、中断运行和群聊建项目按钮。
|
||||
5. 不需要填写公网 Event Callback URL;Hub 使用飞书长连接。
|
||||
|
||||
## 4. 配置 OAuth 回调
|
||||
|
||||
域名 DNS 和 TLS 生效后,在安全设置/重定向 URL 中添加:
|
||||
|
||||
```text
|
||||
https://para-26071100.educraft.paradigm-edu.net/auth/feishu/callback
|
||||
```
|
||||
|
||||
该回调用于 OWNER 登录受控的 Host Console。飞书群机器人长连接本身不依赖这个 URL。
|
||||
|
||||
## 5. 发布并安装应用
|
||||
|
||||
1. 创建应用版本并提交企业管理员审核。
|
||||
2. 将应用可用范围至少包含试点 OWNER 和试点群成员。
|
||||
3. 发布版本。
|
||||
4. 将机器人加入准备试用的飞书群。
|
||||
|
||||
## 6. 获取首位 OWNER 身份
|
||||
|
||||
平台 bootstrap 需要 OWNER 的飞书 Open ID 和显示名称。可通过飞书 API 调试台的用户信息接口查询;Open ID 通常以 `ou_` 开头。Union ID 可选,不影响首次部署。
|
||||
|
||||
请把以下结果通过安全渠道交给平台部署人员:
|
||||
|
||||
```text
|
||||
Organization: para-26071100
|
||||
App ID: cli_...
|
||||
App Secret: (安全渠道发送)
|
||||
OWNER Open ID: ou_...
|
||||
OWNER 显示名称:
|
||||
OWNER Union ID: (可选)
|
||||
试点群名称: (可选,便于验收)
|
||||
```
|
||||
|
||||
## 7. 验收动作
|
||||
|
||||
平台通知部署完成后:
|
||||
|
||||
1. OWNER 打开 Host Console,完成飞书 OAuth 登录。
|
||||
2. 在试点群中 @机器人发送一条纯文本消息。
|
||||
3. 如果群尚未绑定项目,机器人应返回项目创建/绑定卡片。
|
||||
4. 创建项目后再次 @机器人,确认出现处理状态、流式卡片和最终回答。
|
||||
5. 再测试一个小文件附件,以及运行中断按钮。
|
||||
|
||||
任何一步失败时,请保留发生时间、群名、消息截图和飞书 request/log ID;不要在截图中包含 App Secret 或 Provider token。
|
||||
@@ -0,0 +1 @@
|
||||
Hello, World!
|
||||
@@ -0,0 +1,68 @@
|
||||
# Hub runtime configuration. Copy to .env and fill in.
|
||||
|
||||
# Production preflight requires the explicit production runtime mode.
|
||||
NODE_ENV="production"
|
||||
|
||||
# PostgreSQL connection string.
|
||||
DATABASE_URL="postgresql://paradigm:paradigm@127.0.0.1:5432/paradigm"
|
||||
|
||||
# Provider URL and credentials are write-only Organization Provider Connection
|
||||
# data in PostgreSQL, encrypted by ADR-0024. Process-global ANTHROPIC_* provider
|
||||
# settings are rejected in production and must not be added here.
|
||||
|
||||
# Model override (optional). Defaults to anthropic/claude-sonnet-5.
|
||||
# Use an OpenRouter model slug, for example:
|
||||
# ANTHROPIC_DEFAULT_SONNET_MODEL="~anthropic/claude-sonnet-latest"
|
||||
# To use GLM:
|
||||
# ANTHROPIC_DEFAULT_SONNET_MODEL="z-ai/glm-4.7"
|
||||
# ANTHROPIC_DEFAULT_OPUS_MODEL="z-ai/glm-4.7"
|
||||
# ANTHROPIC_DEFAULT_HAIKU_MODEL="z-ai/glm-4.6"
|
||||
|
||||
# Alpha Silo safety limits. max turns may use its default; every other value is
|
||||
# mandatory in production and should be calibrated on the target host.
|
||||
# HUB_AGENT_MAX_TURNS=25
|
||||
HUB_AGENT_MAX_CONCURRENT_RUNS="1"
|
||||
HUB_AGENT_MAX_RUN_SECONDS="900"
|
||||
HUB_HTTP_BODY_LIMIT_BYTES="1048576"
|
||||
HUB_MAX_FILES_PER_MESSAGE="8"
|
||||
HUB_MAX_FILE_BYTES="26214400"
|
||||
HUB_HTTP_REQUESTS_PER_MINUTE="120"
|
||||
HUB_FEISHU_EVENTS_PER_MINUTE="120"
|
||||
|
||||
# Persistent system-managed root for project workspaces. Production must use an
|
||||
# absolute path outside the deployment/release tree; install_service.sh defaults
|
||||
# to this path and rejects any overlap before installing the unit.
|
||||
HUB_PROJECT_WORKSPACE_ROOT="/var/lib/cph-hub/workspaces"
|
||||
|
||||
# This process is pinned to exactly one Organization. Feishu credentials are
|
||||
# resolved from that Organization's encrypted ACTIVE connection.
|
||||
HUB_SILO_ORGANIZATION_ID=""
|
||||
HUB_SYSTEMD_UNIT="cph-hub-example.service"
|
||||
|
||||
# Absolute path to the `cph` binary (ADR-0016). Production preflight requires
|
||||
# the file to be executable and `cph --version` to succeed.
|
||||
CPH_BIN="/usr/local/bin/cph"
|
||||
|
||||
# Hub bind address and port. Production defaults to loopback for a local TLS
|
||||
# reverse proxy; both values are validated and honored by the HTTP server.
|
||||
HOST="127.0.0.1"
|
||||
PORT=8788
|
||||
|
||||
# --- Org admin web (ADR-0021) ---------------------------------------------
|
||||
# Public base URL of this Hub (no trailing slash). Used for Feishu OAuth
|
||||
# redirect_uri = ${HUB_PUBLIC_BASE_URL}/auth/feishu/callback
|
||||
# Configure the same callback URL in the Feishu developer console under
|
||||
# Security Settings → Redirect URLs.
|
||||
HUB_PUBLIC_BASE_URL="https://hub.example.com"
|
||||
|
||||
# HMAC secret for signed session + OAuth state cookies. Generate with:
|
||||
# openssl rand -base64 32
|
||||
HUB_SESSION_SECRET=""
|
||||
|
||||
# Optional OAuth scope (space-separated). Default: contact:user.base:readonly
|
||||
# Apply matching scopes in the Feishu developer console.
|
||||
# HUB_OAUTH_SCOPE="contact:user.base:readonly"
|
||||
|
||||
# Local development/tests only. Production must not set this: systemd injects
|
||||
# the fixed cph-secret-keyring credential through CREDENTIALS_DIRECTORY.
|
||||
# HUB_SECRET_KEYRING_FILE="/absolute/path/to/dev-keyring.json"
|
||||
@@ -0,0 +1,6 @@
|
||||
node_modules/
|
||||
dist/
|
||||
*.log
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
@@ -0,0 +1,188 @@
|
||||
# Alpha Silo service installation
|
||||
|
||||
The supervised alpha runs one Organization per named Silo. The supported host
|
||||
has systemd, PostgreSQL, Node.js 24+, `pg_isready`, `runuser`, `setpriv`,
|
||||
bubblewrap, `socat`, `pg_dump`, `tar`, `sha256sum`, and a compatible `cph`.
|
||||
|
||||
Each Silo needs its own logical database/role, instance id, service user,
|
||||
environment, keyring, workspace, port, domain, and host filesystem quota.
|
||||
Application code lives in immutable versioned directories under
|
||||
`/srv/curriculum-project-hub/releases/`; releases contain no customer state.
|
||||
|
||||
## Install an instance
|
||||
|
||||
```sh
|
||||
sudo BASE=/srv/curriculum-project-hub \
|
||||
HUB_DIR=/srv/curriculum-project-hub/releases/<release-id>/hub \
|
||||
INSTANCE_ID=org-a \
|
||||
WORKSPACE_ROOT=/w/997 \
|
||||
PORT=8788 \
|
||||
MEMORY_MAX=16G CPU_QUOTA=400% TASKS_MAX=512 \
|
||||
bash /srv/curriculum-project-hub/releases/<release-id>/hub/deploy/install_service.sh
|
||||
```
|
||||
|
||||
The first invocation creates root-owned `0600` files below
|
||||
`/srv/curriculum-project-hub/.secrets/org-a/` and exits 78. Copy the keyring to
|
||||
a separately protected recovery destination and fill every blank in
|
||||
`platform.env`. Before rerunning the installer, migrate the empty Silo database:
|
||||
|
||||
```sh
|
||||
sudo bash -c '
|
||||
set -euo pipefail
|
||||
set -a; . /srv/curriculum-project-hub/.secrets/org-a/platform.env; set +a
|
||||
node /srv/curriculum-project-hub/releases/<release-id>/hub/node_modules/prisma/build/index.js \
|
||||
migrate deploy \
|
||||
--schema /srv/curriculum-project-hub/releases/<release-id>/hub/prisma/schema.prisma
|
||||
'
|
||||
```
|
||||
|
||||
Then rerun the installer so it can complete preflight and install the stopped
|
||||
unit. Resource numbers above are examples, not defaults; the operator must set
|
||||
measured ceilings explicitly.
|
||||
|
||||
The installer refuses overlapping release/persistent paths, root service users,
|
||||
missing Node/cph/PostgreSQL/bubblewrap prerequisites, invalid credentials, and
|
||||
unreachable database roles. The installed unit is `cph-hub-org-a.service` and
|
||||
uses systemd `LoadCredential`; the service user cannot read the source keyring.
|
||||
|
||||
Default state paths are:
|
||||
|
||||
```text
|
||||
/var/lib/cph-hub/org-a/home
|
||||
/var/lib/cph-hub/org-a/state
|
||||
/w/997
|
||||
/var/cache/cph-hub/org-a
|
||||
```
|
||||
|
||||
Organization Agent roles and skills are runtime configuration. Skill versions
|
||||
are stored below the Silo state directory (`state/skills`) and are included in
|
||||
`backup_silo.sh` as `agent-skills.tar`; PostgreSQL stores role bundles, skill
|
||||
metadata and role-to-skill selection. Operate them as the Silo service user so
|
||||
content ownership remains correct:
|
||||
|
||||
```sh
|
||||
sudo INSTANCE_ID=org-a \
|
||||
ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
|
||||
bash hub/deploy/agent_config.sh install-skill \
|
||||
--organization org-a --source /staging/typst --version 1
|
||||
sudo INSTANCE_ID=org-a \
|
||||
ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
|
||||
bash hub/deploy/agent_config.sh upsert-role \
|
||||
--organization org-a --role draft --label 草稿 --tools-json null
|
||||
sudo INSTANCE_ID=org-a \
|
||||
ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
|
||||
bash hub/deploy/agent_config.sh set-role-skills \
|
||||
--organization org-a --role draft --skills outline,lesson-project,typst
|
||||
sudo INSTANCE_ID=org-a \
|
||||
ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
|
||||
bash hub/deploy/agent_config.sh list --organization org-a
|
||||
```
|
||||
|
||||
`--tools-json null` means the full registered tool surface; `[]` means no
|
||||
ordinary tools. SDK-bundled skills and workspace/user setting sources remain
|
||||
disabled regardless of runtime configuration.
|
||||
|
||||
## Bootstrap the only Organization
|
||||
|
||||
Prepare a root-owned `0600` JSON file containing
|
||||
`organization`, `owner`, `feishu`, `provider`, and optionally `teams`. Secrets
|
||||
never appear in command-line arguments.
|
||||
|
||||
```json
|
||||
{
|
||||
"organization": { "id": "org_a", "slug": "org-a", "name": "Org A" },
|
||||
"owner": { "openId": "ou_owner", "displayName": "Owner" },
|
||||
"feishu": {
|
||||
"appId": "cli_xxx",
|
||||
"appSecret": "...",
|
||||
"botOpenId": "ou_bot"
|
||||
},
|
||||
"provider": {
|
||||
"providerId": "openrouter",
|
||||
"baseUrl": "https://openrouter.ai/api",
|
||||
"authToken": "..."
|
||||
},
|
||||
"teams": [{ "slug": "teachers", "name": "Teachers" }]
|
||||
}
|
||||
```
|
||||
|
||||
```sh
|
||||
sudo bash -c '
|
||||
set -euo pipefail
|
||||
set -a; . /srv/curriculum-project-hub/.secrets/org-a/platform.env; set +a
|
||||
node /srv/curriculum-project-hub/releases/<release-id>/hub/dist/deployment/bootstrap-silo-cli.js \
|
||||
--config-file /root/org-a-bootstrap.json \
|
||||
--keyring-file /srv/curriculum-project-hub/.secrets/org-a/secret-keyring.json
|
||||
'
|
||||
```
|
||||
|
||||
Bootstrap encrypts and activates both connection records and is rerunnable when
|
||||
provider configuration fails after Organization creation. Hub startup refuses
|
||||
to become healthy unless the sole Organization, active provider, active Feishu
|
||||
application, and Feishu listener are ready. Delete the bootstrap input after
|
||||
the recovery copy is current. Production has no process-global Feishu/provider
|
||||
credential fallback.
|
||||
|
||||
## Start and rotate
|
||||
|
||||
```sh
|
||||
sudo systemctl start cph-hub-org-a.service
|
||||
sudo systemctl status cph-hub-org-a.service
|
||||
```
|
||||
|
||||
KEK rotation requires the named unit to be stopped. Source its instance env so
|
||||
`HUB_SYSTEMD_UNIT` and `DATABASE_URL` identify the same Silo:
|
||||
|
||||
```sh
|
||||
sudo bash -c '
|
||||
set -euo pipefail
|
||||
systemctl stop cph-hub-org-a.service
|
||||
set -a; . /srv/curriculum-project-hub/.secrets/org-a/platform.env; set +a
|
||||
node /srv/curriculum-project-hub/hub/dist/deployment/rotate-secret-kek.js \
|
||||
--keyring-file /srv/curriculum-project-hub/.secrets/org-a/secret-keyring.json
|
||||
systemctl start cph-hub-org-a.service
|
||||
'
|
||||
```
|
||||
|
||||
## Backup and restore drill (recommended after the demo is running)
|
||||
|
||||
Stop the unit and run `backup_silo.sh` with distinct root-owned destinations:
|
||||
|
||||
```sh
|
||||
sudo INSTANCE_ID=org-a \
|
||||
ENV_FILE=/srv/curriculum-project-hub/.secrets/org-a/platform.env \
|
||||
KEYRING_FILE=/srv/curriculum-project-hub/.secrets/org-a/secret-keyring.json \
|
||||
BUSINESS_BACKUP_DIR=/backup/business \
|
||||
RECOVERY_BACKUP_DIR=/separate-recovery \
|
||||
bash hub/deploy/backup_silo.sh
|
||||
```
|
||||
|
||||
The business set contains the PostgreSQL custom dump, workspace archive and
|
||||
`agent-skills.tar`. The
|
||||
separate recovery set contains the keyring and environment. Both include
|
||||
checksums; neither destination may be the live host's only disk.
|
||||
|
||||
Restore into a separate drill database, workspace and skill-store directory;
|
||||
verify checksums before extracting both tar archives, then run:
|
||||
|
||||
```sh
|
||||
set -a; . /path/to/restored/platform.env; set +a
|
||||
mkdir -p "$HUB_PROJECT_WORKSPACE_ROOT" "$HUB_SKILL_STORE_ROOT"
|
||||
tar -xf /path/to/business/workspaces.tar -C "$HUB_PROJECT_WORKSPACE_ROOT"
|
||||
tar -xf /path/to/business/agent-skills.tar -C "$HUB_SKILL_STORE_ROOT"
|
||||
node hub/dist/deployment/restore-preflight.js \
|
||||
--keyring-file /path/to/restored/secret-keyring.json
|
||||
sudo INSTANCE_ID=org-a ENV_FILE=/path/to/restored/platform.env \
|
||||
HUB_DIR=/path/to/restored/release/hub \
|
||||
bash hub/deploy/agent_config.sh verify-store --organization org-a
|
||||
```
|
||||
|
||||
Traffic stays disabled until the sole Organization and every Feishu/provider
|
||||
envelope authenticate, the workspace exists, and an end-to-end test run passes.
|
||||
|
||||
For the first supervised demo, install → bootstrap → start → health check is the
|
||||
deployment gate. A completed off-host restore drill is an Alpha hardening item,
|
||||
not a prerequisite for bringing up that first controlled Silo.
|
||||
|
||||
The default bind remains loopback; expose it only through a TLS reverse proxy.
|
||||
The platform admin surface is not part of the alpha and must not be exposed.
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
INSTANCE_ID="${INSTANCE_ID:?INSTANCE_ID required}"
|
||||
ENV_FILE="${ENV_FILE:?ENV_FILE required}"
|
||||
SERVICE_USER="${SERVICE_USER:-cph-$INSTANCE_ID}"
|
||||
HUB_DIR="${HUB_DIR:-/srv/curriculum-project-hub/current/hub}"
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || { echo "agent config console must run as root" >&2; exit 1; }
|
||||
[ -r "$ENV_FILE" ] || { echo "environment file is not readable: $ENV_FILE" >&2; exit 1; }
|
||||
[ -f "$HUB_DIR/dist/deployment/agent-config-cli.js" ] || { echo "Agent config CLI missing below $HUB_DIR" >&2; exit 1; }
|
||||
id "$SERVICE_USER" >/dev/null 2>&1 || { echo "service user missing: $SERVICE_USER" >&2; exit 1; }
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
: "${DATABASE_URL:?DATABASE_URL missing from ENV_FILE}"
|
||||
: "${HUB_SILO_ORGANIZATION_ID:?HUB_SILO_ORGANIZATION_ID missing from ENV_FILE}"
|
||||
|
||||
exec runuser --user "$SERVICE_USER" -- \
|
||||
env -i \
|
||||
DATABASE_URL="$DATABASE_URL" \
|
||||
HUB_SILO_ORGANIZATION_ID="$HUB_SILO_ORGANIZATION_ID" \
|
||||
HUB_SKILL_STORE_ROOT="${HUB_SKILL_STORE_ROOT:-/var/lib/cph-hub/$INSTANCE_ID/state/skills}" \
|
||||
XDG_STATE_HOME="${XDG_STATE_HOME:-/var/lib/cph-hub/$INSTANCE_ID/state}" \
|
||||
PATH="${PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}" \
|
||||
node "$HUB_DIR/dist/deployment/agent-config-cli.js" "$@"
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create a stopped-instance backup. Business data and recovery credentials are
|
||||
# deliberately written to different root-owned destinations.
|
||||
set -euo pipefail
|
||||
|
||||
INSTANCE_ID="${INSTANCE_ID:?INSTANCE_ID required}"
|
||||
ENV_FILE="${ENV_FILE:?ENV_FILE required}"
|
||||
KEYRING_FILE="${KEYRING_FILE:?KEYRING_FILE required}"
|
||||
BUSINESS_BACKUP_DIR="${BUSINESS_BACKUP_DIR:?BUSINESS_BACKUP_DIR required}"
|
||||
RECOVERY_BACKUP_DIR="${RECOVERY_BACKUP_DIR:?RECOVERY_BACKUP_DIR required}"
|
||||
SERVICE_UNIT="cph-hub-$INSTANCE_ID.service"
|
||||
SKILL_STORE_ROOT="${SKILL_STORE_ROOT:-/var/lib/cph-hub/$INSTANCE_ID/state/skills}"
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || { echo "backup must run as root" >&2; exit 1; }
|
||||
umask 077
|
||||
|
||||
validate_root_secret_file() {
|
||||
local label="$1" path="$2" metadata
|
||||
[[ "$path" = /* ]] || { echo "$label must be an absolute path" >&2; exit 1; }
|
||||
[ ! -L "$path" ] || { echo "$label must not be a symlink: $path" >&2; exit 1; }
|
||||
[ -f "$path" ] || { echo "$label must be a regular file: $path" >&2; exit 1; }
|
||||
metadata="$(stat -c '%u:%g:%a' -- "$path")"
|
||||
[ "$metadata" = "0:0:600" ] || {
|
||||
echo "$label must be root:root mode 0600; found $metadata" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
validate_root_secret_file "ENV_FILE" "$ENV_FILE"
|
||||
validate_root_secret_file "KEYRING_FILE" "$KEYRING_FILE"
|
||||
state="$(systemctl show --property=ActiveState --value "$SERVICE_UNIT")"
|
||||
case "$state" in inactive|failed) ;; *) echo "$SERVICE_UNIT must be stopped; state=$state" >&2; exit 1;; esac
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
: "${DATABASE_URL:?DATABASE_URL missing from ENV_FILE}"
|
||||
: "${HUB_PROJECT_WORKSPACE_ROOT:?HUB_PROJECT_WORKSPACE_ROOT missing from ENV_FILE}"
|
||||
[ -d "$HUB_PROJECT_WORKSPACE_ROOT" ] || { echo "workspace root missing" >&2; exit 1; }
|
||||
[ -d "$SKILL_STORE_ROOT" ] || { echo "skill store root missing" >&2; exit 1; }
|
||||
|
||||
install -d -o root -g root -m 0700 "$BUSINESS_BACKUP_DIR" "$RECOVERY_BACKUP_DIR"
|
||||
business_root="$(realpath -m "$BUSINESS_BACKUP_DIR")"
|
||||
recovery_root="$(realpath -m "$RECOVERY_BACKUP_DIR")"
|
||||
workspace_root="$(realpath -m "$HUB_PROJECT_WORKSPACE_ROOT")"
|
||||
secret_root="$(realpath -m "$(dirname "$KEYRING_FILE")")"
|
||||
skill_root="$(realpath -m "$SKILL_STORE_ROOT")"
|
||||
paths_overlap() {
|
||||
local left="$1" right="$2"
|
||||
[ "$left" = "$right" ] || [[ "$left/" == "$right/"* ]] || [[ "$right/" == "$left/"* ]]
|
||||
}
|
||||
for pair in \
|
||||
"$business_root|$recovery_root" \
|
||||
"$business_root|$workspace_root" \
|
||||
"$recovery_root|$workspace_root" \
|
||||
"$recovery_root|$secret_root"; do
|
||||
left="${pair%%|*}"; right="${pair#*|}"
|
||||
if paths_overlap "$left" "$right"; then
|
||||
echo "backup destinations must not overlap each other or live state: $left <> $right" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if paths_overlap "$business_root" "$skill_root" || paths_overlap "$recovery_root" "$skill_root" || paths_overlap "$workspace_root" "$skill_root"; then
|
||||
echo "backup destinations, workspace and skill store must not overlap: $skill_root" >&2
|
||||
exit 1
|
||||
fi
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
business="$business_root/$INSTANCE_ID-$stamp"
|
||||
recovery="$recovery_root/$INSTANCE_ID-$stamp"
|
||||
install -d -o root -g root -m 0700 "$business" "$recovery"
|
||||
|
||||
pg_dump --format=custom --file="$business/database.dump" "$DATABASE_URL"
|
||||
tar --create --file="$business/workspaces.tar" --directory="$HUB_PROJECT_WORKSPACE_ROOT" .
|
||||
tar --create --file="$business/agent-skills.tar" --directory="$SKILL_STORE_ROOT" .
|
||||
cp --preserve=mode,ownership,timestamps "$KEYRING_FILE" "$recovery/secret-keyring.json"
|
||||
cp --preserve=mode,ownership,timestamps "$ENV_FILE" "$recovery/platform.env"
|
||||
chmod 0600 "$recovery/secret-keyring.json" "$recovery/platform.env"
|
||||
|
||||
(
|
||||
cd "$business"
|
||||
sha256sum database.dump workspaces.tar agent-skills.tar > SHA256SUMS
|
||||
)
|
||||
(
|
||||
cd "$recovery"
|
||||
sha256sum secret-keyring.json platform.env > SHA256SUMS
|
||||
)
|
||||
echo "business backup: $business"
|
||||
echo "separate recovery backup: $recovery"
|
||||
@@ -0,0 +1,46 @@
|
||||
[Unit]
|
||||
Description=Curriculum Project Hub Silo (__INSTANCE_ID__)
|
||||
After=network-online.target postgresql.service
|
||||
Wants=network-online.target
|
||||
# ADR-0018: refuse to start when the configured OS sandbox binary disappears
|
||||
# after installation. Deployment preflight validates the executable itself;
|
||||
# this start-time assertion keeps later package/host changes fail-closed.
|
||||
AssertPathExists=__BWRAP_BIN__
|
||||
AssertPathExists=__SOCAT_BIN__
|
||||
AssertPathExists=__KEYRING_FILE__
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=__SERVICE_USER__
|
||||
Group=__SERVICE_GROUP__
|
||||
WorkingDirectory=__HUB_DIR__
|
||||
EnvironmentFile=__ENV_FILE__
|
||||
Environment=HOME=__SERVICE_HOME__
|
||||
Environment=XDG_STATE_HOME=__STATE_DIR__
|
||||
Environment=XDG_CACHE_HOME=__CACHE_DIR__
|
||||
Environment=HUB_SKILL_STORE_ROOT=__SKILL_STORE_ROOT__
|
||||
Environment=PATH=__RUNTIME_PATH__
|
||||
# ADR-0024: the root-owned source remains unreadable by the service account;
|
||||
# systemd materializes a read-only per-unit credential at runtime.
|
||||
LoadCredential=cph-secret-keyring:__KEYRING_FILE__
|
||||
UMask=0027
|
||||
# Run prisma migrations before each start, then launch the Hub.
|
||||
ExecStartPre=__NODE_BIN__ __HUB_DIR__/node_modules/prisma/build/index.js migrate deploy --schema __HUB_DIR__/prisma/schema.prisma
|
||||
ExecStart=__NODE_BIN__ __HUB_DIR__/dist/server.js
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
MemoryMax=__MEMORY_MAX__
|
||||
CPUQuota=__CPU_QUOTA__
|
||||
TasksMax=__TASKS_MAX__
|
||||
OOMPolicy=stop
|
||||
# Graceful shutdown: lark ws close + in-flight runs.
|
||||
KillSignal=SIGTERM
|
||||
TimeoutStopSec=30
|
||||
# Hardening: no new privileges. ProtectSystem/ProtectHome are NOT set here —
|
||||
# they would break cph's render-cache extraction (~/.cache/cph/) and the
|
||||
# bwrap user-namespace setup. The OS-level sandbox (ADR-0018) is the hard
|
||||
# boundary; systemd hardening is kept minimal to avoid interfering with it.
|
||||
NoNewPrivileges=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deploy the Hub to a host: rsync the repo, install deps, build, restart service.
|
||||
#
|
||||
# Configure these env vars (e.g. in CI secrets):
|
||||
# PLATFORM_DEPLOY_HOST required
|
||||
# PLATFORM_DEPLOY_SSH_KEY required (private key for the deploy user)
|
||||
# PLATFORM_DEPLOY_USER optional, defaults to deploy
|
||||
# PLATFORM_DEPLOY_PORT optional SSH port, defaults to 22
|
||||
# PLATFORM_DEPLOY_HUB_PORT required Silo listener port
|
||||
# PLATFORM_DEPLOY_BASE optional, defaults to /srv/curriculum-project-hub
|
||||
# PLATFORM_DEPLOY_RELEASE optional immutable release id, defaults to git HEAD
|
||||
# PLATFORM_DEPLOY_INSTANCE required, Silo instance id
|
||||
# PLATFORM_DEPLOY_WORKSPACE_ROOT required short per-Silo path (for example /w/997)
|
||||
# PLATFORM_DEPLOY_MEMORY_MAX / CPU_QUOTA / TASKS_MAX required ceilings
|
||||
# PLATFORM_DEPLOY_HEALTH_URL optional, defaults to http://127.0.0.1:8788/api/healthz
|
||||
# The target host must have Node.js 24+, npm, rsync, PostgreSQL client tools
|
||||
# (`pg_isready`), systemd, bubblewrap (`bwrap`) and `socat` (for the ADR-0018 agent sandbox),
|
||||
# and a compatible `cph` binary named by platform.env. The service installer
|
||||
# provisions the dedicated non-root identity and persistent directories.
|
||||
# Use a dedicated `deploy` user that has passwordless sudo for systemctl and
|
||||
# writing the named /etc/systemd/system/cph-hub-<instance>.service through the installer.
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${PLATFORM_DEPLOY_HOST:?PLATFORM_DEPLOY_HOST required}"
|
||||
SSH_KEY="${PLATFORM_DEPLOY_SSH_KEY:?PLATFORM_DEPLOY_SSH_KEY required}"
|
||||
DEPLOY_USER="${PLATFORM_DEPLOY_USER:-deploy}"
|
||||
PORT="${PLATFORM_DEPLOY_PORT:-22}"
|
||||
HUB_PORT="${PLATFORM_DEPLOY_HUB_PORT:?PLATFORM_DEPLOY_HUB_PORT required}"
|
||||
BASE="${PLATFORM_DEPLOY_BASE:-/srv/curriculum-project-hub}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
RELEASE_ID="${PLATFORM_DEPLOY_RELEASE:-$(git -C "$REPO_ROOT" rev-parse --verify HEAD)}"
|
||||
[[ "$RELEASE_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { echo "invalid PLATFORM_DEPLOY_RELEASE" >&2; exit 1; }
|
||||
INSTANCE_ID="${PLATFORM_DEPLOY_INSTANCE:?PLATFORM_DEPLOY_INSTANCE required}"
|
||||
WORKSPACE_ROOT="${PLATFORM_DEPLOY_WORKSPACE_ROOT:?PLATFORM_DEPLOY_WORKSPACE_ROOT required}"
|
||||
SERVICE_UNIT="cph-hub-$INSTANCE_ID.service"
|
||||
MEMORY_MAX="${PLATFORM_DEPLOY_MEMORY_MAX:?PLATFORM_DEPLOY_MEMORY_MAX required}"
|
||||
CPU_QUOTA="${PLATFORM_DEPLOY_CPU_QUOTA:?PLATFORM_DEPLOY_CPU_QUOTA required}"
|
||||
TASKS_MAX="${PLATFORM_DEPLOY_TASKS_MAX:?PLATFORM_DEPLOY_TASKS_MAX required}"
|
||||
HEALTH_URL="${PLATFORM_DEPLOY_HEALTH_URL:-http://127.0.0.1:$HUB_PORT/api/healthz}"
|
||||
HUB_DIR="$BASE/releases/$RELEASE_ID/hub"
|
||||
RELEASE_DIR="$BASE/releases/$RELEASE_ID"
|
||||
|
||||
SSH_OPTS=(-i "$SSH_KEY" -p "$PORT" -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
release_ready=false
|
||||
if ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "test -f '$RELEASE_DIR/.complete'"; then
|
||||
release_ready=true
|
||||
elif ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "test -e '$RELEASE_DIR'"; then
|
||||
echo "incomplete release already exists: $RELEASE_DIR" >&2
|
||||
exit 1
|
||||
else
|
||||
ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "mkdir -p '$HUB_DIR'"
|
||||
fi
|
||||
|
||||
if [ "$release_ready" = false ]; then
|
||||
# 1. Populate a new release directory. A completed release is never mutated.
|
||||
rsync -az --delete \
|
||||
--exclude node_modules --exclude dist --exclude .env \
|
||||
-e "ssh ${SSH_OPTS[*]}" \
|
||||
"$REPO_ROOT/hub/" "$DEPLOY_USER@$HOST:$HUB_DIR/"
|
||||
|
||||
# 2. Install deps, audit and build, then atomically mark the release complete.
|
||||
ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" \
|
||||
"cd '$HUB_DIR' && npm ci && npm run audit:production && npm run build && touch '$RELEASE_DIR/.complete'"
|
||||
fi
|
||||
|
||||
# 3. Ensure the service is installed (idempotent), then restart.
|
||||
ssh "${SSH_OPTS[@]}" "$DEPLOY_USER@$HOST" "
|
||||
set -euo pipefail
|
||||
if [ \"\$(id -u)\" = \"0\" ]; then
|
||||
BASE='$BASE' HUB_DIR='$HUB_DIR' INSTANCE_ID='$INSTANCE_ID' WORKSPACE_ROOT='$WORKSPACE_ROOT' PORT='$HUB_PORT' MEMORY_MAX='$MEMORY_MAX' CPU_QUOTA='$CPU_QUOTA' TASKS_MAX='$TASKS_MAX' bash '$HUB_DIR/deploy/install_service.sh'
|
||||
systemctl restart '$SERVICE_UNIT'
|
||||
systemctl is-active --quiet '$SERVICE_UNIT'
|
||||
else
|
||||
sudo -n BASE='$BASE' HUB_DIR='$HUB_DIR' INSTANCE_ID='$INSTANCE_ID' WORKSPACE_ROOT='$WORKSPACE_ROOT' PORT='$HUB_PORT' MEMORY_MAX='$MEMORY_MAX' CPU_QUOTA='$CPU_QUOTA' TASKS_MAX='$TASKS_MAX' bash '$HUB_DIR/deploy/install_service.sh'
|
||||
sudo -n systemctl restart '$SERVICE_UNIT'
|
||||
sudo -n systemctl is-active --quiet '$SERVICE_UNIT'
|
||||
fi
|
||||
for attempt in \$(seq 1 30); do
|
||||
if curl --fail --silent '$HEALTH_URL' >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
curl --fail --silent --show-error '$HEALTH_URL' >/dev/null
|
||||
"
|
||||
|
||||
echo "Deployed cph-hub to $HOST ($HUB_DIR)"
|
||||
Executable
+323
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env bash
|
||||
# Provision and install the cph-hub systemd service on a supported Linux host.
|
||||
# The script is idempotent: existing identities/directories are validated and
|
||||
# never silently rewritten. An absent environment file is seeded completely,
|
||||
# then the script exits EX_CONFIG so no unit is installed with placeholders.
|
||||
#
|
||||
# Usage:
|
||||
# sudo BASE=/srv/curriculum-project-hub \
|
||||
# HUB_DIR=/srv/curriculum-project-hub/hub \
|
||||
# bash deploy/install_service.sh
|
||||
set -euo pipefail
|
||||
|
||||
BASE="${BASE:-/srv/curriculum-project-hub}"
|
||||
HUB_DIR="${HUB_DIR:-$BASE/hub}"
|
||||
INSTANCE_ID="${INSTANCE_ID:?INSTANCE_ID is required (lowercase letters, digits, hyphens)}"
|
||||
if [[ ! "$INSTANCE_ID" =~ ^[a-z0-9]([a-z0-9-]{0,22}[a-z0-9])?$ ]]; then
|
||||
echo "[install] error: invalid INSTANCE_ID: $INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
SERVICE_UNIT="cph-hub-$INSTANCE_ID.service"
|
||||
SERVICE_USER="${SERVICE_USER:-cph-$INSTANCE_ID}"
|
||||
SERVICE_GROUP="${SERVICE_GROUP:-$SERVICE_USER}"
|
||||
SERVICE_HOME="${SERVICE_HOME:-/var/lib/cph-hub/$INSTANCE_ID/home}"
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/cph-hub/$INSTANCE_ID/state}"
|
||||
CACHE_DIR="${CACHE_DIR:-/var/cache/cph-hub/$INSTANCE_ID}"
|
||||
SKILL_STORE_ROOT="${SKILL_STORE_ROOT:-$STATE_DIR/skills}"
|
||||
WORKSPACE_ROOT="${WORKSPACE_ROOT:?WORKSPACE_ROOT required (use a short per-Silo path such as /w/997)}"
|
||||
HOST="${HOST:-127.0.0.1}"
|
||||
PORT="${PORT:?PORT is required and must be unique on the host}"
|
||||
ENV_FILE="${ENV_FILE:-$BASE/.secrets/$INSTANCE_ID/platform.env}"
|
||||
KEYRING_FILE="${KEYRING_FILE:-$BASE/.secrets/$INSTANCE_ID/secret-keyring.json}"
|
||||
MEMORY_MAX="${MEMORY_MAX:?MEMORY_MAX is required, for example 16G}"
|
||||
CPU_QUOTA="${CPU_QUOTA:?CPU_QUOTA is required, for example 400%}"
|
||||
TASKS_MAX="${TASKS_MAX:?TASKS_MAX is required, for example 512}"
|
||||
SYSTEMD_UNIT_DIR="${SYSTEMD_UNIT_DIR:-/etc/systemd/system}"
|
||||
NODE_BIN="${NODE_BIN:-$(command -v node || true)}"
|
||||
BWRAP_BIN="${BWRAP_BIN:-$(command -v bwrap || true)}"
|
||||
SOCAT_BIN="${SOCAT_BIN:-$(command -v socat || true)}"
|
||||
PG_ISREADY_BIN="${PG_ISREADY_BIN:-$(command -v pg_isready || true)}"
|
||||
RUNUSER_BIN="${RUNUSER_BIN:-$(command -v runuser || true)}"
|
||||
SETPRIV_BIN="${SETPRIV_BIN:-$(command -v setpriv || true)}"
|
||||
CPH_BIN_DEFAULT="${CPH_BIN_DEFAULT:-/usr/local/bin/cph}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PREFLIGHT_JS="$HUB_DIR/dist/deployment/preflight-cli.js"
|
||||
RUNTIME_PATH="$(dirname "$NODE_BIN"):$(dirname "$BWRAP_BIN"):$(dirname "$SOCAT_BIN"):$(dirname "$SETPRIV_BIN"):/usr/local/bin:/usr/bin:/bin"
|
||||
|
||||
fail() {
|
||||
echo "[install] error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail "required command is missing: $1"
|
||||
}
|
||||
|
||||
require_safe_unit_value() {
|
||||
local label="$1"
|
||||
local value="$2"
|
||||
if [[ ! "$value" =~ ^[A-Za-z0-9_./:@+-]+$ ]]; then
|
||||
fail "$label contains characters that cannot be rendered safely into the systemd unit: $value"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
fail "must run as root (use sudo)"
|
||||
fi
|
||||
if [[ ! "$SERVICE_USER" =~ ^[a-z_][a-z0-9_-]*[$]?$ ]]; then
|
||||
fail "invalid SERVICE_USER: $SERVICE_USER"
|
||||
fi
|
||||
if [[ ! "$SERVICE_GROUP" =~ ^[a-z_][a-z0-9_-]*[$]?$ ]]; then
|
||||
fail "invalid SERVICE_GROUP: $SERVICE_GROUP"
|
||||
fi
|
||||
if [ "$SERVICE_USER" = "root" ] || [ "$SERVICE_GROUP" = "root" ]; then
|
||||
fail "the Hub service identity must not be root"
|
||||
fi
|
||||
|
||||
for command in getent groupadd useradd install stat systemctl id openssl date mktemp; do
|
||||
require_command "$command"
|
||||
done
|
||||
for pair in \
|
||||
"NODE_BIN:$NODE_BIN" \
|
||||
"BWRAP_BIN:$BWRAP_BIN" \
|
||||
"SOCAT_BIN:$SOCAT_BIN" \
|
||||
"PG_ISREADY_BIN:$PG_ISREADY_BIN"; do
|
||||
label="${pair%%:*}"
|
||||
value="${pair#*:}"
|
||||
[ -n "$value" ] || fail "$label is required and its executable was not found on PATH"
|
||||
[ -x "$value" ] || fail "$label is not executable: $value"
|
||||
done
|
||||
[ -n "$RUNUSER_BIN" ] || fail "RUNUSER_BIN is required and runuser was not found on PATH"
|
||||
[ -x "$RUNUSER_BIN" ] || fail "RUNUSER_BIN is not executable: $RUNUSER_BIN"
|
||||
[ -n "$SETPRIV_BIN" ] || fail "SETPRIV_BIN is required and setpriv was not found on PATH"
|
||||
[ -x "$SETPRIV_BIN" ] || fail "SETPRIV_BIN is not executable: $SETPRIV_BIN"
|
||||
|
||||
[ -d "$HUB_DIR" ] || fail "HUB_DIR not found: $HUB_DIR"
|
||||
[ -f "$HUB_DIR/dist/server.js" ] || fail "Hub build missing: $HUB_DIR/dist/server.js"
|
||||
[ -f "$PREFLIGHT_JS" ] || fail "deployment preflight missing from Hub build: $PREFLIGHT_JS"
|
||||
[ -f "$HUB_DIR/dist/deployment/service-runtime-probe.js" ] || fail "service runtime probe missing from Hub build"
|
||||
[ -f "$HUB_DIR/node_modules/prisma/build/index.js" ] || fail "production dependencies missing: run npm ci in $HUB_DIR"
|
||||
[ -f "$HUB_DIR/prisma/schema.prisma" ] || fail "Prisma schema missing: $HUB_DIR/prisma/schema.prisma"
|
||||
[ -f "$SCRIPT_DIR/cph-hub.service" ] || fail "service template missing: $SCRIPT_DIR/cph-hub.service"
|
||||
|
||||
for pair in \
|
||||
"BASE:$BASE" \
|
||||
"HUB_DIR:$HUB_DIR" \
|
||||
"SERVICE_HOME:$SERVICE_HOME" \
|
||||
"STATE_DIR:$STATE_DIR" \
|
||||
"CACHE_DIR:$CACHE_DIR" \
|
||||
"SKILL_STORE_ROOT:$SKILL_STORE_ROOT" \
|
||||
"WORKSPACE_ROOT:$WORKSPACE_ROOT" \
|
||||
"ENV_FILE:$ENV_FILE" \
|
||||
"KEYRING_FILE:$KEYRING_FILE" \
|
||||
"SYSTEMD_UNIT_DIR:$SYSTEMD_UNIT_DIR" \
|
||||
"NODE_BIN:$NODE_BIN" \
|
||||
"BWRAP_BIN:$BWRAP_BIN" \
|
||||
"SOCAT_BIN:$SOCAT_BIN" \
|
||||
"RUNUSER_BIN:$RUNUSER_BIN" \
|
||||
"SETPRIV_BIN:$SETPRIV_BIN" \
|
||||
"RUNTIME_PATH:$RUNTIME_PATH"; do
|
||||
require_safe_unit_value "${pair%%:*}" "${pair#*:}"
|
||||
done
|
||||
[[ "$MEMORY_MAX" =~ ^[1-9][0-9]*[KMGT]$ ]] || fail "MEMORY_MAX must be a systemd byte size such as 16G"
|
||||
[[ "$CPU_QUOTA" =~ ^[1-9][0-9]*%$ ]] || fail "CPU_QUOTA must be a positive percentage such as 400%"
|
||||
[[ "$TASKS_MAX" =~ ^[1-9][0-9]*$ ]] || fail "TASKS_MAX must be a positive integer"
|
||||
[ "${#WORKSPACE_ROOT}" -le 16 ] || fail "WORKSPACE_ROOT must be at most 16 bytes for Agent sandbox sockets: $WORKSPACE_ROOT"
|
||||
|
||||
KEYRING_CREATED=false
|
||||
if [ -L "$KEYRING_FILE" ]; then
|
||||
fail "local secret keyring must not be a symlink: $KEYRING_FILE"
|
||||
fi
|
||||
if [ ! -e "$KEYRING_FILE" ]; then
|
||||
install -d -o root -g root -m 0700 "$(dirname "$KEYRING_FILE")"
|
||||
key_id="local-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
key_value="$(openssl rand -base64 32)"
|
||||
tmp_keyring="$(mktemp "$(dirname "$KEYRING_FILE")/.secret-keyring.XXXXXX")"
|
||||
trap 'rm -f "${tmp_keyring:-}"' EXIT
|
||||
chmod 0600 "$tmp_keyring"
|
||||
printf '{"version":1,"activeKeyId":"%s","keys":{"%s":"%s"}}\n' \
|
||||
"$key_id" "$key_id" "$key_value" >"$tmp_keyring"
|
||||
install -o root -g root -m 0600 "$tmp_keyring" "$KEYRING_FILE"
|
||||
rm -f "$tmp_keyring"
|
||||
tmp_keyring=""
|
||||
trap - EXIT
|
||||
unset key_value
|
||||
KEYRING_CREATED=true
|
||||
echo "[install] generated local secret keyring: $KEYRING_FILE" >&2
|
||||
fi
|
||||
[ -f "$KEYRING_FILE" ] || fail "local secret keyring is not a regular file: $KEYRING_FILE"
|
||||
keyring_metadata="$(stat -c '%u:%g:%a' -- "$KEYRING_FILE")"
|
||||
[ "$keyring_metadata" = "0:0:600" ] || \
|
||||
fail "local secret keyring must be owned by root:root with mode 0600; found $keyring_metadata"
|
||||
|
||||
if [ -L "$ENV_FILE" ]; then
|
||||
fail "environment file must not be a symlink: $ENV_FILE"
|
||||
fi
|
||||
if [ ! -e "$ENV_FILE" ]; then
|
||||
install -d -o root -g root -m 0700 "$(dirname "$ENV_FILE")"
|
||||
umask 077
|
||||
cat >"$ENV_FILE" <<EOF
|
||||
# cph-hub production configuration. Required blanks must be filled before
|
||||
# rerunning install_service.sh; failed preflight never installs the unit.
|
||||
NODE_ENV=production
|
||||
DATABASE_URL=
|
||||
HUB_SILO_ORGANIZATION_ID=
|
||||
HUB_SYSTEMD_UNIT=$SERVICE_UNIT
|
||||
CPH_BIN=$CPH_BIN_DEFAULT
|
||||
HOST=$HOST
|
||||
PORT=$PORT
|
||||
HUB_PROJECT_WORKSPACE_ROOT=$WORKSPACE_ROOT
|
||||
HUB_PUBLIC_BASE_URL=
|
||||
HUB_SESSION_SECRET=
|
||||
HUB_AGENT_MAX_TURNS=25
|
||||
HUB_AGENT_MAX_CONCURRENT_RUNS=
|
||||
HUB_AGENT_MAX_RUN_SECONDS=
|
||||
HUB_HTTP_BODY_LIMIT_BYTES=
|
||||
HUB_MAX_FILES_PER_MESSAGE=
|
||||
HUB_MAX_FILE_BYTES=
|
||||
HUB_HTTP_REQUESTS_PER_MINUTE=
|
||||
HUB_FEISHU_EVENTS_PER_MINUTE=
|
||||
HUB_FEISHU_LISTENER_ENABLED=true
|
||||
CPH_SANDBOX_EXTRA_DENY_READ=$ENV_FILE:$KEYRING_FILE
|
||||
EOF
|
||||
chmod 0600 "$ENV_FILE"
|
||||
echo "[install] seeded complete production configuration: $ENV_FILE" >&2
|
||||
echo "[install] copy $KEYRING_FILE to the separately protected keyring backup before continuing" >&2
|
||||
echo "[install] fill every required blank, install cph at CPH_BIN, then rerun" >&2
|
||||
exit 78
|
||||
fi
|
||||
[ -f "$ENV_FILE" ] || fail "environment file is not a regular file: $ENV_FILE"
|
||||
if [ "$KEYRING_CREATED" = true ]; then
|
||||
echo "[install] copy $KEYRING_FILE to the separately protected keyring backup, then rerun" >&2
|
||||
exit 78
|
||||
fi
|
||||
|
||||
PREFLIGHT_ARGS=(
|
||||
"$NODE_BIN" "$PREFLIGHT_JS"
|
||||
--env-file "$ENV_FILE"
|
||||
--keyring-file "$KEYRING_FILE"
|
||||
--node-bin "$NODE_BIN"
|
||||
--runuser-bin "$RUNUSER_BIN"
|
||||
--setpriv-bin "$SETPRIV_BIN"
|
||||
--service-user "$SERVICE_USER"
|
||||
--base-dir "$BASE"
|
||||
--hub-dir "$HUB_DIR"
|
||||
--service-home "$SERVICE_HOME"
|
||||
--state-dir "$STATE_DIR"
|
||||
--cache-dir "$CACHE_DIR"
|
||||
--workspace-root "$WORKSPACE_ROOT"
|
||||
--service-unit "$SERVICE_UNIT"
|
||||
--bwrap-bin "$BWRAP_BIN"
|
||||
--socat-bin "$SOCAT_BIN"
|
||||
--pg-isready-bin "$PG_ISREADY_BIN"
|
||||
)
|
||||
|
||||
# Required configuration, path isolation, binaries and PostgreSQL must pass
|
||||
# before identity/directory provisioning or unit installation begins.
|
||||
"${PREFLIGHT_ARGS[@]}"
|
||||
|
||||
NOLOGIN_SHELL=""
|
||||
for candidate in /usr/sbin/nologin /sbin/nologin /usr/bin/false /bin/false; do
|
||||
if [ -x "$candidate" ]; then
|
||||
NOLOGIN_SHELL="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$NOLOGIN_SHELL" ] || fail "no non-login shell found"
|
||||
|
||||
group_entry="$(getent group "$SERVICE_GROUP" || true)"
|
||||
if [ -z "$group_entry" ]; then
|
||||
groupadd --system "$SERVICE_GROUP"
|
||||
group_entry="$(getent group "$SERVICE_GROUP" || true)"
|
||||
[ -n "$group_entry" ] || fail "created service group cannot be resolved: $SERVICE_GROUP"
|
||||
fi
|
||||
IFS=: read -r resolved_group _ SERVICE_GID _ <<<"$group_entry"
|
||||
[ "$resolved_group" = "$SERVICE_GROUP" ] || fail "service group resolved to unexpected name: $resolved_group"
|
||||
[[ "$SERVICE_GID" =~ ^[0-9]+$ ]] || fail "service group has invalid gid: $SERVICE_GID"
|
||||
[ "$SERVICE_GID" -ne 0 ] || fail "service group must not have gid 0"
|
||||
|
||||
passwd_entry="$(getent passwd "$SERVICE_USER" || true)"
|
||||
if [ -z "$passwd_entry" ]; then
|
||||
useradd \
|
||||
--system \
|
||||
--gid "$SERVICE_GROUP" \
|
||||
--home-dir "$SERVICE_HOME" \
|
||||
--shell "$NOLOGIN_SHELL" \
|
||||
--no-create-home \
|
||||
"$SERVICE_USER"
|
||||
passwd_entry="$(getent passwd "$SERVICE_USER" || true)"
|
||||
[ -n "$passwd_entry" ] || fail "created service user cannot be resolved: $SERVICE_USER"
|
||||
fi
|
||||
IFS=: read -r resolved_user _ SERVICE_UID user_gid _ user_home user_shell <<<"$passwd_entry"
|
||||
[ "$resolved_user" = "$SERVICE_USER" ] || fail "service user resolved to unexpected name: $resolved_user"
|
||||
[[ "$SERVICE_UID" =~ ^[0-9]+$ ]] || fail "service user has invalid uid: $SERVICE_UID"
|
||||
[ "$SERVICE_UID" -ne 0 ] || fail "service user must not have uid 0"
|
||||
[ "$user_gid" = "$SERVICE_GID" ] || fail "service user primary gid $user_gid does not match $SERVICE_GROUP ($SERVICE_GID)"
|
||||
[ "$user_home" = "$SERVICE_HOME" ] || fail "service user home is $user_home; expected $SERVICE_HOME"
|
||||
case "$user_shell" in
|
||||
/usr/sbin/nologin | /sbin/nologin | /usr/bin/false | /bin/false) ;;
|
||||
*) fail "service user must have a non-login shell; found $user_shell" ;;
|
||||
esac
|
||||
service_gids="$(id -G "$SERVICE_USER")" || fail "cannot resolve groups for service user $SERVICE_USER"
|
||||
for service_gid in $service_gids; do
|
||||
if [ "$service_gid" != "$SERVICE_GID" ]; then
|
||||
fail "service user $SERVICE_USER must not have supplementary groups; found gid $service_gid in: $service_gids"
|
||||
fi
|
||||
done
|
||||
|
||||
provision_directory() {
|
||||
local path="$1"
|
||||
if [ -L "$path" ]; then
|
||||
fail "service directory must not be a symlink: $path"
|
||||
fi
|
||||
if [ -e "$path" ]; then
|
||||
[ -d "$path" ] || fail "service path exists but is not a directory: $path"
|
||||
actual="$(stat -c '%u:%g:%a' -- "$path")"
|
||||
expected="$SERVICE_UID:$SERVICE_GID:750"
|
||||
[ "$actual" = "$expected" ] || fail "service directory $path has $actual; expected $expected"
|
||||
return
|
||||
fi
|
||||
install -d -o "$SERVICE_UID" -g "$SERVICE_GID" -m 0750 "$path"
|
||||
}
|
||||
|
||||
provision_directory "$SERVICE_HOME"
|
||||
provision_directory "$STATE_DIR"
|
||||
provision_directory "$CACHE_DIR"
|
||||
provision_directory "$SKILL_STORE_ROOT"
|
||||
provision_directory "$WORKSPACE_ROOT"
|
||||
|
||||
# Resolve every provisioned path again and verify uid/gid/mode before writing
|
||||
# the unit. This catches symlink or ownership changes between the two phases.
|
||||
"${PREFLIGHT_ARGS[@]}" --service-uid "$SERVICE_UID" --service-gid "$SERVICE_GID"
|
||||
|
||||
UNIT="$SYSTEMD_UNIT_DIR/$SERVICE_UNIT"
|
||||
install -d -o root -g root -m 0755 "$SYSTEMD_UNIT_DIR"
|
||||
TMP_UNIT="$(mktemp)"
|
||||
trap 'rm -f "$TMP_UNIT"' EXIT
|
||||
sed \
|
||||
-e "s|__SERVICE_USER__|$SERVICE_USER|g" \
|
||||
-e "s|__SERVICE_GROUP__|$SERVICE_GROUP|g" \
|
||||
-e "s|__SERVICE_HOME__|$SERVICE_HOME|g" \
|
||||
-e "s|__STATE_DIR__|$STATE_DIR|g" \
|
||||
-e "s|__CACHE_DIR__|$CACHE_DIR|g" \
|
||||
-e "s|__SKILL_STORE_ROOT__|$SKILL_STORE_ROOT|g" \
|
||||
-e "s|__WORKSPACE_ROOT__|$WORKSPACE_ROOT|g" \
|
||||
-e "s|__HUB_DIR__|$HUB_DIR|g" \
|
||||
-e "s|__ENV_FILE__|$ENV_FILE|g" \
|
||||
-e "s|__KEYRING_FILE__|$KEYRING_FILE|g" \
|
||||
-e "s|__NODE_BIN__|$NODE_BIN|g" \
|
||||
-e "s|__BWRAP_BIN__|$BWRAP_BIN|g" \
|
||||
-e "s|__SOCAT_BIN__|$SOCAT_BIN|g" \
|
||||
-e "s|__RUNTIME_PATH__|$RUNTIME_PATH|g" \
|
||||
-e "s|__INSTANCE_ID__|$INSTANCE_ID|g" \
|
||||
-e "s|__MEMORY_MAX__|$MEMORY_MAX|g" \
|
||||
-e "s|__CPU_QUOTA__|$CPU_QUOTA|g" \
|
||||
-e "s|__TASKS_MAX__|$TASKS_MAX|g" \
|
||||
"$SCRIPT_DIR/cph-hub.service" >"$TMP_UNIT"
|
||||
install -o root -g root -m 0644 "$TMP_UNIT" "$UNIT"
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable "$SERVICE_UNIT"
|
||||
echo "[install] installed $SERVICE_UNIT for $SERVICE_USER:$SERVICE_GROUP"
|
||||
echo "[install] home=$SERVICE_HOME state=$STATE_DIR cache=$CACHE_DIR skills=$SKILL_STORE_ROOT workspaces=$WORKSPACE_ROOT"
|
||||
echo "[install] start with: systemctl start $SERVICE_UNIT"
|
||||
Generated
+4740
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "@paradigm/hub",
|
||||
"version": "0.0.10",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=24"
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.3.202",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@larksuiteoapi/node-sdk": "^1.70.0",
|
||||
"@prisma/client": "^6.19.3",
|
||||
"ai": "^7.0.16",
|
||||
"dotenv": "^17.4.2",
|
||||
"fastify": "^5.8.5",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"prisma": "^6.19.3",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.7.0",
|
||||
"vitest": "^4.1.10"
|
||||
},
|
||||
"overrides": {
|
||||
"@larksuiteoapi/node-sdk": {
|
||||
"axios": "1.18.1"
|
||||
}
|
||||
},
|
||||
"description": "Curriculum Project Hub — org-scoped Feishu collaboration and confined Agent runtime. Aligns to spec/System through ADR-0024.",
|
||||
"scripts": {
|
||||
"dev": "npm run prisma:migrate && tsx watch src/server.ts",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "npm run prisma:migrate && node dist/server.js",
|
||||
"check": "tsc -p tsconfig.json --noEmit",
|
||||
"audit:production": "npm audit --omit=dev --audit-level=high",
|
||||
"prisma:generate": "prisma generate --schema prisma/schema.prisma",
|
||||
"prisma:validate": "DATABASE_URL=${DATABASE_URL:-postgresql://stub:stub@127.0.0.1:5432/stub} prisma validate --schema prisma/schema.prisma",
|
||||
"prisma:migrate": "DATABASE_URL=${DATABASE_URL:-postgresql://paradigm:paradigm@127.0.0.1:5432/paradigm} prisma migrate deploy --schema prisma/schema.prisma",
|
||||
"secrets:rotate-kek": "node dist/deployment/rotate-secret-kek.js",
|
||||
"agent-config": "node dist/deployment/agent-config-cli.js",
|
||||
"silo:bootstrap": "node dist/deployment/bootstrap-silo-cli.js",
|
||||
"silo:restore-preflight": "node dist/deployment/restore-preflight.js",
|
||||
"deploy": "bash deploy/deploy_platform.sh",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
-- CreateEnum
|
||||
CREATE TYPE "PlatformRole" AS ENUM ('ADMIN', 'TEACHER');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "AgentRunStatus" AS ENUM ('ACTIVE', 'WAITING_FOR_USER', 'COMPLETED', 'FAILED', 'TIMED_OUT', 'CANCELED');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "AgentEntrypoint" AS ENUM ('FEISHU', 'WEB', 'CLI');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "PermissionRole" AS ENUM ('READ', 'EDIT', 'MANAGE');
|
||||
|
||||
-- CreateEnum
|
||||
CREATE TYPE "PermissionResourceType" AS ENUM ('PROJECT', 'ARTIFACT', 'PROJECT_GROUP');
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "User" (
|
||||
"id" TEXT NOT NULL,
|
||||
"feishuOpenId" TEXT NOT NULL,
|
||||
"displayName" TEXT NOT NULL,
|
||||
"avatarUrl" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "User_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "PlatformRoleAssignment" (
|
||||
"id" TEXT NOT NULL,
|
||||
"userId" TEXT NOT NULL,
|
||||
"role" "PlatformRole" NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"revokedAt" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "PlatformRoleAssignment_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "Project" (
|
||||
"id" TEXT NOT NULL,
|
||||
"name" TEXT NOT NULL,
|
||||
"workspaceDir" TEXT NOT NULL,
|
||||
"createdByUserId" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
"archivedAt" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "Project_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "ProjectGroupBinding" (
|
||||
"id" TEXT NOT NULL,
|
||||
"projectId" TEXT NOT NULL,
|
||||
"chatId" TEXT NOT NULL,
|
||||
"createdByUserId" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "ProjectGroupBinding_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "AgentSession" (
|
||||
"id" TEXT NOT NULL,
|
||||
"projectId" TEXT NOT NULL,
|
||||
"provider" TEXT NOT NULL,
|
||||
"model" TEXT NOT NULL,
|
||||
"title" TEXT,
|
||||
"metadata" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
"archivedAt" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "AgentSession_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "AgentRun" (
|
||||
"id" TEXT NOT NULL,
|
||||
"projectId" TEXT NOT NULL,
|
||||
"sessionId" TEXT,
|
||||
"requestedByUserId" TEXT,
|
||||
"entrypoint" "AgentEntrypoint" NOT NULL,
|
||||
"status" "AgentRunStatus" NOT NULL DEFAULT 'ACTIVE',
|
||||
"prompt" TEXT NOT NULL,
|
||||
"model" TEXT NOT NULL,
|
||||
"provider" TEXT NOT NULL,
|
||||
"summary" TEXT,
|
||||
"inputTokens" INTEGER,
|
||||
"outputTokens" INTEGER,
|
||||
"metadata" JSONB NOT NULL,
|
||||
"error" TEXT,
|
||||
"startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"finishedAt" TIMESTAMP(3),
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "AgentRun_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "ProjectAgentLock" (
|
||||
"projectId" TEXT NOT NULL,
|
||||
"runId" TEXT NOT NULL,
|
||||
"holderUserId" TEXT,
|
||||
"acquiredAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"expiresAt" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "ProjectAgentLock_pkey" PRIMARY KEY ("projectId")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "PermissionGrant" (
|
||||
"id" TEXT NOT NULL,
|
||||
"resourceType" "PermissionResourceType" NOT NULL,
|
||||
"resourceId" TEXT NOT NULL,
|
||||
"principal" TEXT NOT NULL,
|
||||
"role" "PermissionRole" NOT NULL,
|
||||
"createdByUserId" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"revokedAt" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "PermissionGrant_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "PermissionSettings" (
|
||||
"id" TEXT NOT NULL,
|
||||
"resourceType" "PermissionResourceType" NOT NULL,
|
||||
"resourceId" TEXT NOT NULL,
|
||||
"externalShare" TEXT NOT NULL,
|
||||
"comment" TEXT NOT NULL,
|
||||
"copyDownload" TEXT NOT NULL,
|
||||
"collaboratorMgmt" TEXT NOT NULL,
|
||||
"agentTrigger" TEXT NOT NULL,
|
||||
"agentCancel" TEXT NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "PermissionSettings_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "AuditEntry" (
|
||||
"id" TEXT NOT NULL,
|
||||
"runId" TEXT,
|
||||
"actorUserId" TEXT,
|
||||
"action" TEXT NOT NULL,
|
||||
"metadata" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "AuditEntry_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "User_feishuOpenId_key" ON "User"("feishuOpenId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "PlatformRoleAssignment_userId_revokedAt_idx" ON "PlatformRoleAssignment"("userId", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "PlatformRoleAssignment_role_revokedAt_idx" ON "PlatformRoleAssignment"("role", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "Project_archivedAt_idx" ON "Project"("archivedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "ProjectGroupBinding_projectId_key" ON "ProjectGroupBinding"("projectId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "ProjectGroupBinding_chatId_key" ON "ProjectGroupBinding"("chatId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "ProjectGroupBinding_chatId_idx" ON "ProjectGroupBinding"("chatId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentSession_projectId_archivedAt_idx" ON "AgentSession"("projectId", "archivedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentSession_provider_model_idx" ON "AgentSession"("provider", "model");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentSession_updatedAt_idx" ON "AgentSession"("updatedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentRun_projectId_status_idx" ON "AgentRun"("projectId", "status");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentRun_sessionId_idx" ON "AgentRun"("sessionId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentRun_requestedByUserId_idx" ON "AgentRun"("requestedByUserId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AgentRun_updatedAt_idx" ON "AgentRun"("updatedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "ProjectAgentLock_runId_key" ON "ProjectAgentLock"("runId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "ProjectAgentLock_expiresAt_idx" ON "ProjectAgentLock"("expiresAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "PermissionGrant_resourceType_resourceId_revokedAt_idx" ON "PermissionGrant"("resourceType", "resourceId", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "PermissionGrant_principal_revokedAt_idx" ON "PermissionGrant"("principal", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "PermissionGrant_resourceType_resourceId_principal_role_revo_key" ON "PermissionGrant"("resourceType", "resourceId", "principal", "role", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "PermissionSettings_resourceType_resourceId_idx" ON "PermissionSettings"("resourceType", "resourceId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "PermissionSettings_resourceType_resourceId_key" ON "PermissionSettings"("resourceType", "resourceId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AuditEntry_runId_idx" ON "AuditEntry"("runId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AuditEntry_actorUserId_idx" ON "AuditEntry"("actorUserId");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "AuditEntry_createdAt_idx" ON "AuditEntry"("createdAt");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "PlatformRoleAssignment" ADD CONSTRAINT "PlatformRoleAssignment_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "Project" ADD CONSTRAINT "Project_createdByUserId_fkey" FOREIGN KEY ("createdByUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "ProjectGroupBinding" ADD CONSTRAINT "ProjectGroupBinding_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "ProjectGroupBinding" ADD CONSTRAINT "ProjectGroupBinding_createdByUserId_fkey" FOREIGN KEY ("createdByUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "AgentSession" ADD CONSTRAINT "AgentSession_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "AgentRun" ADD CONSTRAINT "AgentRun_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "AgentRun" ADD CONSTRAINT "AgentRun_sessionId_fkey" FOREIGN KEY ("sessionId") REFERENCES "AgentSession"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "AgentRun" ADD CONSTRAINT "AgentRun_requestedByUserId_fkey" FOREIGN KEY ("requestedByUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "ProjectAgentLock" ADD CONSTRAINT "ProjectAgentLock_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "ProjectAgentLock" ADD CONSTRAINT "ProjectAgentLock_runId_fkey" FOREIGN KEY ("runId") REFERENCES "AgentRun"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "ProjectAgentLock" ADD CONSTRAINT "ProjectAgentLock_holderUserId_fkey" FOREIGN KEY ("holderUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "PermissionGrant" ADD CONSTRAINT "PermissionGrant_resourceId_fkey" FOREIGN KEY ("resourceId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "PermissionGrant" ADD CONSTRAINT "PermissionGrant_createdByUserId_fkey" FOREIGN KEY ("createdByUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "PermissionSettings" ADD CONSTRAINT "PermissionSettings_resourceId_fkey" FOREIGN KEY ("resourceId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "AuditEntry" ADD CONSTRAINT "AuditEntry_actorUserId_fkey" FOREIGN KEY ("actorUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- CreateTable
|
||||
CREATE TABLE "RoleTriggerGrant" (
|
||||
"id" TEXT NOT NULL,
|
||||
"projectId" TEXT NOT NULL,
|
||||
"roleId" TEXT NOT NULL,
|
||||
"principal" TEXT NOT NULL,
|
||||
"createdByUserId" TEXT,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"revokedAt" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "RoleTriggerGrant_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "RoleTriggerGrant_projectId_roleId_revokedAt_idx" ON "RoleTriggerGrant"("projectId", "roleId", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "RoleTriggerGrant_principal_revokedAt_idx" ON "RoleTriggerGrant"("principal", "revokedAt");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "RoleTriggerGrant_projectId_roleId_principal_revokedAt_key" ON "RoleTriggerGrant"("projectId", "roleId", "principal", "revokedAt");
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "RoleTriggerGrant" ADD CONSTRAINT "RoleTriggerGrant_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- AddForeignKey
|
||||
ALTER TABLE "RoleTriggerGrant" ADD CONSTRAINT "RoleTriggerGrant_createdByUserId_fkey" FOREIGN KEY ("createdByUserId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,23 @@
|
||||
-- FeishuEventReceipt: idempotency for inbound ws events (at-least-once redelivery).
|
||||
CREATE TABLE "FeishuEventReceipt" (
|
||||
"id" TEXT NOT NULL,
|
||||
"eventId" TEXT NOT NULL,
|
||||
"eventType" TEXT NOT NULL,
|
||||
"messageId" TEXT,
|
||||
"receivedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "FeishuEventReceipt_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "FeishuEventReceipt_eventId_key" ON "FeishuEventReceipt"("eventId");
|
||||
CREATE INDEX "FeishuEventReceipt_eventType_idx" ON "FeishuEventReceipt"("eventType");
|
||||
CREATE INDEX "FeishuEventReceipt_messageId_idx" ON "FeishuEventReceipt"("messageId");
|
||||
CREATE INDEX "FeishuEventReceipt_receivedAt_idx" ON "FeishuEventReceipt"("receivedAt");
|
||||
|
||||
-- AuditEntry: add projectId so audit points before/without a run (permission
|
||||
-- denials, slash commands) are locatable to a project.
|
||||
ALTER TABLE "AuditEntry" ADD COLUMN "projectId" TEXT;
|
||||
|
||||
CREATE INDEX "AuditEntry_projectId_createdAt_idx" ON "AuditEntry"("projectId", "createdAt");
|
||||
|
||||
ALTER TABLE "AuditEntry" ADD CONSTRAINT "AuditEntry_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,45 @@
|
||||
-- AgentMessage: per-message record of the agent loop (queryable projection
|
||||
-- alongside the transcript JSONL file, which remains the agent's read-back memory).
|
||||
CREATE TABLE "AgentMessage" (
|
||||
"id" TEXT NOT NULL,
|
||||
"sessionId" TEXT NOT NULL,
|
||||
"runId" TEXT,
|
||||
"role" TEXT NOT NULL,
|
||||
"content" TEXT NOT NULL,
|
||||
"attachments" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "AgentMessage_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX "AgentMessage_sessionId_createdAt_idx" ON "AgentMessage"("sessionId", "createdAt");
|
||||
CREATE INDEX "AgentMessage_runId_idx" ON "AgentMessage"("runId");
|
||||
|
||||
ALTER TABLE "AgentMessage" ADD CONSTRAINT "AgentMessage_sessionId_fkey" FOREIGN KEY ("sessionId") REFERENCES "AgentSession"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
ALTER TABLE "AgentMessage" ADD CONSTRAINT "AgentMessage_runId_fkey" FOREIGN KEY ("runId") REFERENCES "AgentRun"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||
|
||||
-- AgentFileChange: file change captured during a run (write/rename/delete).
|
||||
CREATE TABLE "AgentFileChange" (
|
||||
"id" TEXT NOT NULL,
|
||||
"runId" TEXT NOT NULL,
|
||||
"projectId" TEXT NOT NULL,
|
||||
"path" TEXT NOT NULL,
|
||||
"operation" TEXT NOT NULL,
|
||||
"beforeHash" TEXT,
|
||||
"afterHash" TEXT,
|
||||
"diff" TEXT,
|
||||
"metadata" JSONB NOT NULL,
|
||||
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "AgentFileChange_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX "AgentFileChange_runId_idx" ON "AgentFileChange"("runId");
|
||||
CREATE INDEX "AgentFileChange_projectId_idx" ON "AgentFileChange"("projectId");
|
||||
CREATE INDEX "AgentFileChange_path_idx" ON "AgentFileChange"("path");
|
||||
|
||||
ALTER TABLE "AgentFileChange" ADD CONSTRAINT "AgentFileChange_runId_fkey" FOREIGN KEY ("runId") REFERENCES "AgentRun"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
ALTER TABLE "AgentFileChange" ADD CONSTRAINT "AgentFileChange_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
-- ProjectAgentLock: add heartbeatAt for liveness (A-5).
|
||||
ALTER TABLE "ProjectAgentLock" ADD COLUMN "heartbeatAt" TIMESTAMP(3);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user