forked from bai/curriculum-project-hub
ci(hub): 补齐 Hub 检查和健康 smoke
This commit is contained in:
@@ -13,6 +13,20 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
hub-check:
|
hub-check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: paradigm
|
||||||
|
POSTGRES_PASSWORD: paradigm
|
||||||
|
POSTGRES_DB: cph_hub_test
|
||||||
|
ports:
|
||||||
|
- 5432:5432
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U paradigm -d cph_hub_test"
|
||||||
|
--health-interval 5s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 20
|
||||||
defaults:
|
defaults:
|
||||||
run:
|
run:
|
||||||
working-directory: hub
|
working-directory: hub
|
||||||
@@ -29,6 +43,29 @@ jobs:
|
|||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: npm ci
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Wait for Postgres
|
||||||
|
run: |
|
||||||
|
node <<'NODE'
|
||||||
|
const net = require("node:net");
|
||||||
|
const deadline = Date.now() + 60000;
|
||||||
|
function tryConnect() {
|
||||||
|
const socket = net.createConnection({ host: "127.0.0.1", port: 5432 });
|
||||||
|
socket.once("connect", () => {
|
||||||
|
socket.end();
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
socket.once("error", () => {
|
||||||
|
socket.destroy();
|
||||||
|
if (Date.now() > deadline) {
|
||||||
|
console.error("Postgres did not become reachable at 127.0.0.1:5432");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
setTimeout(tryConnect, 1000);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
tryConnect();
|
||||||
|
NODE
|
||||||
|
|
||||||
# Prisma client must be generated before tsc can check imports from
|
# Prisma client must be generated before tsc can check imports from
|
||||||
# @prisma/client. Stub DATABASE_URL so prisma generate works.
|
# @prisma/client. Stub DATABASE_URL so prisma generate works.
|
||||||
- name: Generate Prisma client
|
- name: Generate Prisma client
|
||||||
@@ -37,6 +74,9 @@ jobs:
|
|||||||
- name: Type-check
|
- name: Type-check
|
||||||
run: npx tsc -p tsconfig.json --noEmit
|
run: npx tsc -p tsconfig.json --noEmit
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: npm run build
|
||||||
|
|
||||||
- name: Validate Prisma schema
|
- name: Validate Prisma schema
|
||||||
run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma validate --schema prisma/schema.prisma
|
run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma validate --schema prisma/schema.prisma
|
||||||
|
|
||||||
@@ -57,25 +97,36 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
DATABASE_URL: postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test
|
DATABASE_URL: postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test
|
||||||
|
|
||||||
# Real-model tests are skipped automatically when OPENROUTER_API_KEY is
|
- name: Smoke health endpoint
|
||||||
# not set. When the secret is available (e.g. on the main branch), they
|
run: |
|
||||||
# run against live OpenRouter — verifying the tool-calls loop end-to-end.
|
PORT=8878 \
|
||||||
|
NODE_ENV=production \
|
||||||
|
DATABASE_URL=postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test \
|
||||||
|
ANTHROPIC_AUTH_TOKEN=smoke \
|
||||||
|
FEISHU_APP_ID=cli_smoke \
|
||||||
|
FEISHU_APP_SECRET=smoke_secret \
|
||||||
|
FEISHU_BOT_OPEN_ID=ou_smoke \
|
||||||
|
HUB_FEISHU_LISTENER_ENABLED=false \
|
||||||
|
npm start &
|
||||||
|
pid=$!
|
||||||
|
trap 'kill "$pid" 2>/dev/null || true' EXIT
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
if curl --fail --silent "http://127.0.0.1:8878/api/healthz" >/dev/null; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 "$pid" 2>/dev/null; then
|
||||||
|
echo "hub server exited before health check passed"
|
||||||
|
wait "$pid"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
curl --fail --silent --show-error "http://127.0.0.1:8878/api/healthz" >/dev/null
|
||||||
|
|
||||||
|
# Real-model tests are opt-in: set RUN_REAL_MODEL_TESTS=true and provide
|
||||||
|
# OPENROUTER_API_KEY when a branch should hit live OpenRouter.
|
||||||
- name: Run real-model integration tests (optional, needs secret)
|
- name: Run real-model integration tests (optional, needs secret)
|
||||||
run: npx vitest run test/integration/real-model.test.ts
|
run: npx vitest run test/integration/real-model.test.ts
|
||||||
env:
|
env:
|
||||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||||
|
RUN_REAL_MODEL_TESTS: ${{ vars.RUN_REAL_MODEL_TESTS }}
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16
|
|
||||||
env:
|
|
||||||
POSTGRES_USER: paradigm
|
|
||||||
POSTGRES_PASSWORD: paradigm
|
|
||||||
POSTGRES_DB: cph_hub_test
|
|
||||||
ports:
|
|
||||||
- 5432:5432
|
|
||||||
options: >-
|
|
||||||
--health-cmd pg_isready
|
|
||||||
--health-interval 10s
|
|
||||||
--health-timeout 5s
|
|
||||||
--health-retries 5
|
|
||||||
|
|||||||
+16
-5
@@ -14,7 +14,7 @@ import "dotenv/config";
|
|||||||
import Fastify from "fastify";
|
import Fastify from "fastify";
|
||||||
import { prisma } from "./db.js";
|
import { prisma } from "./db.js";
|
||||||
import { createDefaultModelRegistry } from "./agent/defaultModels.js";
|
import { createDefaultModelRegistry } from "./agent/defaultModels.js";
|
||||||
import { createLarkClient, startFeishuListenerWithClient, type FeishuRuntime } from "./feishu/client.js";
|
import { createLarkClient, startFeishuListenerWithClient } from "./feishu/client.js";
|
||||||
import { makeTriggerHandler } from "./feishu/trigger.js";
|
import { makeTriggerHandler } from "./feishu/trigger.js";
|
||||||
|
|
||||||
function requireEnv(name: string): string {
|
function requireEnv(name: string): string {
|
||||||
@@ -26,6 +26,12 @@ function requireEnv(name: string): string {
|
|||||||
return v;
|
return v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function booleanEnv(name: string, fallback: boolean): boolean {
|
||||||
|
const raw = process.env[name];
|
||||||
|
if (raw === undefined || raw === "") return fallback;
|
||||||
|
return !["0", "false", "no", "off"].includes(raw.trim().toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
async function main(): Promise<void> {
|
||||||
const databaseUrl = requireEnv("DATABASE_URL");
|
const databaseUrl = requireEnv("DATABASE_URL");
|
||||||
void databaseUrl;
|
void databaseUrl;
|
||||||
@@ -59,10 +65,15 @@ async function main(): Promise<void> {
|
|||||||
const models = createDefaultModelRegistry();
|
const models = createDefaultModelRegistry();
|
||||||
|
|
||||||
// --- Feishu listener ---
|
// --- Feishu listener ---
|
||||||
const feishuConfig = { appId: feishuAppId, appSecret: feishuAppSecret, botOpenId: feishuBotOpenId };
|
const feishuListenerEnabled = booleanEnv("HUB_FEISHU_LISTENER_ENABLED", true);
|
||||||
const larkClient = createLarkClient(feishuConfig);
|
if (feishuListenerEnabled) {
|
||||||
const trigger = makeTriggerHandler({ prisma, models, logger: app.log });
|
const feishuConfig = { appId: feishuAppId, appSecret: feishuAppSecret, botOpenId: feishuBotOpenId };
|
||||||
startFeishuListenerWithClient(feishuConfig, larkClient, app.log, trigger);
|
const larkClient = createLarkClient(feishuConfig);
|
||||||
|
const trigger = makeTriggerHandler({ prisma, models, logger: app.log });
|
||||||
|
startFeishuListenerWithClient(feishuConfig, larkClient, app.log, trigger);
|
||||||
|
} else {
|
||||||
|
app.log.info("feishu listener disabled by HUB_FEISHU_LISTENER_ENABLED");
|
||||||
|
}
|
||||||
|
|
||||||
app.listen({ port, host: "0.0.0.0" }, (err, address) => {
|
app.listen({ port, host: "0.0.0.0" }, (err, address) => {
|
||||||
if (err !== null) {
|
if (err !== null) {
|
||||||
|
|||||||
@@ -1,143 +1,63 @@
|
|||||||
/**
|
/**
|
||||||
* Real-model integration test — exercises the full agent tool-calls loop against
|
* Optional live-model smoke for the current Claude SDK runner.
|
||||||
* a live OpenRouter model + real cph binary + real workspace.
|
|
||||||
*
|
*
|
||||||
* Unlike MockProvider tests, this verifies:
|
* Skips unless explicitly enabled with RUN_REAL_MODEL_TESTS=true and an
|
||||||
* - generateText's tool-calls loop with a real model (GLM/Claude/…)
|
* OpenRouter/Anthropic token is available. When enabled, it verifies that
|
||||||
* - model returns tool_calls → runner dispatches → tool_result back → continue
|
* `runAgent` can complete one real SDK turn and project the user/assistant
|
||||||
* - cph_check / read_file / list_files against a real engineering-file tree
|
* messages into the structured AgentMessage table shape.
|
||||||
* - transcript JSONL read/append on a real filesystem
|
|
||||||
*
|
|
||||||
* Skipped automatically when OPENROUTER_API_KEY is not set (CI without secrets,
|
|
||||||
* or local dev without a key). To run locally:
|
|
||||||
* cp .env.example .env # fill in OPENROUTER_API_KEY
|
|
||||||
* npx vitest run test/integration/real-model.test.ts
|
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect } from "vitest";
|
import { describe, it, expect } from "vitest";
|
||||||
import { cp, rm, readFile } from "node:fs/promises";
|
import { mkdtemp, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
import { createModelFactory } from "../../src/agent/provider.js";
|
|
||||||
import { ToolRegistry, feishuContextTool } from "../../src/agent/tools.js";
|
|
||||||
import { readFileTool, writeFileTool, listFilesTool } from "../../src/agent/workspace.js";
|
|
||||||
import { cphCheckTool, cphBuildTool } from "../../src/agent/cph.js";
|
|
||||||
import { runAgent } from "../../src/agent/runner.js";
|
import { runAgent } from "../../src/agent/runner.js";
|
||||||
import type { FeishuRuntime } from "../../src/feishu/client.js";
|
|
||||||
import type { PrismaClient } from "@prisma/client";
|
import type { PrismaClient } from "@prisma/client";
|
||||||
|
|
||||||
const API_KEY = process.env["OPENROUTER_API_KEY"];
|
const API_KEY = process.env["OPENROUTER_API_KEY"] ?? process.env["ANTHROPIC_AUTH_TOKEN"];
|
||||||
const HAS_KEY = API_KEY !== undefined && API_KEY !== "";
|
const HAS_KEY = API_KEY !== undefined && API_KEY !== "";
|
||||||
const describeOrSkip = HAS_KEY ? describe : describe.skip;
|
const ENABLED = HAS_KEY && process.env["RUN_REAL_MODEL_TESTS"] === "true";
|
||||||
|
const describeOrSkip = ENABLED ? describe : describe.skip;
|
||||||
|
|
||||||
const EXAMPLES_DIR = join(process.cwd(), "..", "examples", "TH-141");
|
if (ENABLED) {
|
||||||
const MODEL = process.env["SMOKE_MODEL"] ?? "z-ai/glm-4.6";
|
process.env["ANTHROPIC_AUTH_TOKEN"] = API_KEY;
|
||||||
|
process.env["ANTHROPIC_BASE_URL"] ??= "https://openrouter.ai/api";
|
||||||
|
process.env["ANTHROPIC_API_KEY"] ??= "";
|
||||||
|
}
|
||||||
|
|
||||||
// Stub prisma: runner touches it for lock heartbeat + message persistence,
|
const MODEL = process.env["SMOKE_MODEL"] ?? process.env["ANTHROPIC_DEFAULT_SONNET_MODEL"] ?? "z-ai/glm-4.7";
|
||||||
// neither of which need a real DB for this test.
|
|
||||||
const stubPrisma = {
|
|
||||||
projectAgentLock: { update: async () => ({}), deleteMany: async () => ({}) },
|
|
||||||
agentMessage: { createMany: async () => ({}) },
|
|
||||||
agentFileChange: { createMany: async () => ({}) },
|
|
||||||
} as unknown as PrismaClient;
|
|
||||||
|
|
||||||
const stubFeishuRt: FeishuRuntime = {
|
|
||||||
client: {
|
|
||||||
im: { v1: { message: { create: async () => ({}), get: async () => ({ data: {} }), list: async () => ({ data: {} }) } } },
|
|
||||||
} as never,
|
|
||||||
logger: { info() {}, warn() {}, error() {}, debug() {}, fatal() {}, child() { return this; }, level: "silent" } as never,
|
|
||||||
};
|
|
||||||
|
|
||||||
describeOrSkip("real model integration (OpenRouter)", () => {
|
describeOrSkip("real model integration (OpenRouter)", () => {
|
||||||
let ws: string;
|
it("completes one live SDK turn and records structured messages", async () => {
|
||||||
|
const workspace = await mkdtemp(join(tmpdir(), "hub-real-model-"));
|
||||||
|
const messages: unknown[] = [];
|
||||||
|
const stubPrisma = {
|
||||||
|
projectAgentLock: { update: async () => ({}) },
|
||||||
|
agentMessage: {
|
||||||
|
create: async (input: unknown) => {
|
||||||
|
messages.push(input);
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as unknown as PrismaClient;
|
||||||
|
|
||||||
async function setupWorkspace(): Promise<string> {
|
|
||||||
ws = join(process.cwd(), ".test-ws-real");
|
|
||||||
await rm(ws, { recursive: true, force: true });
|
|
||||||
await cp(EXAMPLES_DIR, ws, { recursive: true });
|
|
||||||
return ws;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function teardownWorkspace(): Promise<void> {
|
|
||||||
await rm(ws, { recursive: true, force: true });
|
|
||||||
}
|
|
||||||
|
|
||||||
it("runs a multi-step tool-calls loop: list files → cph check → answer", async () => {
|
|
||||||
const workspace = await setupWorkspace();
|
|
||||||
try {
|
try {
|
||||||
const tools = new ToolRegistry();
|
const result = await runAgent({
|
||||||
tools.register("feishu_read_context", feishuContextTool(async () => JSON.stringify({ stub: true }), stubFeishuRt));
|
|
||||||
tools.register("read_file", readFileTool);
|
|
||||||
tools.register("write_file", writeFileTool);
|
|
||||||
tools.register("list_files", listFilesTool);
|
|
||||||
tools.register("cph_check", cphCheckTool);
|
|
||||||
tools.register("cph_build", cphBuildTool);
|
|
||||||
|
|
||||||
const modelFactory = createModelFactory();
|
|
||||||
const transcript = join(workspace, ".cph", "sessions", "real-test.jsonl");
|
|
||||||
|
|
||||||
const result = await runAgent(modelFactory, tools, {
|
|
||||||
runId: "real-test-run",
|
runId: "real-test-run",
|
||||||
projectId: "real-test-project",
|
sessionId: "real-test-session",
|
||||||
prompt: "列出工程文件的目录结构,然后跑 cph check,用中文告诉我结果。",
|
prompt: "请只回复: OK",
|
||||||
model: MODEL,
|
model: MODEL,
|
||||||
project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace },
|
project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace },
|
||||||
prisma: stubPrisma,
|
prisma: stubPrisma,
|
||||||
systemPrompt: "你是一个教研工程文件的助手。你可以读写工程文件、跑 cph check 校验、跑 cph build 渲染。请用中文回复。",
|
systemPrompt: "你是一个极简 smoke test 助手。请严格按用户要求回复。",
|
||||||
transcriptPath: transcript,
|
maxTurns: 3,
|
||||||
maxIterations: 10,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// The model should complete (not hit the iteration cap or fail).
|
expect(result.status, result.error).toBe("completed");
|
||||||
expect(result.status).toBe("completed");
|
expect(result.text.length).toBeGreaterThan(0);
|
||||||
|
expect(messages.length).toBeGreaterThanOrEqual(2);
|
||||||
// It should have produced messages beyond just the initial prompt.
|
|
||||||
expect(result.messages.length).toBeGreaterThan(1);
|
|
||||||
|
|
||||||
// The transcript file should have been written.
|
|
||||||
const transcriptContent = await readFile(transcript, "utf8");
|
|
||||||
expect(transcriptContent.length).toBeGreaterThan(0);
|
|
||||||
const lines = transcriptContent.trim().split("\n");
|
|
||||||
expect(lines.length).toBeGreaterThan(1);
|
|
||||||
// Each line should be valid JSON.
|
|
||||||
for (const line of lines) {
|
|
||||||
expect(() => JSON.parse(line)).not.toThrow();
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
await teardownWorkspace();
|
await rm(workspace, { recursive: true, force: true });
|
||||||
}
|
|
||||||
}, 60_000); // 60s timeout — real model calls take time.
|
|
||||||
|
|
||||||
it("can write a file and read it back through the agent loop", async () => {
|
|
||||||
const workspace = await setupWorkspace();
|
|
||||||
try {
|
|
||||||
const tools = new ToolRegistry();
|
|
||||||
tools.register("read_file", readFileTool);
|
|
||||||
tools.register("write_file", writeFileTool);
|
|
||||||
tools.register("list_files", listFilesTool);
|
|
||||||
tools.register("cph_check", cphCheckTool);
|
|
||||||
tools.register("cph_build", cphBuildTool);
|
|
||||||
|
|
||||||
const modelFactory = createModelFactory();
|
|
||||||
|
|
||||||
const result = await runAgent(modelFactory, tools, {
|
|
||||||
runId: "real-test-run-2",
|
|
||||||
projectId: "real-test-project",
|
|
||||||
prompt: "在工程文件根目录创建一个文件 note.txt,内容写'测试笔记',然后用 read_file 读回来,告诉我内容。",
|
|
||||||
model: MODEL,
|
|
||||||
project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace },
|
|
||||||
prisma: stubPrisma,
|
|
||||||
systemPrompt: "你是一个教研工程文件的助手。请用中文回复。",
|
|
||||||
maxIterations: 10,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result.status).toBe("completed");
|
|
||||||
|
|
||||||
// The file should actually exist on disk (the agent called write_file).
|
|
||||||
const { readFile } = await import("node:fs/promises");
|
|
||||||
const note = await readFile(join(workspace, "note.txt"), "utf8");
|
|
||||||
expect(note).toContain("测试笔记");
|
|
||||||
} finally {
|
|
||||||
await teardownWorkspace();
|
|
||||||
}
|
}
|
||||||
}, 60_000);
|
}, 60_000);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user