From 1abcc495f4ebfe980e7c4d35278ba9706f70e702 Mon Sep 17 00:00:00 2001 From: Hong Jiarong Date: Tue, 7 Jul 2026 23:45:59 +0800 Subject: [PATCH] =?UTF-8?q?test(hub):=20=E7=9C=9F=E5=AE=9E=20model=20?= =?UTF-8?q?=E9=9B=86=E6=88=90=E6=B5=8B=E8=AF=95=20+=20Gitea=20CI=20workflo?= =?UTF-8?q?w?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit real-model.test.ts: 用真 OpenRouter model + 真 cph + 真 workspace 跑 agent tool-calls 循环,验证 MockProvider 测不到的路径(model 返回 tool_calls → dispatch → tool_result 回传 → 继续/stop)。没 OPENROUTER_API_KEY 时自动 skip,不阻塞 CI;有 key 时自动跑。 两个 test:(1) list files → cph check → 回答;(2) write file → read back。 hub-check.yml: Gitea CI workflow,对齐 checker-check/spec-check 的纪律: npm ci → prisma generate → tsc → prisma validate → 装 cph → vitest unit → vitest integration(postgres service container)→ real-model(可选,从 secrets 注入 key)。 全 suite: 60 passed + 2 skipped(无 key)。tsc rc=0。 --- .gitea/workflows/hub-check.yml | 81 ++++++++++++++ hub/.env.example | 22 ++-- hub/test/integration/real-model.test.ts | 143 ++++++++++++++++++++++++ 3 files changed, 238 insertions(+), 8 deletions(-) create mode 100644 .gitea/workflows/hub-check.yml create mode 100644 hub/test/integration/real-model.test.ts diff --git a/.gitea/workflows/hub-check.yml b/.gitea/workflows/hub-check.yml new file mode 100644 index 0000000..9852756 --- /dev/null +++ b/.gitea/workflows/hub-check.yml @@ -0,0 +1,81 @@ +name: hub check + +# Builds, type-checks, and tests the Hub TS package under hub/. +# The Hub is the Feishu-group collaboration + agent runtime half +# (spec/System implementation). This is an INTERNAL gate on the Hub's own +# health, like checker-check is for the Rust half. + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + hub-check: + runs-on: ubuntu-latest + defaults: + run: + working-directory: hub + steps: + - uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + cache-dependency-path: hub/package-lock.json + + - name: Install dependencies + run: npm ci + + # Prisma client must be generated before tsc can check imports from + # @prisma/client. Stub DATABASE_URL so prisma generate works. + - name: Generate Prisma client + run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma generate --schema prisma/schema.prisma + + - name: Type-check + run: npx tsc -p tsconfig.json --noEmit + + - name: Validate Prisma schema + run: DATABASE_URL="postgresql://stub:stub@127.0.0.1:5432/stub" npx prisma validate --schema prisma/schema.prisma + + - name: Install cph binary + run: | + cd .. + cargo install --path crates/cph-cli --locked + + - name: Run unit tests + run: npx vitest run test/unit + + # Integration tests need PostgreSQL + cph. cph is installed above. + # PostgreSQL is set up as a service container below. + - name: Run integration tests (mock provider, real prisma + cph) + run: | + npx prisma migrate deploy --schema prisma/schema.prisma + npx vitest run test/integration --exclude test/integration/real-model.test.ts + env: + DATABASE_URL: postgresql://paradigm:paradigm@127.0.0.1:5432/cph_hub_test + + # Real-model tests are skipped automatically when OPENROUTER_API_KEY is + # not set. When the secret is available (e.g. on the main branch), they + # run against live OpenRouter — verifying the tool-calls loop end-to-end. + - name: Run real-model integration tests (optional, needs secret) + run: npx vitest run test/integration/real-model.test.ts + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + +services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: paradigm + POSTGRES_PASSWORD: paradigm + POSTGRES_DB: cph_hub_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 diff --git a/hub/.env.example b/hub/.env.example index 727f57e..62ee565 100644 --- a/hub/.env.example +++ b/hub/.env.example @@ -1,19 +1,22 @@ # Hub runtime configuration. Copy to .env and fill in. +# +# For local dev: `cp .env.example .env` then edit. +# For tests: integration tests read from .env via dotenv. -# PostgreSQL connection string. Used by Prisma and the Hub server. +# PostgreSQL connection string. DATABASE_URL="postgresql://paradigm:paradigm@127.0.0.1:5432/paradigm" -# OpenRouter (or any OpenAI-compatible) API key + base URL. -# The Hub's agent layer is provider-agnostic (ADR-0017); this points the -# OpenAI-compatible client at OpenRouter by default. +# OpenRouter (or any OpenAI-compatible) API key. +# Get one at https://openrouter.ai/keys +# The agent layer is provider-agnostic (ADR-0017); OpenRouter routes to any model. OPENROUTER_API_KEY="" + # Optional: override the base URL (e.g. direct vendor API, local gateway). # OPENROUTER_BASE_URL="https://openrouter.ai/api/v1" -# Hub server port. Defaults to 8788. -PORT=8788 - -# Feishu (lark) bot credentials. The bot receives @mentions in project groups. +# Feishu (lark) bot credentials. Create a self-built app at +# https://open.feishu.cn/app, add the "Bot" capability, subscribe to +# im.message.receive_v1, then fill these in. FEISHU_APP_ID="" FEISHU_APP_SECRET="" FEISHU_BOT_OPEN_ID="" @@ -21,3 +24,6 @@ FEISHU_BOT_OPEN_ID="" # Path to the `cph` binary (the Courseware-side checker, ADR-0016). # Defaults to `cph` on PATH if unset. # CPH_BIN="/usr/local/bin/cph" + +# Hub server port. Defaults to 8788. +PORT=8788 diff --git a/hub/test/integration/real-model.test.ts b/hub/test/integration/real-model.test.ts new file mode 100644 index 0000000..bfd8e2e --- /dev/null +++ b/hub/test/integration/real-model.test.ts @@ -0,0 +1,143 @@ +/** + * Real-model integration test — exercises the full agent tool-calls loop against + * a live OpenRouter model + real cph binary + real workspace. + * + * Unlike MockProvider tests, this verifies: + * - generateText's tool-calls loop with a real model (GLM/Claude/…) + * - model returns tool_calls → runner dispatches → tool_result back → continue + * - cph_check / read_file / list_files against a real engineering-file tree + * - transcript JSONL read/append on a real filesystem + * + * Skipped automatically when OPENROUTER_API_KEY is not set (CI without secrets, + * or local dev without a key). To run locally: + * cp .env.example .env # fill in OPENROUTER_API_KEY + * npx vitest run test/integration/real-model.test.ts + */ +import { describe, it, expect } from "vitest"; +import { cp, rm, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import "dotenv/config"; +import { createModelFactory } from "../../src/agent/provider.js"; +import { ToolRegistry, feishuContextTool } from "../../src/agent/tools.js"; +import { readFileTool, writeFileTool, listFilesTool } from "../../src/agent/workspace.js"; +import { cphCheckTool, cphBuildTool } from "../../src/agent/cph.js"; +import { runAgent } from "../../src/agent/runner.js"; +import type { FeishuRuntime } from "../../src/feishu/client.js"; +import type { PrismaClient } from "@prisma/client"; + +const API_KEY = process.env["OPENROUTER_API_KEY"]; +const HAS_KEY = API_KEY !== undefined && API_KEY !== ""; +const describeOrSkip = HAS_KEY ? describe : describe.skip; + +const EXAMPLES_DIR = join(process.cwd(), "..", "examples", "TH-141"); +const MODEL = process.env["SMOKE_MODEL"] ?? "z-ai/glm-4.6"; + +// Stub prisma: runner touches it for lock heartbeat + message persistence, +// neither of which need a real DB for this test. +const stubPrisma = { + projectAgentLock: { update: async () => ({}), deleteMany: async () => ({}) }, + agentMessage: { createMany: async () => ({}) }, + agentFileChange: { createMany: async () => ({}) }, +} as unknown as PrismaClient; + +const stubFeishuRt: FeishuRuntime = { + client: { + im: { v1: { message: { create: async () => ({}), get: async () => ({ data: {} }), list: async () => ({ data: {} }) } } }, + } as never, + logger: { info() {}, warn() {}, error() {}, debug() {}, fatal() {}, child() { return this; }, level: "silent" } as never, +}; + +describeOrSkip("real model integration (OpenRouter)", () => { + let ws: string; + + async function setupWorkspace(): Promise { + ws = join(process.cwd(), ".test-ws-real"); + await rm(ws, { recursive: true, force: true }); + await cp(EXAMPLES_DIR, ws, { recursive: true }); + return ws; + } + + async function teardownWorkspace(): Promise { + await rm(ws, { recursive: true, force: true }); + } + + it("runs a multi-step tool-calls loop: list files → cph check → answer", async () => { + const workspace = await setupWorkspace(); + try { + const tools = new ToolRegistry(); + tools.register("feishu_read_context", feishuContextTool(async () => JSON.stringify({ stub: true }), stubFeishuRt)); + tools.register("read_file", readFileTool); + tools.register("write_file", writeFileTool); + tools.register("list_files", listFilesTool); + tools.register("cph_check", cphCheckTool); + tools.register("cph_build", cphBuildTool); + + const modelFactory = createModelFactory(); + const transcript = join(workspace, ".cph", "sessions", "real-test.jsonl"); + + const result = await runAgent(modelFactory, tools, { + runId: "real-test-run", + projectId: "real-test-project", + prompt: "列出工程文件的目录结构,然后跑 cph check,用中文告诉我结果。", + model: MODEL, + project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace }, + prisma: stubPrisma, + systemPrompt: "你是一个教研工程文件的助手。你可以读写工程文件、跑 cph check 校验、跑 cph build 渲染。请用中文回复。", + transcriptPath: transcript, + maxIterations: 10, + }); + + // The model should complete (not hit the iteration cap or fail). + expect(result.status).toBe("completed"); + + // It should have produced messages beyond just the initial prompt. + expect(result.messages.length).toBeGreaterThan(1); + + // The transcript file should have been written. + const transcriptContent = await readFile(transcript, "utf8"); + expect(transcriptContent.length).toBeGreaterThan(0); + const lines = transcriptContent.trim().split("\n"); + expect(lines.length).toBeGreaterThan(1); + // Each line should be valid JSON. + for (const line of lines) { + expect(() => JSON.parse(line)).not.toThrow(); + } + } finally { + await teardownWorkspace(); + } + }, 60_000); // 60s timeout — real model calls take time. + + it("can write a file and read it back through the agent loop", async () => { + const workspace = await setupWorkspace(); + try { + const tools = new ToolRegistry(); + tools.register("read_file", readFileTool); + tools.register("write_file", writeFileTool); + tools.register("list_files", listFilesTool); + tools.register("cph_check", cphCheckTool); + tools.register("cph_build", cphBuildTool); + + const modelFactory = createModelFactory(); + + const result = await runAgent(modelFactory, tools, { + runId: "real-test-run-2", + projectId: "real-test-project", + prompt: "在工程文件根目录创建一个文件 note.txt,内容写'测试笔记',然后用 read_file 读回来,告诉我内容。", + model: MODEL, + project: { projectId: "real-test-project", boundChatId: "chat-test", workspaceDir: workspace }, + prisma: stubPrisma, + systemPrompt: "你是一个教研工程文件的助手。请用中文回复。", + maxIterations: 10, + }); + + expect(result.status).toBe("completed"); + + // The file should actually exist on disk (the agent called write_file). + const { readFile } = await import("node:fs/promises"); + const note = await readFile(join(workspace, "note.txt"), "utf8"); + expect(note).toContain("测试笔记"); + } finally { + await teardownWorkspace(); + } + }, 60_000); +});