diff --git a/docs/para-26071100-feishu-setup.md b/docs/para-26071100-feishu-setup.md index da2ceee..fb13832 100644 --- a/docs/para-26071100-feishu-setup.md +++ b/docs/para-26071100-feishu-setup.md @@ -32,7 +32,7 @@ App ID 通常以 `cli_` 开头，可以写入交付单。App Secret 必须通过 | 接收群聊中 @ 机器人的消息 | `im:message.group_at_msg:readonly` | | 以应用身份发送消息 | `im:message:send_as_bot` | | 读取触发消息和线程上下文 | `im:message:readonly` | -| 获取与上传图片或文件 | `im:resource` | +| 获取消息中的图片/文件，并向飞书上传图片或文件（含 Agent 回答中的图片发送） | `im:resource` | | 添加、删除消息表情回复 | `im:message.reactions:write_only` | | 获取用户基本信息 | `contact:user.base:readonly` | | 获取用户基本资料 | `contact:user.basic_profile:readonly` | @@ -66,6 +66,9 @@ Educraft 机器人以应用身份调用上述 API，因此这些 scope 全部放 如果 API 调试台提示缺少更细粒度权限，请把错误提示和发生时间截图给部署人员。不要自行开通通讯录全量读取等超出本表的权限。 +说明：`im:resource` 既用于下载用户发来的图片/文件，也用于 Agent 回复时把本地或远程图片上传为飞书 `image_key` 后嵌入消息卡片。缺少该权限时，带图回答会发送失败或降级为无图文本。已开通该 scope 的存量应用一般无需新增权限，但若权限尚未随最新版本发布，请创建新版本并审核发布。 + + ## 4. 配置事件与卡片回调 进入“事件与回调”。 diff --git a/hub/package-lock.json b/hub/package-lock.json index 126d0b6..384f9ce 100644 --- a/hub/package-lock.json +++ b/hub/package-lock.json @@ -1,12 +1,12 @@ { "name": "@paradigm/hub", - "version": "0.0.35", + "version": "0.0.36", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@paradigm/hub", - "version": "0.0.35", + "version": "0.0.36", "dependencies": { "@alicloud/credentials": "^2.4.5", "@alicloud/docmind-api20220711": "^1.4.15", diff --git a/hub/package.json b/hub/package.json index b8df102..f507746 100644 --- a/hub/package.json +++ b/hub/package.json @@ -1,6 +1,6 @@ { "name": "@paradigm/hub", - "version": "0.0.35", + "version": "0.0.36", "private": true, "type": "module", "engines": { diff --git a/hub/src/feishu/card/builder.ts b/hub/src/feishu/card/builder.ts index 852e22b..54b33bf 100644 --- a/hub/src/feishu/card/builder.ts +++ b/hub/src/feishu/card/builder.ts @@ -12,6 +12,7 @@ */ import type { ToolUseTraceStep } from "./trace-store.js"; +import type { CardContentSegment } from "../outboundImages.js"; // --------------------------------------------------------------------------- // Types @@ -58,6 +59,7 @@ function toolIcon(toolName: string): string { export function buildAgentCard(params: { phase: CardPhase; text: string; + contentSegments?: readonly CardContentSegment[] | undefined; reasoningText: string | undefined; toolUseSteps: ToolUseTraceStep[]; toolUseElapsedMs: number | undefined; @@ -65,19 +67,19 @@ export function buildAgentCard(params: { interrupted: boolean | undefined; runId: string | undefined; }): Record { - const { phase, text, reasoningText, toolUseSteps, toolUseElapsedMs, isError, interrupted } = params; + const { phase, text, contentSegments, reasoningText, toolUseSteps, toolUseElapsedMs, isError, interrupted } = params; const elements: unknown[] = []; // Tool-use panel (always present if there are steps) if (toolUseSteps.length > 0) { elements.push(buildToolUsePanel(toolUseSteps, toolUseElapsedMs, phase !== "complete")); - } else if (phase === "thinking" || (phase === "streaming" && text === "")) { + } else if (phase === "thinking" || (phase === "streaming" && text === "" && (contentSegments === undefined || contentSegments.length === 0))) { elements.push(buildPendingToolUsePanel()); } // Reasoning panel if (reasoningText !== undefined && reasoningText !== "") { - if (phase === "streaming" && text === "") { + if (phase === "streaming" && text === "" && (contentSegments === undefined || contentSegments.length === 0)) { // Still thinking: show reasoning inline elements.push({ tag: "markdown", @@ -90,12 +92,11 @@ export function buildAgentCard(params: { } } - // Main text content - if (text !== "") { - elements.push({ - tag: "markdown", - content: truncateText(text, MAX_TEXT_LENGTH), - }); + // Main answer: either materialized segments (markdown + Feishu-hosted images) + // or a single markdown block. + const answerElements = buildAnswerElements(text, contentSegments); + if (answerElements.length > 0) { + elements.push(...answerElements); } else if (phase === "thinking" && toolUseSteps.length === 0 && (reasoningText === undefined || reasoningText === "")) { elements.push({ tag: "markdown", @@ -401,6 +402,43 @@ function escapeMarkdown(value: string): string { return value.replace(/\\/g, "\\\\").replace(/([`*_{}[\]<>])/g, "\\$1"); } +function buildAnswerElements( + text: string, + contentSegments: readonly CardContentSegment[] | undefined, +): unknown[] { + if (contentSegments !== undefined && contentSegments.length > 0) { + const elements: unknown[] = []; + let remaining = MAX_TEXT_LENGTH; + for (const segment of contentSegments) { + if (segment.type === "image") { + elements.push({ + tag: "img", + img_key: segment.imgKey, + alt: { tag: "plain_text", content: segment.alt }, + mode: "fit_horizontal", + preview: true, + }); + continue; + } + if (segment.content === "" || remaining <= 0) continue; + const slice = segment.content.length <= remaining + ? segment.content + : truncateText(segment.content, remaining); + remaining -= slice.length; + elements.push({ + tag: "markdown", + content: slice, + }); + } + return elements; + } + if (text === "") return []; + return [{ + tag: "markdown", + content: truncateText(text, MAX_TEXT_LENGTH), + }]; +} + function truncateText(value: string, maxLength: number): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`; } diff --git a/hub/src/feishu/card/streaming-card.ts b/hub/src/feishu/card/streaming-card.ts index 59bbd2a..3ab5473 100644 --- a/hub/src/feishu/card/streaming-card.ts +++ b/hub/src/feishu/card/streaming-card.ts @@ -18,10 +18,13 @@ * 4. onToolEnd(name, id, input, result?, error?) — complete a tool step * 5. finish(finalText) — flush + transition to complete card * 6. fail(errorText) — flush + transition to error card + * + * On finish, markdown image references (`![](url|path)`) are downloaded / + * read, uploaded to Feishu as message images, and embedded as native card + * `img` elements so external URLs never hit Feishu content-security checks. */ - import type { FeishuRuntime, SendMessageOptions } from "../client.js"; -import { sendCard, patchCard, sendText } from "../client.js"; +import { sendCard, patchCard, sendText, sendLongText } from "../client.js"; import { DEFAULT_MAX_MESSAGE_LENGTH, splitAtBoundary } from "../textStream.js"; import { startToolUseTraceRun, @@ -31,6 +34,12 @@ import { getToolUseTraceSteps, } from "./trace-store.js"; import { buildAgentCard, type CardPhase } from "./builder.js"; +import { + type CardContentSegment, + maskMarkdownImagesForStreaming, + materializeAnswerSegments, + sendImageMessage, +} from "../outboundImages.js"; export interface StreamingCardSink { readonly create: (card: Record) => Promise; @@ -44,6 +53,11 @@ export interface StreamingCardOptions { readonly sendOptions?: SendMessageOptions | undefined; readonly patchIntervalMs: number | undefined; readonly maxMessageLength: number | undefined; + /** Project workspace root; required to resolve local image paths. */ + readonly workspaceRoot?: string | undefined; + /** Project workspace directory; required to resolve local image paths. */ + readonly workspaceDir?: string | undefined; + readonly maxImageBytes?: number | undefined; } const DEFAULT_PATCH_INTERVAL_MS = 400; @@ -65,6 +79,9 @@ export class StreamingAgentCard { private readonly sendOptions: SendMessageOptions | undefined; private readonly patchIntervalMs: number; private readonly maxMessageLength: number; + private readonly workspaceRoot: string | undefined; + private readonly workspaceDir: string | undefined; + private readonly maxImageBytes: number | undefined; constructor(options: StreamingCardOptions) { this.runId = options.runId; @@ -73,6 +90,9 @@ export class StreamingAgentCard { this.sendOptions = options.sendOptions; this.patchIntervalMs = options.patchIntervalMs ?? DEFAULT_PATCH_INTERVAL_MS; this.maxMessageLength = options.maxMessageLength ?? DEFAULT_MAX_MESSAGE_LENGTH; + this.workspaceRoot = options.workspaceRoot; + this.workspaceDir = options.workspaceDir; + this.maxImageBytes = options.maxImageBytes; startToolUseTraceRun(this.runId); } @@ -106,23 +126,43 @@ export class StreamingAgentCard { recordToolUseEnd({ runId: this.runId, ...params }); this.scheduleFlush(); } - async finish(fallbackText: string, options: { readonly interrupted?: boolean; readonly footerText?: string | undefined } = {}): Promise { + + async finish( + fallbackText: string, + options: { readonly interrupted?: boolean; readonly footerText?: string | undefined } = {}, + ): Promise { await this.flushChain; this.interrupted = options.interrupted === true; const footerText = options.footerText ?? ""; const fallbackWithFooter = appendFooter(fallbackText, footerText); try { + let answerText = + this.text.length > 0 ? appendFooter(this.text, footerText) : fallbackWithFooter; + this.text = answerText; + + const { segments, unresolved } = await materializeAnswerSegments(answerText, { + rt: this.rt, + workspaceRoot: this.workspaceRoot, + workspaceDir: this.workspaceDir, + maxImageBytes: this.maxImageBytes, + }); + if (unresolved.length > 0) { + this.rt.logger.warn( + { runId: this.runId, unresolvedCount: unresolved.length, unresolved: unresolved.slice(0, 5) }, + "some answer images could not be uploaded to Feishu", + ); + } + let updated = true; - if (this.text.length > 0) { - this.text = appendFooter(this.text, footerText); - updated = await this.flushCard("complete", this.text); - } else if (this.currentMessageId === null && fallbackWithFooter.length > 0) { - // No streaming text was sent. If we never created a card, send one now. - this.text = fallbackWithFooter; - updated = await this.flushCard("complete", this.text); + if (answerText.length > 0 || segments.length > 0) { + updated = await this.flushCard("complete", answerText, false, segments); } else if (this.currentMessageId !== null) { - // Patch the existing card with the final text. - updated = await this.flushCard("complete", fallbackWithFooter); + updated = await this.flushCard("complete", "", false, []); + } + + if (!updated) { + // Card path failed (e.g. residual content policy). Deliver text + standalone images. + updated = await this.deliverPlainFallback(segments, answerText); } if (!updated && this.interrupted) { await sendText(this.rt, this.chatId, "\u5DF2\u4E2D\u65AD\u5F53\u524D\u8FD0\u884C\u3002", this.sendOptions); @@ -166,15 +206,30 @@ export class StreamingAgentCard { return this.flushCard(this.currentPhase(), this.text); } - private async flushCard(phase: CardPhase, text: string, isError = false): Promise { - const chunks = splitAtBoundary(text, this.maxMessageLength); - const firstChunk = chunks[0]; - if (firstChunk === undefined) return true; + private async flushCard( + phase: CardPhase, + text: string, + isError = false, + contentSegments?: readonly CardContentSegment[], + ): Promise { + // During live streaming, strip image URLs so Feishu never fetches remote + // ranks mid-run. Materialized segments are only used on the complete pass. + const displayText = + phase === "complete" && contentSegments !== undefined + ? text + : maskMarkdownImagesForStreaming(text); + const chunks = splitAtBoundary(displayText, this.maxMessageLength); + const firstChunk = chunks[0] ?? ""; + // When we have segments (complete+images), keep first-card complete content + // on segments only; overflow text (rare) falls back to plain chunked cards. const toolUseSteps = getToolUseTraceSteps(this.runId); const card = buildAgentCard({ phase, - text: firstChunk, + text: contentSegments !== undefined && contentSegments.length > 0 ? "" : firstChunk, + contentSegments: contentSegments !== undefined && contentSegments.length > 0 + ? contentSegments + : undefined, reasoningText: this.reasoningText || undefined, toolUseSteps, toolUseElapsedMs: this.toolUseElapsedMs, @@ -186,43 +241,89 @@ export class StreamingAgentCard { if (this.currentMessageId === null) { this.currentMessageId = await sendCard(this.rt, this.chatId, card, this.sendOptions); let updated = this.currentMessageId !== null; - // Send overflow chunks as new messages (rare for agent output) - for (const chunk of chunks.slice(1)) { - const overflowCard = buildAgentCard({ - phase, - text: chunk, - reasoningText: undefined, - toolUseSteps: [], - toolUseElapsedMs: undefined, - isError, - interrupted: this.interrupted, - runId: undefined, - }); - const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions); - updated = updated && overflowMessageId !== null; - this.currentMessageId = overflowMessageId; - } - return updated; - } else { - let updated = await patchCard(this.rt, this.currentMessageId, card); - // For overflow, create new messages - for (const chunk of chunks.slice(1)) { - const overflowCard = buildAgentCard({ - phase, - text: chunk, - reasoningText: undefined, - toolUseSteps: [], - toolUseElapsedMs: undefined, - isError, - interrupted: this.interrupted, - runId: undefined, - }); - const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions); - updated = updated && overflowMessageId !== null; - this.currentMessageId = overflowMessageId; + // Send overflow chunks as new messages (rare for agent output). Segments + // already include the whole answer; only plain text overflows. + if (contentSegments === undefined || contentSegments.length === 0) { + for (const chunk of chunks.slice(1)) { + const overflowCard = buildAgentCard({ + phase, + text: chunk, + reasoningText: undefined, + toolUseSteps: [], + toolUseElapsedMs: undefined, + isError, + interrupted: this.interrupted, + runId: undefined, + }); + const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions); + updated = updated && overflowMessageId !== null; + this.currentMessageId = overflowMessageId; + } } return updated; } + + let updated = await patchCard(this.rt, this.currentMessageId, card); + if (contentSegments === undefined || contentSegments.length === 0) { + for (const chunk of chunks.slice(1)) { + const overflowCard = buildAgentCard({ + phase, + text: chunk, + reasoningText: undefined, + toolUseSteps: [], + toolUseElapsedMs: undefined, + isError, + interrupted: this.interrupted, + runId: undefined, + }); + const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions); + updated = updated && overflowMessageId !== null; + this.currentMessageId = overflowMessageId; + } + } + return updated; + } + + private async deliverPlainFallback( + segments: readonly CardContentSegment[], + answerText: string, + ): Promise { + const textParts: string[] = []; + const imageKeys: string[] = []; + if (segments.length > 0) { + for (const segment of segments) { + if (segment.type === "markdown") { + if (segment.content.trim() !== "") textParts.push(segment.content); + } else { + imageKeys.push(segment.imgKey); + } + } + } else if (answerText.trim() !== "") { + textParts.push(maskMarkdownImagesForStreaming(answerText)); + } + + let any = false; + if (textParts.length > 0) { + const messageId = await sendLongText( + this.rt, + this.chatId, + textParts.join("\n\n"), + this.sendOptions, + ); + any = messageId !== null; + } + for (const imageKey of imageKeys) { + try { + const messageId = await sendImageMessage(this.rt, this.chatId, imageKey, this.sendOptions); + any = any || messageId !== null; + } catch (error) { + this.rt.logger.warn( + { runId: this.runId, err: error instanceof Error ? error.message : String(error) }, + "standalone image fallback failed", + ); + } + } + return any; } private currentPhase(): CardPhase { @@ -235,5 +336,5 @@ export class StreamingAgentCard { function appendFooter(text: string, footerText: string): string { if (footerText === "") return text; if (text === "") return footerText; - return `${text.trimEnd()}\n\n${footerText}`; + return `${text}\n\n${footerText}`; } diff --git a/hub/src/feishu/client.ts b/hub/src/feishu/client.ts index d9d7b64..4daf3dc 100644 --- a/hub/src/feishu/client.ts +++ b/hub/src/feishu/client.ts @@ -316,7 +316,8 @@ export async function sendCard( { msgType: "interactive", content: JSON.stringify(card) }, options, ); - } catch { + } catch (e) { + rt.logger.warn({ chatId, err: errorText(e) }, "sendCard failed"); return null; } } @@ -334,7 +335,7 @@ export async function patchCard(rt: FeishuRuntime, messageId: string, card: Reco }); return true; } catch (e) { - rt.logger.warn({ messageId, err: e instanceof Error ? e.message : String(e) }, "patchCard failed"); + rt.logger.warn({ messageId, err: errorText(e) }, "patchCard failed"); return false; } } diff --git a/hub/src/feishu/outboundImages.ts b/hub/src/feishu/outboundImages.ts new file mode 100644 index 0000000..a4efb33 --- /dev/null +++ b/hub/src/feishu/outboundImages.ts @@ -0,0 +1,389 @@ +/** + * Resolve markdown image references in agent answers into Feishu-hosted + * image_keys so cards can embed them without remote URLs (which trip Feishu + * content-security controls). + */ +import { isIP } from "node:net"; +import type { FeishuRuntime } from "./client.js"; +import { withRetry } from "./client.js"; +import { + WorkspaceFileBoundaryError, + readWorkspaceFileNoFollow, +} from "../security/workspaceFiles.js"; + +export const FEISHU_MAX_IMAGE_BYTES = 10 * 1024 * 1024; +export const DEFAULT_MAX_OUTBOUND_IMAGES = 10; + +const IMAGE_MARKDOWN_RE = /!\[([^\]\n]*)\]\(([^)\n]+)\)/g; +const FENCED_CODE_RE = /```[\s\S]*?```/g; + +export type CardContentSegment = + | { readonly type: "markdown"; readonly content: string } + | { readonly type: "image"; readonly imgKey: string; readonly alt: string }; + +export interface MarkdownImageRef { + readonly fullMatch: string; + readonly alt: string; + readonly src: string; + readonly index: number; + readonly length: number; +} + +export interface OutboundImageContext { + readonly rt: FeishuRuntime; + readonly workspaceRoot?: string | undefined; + readonly workspaceDir?: string | undefined; + readonly maxImageBytes?: number | undefined; + readonly maxImages?: number | undefined; + readonly fetchImpl?: typeof fetch | undefined; +} + +type ImageCreateResponse = { + image_key?: string; + data?: { image_key?: string }; +} | null; + +type MessageCreateResponse = { + message_id?: string; + data?: { message_id?: string }; +} | null; + +/** Streaming-safe view: drop markdown image URLs so partial cards do not hit Feishu URL checks. */ +export function maskMarkdownImagesForStreaming(text: string): string { + return rewriteMarkdownImagesOutsideCode(text, (ref) => { + const alt = ref.alt.trim(); + return alt === "" ? "\u3010\u56fe\u7247\u3011" : alt; + }); +} + +export function findMarkdownImagesOutsideCode(text: string): MarkdownImageRef[] { + const blocked = blockedRanges(text); + const refs: MarkdownImageRef[] = []; + IMAGE_MARKDOWN_RE.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = IMAGE_MARKDOWN_RE.exec(text)) !== null) { + const index = match.index; + if (blocked.some((range) => index >= range.start && index < range.end)) continue; + const fullMatch = match[0]; + const alt = match[1] ?? ""; + const rawSrc = (match[2] ?? "").trim(); + const src = stripUrlTitle(rawSrc); + if (src === "") continue; + refs.push({ fullMatch, alt, src, index, length: fullMatch.length }); + } + return refs; +} + +/** + * Upload reachable markdown images and split the answer into card segments + * (markdown + Feishu img elements). Unresolved images become visible alt text. + */ +export async function materializeAnswerSegments( + text: string, + ctx: OutboundImageContext, +): Promise<{ segments: CardContentSegment[]; unresolved: string[] }> { + const refs = findMarkdownImagesOutsideCode(text); + if (refs.length === 0) { + return { + segments: text === "" ? [] : [{ type: "markdown", content: text }], + unresolved: [], + }; + } + + const maxImages = ctx.maxImages ?? DEFAULT_MAX_OUTBOUND_IMAGES; + const maxBytes = ctx.maxImageBytes ?? FEISHU_MAX_IMAGE_BYTES; + const keyBySrc = new Map(); + const unresolved: string[] = []; + const uniqueSrcs: string[] = []; + for (const ref of refs) { + if (!uniqueSrcs.includes(ref.src)) uniqueSrcs.push(ref.src); + } + + for (const src of uniqueSrcs.slice(0, maxImages)) { + try { + const bytes = await resolveOutboundImageBytes(src, ctx, maxBytes); + if (bytes === null) { + unresolved.push(src); + continue; + } + const imageKey = await uploadMessageImage(ctx.rt, bytes); + keyBySrc.set(src, imageKey); + } catch (error) { + ctx.rt.logger.warn( + { src, err: error instanceof Error ? error.message : String(error) }, + "outbound image materialize failed", + ); + unresolved.push(src); + } + } + for (const src of uniqueSrcs.slice(maxImages)) { + unresolved.push(src); + } + + const segments: CardContentSegment[] = []; + let cursor = 0; + for (const ref of refs) { + if (ref.index > cursor) { + pushMarkdown(segments, text.slice(cursor, ref.index)); + } + const imageKey = keyBySrc.get(ref.src); + if (imageKey !== undefined) { + segments.push({ + type: "image", + imgKey: imageKey, + alt: ref.alt.trim() === "" ? "\u56fe\u7247" : ref.alt.trim(), + }); + } else { + const alt = ref.alt.trim(); + pushMarkdown(segments, alt === "" ? "\u3010\u56fe\u7247\u3011" : alt); + } + cursor = ref.index + ref.length; + } + if (cursor < text.length) { + pushMarkdown(segments, text.slice(cursor)); + } + return { segments, unresolved }; +} + +export async function uploadMessageImage(rt: FeishuRuntime, image: Buffer): Promise { + if (image.byteLength === 0) { + throw new Error("image is empty"); + } + if (image.byteLength > FEISHU_MAX_IMAGE_BYTES) { + throw new Error(`image exceeds Feishu limit of ${FEISHU_MAX_IMAGE_BYTES} bytes`); + } + const client = rt.client as unknown as { + im: { v1: { image: { create: (p: unknown) => Promise } } }; + }; + const res = await withRetry(async () => + client.im.v1.image.create({ + data: { image_type: "message", image }, + }), + ); + const imageKey = res?.image_key ?? res?.data?.image_key; + if (imageKey === undefined || imageKey === "") { + throw new Error("Feishu image upload response is missing image_key"); + } + return imageKey; +} + +/** Send a standalone image message (fallback if card embed is unavailable). */ +export async function sendImageMessage( + rt: FeishuRuntime, + chatId: string, + imageKey: string, + options?: { readonly replyToMessageId?: string | undefined }, +): Promise { + const client = rt.client as unknown as { + im: { + v1: { + message: { + create: (p: unknown) => Promise; + reply: (p: unknown) => Promise; + }; + }; + }; + }; + const replyTo = options?.replyToMessageId; + if (replyTo !== undefined && replyTo !== "") { + const res = await client.im.v1.message.reply({ + path: { message_id: replyTo }, + data: { msg_type: "image", content: JSON.stringify({ image_key: imageKey }) }, + }); + return res?.data?.message_id ?? res?.message_id ?? null; + } + const res = await client.im.v1.message.create({ + params: { receive_id_type: "chat_id" }, + data: { + receive_id: chatId, + msg_type: "image", + content: JSON.stringify({ image_key: imageKey }), + }, + }); + return res?.data?.message_id ?? res?.message_id ?? null; +} + +export async function resolveOutboundImageBytes( + src: string, + ctx: OutboundImageContext, + maxBytes: number, +): Promise { + if (isRemoteUrl(src)) { + return fetchRemoteImage(src, ctx.fetchImpl ?? fetch, maxBytes); + } + const root = ctx.workspaceRoot?.trim(); + const dir = ctx.workspaceDir?.trim(); + if (root === undefined || root === "" || dir === undefined || dir === "") { + return null; + } + try { + const file = await readWorkspaceFileNoFollow(root, dir, src, maxBytes); + return file.data; + } catch (error) { + if (error instanceof WorkspaceFileBoundaryError && error.reason === "not_found") { + return null; + } + throw error; + } +} + +function rewriteMarkdownImagesOutsideCode( + text: string, + replace: (ref: MarkdownImageRef) => string, +): string { + const refs = findMarkdownImagesOutsideCode(text); + if (refs.length === 0) return text; + let out = ""; + let cursor = 0; + for (const ref of refs) { + out += text.slice(cursor, ref.index); + out += replace(ref); + cursor = ref.index + ref.length; + } + out += text.slice(cursor); + return out; +} + +function pushMarkdown(segments: CardContentSegment[], content: string): void { + if (content === "") return; + const last = segments[segments.length - 1]; + if (last !== undefined && last.type === "markdown") { + segments[segments.length - 1] = { type: "markdown", content: last.content + content }; + return; + } + segments.push({ type: "markdown", content }); +} + +function blockedRanges(text: string): Array<{ start: number; end: number }> { + const ranges: Array<{ start: number; end: number }> = []; + FENCED_CODE_RE.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = FENCED_CODE_RE.exec(text)) !== null) { + ranges.push({ start: match.index, end: match.index + match[0].length }); + } + return ranges; +} + +function stripUrlTitle(raw: string): string { + const trimmed = raw.trim(); + // Markdown optional title: url "title" or url 'title' + const titled = /^(\S+)\s+(".*"|'.*')$/.exec(trimmed); + return (titled?.[1] ?? trimmed).trim(); +} + +function isRemoteUrl(src: string): boolean { + try { + const url = new URL(src); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +async function fetchRemoteImage( + src: string, + fetchImpl: typeof fetch, + maxBytes: number, +): Promise { + let url: URL; + try { + url = new URL(src); + } catch { + return null; + } + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (!isPublicHttpHost(url.hostname)) return null; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15_000); + try { + const response = await fetchImpl(url, { + method: "GET", + redirect: "manual", + signal: controller.signal, + headers: { accept: "image/*,*/*;q=0.8" }, + }); + // One safe redirect hop to another public http(s) host. + if (response.status >= 300 && response.status < 400) { + const location = response.headers.get("location"); + if (location === null || location === "") return null; + let redirected: URL; + try { + redirected = new URL(location, url); + } catch { + return null; + } + if (redirected.protocol !== "http:" && redirected.protocol !== "https:") return null; + if (!isPublicHttpHost(redirected.hostname)) return null; + const second = await fetchImpl(redirected, { + method: "GET", + redirect: "manual", + signal: controller.signal, + headers: { accept: "image/*,*/*;q=0.8" }, + }); + return readImageBody(second, maxBytes); + } + return readImageBody(response, maxBytes); + } finally { + clearTimeout(timer); + } +} + +async function readImageBody(response: Response, maxBytes: number): Promise { + if (!response.ok) return null; + const contentType = (response.headers.get("content-type") ?? "").toLowerCase(); + if ( + contentType !== "" && + !contentType.startsWith("image/") && + !contentType.includes("octet-stream") && + (contentType.startsWith("text/") || contentType.includes("json") || contentType.includes("html")) + ) { + return null; + } + const contentLength = Number(response.headers.get("content-length") ?? "NaN"); + if (Number.isFinite(contentLength) && contentLength > maxBytes) return null; + const buf = Buffer.from(await response.arrayBuffer()); + if (buf.byteLength === 0 || buf.byteLength > maxBytes) return null; + return buf; +} + +function isPublicHttpHost(hostname: string): boolean { + const host = hostname.trim().toLowerCase().replace(/\.$/, ""); + if (host === "" || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) { + return false; + } + if (host === "0.0.0.0" || host === "::" || host === "[::]" || host === "::1" || host === "[::1]") { + return false; + } + const unbracketed = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host; + const ipVersion = isIP(unbracketed); + if (ipVersion === 4) return !isPrivateIPv4(unbracketed); + if (ipVersion === 6) return !isPrivateIPv6(unbracketed); + return true; +} + +function isPrivateIPv4(ip: string): boolean { + const parts = ip.split(".").map((part) => Number(part)); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) { + return true; + } + const a = parts[0]!; + const b = parts[1]!; + if (a === 10 || a === 127 || a === 0) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + if (a >= 224) return true; // multicast / reserved + return false; +} + +function isPrivateIPv6(ip: string): boolean { + const normalized = ip.toLowerCase(); + if (normalized === "::1") return true; + if (normalized.startsWith("fc") || normalized.startsWith("fd")) return true; // unique local + if (normalized.startsWith("fe80:")) return true; // link-local + const mapped = /^:ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(normalized); + if (mapped?.[1] !== undefined) return isPrivateIPv4(mapped[1]); + return false; +} diff --git a/hub/src/feishu/trigger.ts b/hub/src/feishu/trigger.ts index d3797c7..d5eb40a 100644 --- a/hub/src/feishu/trigger.ts +++ b/hub/src/feishu/trigger.ts @@ -496,6 +496,9 @@ export function makeTriggerHandler(deps: TriggerDeps): TriggerHandler { sendOptions, patchIntervalMs: undefined, maxMessageLength: undefined, + workspaceRoot: projectWorkspaceRoot, + workspaceDir: project.workspaceDir, + maxImageBytes: deps.resourceLimits?.maxBytesPerFile, }); const fileDeliveryMcpServer = createFileDeliveryMcpServer({ rt, diff --git a/hub/test/unit/feishu-outbound-images.test.ts b/hub/test/unit/feishu-outbound-images.test.ts new file mode 100644 index 0000000..472faf8 --- /dev/null +++ b/hub/test/unit/feishu-outbound-images.test.ts @@ -0,0 +1,163 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { FeishuRuntime } from "../../src/feishu/client.js"; +import { + findMarkdownImagesOutsideCode, + maskMarkdownImagesForStreaming, + materializeAnswerSegments, + uploadMessageImage, +} from "../../src/feishu/outboundImages.js"; +import { buildAgentCard } from "../../src/feishu/card/builder.js"; + +const temps: string[] = []; + +afterEach(async () => { + await Promise.all(temps.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe("outbound markdown image parsing", () => { + it("finds image refs outside fenced code blocks", () => { + const text = [ + "See diagram:", + "![plot](https://cdn.example.com/a.png)", + "", + "```md", + "![not-this](https://cdn.example.com/b.png)", + "```", + "![local](assets/x.png \"title\")", + ].join("\n"); + + const refs = findMarkdownImagesOutsideCode(text); + expect(refs.map((ref) => ref.src)).toEqual([ + "https://cdn.example.com/a.png", + "assets/x.png", + ]); + }); + + it("masks image urls for streaming cards", () => { + expect(maskMarkdownImagesForStreaming("before ![alt text](https://x/y.png) after")).toBe( + "before alt text after", + ); + expect(maskMarkdownImagesForStreaming("![](https://x/y.png)")).toBe("【图片】"); + }); +}); + +describe("materializeAnswerSegments", () => { + it("uploads remote and workspace images and builds card segments", async () => { + const workspaceRoot = await mkdtemp(join(tmpdir(), "cph-img-root-")); + temps.push(workspaceRoot); + const workspaceDir = join(workspaceRoot, "project"); + await mkdir(join(workspaceDir, "assets"), { recursive: true }); + await writeFile( + join(workspaceDir, "assets", "local.png"), + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + ); + + const imageCreate = vi + .fn() + .mockResolvedValueOnce({ image_key: "img_remote_1" }) + .mockResolvedValueOnce({ image_key: "img_local_1" }); + const rt = mockRuntime({ imageCreate }); + + const fetchImpl = vi.fn(async () => + new Response(Buffer.from("remote-bytes"), { + status: 200, + headers: { "content-type": "image/png" }, + }), + ); + + const text = "Intro\n\n![remote](https://cdn.example.com/r.png)\n\nAnd local ![local](assets/local.png)\nDone."; + const { segments, unresolved } = await materializeAnswerSegments(text, { + rt, + workspaceRoot, + workspaceDir, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(unresolved).toEqual([]); + expect(imageCreate).toHaveBeenCalledTimes(2); + expect(segments).toEqual([ + { type: "markdown", content: "Intro\n\n" }, + { type: "image", imgKey: "img_remote_1", alt: "remote" }, + { type: "markdown", content: "\n\nAnd local " }, + { type: "image", imgKey: "img_local_1", alt: "local" }, + { type: "markdown", content: "\nDone." }, + ]); + + const card = buildAgentCard({ + phase: "complete", + text: "", + contentSegments: segments, + reasoningText: undefined, + toolUseSteps: [], + toolUseElapsedMs: undefined, + isError: false, + interrupted: false, + runId: undefined, + }); + expect(card.elements).toEqual(expect.arrayContaining([ + expect.objectContaining({ tag: "img", img_key: "img_remote_1" }), + expect.objectContaining({ tag: "img", img_key: "img_local_1" }), + expect.objectContaining({ tag: "markdown", content: "Intro\n\n" }), + ])); + }); + + it("rejects private remote hosts", async () => { + const imageCreate = vi.fn(); + const rt = mockRuntime({ imageCreate }); + const fetchImpl = vi.fn(); + const { segments, unresolved } = await materializeAnswerSegments( + "![x](http://127.0.0.1/secret.png)", + { rt, fetchImpl: fetchImpl as unknown as typeof fetch }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(imageCreate).not.toHaveBeenCalled(); + expect(unresolved).toEqual(["http://127.0.0.1/secret.png"]); + expect(segments).toEqual([{ type: "markdown", content: "x" }]); + }); +}); + +describe("uploadMessageImage", () => { + it("returns image_key from Feishu upload", async () => { + const imageCreate = vi.fn(async () => ({ data: { image_key: "img_nested" } })); + const rt = mockRuntime({ imageCreate }); + await expect(uploadMessageImage(rt, Buffer.from("png"))).resolves.toBe("img_nested"); + expect(imageCreate).toHaveBeenCalledWith({ + data: { image_type: "message", image: Buffer.from("png") }, + }); + }); +}); + +function mockRuntime(options: { + readonly imageCreate?: (payload: unknown) => Promise; +}): FeishuRuntime { + return { + client: { + im: { + v1: { + image: { + create: options.imageCreate ?? vi.fn(), + }, + message: { + create: vi.fn(), + reply: vi.fn(), + patch: vi.fn(), + }, + }, + }, + } as unknown as FeishuRuntime["client"], + logger: { + warn: vi.fn(), + error: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + child: vi.fn(), + fatal: vi.fn(), + trace: vi.fn(), + silent: vi.fn(), + level: "info", + } as unknown as FeishuRuntime["logger"], + }; +}