forked from EduCraft/curriculum-project-hub
feat(hub): embed agent images via Feishu upload + release v0.0.36
Materialize markdown image refs on agent finish: fetch/read bytes, upload im.v1.image, and render native card img elements so remote image URLs no longer trip Feishu content-security. Stream masks image URLs mid-run; card failure falls back to plain text plus standalone image messages. Docs: clarify im:resource covers outbound Agent image send.
This commit is contained in:
@@ -32,7 +32,7 @@ App ID 通常以 `cli_` 开头，可以写入交付单。App Secret 必须通过
|
||||
| 接收群聊中 @ 机器人的消息 | `im:message.group_at_msg:readonly` |
|
||||
| 以应用身份发送消息 | `im:message:send_as_bot` |
|
||||
| 读取触发消息和线程上下文 | `im:message:readonly` |
|
||||
| 获取与上传图片或文件 | `im:resource` |
|
||||
| 获取消息中的图片/文件，并向飞书上传图片或文件（含 Agent 回答中的图片发送） | `im:resource` |
|
||||
| 添加、删除消息表情回复 | `im:message.reactions:write_only` |
|
||||
| 获取用户基本信息 | `contact:user.base:readonly` |
|
||||
| 获取用户基本资料 | `contact:user.basic_profile:readonly` |
|
||||
@@ -66,6 +66,9 @@ Educraft 机器人以应用身份调用上述 API，因此这些 scope 全部放
|
||||
|
||||
如果 API 调试台提示缺少更细粒度权限，请把错误提示和发生时间截图给部署人员。不要自行开通通讯录全量读取等超出本表的权限。
|
||||
|
||||
说明：`im:resource` 既用于下载用户发来的图片/文件，也用于 Agent 回复时把本地或远程图片上传为飞书 `image_key` 后嵌入消息卡片。缺少该权限时，带图回答会发送失败或降级为无图文本。已开通该 scope 的存量应用一般无需新增权限，但若权限尚未随最新版本发布，请创建新版本并审核发布。
|
||||
|
||||
|
||||
## 4. 配置事件与卡片回调
|
||||
|
||||
进入“事件与回调”。
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@paradigm/hub",
|
||||
"version": "0.0.35",
|
||||
"version": "0.0.36",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@paradigm/hub",
|
||||
"version": "0.0.35",
|
||||
"version": "0.0.36",
|
||||
"dependencies": {
|
||||
"@alicloud/credentials": "^2.4.5",
|
||||
"@alicloud/docmind-api20220711": "^1.4.15",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@paradigm/hub",
|
||||
"version": "0.0.35",
|
||||
"version": "0.0.36",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
|
||||
import type { ToolUseTraceStep } from "./trace-store.js";
|
||||
import type { CardContentSegment } from "../outboundImages.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -58,6 +59,7 @@ function toolIcon(toolName: string): string {
|
||||
export function buildAgentCard(params: {
|
||||
phase: CardPhase;
|
||||
text: string;
|
||||
contentSegments?: readonly CardContentSegment[] | undefined;
|
||||
reasoningText: string | undefined;
|
||||
toolUseSteps: ToolUseTraceStep[];
|
||||
toolUseElapsedMs: number | undefined;
|
||||
@@ -65,19 +67,19 @@ export function buildAgentCard(params: {
|
||||
interrupted: boolean | undefined;
|
||||
runId: string | undefined;
|
||||
}): Record<string, unknown> {
|
||||
const { phase, text, reasoningText, toolUseSteps, toolUseElapsedMs, isError, interrupted } = params;
|
||||
const { phase, text, contentSegments, reasoningText, toolUseSteps, toolUseElapsedMs, isError, interrupted } = params;
|
||||
const elements: unknown[] = [];
|
||||
|
||||
// Tool-use panel (always present if there are steps)
|
||||
if (toolUseSteps.length > 0) {
|
||||
elements.push(buildToolUsePanel(toolUseSteps, toolUseElapsedMs, phase !== "complete"));
|
||||
} else if (phase === "thinking" || (phase === "streaming" && text === "")) {
|
||||
} else if (phase === "thinking" || (phase === "streaming" && text === "" && (contentSegments === undefined || contentSegments.length === 0))) {
|
||||
elements.push(buildPendingToolUsePanel());
|
||||
}
|
||||
|
||||
// Reasoning panel
|
||||
if (reasoningText !== undefined && reasoningText !== "") {
|
||||
if (phase === "streaming" && text === "") {
|
||||
if (phase === "streaming" && text === "" && (contentSegments === undefined || contentSegments.length === 0)) {
|
||||
// Still thinking: show reasoning inline
|
||||
elements.push({
|
||||
tag: "markdown",
|
||||
@@ -90,12 +92,11 @@ export function buildAgentCard(params: {
|
||||
}
|
||||
}
|
||||
|
||||
// Main text content
|
||||
if (text !== "") {
|
||||
elements.push({
|
||||
tag: "markdown",
|
||||
content: truncateText(text, MAX_TEXT_LENGTH),
|
||||
});
|
||||
// Main answer: either materialized segments (markdown + Feishu-hosted images)
|
||||
// or a single markdown block.
|
||||
const answerElements = buildAnswerElements(text, contentSegments);
|
||||
if (answerElements.length > 0) {
|
||||
elements.push(...answerElements);
|
||||
} else if (phase === "thinking" && toolUseSteps.length === 0 && (reasoningText === undefined || reasoningText === "")) {
|
||||
elements.push({
|
||||
tag: "markdown",
|
||||
@@ -401,6 +402,43 @@ function escapeMarkdown(value: string): string {
|
||||
return value.replace(/\\/g, "\\\\").replace(/([`*_{}[\]<>])/g, "\\$1");
|
||||
}
|
||||
|
||||
function buildAnswerElements(
|
||||
text: string,
|
||||
contentSegments: readonly CardContentSegment[] | undefined,
|
||||
): unknown[] {
|
||||
if (contentSegments !== undefined && contentSegments.length > 0) {
|
||||
const elements: unknown[] = [];
|
||||
let remaining = MAX_TEXT_LENGTH;
|
||||
for (const segment of contentSegments) {
|
||||
if (segment.type === "image") {
|
||||
elements.push({
|
||||
tag: "img",
|
||||
img_key: segment.imgKey,
|
||||
alt: { tag: "plain_text", content: segment.alt },
|
||||
mode: "fit_horizontal",
|
||||
preview: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (segment.content === "" || remaining <= 0) continue;
|
||||
const slice = segment.content.length <= remaining
|
||||
? segment.content
|
||||
: truncateText(segment.content, remaining);
|
||||
remaining -= slice.length;
|
||||
elements.push({
|
||||
tag: "markdown",
|
||||
content: slice,
|
||||
});
|
||||
}
|
||||
return elements;
|
||||
}
|
||||
if (text === "") return [];
|
||||
return [{
|
||||
tag: "markdown",
|
||||
content: truncateText(text, MAX_TEXT_LENGTH),
|
||||
}];
|
||||
}
|
||||
|
||||
function truncateText(value: string, maxLength: number): string {
|
||||
return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`;
|
||||
}
|
||||
|
||||
@@ -18,10 +18,13 @@
|
||||
* 4. onToolEnd(name, id, input, result?, error?) — complete a tool step
|
||||
* 5. finish(finalText) — flush + transition to complete card
|
||||
* 6. fail(errorText) — flush + transition to error card
|
||||
*
|
||||
* On finish, markdown image references (``) are downloaded /
|
||||
* read, uploaded to Feishu as message images, and embedded as native card
|
||||
* `img` elements so external URLs never hit Feishu content-security checks.
|
||||
*/
|
||||
|
||||
import type { FeishuRuntime, SendMessageOptions } from "../client.js";
|
||||
import { sendCard, patchCard, sendText } from "../client.js";
|
||||
import { sendCard, patchCard, sendText, sendLongText } from "../client.js";
|
||||
import { DEFAULT_MAX_MESSAGE_LENGTH, splitAtBoundary } from "../textStream.js";
|
||||
import {
|
||||
startToolUseTraceRun,
|
||||
@@ -31,6 +34,12 @@ import {
|
||||
getToolUseTraceSteps,
|
||||
} from "./trace-store.js";
|
||||
import { buildAgentCard, type CardPhase } from "./builder.js";
|
||||
import {
|
||||
type CardContentSegment,
|
||||
maskMarkdownImagesForStreaming,
|
||||
materializeAnswerSegments,
|
||||
sendImageMessage,
|
||||
} from "../outboundImages.js";
|
||||
|
||||
export interface StreamingCardSink {
|
||||
readonly create: (card: Record<string, unknown>) => Promise<string | null>;
|
||||
@@ -44,6 +53,11 @@ export interface StreamingCardOptions {
|
||||
readonly sendOptions?: SendMessageOptions | undefined;
|
||||
readonly patchIntervalMs: number | undefined;
|
||||
readonly maxMessageLength: number | undefined;
|
||||
/** Project workspace root; required to resolve local image paths. */
|
||||
readonly workspaceRoot?: string | undefined;
|
||||
/** Project workspace directory; required to resolve local image paths. */
|
||||
readonly workspaceDir?: string | undefined;
|
||||
readonly maxImageBytes?: number | undefined;
|
||||
}
|
||||
|
||||
const DEFAULT_PATCH_INTERVAL_MS = 400;
|
||||
@@ -65,6 +79,9 @@ export class StreamingAgentCard {
|
||||
private readonly sendOptions: SendMessageOptions | undefined;
|
||||
private readonly patchIntervalMs: number;
|
||||
private readonly maxMessageLength: number;
|
||||
private readonly workspaceRoot: string | undefined;
|
||||
private readonly workspaceDir: string | undefined;
|
||||
private readonly maxImageBytes: number | undefined;
|
||||
|
||||
constructor(options: StreamingCardOptions) {
|
||||
this.runId = options.runId;
|
||||
@@ -73,6 +90,9 @@ export class StreamingAgentCard {
|
||||
this.sendOptions = options.sendOptions;
|
||||
this.patchIntervalMs = options.patchIntervalMs ?? DEFAULT_PATCH_INTERVAL_MS;
|
||||
this.maxMessageLength = options.maxMessageLength ?? DEFAULT_MAX_MESSAGE_LENGTH;
|
||||
this.workspaceRoot = options.workspaceRoot;
|
||||
this.workspaceDir = options.workspaceDir;
|
||||
this.maxImageBytes = options.maxImageBytes;
|
||||
startToolUseTraceRun(this.runId);
|
||||
}
|
||||
|
||||
@@ -106,23 +126,43 @@ export class StreamingAgentCard {
|
||||
recordToolUseEnd({ runId: this.runId, ...params });
|
||||
this.scheduleFlush();
|
||||
}
|
||||
async finish(fallbackText: string, options: { readonly interrupted?: boolean; readonly footerText?: string | undefined } = {}): Promise<void> {
|
||||
|
||||
async finish(
|
||||
fallbackText: string,
|
||||
options: { readonly interrupted?: boolean; readonly footerText?: string | undefined } = {},
|
||||
): Promise<void> {
|
||||
await this.flushChain;
|
||||
this.interrupted = options.interrupted === true;
|
||||
const footerText = options.footerText ?? "";
|
||||
const fallbackWithFooter = appendFooter(fallbackText, footerText);
|
||||
try {
|
||||
let answerText =
|
||||
this.text.length > 0 ? appendFooter(this.text, footerText) : fallbackWithFooter;
|
||||
this.text = answerText;
|
||||
|
||||
const { segments, unresolved } = await materializeAnswerSegments(answerText, {
|
||||
rt: this.rt,
|
||||
workspaceRoot: this.workspaceRoot,
|
||||
workspaceDir: this.workspaceDir,
|
||||
maxImageBytes: this.maxImageBytes,
|
||||
});
|
||||
if (unresolved.length > 0) {
|
||||
this.rt.logger.warn(
|
||||
{ runId: this.runId, unresolvedCount: unresolved.length, unresolved: unresolved.slice(0, 5) },
|
||||
"some answer images could not be uploaded to Feishu",
|
||||
);
|
||||
}
|
||||
|
||||
let updated = true;
|
||||
if (this.text.length > 0) {
|
||||
this.text = appendFooter(this.text, footerText);
|
||||
updated = await this.flushCard("complete", this.text);
|
||||
} else if (this.currentMessageId === null && fallbackWithFooter.length > 0) {
|
||||
// No streaming text was sent. If we never created a card, send one now.
|
||||
this.text = fallbackWithFooter;
|
||||
updated = await this.flushCard("complete", this.text);
|
||||
if (answerText.length > 0 || segments.length > 0) {
|
||||
updated = await this.flushCard("complete", answerText, false, segments);
|
||||
} else if (this.currentMessageId !== null) {
|
||||
// Patch the existing card with the final text.
|
||||
updated = await this.flushCard("complete", fallbackWithFooter);
|
||||
updated = await this.flushCard("complete", "", false, []);
|
||||
}
|
||||
|
||||
if (!updated) {
|
||||
// Card path failed (e.g. residual content policy). Deliver text + standalone images.
|
||||
updated = await this.deliverPlainFallback(segments, answerText);
|
||||
}
|
||||
if (!updated && this.interrupted) {
|
||||
await sendText(this.rt, this.chatId, "\u5DF2\u4E2D\u65AD\u5F53\u524D\u8FD0\u884C\u3002", this.sendOptions);
|
||||
@@ -166,15 +206,30 @@ export class StreamingAgentCard {
|
||||
return this.flushCard(this.currentPhase(), this.text);
|
||||
}
|
||||
|
||||
private async flushCard(phase: CardPhase, text: string, isError = false): Promise<boolean> {
|
||||
const chunks = splitAtBoundary(text, this.maxMessageLength);
|
||||
const firstChunk = chunks[0];
|
||||
if (firstChunk === undefined) return true;
|
||||
private async flushCard(
|
||||
phase: CardPhase,
|
||||
text: string,
|
||||
isError = false,
|
||||
contentSegments?: readonly CardContentSegment[],
|
||||
): Promise<boolean> {
|
||||
// During live streaming, strip image URLs so Feishu never fetches remote
|
||||
// ranks mid-run. Materialized segments are only used on the complete pass.
|
||||
const displayText =
|
||||
phase === "complete" && contentSegments !== undefined
|
||||
? text
|
||||
: maskMarkdownImagesForStreaming(text);
|
||||
|
||||
const chunks = splitAtBoundary(displayText, this.maxMessageLength);
|
||||
const firstChunk = chunks[0] ?? "";
|
||||
// When we have segments (complete+images), keep first-card complete content
|
||||
// on segments only; overflow text (rare) falls back to plain chunked cards.
|
||||
const toolUseSteps = getToolUseTraceSteps(this.runId);
|
||||
const card = buildAgentCard({
|
||||
phase,
|
||||
text: firstChunk,
|
||||
text: contentSegments !== undefined && contentSegments.length > 0 ? "" : firstChunk,
|
||||
contentSegments: contentSegments !== undefined && contentSegments.length > 0
|
||||
? contentSegments
|
||||
: undefined,
|
||||
reasoningText: this.reasoningText || undefined,
|
||||
toolUseSteps,
|
||||
toolUseElapsedMs: this.toolUseElapsedMs,
|
||||
@@ -186,43 +241,89 @@ export class StreamingAgentCard {
|
||||
if (this.currentMessageId === null) {
|
||||
this.currentMessageId = await sendCard(this.rt, this.chatId, card, this.sendOptions);
|
||||
let updated = this.currentMessageId !== null;
|
||||
// Send overflow chunks as new messages (rare for agent output)
|
||||
for (const chunk of chunks.slice(1)) {
|
||||
const overflowCard = buildAgentCard({
|
||||
phase,
|
||||
text: chunk,
|
||||
reasoningText: undefined,
|
||||
toolUseSteps: [],
|
||||
toolUseElapsedMs: undefined,
|
||||
isError,
|
||||
interrupted: this.interrupted,
|
||||
runId: undefined,
|
||||
});
|
||||
const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
|
||||
updated = updated && overflowMessageId !== null;
|
||||
this.currentMessageId = overflowMessageId;
|
||||
}
|
||||
return updated;
|
||||
} else {
|
||||
let updated = await patchCard(this.rt, this.currentMessageId, card);
|
||||
// For overflow, create new messages
|
||||
for (const chunk of chunks.slice(1)) {
|
||||
const overflowCard = buildAgentCard({
|
||||
phase,
|
||||
text: chunk,
|
||||
reasoningText: undefined,
|
||||
toolUseSteps: [],
|
||||
toolUseElapsedMs: undefined,
|
||||
isError,
|
||||
interrupted: this.interrupted,
|
||||
runId: undefined,
|
||||
});
|
||||
const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
|
||||
updated = updated && overflowMessageId !== null;
|
||||
this.currentMessageId = overflowMessageId;
|
||||
// Send overflow chunks as new messages (rare for agent output). Segments
|
||||
// already include the whole answer; only plain text overflows.
|
||||
if (contentSegments === undefined || contentSegments.length === 0) {
|
||||
for (const chunk of chunks.slice(1)) {
|
||||
const overflowCard = buildAgentCard({
|
||||
phase,
|
||||
text: chunk,
|
||||
reasoningText: undefined,
|
||||
toolUseSteps: [],
|
||||
toolUseElapsedMs: undefined,
|
||||
isError,
|
||||
interrupted: this.interrupted,
|
||||
runId: undefined,
|
||||
});
|
||||
const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
|
||||
updated = updated && overflowMessageId !== null;
|
||||
this.currentMessageId = overflowMessageId;
|
||||
}
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
let updated = await patchCard(this.rt, this.currentMessageId, card);
|
||||
if (contentSegments === undefined || contentSegments.length === 0) {
|
||||
for (const chunk of chunks.slice(1)) {
|
||||
const overflowCard = buildAgentCard({
|
||||
phase,
|
||||
text: chunk,
|
||||
reasoningText: undefined,
|
||||
toolUseSteps: [],
|
||||
toolUseElapsedMs: undefined,
|
||||
isError,
|
||||
interrupted: this.interrupted,
|
||||
runId: undefined,
|
||||
});
|
||||
const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
|
||||
updated = updated && overflowMessageId !== null;
|
||||
this.currentMessageId = overflowMessageId;
|
||||
}
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
|
||||
private async deliverPlainFallback(
|
||||
segments: readonly CardContentSegment[],
|
||||
answerText: string,
|
||||
): Promise<boolean> {
|
||||
const textParts: string[] = [];
|
||||
const imageKeys: string[] = [];
|
||||
if (segments.length > 0) {
|
||||
for (const segment of segments) {
|
||||
if (segment.type === "markdown") {
|
||||
if (segment.content.trim() !== "") textParts.push(segment.content);
|
||||
} else {
|
||||
imageKeys.push(segment.imgKey);
|
||||
}
|
||||
}
|
||||
} else if (answerText.trim() !== "") {
|
||||
textParts.push(maskMarkdownImagesForStreaming(answerText));
|
||||
}
|
||||
|
||||
let any = false;
|
||||
if (textParts.length > 0) {
|
||||
const messageId = await sendLongText(
|
||||
this.rt,
|
||||
this.chatId,
|
||||
textParts.join("\n\n"),
|
||||
this.sendOptions,
|
||||
);
|
||||
any = messageId !== null;
|
||||
}
|
||||
for (const imageKey of imageKeys) {
|
||||
try {
|
||||
const messageId = await sendImageMessage(this.rt, this.chatId, imageKey, this.sendOptions);
|
||||
any = any || messageId !== null;
|
||||
} catch (error) {
|
||||
this.rt.logger.warn(
|
||||
{ runId: this.runId, err: error instanceof Error ? error.message : String(error) },
|
||||
"standalone image fallback failed",
|
||||
);
|
||||
}
|
||||
}
|
||||
return any;
|
||||
}
|
||||
|
||||
private currentPhase(): CardPhase {
|
||||
@@ -235,5 +336,5 @@ export class StreamingAgentCard {
|
||||
function appendFooter(text: string, footerText: string): string {
|
||||
if (footerText === "") return text;
|
||||
if (text === "") return footerText;
|
||||
return `${text.trimEnd()}\n\n${footerText}`;
|
||||
return `${text}\n\n${footerText}`;
|
||||
}
|
||||
|
||||
@@ -316,7 +316,8 @@ export async function sendCard(
|
||||
{ msgType: "interactive", content: JSON.stringify(card) },
|
||||
options,
|
||||
);
|
||||
} catch {
|
||||
} catch (e) {
|
||||
rt.logger.warn({ chatId, err: errorText(e) }, "sendCard failed");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -334,7 +335,7 @@ export async function patchCard(rt: FeishuRuntime, messageId: string, card: Reco
|
||||
});
|
||||
return true;
|
||||
} catch (e) {
|
||||
rt.logger.warn({ messageId, err: e instanceof Error ? e.message : String(e) }, "patchCard failed");
|
||||
rt.logger.warn({ messageId, err: errorText(e) }, "patchCard failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
/**
|
||||
* Resolve markdown image references in agent answers into Feishu-hosted
|
||||
* image_keys so cards can embed them without remote URLs (which trip Feishu
|
||||
* content-security controls).
|
||||
*/
|
||||
import { isIP } from "node:net";
|
||||
import type { FeishuRuntime } from "./client.js";
|
||||
import { withRetry } from "./client.js";
|
||||
import {
|
||||
WorkspaceFileBoundaryError,
|
||||
readWorkspaceFileNoFollow,
|
||||
} from "../security/workspaceFiles.js";
|
||||
|
||||
export const FEISHU_MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
||||
export const DEFAULT_MAX_OUTBOUND_IMAGES = 10;
|
||||
|
||||
const IMAGE_MARKDOWN_RE = /!\[([^\]\n]*)\]\(([^)\n]+)\)/g;
|
||||
const FENCED_CODE_RE = /```[\s\S]*?```/g;
|
||||
|
||||
export type CardContentSegment =
|
||||
| { readonly type: "markdown"; readonly content: string }
|
||||
| { readonly type: "image"; readonly imgKey: string; readonly alt: string };
|
||||
|
||||
export interface MarkdownImageRef {
|
||||
readonly fullMatch: string;
|
||||
readonly alt: string;
|
||||
readonly src: string;
|
||||
readonly index: number;
|
||||
readonly length: number;
|
||||
}
|
||||
|
||||
export interface OutboundImageContext {
|
||||
readonly rt: FeishuRuntime;
|
||||
readonly workspaceRoot?: string | undefined;
|
||||
readonly workspaceDir?: string | undefined;
|
||||
readonly maxImageBytes?: number | undefined;
|
||||
readonly maxImages?: number | undefined;
|
||||
readonly fetchImpl?: typeof fetch | undefined;
|
||||
}
|
||||
|
||||
type ImageCreateResponse = {
|
||||
image_key?: string;
|
||||
data?: { image_key?: string };
|
||||
} | null;
|
||||
|
||||
type MessageCreateResponse = {
|
||||
message_id?: string;
|
||||
data?: { message_id?: string };
|
||||
} | null;
|
||||
|
||||
/** Streaming-safe view: drop markdown image URLs so partial cards do not hit Feishu URL checks. */
|
||||
export function maskMarkdownImagesForStreaming(text: string): string {
|
||||
return rewriteMarkdownImagesOutsideCode(text, (ref) => {
|
||||
const alt = ref.alt.trim();
|
||||
return alt === "" ? "\u3010\u56fe\u7247\u3011" : alt;
|
||||
});
|
||||
}
|
||||
|
||||
export function findMarkdownImagesOutsideCode(text: string): MarkdownImageRef[] {
|
||||
const blocked = blockedRanges(text);
|
||||
const refs: MarkdownImageRef[] = [];
|
||||
IMAGE_MARKDOWN_RE.lastIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = IMAGE_MARKDOWN_RE.exec(text)) !== null) {
|
||||
const index = match.index;
|
||||
if (blocked.some((range) => index >= range.start && index < range.end)) continue;
|
||||
const fullMatch = match[0];
|
||||
const alt = match[1] ?? "";
|
||||
const rawSrc = (match[2] ?? "").trim();
|
||||
const src = stripUrlTitle(rawSrc);
|
||||
if (src === "") continue;
|
||||
refs.push({ fullMatch, alt, src, index, length: fullMatch.length });
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload reachable markdown images and split the answer into card segments
|
||||
* (markdown + Feishu img elements). Unresolved images become visible alt text.
|
||||
*/
|
||||
export async function materializeAnswerSegments(
|
||||
text: string,
|
||||
ctx: OutboundImageContext,
|
||||
): Promise<{ segments: CardContentSegment[]; unresolved: string[] }> {
|
||||
const refs = findMarkdownImagesOutsideCode(text);
|
||||
if (refs.length === 0) {
|
||||
return {
|
||||
segments: text === "" ? [] : [{ type: "markdown", content: text }],
|
||||
unresolved: [],
|
||||
};
|
||||
}
|
||||
|
||||
const maxImages = ctx.maxImages ?? DEFAULT_MAX_OUTBOUND_IMAGES;
|
||||
const maxBytes = ctx.maxImageBytes ?? FEISHU_MAX_IMAGE_BYTES;
|
||||
const keyBySrc = new Map<string, string>();
|
||||
const unresolved: string[] = [];
|
||||
const uniqueSrcs: string[] = [];
|
||||
for (const ref of refs) {
|
||||
if (!uniqueSrcs.includes(ref.src)) uniqueSrcs.push(ref.src);
|
||||
}
|
||||
|
||||
for (const src of uniqueSrcs.slice(0, maxImages)) {
|
||||
try {
|
||||
const bytes = await resolveOutboundImageBytes(src, ctx, maxBytes);
|
||||
if (bytes === null) {
|
||||
unresolved.push(src);
|
||||
continue;
|
||||
}
|
||||
const imageKey = await uploadMessageImage(ctx.rt, bytes);
|
||||
keyBySrc.set(src, imageKey);
|
||||
} catch (error) {
|
||||
ctx.rt.logger.warn(
|
||||
{ src, err: error instanceof Error ? error.message : String(error) },
|
||||
"outbound image materialize failed",
|
||||
);
|
||||
unresolved.push(src);
|
||||
}
|
||||
}
|
||||
for (const src of uniqueSrcs.slice(maxImages)) {
|
||||
unresolved.push(src);
|
||||
}
|
||||
|
||||
const segments: CardContentSegment[] = [];
|
||||
let cursor = 0;
|
||||
for (const ref of refs) {
|
||||
if (ref.index > cursor) {
|
||||
pushMarkdown(segments, text.slice(cursor, ref.index));
|
||||
}
|
||||
const imageKey = keyBySrc.get(ref.src);
|
||||
if (imageKey !== undefined) {
|
||||
segments.push({
|
||||
type: "image",
|
||||
imgKey: imageKey,
|
||||
alt: ref.alt.trim() === "" ? "\u56fe\u7247" : ref.alt.trim(),
|
||||
});
|
||||
} else {
|
||||
const alt = ref.alt.trim();
|
||||
pushMarkdown(segments, alt === "" ? "\u3010\u56fe\u7247\u3011" : alt);
|
||||
}
|
||||
cursor = ref.index + ref.length;
|
||||
}
|
||||
if (cursor < text.length) {
|
||||
pushMarkdown(segments, text.slice(cursor));
|
||||
}
|
||||
return { segments, unresolved };
|
||||
}
|
||||
|
||||
export async function uploadMessageImage(rt: FeishuRuntime, image: Buffer): Promise<string> {
|
||||
if (image.byteLength === 0) {
|
||||
throw new Error("image is empty");
|
||||
}
|
||||
if (image.byteLength > FEISHU_MAX_IMAGE_BYTES) {
|
||||
throw new Error(`image exceeds Feishu limit of ${FEISHU_MAX_IMAGE_BYTES} bytes`);
|
||||
}
|
||||
const client = rt.client as unknown as {
|
||||
im: { v1: { image: { create: (p: unknown) => Promise<ImageCreateResponse> } } };
|
||||
};
|
||||
const res = await withRetry(async () =>
|
||||
client.im.v1.image.create({
|
||||
data: { image_type: "message", image },
|
||||
}),
|
||||
);
|
||||
const imageKey = res?.image_key ?? res?.data?.image_key;
|
||||
if (imageKey === undefined || imageKey === "") {
|
||||
throw new Error("Feishu image upload response is missing image_key");
|
||||
}
|
||||
return imageKey;
|
||||
}
|
||||
|
||||
/** Send a standalone image message (fallback if card embed is unavailable). */
|
||||
export async function sendImageMessage(
|
||||
rt: FeishuRuntime,
|
||||
chatId: string,
|
||||
imageKey: string,
|
||||
options?: { readonly replyToMessageId?: string | undefined },
|
||||
): Promise<string | null> {
|
||||
const client = rt.client as unknown as {
|
||||
im: {
|
||||
v1: {
|
||||
message: {
|
||||
create: (p: unknown) => Promise<MessageCreateResponse>;
|
||||
reply: (p: unknown) => Promise<MessageCreateResponse>;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
const replyTo = options?.replyToMessageId;
|
||||
if (replyTo !== undefined && replyTo !== "") {
|
||||
const res = await client.im.v1.message.reply({
|
||||
path: { message_id: replyTo },
|
||||
data: { msg_type: "image", content: JSON.stringify({ image_key: imageKey }) },
|
||||
});
|
||||
return res?.data?.message_id ?? res?.message_id ?? null;
|
||||
}
|
||||
const res = await client.im.v1.message.create({
|
||||
params: { receive_id_type: "chat_id" },
|
||||
data: {
|
||||
receive_id: chatId,
|
||||
msg_type: "image",
|
||||
content: JSON.stringify({ image_key: imageKey }),
|
||||
},
|
||||
});
|
||||
return res?.data?.message_id ?? res?.message_id ?? null;
|
||||
}
|
||||
|
||||
export async function resolveOutboundImageBytes(
|
||||
src: string,
|
||||
ctx: OutboundImageContext,
|
||||
maxBytes: number,
|
||||
): Promise<Buffer | null> {
|
||||
if (isRemoteUrl(src)) {
|
||||
return fetchRemoteImage(src, ctx.fetchImpl ?? fetch, maxBytes);
|
||||
}
|
||||
const root = ctx.workspaceRoot?.trim();
|
||||
const dir = ctx.workspaceDir?.trim();
|
||||
if (root === undefined || root === "" || dir === undefined || dir === "") {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const file = await readWorkspaceFileNoFollow(root, dir, src, maxBytes);
|
||||
return file.data;
|
||||
} catch (error) {
|
||||
if (error instanceof WorkspaceFileBoundaryError && error.reason === "not_found") {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function rewriteMarkdownImagesOutsideCode(
|
||||
text: string,
|
||||
replace: (ref: MarkdownImageRef) => string,
|
||||
): string {
|
||||
const refs = findMarkdownImagesOutsideCode(text);
|
||||
if (refs.length === 0) return text;
|
||||
let out = "";
|
||||
let cursor = 0;
|
||||
for (const ref of refs) {
|
||||
out += text.slice(cursor, ref.index);
|
||||
out += replace(ref);
|
||||
cursor = ref.index + ref.length;
|
||||
}
|
||||
out += text.slice(cursor);
|
||||
return out;
|
||||
}
|
||||
|
||||
function pushMarkdown(segments: CardContentSegment[], content: string): void {
|
||||
if (content === "") return;
|
||||
const last = segments[segments.length - 1];
|
||||
if (last !== undefined && last.type === "markdown") {
|
||||
segments[segments.length - 1] = { type: "markdown", content: last.content + content };
|
||||
return;
|
||||
}
|
||||
segments.push({ type: "markdown", content });
|
||||
}
|
||||
|
||||
function blockedRanges(text: string): Array<{ start: number; end: number }> {
|
||||
const ranges: Array<{ start: number; end: number }> = [];
|
||||
FENCED_CODE_RE.lastIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = FENCED_CODE_RE.exec(text)) !== null) {
|
||||
ranges.push({ start: match.index, end: match.index + match[0].length });
|
||||
}
|
||||
return ranges;
|
||||
}
|
||||
|
||||
function stripUrlTitle(raw: string): string {
|
||||
const trimmed = raw.trim();
|
||||
// Markdown optional title: url "title" or url 'title'
|
||||
const titled = /^(\S+)\s+(".*"|'.*')$/.exec(trimmed);
|
||||
return (titled?.[1] ?? trimmed).trim();
|
||||
}
|
||||
|
||||
function isRemoteUrl(src: string): boolean {
|
||||
try {
|
||||
const url = new URL(src);
|
||||
return url.protocol === "http:" || url.protocol === "https:";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchRemoteImage(
|
||||
src: string,
|
||||
fetchImpl: typeof fetch,
|
||||
maxBytes: number,
|
||||
): Promise<Buffer | null> {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(src);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
||||
if (!isPublicHttpHost(url.hostname)) return null;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 15_000);
|
||||
try {
|
||||
const response = await fetchImpl(url, {
|
||||
method: "GET",
|
||||
redirect: "manual",
|
||||
signal: controller.signal,
|
||||
headers: { accept: "image/*,*/*;q=0.8" },
|
||||
});
|
||||
// One safe redirect hop to another public http(s) host.
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
const location = response.headers.get("location");
|
||||
if (location === null || location === "") return null;
|
||||
let redirected: URL;
|
||||
try {
|
||||
redirected = new URL(location, url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (redirected.protocol !== "http:" && redirected.protocol !== "https:") return null;
|
||||
if (!isPublicHttpHost(redirected.hostname)) return null;
|
||||
const second = await fetchImpl(redirected, {
|
||||
method: "GET",
|
||||
redirect: "manual",
|
||||
signal: controller.signal,
|
||||
headers: { accept: "image/*,*/*;q=0.8" },
|
||||
});
|
||||
return readImageBody(second, maxBytes);
|
||||
}
|
||||
return readImageBody(response, maxBytes);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function readImageBody(response: Response, maxBytes: number): Promise<Buffer | null> {
|
||||
if (!response.ok) return null;
|
||||
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
|
||||
if (
|
||||
contentType !== "" &&
|
||||
!contentType.startsWith("image/") &&
|
||||
!contentType.includes("octet-stream") &&
|
||||
(contentType.startsWith("text/") || contentType.includes("json") || contentType.includes("html"))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const contentLength = Number(response.headers.get("content-length") ?? "NaN");
|
||||
if (Number.isFinite(contentLength) && contentLength > maxBytes) return null;
|
||||
const buf = Buffer.from(await response.arrayBuffer());
|
||||
if (buf.byteLength === 0 || buf.byteLength > maxBytes) return null;
|
||||
return buf;
|
||||
}
|
||||
|
||||
function isPublicHttpHost(hostname: string): boolean {
|
||||
const host = hostname.trim().toLowerCase().replace(/\.$/, "");
|
||||
if (host === "" || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) {
|
||||
return false;
|
||||
}
|
||||
if (host === "0.0.0.0" || host === "::" || host === "[::]" || host === "::1" || host === "[::1]") {
|
||||
return false;
|
||||
}
|
||||
const unbracketed = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
||||
const ipVersion = isIP(unbracketed);
|
||||
if (ipVersion === 4) return !isPrivateIPv4(unbracketed);
|
||||
if (ipVersion === 6) return !isPrivateIPv6(unbracketed);
|
||||
return true;
|
||||
}
|
||||
|
||||
function isPrivateIPv4(ip: string): boolean {
|
||||
const parts = ip.split(".").map((part) => Number(part));
|
||||
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
|
||||
return true;
|
||||
}
|
||||
const a = parts[0]!;
|
||||
const b = parts[1]!;
|
||||
if (a === 10 || a === 127 || a === 0) return true;
|
||||
if (a === 169 && b === 254) return true;
|
||||
if (a === 172 && b >= 16 && b <= 31) return true;
|
||||
if (a === 192 && b === 168) return true;
|
||||
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT
|
||||
if (a >= 224) return true; // multicast / reserved
|
||||
return false;
|
||||
}
|
||||
|
||||
function isPrivateIPv6(ip: string): boolean {
|
||||
const normalized = ip.toLowerCase();
|
||||
if (normalized === "::1") return true;
|
||||
if (normalized.startsWith("fc") || normalized.startsWith("fd")) return true; // unique local
|
||||
if (normalized.startsWith("fe80:")) return true; // link-local
|
||||
const mapped = /^:ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(normalized);
|
||||
if (mapped?.[1] !== undefined) return isPrivateIPv4(mapped[1]);
|
||||
return false;
|
||||
}
|
||||
@@ -496,6 +496,9 @@ export function makeTriggerHandler(deps: TriggerDeps): TriggerHandler {
|
||||
sendOptions,
|
||||
patchIntervalMs: undefined,
|
||||
maxMessageLength: undefined,
|
||||
workspaceRoot: projectWorkspaceRoot,
|
||||
workspaceDir: project.workspaceDir,
|
||||
maxImageBytes: deps.resourceLimits?.maxBytesPerFile,
|
||||
});
|
||||
const fileDeliveryMcpServer = createFileDeliveryMcpServer({
|
||||
rt,
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { FeishuRuntime } from "../../src/feishu/client.js";
|
||||
import {
|
||||
findMarkdownImagesOutsideCode,
|
||||
maskMarkdownImagesForStreaming,
|
||||
materializeAnswerSegments,
|
||||
uploadMessageImage,
|
||||
} from "../../src/feishu/outboundImages.js";
|
||||
import { buildAgentCard } from "../../src/feishu/card/builder.js";
|
||||
|
||||
const temps: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temps.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("outbound markdown image parsing", () => {
|
||||
it("finds image refs outside fenced code blocks", () => {
|
||||
const text = [
|
||||
"See diagram:",
|
||||
"",
|
||||
"",
|
||||
"```md",
|
||||
"",
|
||||
"```",
|
||||
"![local](assets/x.png \"title\")",
|
||||
].join("\n");
|
||||
|
||||
const refs = findMarkdownImagesOutsideCode(text);
|
||||
expect(refs.map((ref) => ref.src)).toEqual([
|
||||
"https://cdn.example.com/a.png",
|
||||
"assets/x.png",
|
||||
]);
|
||||
});
|
||||
|
||||
it("masks image urls for streaming cards", () => {
|
||||
expect(maskMarkdownImagesForStreaming("before  after")).toBe(
|
||||
"before alt text after",
|
||||
);
|
||||
expect(maskMarkdownImagesForStreaming("")).toBe("【图片】");
|
||||
});
|
||||
});
|
||||
|
||||
describe("materializeAnswerSegments", () => {
|
||||
it("uploads remote and workspace images and builds card segments", async () => {
|
||||
const workspaceRoot = await mkdtemp(join(tmpdir(), "cph-img-root-"));
|
||||
temps.push(workspaceRoot);
|
||||
const workspaceDir = join(workspaceRoot, "project");
|
||||
await mkdir(join(workspaceDir, "assets"), { recursive: true });
|
||||
await writeFile(
|
||||
join(workspaceDir, "assets", "local.png"),
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
);
|
||||
|
||||
const imageCreate = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ image_key: "img_remote_1" })
|
||||
.mockResolvedValueOnce({ image_key: "img_local_1" });
|
||||
const rt = mockRuntime({ imageCreate });
|
||||
|
||||
const fetchImpl = vi.fn(async () =>
|
||||
new Response(Buffer.from("remote-bytes"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
);
|
||||
|
||||
const text = "Intro\n\n\n\nAnd local \nDone.";
|
||||
const { segments, unresolved } = await materializeAnswerSegments(text, {
|
||||
rt,
|
||||
workspaceRoot,
|
||||
workspaceDir,
|
||||
fetchImpl: fetchImpl as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(unresolved).toEqual([]);
|
||||
expect(imageCreate).toHaveBeenCalledTimes(2);
|
||||
expect(segments).toEqual([
|
||||
{ type: "markdown", content: "Intro\n\n" },
|
||||
{ type: "image", imgKey: "img_remote_1", alt: "remote" },
|
||||
{ type: "markdown", content: "\n\nAnd local " },
|
||||
{ type: "image", imgKey: "img_local_1", alt: "local" },
|
||||
{ type: "markdown", content: "\nDone." },
|
||||
]);
|
||||
|
||||
const card = buildAgentCard({
|
||||
phase: "complete",
|
||||
text: "",
|
||||
contentSegments: segments,
|
||||
reasoningText: undefined,
|
||||
toolUseSteps: [],
|
||||
toolUseElapsedMs: undefined,
|
||||
isError: false,
|
||||
interrupted: false,
|
||||
runId: undefined,
|
||||
});
|
||||
expect(card.elements).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ tag: "img", img_key: "img_remote_1" }),
|
||||
expect.objectContaining({ tag: "img", img_key: "img_local_1" }),
|
||||
expect.objectContaining({ tag: "markdown", content: "Intro\n\n" }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("rejects private remote hosts", async () => {
|
||||
const imageCreate = vi.fn();
|
||||
const rt = mockRuntime({ imageCreate });
|
||||
const fetchImpl = vi.fn();
|
||||
const { segments, unresolved } = await materializeAnswerSegments(
|
||||
"",
|
||||
{ rt, fetchImpl: fetchImpl as unknown as typeof fetch },
|
||||
);
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
expect(imageCreate).not.toHaveBeenCalled();
|
||||
expect(unresolved).toEqual(["http://127.0.0.1/secret.png"]);
|
||||
expect(segments).toEqual([{ type: "markdown", content: "x" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("uploadMessageImage", () => {
|
||||
it("returns image_key from Feishu upload", async () => {
|
||||
const imageCreate = vi.fn(async () => ({ data: { image_key: "img_nested" } }));
|
||||
const rt = mockRuntime({ imageCreate });
|
||||
await expect(uploadMessageImage(rt, Buffer.from("png"))).resolves.toBe("img_nested");
|
||||
expect(imageCreate).toHaveBeenCalledWith({
|
||||
data: { image_type: "message", image: Buffer.from("png") },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function mockRuntime(options: {
|
||||
readonly imageCreate?: (payload: unknown) => Promise<unknown>;
|
||||
}): FeishuRuntime {
|
||||
return {
|
||||
client: {
|
||||
im: {
|
||||
v1: {
|
||||
image: {
|
||||
create: options.imageCreate ?? vi.fn(),
|
||||
},
|
||||
message: {
|
||||
create: vi.fn(),
|
||||
reply: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as FeishuRuntime["client"],
|
||||
logger: {
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
info: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
child: vi.fn(),
|
||||
fatal: vi.fn(),
|
||||
trace: vi.fn(),
|
||||
silent: vi.fn(),
|
||||
level: "info",
|
||||
} as unknown as FeishuRuntime["logger"],
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user