feat(hub): embed agent images via Feishu upload + release v0.0.36

Materialize markdown image refs on agent finish: fetch/read bytes, upload im.v1.image, and render native card img elements so remote image URLs no longer trip Feishu content-security. Stream masks image URLs mid-run; card failure falls back to plain text plus standalone image messages. Docs: clarify im:resource covers outbound Agent image send.
2026-07-20 10:40:03 +00:00
parent dc2d1c2f9e
commit e21096c642
9 changed files with 765 additions and 67 deletions
@@ -18,10 +18,13 @@
 *   4. onToolEnd(name, id, input, result?, error?) — complete a tool step
 *   5. finish(finalText)    — flush + transition to complete card
 *   6. fail(errorText)      — flush + transition to error card
+ *
+ * On finish, markdown image references (`![](url|path)`) are downloaded /
+ * read, uploaded to Feishu as message images, and embedded as native card
+ * `img` elements so external URLs never hit Feishu content-security checks.
 */
-
 import type { FeishuRuntime, SendMessageOptions } from "../client.js";
-import { sendCard, patchCard, sendText } from "../client.js";
+import { sendCard, patchCard, sendText, sendLongText } from "../client.js";
 import { DEFAULT_MAX_MESSAGE_LENGTH, splitAtBoundary } from "../textStream.js";
 import {
  startToolUseTraceRun,
@@ -31,6 +34,12 @@ import {
  getToolUseTraceSteps,
 } from "./trace-store.js";
 import { buildAgentCard, type CardPhase } from "./builder.js";
+import {
+  type CardContentSegment,
+  maskMarkdownImagesForStreaming,
+  materializeAnswerSegments,
+  sendImageMessage,
+} from "../outboundImages.js";

 export interface StreamingCardSink {
  readonly create: (card: Record<string, unknown>) => Promise<string | null>;
@@ -44,6 +53,11 @@ export interface StreamingCardOptions {
  readonly sendOptions?: SendMessageOptions | undefined;
  readonly patchIntervalMs: number | undefined;
  readonly maxMessageLength: number | undefined;
+  /** Project workspace root; required to resolve local image paths. */
+  readonly workspaceRoot?: string | undefined;
+  /** Project workspace directory; required to resolve local image paths. */
+  readonly workspaceDir?: string | undefined;
+  readonly maxImageBytes?: number | undefined;
 }

 const DEFAULT_PATCH_INTERVAL_MS = 400;
@@ -65,6 +79,9 @@ export class StreamingAgentCard {
  private readonly sendOptions: SendMessageOptions | undefined;
  private readonly patchIntervalMs: number;
  private readonly maxMessageLength: number;
+  private readonly workspaceRoot: string | undefined;
+  private readonly workspaceDir: string | undefined;
+  private readonly maxImageBytes: number | undefined;

  constructor(options: StreamingCardOptions) {
    this.runId = options.runId;
@@ -73,6 +90,9 @@ export class StreamingAgentCard {
    this.sendOptions = options.sendOptions;
    this.patchIntervalMs = options.patchIntervalMs ?? DEFAULT_PATCH_INTERVAL_MS;
    this.maxMessageLength = options.maxMessageLength ?? DEFAULT_MAX_MESSAGE_LENGTH;
+    this.workspaceRoot = options.workspaceRoot;
+    this.workspaceDir = options.workspaceDir;
+    this.maxImageBytes = options.maxImageBytes;
    startToolUseTraceRun(this.runId);
  }

@@ -106,23 +126,43 @@ export class StreamingAgentCard {
    recordToolUseEnd({ runId: this.runId, ...params });
    this.scheduleFlush();
  }
-  async finish(fallbackText: string, options: { readonly interrupted?: boolean; readonly footerText?: string | undefined } = {}): Promise<void> {
+
+  async finish(
+    fallbackText: string,
+    options: { readonly interrupted?: boolean; readonly footerText?: string | undefined } = {},
+  ): Promise<void> {
    await this.flushChain;
    this.interrupted = options.interrupted === true;
    const footerText = options.footerText ?? "";
    const fallbackWithFooter = appendFooter(fallbackText, footerText);
    try {
+      let answerText =
+        this.text.length > 0 ? appendFooter(this.text, footerText) : fallbackWithFooter;
+      this.text = answerText;
+
+      const { segments, unresolved } = await materializeAnswerSegments(answerText, {
+        rt: this.rt,
+        workspaceRoot: this.workspaceRoot,
+        workspaceDir: this.workspaceDir,
+        maxImageBytes: this.maxImageBytes,
+      });
+      if (unresolved.length > 0) {
+        this.rt.logger.warn(
+          { runId: this.runId, unresolvedCount: unresolved.length, unresolved: unresolved.slice(0, 5) },
+          "some answer images could not be uploaded to Feishu",
+        );
+      }
+
      let updated = true;
-      if (this.text.length > 0) {
-        this.text = appendFooter(this.text, footerText);
-        updated = await this.flushCard("complete", this.text);
-      } else if (this.currentMessageId === null && fallbackWithFooter.length > 0) {
-        // No streaming text was sent. If we never created a card, send one now.
-        this.text = fallbackWithFooter;
-        updated = await this.flushCard("complete", this.text);
+      if (answerText.length > 0 || segments.length > 0) {
+        updated = await this.flushCard("complete", answerText, false, segments);
      } else if (this.currentMessageId !== null) {
-        // Patch the existing card with the final text.
-        updated = await this.flushCard("complete", fallbackWithFooter);
+        updated = await this.flushCard("complete", "", false, []);
+      }
+
+      if (!updated) {
+        // Card path failed (e.g. residual content policy). Deliver text + standalone images.
+        updated = await this.deliverPlainFallback(segments, answerText);
      }
      if (!updated && this.interrupted) {
        await sendText(this.rt, this.chatId, "\u5DF2\u4E2D\u65AD\u5F53\u524D\u8FD0\u884C\u3002", this.sendOptions);
@@ -166,15 +206,30 @@ export class StreamingAgentCard {
    return this.flushCard(this.currentPhase(), this.text);
  }

-  private async flushCard(phase: CardPhase, text: string, isError = false): Promise<boolean> {
-    const chunks = splitAtBoundary(text, this.maxMessageLength);
-    const firstChunk = chunks[0];
-    if (firstChunk === undefined) return true;
+  private async flushCard(
+    phase: CardPhase,
+    text: string,
+    isError = false,
+    contentSegments?: readonly CardContentSegment[],
+  ): Promise<boolean> {
+    // During live streaming, strip image URLs so Feishu never fetches remote
+    // ranks mid-run. Materialized segments are only used on the complete pass.
+    const displayText =
+      phase === "complete" && contentSegments !== undefined
+        ? text
+        : maskMarkdownImagesForStreaming(text);

+    const chunks = splitAtBoundary(displayText, this.maxMessageLength);
+    const firstChunk = chunks[0] ?? "";
+    // When we have segments (complete+images), keep first-card complete content
+    // on segments only; overflow text (rare) falls back to plain chunked cards.
    const toolUseSteps = getToolUseTraceSteps(this.runId);
    const card = buildAgentCard({
      phase,
-      text: firstChunk,
+      text: contentSegments !== undefined && contentSegments.length > 0 ? "" : firstChunk,
+      contentSegments: contentSegments !== undefined && contentSegments.length > 0
+        ? contentSegments
+        : undefined,
      reasoningText: this.reasoningText || undefined,
      toolUseSteps,
      toolUseElapsedMs: this.toolUseElapsedMs,
@@ -186,43 +241,89 @@ export class StreamingAgentCard {
    if (this.currentMessageId === null) {
      this.currentMessageId = await sendCard(this.rt, this.chatId, card, this.sendOptions);
      let updated = this.currentMessageId !== null;
-      // Send overflow chunks as new messages (rare for agent output)
-      for (const chunk of chunks.slice(1)) {
-        const overflowCard = buildAgentCard({
-          phase,
-          text: chunk,
-          reasoningText: undefined,
-          toolUseSteps: [],
-          toolUseElapsedMs: undefined,
-          isError,
-          interrupted: this.interrupted,
-          runId: undefined,
-        });
-        const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
-        updated = updated && overflowMessageId !== null;
-        this.currentMessageId = overflowMessageId;
-      }
-      return updated;
-    } else {
-      let updated = await patchCard(this.rt, this.currentMessageId, card);
-      // For overflow, create new messages
-      for (const chunk of chunks.slice(1)) {
-        const overflowCard = buildAgentCard({
-          phase,
-          text: chunk,
-          reasoningText: undefined,
-          toolUseSteps: [],
-          toolUseElapsedMs: undefined,
-          isError,
-          interrupted: this.interrupted,
-          runId: undefined,
-        });
-        const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
-        updated = updated && overflowMessageId !== null;
-        this.currentMessageId = overflowMessageId;
+      // Send overflow chunks as new messages (rare for agent output). Segments
+      // already include the whole answer; only plain text overflows.
+      if (contentSegments === undefined || contentSegments.length === 0) {
+        for (const chunk of chunks.slice(1)) {
+          const overflowCard = buildAgentCard({
+            phase,
+            text: chunk,
+            reasoningText: undefined,
+            toolUseSteps: [],
+            toolUseElapsedMs: undefined,
+            isError,
+            interrupted: this.interrupted,
+            runId: undefined,
+          });
+          const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
+          updated = updated && overflowMessageId !== null;
+          this.currentMessageId = overflowMessageId;
+        }
      }
      return updated;
    }
+
+    let updated = await patchCard(this.rt, this.currentMessageId, card);
+    if (contentSegments === undefined || contentSegments.length === 0) {
+      for (const chunk of chunks.slice(1)) {
+        const overflowCard = buildAgentCard({
+          phase,
+          text: chunk,
+          reasoningText: undefined,
+          toolUseSteps: [],
+          toolUseElapsedMs: undefined,
+          isError,
+          interrupted: this.interrupted,
+          runId: undefined,
+        });
+        const overflowMessageId = await sendCard(this.rt, this.chatId, overflowCard, this.sendOptions);
+        updated = updated && overflowMessageId !== null;
+        this.currentMessageId = overflowMessageId;
+      }
+    }
+    return updated;
+  }
+
+  private async deliverPlainFallback(
+    segments: readonly CardContentSegment[],
+    answerText: string,
+  ): Promise<boolean> {
+    const textParts: string[] = [];
+    const imageKeys: string[] = [];
+    if (segments.length > 0) {
+      for (const segment of segments) {
+        if (segment.type === "markdown") {
+          if (segment.content.trim() !== "") textParts.push(segment.content);
+        } else {
+          imageKeys.push(segment.imgKey);
+        }
+      }
+    } else if (answerText.trim() !== "") {
+      textParts.push(maskMarkdownImagesForStreaming(answerText));
+    }
+
+    let any = false;
+    if (textParts.length > 0) {
+      const messageId = await sendLongText(
+        this.rt,
+        this.chatId,
+        textParts.join("\n\n"),
+        this.sendOptions,
+      );
+      any = messageId !== null;
+    }
+    for (const imageKey of imageKeys) {
+      try {
+        const messageId = await sendImageMessage(this.rt, this.chatId, imageKey, this.sendOptions);
+        any = any || messageId !== null;
+      } catch (error) {
+        this.rt.logger.warn(
+          { runId: this.runId, err: error instanceof Error ? error.message : String(error) },
+          "standalone image fallback failed",
+        );
+      }
+    }
+    return any;
  }

  private currentPhase(): CardPhase {
@@ -235,5 +336,5 @@ export class StreamingAgentCard {
 function appendFooter(text: string, footerText: string): string {
  if (footerText === "") return text;
  if (text === "") return footerText;
-  return `${text.trimEnd()}\n\n${footerText}`;
+  return `${text}\n\n${footerText}`;
 }