heihei/gori-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: send workspace images to QQ chats

Browse Source 
Worker results may report image attachments stored inside the workspace;
the runtime validates them (containment, png/jpg magic, size) and
delivers them with the pending event. Assistants gain a send_image
action so users can ask for an image later. QQ uploads via /files with
base64 file_data and sends msg_type 7 rich media, sharing the same
msg_id/msg_seq counter as text replies; non-image adapters flatten
images to text.
This commit is contained in:
zenord
2026-08-19 14:39:18 +08:00
parent 7aa610294c
commit 6b506b8c55
17 changed files with 659 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/acp/assistant-manager.ts
+69 -14
View File
@@ -4,7 +4,8 @@ import path from "node:path";
import type { AcpConfig, PermissionPolicy } from "../config.js";
import { chatKeyFor, type AssistantBinding, type DurableSessionStore } from "../core/durable-session-store.js";
import type { Proposal, ProposalPending, ProposalStore } from "../core/proposal-store.js";
import type { IncomingAttachment } from "../core/types.js";
import type { IncomingAttachment, OutgoingImageRef } from "../core/types.js";
import { MAX_OUTBOUND_IMAGES, validateWorkspaceImage } from "../core/workspace-images.js";
import type { ResolvedBot } from "../roles/role-registry.js";
import type { ConversationRequest, ConversationResponse, ConversationRuntime, RuntimeEventSink, RuntimeStats } from "./types.js";
import type { AcpPromptContent } from "./client.js";
@@ -20,6 +21,7 @@ export type AssistantAction =
| { type: "adjust_proposal"; id?: string; title?: string; goal?: string; steps?: string[] }
| { type: "follow_up"; id?: string; instruction: string }
| { type: "finish"; id?: string; note?: string }
| { type: "send_image"; path: string }
| { type: "start_next" }
| { type: "cancel"; id?: string }
| { type: "stop" };
@@ -36,6 +38,7 @@ export interface ParsedWorkerResult {
summary: string;
question?: string;
workspaceDirty?: boolean;
attachments?: { path: string; mimeType?: string }[];
}
interface ActiveWorker {
@@ -111,8 +114,8 @@ export class AssistantManager implements ConversationRuntime {
async prompt(request: ConversationRequest): Promise<ConversationResponse> {
if (this.shuttingDown) throw new Error("ACP runtime is shutting down");
const conversationKey = assistantKeyFor(chatKeyFor(request.platform, request.chatId), request.userId);
const reply = await this.enqueueChat(conversationKey, () => this.runAssistantTurn(conversationKey, request));
return { text: reply, botId: this.bot.id, agentId: this.bot.agent.id, mode: "assistant" };
const result = await this.enqueueChat(conversationKey, () => this.runAssistantTurn(conversationKey, request));
return { text: result.text, images: result.images, botId: this.bot.id, agentId: this.bot.agent.id, mode: "assistant" };
}
async cancel(platform: string, chatId: string, userId: string): Promise<boolean> {
@@ -226,7 +229,7 @@ export class AssistantManager implements ConversationRuntime {
await Promise.allSettled([...this.chatChains.values()]);
}
private async runAssistantTurn(conversationKey: string, request: ConversationRequest): Promise<string> {
private async runAssistantTurn(conversationKey: string, request: ConversationRequest): Promise<{ text: string; images?: OutgoingImageRef[] }> {
const worker = await this.acquireAssistantWorker(conversationKey);
try {
const reply = await worker.prompt(this.assistantPromptContent(worker, request));
@@ -234,10 +237,10 @@ export class AssistantManager implements ConversationRuntime {
if (worker.isolationViolated) throw new Error("Assistant session attempted forbidden tool activity");
await this.store.touchBinding(conversationKey);
const chatKey = chatKeyFor(request.platform, request.chatId);
const corrections = await this.executeActions(chatKey, request, parsed.actions);
const { corrections, images } = await this.executeActions(chatKey, request, parsed.actions);
const reminders = this.pendingReminders(chatKey, request.userId, parsed.reply);
const extras = [...corrections, ...reminders];
return extras.length > 0 ? `${parsed.reply}\n${extras.join("\n")}` : parsed.reply;
return { text: extras.length > 0 ? `${parsed.reply}\n${extras.join("\n")}` : parsed.reply, images };
} catch (error) {
if (worker.isolationViolated) await this.store.deleteBinding(conversationKey).catch(() => undefined);
throw error;
@@ -269,8 +272,9 @@ export class AssistantManager implements ConversationRuntime {
return parsed;
}
private async executeActions(chatKey: string, request: ConversationRequest, actions: AssistantAction[]): Promise<string[]> {
private async executeActions(chatKey: string, request: ConversationRequest, actions: AssistantAction[]): Promise<{ corrections: string[]; images?: OutgoingImageRef[] }> {
const corrections: string[] = [];
const images: OutgoingImageRef[] = [];
for (const action of actions) {
if (action.type === "create_proposal") {
await this.proposals.create({
@@ -292,6 +296,10 @@ export class AssistantManager implements ConversationRuntime {
} else if (action.type === "finish") {
const ok = await this.scheduler(() => this.finishLocked(chatKey, request.userId, action.id, action.note));
if (!ok) corrections.push("没有可结束的任务；只有待确认的任务可以 finish，且你只能操作自己发起的任务。");
} else if (action.type === "send_image") {
const image = await validateWorkspaceImage(this.bot.workspace, action.path);
if (image) images.push({ path: image.path, mimeType: image.mimeType, filename: image.filename });
else corrections.push("这张图发不出去：路径不在工作区内，或者不是有效的 png/jpg 图片。");
} else if (action.type === "start_next") {
const started = await this.scheduler(() => this.tryStartLocked(chatKey, request.userId));
if (!started) corrections.push(this.startNextCorrection(chatKey, request.userId));
@@ -303,7 +311,7 @@ export class AssistantManager implements ConversationRuntime {
if (!ok) corrections.push("没有可停止的任务；你只能操作自己发起的任务。");
}
}
return corrections;
return { corrections, images: images.length > 0 ? images.slice(0, MAX_OUTBOUND_IMAGES) : undefined };
}
private startNextCorrection(chatKey: string, userId: string): string {
@@ -517,11 +525,23 @@ export class AssistantManager implements ConversationRuntime {
if (!proposal || proposal.status !== "working") { active.settled = true; return false; }
active.settled = true;
this.active = undefined;
const attachments: { path: string; mimeType?: string }[] = [];
const droppedAttachments: string[] = [];
for (const ref of result.attachments || []) {
const image = await validateWorkspaceImage(this.bot.workspace, ref.path);
if (image) attachments.push({ path: image.path, mimeType: image.mimeType });
else droppedAttachments.push(ref.path);
}
if (droppedAttachments.length > 0) {
console.error(`Worker reported ${droppedAttachments.length} invalid attachment(s) for proposal ${active.proposalId}; dropped (outside workspace or not a valid png/jpg)`);
}
const pending: ProposalPending = {
summary: result.summary,
question: result.question,
workspaceDirty: result.workspaceDirty,
receivedAt: Date.now()
receivedAt: Date.now(),
attachments: attachments.length > 0 ? attachments : undefined,
droppedAttachments: droppedAttachments.length > 0 ? droppedAttachments : undefined
};
await this.proposals.update(active.proposalId, {
status: "pending",
@@ -578,11 +598,11 @@ export class AssistantManager implements ConversationRuntime {
if (this.shuttingDown) return;
try {
const reply = await this.runAssistantEvent(conversationKey, proposal.id);
if (reply) await sink(chatKey, reply);
if (reply) await sink(chatKey, reply, pendingImageRefs(this.proposals.get(proposalId)));
} catch (error) {
console.error(`Assistant event notification failed (${error instanceof Error ? error.name : "unknown error"})`);
const latest = this.proposals.get(proposalId);
if (latest?.pending) await sink(chatKey, fallbackEventText(latest)).catch(() => undefined);
if (latest?.pending) await sink(chatKey, fallbackEventText(latest), pendingImageRefs(latest)).catch(() => undefined);
}
}).catch(() => undefined);
}
@@ -844,6 +864,10 @@ function parseAssistantAction(value: unknown): AssistantAction | undefined {
if ((value.id !== undefined && typeof value.id !== "string") || (value.note !== undefined && typeof value.note !== "string")) return undefined;
return { type: "finish", id: value.id as string | undefined, note: value.note as string | undefined };
}
if (value.type === "send_image") {
if (typeof value.path !== "string" || value.path.length === 0) return undefined;
return { type: "send_image", path: value.path };
}
if (value.type === "start_next") return { type: "start_next" };
if (value.type === "cancel") {
if (value.id !== undefined && typeof value.id !== "string") return undefined;
@@ -862,16 +886,27 @@ export function parseWorkerResult(text: string): ParsedWorkerResult | undefined
if (value.status !== "PENDING") return undefined;
if ((value.question !== undefined && typeof value.question !== "string")
|| (value.workspaceDirty !== undefined && typeof value.workspaceDirty !== "boolean")) return undefined;
let attachments: { path: string; mimeType?: string }[] | undefined;
if (value.attachments !== undefined) {
if (!Array.isArray(value.attachments) || value.attachments.length > MAX_OUTBOUND_IMAGES) return undefined;
attachments = [];
for (const item of value.attachments) {
if (!isRecord(item) || typeof item.path !== "string" || item.path.length === 0
|| (item.mimeType !== undefined && typeof item.mimeType !== "string")) return undefined;
attachments.push({ path: item.path, mimeType: item.mimeType as string | undefined });
}
}
return {
status: "PENDING",
summary: value.summary,
question: value.question as string | undefined,
workspaceDirty: value.workspaceDirty as boolean | undefined
workspaceDirty: value.workspaceDirty as boolean | undefined,
attachments
};
}
function withWorkerResultProtocol(input: string | AcpPromptContent[]): string | AcpPromptContent[] {
const instruction = "When this turn is finished, end your response with exactly one hidden worker result envelope: <GORI_WORKER_RESULT_V2>{\"status\":\"PENDING\",\"summary\":\"...\"}</GORI_WORKER_RESULT_V2>. The summary is a short user-readable description of what you did or what is blocking you. Add a \"question\" when you need the user's decision before continuing, and set \"workspaceDirty\": true when you left the workspace modified or are unsure about its state. PENDING is the only status; do not emit any other status value or any text after the envelope.";
const instruction = "When this turn is finished, end your response with exactly one hidden worker result envelope: <GORI_WORKER_RESULT_V2>{\"status\":\"PENDING\",\"summary\":\"...\"}</GORI_WORKER_RESULT_V2>. The summary is a short user-readable description of what you did or what is blocking you. Add a \"question\" when you need the user's decision before continuing, and set \"workspaceDirty\": true when you left the workspace modified or are unsure about its state. When you produced image files the user should see (png/jpg only), save them inside the workspace (prefer .gori-outbox/) and report up to 3 of them as \"attachments\": [{\"path\":\"relative/or/absolute/path\"}]; never report paths outside the workspace. PENDING is the only status; do not emit any other status value or any text after the envelope.";
if (typeof input === "string") return `${input}\n\n${instruction}`;
return [...input, { type: "text" as const, text: `\n\n${instruction}` }];
}
@@ -908,18 +943,38 @@ function workerFollowUpPrompt(instruction: string, question?: string): string {
function assistantEventPrompt(proposal: Proposal): string {
const pending = proposal.pending!;
const attachments = pending.attachments?.length
? `Image attachments produced by the worker (already sent to the user after your message): ${pending.attachments.map((attachment) => attachment.path).join(", ")}`
: "Image attachments: none";
const dropped = pending.droppedAttachments?.length
? `Note: ${pending.droppedAttachments.length} reported attachment(s) were invalid (outside the workspace or not png/jpg) and were dropped; mention this to the user.`
: "";
return [
"[Internal event from gori-agent. This is not a user message.]",
`The worker for proposal "${proposal.title}" (id ${proposal.id}) reported its result and the proposal is now waiting for the user's decision.`,
`Summary: ${pending.summary}`,
pending.question ? `Question for the user: ${pending.question}` : "Question for the user: none",
attachments,
...(dropped ? [dropped] : []),
"Tell the user, in your own words, what happened and what they can do next (finish to close the task, follow up with new instructions to continue it, or stop). End with the usual GORI_ASSISTANT_ACTION_V2 envelope; its actions array MUST be empty because actions are ignored for internal events."
].join("\n");
}
function fallbackEventText(proposal: Proposal): string {
const pending = proposal.pending!;
return `任务「${proposal.title}」等你确认。摘要：${pending.summary}${pending.question ? ` 问题：${pending.question}` : ""} 说 finish 结束，或直接说要求继续改。`;
const attachments = pending.attachments?.length ? `（附 ${pending.attachments.length} 张图片）` : "";
const dropped = pending.droppedAttachments?.length ? `（${pending.droppedAttachments.length} 个附件校验失败被丢弃）` : "";
return `任务「${proposal.title}」等你确认。摘要：${pending.summary}${pending.question ? ` 问题：${pending.question}` : ""}${attachments}${dropped} 说 finish 结束，或直接说要求继续改。`;
}
function pendingImageRefs(proposal: Proposal | undefined): OutgoingImageRef[] | undefined {
const attachments = proposal?.pending?.attachments;
if (!attachments || attachments.length === 0) return undefined;
return attachments.map((attachment) => ({
path: attachment.path,
mimeType: attachment.mimeType,
filename: attachment.path.split("/").pop()
}));
}
function isRecord(value: unknown): value is Record<string, unknown> {