feat: send workspace images to QQ chats

Worker results may report image attachments stored inside the workspace;
the runtime validates them (containment, png/jpg magic, size) and
delivers them with the pending event. Assistants gain a send_image
action so users can ask for an image later. QQ uploads via /files with
base64 file_data and sends msg_type 7 rich media, sharing the same
msg_id/msg_seq counter as text replies; non-image adapters flatten
images to text.
This commit is contained in:
zenord
2026-08-19 14:39:18 +08:00
parent 7aa610294c
commit 6b506b8c55
17 changed files with 659 additions and 62 deletions
+89 -5
View File
@@ -22,7 +22,7 @@ interface Harness {
store: DurableSessionStore;
proposals: ProposalStore;
manager: AssistantManager;
events: { chatKey: string; text: string }[];
events: { chatKey: string; text: string; images?: { path: string; mimeType?: string; filename?: string }[] }[];
logFile: string;
}
@@ -41,12 +41,12 @@ async function createHarness(options: { initialize?: boolean; agentEnv?: Record<
const proposals = new ProposalStore(path.join(home, "state", "proposals.json"), identity);
await store.open();
await proposals.open();
const events: { chatKey: string; text: string }[] = [];
const events: { chatKey: string; text: string; images?: { path: string; mimeType?: string; filename?: string }[] }[] = [];
const manager = new AssistantManager(config.runtime.acp, new BotProfileResolver(config).bot, store, proposals, {
assistantWorkspaceHome: home,
allowUnverifiedAssistantAgent: true
});
manager.setEventSink(async (chatKey, text) => { events.push({ chatKey, text }); });
manager.setEventSink(async (chatKey, text, images) => { events.push({ chatKey, text, images }); });
if (options.initialize !== false) await manager.initialize();
return { home, workspace, store, proposals, manager, events, logFile };
}
@@ -70,7 +70,7 @@ async function reopenManager(harness: Harness): Promise<void> {
assistantWorkspaceHome: harness.home,
allowUnverifiedAssistantAgent: true
});
harness.manager.setEventSink(async (chatKey, text) => { harness.events.push({ chatKey, text }); });
harness.manager.setEventSink(async (chatKey, text, images) => { harness.events.push({ chatKey, text, images }); });
await harness.manager.initialize();
}
@@ -485,10 +485,21 @@ test("envelope parsers accept valid tails and reject invalid ones", () => {
assert.equal(parseAssistantActions(`x\n<GORI_ASSISTANT_ACTION_V1>{"reply":"hi","actions":[]}</GORI_ASSISTANT_ACTION_V1>`), undefined);
assert.deepEqual(
parseWorkerResult(`text\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","question":"q","workspaceDirty":true}</GORI_WORKER_RESULT_V2>`),
{ status: "PENDING", summary: "s", question: "q", workspaceDirty: true }
{ status: "PENDING", summary: "s", question: "q", workspaceDirty: true, attachments: undefined }
);
assert.deepEqual(
parseWorkerResult(`text\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","attachments":[{"path":".gori-outbox/a.png","mimeType":"image/png"}]}</GORI_WORKER_RESULT_V2>`),
{ status: "PENDING", summary: "s", question: undefined, workspaceDirty: undefined, attachments: [{ path: ".gori-outbox/a.png", mimeType: "image/png" }] }
);
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","attachments":[{"path":"a.png"},{"path":"b.png"},{"path":"c.png"},{"path":"d.png"}]}</GORI_WORKER_RESULT_V2>`), undefined);
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","attachments":[{"mimeType":"image/png"}]}</GORI_WORKER_RESULT_V2>`), undefined);
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V2>{"status":"SUCCESS","summary":"s"}</GORI_WORKER_RESULT_V2>`), undefined);
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V1>{"status":"PENDING","summary":"s"}</GORI_WORKER_RESULT_V1>`), undefined);
assert.deepEqual(
parseAssistantActions(`text\n<GORI_ASSISTANT_ACTION_V2>{"reply":"hi","actions":[{"type":"send_image","path":".gori-outbox/a.png"}]}</GORI_ASSISTANT_ACTION_V2>`),
{ reply: "hi", actions: [{ type: "send_image", path: ".gori-outbox/a.png" }] }
);
assert.equal(parseAssistantActions(`x\n<GORI_ASSISTANT_ACTION_V2>{"reply":"hi","actions":[{"type":"send_image"}]}</GORI_ASSISTANT_ACTION_V2>`), undefined);
});
test("an owner's own pending proposal blocks their start_next with an actionable correction", async () => {
@@ -678,6 +689,79 @@ test("image attachments degrade to a text note when the agent has no image capab
}
});
test("worker attachments are validated against the workspace and ride along with the owner event", async () => {
const harness = await createHarness();
try {
await harness.manager.prompt(request("create proposal: attach"));
const proposal = harness.proposals.list()[0]!;
await harness.manager.prompt(request("confirm"));
await waitFor(() => harness.proposals.get(proposal.id)!.status === "pending");
const pending = harness.proposals.get(proposal.id)!.pending!;
assert.equal(pending.attachments?.length, 1);
const stored = pending.attachments![0]!;
assert.equal(stored.mimeType, "image/png");
const workspaceRoot = fs.realpathSync(harness.workspace);
assert.ok(stored.path.startsWith(`${workspaceRoot}${path.sep}`), stored.path);
assert.ok(stored.path.endsWith(path.join(".gori-outbox", "shot.png")), stored.path);
assert.equal(pending.droppedAttachments, undefined);
await waitFor(() => harness.events.length > 0);
assert.equal(harness.events[0]!.images?.length, 1);
assert.equal(harness.events[0]!.images![0]!.path, stored.path);
assert.equal(harness.events[0]!.images![0]!.filename, "shot.png");
} finally {
await closeHarness(harness);
}
});
test("worker attachments outside the workspace or not png/jpg are dropped without failing the result", async () => {
const harness = await createHarness();
try {
await harness.manager.prompt(request("create proposal: attachbad"));
const proposal = harness.proposals.list()[0]!;
await harness.manager.prompt(request("confirm"));
await waitFor(() => harness.proposals.get(proposal.id)!.status === "pending");
const pending = harness.proposals.get(proposal.id)!.pending!;
assert.equal(pending.summary, "made a pic");
assert.equal(pending.attachments, undefined);
assert.deepEqual(pending.droppedAttachments, ["/tmp/evil.png", "not-an-image.txt"]);
await waitFor(() => harness.events.length > 0);
assert.equal(harness.events[0]!.images, undefined);
} finally {
await closeHarness(harness);
}
});
test("send_image validates the path and attaches the image to the reply", async () => {
const harness = await createHarness();
try {
await harness.manager.prompt(request("create proposal: attach"));
const proposal = harness.proposals.list()[0]!;
await harness.manager.prompt(request("confirm"));
await waitFor(() => harness.proposals.get(proposal.id)!.status === "pending");
const sent = await harness.manager.prompt(request("send image: .gori-outbox/shot.png"));
assert.equal(sent.images?.length, 1);
assert.equal(sent.images![0]!.filename, "shot.png");
assert.equal(sent.images![0]!.mimeType, "image/png");
assert.ok(sent.images![0]!.path.startsWith(fs.realpathSync(harness.workspace)));
assert.doesNotMatch(sent.text, /发不出去/);
const outside = await harness.manager.prompt(request("send image: /tmp/evil.png"));
assert.equal(outside.images, undefined);
assert.match(outside.text, /发不出去/);
const notImage = await harness.manager.prompt(request("send image: not-an-image.txt"));
assert.equal(notImage.images, undefined);
assert.match(notImage.text, /发不出去/);
} finally {
await closeHarness(harness);
}
});
function processGroupExists(pgid: number): boolean {
try { process.kill(-pgid, 0); return true; }
catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; }
+16
View File
@@ -1,9 +1,12 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { Readable, Writable } from "node:stream";
import * as acp from "@agentclientprotocol/sdk";
const PNG_HEADER = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
const pending = new Map();
const logFile = process.env.FAKE_ACP_LOG;
const log = (entry) => { if (logFile) fs.appendFileSync(logFile, `${JSON.stringify(entry)}\n`); };
@@ -84,6 +87,7 @@ const app = acp.agent({ name: "fake-acp-agent" })
else if (userText === "finish") response = assistantEnvelope("finishing", [{ type: "finish" }]);
else if (userText.startsWith("follow up:")) response = assistantEnvelope("following up", [{ type: "follow_up", instruction: userText.slice("follow up:".length).trim() }]);
else if (userText.startsWith("adjust proposal:")) response = assistantEnvelope("adjusting", [{ type: "adjust_proposal", title: userText.slice("adjust proposal:".length).trim() }]);
else if (userText.startsWith("send image:")) response = assistantEnvelope("sending image", [{ type: "send_image", path: userText.slice("send image:".length).trim() }]);
else if (userText === "start next") response = assistantEnvelope("starting next", [{ type: "start_next" }]);
else if (userText === "stop") response = assistantEnvelope("stopping", [{ type: "stop" }]);
else if (userText === "cancel proposal") response = assistantEnvelope("cancelling", [{ type: "cancel" }]);
@@ -117,6 +121,18 @@ const app = acp.agent({ name: "fake-acp-agent" })
if (process.env.FAKE_ACP_WORKER_GATE_FILE) {
while (!fs.existsSync(process.env.FAKE_ACP_WORKER_GATE_FILE)) await new Promise((resolve) => setTimeout(resolve, 5));
}
if (goal.includes("attachbad")) {
await update(client, params.sessionId, workerEnvelope({ status: "PENDING", summary: "made a pic", attachments: [{ path: "/tmp/evil.png" }, { path: "not-an-image.txt" }] }));
return { stopReason: "end_turn" };
}
if (goal.includes("attach")) {
const outbox = path.join(process.cwd(), ".gori-outbox");
fs.mkdirSync(outbox, { recursive: true });
fs.writeFileSync(path.join(outbox, "shot.png"), Buffer.concat([PNG_HEADER, Buffer.from("fake-png-payload-v1")]));
fs.writeFileSync(path.join(process.cwd(), "not-an-image.txt"), "plain text");
await update(client, params.sessionId, workerEnvelope({ status: "PENDING", summary: "made a pic", attachments: [{ path: ".gori-outbox/shot.png", mimeType: "image/png" }] }));
return { stopReason: "end_turn" };
}
const result = goal.includes("ask")
? { status: "PENDING", summary: "hit a dirty target", question: "May I overwrite it?", workspaceDirty: true }
: goal.includes("fail")
+153 -3
View File
@@ -1,4 +1,7 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import test from "node:test";
import type { ConversationRequest, ConversationRuntime } from "../src/acp/types.js";
import type { PlatformAdapter } from "../src/core/adapter.js";
@@ -6,6 +9,15 @@ import { Gateway } from "../src/core/gateway.js";
import type { Proposal } from "../src/core/proposal-store.js";
import type { IncomingMessage, OutgoingMessage } from "../src/core/types.js";
const PNG_HEADER = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
async function tempPng(payload: string): Promise<{ dir: string; file: string }> {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "gori-gateway-img-"));
const file = path.join(dir, "shot.png");
await fs.promises.writeFile(file, Buffer.concat([PNG_HEADER, Buffer.from(payload)]));
return { dir, file };
}
interface PendingTurn {
request: ConversationRequest;
resolve(text: string): void;
@@ -76,12 +88,17 @@ const groupMessage = (text: string, messageId: string): IncomingMessage => ({
});
const flush = async () => { await Promise.resolve(); await Promise.resolve(); await new Promise<void>((resolve) => setImmediate(resolve)); };
const waitFor = async (condition: () => boolean) => {
for (let count = 0; count < 50 && !condition(); count++) await flush();
const deadline = Date.now() + 5_000;
while (!condition()) {
if (Date.now() > deadline) break;
await flush();
await new Promise<void>((resolve) => setTimeout(resolve, 5));
}
assert.equal(condition(), true);
};
function recordingAdapter(sent: OutgoingMessage[] = []): PlatformAdapter {
return { name: "test", async handleWebhook() { return {}; }, async sendMessage(outgoing) { sent.push(outgoing); } };
function recordingAdapter(sent: OutgoingMessage[] = [], supportsImages = false): PlatformAdapter {
return { name: "test", supportsImages, async handleWebhook() { return {}; }, async sendMessage(outgoing) { sent.push(outgoing); } };
}
// Simulates QQ passive-reply dedup: a repeated msg_id + msg_seq pair is rejected.
@@ -286,6 +303,139 @@ test("status reports the last successful event delivery for the current chat", a
assert.match(status.reply || "", /lastEventAt=\d{4}-\d{2}-\d{2}T/);
});
test("sendEvent delivers the text first, then images, all sharing the msg_seq counter", async () => {
const { runtime, gateway } = createGateway();
const { dir, file } = await tempPng("payload-v1");
try {
const sent: OutgoingMessage[] = [];
const turn = gateway.receive(message("work", "img-event"), recordingAdapter(sent, true));
await waitFor(() => runtime.turns.length === 1);
runtime.turns[0].resolve("done");
await turn;
await gateway.sendEvent("qq:chat", "截图好了", [{ path: file, filename: "shot.png" }]);
const pieces = sent.map((outgoing) => [outgoing.text, outgoing.replySequence, outgoing.images?.length || 0]);
assert.deepEqual(pieces, [
["done", 1, 0],
["截图好了", 2, 0],
["", 3, 1]
]);
const image = sent[2]!.images![0]!;
assert.equal(image.mimeType, "image/png");
assert.equal(image.filename, "shot.png");
assert.equal(Buffer.from(image.data, "base64").toString("latin1"), Buffer.concat([PNG_HEADER, Buffer.from("payload-v1")]).toString("latin1"));
} finally {
await fs.promises.rm(dir, { recursive: true, force: true });
}
});
test("a queued event re-reads its image at redelivery time and degrades when the file is gone", async () => {
let now = 1_000_000;
const runtime = new FakeRuntime();
const gateway = new Gateway(policy, runtime, { now: () => now });
const { dir, file } = await tempPng("payload-v1");
try {
const sent: OutgoingMessage[] = [];
const adapter = recordingAdapter(sent, true);
const first = gateway.receive(groupMessage("work", "m1"), adapter);
await waitFor(() => runtime.turns.length === 1);
runtime.turns[0].resolve("done");
await first;
now += 6 * 60_000; // past the group window: the event is queued instead of sent
await gateway.sendEvent("qq:group:g1", "stale event", [{ path: file, filename: "shot.png" }]);
assert.equal(sent.filter((outgoing) => outgoing.text === "stale event").length, 0);
// The worker updated the image after the event was queued: redelivery must send the new content.
await fs.promises.writeFile(file, Buffer.concat([PNG_HEADER, Buffer.from("payload-v2")]));
const second = gateway.receive(groupMessage("hello", "m2"), adapter);
await waitFor(() => runtime.turns.length === 2);
runtime.turns[1].resolve("ok");
await second;
const redelivered = sent.filter((outgoing) => outgoing.replyTo === "m2");
assert.deepEqual(redelivered.map((outgoing) => [outgoing.text, outgoing.replySequence, outgoing.images?.length || 0]), [
["stale event", 1, 0],
["", 2, 1],
["ok", 3, 0]
]);
assert.equal(Buffer.from(redelivered[1]!.images![0]!.data, "base64").toString("latin1"), Buffer.concat([PNG_HEADER, Buffer.from("payload-v2")]).toString("latin1"));
// Queue another event, then delete the file: the redelivery degrades to a text note.
now += 6 * 60_000;
await gateway.sendEvent("qq:group:g1", "another stale event", [{ path: file, filename: "shot.png" }]);
await fs.promises.rm(file);
const third = gateway.receive(groupMessage("hello again", "m3"), adapter);
await waitFor(() => runtime.turns.length === 3);
runtime.turns[2].resolve("ok again");
await third;
const degraded = sent.filter((outgoing) => outgoing.replyTo === "m3");
assert.equal(degraded.filter((outgoing) => outgoing.images?.length).length, 0);
assert.match(degraded[0]!.text, /another stale event/);
assert.match(degraded[0]!.text, /图片 shot\.png 发送失败/);
} finally {
await fs.promises.rm(dir, { recursive: true, force: true });
}
});
test("adapters without image support receive images flattened into text lines", async () => {
const { runtime, gateway } = createGateway();
const { dir, file } = await tempPng("payload-v1");
try {
const sent: OutgoingMessage[] = [];
const turn = gateway.receive(message("work", "flat"), recordingAdapter(sent));
await waitFor(() => runtime.turns.length === 1);
runtime.turns[0].resolve("done");
await turn;
await gateway.sendEvent("qq:chat", "截图好了", [{ path: file, filename: "shot.png" }]);
const event = sent.find((outgoing) => outgoing.text.includes("截图好了"))!;
assert.equal(event.images, undefined);
assert.match(event.text, /截图好了/);
assert.match(event.text, /\[图片\] shot\.png/);
assert.equal(sent.filter((outgoing) => outgoing.images?.length).length, 0);
} finally {
await fs.promises.rm(dir, { recursive: true, force: true });
}
});
test("an image send failure degrades to a text note without blocking the event", async () => {
const { runtime, gateway } = createGateway();
const { dir, file } = await tempPng("payload-v1");
try {
const sent: OutgoingMessage[] = [];
const errors: string[] = [];
const adapter: PlatformAdapter = {
name: "test", supportsImages: true, async handleWebhook() { return {}; },
async sendMessage(outgoing) {
if (outgoing.images?.length) throw new Error("sensitive provider response");
sent.push(outgoing);
}
};
const originalError = console.error;
console.error = (...args: unknown[]) => { errors.push(args.map(String).join(" ")); };
try {
const turn = gateway.receive(message("work", "img-fail"), adapter);
await waitFor(() => runtime.turns.length === 1);
runtime.turns[0].resolve("done");
await turn;
await gateway.sendEvent("qq:chat", "截图好了", [{ path: file, filename: "shot.png" }]);
} finally {
console.error = originalError;
}
const texts = sent.map((outgoing) => [outgoing.text, outgoing.replySequence]);
// The failed image attempt consumed msg_seq 3 (never reused in case QQ actually received it).
assert.deepEqual(texts, [
["done", 1],
["截图好了", 2],
["(图片 shot.png 发送失败)", 4]
]);
assert.deepEqual(errors, ["Gateway event image send failed (platform=qq)"]);
} finally {
await fs.promises.rm(dir, { recursive: true, force: true });
}
});
test("normal replies and fresh events share one msg_seq counter per inbound message", async () => {
const runtime = new FakeRuntime();
const gateway = new Gateway(policy, runtime);
+71
View File
@@ -171,6 +171,77 @@ test("sendMessage uses C2C endpoint and forwards reply sequences as msg_seq", as
} finally { globalThis.fetch = original; }
});
test("sendMessage uploads images via /files and sends msg_type 7 media with the shared msg_seq", async () => {
const original = globalThis.fetch; const urls: string[] = []; const bodies: Array<Record<string, unknown>> = [];
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = String(input);
urls.push(url);
if (url.includes("getAppAccessToken")) return new Response(JSON.stringify({ access_token: "token", expires_in: 7200 }), { status: 200 });
bodies.push(JSON.parse(String(init?.body)) as Record<string, unknown>);
if (url.endsWith("/files")) return new Response(JSON.stringify({ file_info: "FILEINFO", ttl: 60 }), { status: 200 });
return new Response(JSON.stringify({ id: "sent" }), { status: 200 });
}) as typeof fetch;
try {
const adapter = new QqAdapter(config, { receive: async () => ({ ok: true }) } as never);
const target = { platform: "qq", chatId: "group:g1", raw: { group_openid: "g1" } };
const image = { mimeType: "image/png", data: Buffer.from("fake-png").toString("base64"), filename: "shot.png" };
await adapter.sendMessage({ target, text: "", images: [image], replyTo: "m1", replySequence: 3 });
const apiCalls = urls.filter((url) => !url.includes("getAppAccessToken")).map((url) => url.replace("https://api.sgroup.qq.com", ""));
assert.deepEqual(apiCalls, ["/v2/groups/g1/files", "/v2/groups/g1/messages"]);
assert.deepEqual(bodies, [
{ file_type: 1, file_data: image.data, srv_send_msg: false },
{ msg_type: 7, media: { file_info: "FILEINFO" }, content: "", msg_id: "m1", msg_seq: 3 }
]);
} finally { globalThis.fetch = original; }
});
test("sendMessage delivers text and images to C2C with independent upload per target", async () => {
const original = globalThis.fetch; const urls: string[] = []; const bodies: Array<Record<string, unknown>> = [];
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {
const url = String(input);
urls.push(url);
if (url.includes("getAppAccessToken")) return new Response(JSON.stringify({ access_token: "token", expires_in: 7200 }), { status: 200 });
bodies.push(JSON.parse(String(init?.body)) as Record<string, unknown>);
if (url.endsWith("/files")) return new Response(JSON.stringify({ file_info: "FILEINFO" }), { status: 200 });
return new Response(JSON.stringify({ id: "sent" }), { status: 200 });
}) as typeof fetch;
try {
const adapter = new QqAdapter(config, { receive: async () => ({ ok: true }) } as never);
const target = { platform: "qq", chatId: "user:u2", raw: { author: { user_openid: "u2" } } };
const image = { mimeType: "image/jpeg", data: Buffer.from("fake-jpg").toString("base64") };
await adapter.sendMessage({ target, text: "看图", images: [image], replyTo: "m2", replySequence: 5 });
assert.ok(urls.some((url) => url.endsWith("/v2/users/u2/files")));
assert.equal(urls.filter((url) => url.endsWith("/v2/users/u2/messages")).length, 2);
assert.deepEqual(bodies[0], { file_type: 1, file_data: image.data, srv_send_msg: false });
assert.deepEqual(bodies[1], { msg_type: 7, media: { file_info: "FILEINFO" }, content: "", msg_id: "m2", msg_seq: 5 });
assert.deepEqual(bodies[2], { content: "看图", msg_id: "m2", msg_seq: 5 });
} finally { globalThis.fetch = original; }
});
test("image upload failures surface safe errors without the base64 payload", async () => {
const original = globalThis.fetch;
globalThis.fetch = (async (input: string | URL | Request) => {
const url = String(input);
if (url.includes("getAppAccessToken")) return new Response(JSON.stringify({ access_token: "token", expires_in: 7200 }), { status: 200 });
return new Response(JSON.stringify({ code: 40034001, message: "invalid file_data" }), { status: 400 });
}) as typeof fetch;
try {
const adapter = new QqAdapter(config, { receive: async () => ({ ok: true }) } as never);
const target = { platform: "qq", chatId: "group:g1", raw: { group_openid: "g1" } };
const image = { mimeType: "image/png", data: Buffer.from("secret-image-bytes").toString("base64") };
await assert.rejects(
adapter.sendMessage({ target, text: "", images: [image], replyTo: "m1", replySequence: 1 }),
(error: Error) => {
assert.match(error.message, /QQ image upload failed: HTTP 400/);
assert.match(error.message, /code=40034001/);
assert.doesNotMatch(error.message, /secret-image-bytes/);
assert.doesNotMatch(error.message, /base64/);
return true;
}
);
} finally { globalThis.fetch = original; }
});
test("sendMessage without replyTo omits msg_id and msg_seq from the body", async () => {
const original = globalThis.fetch; const bodies: Array<Record<string, unknown>> = [];
globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => {