feat: send workspace images to QQ chats
Worker results may report image attachments stored inside the workspace; the runtime validates them (containment, png/jpg magic, size) and delivers them with the pending event. Assistants gain a send_image action so users can ask for an image later. QQ uploads via /files with base64 file_data and sends msg_type 7 rich media, sharing the same msg_id/msg_seq counter as text replies; non-image adapters flatten images to text.
This commit is contained in:
zenord
@@ -22,7 +22,7 @@ interface Harness {
|
||||
store: DurableSessionStore;
|
||||
proposals: ProposalStore;
|
||||
manager: AssistantManager;
|
||||
events: { chatKey: string; text: string }[];
|
||||
events: { chatKey: string; text: string; images?: { path: string; mimeType?: string; filename?: string }[] }[];
|
||||
logFile: string;
|
||||
}
|
||||
|
||||
@@ -41,12 +41,12 @@ async function createHarness(options: { initialize?: boolean; agentEnv?: Record<
|
||||
const proposals = new ProposalStore(path.join(home, "state", "proposals.json"), identity);
|
||||
await store.open();
|
||||
await proposals.open();
|
||||
const events: { chatKey: string; text: string }[] = [];
|
||||
const events: { chatKey: string; text: string; images?: { path: string; mimeType?: string; filename?: string }[] }[] = [];
|
||||
const manager = new AssistantManager(config.runtime.acp, new BotProfileResolver(config).bot, store, proposals, {
|
||||
assistantWorkspaceHome: home,
|
||||
allowUnverifiedAssistantAgent: true
|
||||
});
|
||||
manager.setEventSink(async (chatKey, text) => { events.push({ chatKey, text }); });
|
||||
manager.setEventSink(async (chatKey, text, images) => { events.push({ chatKey, text, images }); });
|
||||
if (options.initialize !== false) await manager.initialize();
|
||||
return { home, workspace, store, proposals, manager, events, logFile };
|
||||
}
|
||||
@@ -70,7 +70,7 @@ async function reopenManager(harness: Harness): Promise<void> {
|
||||
assistantWorkspaceHome: harness.home,
|
||||
allowUnverifiedAssistantAgent: true
|
||||
});
|
||||
harness.manager.setEventSink(async (chatKey, text) => { harness.events.push({ chatKey, text }); });
|
||||
harness.manager.setEventSink(async (chatKey, text, images) => { harness.events.push({ chatKey, text, images }); });
|
||||
await harness.manager.initialize();
|
||||
}
|
||||
|
||||
@@ -485,10 +485,21 @@ test("envelope parsers accept valid tails and reject invalid ones", () => {
|
||||
assert.equal(parseAssistantActions(`x\n<GORI_ASSISTANT_ACTION_V1>{"reply":"hi","actions":[]}</GORI_ASSISTANT_ACTION_V1>`), undefined);
|
||||
assert.deepEqual(
|
||||
parseWorkerResult(`text\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","question":"q","workspaceDirty":true}</GORI_WORKER_RESULT_V2>`),
|
||||
{ status: "PENDING", summary: "s", question: "q", workspaceDirty: true }
|
||||
{ status: "PENDING", summary: "s", question: "q", workspaceDirty: true, attachments: undefined }
|
||||
);
|
||||
assert.deepEqual(
|
||||
parseWorkerResult(`text\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","attachments":[{"path":".gori-outbox/a.png","mimeType":"image/png"}]}</GORI_WORKER_RESULT_V2>`),
|
||||
{ status: "PENDING", summary: "s", question: undefined, workspaceDirty: undefined, attachments: [{ path: ".gori-outbox/a.png", mimeType: "image/png" }] }
|
||||
);
|
||||
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","attachments":[{"path":"a.png"},{"path":"b.png"},{"path":"c.png"},{"path":"d.png"}]}</GORI_WORKER_RESULT_V2>`), undefined);
|
||||
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V2>{"status":"PENDING","summary":"s","attachments":[{"mimeType":"image/png"}]}</GORI_WORKER_RESULT_V2>`), undefined);
|
||||
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V2>{"status":"SUCCESS","summary":"s"}</GORI_WORKER_RESULT_V2>`), undefined);
|
||||
assert.equal(parseWorkerResult(`x\n<GORI_WORKER_RESULT_V1>{"status":"PENDING","summary":"s"}</GORI_WORKER_RESULT_V1>`), undefined);
|
||||
assert.deepEqual(
|
||||
parseAssistantActions(`text\n<GORI_ASSISTANT_ACTION_V2>{"reply":"hi","actions":[{"type":"send_image","path":".gori-outbox/a.png"}]}</GORI_ASSISTANT_ACTION_V2>`),
|
||||
{ reply: "hi", actions: [{ type: "send_image", path: ".gori-outbox/a.png" }] }
|
||||
);
|
||||
assert.equal(parseAssistantActions(`x\n<GORI_ASSISTANT_ACTION_V2>{"reply":"hi","actions":[{"type":"send_image"}]}</GORI_ASSISTANT_ACTION_V2>`), undefined);
|
||||
});
|
||||
|
||||
test("an owner's own pending proposal blocks their start_next with an actionable correction", async () => {
|
||||
@@ -678,6 +689,79 @@ test("image attachments degrade to a text note when the agent has no image capab
|
||||
}
|
||||
});
|
||||
|
||||
test("worker attachments are validated against the workspace and ride along with the owner event", async () => {
|
||||
const harness = await createHarness();
|
||||
try {
|
||||
await harness.manager.prompt(request("create proposal: attach"));
|
||||
const proposal = harness.proposals.list()[0]!;
|
||||
await harness.manager.prompt(request("confirm"));
|
||||
await waitFor(() => harness.proposals.get(proposal.id)!.status === "pending");
|
||||
|
||||
const pending = harness.proposals.get(proposal.id)!.pending!;
|
||||
assert.equal(pending.attachments?.length, 1);
|
||||
const stored = pending.attachments![0]!;
|
||||
assert.equal(stored.mimeType, "image/png");
|
||||
const workspaceRoot = fs.realpathSync(harness.workspace);
|
||||
assert.ok(stored.path.startsWith(`${workspaceRoot}${path.sep}`), stored.path);
|
||||
assert.ok(stored.path.endsWith(path.join(".gori-outbox", "shot.png")), stored.path);
|
||||
assert.equal(pending.droppedAttachments, undefined);
|
||||
|
||||
await waitFor(() => harness.events.length > 0);
|
||||
assert.equal(harness.events[0]!.images?.length, 1);
|
||||
assert.equal(harness.events[0]!.images![0]!.path, stored.path);
|
||||
assert.equal(harness.events[0]!.images![0]!.filename, "shot.png");
|
||||
} finally {
|
||||
await closeHarness(harness);
|
||||
}
|
||||
});
|
||||
|
||||
test("worker attachments outside the workspace or not png/jpg are dropped without failing the result", async () => {
|
||||
const harness = await createHarness();
|
||||
try {
|
||||
await harness.manager.prompt(request("create proposal: attachbad"));
|
||||
const proposal = harness.proposals.list()[0]!;
|
||||
await harness.manager.prompt(request("confirm"));
|
||||
await waitFor(() => harness.proposals.get(proposal.id)!.status === "pending");
|
||||
|
||||
const pending = harness.proposals.get(proposal.id)!.pending!;
|
||||
assert.equal(pending.summary, "made a pic");
|
||||
assert.equal(pending.attachments, undefined);
|
||||
assert.deepEqual(pending.droppedAttachments, ["/tmp/evil.png", "not-an-image.txt"]);
|
||||
|
||||
await waitFor(() => harness.events.length > 0);
|
||||
assert.equal(harness.events[0]!.images, undefined);
|
||||
} finally {
|
||||
await closeHarness(harness);
|
||||
}
|
||||
});
|
||||
|
||||
test("send_image validates the path and attaches the image to the reply", async () => {
|
||||
const harness = await createHarness();
|
||||
try {
|
||||
await harness.manager.prompt(request("create proposal: attach"));
|
||||
const proposal = harness.proposals.list()[0]!;
|
||||
await harness.manager.prompt(request("confirm"));
|
||||
await waitFor(() => harness.proposals.get(proposal.id)!.status === "pending");
|
||||
|
||||
const sent = await harness.manager.prompt(request("send image: .gori-outbox/shot.png"));
|
||||
assert.equal(sent.images?.length, 1);
|
||||
assert.equal(sent.images![0]!.filename, "shot.png");
|
||||
assert.equal(sent.images![0]!.mimeType, "image/png");
|
||||
assert.ok(sent.images![0]!.path.startsWith(fs.realpathSync(harness.workspace)));
|
||||
assert.doesNotMatch(sent.text, /发不出去/);
|
||||
|
||||
const outside = await harness.manager.prompt(request("send image: /tmp/evil.png"));
|
||||
assert.equal(outside.images, undefined);
|
||||
assert.match(outside.text, /发不出去/);
|
||||
|
||||
const notImage = await harness.manager.prompt(request("send image: not-an-image.txt"));
|
||||
assert.equal(notImage.images, undefined);
|
||||
assert.match(notImage.text, /发不出去/);
|
||||
} finally {
|
||||
await closeHarness(harness);
|
||||
}
|
||||
});
|
||||
|
||||
function processGroupExists(pgid: number): boolean {
|
||||
try { process.kill(-pgid, 0); return true; }
|
||||
catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; }
|
||||
|
||||
Reference in New Issue
Block a user