WFDASH-WEBAPP/src/lib/server/giellie-agent.test.ts

import { describe, it, expect, vi } from "vitest";
import {
  runAgent,
  resolvePending,
  toModelMessages,
  type AgentDeps,
  type ModelClient,
  type ModelStream,
} from "./giellie-agent";
import type { Block, StoredMessage, PendingAction } from "@/lib/giellie-types";
import type { GielieEvent } from "@/lib/giellie-sse";

const tool = (name: string) => ({ name, description: "", input_schema: { type: "object", properties: {} } });
const TOOLS = [
  tool("orders__list"),
  tool("orders__update"),
  tool("quotes__send_email"),
  tool("quotes__update"),
  tool("orders__cancel"),
  tool("companies__delete"),
];

function fakeClient(turns: Block[][]): ModelClient & { calls: Array<Record<string, unknown>> } {
  let i = 0;
  const calls: Array<Record<string, unknown>> = [];
  return {
    calls,
    messages: {
      stream: (params) => {
        calls.push(params);
        const content = turns[i++];
        if (!content) throw new Error("model called more times than scripted");
        const s: ModelStream = {
          on(ev, cb) {
            if (ev === "text") for (const b of content) if (b.type === "text") cb(String(b.text));
            return s;
          },
          finalMessage: async () => ({ content }),
        };
        return s;
      },
    },
  };
}

const text = (t: string): Block => ({ type: "text", text: t });
const use = (id: string, name: string, input: Record<string, unknown> = {}): Block => ({
  type: "tool_use",
  id,
  name,
  input,
});

function setup(turns: Block[][], over: Partial<AgentDeps> = {}) {
  const client = fakeClient(turns);
  const events: GielieEvent[] = [];
  const callTool = vi.fn(async () => ({ text: '{"ok":true}', isError: false }));
  const order: string[] = [];
  const snaps: Array<{ messages: StoredMessage[]; pending: PendingAction | null }> = [];
  const checkpoint = vi.fn(async (m: StoredMessage[], p: PendingAction | null) => {
    order.push("checkpoint");
    snaps.push({ messages: structuredClone(m), pending: p ? structuredClone(p) : null });
  });
  const deps: AgentDeps = {
    client,
    model: "m",
    maxTokens: 100,
    system: "sys",
    tools: TOOLS,
    callTool,
    emit: (e) => {
      order.push(`emit:${e.type}`);
      events.push(e);
    },
    checkpoint,
    trust: false,
    ...over,
  };
  const messages: StoredMessage[] = [
    { role: "user", content: [text("hi")], ts: "t" },
  ];
  return { client, events, callTool, checkpoint, deps, messages, snaps, order };
}

describe("runAgent", () => {
  it("answers directly and streams text", async () => {
    const s = setup([[text("hello")]]);
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("done");
    expect(s.events).toContainEqual({ type: "text", delta: "hello" });
    expect(s.messages.at(-1)?.role).toBe("assistant");
    expect(s.checkpoint).toHaveBeenCalledTimes(1);
  });

  it("runs read tools immediately and loops", async () => {
    const s = setup([[use("t1", "orders__list", { perPage: 5 })], [text("done")]]);
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("done");
    expect(s.callTool).toHaveBeenCalledWith("orders.list", { perPage: 5 });
    expect(s.client.calls).toHaveLength(2);
    const userResult = s.messages.find(
      (m) => m.role === "user" && m.content[0].type === "tool_result",
    );
    expect(userResult?.content[0]).toMatchObject({ tool_use_id: "t1", content: '{"ok":true}' });
    expect(s.events.map((e) => e.type)).toEqual(expect.arrayContaining(["tool_start", "tool_result"]));
  });

  it("pauses for approval on writes without executing them", async () => {
    const s = setup([[text("ok"), use("t1", "orders__update", { id: "a" })]]);
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("approval_required");
    expect(s.callTool).not.toHaveBeenCalled();
    const ev = s.events.find((e) => e.type === "approval_required") as Extract<GielieEvent, { type: "approval_required" }>;
    expect(ev.pending.tool_uses).toEqual([
      { id: "t1", name: "orders.update", input: { id: "a" }, confirmAlways: false },
    ]);
    expect(ev.pending.order).toEqual(["t1"]);
    const last = s.snaps.at(-1)!;
    expect(last.messages).toEqual(s.messages);
    expect(last.pending).toEqual(ev.pending);
  });

  it("persists the pending action before showing the approval card", async () => {
    const s = setup([[use("t1", "orders__update")]]);
    await runAgent(s.messages, s.deps);
    expect(s.order.indexOf("checkpoint")).toBeGreaterThanOrEqual(0);
    expect(s.order.indexOf("checkpoint")).toBeLessThan(s.order.indexOf("emit:approval_required"));
  });

  it("keeps read results in pending.completed when a write pauses the turn", async () => {
    const s = setup([[use("r1", "orders__list"), use("w1", "orders__update")]]);
    await runAgent(s.messages, s.deps);
    const ev = s.events.find((e) => e.type === "approval_required") as Extract<GielieEvent, { type: "approval_required" }>;
    expect(ev.pending.completed).toHaveLength(1);
    expect(ev.pending.completed[0]).toMatchObject({ tool_use_id: "r1" });
    expect(ev.pending.order).toEqual(["r1", "w1"]);
  });

  it("auto-runs ordinary writes when the chat is trusted", async () => {
    const s = setup([[use("t1", "orders__update", { id: "a" })], [text("done")]], { trust: true });
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("done");
    expect(s.callTool).toHaveBeenCalledWith("orders.update", { id: "a" });
  });

  it("still pauses for email sends when trusted", async () => {
    const s = setup([[use("t1", "quotes__send_email", { id: "q" })]], { trust: true });
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("approval_required");
    expect(s.callTool).not.toHaveBeenCalled();
    const ev = s.events.find((e) => e.type === "approval_required") as Extract<GielieEvent, { type: "approval_required" }>;
    expect(ev.pending.tool_uses[0].confirmAlways).toBe(true);
    expect(ev.pending.tool_uses[0].confirmReason).toBe("Sends an email");
  });

  it("pauses a trusted chat's quotes.update that touches email_bcc, with a reason", async () => {
    const s = setup([[use("t1", "quotes__update", { id: "q", patch: { email_bcc: "x@y.com" } })]], {
      trust: true,
    });
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("approval_required");
    expect(s.callTool).not.toHaveBeenCalled();
    const ev = s.events.find((e) => e.type === "approval_required") as Extract<GielieEvent, { type: "approval_required" }>;
    expect(ev.pending.tool_uses[0]).toMatchObject({
      name: "quotes.update",
      confirmAlways: true,
      confirmReason: "Changes email/contact details",
    });
  });

  it("still auto-runs a trusted quotes.update without contact fields", async () => {
    const s = setup([[use("t1", "quotes__update", { id: "q", patch: { notes: "n" } })], [text("done")]], {
      trust: true,
    });
    expect(await runAgent(s.messages, s.deps)).toBe("done");
    expect(s.callTool).toHaveBeenCalledTimes(1);
  });

  it("pauses orders.cancel even when the chat is trusted", async () => {
    const s = setup([[use("t1", "orders__cancel", { id: "o" })]], { trust: true });
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("approval_required");
    expect(s.callTool).not.toHaveBeenCalled();
    const ev = s.events.find((e) => e.type === "approval_required") as Extract<GielieEvent, { type: "approval_required" }>;
    expect(ev.pending.tool_uses[0]).toMatchObject({
      confirmAlways: true,
      confirmReason: "Cancels or merges records — always asks",
    });
  });

  it("refuses denied tools without calling them", async () => {
    const s = setup([[use("t1", "companies__delete", { id: "c" })], [text("ok")]], { trust: true });
    await runAgent(s.messages, s.deps);
    expect(s.callTool).not.toHaveBeenCalled();
    const res = s.messages.find((m) => m.content[0].type === "tool_result");
    expect(res?.content[0]).toMatchObject({ is_error: true });
  });

  it("errors on tools that are not in the catalogue", async () => {
    const s = setup([[use("t1", "made_up__tool")], [text("ok")]]);
    await runAgent(s.messages, s.deps);
    expect(s.callTool).not.toHaveBeenCalled();
    const res = s.messages.find((m) => m.content[0].type === "tool_result");
    expect(res?.content[0]).toMatchObject({ is_error: true });
  });

  it("turns a throwing tool into an error result", async () => {
    const s = setup([[use("t1", "orders__list")], [text("ok")]]);
    s.callTool.mockRejectedValueOnce(new Error("boom"));
    await runAgent(s.messages, s.deps);
    const res = s.messages.find((m) => m.content[0].type === "tool_result");
    expect(res?.content[0]).toMatchObject({ is_error: true });
    expect(String((res?.content[0] as unknown as { content: string }).content)).toContain("boom");
  });

  it("stops at maxTurns with an assistant note so roles keep alternating", async () => {
    const s = setup([[use("t1", "orders__list")], [use("t2", "orders__list")]], { maxTurns: 2 });
    expect(await runAgent(s.messages, s.deps)).toBe("max_turns");
    const last = s.messages.at(-1)!;
    const note = 'Stopped after 2 steps without finishing. Say "continue" and I\'ll pick up where I left off.';
    expect(last.role).toBe("assistant");
    expect(last.content).toEqual([{ type: "text", text: note }]);
    expect(s.events).toContainEqual({ type: "text", delta: note });
    expect(s.snaps.at(-1)!.messages.at(-1)!.role).toBe("assistant");
    expect(s.snaps.at(-1)!.pending).toBeNull();
  });

  it("checkpoints only consistent states when the model call fails", async () => {
    const s = setup([[use("t1", "orders__list")]]);
    // Second model call is unscripted -> throws.
    await expect(runAgent(s.messages, s.deps)).rejects.toThrow();
    expect(s.snaps.length).toBeGreaterThanOrEqual(1);
    for (const snap of s.snaps) {
      const last = snap.messages.at(-1)!;
      const danglingToolUse = last.role === "assistant" && last.content.some((b) => b.type === "tool_use");
      // Only an approval pause may persist a tool_use without its result.
      if (danglingToolUse) expect(snap.pending).not.toBeNull();
    }
    expect(s.snaps.some((snap) => snap.messages.at(-1)!.role === "user" && snap.messages.length === 3)).toBe(true);
    // Nothing was ever checkpointed with an unanswered tool_use and no pending.
    expect(
      s.snaps.some((snap) => {
        const last = snap.messages.at(-1)!;
        return snap.pending === null && last.role === "assistant" && last.content.some((b) => b.type === "tool_use");
      }),
    ).toBe(false);
  });

  it("executes nothing when the response was cut off by max_tokens", async () => {
    const s = setup([[use("t1", "orders__list"), use("w1", "orders__update")], [text("ok")]], {
      maxTurns: 5,
    });
    // Script stop_reason on the first response.
    const orig = s.deps.client.messages.stream;
    let n = 0;
    s.deps.client.messages.stream = (params) => {
      const st = orig(params);
      if (n++ === 0) {
        return { on: (ev, cb) => (st.on(ev, cb), st), finalMessage: async () => ({ ...(await st.finalMessage()), stop_reason: "max_tokens" }) };
      }
      return st;
    };
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("done");
    expect(s.callTool).not.toHaveBeenCalled();
    expect(s.events.some((e) => e.type === "approval_required")).toBe(false);
    const res = s.messages.find((m) => m.role === "user" && m.content[0].type === "tool_result")!;
    expect(res.content.map((b) => b.tool_use_id)).toEqual(["t1", "w1"]);
    for (const b of res.content) {
      expect(b).toMatchObject({ is_error: true });
      expect(String(b.content)).toContain("cut off");
    }
  });

  it.each([
    ["only an empty text block", [text("")]],
    ["only whitespace text blocks", [text("  "), text("\n\n")]],
  ])("replaces a response with %s by the placeholder", async (_l, blocks) => {
    const s = setup([blocks]);
    expect(await runAgent(s.messages, s.deps)).toBe("done");
    expect(s.messages.at(-1)!.content).toEqual([{ type: "text", text: "(no response)" }]);
  });

  it("keeps a response whose text is blank but which has another block type", async () => {
    const thinking: Block = { type: "thinking", thinking: "hm", signature: "s" };
    const s = setup([[thinking, text(" ")]]);
    await runAgent(s.messages, s.deps);
    expect(s.messages.at(-1)!.content).toEqual([thinking, text(" ")]);
  });

  it("replaces an empty model response with a placeholder", async () => {
    const s = setup([[]]);
    const out = await runAgent(s.messages, s.deps);
    expect(out).toBe("done");
    expect(s.messages.at(-1)!.content).toEqual([{ type: "text", text: "(no response)" }]);
  });
});

const UNAVAILABLE =
  "Result unavailable — this action may or may not have executed. Verify the current state (read the record) before retrying.";

describe("resolvePending", () => {
  const pending: PendingAction = {
    tool_uses: [{ id: "w1", name: "orders.update", input: { id: "a" }, confirmAlways: false }],
    completed: [{ type: "tool_result", tool_use_id: "r1", content: "read-result" }],
    order: ["r1", "w1"],
  };

  it("executes approved tools and merges results in original order", async () => {
    const s = setup([]);
    await resolvePending(s.messages, pending, true, s.deps);
    expect(s.callTool).toHaveBeenCalledWith("orders.update", { id: "a" });
    const last = s.messages.at(-1)!;
    expect(last.role).toBe("user");
    expect(last.content.map((b) => b.tool_use_id)).toEqual(["r1", "w1"]);
  });

  it("checkpoints the resolved state once with pending cleared", async () => {
    const s = setup([]);
    await resolvePending(s.messages, pending, true, s.deps);
    expect(s.checkpoint).toHaveBeenCalledTimes(1);
    const snap = s.snaps[0];
    expect(snap.pending).toBeNull();
    const last = snap.messages.at(-1)!;
    expect(last.role).toBe("user");
    expect(last.content.map((b) => b.tool_use_id)).toEqual(["r1", "w1"]);
  });

  it("never executes denied or uncatalogued tools from a tampered pending action", async () => {
    const s = setup([]);
    const tampered: PendingAction = {
      tool_uses: [
        { id: "d1", name: "companies.delete", input: { id: "c" }, confirmAlways: false },
        { id: "u1", name: "made_up.tool", input: {}, confirmAlways: false },
      ],
      completed: [],
      order: ["d1", "u1"],
    };
    await resolvePending(s.messages, tampered, true, s.deps);
    expect(s.callTool).not.toHaveBeenCalled();
    const last = s.messages.at(-1)!;
    expect(last.content.map((b) => b.tool_use_id)).toEqual(["d1", "u1"]);
    for (const b of last.content) expect(b).toMatchObject({ is_error: true });
  });

  it("merges results in pending.order even when it differs from tool_use order", async () => {
    const s = setup([]);
    const p: PendingAction = {
      tool_uses: [{ id: "w1", name: "orders.update", input: {}, confirmAlways: false }],
      completed: [{ type: "tool_result", tool_use_id: "r1", content: "x" }],
      order: ["w1", "r1"],
    };
    await resolvePending(s.messages, p, true, s.deps);
    expect(s.messages.at(-1)!.content.map((b) => b.tool_use_id)).toEqual(["w1", "r1"]);
  });

  it("synthesises an error result for an order id with no result", async () => {
    const s = setup([]);
    const p: PendingAction = { ...pending, order: ["r1", "w1", "ghost"] };
    await resolvePending(s.messages, p, true, s.deps);
    const last = s.messages.at(-1)!;
    expect(last.content.map((b) => b.tool_use_id)).toEqual(["r1", "w1", "ghost"]);
    expect(last.content[2]).toMatchObject({
      is_error: true,
      content: UNAVAILABLE,
    });
  });

  it("answers a duplicated id in pending.order only once", async () => {
    const s = setup([]);
    const p: PendingAction = { ...pending, order: ["r1", "w1", "w1", "r1"] };
    await resolvePending(s.messages, p, true, s.deps);
    expect(s.messages.at(-1)!.content.map((b) => b.tool_use_id)).toEqual(["r1", "w1"]);
    expect(s.callTool).toHaveBeenCalledTimes(1);
  });

  it("does not execute a tool_use whose id is not in pending.order", async () => {
    const s = setup([]);
    const p: PendingAction = {
      tool_uses: [
        { id: "w1", name: "orders.update", input: { id: "a" }, confirmAlways: false },
        { id: "x1", name: "orders.update", input: { id: "evil" }, confirmAlways: false },
      ],
      completed: [],
      order: ["w1"],
    };
    await resolvePending(s.messages, p, true, s.deps);
    expect(s.callTool).toHaveBeenCalledTimes(1);
    expect(s.callTool).toHaveBeenCalledWith("orders.update", { id: "a" });
    expect(s.messages.at(-1)!.content.map((b) => b.tool_use_id)).toEqual(["w1"]);
  });

  it("returns a rejection result without executing", async () => {
    const s = setup([]);
    await resolvePending(s.messages, pending, false, s.deps);
    expect(s.callTool).not.toHaveBeenCalled();
    const w = s.messages.at(-1)!.content[1];
    expect(w).toMatchObject({ tool_use_id: "w1", is_error: true });
    expect(String(w.content)).toContain("rejected");
  });
});

describe("toModelMessages", () => {
  it("maps attachment chips to text and drops ts", () => {
    const out = toModelMessages([
      { role: "user", ts: "t", content: [{ type: "attachment", filename: "a.jpg", kind: "image" }, text("x")] },
    ]);
    expect(out[0]).toEqual({
      role: "user",
      content: [{ type: "text", text: "[attached image: a.jpg]" }, text("x")],
    });
  });
  it("substitutes live blocks at the live index (also mapping chips)", () => {
    const msgs: StoredMessage[] = [
      { role: "user", ts: "t", content: [text("stored")] },
    ];
    const img: Block = { type: "image", source: { type: "base64", media_type: "image/png", data: "AA" } };
    const out = toModelMessages(msgs, {
      index: 0,
      blocks: [img, { type: "attachment", filename: "p.png", kind: "image" }, text("live")],
    });
    expect(out[0].content).toEqual([img, { type: "text", text: "[attached image: p.png]" }, text("live")]);
  });
});