add complete sequence tool calling with no hard-coded responses

.claude/settings.local.json

@@ -3,7 +3,8 @@
     "allow": [
       "WebSearch",
       "Bash(npm run typecheck:*)",
-      "Bash(npm run lint:*)"
+      "Bash(npm run lint:*)",
+      "WebFetch(domain:ai.google.dev)"
     ]
   }
 }

src/components/AgenticInterface.tsx

@@ -16,7 +16,8 @@ const INITIAL_MESSAGES: Message[] = [
 
 export function AgenticInterface() {
   const { squareColor, setSquareColor } = useColorStore();
-  const { processMessage, isProcessing, loadingStatus } = useFunctionCalling();
+  const { processMessage, continueWithToolResult, isProcessing, loadingStatus } =
+    useFunctionCalling();
   const [messages, setMessages] = useState<Message[]>(INITIAL_MESSAGES);
 
   const handleSendMessage = async (text: string) => {
@@ -31,13 +32,20 @@ export function AgenticInterface() {
         const color = result.functionCall.args.color as string | undefined;
         if (color) {
           setSquareColor(color);
-          botResponse = `Done! I've changed the square color to ${color}.`;
+          // Get response from model after executing the function
+          botResponse = await continueWithToolResult("set_square_color", {
+            success: true,
+            color,
+          });
         } else {
           botResponse =
             "I understood you want to change the color, but I couldn't determine which color. Please try again with a specific color.";
         }
       } else if (result.functionCall?.functionName === "getSquareColor") {
-        botResponse = `The current color of the square is ${squareColor}.`;
+        // Get response from model after executing the function
+        botResponse = await continueWithToolResult("get_square_color", {
+          color: squareColor,
+        });
       } else if (result.textResponse) {
         botResponse = result.textResponse;
       } else {

src/hooks/useFunctionCalling.ts

@@ -138,5 +138,20 @@ export function useFunctionCalling() {
     [initModel],
   );
 
-  return { processMessage, isProcessing, loadingStatus };
+  const continueWithToolResult = useCallback(
+    async (functionName: string, functionResult: unknown): Promise<string> => {
+      const api = apiRef.current;
+      if (!api) return "Error: Worker not available";
+
+      setIsProcessing(true);
+      try {
+        return await api.continueWithToolResult(functionName, functionResult);
+      } finally {
+        setIsProcessing(false);
+      }
+    },
+    [],
+  );
+
+  return { processMessage, continueWithToolResult, isProcessing, loadingStatus };
 }

src/worker.ts

@@ -5,6 +5,7 @@ import {
   type PreTrainedTokenizer,
 } from "@huggingface/transformers";
 import * as Comlink from "comlink";
+import { createStore } from "zustand/vanilla";
 
 const MODEL_ID = "onnx-community/functiongemma-270m-it-ONNX";
 
@@ -23,6 +24,42 @@ export type LoadingProgress = {
 
 type ProgressCallback = (progress: LoadingProgress) => void;
 
+// Types for FunctionGemma multi-turn conversation
+type ToolCall = {
+  type: "function";
+  function: {
+    name: string;
+    arguments: Record<string, unknown>;
+  };
+};
+
+type ChatMessage =
+  | { role: "developer" | "user"; content: string }
+  | { role: "assistant"; content?: string; tool_calls?: ToolCall[] }
+  | { role: "tool"; content: Array<{ name: string; response: unknown }> };
+
+export type FunctionCallResult = {
+  functionName: string;
+  args: Record<string, unknown>;
+} | null;
+
+type ConversationState = {
+  messages: ChatMessage[];
+  lastFunctionCall: FunctionCallResult;
+  setMessages: (messages: ChatMessage[]) => void;
+  setLastFunctionCall: (fc: FunctionCallResult) => void;
+  clear: () => void;
+};
+
+// Store to maintain conversation state between processMessage and continueWithToolResult
+const conversationStore = createStore<ConversationState>((set) => ({
+  messages: [],
+  lastFunctionCall: null,
+  setMessages: (messages) => set({ messages }),
+  setLastFunctionCall: (fc) => set({ lastFunctionCall: fc }),
+  clear: () => set({ messages: [], lastFunctionCall: null }),
+}));
+
 function getModel(onProgress?: ProgressCallback) {
   modelPromise ??= (async () => {
     console.log("[Worker] Loading model...");
@@ -79,11 +116,6 @@ const tools = [
 const developerPrompt =
   "You are a model that can do function calling with the following functions";
 
-export type FunctionCallResult = {
-  functionName: string;
-  args: Record<string, unknown>;
-} | null;
-
 // Parses functiongemma output: <start_function_call>call:func_name{param:<escape>value<escape>}<end_function_call>
 function parseFunctionCall(output: string): FunctionCallResult {
   const match = output.match(
@@ -186,12 +218,21 @@ const workerAPI = {
     const functionCall = parseFunctionCall(decoded as string);
     console.log("[Worker] Parsed function call:", functionCall);
 
-    // If function was called, don't return text response (it's usually garbage)
+    // If function was called, save conversation state for continueWithToolResult
     if (functionCall) {
       console.log(
         "[Worker] ✅ Function call successful:",
         functionCall.functionName,
       );
+
+      // Save conversation state for the next turn
+      const { setMessages, setLastFunctionCall } = conversationStore.getState();
+      setMessages([
+        { role: "developer", content: developerPrompt },
+        { role: "user", content: text },
+      ]);
+      setLastFunctionCall(functionCall);
+
       return { functionCall };
     }
 
@@ -216,6 +257,119 @@ const workerAPI = {
 
     return { functionCall: null, textResponse: textResponse || undefined };
   },
+
+  async continueWithToolResult(
+    functionName: string,
+    functionResult: unknown,
+  ): Promise<string> {
+    console.log("[Worker] ========== CONTINUE WITH TOOL RESULT ==========");
+    console.log("[Worker] Function:", functionName);
+    console.log("[Worker] Result:", functionResult);
+
+    const { messages, lastFunctionCall, clear } = conversationStore.getState();
+
+    if (messages.length === 0 || !lastFunctionCall) {
+      console.log("[Worker] ⚠️ No conversation state found");
+      return "I apologize, but I lost track of our conversation. Could you please try again?";
+    }
+
+    const { model, tokenizer } = await getModel();
+
+    // Build the full conversation with tool result
+    // Convert camelCase back to snake_case for the model
+    const snakeCaseName =
+      lastFunctionCall.functionName === "setSquareColor"
+        ? "set_square_color"
+        : lastFunctionCall.functionName === "getSquareColor"
+          ? "get_square_color"
+          : functionName;
+
+    // Build messages in the format expected by the tokenizer
+    // Note: Using 'as unknown' to bypass strict typing since the tokenizer
+    // accepts a more flexible format than what TypeScript infers
+    const fullMessages = [
+      ...messages,
+      // Assistant turn with tool_calls
+      {
+        role: "assistant",
+        tool_calls: [
+          {
+            type: "function",
+            function: {
+              name: snakeCaseName,
+              arguments: lastFunctionCall.args,
+            },
+          },
+        ],
+      },
+      // Tool response
+      {
+        role: "tool",
+        content: [{ name: snakeCaseName, response: functionResult }],
+      },
+    ] as unknown[];
+
+    console.log(
+      "[Worker] Full messages for final response:",
+      JSON.stringify(fullMessages, null, 2),
+    );
+
+    // biome-ignore lint/suspicious/noExplicitAny: tokenizer accepts flexible message format
+    const inputs = tokenizer.apply_chat_template(fullMessages as any, {
+      tools,
+      tokenize: true,
+      add_generation_prompt: true,
+      return_dict: true,
+    }) as { input_ids: { dims: number[]; data: unknown } };
+
+    console.log("[Worker] Input token count:", inputs.input_ids.dims[1]);
+
+    try {
+      const inputIds = Array.from(inputs.input_ids.data as Iterable<number>);
+      const decodedInput = tokenizer.decode(inputIds, {
+        skip_special_tokens: false,
+      });
+      console.log(
+        "[Worker] Decoded input for final response:",
+        decodedInput,
+      );
+    } catch (e) {
+      console.log("[Worker] Could not decode input:", e);
+    }
+
+    // biome-ignore lint/complexity/noBannedTypes: model.generate type is not properly typed in transformers.js
+    const output = await (model.generate as Function)({
+      ...inputs,
+      max_new_tokens: 256,
+    });
+
+    const inputLength = inputs.input_ids.dims[1];
+    const decoded = tokenizer.decode(output.slice(0, [inputLength, null]), {
+      skip_special_tokens: false,
+    });
+
+    console.log("[Worker] Raw final response:", decoded);
+
+    // Clear conversation state
+    clear();
+
+    // Clean up the response
+    const cleanResponse = (decoded as string)
+      .replace(/<start_function_call>[\s\S]*?<end_function_call>/g, "")
+      .replace(/<start_function_response>[\s\S]*?<end_function_response>/g, "")
+      .replace(/<\|im_end\|>/g, "")
+      .replace(/<\|im_start\|>/g, "")
+      .replace(/<\|endoftext\|>/g, "")
+      .replace(/<end_of_turn>/g, "")
+      .replace(/<start_of_turn>/g, "")
+      .replace(/^model\s*/i, "")
+      .replace(/^assistant\s*/i, "")
+      .trim();
+
+    console.log("[Worker] ✅ Final response:", cleanResponse);
+
+    return cleanResponse || "Done!";
+  },
 };
 
 export type WorkerAPI = typeof workerAPI;