From 18c53e8e83f4e621c6f258d78729d60ea32bc639 Mon Sep 17 00:00:00 2001
From: Pete Miller <miller.pete@gmail.com>
Date: Fri, 20 Jun 2025 11:33:08 -0700
Subject: [PATCH] AI Chat Tool Use Conversation Events (#29643)

A Tool is a feature the LLM can "use" by sending a tool request with some input parameters specified.
This will eventually get added to the events of a conversation history entry (via `ToolUseEvent` added in this PR).
We will then provide a response to the tool in the ToolUseEvent's output field (not implemented in this PR).
The LLM will be called again providing an additional role=Tool conversation history entry (not implemented in this PR).

We put tool use requests and responses in the same object (`ToolUseEvent`) for ease of access and UI rendering. Otherwise there would be a lot of cases of searching history to find a relevant tool use response for a given tool use request if we kept them in assistant and user (or tool) `ConversationTurn` items.

In order to keep this submission size as small as possible, it only includes what's necessary to define the mojom structs, so that other PRs that use them can be created in parallel.

Subsequent submissions will be:
- Tool base class
- Conversation API support for sending defined tools, tool use requests, and tool use responses
- ConversationHandler getting an agentic loop to perform the usage of the Tools and send the responses back to the engine
- UI representation for tool uses
- Tool implementations
---
 .../ai_chat/core/common/mojom/ai_chat.mojom   | 37 +++++++++++++++++++
 .../page/stories/components_panel.tsx         |  3 +-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/components/ai_chat/core/common/mojom/ai_chat.mojom b/components/ai_chat/core/common/mojom/ai_chat.mojom
index 680e6e6e3c0..e91bf80d8b9 100644
--- a/components/ai_chat/core/common/mojom/ai_chat.mojom
+++ b/components/ai_chat/core/common/mojom/ai_chat.mojom
@@ -8,6 +8,7 @@ module ai_chat.mojom;
 import "brave/components/ai_chat/core/common/mojom/untrusted_frame.mojom";
 import "brave/components/ai_chat/core/common/mojom/tab_tracker.mojom";
 import "mojo/public/mojom/base/time.mojom";
+import "mojo/public/mojom/base/values.mojom";
 import "url/mojom/url.mojom";
 
 enum CharacterType {
@@ -170,6 +171,8 @@ struct WebSourcesEvent {
 };
 
 struct CompletionEvent {
+  // TODO(petemill): Use ContentBlock or even array<ContentBlock>
+  // and rely less on multiple events.
   string completion;
 };
 
@@ -198,12 +201,46 @@ struct UploadedFile {
   UploadedFileType type;
 };
 
+struct ImageContentBlock {
+  // Only base64 encoded data uri is supported
+  url.mojom.Url image_url;
+};
+
+struct TextContentBlock {
+  string text;
+};
+
+union ContentBlock {
+  ImageContentBlock image_content_block;
+  TextContentBlock text_content_block;
+};
+
+struct ToolUseEvent {
+  // Name of the tool, used to identify which
+  // tool to run in order to produce the output
+  // from the given arguments.
+  string tool_name;
+
+  // ID of the tool use request
+  string id;
+
+  // Arguments for the Tool. The LLM may hallucinate and generate
+  // incorrect arguments or even invalid JSON, so this should be
+  // interpreted defensively.
+  mojo_base.mojom.DictionaryValue arguments;
+
+  // Output from the Tool. When null, the Tool
+  // has not yet run or completed.
+  array<ContentBlock>? output;
+};
+
 // Events that occur during a conversation turn (only assistant for now)
 union ConversationEntryEvent {
   CompletionEvent completion_event;
   SearchQueriesEvent search_queries_event;
   SearchStatusEvent search_status_event;
   WebSourcesEvent sources_event;
+  ToolUseEvent tool_use_event;
 
   // These events don't normally get added to the conversation entry
   // but are used in engine responses.
diff --git a/components/ai_chat/resources/page/stories/components_panel.tsx b/components/ai_chat/resources/page/stories/components_panel.tsx
index 9ebe4f355ec..b278c57e544 100644
--- a/components/ai_chat/resources/page/stories/components_panel.tsx
+++ b/components/ai_chat/resources/page/stories/components_panel.tsx
@@ -42,7 +42,8 @@ const eventTemplate: Mojom.ConversationEntryEvent = {
   selectedLanguageEvent: undefined,
   conversationTitleEvent: undefined,
   sourcesEvent: undefined,
-  contentReceiptEvent: undefined
+  contentReceiptEvent: undefined,
+  toolUseEvent: undefined,
 }
 
 function getCompletionEvent(text: string): Mojom.ConversationEntryEvent {