AI Chat Tool Use Conversation Events (#29643)

A Tool is a feature the LLM can "use" by sending a tool request with some input parameters specified.
This will eventually get added to the events of a conversation history entry (via `ToolUseEvent` added in this PR).
We will then provide a response to the tool in the ToolUseEvent's output field (not implemented in this PR).
The LLM will be called again providing an additional role=Tool conversation history entry (not implemented in this PR).

We put tool use requests and responses in the same object (`ToolUseEvent`) for ease of access and UI rendering. Otherwise there would be a lot of cases of searching history to find a relevant tool use response for a given tool use request if we kept them in assistant and user (or tool) `ConversationTurn` items.

In order to keep this submission size as small as possible, it only includes what's necessary to define the mojom structs, so that other PRs that use them can be created in parallel.

Subsequent submissions will be:
- Tool base class
- Conversation API support for sending defined tools, tool use requests, and tool use responses
- ConversationHandler getting an agentic loop to perform the usage of the Tools and send the responses back to the engine
- UI representation for tool uses
- Tool implementations
This commit is contained in:
Pete Miller
2025-06-20 11:33:08 -07:00
committed by GitHub
parent c394cf6f46
commit 18c53e8e83
2 changed files with 39 additions and 1 deletions
@@ -8,6 +8,7 @@ module ai_chat.mojom;
import "brave/components/ai_chat/core/common/mojom/untrusted_frame.mojom";
import "brave/components/ai_chat/core/common/mojom/tab_tracker.mojom";
import "mojo/public/mojom/base/time.mojom";
import "mojo/public/mojom/base/values.mojom";
import "url/mojom/url.mojom";
enum CharacterType {
@@ -170,6 +171,8 @@ struct WebSourcesEvent {
};
struct CompletionEvent {
// TODO(petemill): Use ContentBlock or even array<ContentBlock>
// and rely less on multiple events.
string completion;
};
@@ -198,12 +201,46 @@ struct UploadedFile {
UploadedFileType type;
};
struct ImageContentBlock {
// Only base64 encoded data uri is supported
url.mojom.Url image_url;
};
struct TextContentBlock {
string text;
};
union ContentBlock {
ImageContentBlock image_content_block;
TextContentBlock text_content_block;
};
struct ToolUseEvent {
// Name of the tool, used to identify which
// tool to run in order to produce the output
// from the given arguments.
string tool_name;
// ID of the tool use request
string id;
// Arguments for the Tool. The LLM may hallucinate and generate
// incorrect arguments or even invalid JSON, so this should be
// interpreted defensively.
mojo_base.mojom.DictionaryValue arguments;
// Output from the Tool. When null, the Tool
// has not yet run or completed.
array<ContentBlock>? output;
};
// Events that occur during a conversation turn (only assistant for now)
union ConversationEntryEvent {
CompletionEvent completion_event;
SearchQueriesEvent search_queries_event;
SearchStatusEvent search_status_event;
WebSourcesEvent sources_event;
ToolUseEvent tool_use_event;
// These events don't normally get added to the conversation entry
// but are used in engine responses.
@@ -42,7 +42,8 @@ const eventTemplate: Mojom.ConversationEntryEvent = {
selectedLanguageEvent: undefined,
conversationTitleEvent: undefined,
sourcesEvent: undefined,
contentReceiptEvent: undefined
contentReceiptEvent: undefined,
toolUseEvent: undefined,
}
function getCompletionEvent(text: string): Mojom.ConversationEntryEvent {