[AI Chat] [BYOM] Add tool support for custom models in normal profiles (#34456)

* [AI Chat] [BYOM] Add supports_tools toggle to custom model settings UI

Expose the supports_tools flag in the custom model configuration UI
and persist it to prefs, allowing users to enable tool support
(e.g., web search) for custom models that support function calling.

- Add kCustomModelSupportsTools pref key and persist in GetModelDict/GetCustomModels
- Add supportsTools property, handler, and toggle in model_config_ui
- Add localized strings for the tool support toggle and tooltip

* [AI Chat] [BYOM] Restrict BYOM tool support to normal profiles

Add array<ConversationCapability> supported_capabilities model property
to filter which models are available for different conversation modes
(chat, content agent, etc.). Automatic (depends on tool support flag),
Claude Haiku and Sonnet have {CHAT, CONTENT_AGENT}, all other models
(including BYOM) have {CHAT} only.
This commit is contained in:
Jocelyn Liu
2026-03-06 02:13:24 +00:00
committed by GitHub
parent dcb5f18f25
commit bba0efea11
21 changed files with 151 additions and 16 deletions
+6
View File
@@ -1683,6 +1683,12 @@
<message name="IDS_SETTINGS_LEO_ASSISTANT_INPUT_MODEL_VISION_SUPPORT_TOOLTIP_INFO" desc="A brief description for model vision support for adding a custom model.">
If this model takes image as input, e.g. llama 3.2b vision
</message>
<message name="IDS_SETTINGS_LEO_ASSISTANT_MODEL_TOOL_SUPPORT_TOGGLE" desc="An label for model tool support toggle.">
Tool Support
</message>
<message name="IDS_SETTINGS_LEO_ASSISTANT_INPUT_MODEL_TOOL_SUPPORT_TOOLTIP_INFO" desc="A brief description for model tool support for adding a custom model.">
If this model supports function calling / tool use
</message>
<message name="IDS_SETTINGS_LEO_ASSISTANT_INPUT_MODEL_LABEL_TOOLTIP_INFO" desc="A brief description for the input title for adding a custom model">
The name of the model as it will appear in the model selection menu
</message>
@@ -316,6 +316,32 @@
on-change="onVisionSupportChanged_">
</leo-toggle>
</div>
<div class="input-container">
<div class="custom-label">
<div class="label-text">
<span>$i18n{braveLeoAssistantInputModelToolSupport}</span>
<leo-tooltip mode="default">
<div
slot="content"
class="tooltip-content"
>
$i18n{braveLeoAssistantInputModelToolSupportTooltipInfo}
</div>
<leo-button
size="small"
kind="plain-faint"
fab
>
<leo-icon name="info-outline"></leo-icon>
</leo-button>
</leo-tooltip>
</div>
</div>
<leo-toggle
checked="{{supportsTools}}"
on-change="onSupportsToolsChanged_">
</leo-toggle>
</div>
<div class="actions-container">
<leo-button
@@ -90,6 +90,9 @@ export class ModelConfigUI extends ModelConfigUIBase {
},
hasVisionSupport : {
type: Boolean
},
supportsTools: {
type: Boolean
}
}
}
@@ -111,6 +114,7 @@ export class ModelConfigUI extends ModelConfigUIBase {
declare shouldShowUnsafeEndpointModal: boolean
declare invalidUrlErrorMessage: string
declare hasVisionSupport: boolean
declare supportsTools: boolean
override ready() {
super.ready()
@@ -158,7 +162,8 @@ export class ModelConfigUI extends ModelConfigUIBase {
key: modelKey,
displayName: this.label,
visionSupport: this.hasVisionSupport,
supportsTools: false,
supportsTools: this.supportsTools,
supportedCapabilities: [mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
};
@@ -218,6 +223,10 @@ export class ModelConfigUI extends ModelConfigUIBase {
this.hasVisionSupport = e.checked
}
onSupportsToolsChanged_(e: any) {
this.supportsTools = e.checked
}
private saveEnabled_() {
// Make sure all required fields are filled
return this.label && this.modelRequestName && this.endpointUrl && !this.isUrlInvalid
@@ -250,6 +259,7 @@ export class ModelConfigUI extends ModelConfigUIBase {
this.modelSystemPrompt =
newValue.options.customModelOptions.modelSystemPrompt;
this.hasVisionSupport = newValue.visionSupport;
this.supportsTools = newValue.supportsTools
}
this.constructTokenEstimateString_()
}
@@ -643,6 +643,10 @@ void BraveAddCommonStrings(content::WebUIDataSource* html_source,
IDS_SETTINGS_LEO_ASSISTANT_INPUT_MODEL_VISION_SUPPORT},
{"braveLeoAssistantInputModelVisionSupportTooltipInfo",
IDS_SETTINGS_LEO_ASSISTANT_INPUT_MODEL_VISION_SUPPORT_TOOLTIP_INFO},
{"braveLeoAssistantInputModelToolSupport",
IDS_SETTINGS_LEO_ASSISTANT_MODEL_TOOL_SUPPORT_TOGGLE},
{"braveLeoAssistantInputModelToolSupportTooltipInfo",
IDS_SETTINGS_LEO_ASSISTANT_INPUT_MODEL_TOOL_SUPPORT_TOOLTIP_INFO},
{"braveLeoAssistantInputContextSizeLabel",
IDS_SETTINGS_LEO_ASSISTANT_INPUT_CONTEXT_SIZE},
{"braveLeoAssistantInputContextSizeTooltipInfo",
@@ -2053,11 +2053,12 @@ std::vector<base::WeakPtr<Tool>> ConversationHandler::GetTools() {
std::remove_if(
tools.begin(), tools.end(),
[&](auto& tool) {
return (!tool->IsSupportedByModel(model) ||
!tool->SupportsConversation(
GetIsTemporary(),
associated_content_manager_->HasAssociatedContent(),
conversation_capability_));
return (
!tool->IsSupportedByModel(model, conversation_capability_) ||
!tool->SupportsConversation(
GetIsTemporary(),
associated_content_manager_->HasAssociatedContent(),
conversation_capability_));
}),
tools.end());
@@ -64,6 +64,7 @@ constexpr char kCustomModelSystemPromptKey[] = "model_system_prompt";
constexpr char kCustomModelItemApiKey[] = "api_key";
constexpr char kCustomModelItemKey[] = "key";
constexpr char kCustomModelVisionSupport[] = "vision_support";
constexpr char kCustomModelSupportsTools[] = "supports_tools";
// When adding new models, especially for display, make sure to add the UI
// strings to ai_chat_ui_strings.grdp and ai_chat/core/constants.cc.
@@ -111,6 +112,11 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Automatic";
model->vision_support = true;
model->supports_tools = features::kAutomaticModelSupportsTools.Get();
model->supported_capabilities =
model->supports_tools
? std::vector{mojom::ConversationCapability::CHAT,
mojom::ConversationCapability::CONTENT_AGENT}
: std::vector{mojom::ConversationCapability::CHAT};
model->is_suggested_model = true;
model->is_near_model = false;
model->options =
@@ -132,6 +138,9 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Claude Haiku";
model->vision_support = true;
model->supports_tools = true;
model->supported_capabilities = {
mojom::ConversationCapability::CHAT,
mojom::ConversationCapability::CONTENT_AGENT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -154,6 +163,9 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Claude Sonnet";
model->vision_support = true;
model->supports_tools = true;
model->supported_capabilities = {
mojom::ConversationCapability::CHAT,
mojom::ConversationCapability::CONTENT_AGENT};
model->is_suggested_model = true;
model->is_near_model = false;
model->options =
@@ -178,6 +190,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Llama 3.1 8B";
model->vision_support = false;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -202,6 +215,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Qwen VL 30B";
model->vision_support = true;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = true;
model->is_near_model = false;
model->options =
@@ -227,6 +241,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "GLM 4.7 Flash";
model->vision_support = true;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -252,6 +267,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Llama 4 Maverick";
model->vision_support = true;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -277,6 +293,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "GPT OSS 20B";
model->vision_support = false;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -300,6 +317,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "GPT OSS 120B";
model->vision_support = false;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -323,6 +341,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Mistral Large";
model->vision_support = true;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -346,6 +365,7 @@ const std::vector<mojom::ModelPtr>& GetLeoModels() {
model->display_name = "Kimi K2.5";
model->vision_support = false;
model->supports_tools = false;
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->is_suggested_model = false;
model->is_near_model = false;
model->options =
@@ -544,6 +564,7 @@ base::DictValue GetModelDict(mojom::ModelPtr model) {
model_dict.Set(kCustomModelItemKey, model->key);
model_dict.Set(kCustomModelItemLabelKey, model->display_name);
model_dict.Set(kCustomModelVisionSupport, model->vision_support);
model_dict.Set(kCustomModelSupportsTools, model->supports_tools);
model_dict.Set(kCustomModelItemModelKey, options.model_request_name);
model_dict.Set(kCustomModelItemEndpointUrlKey, options.endpoint.spec());
model_dict.Set(kCustomModelItemApiKey, EncryptAPIKey(options.api_key));
@@ -1075,6 +1096,9 @@ const std::vector<mojom::ModelPtr> ModelService::GetCustomModels() {
model->display_name = *model_pref.FindString(kCustomModelItemLabelKey);
model->vision_support =
model_pref.FindBool(kCustomModelVisionSupport).value_or(false);
model->supports_tools =
model_pref.FindBool(kCustomModelSupportsTools).value_or(false);
model->supported_capabilities = {mojom::ConversationCapability::CHAT};
model->options = mojom::ModelOptions::NewCustomModelOptions(
std::move(custom_model_opts));
@@ -65,7 +65,9 @@ std::optional<base::DictValue> MockTool::ExtraParams() const {
return std::nullopt;
}
bool MockTool::IsSupportedByModel(const mojom::Model& model) const {
bool MockTool::IsSupportedByModel(
const mojom::Model& model,
mojom::ConversationCapability conversation_capability) const {
return is_supported_by_model_;
}
@@ -41,7 +41,9 @@ class MockTool : public Tool {
std::optional<base::DictValue> InputProperties() const override;
std::optional<std::vector<std::string>> RequiredProperties() const override;
std::optional<base::DictValue> ExtraParams() const override;
bool IsSupportedByModel(const mojom::Model& model) const override;
bool IsSupportedByModel(
const mojom::Model& model,
mojom::ConversationCapability conversation_capability) const override;
bool SupportsConversation(
bool is_temporary,
bool has_untrusted_content,
@@ -5,6 +5,8 @@
#include "brave/components/ai_chat/core/browser/tools/tool.h"
#include <algorithm>
#include "brave/components/ai_chat/core/common/mojom/ai_chat.mojom.h"
#include "brave/components/ai_chat/core/common/mojom/common.mojom.h"
@@ -29,9 +31,13 @@ bool Tool::IsAgentTool() const {
return false;
}
bool Tool::IsSupportedByModel(const mojom::Model& model) const {
bool Tool::IsSupportedByModel(
const mojom::Model& model,
mojom::ConversationCapability conversation_capability) const {
// Implementors should add any extra checks in an override.
return model.supports_tools;
return model.supports_tools && std::ranges::find(model.supported_capabilities,
conversation_capability) !=
model.supported_capabilities.end();
}
std::variant<bool, mojom::PermissionChallengePtr>
+3 -1
View File
@@ -74,7 +74,9 @@ class Tool {
// Implementor can check features of the model to determine if the tool is
// supported.
virtual bool IsSupportedByModel(const mojom::Model& model) const;
virtual bool IsSupportedByModel(
const mojom::Model& model,
mojom::ConversationCapability conversation_capability) const;
// Check if this tool requires user interaction before handling
// Returns:
@@ -595,6 +595,13 @@ struct Model {
bool vision_support;
// Model supports tool use
bool supports_tools = false;
// Conversation capabilities this model supports. Used to filter which models
// are available for different conversation types (e.g. basic chat, content
// agent, deep research).
// All models that can participate in regular conversations should include
// CHAT. Models approved for agentic contexts (content agent, agent profile)
// should include CONTENT_AGENT, which should be approved by security team.
array<ConversationCapability> supported_capabilities;
// Is a suggested model
bool is_suggested_model;
// Is a Near model
@@ -22,6 +22,7 @@ describe('ModelIntro', () => {
visionSupport: false,
isSuggestedModel: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isNearModel: false,
options: {
customModelOptions: undefined,
@@ -24,6 +24,10 @@ describe('ModelSelector', () => {
isNearModel: false,
visionSupport: true,
supportsTools: true,
supportedCapabilities: [
Mojom.ConversationCapability.CHAT,
Mojom.ConversationCapability.CONTENT_AGENT,
],
options: {
customModelOptions: undefined,
leoModelOptions: {
@@ -42,6 +46,10 @@ describe('ModelSelector', () => {
isSuggestedModel: true,
isNearModel: false,
supportsTools: true,
supportedCapabilities: [
Mojom.ConversationCapability.CHAT,
Mojom.ConversationCapability.CONTENT_AGENT,
],
visionSupport: true,
options: {
customModelOptions: undefined,
@@ -61,6 +69,7 @@ describe('ModelSelector', () => {
isSuggestedModel: false,
isNearModel: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
visionSupport: true,
options: {
customModelOptions: undefined,
@@ -80,6 +89,10 @@ describe('ModelSelector', () => {
isSuggestedModel: true,
isNearModel: true,
supportsTools: true,
supportedCapabilities: [
Mojom.ConversationCapability.CHAT,
Mojom.ConversationCapability.CONTENT_AGENT,
],
visionSupport: true,
options: {
customModelOptions: undefined,
@@ -99,6 +112,7 @@ describe('ModelSelector', () => {
isSuggestedModel: false,
isNearModel: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
visionSupport: true,
options: {
customModelOptions: undefined,
@@ -119,6 +133,7 @@ describe('ModelSelector', () => {
isNearModel: false,
visionSupport: true,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
options: {
customModelOptions: undefined,
leoModelOptions: {
@@ -138,6 +153,7 @@ describe('ModelSelector', () => {
isNearModel: false,
visionSupport: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
options: {
leoModelOptions: undefined,
@@ -354,10 +370,12 @@ describe('ModelSelector', () => {
+ 'agent mode',
async () => {
// Simulate the real context behavior: in agent mode,
// filter models by supportsTools. This mimics the logic
// filter models by supportedCapabilities. This mimics the logic
// in Conversation Context.
const filteredModels = mockModels.filter(
(model) => model.supportsTools === true,
const filteredModels = mockModels.filter((model) =>
model.supportedCapabilities.includes(
Mojom.ConversationCapability.CONTENT_AGENT,
),
)
renderModelSelector({
@@ -124,7 +124,11 @@ export function useProvideConversationContext(props: ConversationContextProps) {
const availableModels = React.useMemo(() => {
return aiChat.isAIChatAgentProfileFeatureEnabled
&& aiChat.isAIChatAgentProfile
? conversationState.allModels.filter((m) => m.supportsTools)
? conversationState.allModels.filter((m) =>
m.supportedCapabilities.includes(
Mojom.ConversationCapability.CONTENT_AGENT,
),
)
: conversationState.allModels
}, [
conversationState.allModels,
@@ -853,6 +853,7 @@ const MODELS: Mojom.Model[] = [
displayName: 'Model One',
visionSupport: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: true,
isNearModel: false,
options: {
@@ -872,6 +873,10 @@ const MODELS: Mojom.Model[] = [
displayName: 'Model Two',
visionSupport: true,
supportsTools: true,
supportedCapabilities: [
Mojom.ConversationCapability.CHAT,
Mojom.ConversationCapability.CONTENT_AGENT,
],
isSuggestedModel: true,
isNearModel: false,
options: {
@@ -891,6 +896,7 @@ const MODELS: Mojom.Model[] = [
displayName: 'Model Three',
visionSupport: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
options: {
@@ -910,6 +916,7 @@ const MODELS: Mojom.Model[] = [
displayName: 'Microsoft Phi-3',
visionSupport: false,
supportsTools: true,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
options: {
@@ -165,7 +165,11 @@ export default function createUntrustedConversationApi(
state.allModels =
state.conversationCapability
=== Mojom.ConversationCapability.CONTENT_AGENT
? state.allModels.filter((model) => model.supportsTools)
? state.allModels.filter((m) =>
m.supportedCapabilities.includes(
Mojom.ConversationCapability.CONTENT_AGENT,
),
)
: state.allModels
api.state.update(state)
},
@@ -17,6 +17,7 @@ const mockModels: Mojom.Model[] = [
displayName: 'Model One',
visionSupport: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
options: {
@@ -36,6 +37,7 @@ const mockModels: Mojom.Model[] = [
displayName: 'Brave Summary',
visionSupport: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
options: {
@@ -15,6 +15,7 @@ const mockModels = [
displayName: 'Model One',
visionSupport: false,
supportsTools: false,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
options: {
@@ -34,6 +35,7 @@ const mockModels = [
displayName: 'Model Two',
visionSupport: true,
supportsTools: true,
supportedCapabilities: [Mojom.ConversationCapability.CHAT],
isSuggestedModel: false,
isNearModel: false,
options: {
@@ -114,6 +114,7 @@ struct AIChatIntroMessageView_Previews: PreviewProvider {
displayName: "Mixtral 8x7b",
visionSupport: false,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
)
@@ -295,6 +295,7 @@ struct AIChatMenuView_Preview: PreviewProvider {
displayName: "Mixtral 8x7b",
visionSupport: false,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
),
@@ -314,6 +315,7 @@ struct AIChatMenuView_Preview: PreviewProvider {
displayName: "Mixtral 8x7b",
visionSupport: false,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
),
@@ -332,6 +334,7 @@ struct AIChatMenuView_Preview: PreviewProvider {
displayName: "Claude-Instant",
visionSupport: false,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
),
@@ -350,6 +353,7 @@ struct AIChatMenuView_Preview: PreviewProvider {
displayName: "Llama-2 13b",
visionSupport: false,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
),
@@ -368,6 +372,7 @@ struct AIChatMenuView_Preview: PreviewProvider {
displayName: "Llama-2 70b",
visionSupport: false,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
),
@@ -438,6 +438,7 @@ extension AiChat.Model {
displayName: "Custom Model",
visionSupport: true,
supportsTools: false,
supportedCapabilities: [NSNumber(value: AiChat.ConversationCapability.chat.rawValue)],
isSuggestedModel: false,
isNearModel: false
)