diff --git a/crates/anthropic/src/anthropic.rs b/crates/anthropic/src/anthropic.rs index f960dc541a..91b6723e90 100644 --- a/crates/anthropic/src/anthropic.rs +++ b/crates/anthropic/src/anthropic.rs @@ -49,6 +49,7 @@ pub enum Model { /// Indicates whether this custom model supports caching. cache_configuration: Option, max_output_tokens: Option, + default_temperature: Option, }, } @@ -124,6 +125,19 @@ impl Model { } } + pub fn default_temperature(&self) -> f32 { + match self { + Self::Claude3_5Sonnet + | Self::Claude3Opus + | Self::Claude3Sonnet + | Self::Claude3Haiku => 1.0, + Self::Custom { + default_temperature, + .. + } => default_temperature.unwrap_or(1.0), + } + } + pub fn tool_model_id(&self) -> &str { if let Self::Custom { tool_override: Some(tool_override), diff --git a/crates/assistant/src/context.rs b/crates/assistant/src/context.rs index 830c098049..97a5b3ea98 100644 --- a/crates/assistant/src/context.rs +++ b/crates/assistant/src/context.rs @@ -2180,7 +2180,7 @@ impl Context { messages: Vec::new(), tools: Vec::new(), stop: Vec::new(), - temperature: 1.0, + temperature: None, }; for message in self.messages(cx) { if message.status != MessageStatus::Done { diff --git a/crates/assistant/src/inline_assistant.rs b/crates/assistant/src/inline_assistant.rs index d95b54d3c6..f2428c3a2e 100644 --- a/crates/assistant/src/inline_assistant.rs +++ b/crates/assistant/src/inline_assistant.rs @@ -2732,7 +2732,7 @@ impl CodegenAlternative { messages, tools: Vec::new(), stop: Vec::new(), - temperature: 1., + temperature: None, }) } diff --git a/crates/assistant/src/prompt_library.rs b/crates/assistant/src/prompt_library.rs index 76ee95d507..24e20a18a7 100644 --- a/crates/assistant/src/prompt_library.rs +++ b/crates/assistant/src/prompt_library.rs @@ -796,7 +796,7 @@ impl PromptLibrary { }], tools: Vec::new(), stop: Vec::new(), - temperature: 1., + temperature: None, }, cx, ) diff --git a/crates/assistant/src/slash_command/auto_command.rs b/crates/assistant/src/slash_command/auto_command.rs index e1f20c311b..14cee29682 100644 --- a/crates/assistant/src/slash_command/auto_command.rs +++ b/crates/assistant/src/slash_command/auto_command.rs @@ -216,7 +216,7 @@ async fn commands_for_summaries( }], tools: Vec::new(), stop: Vec::new(), - temperature: 1.0, + temperature: None, }; while let Some(current_summaries) = stack.pop() { diff --git a/crates/assistant/src/terminal_inline_assistant.rs b/crates/assistant/src/terminal_inline_assistant.rs index caf819bae5..e1a26d8510 100644 --- a/crates/assistant/src/terminal_inline_assistant.rs +++ b/crates/assistant/src/terminal_inline_assistant.rs @@ -284,7 +284,7 @@ impl TerminalInlineAssistant { messages, tools: Vec::new(), stop: Vec::new(), - temperature: 1.0, + temperature: None, }) } diff --git a/crates/language_model/src/provider/anthropic.rs b/crates/language_model/src/provider/anthropic.rs index 1e3d275094..86538bec49 100644 --- a/crates/language_model/src/provider/anthropic.rs +++ b/crates/language_model/src/provider/anthropic.rs @@ -51,6 +51,7 @@ pub struct AvailableModel { /// Configuration of Anthropic's caching API. pub cache_configuration: Option, pub max_output_tokens: Option, + pub default_temperature: Option, } pub struct AnthropicLanguageModelProvider { @@ -200,6 +201,7 @@ impl LanguageModelProvider for AnthropicLanguageModelProvider { } }), max_output_tokens: model.max_output_tokens, + default_temperature: model.default_temperature, }, ); } @@ -375,8 +377,11 @@ impl LanguageModel for AnthropicModel { request: LanguageModelRequest, cx: &AsyncAppContext, ) -> BoxFuture<'static, Result>>> { - let request = - request.into_anthropic(self.model.id().into(), self.model.max_output_tokens()); + let request = request.into_anthropic( + self.model.id().into(), + self.model.default_temperature(), + self.model.max_output_tokens(), + ); let request = self.stream_completion(request, cx); let future = self.request_limiter.stream(async move { let response = request.await.map_err(|err| anyhow!(err))?; @@ -405,6 +410,7 @@ impl LanguageModel for AnthropicModel { ) -> BoxFuture<'static, Result>>> { let mut request = request.into_anthropic( self.model.tool_model_id().into(), + self.model.default_temperature(), self.model.max_output_tokens(), ); request.tool_choice = Some(anthropic::ToolChoice::Tool { diff --git a/crates/language_model/src/provider/cloud.rs b/crates/language_model/src/provider/cloud.rs index 58efb4cfe1..606a6fbace 100644 --- a/crates/language_model/src/provider/cloud.rs +++ b/crates/language_model/src/provider/cloud.rs @@ -87,6 +87,8 @@ pub struct AvailableModel { pub tool_override: Option, /// Indicates whether this custom model supports caching. pub cache_configuration: Option, + /// The default temperature to use for this model. + pub default_temperature: Option, } pub struct CloudLanguageModelProvider { @@ -255,6 +257,7 @@ impl LanguageModelProvider for CloudLanguageModelProvider { min_total_token: config.min_total_token, } }), + default_temperature: model.default_temperature, max_output_tokens: model.max_output_tokens, }), AvailableProvider::OpenAi => CloudModel::OpenAi(open_ai::Model::Custom { @@ -516,7 +519,11 @@ impl LanguageModel for CloudLanguageModel { match &self.model { CloudModel::Anthropic(model) => { - let request = request.into_anthropic(model.id().into(), model.max_output_tokens()); + let request = request.into_anthropic( + model.id().into(), + model.default_temperature(), + model.max_output_tokens(), + ); let client = self.client.clone(); let llm_api_token = self.llm_api_token.clone(); let future = self.request_limiter.stream(async move { @@ -642,8 +649,11 @@ impl LanguageModel for CloudLanguageModel { match &self.model { CloudModel::Anthropic(model) => { - let mut request = - request.into_anthropic(model.tool_model_id().into(), model.max_output_tokens()); + let mut request = request.into_anthropic( + model.tool_model_id().into(), + model.default_temperature(), + model.max_output_tokens(), + ); request.tool_choice = Some(anthropic::ToolChoice::Tool { name: tool_name.clone(), }); diff --git a/crates/language_model/src/provider/ollama.rs b/crates/language_model/src/provider/ollama.rs index 6a3190dee7..a29ff3cf6a 100644 --- a/crates/language_model/src/provider/ollama.rs +++ b/crates/language_model/src/provider/ollama.rs @@ -235,7 +235,7 @@ impl OllamaLanguageModel { options: Some(ChatOptions { num_ctx: Some(self.model.max_tokens), stop: Some(request.stop), - temperature: Some(request.temperature), + temperature: request.temperature.or(Some(1.0)), ..Default::default() }), tools: vec![], diff --git a/crates/language_model/src/request.rs b/crates/language_model/src/request.rs index dd480b8aaf..06dde1862a 100644 --- a/crates/language_model/src/request.rs +++ b/crates/language_model/src/request.rs @@ -236,7 +236,7 @@ pub struct LanguageModelRequest { pub messages: Vec, pub tools: Vec, pub stop: Vec, - pub temperature: f32, + pub temperature: Option, } impl LanguageModelRequest { @@ -262,7 +262,7 @@ impl LanguageModelRequest { .collect(), stream, stop: self.stop, - temperature: self.temperature, + temperature: self.temperature.unwrap_or(1.0), max_tokens: max_output_tokens, tools: Vec::new(), tool_choice: None, @@ -290,7 +290,7 @@ impl LanguageModelRequest { candidate_count: Some(1), stop_sequences: Some(self.stop), max_output_tokens: None, - temperature: Some(self.temperature as f64), + temperature: self.temperature.map(|t| t as f64).or(Some(1.0)), top_p: None, top_k: None, }), @@ -298,7 +298,12 @@ impl LanguageModelRequest { } } - pub fn into_anthropic(self, model: String, max_output_tokens: u32) -> anthropic::Request { + pub fn into_anthropic( + self, + model: String, + default_temperature: f32, + max_output_tokens: u32, + ) -> anthropic::Request { let mut new_messages: Vec = Vec::new(); let mut system_message = String::new(); @@ -400,7 +405,7 @@ impl LanguageModelRequest { tool_choice: None, metadata: None, stop_sequences: Vec::new(), - temperature: Some(self.temperature), + temperature: self.temperature.or(Some(default_temperature)), top_k: None, top_p: None, } diff --git a/crates/language_model/src/settings.rs b/crates/language_model/src/settings.rs index 8888d51e11..2bf8deb042 100644 --- a/crates/language_model/src/settings.rs +++ b/crates/language_model/src/settings.rs @@ -99,6 +99,7 @@ impl AnthropicSettingsContent { tool_override, cache_configuration, max_output_tokens, + default_temperature, } => Some(provider::anthropic::AvailableModel { name, display_name, @@ -112,6 +113,7 @@ impl AnthropicSettingsContent { }, ), max_output_tokens, + default_temperature, }), _ => None, }) diff --git a/crates/semantic_index/src/summary_index.rs b/crates/semantic_index/src/summary_index.rs index 08f25ae028..f4c6d4726c 100644 --- a/crates/semantic_index/src/summary_index.rs +++ b/crates/semantic_index/src/summary_index.rs @@ -562,7 +562,7 @@ impl SummaryIndex { }], tools: Vec::new(), stop: Vec::new(), - temperature: 1.0, + temperature: None, }; let code_len = code.len();