Implement Anthropic prompt caching (#16274)
Release Notes: - Adds support for Prompt Caching in Anthropic. For models that support it this can dramatically lower cost while improving performance.
This commit is contained in:
parent
09b6e3f2a6
commit
46fb917e02
11 changed files with 338 additions and 70 deletions
|
@ -784,6 +784,7 @@ impl PromptLibrary {
|
|||
messages: vec![LanguageModelRequestMessage {
|
||||
role: Role::System,
|
||||
content: vec![body.to_string().into()],
|
||||
cache: false,
|
||||
}],
|
||||
stop: Vec::new(),
|
||||
temperature: 1.,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue