Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix for error when truncating a length less than the string length

Browse source
This commit is contained in:
KCaverly 2023-10-18 16:31:29 -04:00
parent 32853c2044
commit a0e01e075d
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crates/ai/src/models.rs
View file

@ -38,7 +38,11 @@ impl LanguageModel for OpenAILanguageModel {
fn truncate(&self, content: &str, length: usize) -> anyhow::Result<String> { fn truncate(&self, content: &str, length: usize) -> anyhow::Result<String> {
if let Some(bpe) = &self.bpe { if let Some(bpe) = &self.bpe {
let tokens = bpe.encode_with_special_tokens(content); let tokens = bpe.encode_with_special_tokens(content);
bpe.decode(tokens[..length].to_vec()) if tokens.len() > length {
bpe.decode(tokens[..length].to_vec())
} else {
bpe.decode(tokens)
}
} else { } else {
Err(anyhow!("bpe for open ai model was not retrieved")) Err(anyhow!("bpe for open ai model was not retrieved"))
} }