Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use gpt-4o tokenizer for gpt-5 for now

Browse source
This commit is contained in:
Richard Feldman 2025-08-07 14:39:58 -04:00 • committed by Joseph T. Lyons
parent d5b6a4d710
commit a6b9668355
1 changed files with 5 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/language_models/src/provider/open_ai.rs
View file

@ -673,10 +673,11 @@ pub fn count_open_ai_tokens(
| Model::O1
| Model::O3
| Model::O3Mini
| Model::O4Mini
| Model::Five
| Model::FiveMini
| Model::FiveNano => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),
| Model::O4Mini => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),
// GPT-5 models don't have tiktoken support yet; fall back on gpt-4o tokenizer
Model::Five | Model::FiveMini | Model::FiveNano => {
tiktoken_rs::num_tokens_from_messages("gpt-4o", &messages)
}
}
.map(|tokens| tokens as u64)
})