Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use gpt-4o tokenizer for gpt-5 for now

Browse source
This commit is contained in:
Richard Feldman 2025-08-07 14:39:58 -04:00 • committed by Joseph T. Lyons
parent 52fad99449
commit e2be6611db
1 changed files with 5 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/language_models/src/provider/open_ai.rs
View file

@ -673,10 +673,11 @@ pub fn count_open_ai_tokens(
| Model::O1 | Model::O1
| Model::O3 | Model::O3
| Model::O3Mini | Model::O3Mini
| Model::O4Mini | Model::O4Mini => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),
| Model::Five // GPT-5 models don't have tiktoken support yet; fall back on gpt-4o tokenizer
| Model::FiveMini Model::Five | Model::FiveMini | Model::FiveNano => {
| Model::FiveNano => tiktoken_rs::num_tokens_from_messages(model.id(), &messages), tiktoken_rs::num_tokens_from_messages("gpt-4o", &messages)
}
} }
.map(|tokens| tokens as u64) .map(|tokens| tokens as u64)
}) })