Standardize on u64 for token counts (#32869)

Previously we were using a mix of `u32` and `usize`, e.g. `max_tokens:
usize, max_output_tokens: Option<u32>` in the same `struct`.

Although [tiktoken](https://github.com/openai/tiktoken) uses `usize`,
token counts should be consistent across targets (e.g. the same model
doesn't suddenly get a smaller context window if you're compiling for
wasm32), and these token counts could end up getting serialized using a
binary protocol, so `usize` is not the right choice for token counts.

I chose to standardize on `u64` over `u32` because we don't store many
of them (so the extra size should be insignificant) and future models
may exceed `u32::MAX` tokens.

Release Notes:

- N/A
This commit is contained in:
Richard Feldman 2025-06-17 10:43:07 -04:00 committed by GitHub
parent a391d67366
commit 5405c2c2d3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 191 additions and 192 deletions

View file

@ -50,7 +50,7 @@ impl From<Role> for String {
pub struct Model {
pub name: String,
pub display_name: Option<String>,
pub max_tokens: usize,
pub max_tokens: u64,
pub supports_tools: Option<bool>,
pub supports_images: Option<bool>,
}
@ -73,7 +73,7 @@ impl Model {
pub fn new(
name: &str,
display_name: Option<&str>,
max_tokens: Option<usize>,
max_tokens: Option<u64>,
supports_tools: Option<bool>,
supports_images: Option<bool>,
) -> Self {
@ -94,11 +94,11 @@ impl Model {
self.display_name.as_ref().unwrap_or(&self.name)
}
pub fn max_token_count(&self) -> usize {
pub fn max_token_count(&self) -> u64 {
self.max_tokens
}
pub fn max_output_tokens(&self) -> Option<u32> {
pub fn max_output_tokens(&self) -> Option<u64> {
None
}
@ -117,7 +117,7 @@ pub struct Request {
pub messages: Vec<RequestMessage>,
pub stream: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
pub max_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub stop: Vec<String>,
pub temperature: f32,
@ -318,9 +318,9 @@ pub struct FunctionChunk {
#[derive(Serialize, Deserialize, Debug)]
pub struct Usage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
}
#[derive(Serialize, Deserialize, Debug)]
@ -369,7 +369,7 @@ pub struct ModelEntry {
pub created: usize,
pub description: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub context_length: Option<usize>,
pub context_length: Option<u64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub supported_parameters: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]