Standardize on u64 for token counts (#32869)
Previously we were using a mix of `u32` and `usize`, e.g. `max_tokens: usize, max_output_tokens: Option<u32>` in the same `struct`. Although [tiktoken](https://github.com/openai/tiktoken) uses `usize`, token counts should be consistent across targets (e.g. the same model doesn't suddenly get a smaller context window if you're compiling for wasm32), and these token counts could end up getting serialized using a binary protocol, so `usize` is not the right choice for token counts. I chose to standardize on `u64` over `u32` because we don't store many of them (so the extra size should be insignificant) and future models may exceed `u32::MAX` tokens. Release Notes: - N/A
This commit is contained in:
parent
a391d67366
commit
5405c2c2d3
32 changed files with 191 additions and 192 deletions
|
@ -70,9 +70,9 @@ pub enum Model {
|
|||
name: String,
|
||||
/// The name displayed in the UI, such as in the assistant panel model dropdown menu.
|
||||
display_name: Option<String>,
|
||||
max_tokens: usize,
|
||||
max_output_tokens: Option<u32>,
|
||||
max_completion_tokens: Option<u32>,
|
||||
max_tokens: u64,
|
||||
max_output_tokens: Option<u64>,
|
||||
max_completion_tokens: Option<u64>,
|
||||
supports_tools: Option<bool>,
|
||||
supports_images: Option<bool>,
|
||||
},
|
||||
|
@ -130,7 +130,7 @@ impl Model {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn max_token_count(&self) -> usize {
|
||||
pub fn max_token_count(&self) -> u64 {
|
||||
match self {
|
||||
Self::CodestralLatest => 256000,
|
||||
Self::MistralLargeLatest => 131000,
|
||||
|
@ -145,7 +145,7 @@ impl Model {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn max_output_tokens(&self) -> Option<u32> {
|
||||
pub fn max_output_tokens(&self) -> Option<u64> {
|
||||
match self {
|
||||
Self::Custom {
|
||||
max_output_tokens, ..
|
||||
|
@ -193,7 +193,7 @@ pub struct Request {
|
|||
pub messages: Vec<RequestMessage>,
|
||||
pub stream: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
pub max_tokens: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
|
@ -360,9 +360,9 @@ pub struct Response {
|
|||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue