From cf0d1e4229eab92d0832012389e6350d53fffb42 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Fri, 4 Apr 2025 10:33:15 -0400 Subject: [PATCH] collab: Add granular tokens per minute columns to `models` table (#28090) This PR adds new granular tokens per minute columns to the `models` table in preparation for more fine-grained rate limits. The following columns have been added: - `max_input_tokens_per_minute` - `max_output_tokens_per_minute` These mirror the "Maximum input tokens per minute (ITPM)" and "Maximum output tokens per minute (OTPM)" [rate limits from Anthropic](https://docs.anthropic.com/en/api/rate-limits#rate-limits). Release Notes: - N/A --- .../20250404141155_add_granular_token_limits_to_models.sql | 3 +++ crates/collab/src/llm/db/tables/model.rs | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 crates/collab/migrations_llm/20250404141155_add_granular_token_limits_to_models.sql diff --git a/crates/collab/migrations_llm/20250404141155_add_granular_token_limits_to_models.sql b/crates/collab/migrations_llm/20250404141155_add_granular_token_limits_to_models.sql new file mode 100644 index 0000000000..e5c50d8385 --- /dev/null +++ b/crates/collab/migrations_llm/20250404141155_add_granular_token_limits_to_models.sql @@ -0,0 +1,3 @@ +alter table models + add column max_input_tokens_per_minute bigint not null default 0, + add column max_output_tokens_per_minute bigint not null default 0; diff --git a/crates/collab/src/llm/db/tables/model.rs b/crates/collab/src/llm/db/tables/model.rs index 6c52184185..3453c34726 100644 --- a/crates/collab/src/llm/db/tables/model.rs +++ b/crates/collab/src/llm/db/tables/model.rs @@ -12,6 +12,8 @@ pub struct Model { pub name: String, pub max_requests_per_minute: i64, pub max_tokens_per_minute: i64, + pub max_input_tokens_per_minute: i64, + pub max_output_tokens_per_minute: i64, pub max_tokens_per_day: i64, pub price_per_million_input_tokens: i32, pub price_per_million_cache_creation_input_tokens: i32,