Trying local mistralrs build

This commit is contained in:
Richard Feldman 2025-07-29 18:24:33 -04:00
parent f62e693b8f
commit 5830628568
No known key found for this signature in database
4 changed files with 255 additions and 62 deletions

View file

@ -98,9 +98,9 @@ impl State {
}
async fn load_mistral_model() -> Result<Arc<MistralModel>> {
println!("\n\n\n\nLoading mistral model...\n\n\n");
let model = TextModelBuilder::new(DEFAULT_MODEL)
.with_isq(IsqType::Q4_0)
.with_logging()
.build()
.await?;