Trying local mistralrs build
This commit is contained in:
parent
f62e693b8f
commit
5830628568
4 changed files with 255 additions and 62 deletions
|
@ -98,9 +98,9 @@ impl State {
|
|||
}
|
||||
|
||||
async fn load_mistral_model() -> Result<Arc<MistralModel>> {
|
||||
println!("\n\n\n\nLoading mistral model...\n\n\n");
|
||||
let model = TextModelBuilder::new(DEFAULT_MODEL)
|
||||
.with_isq(IsqType::Q4_0)
|
||||
.with_logging()
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue