Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llm: Sample ~10% of staff members inputs/outputs to LLM (#23537)

Browse source 
Release Notes:

- N/A
This commit is contained in:
Thorsten Ball 2025-01-23 15:32:25 +01:00 • committed by GitHub
parent 95c045ad93
commit 0cb41754e2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 29 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

36
crates/collab/src/llm.rs
View file

@ -447,6 +447,8 @@ async fn predict_edits(
)); ));
} }
let sample_input_output = claims.is_staff && rand::random::<f32>() < 0.1;
let api_url = state let api_url = state
.config .config
.prediction_api_url .prediction_api_url
@ -489,7 +491,7 @@ async fn predict_edits(
max_tokens: 2048, max_tokens: 2048,
temperature: 0., temperature: 0.,
prediction: Some(fireworks::Prediction::Content { prediction: Some(fireworks::Prediction::Content {
content: params.input_excerpt, content: params.input_excerpt.clone(),
}), }),
rewrite_speculation: Some(true), rewrite_speculation: Some(true),
}, },
@ -536,18 +538,38 @@ async fn predict_edits(
let kinesis_client = state.kinesis_client.clone(); let kinesis_client = state.kinesis_client.clone();
let kinesis_stream = state.config.kinesis_stream.clone(); let kinesis_stream = state.config.kinesis_stream.clone();
let model = model.clone(); let model = model.clone();
let output = choice.text.clone();
async move { async move {
SnowflakeRow::new( let properties = if sample_input_output {
"Fireworks Completion Requested",
claims.metrics_id,
claims.is_staff,
claims.system_id.clone(),
json!({ json!({
"model": model.to_string(), "model": model.to_string(),
"headers": response.headers, "headers": response.headers,
"usage": response.completion.usage, "usage": response.completion.usage,
"duration": duration.as_secs_f64(), "duration": duration.as_secs_f64(),
}), "prompt": prompt,
"input_excerpt": params.input_excerpt,
"input_events": params.input_events,
"outline": params.outline,
"output": output,
"is_sampled": true,
})
} else {
json!({
"model": model.to_string(),
"headers": response.headers,
"usage": response.completion.usage,
"duration": duration.as_secs_f64(),
"is_sampled": false,
})
};
SnowflakeRow::new(
"Fireworks Completion Requested",
claims.metrics_id,
claims.is_staff,
claims.system_id.clone(),
properties,
) )
.write(&kinesis_client, &kinesis_stream) .write(&kinesis_client, &kinesis_stream)
.await .await