Add telemetry for supermaven (#11821)
Data migration plan:
- [X] Make a duplicate table of `copilot_events`
- Name: `inline_completion_events`
- Omit `suggestion_id` column
- [X-reverted-skipping] In collab, continue to match on copilot_events,
but simply stuff their data into inline_completion_events, to forward it
to the new table
- [skipping] Once collab is deployed, ensure no events are being sent to
copilot_events, migrate `copilot_events` to new table via a transaction
- [skipping] Delete `copilot_events` table
---
- [X] Locally test that copilot events sent from old clients get put
into inline_completions_table
- [X] Locally test that copilot events and supermaven events sent from
new clients get put into inline_completions_table
---
- [X] Why are discard events being spammed?
- A:
8d4315712b/crates/editor/src/editor.rs (L2147)

This will throw off the past results for accepted / dismissed that I was
wanting to use to evaluate Supermaven quality, by comparing its rate
with copilot's rate.
I'm not super thrilled with this fix, but I think it'll do. In the
`supermaven_completions_provider`, we check if there's a `completion_id`
before sending either an accepted or discard completion event. I don't
see a similar construct in the `copilot_completions_provider` to
piggyback off of, so I begrudgingly introduced
`should_allow_event_to_send` and had it follow the same pattern that
`completion_id` does. Maybe there's a better way?
---
Adds events to supermaven suggestions. Makes "CopilotEvents" generic ->
"InlineCompletionEvents".
Release Notes:
- N/A
This commit is contained in:
parent
55f08c0511
commit
b6189b05f9
9 changed files with 123 additions and 42 deletions
|
@ -15,8 +15,9 @@ use serde::{Serialize, Serializer};
|
|||
use sha2::{Digest, Sha256};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use telemetry_events::{
|
||||
ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent,
|
||||
EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent,
|
||||
ActionEvent, AppEvent, AssistantEvent, CallEvent, CpuEvent, EditEvent, EditorEvent, Event,
|
||||
EventRequestBody, EventWrapper, ExtensionEvent, InlineCompletionEvent, MemoryEvent,
|
||||
SettingEvent,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
|
@ -424,13 +425,19 @@ pub async fn post_events(
|
|||
first_event_at,
|
||||
country_code.clone(),
|
||||
)),
|
||||
Event::Copilot(event) => to_upload.copilot_events.push(CopilotEventRow::from_event(
|
||||
event.clone(),
|
||||
&wrapper,
|
||||
&request_body,
|
||||
first_event_at,
|
||||
country_code.clone(),
|
||||
)),
|
||||
// Needed for clients sending old copilot_event types
|
||||
Event::Copilot(_) => {}
|
||||
Event::InlineCompletion(event) => {
|
||||
to_upload
|
||||
.inline_completion_events
|
||||
.push(InlineCompletionEventRow::from_event(
|
||||
event.clone(),
|
||||
&wrapper,
|
||||
&request_body,
|
||||
first_event_at,
|
||||
country_code.clone(),
|
||||
))
|
||||
}
|
||||
Event::Call(event) => to_upload.call_events.push(CallEventRow::from_event(
|
||||
event.clone(),
|
||||
&wrapper,
|
||||
|
@ -512,7 +519,7 @@ pub async fn post_events(
|
|||
#[derive(Default)]
|
||||
struct ToUpload {
|
||||
editor_events: Vec<EditorEventRow>,
|
||||
copilot_events: Vec<CopilotEventRow>,
|
||||
inline_completion_events: Vec<InlineCompletionEventRow>,
|
||||
assistant_events: Vec<AssistantEventRow>,
|
||||
call_events: Vec<CallEventRow>,
|
||||
cpu_events: Vec<CpuEventRow>,
|
||||
|
@ -531,14 +538,14 @@ impl ToUpload {
|
|||
.await
|
||||
.with_context(|| format!("failed to upload to table '{EDITOR_EVENTS_TABLE}'"))?;
|
||||
|
||||
const COPILOT_EVENTS_TABLE: &str = "copilot_events";
|
||||
const INLINE_COMPLETION_EVENTS_TABLE: &str = "inline_completion_events";
|
||||
Self::upload_to_table(
|
||||
COPILOT_EVENTS_TABLE,
|
||||
&self.copilot_events,
|
||||
INLINE_COMPLETION_EVENTS_TABLE,
|
||||
&self.inline_completion_events,
|
||||
clickhouse_client,
|
||||
)
|
||||
.await
|
||||
.with_context(|| format!("failed to upload to table '{COPILOT_EVENTS_TABLE}'"))?;
|
||||
.with_context(|| format!("failed to upload to table '{INLINE_COMPLETION_EVENTS_TABLE}'"))?;
|
||||
|
||||
const ASSISTANT_EVENTS_TABLE: &str = "assistant_events";
|
||||
Self::upload_to_table(
|
||||
|
@ -708,9 +715,9 @@ impl EditorEventRow {
|
|||
}
|
||||
|
||||
#[derive(Serialize, Debug, clickhouse::Row)]
|
||||
pub struct CopilotEventRow {
|
||||
pub struct InlineCompletionEventRow {
|
||||
pub installation_id: String,
|
||||
pub suggestion_id: String,
|
||||
pub provider: String,
|
||||
pub suggestion_accepted: bool,
|
||||
pub app_version: String,
|
||||
pub file_extension: String,
|
||||
|
@ -730,9 +737,9 @@ pub struct CopilotEventRow {
|
|||
pub patch: Option<i32>,
|
||||
}
|
||||
|
||||
impl CopilotEventRow {
|
||||
impl InlineCompletionEventRow {
|
||||
fn from_event(
|
||||
event: CopilotEvent,
|
||||
event: InlineCompletionEvent,
|
||||
wrapper: &EventWrapper,
|
||||
body: &EventRequestBody,
|
||||
first_event_at: chrono::DateTime<chrono::Utc>,
|
||||
|
@ -759,7 +766,7 @@ impl CopilotEventRow {
|
|||
country_code: country_code.unwrap_or("XX".to_string()),
|
||||
region_code: "".to_string(),
|
||||
city: "".to_string(),
|
||||
suggestion_id: event.suggestion_id.unwrap_or_default(),
|
||||
provider: event.provider,
|
||||
suggestion_accepted: event.suggestion_accepted,
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue