Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add telemetry events for loading extensions (#9793)

Browse source 
* Store extensions versions' wasm API version in the database
* Share a common struct for extension API responses between collab and
client
* Add wasm API version and schema version to extension API responses

Release Notes:

- N/A

Co-authored-by: Marshall <marshall@zed.dev>
This commit is contained in:
Max Brunsfeld 2024-03-25 14:30:48 -07:00 • committed by GitHub
parent 9b62e461ed
commit 5adc51f113
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 531 additions and 306 deletions
Download patch file Download diff file Expand all files Collapse all files

99
crates/collab/src/api/events.rs
View file

@ -1,5 +1,5 @@
use std::sync::{Arc, OnceLock};
use super::ips_file::IpsFile;
use crate::{api::slack, AppState, Error, Result};
use anyhow::{anyhow, Context};
use aws_sdk_s3::primitives::ByteStream;
use axum::{
@ -9,18 +9,16 @@ use axum::{
routing::post,
Extension, Router, TypedHeader,
};
use rpc::ExtensionMetadata;
use serde::{Serialize, Serializer};
use sha2::{Digest, Sha256};
use std::sync::{Arc, OnceLock};
use telemetry_events::{
ActionEvent, AppEvent, AssistantEvent, CallEvent, CopilotEvent, CpuEvent, EditEvent,
EditorEvent, Event, EventRequestBody, EventWrapper, MemoryEvent, SettingEvent,
EditorEvent, Event, EventRequestBody, EventWrapper, ExtensionEvent, MemoryEvent, SettingEvent,
};
use util::SemanticVersion;
use crate::{api::slack, AppState, Error, Result};
use super::ips_file::IpsFile;
pub fn router() -> Router {
Router::new()
.route("/telemetry/events", post(post_events))
@ -331,6 +329,21 @@ pub async fn post_events(
&request_body,
first_event_at,
)),
Event::Extension(event) => {
let metadata = app
.db
.get_extension_version(&event.extension_id, &event.version)
.await?;
to_upload
.extension_events
.push(ExtensionEventRow::from_event(
event.clone(),
&wrapper,
&request_body,
metadata,
first_event_at,
))
}
}
}
@ -352,6 +365,7 @@ struct ToUpload {
memory_events: Vec<MemoryEventRow>,
app_events: Vec<AppEventRow>,
setting_events: Vec<SettingEventRow>,
extension_events: Vec<ExtensionEventRow>,
edit_events: Vec<EditEventRow>,
action_events: Vec<ActionEventRow>,
}
@ -410,6 +424,15 @@ impl ToUpload {
.await
.with_context(|| format!("failed to upload to table '{SETTING_EVENTS_TABLE}'"))?;
const EXTENSION_EVENTS_TABLE: &str = "extension_events";
Self::upload_to_table(
EXTENSION_EVENTS_TABLE,
&self.extension_events,
clickhouse_client,
)
.await
.with_context(|| format!("failed to upload to table '{EXTENSION_EVENTS_TABLE}'"))?;
const EDIT_EVENTS_TABLE: &str = "edit_events";
Self::upload_to_table(EDIT_EVENTS_TABLE, &self.edit_events, clickhouse_client)
.await
@ -861,6 +884,68 @@ impl SettingEventRow {
}
}
#[derive(Serialize, Debug, clickhouse::Row)]
pub struct ExtensionEventRow {
// AppInfoBase
app_version: String,
major: Option<i32>,
minor: Option<i32>,
patch: Option<i32>,
release_channel: String,
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
time: i64,
// ExtensionEventRow
extension_id: Arc<str>,
extension_version: Arc<str>,
dev: bool,
schema_version: Option<i32>,
wasm_api_version: Option<String>,
}
impl ExtensionEventRow {
fn from_event(
event: ExtensionEvent,
wrapper: &EventWrapper,
body: &EventRequestBody,
extension_metadata: Option<ExtensionMetadata>,
first_event_at: chrono::DateTime<chrono::Utc>,
) -> Self {
let semver = body.semver();
let time =
first_event_at + chrono::Duration::milliseconds(wrapper.milliseconds_since_first_event);
Self {
app_version: body.app_version.clone(),
major: semver.map(|s| s.major as i32),
minor: semver.map(|s| s.minor as i32),
patch: semver.map(|s| s.patch as i32),
release_channel: body.release_channel.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
time: time.timestamp_millis(),
extension_id: event.extension_id,
extension_version: event.version,
dev: extension_metadata.is_none(),
schema_version: extension_metadata
.as_ref()
.and_then(|metadata| metadata.manifest.schema_version),
wasm_api_version: extension_metadata.as_ref().and_then(|metadata| {
metadata
.manifest
.wasm_api_version
.as_ref()
.map(|version| version.to_string())
}),
}
}
}
#[derive(Serialize, Debug, clickhouse::Row)]
pub struct EditEventRow {
// AppInfoBase

10
crates/collab/src/api/extensions.rs
View file

@ -1,7 +1,4 @@
use crate::{
db::{ExtensionMetadata, NewExtensionVersion},
AppState, Error, Result,
};
use crate::{db::NewExtensionVersion, AppState, Error, Result};
use anyhow::{anyhow, Context as _};
use aws_sdk_s3::presigning::PresigningConfig;
use axum::{
@ -12,7 +9,7 @@ use axum::{
Extension, Json, Router,
};
use collections::HashMap;
use rpc::ExtensionApiManifest;
use rpc::{ExtensionApiManifest, ExtensionMetadata};
use serde::{Deserialize, Serialize};
use std::{sync::Arc, time::Duration};
use time::PrimitiveDateTime;
@ -78,7 +75,7 @@ async fn download_latest_extension(
Extension(app),
Path(DownloadExtensionParams {
extension_id: params.extension_id,
version: extension.version,
version: extension.manifest.version,
}),
)
.await
@ -285,6 +282,7 @@ async fn fetch_extension_manifest(
authors: manifest.authors,
repository: manifest.repository,
schema_version: manifest.schema_version.unwrap_or(0),
wasm_api_version: manifest.wasm_api_version,
published_at,
})
}

16
crates/collab/src/db.rs
View file

@ -12,7 +12,7 @@ use futures::StreamExt;
use rand::{prelude::StdRng, Rng, SeedableRng};
use rpc::{
proto::{self},
ConnectionId,
ConnectionId, ExtensionMetadata,
};
use sea_orm::{
entity::prelude::*,
@ -726,22 +726,10 @@ pub struct NewExtensionVersion {
pub authors: Vec<String>,
pub repository: String,
pub schema_version: i32,
pub wasm_api_version: Option<String>,
pub published_at: PrimitiveDateTime,
}
#[derive(Debug, Serialize, PartialEq)]
pub struct ExtensionMetadata {
pub id: String,
pub name: String,
pub version: String,
pub authors: Vec<String>,
pub description: String,
pub repository: String,
#[serde(serialize_with = "serialize_iso8601")]
pub published_at: PrimitiveDateTime,
pub download_count: u64,
}
pub fn serialize_iso8601<S: Serializer>(
datetime: &PrimitiveDateTime,
serializer: S,

92
crates/collab/src/db/queries/extensions.rs
View file

@ -1,3 +1,5 @@
use chrono::Utc;
use super::*;
impl Database {
@ -31,22 +33,8 @@ impl Database {
Ok(extensions
.into_iter()
.filter_map(|(extension, latest_version)| {
let version = latest_version?;
Some(ExtensionMetadata {
id: extension.external_id,
name: extension.name,
version: version.version,
authors: version
.authors
.split(',')
.map(|author| author.trim().to_string())
.collect::<Vec<_>>(),
description: version.description,
repository: version.repository,
published_at: version.published_at,
download_count: extension.total_download_count as u64,
})
.filter_map(|(extension, version)| {
Some(metadata_from_extension_and_version(extension, version?))
})
.collect())
})
@ -67,22 +55,29 @@ impl Database {
.one(&*tx)
.await?;
Ok(extension.and_then(|(extension, latest_version)| {
let version = latest_version?;
Some(ExtensionMetadata {
id: extension.external_id,
name: extension.name,
version: version.version,
authors: version
.authors
.split(',')
.map(|author| author.trim().to_string())
.collect::<Vec<_>>(),
description: version.description,
repository: version.repository,
published_at: version.published_at,
download_count: extension.total_download_count as u64,
})
Ok(extension.and_then(|(extension, version)| {
Some(metadata_from_extension_and_version(extension, version?))
}))
})
.await
}
pub async fn get_extension_version(
&self,
extension_id: &str,
version: &str,
) -> Result<Option<ExtensionMetadata>> {
self.transaction(|tx| async move {
let extension = extension::Entity::find()
.filter(extension::Column::ExternalId.eq(extension_id))
.filter(extension_version::Column::Version.eq(version))
.inner_join(extension_version::Entity)
.select_also(extension_version::Entity)
.one(&*tx)
.await?;
Ok(extension.and_then(|(extension, version)| {
Some(metadata_from_extension_and_version(extension, version?))
}))
})
.await
@ -172,6 +167,7 @@ impl Database {
repository: ActiveValue::Set(version.repository.clone()),
description: ActiveValue::Set(version.description.clone()),
schema_version: ActiveValue::Set(version.schema_version),
wasm_api_version: ActiveValue::Set(version.wasm_api_version.clone()),
download_count: ActiveValue::NotSet,
}
}))
@ -241,3 +237,35 @@ impl Database {
.await
}
}
fn metadata_from_extension_and_version(
extension: extension::Model,
version: extension_version::Model,
) -> ExtensionMetadata {
ExtensionMetadata {
id: extension.external_id,
manifest: rpc::ExtensionApiManifest {
name: extension.name,
version: version.version,
authors: version
.authors
.split(',')
.map(|author| author.trim().to_string())
.collect::<Vec<_>>(),
description: Some(version.description),
repository: version.repository,
schema_version: Some(version.schema_version),
wasm_api_version: version.wasm_api_version,
},
published_at: convert_time_to_chrono(version.published_at),
download_count: extension.total_download_count as u64,
}
}
pub fn convert_time_to_chrono(time: time::PrimitiveDateTime) -> chrono::DateTime<Utc> {
chrono::DateTime::from_naive_utc_and_offset(
chrono::NaiveDateTime::from_timestamp_opt(time.assume_utc().unix_timestamp(), 0).unwrap(),
Utc,
)
}

1
crates/collab/src/db/tables/extension_version.rs
View file

@ -14,6 +14,7 @@ pub struct Model {
pub repository: String,
pub description: String,
pub schema_version: i32,
pub wasm_api_version: Option<String>,
pub download_count: i64,
}

126
crates/collab/src/db/tests/extension_tests.rs
View file

@ -1,10 +1,9 @@
use super::Database;
use crate::{
db::{ExtensionMetadata, NewExtensionVersion},
db::{queries::extensions::convert_time_to_chrono, ExtensionMetadata, NewExtensionVersion},
test_both_dbs,
};
use std::sync::Arc;
use time::{OffsetDateTime, PrimitiveDateTime};
test_both_dbs!(
test_extensions,
@ -19,8 +18,10 @@ async fn test_extensions(db: &Arc<Database>) {
let extensions = db.get_extensions(None, 1, 5).await.unwrap();
assert!(extensions.is_empty());
let t0 = OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
let t0 = PrimitiveDateTime::new(t0.date(), t0.time());
let t0 = time::OffsetDateTime::from_unix_timestamp_nanos(0).unwrap();
let t0 = time::PrimitiveDateTime::new(t0.date(), t0.time());
let t0_chrono = convert_time_to_chrono(t0);
db.insert_extension_versions(
&[
@ -34,6 +35,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into()],
repository: "ext1/repo".into(),
schema_version: 1,
wasm_api_version: None,
published_at: t0,
},
NewExtensionVersion {
@ -43,6 +45,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into(), "marshall".into()],
repository: "ext1/repo".into(),
schema_version: 1,
wasm_api_version: None,
published_at: t0,
},
],
@ -56,6 +59,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["marshall".into()],
repository: "ext2/repo".into(),
schema_version: 0,
wasm_api_version: None,
published_at: t0,
}],
),
@ -84,22 +88,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext1".into(),
name: "Extension One".into(),
version: "0.0.2".into(),
authors: vec!["max".into(), "marshall".into()],
description: "a good extension".into(),
repository: "ext1/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension One".into(),
version: "0.0.2".into(),
authors: vec!["max".into(), "marshall".into()],
description: Some("a good extension".into()),
repository: "ext1/repo".into(),
schema_version: Some(1),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 0,
},
ExtensionMetadata {
id: "ext2".into(),
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
repository: "ext2/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 0
},
]
@ -111,12 +123,16 @@ async fn test_extensions(db: &Arc<Database>) {
extensions,
&[ExtensionMetadata {
id: "ext2".into(),
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
repository: "ext2/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 0
},]
);
@ -147,22 +163,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext2".into(),
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
repository: "ext2/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 7
},
ExtensionMetadata {
id: "ext1".into(),
name: "Extension One".into(),
version: "0.0.2".into(),
authors: vec!["max".into(), "marshall".into()],
description: "a good extension".into(),
repository: "ext1/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension One".into(),
version: "0.0.2".into(),
authors: vec!["max".into(), "marshall".into()],
description: Some("a good extension".into()),
repository: "ext1/repo".into(),
schema_version: Some(1),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 5,
},
]
@ -181,6 +205,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["max".into(), "marshall".into()],
repository: "ext1/repo".into(),
schema_version: 1,
wasm_api_version: None,
published_at: t0,
}],
),
@ -193,6 +218,7 @@ async fn test_extensions(db: &Arc<Database>) {
authors: vec!["marshall".into()],
repository: "ext2/repo".into(),
schema_version: 0,
wasm_api_version: None,
published_at: t0,
}],
),
@ -223,22 +249,30 @@ async fn test_extensions(db: &Arc<Database>) {
&[
ExtensionMetadata {
id: "ext2".into(),
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: "a great extension".into(),
repository: "ext2/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension Two".into(),
version: "0.2.0".into(),
authors: vec!["marshall".into()],
description: Some("a great extension".into()),
repository: "ext2/repo".into(),
schema_version: Some(0),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 7
},
ExtensionMetadata {
id: "ext1".into(),
name: "Extension One".into(),
version: "0.0.3".into(),
authors: vec!["max".into(), "marshall".into()],
description: "a real good extension".into(),
repository: "ext1/repo".into(),
published_at: t0,
manifest: rpc::ExtensionApiManifest {
name: "Extension One".into(),
version: "0.0.3".into(),
authors: vec!["max".into(), "marshall".into()],
description: Some("a real good extension".into()),
repository: "ext1/repo".into(),
schema_version: Some(1),
wasm_api_version: None,
},
published_at: t0_chrono,
download_count: 5,
},
]