Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Denormalize buffer operations (#9026)

Browse source 
This should significantly reduce database load on redeploy.

Co-Authored-By: Max <max@zed.dev>
Co-Authored-By: Nathan <nathan@zed.dev>

Release Notes:

- Reduced likelihood of being disconnected during deploys

Co-authored-by: Max <max@zed.dev>
Co-authored-by: Nathan <nathan@zed.dev>
This commit is contained in:
Conrad Irwin 2024-03-07 11:35:47 -07:00 • committed by GitHub
parent b5370cd15a
commit 86748a09e7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 62 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

89
crates/collab/src/db/queries/buffers.rs
View file

@ -558,6 +558,17 @@ impl Database {
lamport_timestamp: i32,
tx: &DatabaseTransaction,
) -> Result<()> {
buffer::Entity::update(buffer::ActiveModel {
id: ActiveValue::Unchanged(buffer_id),
epoch: ActiveValue::Unchanged(epoch),
latest_operation_epoch: ActiveValue::Set(Some(epoch)),
latest_operation_replica_id: ActiveValue::Set(Some(replica_id)),
latest_operation_lamport_timestamp: ActiveValue::Set(Some(lamport_timestamp)),
channel_id: ActiveValue::NotSet,
})
.exec(tx)
.await?;
use observed_buffer_edits::Column;
observed_buffer_edits::Entity::insert(observed_buffer_edits::ActiveModel {
user_id: ActiveValue::Set(user_id),
@ -711,7 +722,10 @@ impl Database {
buffer::ActiveModel {
id: ActiveValue::Unchanged(buffer.id),
epoch: ActiveValue::Set(epoch),
..Default::default()
latest_operation_epoch: ActiveValue::NotSet,
latest_operation_replica_id: ActiveValue::NotSet,
latest_operation_lamport_timestamp: ActiveValue::NotSet,
channel_id: ActiveValue::NotSet,
}
.save(tx)
.await?;
@ -745,30 +759,6 @@ impl Database {
.await
}
pub async fn latest_channel_buffer_changes(
&self,
channel_ids_by_buffer_id: &HashMap<BufferId, ChannelId>,
tx: &DatabaseTransaction,
) -> Result<Vec<proto::ChannelBufferVersion>> {
let latest_operations = self
.get_latest_operations_for_buffers(channel_ids_by_buffer_id.keys().copied(), tx)
.await?;
Ok(latest_operations
.iter()
.flat_map(|op| {
Some(proto::ChannelBufferVersion {
channel_id: channel_ids_by_buffer_id.get(&op.buffer_id)?.to_proto(),
epoch: op.epoch as u64,
version: vec![proto::VectorClockEntry {
replica_id: op.replica_id as u32,
timestamp: op.lamport_timestamp as u32,
}],
})
})
.collect())
}
pub async fn observed_channel_buffer_changes(
&self,
channel_ids_by_buffer_id: &HashMap<BufferId, ChannelId>,
@ -798,55 +788,6 @@ impl Database {
})
.collect())
}
/// Returns the latest operations for the buffers with the specified IDs.
pub async fn get_latest_operations_for_buffers(
&self,
buffer_ids: impl IntoIterator<Item = BufferId>,
tx: &DatabaseTransaction,
) -> Result<Vec<buffer_operation::Model>> {
let mut values = String::new();
for id in buffer_ids {
if !values.is_empty() {
values.push_str(", ");
}
write!(&mut values, "({})", id).unwrap();
}
if values.is_empty() {
return Ok(Vec::default());
}
let sql = format!(
r#"
SELECT
*
FROM
(
SELECT
*,
row_number() OVER (
PARTITION BY buffer_id
ORDER BY
epoch DESC,
lamport_timestamp DESC,
replica_id DESC
) as row_number
FROM buffer_operations
WHERE
buffer_id in ({values})
) AS last_operations
WHERE
row_number = 1
"#,
);
let stmt = Statement::from_string(self.pool.get_database_backend(), sql);
Ok(buffer_operation::Entity::find()
.from_raw_sql(stmt)
.all(tx)
.await?)
}
}
fn operation_to_storage(

25
crates/collab/src/db/queries/channels.rs
View file

@ -1,5 +1,8 @@
use super::*;
use rpc::{proto::channel_member::Kind, ErrorCode, ErrorCodeExt};
use rpc::{
proto::{channel_member::Kind, ChannelBufferVersion, VectorClockEntry},
ErrorCode, ErrorCodeExt,
};
use sea_orm::TryGetableMany;
impl Database {
@ -625,6 +628,7 @@ impl Database {
let channel_ids = channels.iter().map(|c| c.id).collect::<Vec<_>>();
let mut channel_ids_by_buffer_id = HashMap::default();
let mut latest_buffer_versions: Vec<ChannelBufferVersion> = vec![];
let mut rows = buffer::Entity::find()
.filter(buffer::Column::ChannelId.is_in(channel_ids.iter().copied()))
.stream(tx)
@ -632,13 +636,24 @@ impl Database {
while let Some(row) = rows.next().await {
let row = row?;
channel_ids_by_buffer_id.insert(row.id, row.channel_id);
latest_buffer_versions.push(ChannelBufferVersion {
channel_id: row.channel_id.0 as u64,
epoch: row.latest_operation_epoch.unwrap_or_default() as u64,
version: if let Some((latest_lamport_timestamp, latest_replica_id)) = row
.latest_operation_lamport_timestamp
.zip(row.latest_operation_replica_id)
{
vec![VectorClockEntry {
timestamp: latest_lamport_timestamp as u32,
replica_id: latest_replica_id as u32,
}]
} else {
vec![]
},
});
}
drop(rows);
let latest_buffer_versions = self
.latest_channel_buffer_changes(&channel_ids_by_buffer_id, tx)
.await?;
let latest_channel_messages = self.latest_channel_messages(&channel_ids, tx).await?;
let observed_buffer_versions = self