Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove non-determinism from Peer caused by smol's timeout helper

Browse source
This commit is contained in:
Antonio Scandurra 2022-06-22 18:39:12 +02:00
parent 071d940a88
commit f2d134917e
2 changed files with 34 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/collab/src/integration_tests.rs
View file

@ -4722,7 +4722,7 @@ async fn test_random_collaboration(
op_start_signals.remove(guest_ix); op_start_signals.remove(guest_ix);
server.forbid_connections(); server.forbid_connections();
server.disconnect_client(removed_guest_id); server.disconnect_client(removed_guest_id);
deterministic.advance_clock(5 * RECEIVE_TIMEOUT); deterministic.advance_clock(RECEIVE_TIMEOUT);
deterministic.start_waiting(); deterministic.start_waiting();
let (guest, guest_project, mut guest_cx, guest_err) = guest.await; let (guest, guest_project, mut guest_cx, guest_err) = guest.await;
deterministic.finish_waiting(); deterministic.finish_waiting();

59
crates/rpc/src/peer.rs
View file

@ -11,7 +11,6 @@ use futures::{
}; };
use parking_lot::{Mutex, RwLock}; use parking_lot::{Mutex, RwLock};
use serde::{ser::SerializeStruct, Serialize}; use serde::{ser::SerializeStruct, Serialize};
use smol_timeout::TimeoutExt;
use std::sync::atomic::Ordering::SeqCst; use std::sync::atomic::Ordering::SeqCst;
use std::{ use std::{
fmt, fmt,
@ -177,14 +176,17 @@ impl Peer {
outgoing = outgoing_rx.next().fuse() => match outgoing { outgoing = outgoing_rx.next().fuse() => match outgoing {
Some(outgoing) => { Some(outgoing) => {
tracing::debug!(%connection_id, "outgoing rpc message: writing"); tracing::debug!(%connection_id, "outgoing rpc message: writing");
if let Some(result) = writer.write(outgoing).timeout(WRITE_TIMEOUT).await { futures::select_biased! {
tracing::debug!(%connection_id, "outgoing rpc message: done writing"); result = writer.write(outgoing).fuse() => {
result.context("failed to write RPC message")?; tracing::debug!(%connection_id, "outgoing rpc message: done writing");
tracing::debug!(%connection_id, "keepalive interval: resetting after sending message"); result.context("failed to write RPC message")?;
keepalive_timer.set(create_timer(KEEPALIVE_INTERVAL).fuse()); tracing::debug!(%connection_id, "keepalive interval: resetting after sending message");
} else { keepalive_timer.set(create_timer(KEEPALIVE_INTERVAL).fuse());
tracing::debug!(%connection_id, "outgoing rpc message: writing timed out"); }
Err(anyhow!("timed out writing message"))?; _ = create_timer(WRITE_TIMEOUT).fuse() => {
tracing::debug!(%connection_id, "outgoing rpc message: writing timed out");
Err(anyhow!("timed out writing message"))?;
}
} }
} }
None => { None => {
@ -199,32 +201,37 @@ impl Peer {
receive_timeout.set(create_timer(RECEIVE_TIMEOUT).fuse()); receive_timeout.set(create_timer(RECEIVE_TIMEOUT).fuse());
if let proto::Message::Envelope(incoming) = incoming { if let proto::Message::Envelope(incoming) = incoming {
tracing::debug!(%connection_id, "incoming rpc message: processing"); tracing::debug!(%connection_id, "incoming rpc message: processing");
match incoming_tx.send(incoming).timeout(RECEIVE_TIMEOUT).await { futures::select_biased! {
Some(Ok(_)) => { result = incoming_tx.send(incoming).fuse() => match result {
tracing::debug!(%connection_id, "incoming rpc message: processed"); Ok(_) => {
tracing::debug!(%connection_id, "incoming rpc message: processed");
}
Err(_) => {
tracing::debug!(%connection_id, "incoming rpc message: channel closed");
return Ok(())
}
}, },
Some(Err(_)) => { _ = create_timer(WRITE_TIMEOUT).fuse() => {
tracing::debug!(%connection_id, "incoming rpc message: channel closed");
return Ok(())
},
None => {
tracing::debug!(%connection_id, "incoming rpc message: processing timed out"); tracing::debug!(%connection_id, "incoming rpc message: processing timed out");
Err(anyhow!("timed out processing incoming message"))? Err(anyhow!("timed out processing incoming message"))?
}, }
} }
} }
break; break;
}, },
_ = keepalive_timer => { _ = keepalive_timer => {
tracing::debug!(%connection_id, "keepalive interval: pinging"); tracing::debug!(%connection_id, "keepalive interval: pinging");
if let Some(result) = writer.write(proto::Message::Ping).timeout(WRITE_TIMEOUT).await { futures::select_biased! {
tracing::debug!(%connection_id, "keepalive interval: done pinging"); result = writer.write(proto::Message::Ping).fuse() => {
result.context("failed to send keepalive")?; tracing::debug!(%connection_id, "keepalive interval: done pinging");
tracing::debug!(%connection_id, "keepalive interval: resetting after pinging"); result.context("failed to send keepalive")?;
keepalive_timer.set(create_timer(KEEPALIVE_INTERVAL).fuse()); tracing::debug!(%connection_id, "keepalive interval: resetting after pinging");
} else { keepalive_timer.set(create_timer(KEEPALIVE_INTERVAL).fuse());
tracing::debug!(%connection_id, "keepalive interval: pinging timed out"); }
Err(anyhow!("timed out sending keepalive"))?; _ = create_timer(WRITE_TIMEOUT).fuse() => {
tracing::debug!(%connection_id, "keepalive interval: pinging timed out");
Err(anyhow!("timed out sending keepalive"))?;
}
} }
} }
_ = receive_timeout => { _ = receive_timeout => {