Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replace isahc with async ureq (#18414)

Browse source 
REplace isahc with ureq everywhere gpui is used.

This should allow us to make http requests without libssl; and avoid a
long-tail of panics caused by ishac.

Release Notes:

- (potentially breaking change) updated our http client

---------

Co-authored-by: Mikayla <mikayla@zed.dev>
This commit is contained in:
Conrad Irwin 2024-10-03 01:00:48 +05:30 • committed by GitHub
parent f809787275
commit 3a5deb5c6f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 1068 additions and 401 deletions
Download patch file Download diff file Expand all files Collapse all files

612
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

17
Cargo.toml
View file

@ -7,6 +7,7 @@ members = [
"crates/assistant",
"crates/assistant_slash_command",
"crates/assistant_tool",
"crates/ureq_client",
"crates/audio",
"crates/auto_update",
"crates/breadcrumbs",
@ -52,7 +53,6 @@ members = [
"crates/indexed_docs",
"crates/inline_completion_button",
"crates/install_cli",
"crates/isahc_http_client",
"crates/journal",
"crates/language",
"crates/language_model",
@ -87,6 +87,7 @@ members = [
"crates/release_channel",
"crates/remote",
"crates/remote_server",
"crates/reqwest_client",
"crates/repl",
"crates/rich_text",
"crates/rope",
@ -186,6 +187,8 @@ assets = { path = "crates/assets" }
assistant = { path = "crates/assistant" }
assistant_slash_command = { path = "crates/assistant_slash_command" }
assistant_tool = { path = "crates/assistant_tool" }
ureq_client = { path = "crates/ureq_client" }
async-compat = { version = "0.2.1" }
audio = { path = "crates/audio" }
auto_update = { path = "crates/auto_update" }
breadcrumbs = { path = "crates/breadcrumbs" }
@ -228,7 +231,6 @@ image_viewer = { path = "crates/image_viewer" }
indexed_docs = { path = "crates/indexed_docs" }
inline_completion_button = { path = "crates/inline_completion_button" }
install_cli = { path = "crates/install_cli" }
isahc_http_client = { path = "crates/isahc_http_client" }
journal = { path = "crates/journal" }
language = { path = "crates/language" }
language_model = { path = "crates/language_model" }
@ -265,6 +267,7 @@ release_channel = { path = "crates/release_channel" }
remote = { path = "crates/remote" }
remote_server = { path = "crates/remote_server" }
repl = { path = "crates/repl" }
reqwest_client = { path = "crates/reqwest_client" }
rich_text = { path = "crates/rich_text" }
rope = { path = "crates/rope" }
rpc = { path = "crates/rpc" }
@ -325,7 +328,7 @@ async-pipe = { git = "https://github.com/zed-industries/async-pipe-rs", rev = "8
async-recursion = "1.0.0"
async-tar = "0.5.0"
async-trait = "0.1"
async-tungstenite = "0.23"
async-tungstenite = "0.28"
async-watch = "0.3.1"
async_zip = { version = "0.0.17", features = ["deflate", "deflate64"] }
base64 = "0.22"
@ -364,10 +367,7 @@ ignore = "0.4.22"
image = "0.25.1"
indexmap = { version = "1.6.2", features = ["serde"] }
indoc = "2"
# We explicitly disable http2 support in isahc.
isahc = { version = "1.7.2", default-features = false, features = [
"text-decoding",
] }
itertools = "0.13.0"
jsonwebtoken = "9.3"
libc = "0.2"
@ -392,13 +392,14 @@ pulldown-cmark = { version = "0.12.0", default-features = false }
rand = "0.8.5"
regex = "1.5"
repair_json = "0.1.0"
reqwest = { git = "https://github.com/zed-industries/reqwest.git", rev = "fd110f6998da16bbca97b6dddda9be7827c50e29" }
rsa = "0.9.6"
runtimelib = { version = "0.15", default-features = false, features = [
"async-dispatcher-runtime",
] }
rustc-demangle = "0.1.23"
rust-embed = { version = "8.4", features = ["include-exclude"] }
rustls = "0.20.3"
rustls = "0.21.12"
rustls-native-certs = "0.8.0"
schemars = { version = "0.8", features = ["impl_json_schema"] }
semver = "1.0"

3
crates/client/Cargo.toml
View file

@ -18,6 +18,7 @@ test-support = ["clock/test-support", "collections/test-support", "gpui/test-sup
[dependencies]
anyhow.workspace = true
async-recursion = "0.3"
async-tls = "0.13"
async-tungstenite = { workspace = true, features = ["async-std", "async-tls"] }
chrono = { workspace = true, features = ["serde"] }
clock.workspace = true
@ -34,8 +35,6 @@ postage.workspace = true
rand.workspace = true
release_channel.workspace = true
rpc = { workspace = true, features = ["gpui"] }
rustls.workspace = true
rustls-native-certs.workspace = true
schemars.workspace = true
serde.workspace = true
serde_json.workspace = true

27
crates/client/src/client.rs
View file

@ -1023,7 +1023,7 @@ impl Client {
&self,
http: Arc<HttpClientWithUrl>,
release_channel: Option<ReleaseChannel>,
) -> impl Future<Output = Result<Url>> {
) -> impl Future<Output = Result<url::Url>> {
#[cfg(any(test, feature = "test-support"))]
let url_override = self.rpc_url.read().clone();
@ -1117,7 +1117,7 @@ impl Client {
// for us from the RPC URL.
//
// Among other things, it will generate and set a `Sec-WebSocket-Key` header for us.
let mut request = rpc_url.into_client_request()?;
let mut request = IntoClientRequest::into_client_request(rpc_url.as_str())?;
// We then modify the request to add our desired headers.
let request_headers = request.headers_mut();
@ -1137,30 +1137,13 @@ impl Client {
match url_scheme {
Https => {
let client_config = {
let mut root_store = rustls::RootCertStore::empty();
let root_certs = rustls_native_certs::load_native_certs();
for error in root_certs.errors {
log::warn!("error loading native certs: {:?}", error);
}
root_store.add_parsable_certificates(
&root_certs
.certs
.into_iter()
.map(|cert| cert.as_ref().to_owned())
.collect::<Vec<_>>(),
);
rustls::ClientConfig::builder()
.with_safe_defaults()
.with_root_certificates(root_store)
.with_no_client_auth()
};
let (stream, _) =
async_tungstenite::async_tls::client_async_tls_with_connector(
request,
stream,
Some(client_config.into()),
Some(async_tls::TlsConnector::from(
http_client::TLS_CONFIG.clone(),
)),
)
.await?;
Ok(Connection::new(

2
crates/collab/Cargo.toml
View file

@ -37,7 +37,7 @@ futures.workspace = true
google_ai.workspace = true
hex.workspace = true
http_client.workspace = true
isahc_http_client.workspace = true
reqwest_client.workspace = true
jsonwebtoken.workspace = true
live_kit_server.workspace = true
log.workspace = true

40
crates/collab/src/api/events.rs
View file

@ -674,7 +674,7 @@ pub struct EditorEventRow {
copilot_enabled_for_language: bool,
historical_event: bool,
architecture: String,
is_staff: Option<bool>,
is_staff: bool,
major: Option<i32>,
minor: Option<i32>,
patch: Option<i32>,
@ -708,7 +708,7 @@ impl EditorEventRow {
installation_id: body.installation_id.clone().unwrap_or_default(),
session_id: body.session_id.clone(),
metrics_id: body.metrics_id.clone().unwrap_or_default(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
operation: event.operation,
file_extension: event.file_extension.unwrap_or_default(),
@ -741,7 +741,7 @@ pub struct InlineCompletionEventRow {
region_code: String,
city: String,
time: i64,
is_staff: Option<bool>,
is_staff: bool,
major: Option<i32>,
minor: Option<i32>,
patch: Option<i32>,
@ -772,7 +772,7 @@ impl InlineCompletionEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone().unwrap_or_default(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
file_extension: event.file_extension.unwrap_or_default(),
signed_in: wrapper.signed_in,
@ -800,7 +800,7 @@ pub struct CallEventRow {
// ClientEventBase
installation_id: String,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// CallEventRow
@ -832,7 +832,7 @@ impl CallEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone().unwrap_or_default(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
operation: event.operation,
room_id: event.room_id,
@ -856,7 +856,7 @@ pub struct AssistantEventRow {
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// AssistantEventRow
@ -891,7 +891,7 @@ impl AssistantEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
conversation_id: event.conversation_id.unwrap_or_default(),
kind: event.kind.to_string(),
@ -909,7 +909,7 @@ impl AssistantEventRow {
pub struct CpuEventRow {
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
usage_as_percentage: f32,
core_count: u32,
app_version: String,
@ -947,7 +947,7 @@ impl CpuEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
usage_as_percentage: event.usage_as_percentage,
core_count: event.core_count,
@ -970,7 +970,7 @@ pub struct MemoryEventRow {
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// MemoryEventRow
@ -1001,7 +1001,7 @@ impl MemoryEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
memory_in_bytes: event.memory_in_bytes,
virtual_memory_in_bytes: event.virtual_memory_in_bytes,
@ -1024,7 +1024,7 @@ pub struct AppEventRow {
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// AppEventRow
@ -1054,7 +1054,7 @@ impl AppEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
operation: event.operation,
}
@ -1076,7 +1076,7 @@ pub struct SettingEventRow {
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// SettingEventRow
setting: String,
@ -1106,7 +1106,7 @@ impl SettingEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
setting: event.setting,
value: event.value,
@ -1129,7 +1129,7 @@ pub struct ExtensionEventRow {
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// ExtensionEventRow
@ -1164,7 +1164,7 @@ impl ExtensionEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
extension_id: event.extension_id,
extension_version: event.version,
@ -1198,7 +1198,7 @@ pub struct ReplEventRow {
// ClientEventBase
installation_id: Option<String>,
session_id: Option<String>,
is_staff: Option<bool>,
is_staff: bool,
time: i64,
// ReplEventRow
@ -1230,7 +1230,7 @@ impl ReplEventRow {
os_version: body.os_version.clone().unwrap_or_default(),
installation_id: body.installation_id.clone(),
session_id: body.session_id.clone(),
is_staff: body.is_staff,
is_staff: body.is_staff.unwrap_or_default(),
time: time.timestamp_millis(),
kernel_language: event.kernel_language,
kernel_status: event.kernel_status,

12
crates/collab/src/llm.rs
View file

@ -22,7 +22,8 @@ use chrono::{DateTime, Duration, Utc};
use collections::HashMap;
use db::{usage_measure::UsageMeasure, ActiveUserCount, LlmDatabase};
use futures::{Stream, StreamExt as _};
use isahc_http_client::IsahcHttpClient;
use reqwest_client::ReqwestClient;
use rpc::ListModelsResponse;
use rpc::{
proto::Plan, LanguageModelProvider, PerformCompletionParams, EXPIRED_LLM_TOKEN_HEADER_NAME,
@ -43,7 +44,7 @@ pub struct LlmState {
pub config: Config,
pub executor: Executor,
pub db: Arc<LlmDatabase>,
pub http_client: IsahcHttpClient,
pub http_client: ReqwestClient,
pub clickhouse_client: Option<clickhouse::Client>,
active_user_count_by_model:
RwLock<HashMap<(LanguageModelProvider, String), (DateTime<Utc>, ActiveUserCount)>>,
@ -69,11 +70,8 @@ impl LlmState {
let db = Arc::new(db);
let user_agent = format!("Zed Server/{}", env!("CARGO_PKG_VERSION"));
let http_client = IsahcHttpClient::builder()
.default_header("User-Agent", user_agent)
.build()
.map(IsahcHttpClient::from)
.context("failed to construct http client")?;
let http_client =
ReqwestClient::user_agent(&user_agent).context("failed to construct http client")?;
let this = Self {
executor,

6
crates/collab/src/rpc.rs
View file

@ -36,8 +36,8 @@ use collections::{HashMap, HashSet};
pub use connection_pool::{ConnectionPool, ZedVersion};
use core::fmt::{self, Debug, Formatter};
use http_client::HttpClient;
use isahc_http_client::IsahcHttpClient;
use open_ai::{OpenAiEmbeddingModel, OPEN_AI_API_URL};
use reqwest_client::ReqwestClient;
use sha2::Digest;
use supermaven_api::{CreateExternalUserRequest, SupermavenAdminApi};
@ -954,8 +954,8 @@ impl Server {
tracing::info!("connection opened");
let user_agent = format!("Zed Server/{}", env!("CARGO_PKG_VERSION"));
let http_client = match IsahcHttpClient::builder().default_header("User-Agent", user_agent).build() {
Ok(http_client) => Arc::new(IsahcHttpClient::from(http_client)),
let http_client = match ReqwestClient::user_agent(&user_agent) {
Ok(http_client) => Arc::new(http_client),
Err(error) => {
tracing::error!(?error, "failed to create HTTP client");
return;

2
crates/evals/Cargo.toml
View file

@ -16,6 +16,7 @@ path = "src/eval.rs"
[dependencies]
clap.workspace = true
anyhow.workspace = true
ureq_client.workspace = true
client.workspace = true
clock.workspace = true
collections.workspace = true
@ -24,7 +25,6 @@ feature_flags.workspace = true
fs.workspace = true
git.workspace = true
gpui.workspace = true
isahc_http_client.workspace = true
language.workspace = true
languages.workspace = true
http_client.workspace = true

7
crates/evals/src/eval.rs
View file

@ -32,6 +32,7 @@ use std::{
Arc,
},
};
use ureq_client::UreqClient;
const CODESEARCH_NET_DIR: &'static str = "target/datasets/code-search-net";
const EVAL_REPOS_DIR: &'static str = "target/datasets/eval-repos";
@ -100,7 +101,11 @@ fn main() -> Result<()> {
gpui::App::headless().run(move |cx| {
let executor = cx.background_executor().clone();
let client = isahc_http_client::IsahcHttpClient::new(None, None);
let client = Arc::new(UreqClient::new(
None,
"Zed LLM evals".to_string(),
executor.clone(),
));
cx.set_http_client(client.clone());
match cli.command {
Commands::Fetch {} => {

4
crates/extension/Cargo.toml
View file

@ -56,10 +56,12 @@ task.workspace = true
serde_json_lenient.workspace = true
[dev-dependencies]
isahc_http_client.workspace = true
ureq_client.workspace = true
ctor.workspace = true
env_logger.workspace = true
parking_lot.workspace = true
reqwest_client.workspace = true
tokio.workspace = true
fs = { workspace = true, features = ["test-support"] }
gpui = { workspace = true, features = ["test-support"] }

2
crates/extension/src/extension_builder.rs
View file

@ -25,7 +25,7 @@ use wit_component::ComponentEncoder;
/// Once Rust 1.78 is released, there will be a `wasm32-wasip2` target available, so we will
/// not need the adapter anymore.
const RUST_TARGET: &str = "wasm32-wasip1";
const WASI_ADAPTER_URL: &str =
pub const WASI_ADAPTER_URL: &str =
"https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.reactor.wasm";
/// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc

52
crates/extension/src/extension_store_test.rs
View file

@ -1,3 +1,4 @@
use crate::extension_builder::WASI_ADAPTER_URL;
use crate::extension_manifest::SchemaVersion;
use crate::extension_settings::ExtensionSettings;
use crate::{
@ -11,14 +12,14 @@ use collections::BTreeMap;
use fs::{FakeFs, Fs, RealFs};
use futures::{io::BufReader, AsyncReadExt, StreamExt};
use gpui::{Context, SemanticVersion, TestAppContext};
use http_client::{FakeHttpClient, Response};
use http_client::{AsyncBody, FakeHttpClient, HttpClient, Response};
use indexed_docs::IndexedDocsRegistry;
use isahc_http_client::IsahcHttpClient;
use language::{LanguageMatcher, LanguageRegistry, LanguageServerBinaryStatus, LanguageServerName};
use node_runtime::NodeRuntime;
use parking_lot::Mutex;
use project::{Project, DEFAULT_COMPLETION_CONTEXT};
use release_channel::AppVersion;
use reqwest_client::ReqwestClient;
use serde_json::json;
use settings::{Settings as _, SettingsStore};
use snippet_provider::SnippetRegistry;
@ -28,6 +29,7 @@ use std::{
sync::Arc,
};
use theme::ThemeRegistry;
use ureq_client::UreqClient;
use util::test::temp_tree;
#[cfg(test)]
@ -576,7 +578,7 @@ async fn test_extension_store_with_test_extension(cx: &mut TestAppContext) {
std::env::consts::ARCH
)
});
let builder_client = IsahcHttpClient::new(None, Some(user_agent));
let builder_client = Arc::new(UreqClient::new(None, user_agent, cx.executor().clone()));
let extension_store = cx.new_model(|cx| {
ExtensionStore::new(
@ -769,6 +771,50 @@ async fn test_extension_store_with_test_extension(cx: &mut TestAppContext) {
assert!(fs.metadata(&expected_server_path).await.unwrap().is_none());
}
#[gpui::test]
async fn test_wasi_adapter_download(cx: &mut TestAppContext) {
let client = Arc::new(UreqClient::new(
None,
"zed-test-wasi-adapter-download".to_string(),
cx.executor().clone(),
));
let mut response = client
.get(WASI_ADAPTER_URL, AsyncBody::default(), true)
.await
.unwrap();
let mut content = Vec::new();
let mut body = BufReader::new(response.body_mut());
body.read_to_end(&mut content).await.unwrap();
assert!(wasmparser::Parser::is_core_wasm(&content));
assert_eq!(content.len(), 96801); // Determined by downloading this to my computer
wit_component::ComponentEncoder::default()
.adapter("wasi_snapshot_preview1", &content)
.unwrap();
}
#[tokio::test]
async fn test_wasi_adapter_download_tokio() {
let client = Arc::new(ReqwestClient::new());
let mut response = client
.get(WASI_ADAPTER_URL, AsyncBody::default(), true)
.await
.unwrap();
let mut content = Vec::new();
let mut body = BufReader::new(response.body_mut());
body.read_to_end(&mut content).await.unwrap();
assert!(wasmparser::Parser::is_core_wasm(&content));
assert_eq!(content.len(), 96801); // Determined by downloading this to my computer
wit_component::ComponentEncoder::default()
.adapter("wasi_snapshot_preview1", &content)
.unwrap();
}
fn init_test(cx: &mut TestAppContext) {
cx.update(|cx| {
let store = SettingsStore::test(cx);

2
crates/extension_cli/Cargo.toml
View file

@ -18,7 +18,7 @@ clap = { workspace = true, features = ["derive"] }
env_logger.workspace = true
extension = { workspace = true, features = ["no-webrtc"] }
fs.workspace = true
isahc_http_client.workspace = true
reqwest_client.workspace = true
language.workspace = true
log.workspace = true
rpc.workspace = true

9
crates/extension_cli/src/main.rs
View file

@ -13,8 +13,8 @@ use extension::{
extension_builder::{CompileExtensionOptions, ExtensionBuilder},
ExtensionManifest,
};
use isahc_http_client::IsahcHttpClient;
use language::LanguageConfig;
use reqwest_client::ReqwestClient;
use theme::ThemeRegistry;
use tree_sitter::{Language, Query, WasmStore};
@ -66,12 +66,7 @@ async fn main() -> Result<()> {
std::env::consts::OS,
std::env::consts::ARCH
);
let http_client = Arc::new(
IsahcHttpClient::builder()
.default_header("User-Agent", user_agent)
.build()
.map(IsahcHttpClient::from)?,
);
let http_client = Arc::new(ReqwestClient::user_agent(&user_agent)?);
let builder = ExtensionBuilder::new(http_client, scratch_dir);
builder

4
crates/http_client/Cargo.toml
View file

@ -16,7 +16,9 @@ path = "src/http_client.rs"
doctest = true
[dependencies]
http = "0.2"
http = "1.1"
rustls.workspace = true
rustls-native-certs.workspace = true
anyhow.workspace = true
derive_more.workspace = true
futures.workspace = true

29
crates/http_client/src/http_client.rs
View file

@ -11,13 +11,21 @@ use http::request::Builder;
#[cfg(feature = "test-support")]
use std::fmt;
use std::{
sync::{Arc, Mutex},
sync::{Arc, LazyLock, Mutex},
time::Duration,
};
pub use url::Url;
#[derive(Clone)]
pub struct ReadTimeout(pub Duration);
#[derive(Default, Debug, Clone)]
impl Default for ReadTimeout {
fn default() -> Self {
Self(Duration::from_secs(5))
}
}
#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)]
pub enum RedirectPolicy {
#[default]
NoFollow,
@ -26,6 +34,23 @@ pub enum RedirectPolicy {
}
pub struct FollowRedirects(pub bool);
pub static TLS_CONFIG: LazyLock<Arc<rustls::ClientConfig>> = LazyLock::new(|| {
let mut root_store = rustls::RootCertStore::empty();
let root_certs = rustls_native_certs::load_native_certs();
for error in root_certs.errors {
log::warn!("error loading native certs: {:?}", error);
}
root_store.add_parsable_certificates(&root_certs.certs);
Arc::new(
rustls::ClientConfig::builder()
.with_safe_defaults()
.with_root_certificates(root_store)
.with_no_client_auth(),
)
});
pub trait HttpRequestExt {
/// Set a read timeout on the request.
/// For isahc, this is the low_speed_timeout.

1
crates/isahc_http_client/LICENSE-APACHE
View file

@ -1 +0,0 @@
../../LICENSE-APACHE

105
crates/isahc_http_client/src/isahc_http_client.rs
View file

@ -1,105 +0,0 @@
use std::{mem, sync::Arc, time::Duration};
use futures::future::BoxFuture;
use util::maybe;
pub use isahc::config::Configurable;
pub struct IsahcHttpClient(isahc::HttpClient);
pub use http_client::*;
impl IsahcHttpClient {
pub fn new(proxy: Option<Uri>, user_agent: Option<String>) -> Arc<IsahcHttpClient> {
let mut builder = isahc::HttpClient::builder()
.connect_timeout(Duration::from_secs(5))
.low_speed_timeout(100, Duration::from_secs(5))
.proxy(proxy.clone());
if let Some(agent) = user_agent {
builder = builder.default_header("User-Agent", agent);
}
Arc::new(IsahcHttpClient(builder.build().unwrap()))
}
pub fn builder() -> isahc::HttpClientBuilder {
isahc::HttpClientBuilder::new()
}
}
impl From<isahc::HttpClient> for IsahcHttpClient {
fn from(client: isahc::HttpClient) -> Self {
Self(client)
}
}
impl HttpClient for IsahcHttpClient {
fn proxy(&self) -> Option<&Uri> {
None
}
fn send(
&self,
req: http_client::http::Request<http_client::AsyncBody>,
) -> BoxFuture<'static, Result<http_client::Response<http_client::AsyncBody>, anyhow::Error>>
{
let redirect_policy = req
.extensions()
.get::<http_client::RedirectPolicy>()
.cloned()
.unwrap_or_default();
let read_timeout = req
.extensions()
.get::<http_client::ReadTimeout>()
.map(|t| t.0);
let req = maybe!({
let (mut parts, body) = req.into_parts();
let mut builder = isahc::Request::builder()
.method(parts.method)
.uri(parts.uri)
.version(parts.version);
if let Some(read_timeout) = read_timeout {
builder = builder.low_speed_timeout(100, read_timeout);
}
let headers = builder.headers_mut()?;
mem::swap(headers, &mut parts.headers);
let extensions = builder.extensions_mut()?;
mem::swap(extensions, &mut parts.extensions);
let isahc_body = match body.0 {
http_client::Inner::Empty => isahc::AsyncBody::empty(),
http_client::Inner::AsyncReader(reader) => isahc::AsyncBody::from_reader(reader),
http_client::Inner::SyncReader(reader) => {
isahc::AsyncBody::from_bytes_static(reader.into_inner())
}
};
builder
.redirect_policy(match redirect_policy {
http_client::RedirectPolicy::FollowAll => isahc::config::RedirectPolicy::Follow,
http_client::RedirectPolicy::FollowLimit(limit) => {
isahc::config::RedirectPolicy::Limit(limit)
}
http_client::RedirectPolicy::NoFollow => isahc::config::RedirectPolicy::None,
})
.body(isahc_body)
.ok()
});
let client = self.0.clone();
Box::pin(async move {
match req {
Some(req) => client
.send_async(req)
.await
.map_err(Into::into)
.map(|response| {
let (parts, body) = response.into_parts();
let body = http_client::AsyncBody::from_reader(body);
http_client::Response::from_parts(parts, body)
}),
None => Err(anyhow::anyhow!("Request was malformed")),
}
})
}
}

2
crates/live_kit_server/Cargo.toml
View file

@ -20,7 +20,7 @@ jsonwebtoken.workspace = true
log.workspace = true
prost.workspace = true
prost-types.workspace = true
reqwest = "0.11"
reqwest.workspace = true
serde.workspace = true
[build-dependencies]

31
crates/reqwest_client/Cargo.toml Normal file
View file

@ -0,0 +1,31 @@
[package]
name = "reqwest_client"
version = "0.1.0"
edition = "2021"
publish = false
license = "Apache-2.0"
[lints]
workspace = true
[features]
test-support = []
[lib]
path = "src/reqwest_client.rs"
doctest = true
[[example]]
name = "client"
path = "examples/client.rs"
[dependencies]
anyhow.workspace = true
futures.workspace = true
serde.workspace = true
smol.workspace = true
http_client.workspace = true
tokio.workspace = true
bytes = "1.0"
reqwest = { workspace = true, features = ["rustls-tls-manual-roots", "stream"] }

1
crates/reqwest_client/LICENSE-GPL Symbolic link
View file

@ -0,0 +1 @@
../../LICENSE-GPL

16
crates/reqwest_client/examples/client.rs Normal file
View file

@ -0,0 +1,16 @@
use futures::AsyncReadExt as _;
use http_client::AsyncBody;
use http_client::HttpClient;
use reqwest_client::ReqwestClient;
#[tokio::main]
async fn main() {
let resp = ReqwestClient::new()
.get("http://zed.dev", AsyncBody::empty(), true)
.await
.unwrap();
let mut body = String::new();
resp.into_body().read_to_string(&mut body).await.unwrap();
println!("{}", &body);
}

232
crates/reqwest_client/src/reqwest_client.rs Normal file
View file

@ -0,0 +1,232 @@
use std::{borrow::Cow, io::Read, pin::Pin, task::Poll};
use anyhow::anyhow;
use bytes::{BufMut, Bytes, BytesMut};
use futures::{AsyncRead, TryStreamExt};
use http_client::{http, AsyncBody, ReadTimeout};
use reqwest::header::{HeaderMap, HeaderValue};
use smol::future::FutureExt;
const DEFAULT_CAPACITY: usize = 4096;
pub struct ReqwestClient {
client: reqwest::Client,
}
impl ReqwestClient {
pub fn new() -> Self {
Self {
client: reqwest::Client::new(),
}
}
pub fn user_agent(agent: &str) -> anyhow::Result<Self> {
let mut map = HeaderMap::new();
map.insert(http::header::USER_AGENT, HeaderValue::from_str(agent)?);
Ok(Self {
client: reqwest::Client::builder().default_headers(map).build()?,
})
}
}
impl From<reqwest::Client> for ReqwestClient {
fn from(client: reqwest::Client) -> Self {
Self { client }
}
}
// This struct is essentially a re-implementation of
// https://docs.rs/tokio-util/0.7.12/tokio_util/io/struct.ReaderStream.html
// except outside of Tokio's aegis
struct ReaderStream {
reader: Option<Pin<Box<dyn futures::AsyncRead + Send + Sync>>>,
buf: BytesMut,
capacity: usize,
}
impl ReaderStream {
fn new(reader: Pin<Box<dyn futures::AsyncRead + Send + Sync>>) -> Self {
Self {
reader: Some(reader),
buf: BytesMut::new(),
capacity: DEFAULT_CAPACITY,
}
}
}
impl futures::Stream for ReaderStream {
type Item = std::io::Result<Bytes>;
fn poll_next(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> Poll<Option<Self::Item>> {
let mut this = self.as_mut();
let mut reader = match this.reader.take() {
Some(r) => r,
None => return Poll::Ready(None),
};
if this.buf.capacity() == 0 {
let capacity = this.capacity;
this.buf.reserve(capacity);
}
match poll_read_buf(&mut reader, cx, &mut this.buf) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(err)) => {
self.reader = None;
Poll::Ready(Some(Err(err)))
}
Poll::Ready(Ok(0)) => {
self.reader = None;
Poll::Ready(None)
}
Poll::Ready(Ok(_)) => {
let chunk = this.buf.split();
self.reader = Some(reader);
Poll::Ready(Some(Ok(chunk.freeze())))
}
}
}
}
/// Implementation from https://docs.rs/tokio-util/0.7.12/src/tokio_util/util/poll_buf.rs.html
/// Specialized for this use case
pub fn poll_read_buf(
io: &mut Pin<Box<dyn futures::AsyncRead + Send + Sync>>,
cx: &mut std::task::Context<'_>,
buf: &mut BytesMut,
) -> Poll<std::io::Result<usize>> {
if !buf.has_remaining_mut() {
return Poll::Ready(Ok(0));
}
let n = {
let dst = buf.chunk_mut();
// Safety: `chunk_mut()` returns a `&mut UninitSlice`, and `UninitSlice` is a
// transparent wrapper around `[MaybeUninit<u8>]`.
let dst = unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>]) };
let mut buf = tokio::io::ReadBuf::uninit(dst);
let ptr = buf.filled().as_ptr();
let unfilled_portion = buf.initialize_unfilled();
// SAFETY: Pin projection
let io_pin = unsafe { Pin::new_unchecked(io) };
std::task::ready!(io_pin.poll_read(cx, unfilled_portion)?);
// Ensure the pointer does not change from under us
assert_eq!(ptr, buf.filled().as_ptr());
buf.filled().len()
};
// Safety: This is guaranteed to be the number of initialized (and read)
// bytes due to the invariants provided by `ReadBuf::filled`.
unsafe {
buf.advance_mut(n);
}
Poll::Ready(Ok(n))
}
enum WrappedBodyInner {
None,
SyncReader(std::io::Cursor<Cow<'static, [u8]>>),
Stream(ReaderStream),
}
struct WrappedBody(WrappedBodyInner);
impl WrappedBody {
fn new(body: AsyncBody) -> Self {
match body.0 {
http_client::Inner::Empty => Self(WrappedBodyInner::None),
http_client::Inner::SyncReader(cursor) => Self(WrappedBodyInner::SyncReader(cursor)),
http_client::Inner::AsyncReader(pin) => {
Self(WrappedBodyInner::Stream(ReaderStream::new(pin)))
}
}
}
}
impl futures::stream::Stream for WrappedBody {
type Item = Result<Bytes, std::io::Error>;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
match &mut self.0 {
WrappedBodyInner::None => Poll::Ready(None),
WrappedBodyInner::SyncReader(cursor) => {
let mut buf = Vec::new();
match cursor.read_to_end(&mut buf) {
Ok(_) => {
return Poll::Ready(Some(Ok(Bytes::from(buf))));
}
Err(e) => return Poll::Ready(Some(Err(e))),
}
}
WrappedBodyInner::Stream(stream) => {
// SAFETY: Pin projection
let stream = unsafe { Pin::new_unchecked(stream) };
futures::Stream::poll_next(stream, cx)
}
}
}
}
impl http_client::HttpClient for ReqwestClient {
fn proxy(&self) -> Option<&http::Uri> {
None
}
fn send(
&self,
req: http::Request<http_client::AsyncBody>,
) -> futures::future::BoxFuture<
'static,
Result<http_client::Response<http_client::AsyncBody>, anyhow::Error>,
> {
let (parts, body) = req.into_parts();
let mut request = self.client.request(parts.method, parts.uri.to_string());
request = request.headers(parts.headers);
if let Some(redirect_policy) = parts.extensions.get::<http_client::RedirectPolicy>() {
request = request.redirect_policy(match redirect_policy {
http_client::RedirectPolicy::NoFollow => reqwest::redirect::Policy::none(),
http_client::RedirectPolicy::FollowLimit(limit) => {
reqwest::redirect::Policy::limited(*limit as usize)
}
http_client::RedirectPolicy::FollowAll => reqwest::redirect::Policy::limited(100),
});
}
if let Some(ReadTimeout(timeout)) = parts.extensions.get::<ReadTimeout>() {
request = request.timeout(*timeout);
}
let body = WrappedBody::new(body);
let request = request.body(reqwest::Body::wrap_stream(body));
async move {
let response = request.send().await.map_err(|e| anyhow!(e))?;
let status = response.status();
let mut builder = http::Response::builder().status(status.as_u16());
for (name, value) in response.headers() {
builder = builder.header(name, value);
}
let bytes = response.bytes_stream();
let bytes = bytes
.map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e))
.into_async_read();
let body = http_client::AsyncBody::from_reader(bytes);
builder.body(body).map_err(|e| anyhow!(e))
}
.boxed()
}
}

2
crates/semantic_index/Cargo.toml
View file

@ -51,7 +51,7 @@ workspace.workspace = true
worktree.workspace = true
[dev-dependencies]
isahc_http_client.workspace = true
ureq_client.workspace = true
env_logger.workspace = true
client = { workspace = true, features = ["test-support"] }
fs = { workspace = true, features = ["test-support"] }

7
crates/semantic_index/examples/index.rs
View file

@ -2,7 +2,6 @@ use client::Client;
use futures::channel::oneshot;
use gpui::App;
use http_client::HttpClientWithUrl;
use isahc_http_client::IsahcHttpClient;
use language::language_settings::AllLanguageSettings;
use project::Project;
use semantic_index::{OpenAiEmbeddingModel, OpenAiEmbeddingProvider, SemanticDb};
@ -29,7 +28,11 @@ fn main() {
let clock = Arc::new(FakeSystemClock::default());
let http = Arc::new(HttpClientWithUrl::new(
IsahcHttpClient::new(None, None),
Arc::new(ureq_client::UreqClient::new(
None,
"Zed semantic index example".to_string(),
cx.background_executor().clone(),
)),
"http://localhost:11434",
None,
));

16
crates/isahc_http_client/Cargo.toml → crates/ureq_client/Cargo.toml
View file

@ -1,5 +1,5 @@
[package]
name = "isahc_http_client"
name = "ureq_client"
version = "0.1.0"
edition = "2021"
publish = false
@ -12,11 +12,21 @@ workspace = true
test-support = []
[lib]
path = "src/isahc_http_client.rs"
path = "src/ureq_client.rs"
doctest = true
[[example]]
name = "client"
path = "examples/client.rs"
[dependencies]
anyhow.workspace = true
futures.workspace = true
serde.workspace = true
smol.workspace = true
gpui.workspace = true
http_client.workspace = true
isahc.workspace = true
util.workspace = true
parking_lot.workspace = true
ureq = "=2.9.1"

1
crates/ureq_client/LICENSE-GPL Symbolic link
View file

@ -0,0 +1 @@
../../LICENSE-GPL

24
crates/ureq_client/examples/client.rs Normal file
View file

@ -0,0 +1,24 @@
use futures::AsyncReadExt;
use http_client::{AsyncBody, HttpClient};
use ureq_client::UreqClient;
fn main() {
gpui::App::headless().run(|cx| {
println!("{:?}", std::thread::current().id());
cx.spawn(|cx| async move {
let resp = UreqClient::new(
None,
"Conrad's bot".to_string(),
cx.background_executor().clone(),
)
.get("http://zed.dev", AsyncBody::empty(), true)
.await
.unwrap();
let mut body = String::new();
resp.into_body().read_to_string(&mut body).await.unwrap();
println!("{}", body);
})
.detach();
})
}

187
crates/ureq_client/src/ureq_client.rs Normal file
View file

@ -0,0 +1,187 @@
use std::collections::HashMap;
use std::io::Read;
use std::sync::Arc;
use std::time::Duration;
use std::{pin::Pin, task::Poll};
use anyhow::Error;
use futures::channel::mpsc;
use futures::future::BoxFuture;
use futures::{AsyncRead, SinkExt, StreamExt};
use http_client::{http, AsyncBody, HttpClient, RedirectPolicy, Uri};
use smol::future::FutureExt;
use util::ResultExt;
pub struct UreqClient {
// Note in ureq 2.x the options are stored on the Agent.
// In ureq 3.x we'll be able to set these on the request.
// In practice it's probably "fine" to have many clients, the number of distinct options
// is low; and most requests to the same connection will have the same options so the
// connection pool will work.
clients: Arc<parking_lot::Mutex<HashMap<(Duration, RedirectPolicy), ureq::Agent>>>,
proxy_url: Option<Uri>,
proxy: Option<ureq::Proxy>,
user_agent: String,
background_executor: gpui::BackgroundExecutor,
}
impl UreqClient {
pub fn new(
proxy_url: Option<Uri>,
user_agent: String,
background_executor: gpui::BackgroundExecutor,
) -> Self {
Self {
clients: Arc::default(),
proxy_url: proxy_url.clone(),
proxy: proxy_url.and_then(|url| ureq::Proxy::new(url.to_string()).log_err()),
user_agent,
background_executor,
}
}
fn agent_for(&self, redirect_policy: RedirectPolicy, timeout: Duration) -> ureq::Agent {
let mut clients = self.clients.lock();
// in case our assumption of distinct options is wrong, we'll sporadically clean it out.
if clients.len() > 50 {
clients.clear()
}
clients
.entry((timeout, redirect_policy.clone()))
.or_insert_with(|| {
let mut builder = ureq::AgentBuilder::new()
.timeout_connect(Duration::from_secs(5))
.timeout_read(timeout)
.timeout_write(timeout)
.user_agent(&self.user_agent)
.tls_config(http_client::TLS_CONFIG.clone())
.redirects(match redirect_policy {
RedirectPolicy::NoFollow => 0,
RedirectPolicy::FollowLimit(limit) => limit,
RedirectPolicy::FollowAll => 100,
});
if let Some(proxy) = &self.proxy {
builder = builder.proxy(proxy.clone());
}
builder.build()
})
.clone()
}
}
impl HttpClient for UreqClient {
fn proxy(&self) -> Option<&Uri> {
self.proxy_url.as_ref()
}
fn send(
&self,
request: http::Request<AsyncBody>,
) -> BoxFuture<'static, Result<http::Response<AsyncBody>, Error>> {
let agent = self.agent_for(
request
.extensions()
.get::<RedirectPolicy>()
.cloned()
.unwrap_or_default(),
request
.extensions()
.get::<http_client::ReadTimeout>()
.cloned()
.unwrap_or_default()
.0,
);
let mut req = agent.request(&request.method().as_ref(), &request.uri().to_string());
for (name, value) in request.headers().into_iter() {
req = req.set(name.as_str(), value.to_str().unwrap());
}
let body = request.into_body();
let executor = self.background_executor.clone();
self.background_executor
.spawn(async move {
let response = req.send(body)?;
let mut builder = http::Response::builder()
.status(response.status())
.version(http::Version::HTTP_11);
for name in response.headers_names() {
if let Some(value) = response.header(&name) {
builder = builder.header(name, value);
}
}
let body = AsyncBody::from_reader(UreqResponseReader::new(executor, response));
let http_response = builder.body(body)?;
Ok(http_response)
})
.boxed()
}
}
struct UreqResponseReader {
receiver: mpsc::Receiver<std::io::Result<Vec<u8>>>,
buffer: Vec<u8>,
idx: usize,
_task: gpui::Task<()>,
}
impl UreqResponseReader {
fn new(background_executor: gpui::BackgroundExecutor, response: ureq::Response) -> Self {
let (mut sender, receiver) = mpsc::channel(1);
let mut reader = response.into_reader();
let task = background_executor.spawn(async move {
let mut buffer = vec![0; 8192];
loop {
let n = match reader.read(&mut buffer) {
Ok(0) => break,
Ok(n) => n,
Err(e) => {
let _ = sender.send(Err(e)).await;
break;
}
};
let _ = sender.send(Ok(buffer[..n].to_vec())).await;
}
});
UreqResponseReader {
_task: task,
receiver,
buffer: Vec::new(),
idx: 0,
}
}
}
impl AsyncRead for UreqResponseReader {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &mut [u8],
) -> Poll<std::io::Result<usize>> {
if self.buffer.is_empty() {
match self.receiver.poll_next_unpin(cx) {
Poll::Ready(Some(Ok(data))) => self.buffer = data,
Poll::Ready(Some(Err(e))) => {
return Poll::Ready(Err(e));
}
Poll::Ready(None) => {
return Poll::Ready(Ok(0));
}
Poll::Pending => {
return Poll::Pending;
}
}
}
let n = std::cmp::min(buf.len(), self.buffer.len() - self.idx);
buf[..n].copy_from_slice(&self.buffer[self.idx..self.idx + n]);
self.idx += n;
if self.idx == self.buffer.len() {
self.buffer.clear();
self.idx = 0;
}
Poll::Ready(Ok(n))
}
}

2
crates/vim/Cargo.toml
View file

@ -17,7 +17,7 @@ neovim = ["nvim-rs", "async-compat", "async-trait", "tokio"]
[dependencies]
anyhow.workspace = true
async-compat = { version = "0.2.1", "optional" = true }
async-compat = { workspace = true, "optional" = true }
async-trait = { workspace = true, "optional" = true }
collections.workspace = true
command_palette.workspace = true

2
crates/zed/Cargo.toml
View file

@ -57,7 +57,7 @@ http_client.workspace = true
image_viewer.workspace = true
inline_completion_button.workspace = true
install_cli.workspace = true
isahc_http_client.workspace = true
ureq_client.workspace = true
journal.workspace = true
language.workspace = true
language_model.workspace = true

10
crates/zed/src/main.rs
View file

@ -24,9 +24,9 @@ use gpui::{
UpdateGlobal as _, VisualContext,
};
use http_client::{read_proxy_from_env, Uri};
use isahc_http_client::IsahcHttpClient;
use language::LanguageRegistry;
use log::LevelFilter;
use ureq_client::UreqClient;
use assets::Assets;
use node_runtime::{NodeBinaryOptions, NodeRuntime};
@ -334,9 +334,7 @@ fn main() {
log::info!("========== starting zed ==========");
let app = App::new()
.with_assets(Assets)
.with_http_client(IsahcHttpClient::new(None, None));
let app = App::new().with_assets(Assets);
let system_id = app.background_executor().block(system_id()).ok();
let installation_id = app.background_executor().block(installation_id()).ok();
@ -470,8 +468,8 @@ fn main() {
.ok()
})
.or_else(read_proxy_from_env);
let http = IsahcHttpClient::new(proxy_url, Some(user_agent));
cx.set_http_client(http);
let http = UreqClient::new(proxy_url, user_agent, cx.background_executor().clone());
cx.set_http_client(Arc::new(http));
<dyn Fs>::set_global(fs.clone(), cx);