diff --git a/crates/extension/src/wasm_host/wit/since_v0_0_7.rs b/crates/extension/src/wasm_host/wit/since_v0_0_7.rs index ca52f1d473..e5cf38c076 100644 --- a/crates/extension/src/wasm_host/wit/since_v0_0_7.rs +++ b/crates/extension/src/wasm_host/wit/since_v0_0_7.rs @@ -5,9 +5,10 @@ use anyhow::{anyhow, bail, Context, Result}; use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; use async_trait::async_trait; -use futures::AsyncReadExt; use futures::{io::BufReader, FutureExt as _}; +use futures::{lock::Mutex, AsyncReadExt}; use indexed_docs::IndexedDocsDatabase; +use isahc::config::{Configurable, RedirectPolicy}; use language::{ language_settings::AllLanguageSettings, LanguageServerBinaryStatus, LspAdapterDelegate, }; @@ -30,7 +31,8 @@ wasmtime::component::bindgen!({ path: "../extension_api/wit/since_v0.0.7", with: { "worktree": ExtensionWorktree, - "key-value-store": ExtensionKeyValueStore + "key-value-store": ExtensionKeyValueStore, + "zed:extension/http-client/http-response-stream": ExtensionHttpResponseStream }, }); @@ -41,8 +43,8 @@ mod settings { } pub type ExtensionWorktree = Arc; - pub type ExtensionKeyValueStore = Arc; +pub type ExtensionHttpResponseStream = Arc>>; pub fn linker() -> &'static Linker { static LINKER: OnceLock> = OnceLock::new(); @@ -130,35 +132,123 @@ impl common::Host for WasmState {} impl http_client::Host for WasmState { async fn fetch( &mut self, - req: http_client::HttpRequest, + request: http_client::HttpRequest, ) -> wasmtime::Result> { maybe!(async { - let url = &req.url; - - let mut response = self - .host - .http_client - .get(url, AsyncBody::default(), true) - .await?; + let url = &request.url; + let request = convert_request(&request, true)?; + let mut response = self.host.http_client.send(request).await?; if response.status().is_client_error() || response.status().is_server_error() { bail!("failed to fetch '{url}': status code {}", response.status()) } - - let mut body = Vec::new(); - response - .body_mut() - .read_to_end(&mut body) - .await - .with_context(|| format!("failed to read response body from '{url}'"))?; - - Ok(http_client::HttpResponse { - body: String::from_utf8(body)?, - }) + convert_response(&mut response).await }) .await .to_wasmtime_result() } + + async fn fetch_stream( + &mut self, + request: http_client::HttpRequest, + ) -> wasmtime::Result, String>> { + let request = convert_request(&request, true)?; + let response = self.host.http_client.send(request); + maybe!(async { + let response = response.await?; + let stream = Arc::new(Mutex::new(response)); + let resource = self.table.push(stream)?; + Ok(resource) + }) + .await + .to_wasmtime_result() + } +} + +#[async_trait] +impl http_client::HostHttpResponseStream for WasmState { + async fn next_chunk( + &mut self, + resource: Resource, + ) -> wasmtime::Result>, String>> { + let stream = self.table.get(&resource)?.clone(); + maybe!(async move { + let mut response = stream.lock().await; + let mut buffer = vec![0; 8192]; // 8KB buffer + let bytes_read = response.body_mut().read(&mut buffer).await?; + if bytes_read == 0 { + Ok(None) + } else { + buffer.truncate(bytes_read); + Ok(Some(buffer)) + } + }) + .await + .to_wasmtime_result() + } + + fn drop(&mut self, _resource: Resource) -> Result<()> { + Ok(()) + } +} + +impl From for ::http_client::Method { + fn from(value: http_client::HttpMethod) -> Self { + match value { + http_client::HttpMethod::Get => Self::GET, + http_client::HttpMethod::Post => Self::POST, + http_client::HttpMethod::Put => Self::PUT, + http_client::HttpMethod::Delete => Self::DELETE, + http_client::HttpMethod::Head => Self::HEAD, + http_client::HttpMethod::Options => Self::OPTIONS, + http_client::HttpMethod::Patch => Self::PATCH, + } + } +} + +fn convert_request( + extension_request: &http_client::HttpRequest, + follow_redirects: bool, +) -> Result<::http_client::Request, anyhow::Error> { + let mut request = ::http_client::Request::builder() + .method(::http_client::Method::from(extension_request.method)) + .uri(&extension_request.url) + .redirect_policy(if follow_redirects { + RedirectPolicy::Follow + } else { + RedirectPolicy::None + }); + for (key, value) in &extension_request.headers { + request = request.header(key, value); + } + let body = extension_request + .body + .clone() + .map(AsyncBody::from) + .unwrap_or_default(); + request.body(body).map_err(anyhow::Error::from) +} + +async fn convert_response( + response: &mut ::http_client::Response, +) -> Result { + let mut extension_response = http_client::HttpResponse { + body: Vec::new(), + headers: Vec::new(), + }; + + for (key, value) in response.headers() { + extension_response + .headers + .push((key.to_string(), value.to_str().unwrap_or("").to_string())); + } + + response + .body_mut() + .read_to_end(&mut extension_response.body) + .await?; + + Ok(extension_response) } #[async_trait] diff --git a/crates/extension_api/src/extension_api.rs b/crates/extension_api/src/extension_api.rs index e07df1171d..ed5b0b22a0 100644 --- a/crates/extension_api/src/extension_api.rs +++ b/crates/extension_api/src/extension_api.rs @@ -19,7 +19,9 @@ pub use wit::{ github_release_by_tag_name, latest_github_release, GithubRelease, GithubReleaseAsset, GithubReleaseOptions, }, - zed::extension::http_client::{fetch, HttpRequest, HttpResponse}, + zed::extension::http_client::{ + fetch, fetch_stream, HttpMethod, HttpRequest, HttpResponse, HttpResponseStream, + }, zed::extension::nodejs::{ node_binary_path, npm_install_package, npm_package_installed_version, npm_package_latest_version, diff --git a/crates/extension_api/wit/since_v0.0.7/http-client.wit b/crates/extension_api/wit/since_v0.0.7/http-client.wit index e1f7b69d49..a2a847c72d 100644 --- a/crates/extension_api/wit/since_v0.0.7/http-client.wit +++ b/crates/extension_api/wit/since_v0.0.7/http-client.wit @@ -1,16 +1,45 @@ interface http-client { /// An HTTP request. record http-request { + /// The HTTP method for the request. + method: http-method, /// The URL to which the request should be made. url: string, + /// Headers for the request. + headers: list>, + /// The request body. + body: option>, + } + + /// HTTP methods. + enum http-method { + get, + post, + put, + delete, + head, + options, + patch, } /// An HTTP response. record http-response { + /// The response headers. + headers: list>, /// The response body. - body: string, + body: list, } /// Performs an HTTP request and returns the response. fetch: func(req: http-request) -> result; + + /// An HTTP response stream. + resource http-response-stream { + /// Retrieves the next chunk of data from the response stream. + /// Returns None if the stream has ended. + next-chunk: func() -> result>, string>; + } + + /// Performs an HTTP request and returns a response stream. + fetch-stream: func(req: http-request) -> result; } diff --git a/extensions/gleam/src/gleam.rs b/extensions/gleam/src/gleam.rs index 244bc8ec9b..a95231e15d 100644 --- a/extensions/gleam/src/gleam.rs +++ b/extensions/gleam/src/gleam.rs @@ -1,10 +1,10 @@ mod hexdocs; -use std::fs; +use std::{fs, io}; use zed::lsp::CompletionKind; use zed::{ - CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand, - SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection, + CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId, + SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection, }; use zed_extension_api::{self as zed, Result}; @@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension { let module_path = components.map(ToString::to_string).collect::>(); let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!( "https://hexdocs.pm/{package_name}{maybe_path}", maybe_path = if !module_path.is_empty() { @@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension { String::new() } ), + headers: vec![( + "User-Agent".to_string(), + "Zed (Gleam Extension)".to_string(), + )], + body: None, })?; - let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (markdown, _modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?; let mut text = String::new(); text.push_str(&markdown); diff --git a/extensions/gleam/src/hexdocs.rs b/extensions/gleam/src/hexdocs.rs index 26198fc5d1..07be1424da 100644 --- a/extensions/gleam/src/hexdocs.rs +++ b/extensions/gleam/src/hexdocs.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::collections::BTreeSet; -use std::io::Read; +use std::io::{self, Read}; use std::rc::Rc; use html_to_markdown::markdown::{ @@ -10,23 +10,36 @@ use html_to_markdown::{ convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter, StartTagOutcome, TagHandler, }; -use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result}; +use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result}; pub fn index(package: String, database: &KeyValueStore) -> Result<()> { + let headers = vec![( + "User-Agent".to_string(), + "Zed (Gleam Extension)".to_string(), + )]; + let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!("https://hexdocs.pm/{package}"), + headers: headers.clone(), + body: None, })?; - let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (package_root_markdown, modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?; database.insert(&package, &package_root_markdown)?; for module in modules { let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!("https://hexdocs.pm/{package}/{module}.html"), + headers: headers.clone(), + body: None, })?; - let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (markdown, _modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?; database.insert(&format!("{module} ({package})"), &markdown)?; }