From fc64843dd55a7e6d937aa1a7b3a08337cf28e616 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 12 Aug 2024 08:36:49 -0600 Subject: [PATCH] Enhance HTTP API for extensions (#16067) # HTTP Client Improvements for Extension API This PR enhances the HTTP client functionality in the Zed extension API, providing more control over requests and allowing for streaming responses. ## Key Changes 1. Extended `HttpRequest` struct: - Added `method` field to specify HTTP method - Added `headers` field for custom headers - Added optional `body` field for request payload 2. Introduced `HttpMethod` enum for supported HTTP methods 3. Updated `HttpResponse` struct: - Added `headers` field to access response headers - Changed `body` type from `String` to `Vec` for binary data support 4. Added streaming support: - New `fetch_stream` function to get a response stream - Introduced `HttpResponseStream` resource for chunked reading 5. Updated internal implementations to support these new features 6. Modified the Gleam extension to use the new API structure ## Motivation These changes provide extension developers with more flexibility and control over HTTP requests. The streaming support is particularly useful for handling large responses efficiently or ideally streaming into the UI. ## Testing - [x] Updated existing tests - [ ] Added new tests for streaming functionality ## Next Steps - Consider adding more comprehensive examples in the documentation - Evaluate performance impact of streaming for large responses Please review and let me know if any adjustments are needed. Release Notes: - N/A --------- Co-authored-by: Marshall Bowers --- .../src/wasm_host/wit/since_v0_0_7.rs | 134 +++++++++++++++--- crates/extension_api/src/extension_api.rs | 4 +- .../wit/since_v0.0.7/http-client.wit | 31 +++- extensions/gleam/src/gleam.rs | 15 +- extensions/gleam/src/hexdocs.rs | 21 ++- 5 files changed, 173 insertions(+), 32 deletions(-) diff --git a/crates/extension/src/wasm_host/wit/since_v0_0_7.rs b/crates/extension/src/wasm_host/wit/since_v0_0_7.rs index ca52f1d473..e5cf38c076 100644 --- a/crates/extension/src/wasm_host/wit/since_v0_0_7.rs +++ b/crates/extension/src/wasm_host/wit/since_v0_0_7.rs @@ -5,9 +5,10 @@ use anyhow::{anyhow, bail, Context, Result}; use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; use async_trait::async_trait; -use futures::AsyncReadExt; use futures::{io::BufReader, FutureExt as _}; +use futures::{lock::Mutex, AsyncReadExt}; use indexed_docs::IndexedDocsDatabase; +use isahc::config::{Configurable, RedirectPolicy}; use language::{ language_settings::AllLanguageSettings, LanguageServerBinaryStatus, LspAdapterDelegate, }; @@ -30,7 +31,8 @@ wasmtime::component::bindgen!({ path: "../extension_api/wit/since_v0.0.7", with: { "worktree": ExtensionWorktree, - "key-value-store": ExtensionKeyValueStore + "key-value-store": ExtensionKeyValueStore, + "zed:extension/http-client/http-response-stream": ExtensionHttpResponseStream }, }); @@ -41,8 +43,8 @@ mod settings { } pub type ExtensionWorktree = Arc; - pub type ExtensionKeyValueStore = Arc; +pub type ExtensionHttpResponseStream = Arc>>; pub fn linker() -> &'static Linker { static LINKER: OnceLock> = OnceLock::new(); @@ -130,35 +132,123 @@ impl common::Host for WasmState {} impl http_client::Host for WasmState { async fn fetch( &mut self, - req: http_client::HttpRequest, + request: http_client::HttpRequest, ) -> wasmtime::Result> { maybe!(async { - let url = &req.url; - - let mut response = self - .host - .http_client - .get(url, AsyncBody::default(), true) - .await?; + let url = &request.url; + let request = convert_request(&request, true)?; + let mut response = self.host.http_client.send(request).await?; if response.status().is_client_error() || response.status().is_server_error() { bail!("failed to fetch '{url}': status code {}", response.status()) } - - let mut body = Vec::new(); - response - .body_mut() - .read_to_end(&mut body) - .await - .with_context(|| format!("failed to read response body from '{url}'"))?; - - Ok(http_client::HttpResponse { - body: String::from_utf8(body)?, - }) + convert_response(&mut response).await }) .await .to_wasmtime_result() } + + async fn fetch_stream( + &mut self, + request: http_client::HttpRequest, + ) -> wasmtime::Result, String>> { + let request = convert_request(&request, true)?; + let response = self.host.http_client.send(request); + maybe!(async { + let response = response.await?; + let stream = Arc::new(Mutex::new(response)); + let resource = self.table.push(stream)?; + Ok(resource) + }) + .await + .to_wasmtime_result() + } +} + +#[async_trait] +impl http_client::HostHttpResponseStream for WasmState { + async fn next_chunk( + &mut self, + resource: Resource, + ) -> wasmtime::Result>, String>> { + let stream = self.table.get(&resource)?.clone(); + maybe!(async move { + let mut response = stream.lock().await; + let mut buffer = vec![0; 8192]; // 8KB buffer + let bytes_read = response.body_mut().read(&mut buffer).await?; + if bytes_read == 0 { + Ok(None) + } else { + buffer.truncate(bytes_read); + Ok(Some(buffer)) + } + }) + .await + .to_wasmtime_result() + } + + fn drop(&mut self, _resource: Resource) -> Result<()> { + Ok(()) + } +} + +impl From for ::http_client::Method { + fn from(value: http_client::HttpMethod) -> Self { + match value { + http_client::HttpMethod::Get => Self::GET, + http_client::HttpMethod::Post => Self::POST, + http_client::HttpMethod::Put => Self::PUT, + http_client::HttpMethod::Delete => Self::DELETE, + http_client::HttpMethod::Head => Self::HEAD, + http_client::HttpMethod::Options => Self::OPTIONS, + http_client::HttpMethod::Patch => Self::PATCH, + } + } +} + +fn convert_request( + extension_request: &http_client::HttpRequest, + follow_redirects: bool, +) -> Result<::http_client::Request, anyhow::Error> { + let mut request = ::http_client::Request::builder() + .method(::http_client::Method::from(extension_request.method)) + .uri(&extension_request.url) + .redirect_policy(if follow_redirects { + RedirectPolicy::Follow + } else { + RedirectPolicy::None + }); + for (key, value) in &extension_request.headers { + request = request.header(key, value); + } + let body = extension_request + .body + .clone() + .map(AsyncBody::from) + .unwrap_or_default(); + request.body(body).map_err(anyhow::Error::from) +} + +async fn convert_response( + response: &mut ::http_client::Response, +) -> Result { + let mut extension_response = http_client::HttpResponse { + body: Vec::new(), + headers: Vec::new(), + }; + + for (key, value) in response.headers() { + extension_response + .headers + .push((key.to_string(), value.to_str().unwrap_or("").to_string())); + } + + response + .body_mut() + .read_to_end(&mut extension_response.body) + .await?; + + Ok(extension_response) } #[async_trait] diff --git a/crates/extension_api/src/extension_api.rs b/crates/extension_api/src/extension_api.rs index e07df1171d..ed5b0b22a0 100644 --- a/crates/extension_api/src/extension_api.rs +++ b/crates/extension_api/src/extension_api.rs @@ -19,7 +19,9 @@ pub use wit::{ github_release_by_tag_name, latest_github_release, GithubRelease, GithubReleaseAsset, GithubReleaseOptions, }, - zed::extension::http_client::{fetch, HttpRequest, HttpResponse}, + zed::extension::http_client::{ + fetch, fetch_stream, HttpMethod, HttpRequest, HttpResponse, HttpResponseStream, + }, zed::extension::nodejs::{ node_binary_path, npm_install_package, npm_package_installed_version, npm_package_latest_version, diff --git a/crates/extension_api/wit/since_v0.0.7/http-client.wit b/crates/extension_api/wit/since_v0.0.7/http-client.wit index e1f7b69d49..a2a847c72d 100644 --- a/crates/extension_api/wit/since_v0.0.7/http-client.wit +++ b/crates/extension_api/wit/since_v0.0.7/http-client.wit @@ -1,16 +1,45 @@ interface http-client { /// An HTTP request. record http-request { + /// The HTTP method for the request. + method: http-method, /// The URL to which the request should be made. url: string, + /// Headers for the request. + headers: list>, + /// The request body. + body: option>, + } + + /// HTTP methods. + enum http-method { + get, + post, + put, + delete, + head, + options, + patch, } /// An HTTP response. record http-response { + /// The response headers. + headers: list>, /// The response body. - body: string, + body: list, } /// Performs an HTTP request and returns the response. fetch: func(req: http-request) -> result; + + /// An HTTP response stream. + resource http-response-stream { + /// Retrieves the next chunk of data from the response stream. + /// Returns None if the stream has ended. + next-chunk: func() -> result>, string>; + } + + /// Performs an HTTP request and returns a response stream. + fetch-stream: func(req: http-request) -> result; } diff --git a/extensions/gleam/src/gleam.rs b/extensions/gleam/src/gleam.rs index 244bc8ec9b..a95231e15d 100644 --- a/extensions/gleam/src/gleam.rs +++ b/extensions/gleam/src/gleam.rs @@ -1,10 +1,10 @@ mod hexdocs; -use std::fs; +use std::{fs, io}; use zed::lsp::CompletionKind; use zed::{ - CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand, - SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection, + CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId, + SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection, }; use zed_extension_api::{self as zed, Result}; @@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension { let module_path = components.map(ToString::to_string).collect::>(); let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!( "https://hexdocs.pm/{package_name}{maybe_path}", maybe_path = if !module_path.is_empty() { @@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension { String::new() } ), + headers: vec![( + "User-Agent".to_string(), + "Zed (Gleam Extension)".to_string(), + )], + body: None, })?; - let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (markdown, _modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?; let mut text = String::new(); text.push_str(&markdown); diff --git a/extensions/gleam/src/hexdocs.rs b/extensions/gleam/src/hexdocs.rs index 26198fc5d1..07be1424da 100644 --- a/extensions/gleam/src/hexdocs.rs +++ b/extensions/gleam/src/hexdocs.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::collections::BTreeSet; -use std::io::Read; +use std::io::{self, Read}; use std::rc::Rc; use html_to_markdown::markdown::{ @@ -10,23 +10,36 @@ use html_to_markdown::{ convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter, StartTagOutcome, TagHandler, }; -use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result}; +use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result}; pub fn index(package: String, database: &KeyValueStore) -> Result<()> { + let headers = vec![( + "User-Agent".to_string(), + "Zed (Gleam Extension)".to_string(), + )]; + let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!("https://hexdocs.pm/{package}"), + headers: headers.clone(), + body: None, })?; - let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (package_root_markdown, modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?; database.insert(&package, &package_root_markdown)?; for module in modules { let response = zed::fetch(&HttpRequest { + method: HttpMethod::Get, url: format!("https://hexdocs.pm/{package}/{module}.html"), + headers: headers.clone(), + body: None, })?; - let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?; + let (markdown, _modules) = + convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?; database.insert(&format!("{module} ({package})"), &markdown)?; }