Enhance HTTP API for extensions (#16067)

# HTTP Client Improvements for Extension API

This PR enhances the HTTP client functionality in the Zed extension API,
providing more control over requests and allowing for streaming
responses.

## Key Changes

1. Extended `HttpRequest` struct:
   - Added `method` field to specify HTTP method
   - Added `headers` field for custom headers
   - Added optional `body` field for request payload

2. Introduced `HttpMethod` enum for supported HTTP methods

3. Updated `HttpResponse` struct:
   - Added `headers` field to access response headers
- Changed `body` type from `String` to `Vec<u8>` for binary data support

4. Added streaming support:
   - New `fetch_stream` function to get a response stream
   - Introduced `HttpResponseStream` resource for chunked reading

5. Updated internal implementations to support these new features

6. Modified the Gleam extension to use the new API structure

## Motivation

These changes provide extension developers with more flexibility and
control over HTTP requests. The streaming support is particularly useful
for handling large responses efficiently or ideally streaming into the
UI.

## Testing

- [x] Updated existing tests
- [ ] Added new tests for streaming functionality

## Next Steps

- Consider adding more comprehensive examples in the documentation
- Evaluate performance impact of streaming for large responses

Please review and let me know if any adjustments are needed.

Release Notes:

- N/A

---------

Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>

crates/extension/src/wasm_host/wit/since_v0_0_7.rs

@@ -5,9 +5,10 @@ use anyhow::{anyhow, bail, Context, Result};
 use async_compression::futures::bufread::GzipDecoder;
 use async_tar::Archive;
 use async_trait::async_trait;
-use futures::AsyncReadExt;
 use futures::{io::BufReader, FutureExt as _};
+use futures::{lock::Mutex, AsyncReadExt};
 use indexed_docs::IndexedDocsDatabase;
+use isahc::config::{Configurable, RedirectPolicy};
 use language::{
     language_settings::AllLanguageSettings, LanguageServerBinaryStatus, LspAdapterDelegate,
 };
@@ -30,7 +31,8 @@ wasmtime::component::bindgen!({
     path: "../extension_api/wit/since_v0.0.7",
     with: {
          "worktree": ExtensionWorktree,
-         "key-value-store": ExtensionKeyValueStore
+         "key-value-store": ExtensionKeyValueStore,
+         "zed:extension/http-client/http-response-stream": ExtensionHttpResponseStream
     },
 });
 
@@ -41,8 +43,8 @@ mod settings {
 }
 
 pub type ExtensionWorktree = Arc<dyn LspAdapterDelegate>;
-
 pub type ExtensionKeyValueStore = Arc<IndexedDocsDatabase>;
+pub type ExtensionHttpResponseStream = Arc<Mutex<::http_client::Response<AsyncBody>>>;
 
 pub fn linker() -> &'static Linker<WasmState> {
     static LINKER: OnceLock<Linker<WasmState>> = OnceLock::new();
@@ -130,35 +132,123 @@ impl common::Host for WasmState {}
 impl http_client::Host for WasmState {
     async fn fetch(
         &mut self,
-        req: http_client::HttpRequest,
+        request: http_client::HttpRequest,
     ) -> wasmtime::Result<Result<http_client::HttpResponse, String>> {
         maybe!(async {
-            let url = &req.url;
-
-            let mut response = self
-                .host
-                .http_client
-                .get(url, AsyncBody::default(), true)
-                .await?;
+            let url = &request.url;
+            let request = convert_request(&request, true)?;
+            let mut response = self.host.http_client.send(request).await?;
 
             if response.status().is_client_error() || response.status().is_server_error() {
                 bail!("failed to fetch '{url}': status code {}", response.status())
             }
-
-            let mut body = Vec::new();
-            response
-                .body_mut()
-                .read_to_end(&mut body)
-                .await
-                .with_context(|| format!("failed to read response body from '{url}'"))?;
-
-            Ok(http_client::HttpResponse {
-                body: String::from_utf8(body)?,
-            })
+            convert_response(&mut response).await
         })
         .await
         .to_wasmtime_result()
     }
+
+    async fn fetch_stream(
+        &mut self,
+        request: http_client::HttpRequest,
+    ) -> wasmtime::Result<Result<Resource<ExtensionHttpResponseStream>, String>> {
+        let request = convert_request(&request, true)?;
+        let response = self.host.http_client.send(request);
+        maybe!(async {
+            let response = response.await?;
+            let stream = Arc::new(Mutex::new(response));
+            let resource = self.table.push(stream)?;
+            Ok(resource)
+        })
+        .await
+        .to_wasmtime_result()
+    }
+}
+
+#[async_trait]
+impl http_client::HostHttpResponseStream for WasmState {
+    async fn next_chunk(
+        &mut self,
+        resource: Resource<ExtensionHttpResponseStream>,
+    ) -> wasmtime::Result<Result<Option<Vec<u8>>, String>> {
+        let stream = self.table.get(&resource)?.clone();
+        maybe!(async move {
+            let mut response = stream.lock().await;
+            let mut buffer = vec![0; 8192]; // 8KB buffer
+            let bytes_read = response.body_mut().read(&mut buffer).await?;
+            if bytes_read == 0 {
+                Ok(None)
+            } else {
+                buffer.truncate(bytes_read);
+                Ok(Some(buffer))
+            }
+        })
+        .await
+        .to_wasmtime_result()
+    }
+
+    fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
+        Ok(())
+    }
+}
+
+impl From<http_client::HttpMethod> for ::http_client::Method {
+    fn from(value: http_client::HttpMethod) -> Self {
+        match value {
+            http_client::HttpMethod::Get => Self::GET,
+            http_client::HttpMethod::Post => Self::POST,
+            http_client::HttpMethod::Put => Self::PUT,
+            http_client::HttpMethod::Delete => Self::DELETE,
+            http_client::HttpMethod::Head => Self::HEAD,
+            http_client::HttpMethod::Options => Self::OPTIONS,
+            http_client::HttpMethod::Patch => Self::PATCH,
+        }
+    }
+}
+
+fn convert_request(
+    extension_request: &http_client::HttpRequest,
+    follow_redirects: bool,
+) -> Result<::http_client::Request<AsyncBody>, anyhow::Error> {
+    let mut request = ::http_client::Request::builder()
+        .method(::http_client::Method::from(extension_request.method))
+        .uri(&extension_request.url)
+        .redirect_policy(if follow_redirects {
+            RedirectPolicy::Follow
+        } else {
+            RedirectPolicy::None
+        });
+    for (key, value) in &extension_request.headers {
+        request = request.header(key, value);
+    }
+    let body = extension_request
+        .body
+        .clone()
+        .map(AsyncBody::from)
+        .unwrap_or_default();
+    request.body(body).map_err(anyhow::Error::from)
+}
+
+async fn convert_response(
+    response: &mut ::http_client::Response<AsyncBody>,
+) -> Result<http_client::HttpResponse, anyhow::Error> {
+    let mut extension_response = http_client::HttpResponse {
+        body: Vec::new(),
+        headers: Vec::new(),
+    };
+
+    for (key, value) in response.headers() {
+        extension_response
+            .headers
+            .push((key.to_string(), value.to_str().unwrap_or("").to_string()));
+    }
+
+    response
+        .body_mut()
+        .read_to_end(&mut extension_response.body)
+        .await?;
+
+    Ok(extension_response)
 }
 
 #[async_trait]

crates/extension_api/src/extension_api.rs

@@ -19,7 +19,9 @@ pub use wit::{
         github_release_by_tag_name, latest_github_release, GithubRelease, GithubReleaseAsset,
         GithubReleaseOptions,
     },
-    zed::extension::http_client::{fetch, HttpRequest, HttpResponse},
+    zed::extension::http_client::{
+        fetch, fetch_stream, HttpMethod, HttpRequest, HttpResponse, HttpResponseStream,
+    },
     zed::extension::nodejs::{
         node_binary_path, npm_install_package, npm_package_installed_version,
         npm_package_latest_version,

crates/extension_api/wit/since_v0.0.7/http-client.wit

@@ -1,16 +1,45 @@
 interface http-client {
     /// An HTTP request.
     record http-request {
+        /// The HTTP method for the request.
+        method: http-method,
         /// The URL to which the request should be made.
         url: string,
+        /// Headers for the request.
+        headers: list<tuple<string, string>>,
+        /// The request body.
+        body: option<list<u8>>,
+    }
+
+    /// HTTP methods.
+    enum http-method {
+        get,
+        post,
+        put,
+        delete,
+        head,
+        options,
+        patch,
     }
 
     /// An HTTP response.
     record http-response {
+        /// The response headers.
+        headers: list<tuple<string, string>>,
         /// The response body.
-        body: string,
+        body: list<u8>,
     }
 
     /// Performs an HTTP request and returns the response.
     fetch: func(req: http-request) -> result<http-response, string>;
+
+    /// An HTTP response stream.
+    resource http-response-stream {
+        /// Retrieves the next chunk of data from the response stream.
+        /// Returns None if the stream has ended.
+        next-chunk: func() -> result<option<list<u8>>, string>;
+    }
+
+    /// Performs an HTTP request and returns a response stream.
+    fetch-stream: func(req: http-request) -> result<http-response-stream, string>;
 }

extensions/gleam/src/gleam.rs

@@ -1,10 +1,10 @@
 mod hexdocs;
 
-use std::fs;
+use std::{fs, io};
 use zed::lsp::CompletionKind;
 use zed::{
-    CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand,
-    SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
+    CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId,
+    SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
 };
 use zed_extension_api::{self as zed, Result};
 
@@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension {
                 let module_path = components.map(ToString::to_string).collect::<Vec<_>>();
 
                 let response = zed::fetch(&HttpRequest {
+                    method: HttpMethod::Get,
                     url: format!(
                         "https://hexdocs.pm/{package_name}{maybe_path}",
                         maybe_path = if !module_path.is_empty() {
@@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension {
                             String::new()
                         }
                     ),
+                    headers: vec![(
+                        "User-Agent".to_string(),
+                        "Zed (Gleam Extension)".to_string(),
+                    )],
+                    body: None,
                 })?;
 
-                let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+                let (markdown, _modules) =
+                    convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?;
 
                 let mut text = String::new();
                 text.push_str(&markdown);

extensions/gleam/src/hexdocs.rs

@@ -1,6 +1,6 @@
 use std::cell::RefCell;
 use std::collections::BTreeSet;
-use std::io::Read;
+use std::io::{self, Read};
 use std::rc::Rc;
 
 use html_to_markdown::markdown::{
@@ -10,23 +10,36 @@ use html_to_markdown::{
     convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter,
     StartTagOutcome, TagHandler,
 };
-use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result};
+use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result};
 
 pub fn index(package: String, database: &KeyValueStore) -> Result<()> {
+    let headers = vec![(
+        "User-Agent".to_string(),
+        "Zed (Gleam Extension)".to_string(),
+    )];
+
     let response = zed::fetch(&HttpRequest {
+        method: HttpMethod::Get,
         url: format!("https://hexdocs.pm/{package}"),
+        headers: headers.clone(),
+        body: None,
     })?;
 
-    let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+    let (package_root_markdown, modules) =
+        convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
 
     database.insert(&package, &package_root_markdown)?;
 
     for module in modules {
         let response = zed::fetch(&HttpRequest {
+            method: HttpMethod::Get,
             url: format!("https://hexdocs.pm/{package}/{module}.html"),
+            headers: headers.clone(),
+            body: None,
         })?;
 
-        let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+        let (markdown, _modules) =
+            convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
 
         database.insert(&format!("{module} ({package})"), &markdown)?;
     }