Enhance HTTP API for extensions (#16067)

# HTTP Client Improvements for Extension API

This PR enhances the HTTP client functionality in the Zed extension API,
providing more control over requests and allowing for streaming
responses.

## Key Changes

1. Extended `HttpRequest` struct:
   - Added `method` field to specify HTTP method
   - Added `headers` field for custom headers
   - Added optional `body` field for request payload

2. Introduced `HttpMethod` enum for supported HTTP methods

3. Updated `HttpResponse` struct:
   - Added `headers` field to access response headers
- Changed `body` type from `String` to `Vec<u8>` for binary data support

4. Added streaming support:
   - New `fetch_stream` function to get a response stream
   - Introduced `HttpResponseStream` resource for chunked reading

5. Updated internal implementations to support these new features

6. Modified the Gleam extension to use the new API structure

## Motivation

These changes provide extension developers with more flexibility and
control over HTTP requests. The streaming support is particularly useful
for handling large responses efficiently or ideally streaming into the
UI.

## Testing

- [x] Updated existing tests
- [ ] Added new tests for streaming functionality

## Next Steps

- Consider adding more comprehensive examples in the documentation
- Evaluate performance impact of streaming for large responses

Please review and let me know if any adjustments are needed.

Release Notes:

- N/A

---------

Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>

extensions/gleam/src/gleam.rs

@@ -1,10 +1,10 @@
 mod hexdocs;
 
-use std::fs;
+use std::{fs, io};
 use zed::lsp::CompletionKind;
 use zed::{
-    CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand,
-    SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
+    CodeLabel, CodeLabelSpan, HttpMethod, HttpRequest, KeyValueStore, LanguageServerId,
+    SlashCommand, SlashCommandArgumentCompletion, SlashCommandOutput, SlashCommandOutputSection,
 };
 use zed_extension_api::{self as zed, Result};
 
@@ -194,6 +194,7 @@ impl zed::Extension for GleamExtension {
                 let module_path = components.map(ToString::to_string).collect::<Vec<_>>();
 
                 let response = zed::fetch(&HttpRequest {
+                    method: HttpMethod::Get,
                     url: format!(
                         "https://hexdocs.pm/{package_name}{maybe_path}",
                         maybe_path = if !module_path.is_empty() {
@@ -202,9 +203,15 @@ impl zed::Extension for GleamExtension {
                             String::new()
                         }
                     ),
+                    headers: vec![(
+                        "User-Agent".to_string(),
+                        "Zed (Gleam Extension)".to_string(),
+                    )],
+                    body: None,
                 })?;
 
-                let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+                let (markdown, _modules) =
+                    convert_hexdocs_to_markdown(&mut io::Cursor::new(response.body))?;
 
                 let mut text = String::new();
                 text.push_str(&markdown);

extensions/gleam/src/hexdocs.rs

@@ -1,6 +1,6 @@
 use std::cell::RefCell;
 use std::collections::BTreeSet;
-use std::io::Read;
+use std::io::{self, Read};
 use std::rc::Rc;
 
 use html_to_markdown::markdown::{
@@ -10,23 +10,36 @@ use html_to_markdown::{
     convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter,
     StartTagOutcome, TagHandler,
 };
-use zed_extension_api::{self as zed, HttpRequest, KeyValueStore, Result};
+use zed_extension_api::{self as zed, HttpMethod, HttpRequest, KeyValueStore, Result};
 
 pub fn index(package: String, database: &KeyValueStore) -> Result<()> {
+    let headers = vec![(
+        "User-Agent".to_string(),
+        "Zed (Gleam Extension)".to_string(),
+    )];
+
     let response = zed::fetch(&HttpRequest {
+        method: HttpMethod::Get,
         url: format!("https://hexdocs.pm/{package}"),
+        headers: headers.clone(),
+        body: None,
     })?;
 
-    let (package_root_markdown, modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+    let (package_root_markdown, modules) =
+        convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
 
     database.insert(&package, &package_root_markdown)?;
 
     for module in modules {
         let response = zed::fetch(&HttpRequest {
+            method: HttpMethod::Get,
             url: format!("https://hexdocs.pm/{package}/{module}.html"),
+            headers: headers.clone(),
+            body: None,
         })?;
 
-        let (markdown, _modules) = convert_hexdocs_to_markdown(response.body.as_bytes())?;
+        let (markdown, _modules) =
+            convert_hexdocs_to_markdown(&mut io::Cursor::new(&response.body))?;
 
         database.insert(&format!("{module} ({package})"), &markdown)?;
     }