Allow extensions to define providers for indexing docs (#13755)

This PR provides extensions with the ability to define providers for
indexing docs.

Release Notes:

- N/A

extensions/gleam/extension.toml

@@ -23,3 +23,5 @@ tooltip_text = "Insert Gleam project data"
 description = "Returns Gleam docs."
 requires_argument = true
 tooltip_text = "Insert Gleam docs"
+
+# [indexed_docs_providers.gleam-hexdocs]

extensions/gleam/src/gleam.rs

@@ -4,10 +4,10 @@ use std::fs;
 use std::rc::Rc;
 use zed::lsp::CompletionKind;
 use zed::{
-    CodeLabel, CodeLabelSpan, LanguageServerId, SlashCommand, SlashCommandOutput,
-    SlashCommandOutputSection,
+    CodeLabel, CodeLabelSpan, HttpRequest, KeyValueStore, LanguageServerId, SlashCommand,
+    SlashCommandOutput, SlashCommandOutputSection,
 };
-use zed_extension_api::{self as zed, fetch, HttpRequest, Result};
+use zed_extension_api::{self as zed, Result};
 
 struct GleamExtension {
     cached_binary_path: Option<String>,
@@ -180,7 +180,7 @@ impl zed::Extension for GleamExtension {
                     .ok_or_else(|| "missing package name".to_string())?;
                 let module_path = components.map(ToString::to_string).collect::<Vec<_>>();
 
-                let response = fetch(&HttpRequest {
+                let response = zed::fetch(&HttpRequest {
                     url: format!(
                         "https://hexdocs.pm/{package_name}{maybe_path}",
                         maybe_path = if !module_path.is_empty() {
@@ -236,6 +236,38 @@ impl zed::Extension for GleamExtension {
             command => Err(format!("unknown slash command: \"{command}\"")),
         }
     }
+
+    fn index_docs(
+        &self,
+        provider: String,
+        package: String,
+        database: &KeyValueStore,
+    ) -> Result<(), String> {
+        match provider.as_str() {
+            "gleam-hexdocs" => {
+                let response = zed::fetch(&HttpRequest {
+                    url: format!("https://hexdocs.pm/{package}"),
+                })?;
+
+                let mut handlers: Vec<TagHandler> = vec![
+                    Rc::new(RefCell::new(
+                        html_to_markdown::markdown::WebpageChromeRemover,
+                    )),
+                    Rc::new(RefCell::new(html_to_markdown::markdown::ParagraphHandler)),
+                    Rc::new(RefCell::new(html_to_markdown::markdown::HeadingHandler)),
+                    Rc::new(RefCell::new(html_to_markdown::markdown::ListHandler)),
+                    Rc::new(RefCell::new(html_to_markdown::markdown::TableHandler::new())),
+                    Rc::new(RefCell::new(html_to_markdown::markdown::StyledTextHandler)),
+                ];
+
+                let markdown = convert_html_to_markdown(response.body.as_bytes(), &mut handlers)
+                    .map_err(|err| format!("failed to convert docs to Markdown {err}"))?;
+
+                Ok(database.insert(&package, &markdown)?)
+            }
+            _ => Ok(()),
+        }
+    }
 }
 
 zed::register_extension!(GleamExtension);