use std::cell::RefCell; use std::io::Read; use std::rc::Rc; use anyhow::Result; use html_to_markdown::markdown::{ HeadingHandler, ListHandler, ParagraphHandler, StyledTextHandler, TableHandler, }; use html_to_markdown::{ convert_html_to_markdown, HandleTag, HandlerOutcome, HtmlElement, MarkdownWriter, StartTagOutcome, TagHandler, }; use indexmap::IndexSet; use strum::IntoEnumIterator; use crate::{RustdocItem, RustdocItemKind}; /// Converts the provided rustdoc HTML to Markdown. pub fn convert_rustdoc_to_markdown(html: impl Read) -> Result<(String, Vec)> { let item_collector = Rc::new(RefCell::new(RustdocItemCollector::new())); let mut handlers: Vec = vec![ Rc::new(RefCell::new(ParagraphHandler)), Rc::new(RefCell::new(HeadingHandler)), Rc::new(RefCell::new(ListHandler)), Rc::new(RefCell::new(TableHandler::new())), Rc::new(RefCell::new(StyledTextHandler)), Rc::new(RefCell::new(RustdocChromeRemover)), Rc::new(RefCell::new(RustdocHeadingHandler)), Rc::new(RefCell::new(RustdocCodeHandler)), Rc::new(RefCell::new(RustdocItemHandler)), item_collector.clone(), ]; let markdown = convert_html_to_markdown(html, &mut handlers)?; let items = item_collector .borrow() .items .iter() .cloned() .collect::>(); Ok((markdown, items)) } pub struct RustdocHeadingHandler; impl HandleTag for RustdocHeadingHandler { fn should_handle(&self, _tag: &str) -> bool { // We're only handling text, so we don't need to visit any tags. false } fn handle_text(&mut self, text: &str, writer: &mut MarkdownWriter) -> HandlerOutcome { if writer.is_inside("h1") || writer.is_inside("h2") || writer.is_inside("h3") || writer.is_inside("h4") || writer.is_inside("h5") || writer.is_inside("h6") { let text = text .trim_matches(|char| char == '\n' || char == '\r') .replace('\n', " "); writer.push_str(&text); return HandlerOutcome::Handled; } HandlerOutcome::NoOp } } pub struct RustdocCodeHandler; impl HandleTag for RustdocCodeHandler { fn should_handle(&self, tag: &str) -> bool { match tag { "pre" | "code" => true, _ => false, } } fn handle_tag_start( &mut self, tag: &HtmlElement, writer: &mut MarkdownWriter, ) -> StartTagOutcome { match tag.tag() { "code" => { if !writer.is_inside("pre") { writer.push_str("`"); } } "pre" => { let classes = tag.classes(); let is_rust = classes.iter().any(|class| class == "rust"); let language = is_rust .then(|| "rs") .or_else(|| { classes.iter().find_map(|class| { if let Some((_, language)) = class.split_once("language-") { Some(language.trim()) } else { None } }) }) .unwrap_or(""); writer.push_str(&format!("\n\n```{language}\n")); } _ => {} } StartTagOutcome::Continue } fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) { match tag.tag() { "code" => { if !writer.is_inside("pre") { writer.push_str("`"); } } "pre" => writer.push_str("\n```\n"), _ => {} } } fn handle_text(&mut self, text: &str, writer: &mut MarkdownWriter) -> HandlerOutcome { if writer.is_inside("pre") { writer.push_str(&text); return HandlerOutcome::Handled; } HandlerOutcome::NoOp } } const RUSTDOC_ITEM_NAME_CLASS: &str = "item-name"; pub struct RustdocItemHandler; impl RustdocItemHandler { /// Returns whether we're currently inside of an `.item-name` element, which /// rustdoc uses to display Rust items in a list. fn is_inside_item_name(writer: &MarkdownWriter) -> bool { writer .current_element_stack() .iter() .any(|element| element.has_class(RUSTDOC_ITEM_NAME_CLASS)) } } impl HandleTag for RustdocItemHandler { fn should_handle(&self, tag: &str) -> bool { match tag { "div" | "span" => true, _ => false, } } fn handle_tag_start( &mut self, tag: &HtmlElement, writer: &mut MarkdownWriter, ) -> StartTagOutcome { match tag.tag() { "div" | "span" => { if Self::is_inside_item_name(writer) && tag.has_class("stab") { writer.push_str(" ["); } } _ => {} } StartTagOutcome::Continue } fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) { match tag.tag() { "div" | "span" => { if tag.has_class(RUSTDOC_ITEM_NAME_CLASS) { writer.push_str(": "); } if Self::is_inside_item_name(writer) && tag.has_class("stab") { writer.push_str("]"); } } _ => {} } } fn handle_text(&mut self, text: &str, writer: &mut MarkdownWriter) -> HandlerOutcome { if Self::is_inside_item_name(writer) && !writer.is_inside("span") && !writer.is_inside("code") { writer.push_str(&format!("`{text}`")); return HandlerOutcome::Handled; } HandlerOutcome::NoOp } } pub struct RustdocChromeRemover; impl HandleTag for RustdocChromeRemover { fn should_handle(&self, tag: &str) -> bool { match tag { "head" | "script" | "nav" | "summary" | "button" | "a" | "div" | "span" => true, _ => false, } } fn handle_tag_start( &mut self, tag: &HtmlElement, _writer: &mut MarkdownWriter, ) -> StartTagOutcome { match tag.tag() { "head" | "script" | "nav" => return StartTagOutcome::Skip, "summary" => { if tag.has_class("hideme") { return StartTagOutcome::Skip; } } "button" => { if tag.attr("id").as_deref() == Some("copy-path") { return StartTagOutcome::Skip; } } "a" => { if tag.has_any_classes(&["anchor", "doc-anchor", "src"]) { return StartTagOutcome::Skip; } } "div" | "span" => { if tag.has_any_classes(&["nav-container", "sidebar-elems", "out-of-band"]) { return StartTagOutcome::Skip; } } _ => {} } StartTagOutcome::Continue } } pub struct RustdocItemCollector { pub items: IndexSet, } impl RustdocItemCollector { pub fn new() -> Self { Self { items: IndexSet::new(), } } fn parse_item(tag: &HtmlElement) -> Option { if tag.tag() != "a" { return None; } let href = tag.attr("href")?; if href.starts_with('#') || href.starts_with("https://") || href.starts_with("../") { return None; } for kind in RustdocItemKind::iter() { if tag.has_class(kind.class()) { let mut parts = href.trim_end_matches("/index.html").split('/'); if let Some(last_component) = parts.next_back() { let last_component = match last_component.split_once('#') { Some((component, _fragment)) => component, None => last_component, }; let name = last_component .trim_start_matches(&format!("{}.", kind.class())) .trim_end_matches(".html"); return Some(RustdocItem { kind, name: name.into(), path: parts.map(Into::into).collect(), }); } } } None } } impl HandleTag for RustdocItemCollector { fn should_handle(&self, tag: &str) -> bool { tag == "a" } fn handle_tag_start( &mut self, tag: &HtmlElement, writer: &mut MarkdownWriter, ) -> StartTagOutcome { match tag.tag() { "a" => { let is_reexport = writer.current_element_stack().iter().any(|element| { if let Some(id) = element.attr("id") { id.starts_with("reexport.") || id.starts_with("method.") } else { false } }); if !is_reexport { if let Some(item) = Self::parse_item(tag) { self.items.insert(item); } } } _ => {} } StartTagOutcome::Continue } } #[cfg(test)] mod tests { use html_to_markdown::{convert_html_to_markdown, TagHandler}; use indoc::indoc; use pretty_assertions::assert_eq; use super::*; fn rustdoc_handlers() -> Vec { vec![ Rc::new(RefCell::new(ParagraphHandler)), Rc::new(RefCell::new(HeadingHandler)), Rc::new(RefCell::new(ListHandler)), Rc::new(RefCell::new(TableHandler::new())), Rc::new(RefCell::new(StyledTextHandler)), Rc::new(RefCell::new(RustdocChromeRemover)), Rc::new(RefCell::new(RustdocHeadingHandler)), Rc::new(RefCell::new(RustdocCodeHandler)), Rc::new(RefCell::new(RustdocItemHandler)), ] } #[test] fn test_main_heading_buttons_get_removed() { let html = indoc! {r##"

Crate serde

source ·
"##}; let expected = indoc! {" # Crate serde "} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_single_paragraph() { let html = indoc! {r#"

In particular, the last point is what sets axum apart from other frameworks. axum doesn’t have its own middleware system but instead uses tower::Service. This means axum gets timeouts, tracing, compression, authorization, and more, for free. It also enables you to share middleware with applications written using hyper or tonic.

"#}; let expected = indoc! {" In particular, the last point is what sets `axum` apart from other frameworks. `axum` doesn’t have its own middleware system but instead uses `tower::Service`. This means `axum` gets timeouts, tracing, compression, authorization, and more, for free. It also enables you to share middleware with applications written using `hyper` or `tonic`. "} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_multiple_paragraphs() { let html = indoc! {r##"

§Serde

Serde is a framework for serializing and deserializing Rust data structures efficiently and generically.

The Serde ecosystem consists of data structures that know how to serialize and deserialize themselves along with data formats that know how to serialize and deserialize other things. Serde provides the layer by which these two groups interact with each other, allowing any supported data structure to be serialized and deserialized using any supported data format.

See the Serde website https://serde.rs/ for additional documentation and usage examples.

§Design

Where many other languages rely on runtime reflection for serializing data, Serde is instead built on Rust’s powerful trait system. A data structure that knows how to serialize and deserialize itself is one that implements Serde’s Serialize and Deserialize traits (or uses Serde’s derive attribute to automatically generate implementations at compile time). This avoids any overhead of reflection or runtime type information. In fact in many situations the interaction between data structure and data format can be completely optimized away by the Rust compiler, leaving Serde serialization to perform the same speed as a handwritten serializer for the specific selection of data structure and data format.

"##}; let expected = indoc! {" ## Serde Serde is a framework for _**ser**_ializing and _**de**_serializing Rust data structures efficiently and generically. The Serde ecosystem consists of data structures that know how to serialize and deserialize themselves along with data formats that know how to serialize and deserialize other things. Serde provides the layer by which these two groups interact with each other, allowing any supported data structure to be serialized and deserialized using any supported data format. See the Serde website https://serde.rs/ for additional documentation and usage examples. ### Design Where many other languages rely on runtime reflection for serializing data, Serde is instead built on Rust’s powerful trait system. A data structure that knows how to serialize and deserialize itself is one that implements Serde’s `Serialize` and `Deserialize` traits (or uses Serde’s derive attribute to automatically generate implementations at compile time). This avoids any overhead of reflection or runtime type information. In fact in many situations the interaction between data structure and data format can be completely optimized away by the Rust compiler, leaving Serde serialization to perform the same speed as a handwritten serializer for the specific selection of data structure and data format. "} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_styled_text() { let html = indoc! {r#"

This text is bolded.

This text is italicized.

"#}; let expected = indoc! {" This text is **bolded**. This text is _italicized_. "} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_rust_code_block() { let html = indoc! {r#"
use axum::extract::{Path, Query, Json};
            use std::collections::HashMap;

            // `Path` gives you the path parameters and deserializes them.
            async fn path(Path(user_id): Path<u32>) {}

            // `Query` gives you the query parameters and deserializes them.
            async fn query(Query(params): Query<HashMap<String, String>>) {}

            // Buffer the request body and deserialize it as JSON into a
            // `serde_json::Value`. `Json` supports any type that implements
            // `serde::Deserialize`.
            async fn json(Json(payload): Json<serde_json::Value>) {}
"#}; let expected = indoc! {" ```rs use axum::extract::{Path, Query, Json}; use std::collections::HashMap; // `Path` gives you the path parameters and deserializes them. async fn path(Path(user_id): Path) {} // `Query` gives you the query parameters and deserializes them. async fn query(Query(params): Query>) {} // Buffer the request body and deserialize it as JSON into a // `serde_json::Value`. `Json` supports any type that implements // `serde::Deserialize`. async fn json(Json(payload): Json) {} ``` "} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_toml_code_block() { let html = indoc! {r##"

§Required dependencies

To use axum there are a few dependencies you have to pull in as well:

[dependencies]
            axum = "<latest-version>"
            tokio = { version = "<latest-version>", features = ["full"] }
            tower = "<latest-version>"
            
"##}; let expected = indoc! {r#" ## Required dependencies To use axum there are a few dependencies you have to pull in as well: ```toml [dependencies] axum = "" tokio = { version = "", features = ["full"] } tower = "" ``` "#} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_item_table() { let html = indoc! {r##"

Structs§

  • Errors that can happen when using axum.
  • Extractor and response for extensions.
  • Formform
    URL encoded extractor and response.
  • Jsonjson
    JSON Extractor / Response.
  • The router type for composing handlers and services.

Functions§

  • servetokio and (http1 or http2)
    Serve the service with the supplied listener.
"##}; let expected = indoc! {r#" ## Structs - `Error`: Errors that can happen when using axum. - `Extension`: Extractor and response for extensions. - `Form` [`form`]: URL encoded extractor and response. - `Json` [`json`]: JSON Extractor / Response. - `Router`: The router type for composing handlers and services. ## Functions - `serve` [`tokio` and (`http1` or `http2`)]: Serve the service with the supplied listener. "#} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } #[test] fn test_table() { let html = indoc! {r##"

§Feature flags

axum uses a set of feature flags to reduce the amount of compiled and optional dependencies.

The following optional features are available:

NameDescriptionDefault?
http1Enables hyper’s http1 featureYes
http2Enables hyper’s http2 featureNo
jsonEnables the Json type and some similar convenience functionalityYes
macrosEnables optional utility macrosNo
matched-pathEnables capturing of every request’s router path and the MatchedPath extractorYes
multipartEnables parsing multipart/form-data requests with MultipartNo
original-uriEnables capturing of every request’s original URI and the OriginalUri extractorYes
tokioEnables tokio as a dependency and axum::serve, SSE and extract::connect_info types.Yes
tower-logEnables tower’s log featureYes
tracingLog rejections from built-in extractorsYes
wsEnables WebSockets support via extract::wsNo
formEnables the Form extractorYes
queryEnables the Query extractorYes
"##}; let expected = indoc! {r#" ## Feature flags axum uses a set of feature flags to reduce the amount of compiled and optional dependencies. The following optional features are available: | Name | Description | Default? | | --- | --- | --- | | `http1` | Enables hyper’s `http1` feature | Yes | | `http2` | Enables hyper’s `http2` feature | No | | `json` | Enables the `Json` type and some similar convenience functionality | Yes | | `macros` | Enables optional utility macros | No | | `matched-path` | Enables capturing of every request’s router path and the `MatchedPath` extractor | Yes | | `multipart` | Enables parsing `multipart/form-data` requests with `Multipart` | No | | `original-uri` | Enables capturing of every request’s original URI and the `OriginalUri` extractor | Yes | | `tokio` | Enables `tokio` as a dependency and `axum::serve`, `SSE` and `extract::connect_info` types. | Yes | | `tower-log` | Enables `tower`’s `log` feature | Yes | | `tracing` | Log rejections from built-in extractors | Yes | | `ws` | Enables WebSockets support via `extract::ws` | No | | `form` | Enables the `Form` extractor | Yes | | `query` | Enables the `Query` extractor | Yes | "#} .trim(); assert_eq!( convert_html_to_markdown(html.as_bytes(), &mut rustdoc_handlers()).unwrap(), expected ) } }