diff --git a/Cargo.lock b/Cargo.lock index 1cd9e521f1..d91c5d5eca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4758,7 +4758,6 @@ name = "docs_preprocessor" version = "0.1.0" dependencies = [ "anyhow", - "clap", "command_palette", "gpui", "mdbook", @@ -4769,6 +4768,7 @@ dependencies = [ "util", "workspace-hack", "zed", + "zlog", ] [[package]] diff --git a/crates/docs_preprocessor/Cargo.toml b/crates/docs_preprocessor/Cargo.toml index a0df669abe..a9eff17fa1 100644 --- a/crates/docs_preprocessor/Cargo.toml +++ b/crates/docs_preprocessor/Cargo.toml @@ -7,17 +7,17 @@ license = "GPL-3.0-or-later" [dependencies] anyhow.workspace = true -clap.workspace = true +command_palette.workspace = true +gpui.workspace = true mdbook = "0.4.40" +regex.workspace = true serde.workspace = true serde_json.workspace = true settings.workspace = true -regex.workspace = true util.workspace = true workspace-hack.workspace = true zed.workspace = true -gpui.workspace = true -command_palette.workspace = true +zlog.workspace = true [lints] workspace = true diff --git a/crates/docs_preprocessor/src/main.rs b/crates/docs_preprocessor/src/main.rs index 8eeeb6f0c5..1448f4cb52 100644 --- a/crates/docs_preprocessor/src/main.rs +++ b/crates/docs_preprocessor/src/main.rs @@ -1,14 +1,15 @@ -use anyhow::Result; -use clap::{Arg, ArgMatches, Command}; +use anyhow::{Context, Result}; use mdbook::BookItem; use mdbook::book::{Book, Chapter}; use mdbook::preprocess::CmdPreprocessor; use regex::Regex; use settings::KeymapFile; -use std::collections::HashSet; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; use std::io::{self, Read}; use std::process; use std::sync::LazyLock; +use util::paths::PathExt; static KEYMAP_MACOS: LazyLock = LazyLock::new(|| { load_keymap("keymaps/default-macos.json").expect("Failed to load MacOS keymap") @@ -20,60 +21,68 @@ static KEYMAP_LINUX: LazyLock = LazyLock::new(|| { static ALL_ACTIONS: LazyLock> = LazyLock::new(dump_all_gpui_actions); -pub fn make_app() -> Command { - Command::new("zed-docs-preprocessor") - .about("Preprocesses Zed Docs content to provide rich action & keybinding support and more") - .subcommand( - Command::new("supports") - .arg(Arg::new("renderer").required(true)) - .about("Check whether a renderer is supported by this preprocessor"), - ) -} +const FRONT_MATTER_COMMENT: &'static str = ""; fn main() -> Result<()> { - let matches = make_app().get_matches(); + zlog::init(); + zlog::init_output_stderr(); // call a zed:: function so everything in `zed` crate is linked and // all actions in the actual app are registered zed::stdout_is_a_pty(); + let args = std::env::args().skip(1).collect::>(); - if let Some(sub_args) = matches.subcommand_matches("supports") { - handle_supports(sub_args); - } else { - handle_preprocessing()?; + match args.get(0).map(String::as_str) { + Some("supports") => { + let renderer = args.get(1).expect("Required argument"); + let supported = renderer != "not-supported"; + if supported { + process::exit(0); + } else { + process::exit(1); + } + } + Some("postprocess") => handle_postprocessing()?, + _ => handle_preprocessing()?, } Ok(()) } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -enum Error { +enum PreprocessorError { ActionNotFound { action_name: String }, DeprecatedActionUsed { used: String, should_be: String }, + InvalidFrontmatterLine(String), } -impl Error { +impl PreprocessorError { fn new_for_not_found_action(action_name: String) -> Self { for action in &*ALL_ACTIONS { for alias in action.deprecated_aliases { if alias == &action_name { - return Error::DeprecatedActionUsed { + return PreprocessorError::DeprecatedActionUsed { used: action_name.clone(), should_be: action.name.to_string(), }; } } } - Error::ActionNotFound { + PreprocessorError::ActionNotFound { action_name: action_name.to_string(), } } } -impl std::fmt::Display for Error { +impl std::fmt::Display for PreprocessorError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Error::ActionNotFound { action_name } => write!(f, "Action not found: {}", action_name), - Error::DeprecatedActionUsed { used, should_be } => write!( + PreprocessorError::InvalidFrontmatterLine(line) => { + write!(f, "Invalid frontmatter line: {}", line) + } + PreprocessorError::ActionNotFound { action_name } => { + write!(f, "Action not found: {}", action_name) + } + PreprocessorError::DeprecatedActionUsed { used, should_be } => write!( f, "Deprecated action used: {} should be {}", used, should_be @@ -89,8 +98,9 @@ fn handle_preprocessing() -> Result<()> { let (_ctx, mut book) = CmdPreprocessor::parse_input(input.as_bytes())?; - let mut errors = HashSet::::new(); + let mut errors = HashSet::::new(); + handle_frontmatter(&mut book, &mut errors); template_and_validate_keybindings(&mut book, &mut errors); template_and_validate_actions(&mut book, &mut errors); @@ -108,19 +118,41 @@ fn handle_preprocessing() -> Result<()> { Ok(()) } -fn handle_supports(sub_args: &ArgMatches) -> ! { - let renderer = sub_args - .get_one::("renderer") - .expect("Required argument"); - let supported = renderer != "not-supported"; - if supported { - process::exit(0); - } else { - process::exit(1); - } +fn handle_frontmatter(book: &mut Book, errors: &mut HashSet) { + let frontmatter_regex = Regex::new(r"(?s)^\s*---(.*?)---").unwrap(); + for_each_chapter_mut(book, |chapter| { + let new_content = frontmatter_regex.replace(&chapter.content, |caps: ®ex::Captures| { + let frontmatter = caps[1].trim(); + let frontmatter = frontmatter.trim_matches(&[' ', '-', '\n']); + let mut metadata = HashMap::::default(); + for line in frontmatter.lines() { + let Some((name, value)) = line.split_once(':') else { + errors.insert(PreprocessorError::InvalidFrontmatterLine(format!( + "{}: {}", + chapter_breadcrumbs(&chapter), + line + ))); + continue; + }; + let name = name.trim(); + let value = value.trim(); + metadata.insert(name.to_string(), value.to_string()); + } + FRONT_MATTER_COMMENT.replace( + "{}", + &serde_json::to_string(&metadata).expect("Failed to serialize metadata"), + ) + }); + match new_content { + Cow::Owned(content) => { + chapter.content = content; + } + Cow::Borrowed(_) => {} + } + }); } -fn template_and_validate_keybindings(book: &mut Book, errors: &mut HashSet) { +fn template_and_validate_keybindings(book: &mut Book, errors: &mut HashSet) { let regex = Regex::new(r"\{#kb (.*?)\}").unwrap(); for_each_chapter_mut(book, |chapter| { @@ -128,7 +160,9 @@ fn template_and_validate_keybindings(book: &mut Book, errors: &mut HashSet) { +fn template_and_validate_actions(book: &mut Book, errors: &mut HashSet) { let regex = Regex::new(r"\{#action (.*?)\}").unwrap(); for_each_chapter_mut(book, |chapter| { @@ -152,7 +186,9 @@ fn template_and_validate_actions(book: &mut Book, errors: &mut HashSet) { .replace_all(&chapter.content, |caps: ®ex::Captures| { let name = caps[1].trim(); let Some(action) = find_action_by_name(name) else { - errors.insert(Error::new_for_not_found_action(name.to_string())); + errors.insert(PreprocessorError::new_for_not_found_action( + name.to_string(), + )); return String::new(); }; format!("{}", &action.human_name) @@ -217,6 +253,13 @@ fn name_for_action(action_as_str: String) -> String { .unwrap_or(action_as_str) } +fn chapter_breadcrumbs(chapter: &Chapter) -> String { + let mut breadcrumbs = Vec::with_capacity(chapter.parent_names.len() + 1); + breadcrumbs.extend(chapter.parent_names.iter().map(String::as_str)); + breadcrumbs.push(chapter.name.as_str()); + format!("[{:?}] {}", chapter.source_path, breadcrumbs.join(" > ")) +} + fn load_keymap(asset_path: &str) -> Result { let content = util::asset_str::(asset_path); KeymapFile::parse(content.as_ref()) @@ -254,3 +297,126 @@ fn dump_all_gpui_actions() -> Vec { return actions; } + +fn handle_postprocessing() -> Result<()> { + let logger = zlog::scoped!("render"); + let mut ctx = mdbook::renderer::RenderContext::from_json(io::stdin())?; + let output = ctx + .config + .get_mut("output") + .expect("has output") + .as_table_mut() + .expect("output is table"); + let zed_html = output.remove("zed-html").expect("zed-html output defined"); + let default_description = zed_html + .get("default-description") + .expect("Default description not found") + .as_str() + .expect("Default description not a string") + .to_string(); + let default_title = zed_html + .get("default-title") + .expect("Default title not found") + .as_str() + .expect("Default title not a string") + .to_string(); + + output.insert("html".to_string(), zed_html); + mdbook::Renderer::render(&mdbook::renderer::HtmlHandlebars::new(), &ctx)?; + let ignore_list = ["toc.html"]; + + let root_dir = ctx.destination.clone(); + let mut files = Vec::with_capacity(128); + let mut queue = Vec::with_capacity(64); + queue.push(root_dir.clone()); + while let Some(dir) = queue.pop() { + for entry in std::fs::read_dir(&dir).context(dir.to_sanitized_string())? { + let Ok(entry) = entry else { + continue; + }; + let file_type = entry.file_type().context("Failed to determine file type")?; + if file_type.is_dir() { + queue.push(entry.path()); + } + if file_type.is_file() + && matches!( + entry.path().extension().and_then(std::ffi::OsStr::to_str), + Some("html") + ) + { + if ignore_list.contains(&&*entry.file_name().to_string_lossy()) { + zlog::info!(logger => "Ignoring {}", entry.path().to_string_lossy()); + } else { + files.push(entry.path()); + } + } + } + } + + zlog::info!(logger => "Processing {} `.html` files", files.len()); + let meta_regex = Regex::new(&FRONT_MATTER_COMMENT.replace("{}", "(.*)")).unwrap(); + for file in files { + let contents = std::fs::read_to_string(&file)?; + let mut meta_description = None; + let mut meta_title = None; + let contents = meta_regex.replace(&contents, |caps: ®ex::Captures| { + let metadata: HashMap = serde_json::from_str(&caps[1]).with_context(|| format!("JSON Metadata: {:?}", &caps[1])).expect("Failed to deserialize metadata"); + for (kind, content) in metadata { + match kind.as_str() { + "description" => { + meta_description = Some(content); + } + "title" => { + meta_title = Some(content); + } + _ => { + zlog::warn!(logger => "Unrecognized frontmatter key: {} in {:?}", kind, pretty_path(&file, &root_dir)); + } + } + } + String::new() + }); + let meta_description = meta_description.as_ref().unwrap_or_else(|| { + zlog::warn!(logger => "No meta description found for {:?}", pretty_path(&file, &root_dir)); + &default_description + }); + let page_title = extract_title_from_page(&contents, pretty_path(&file, &root_dir)); + let meta_title = meta_title.as_ref().unwrap_or_else(|| { + zlog::debug!(logger => "No meta title found for {:?}", pretty_path(&file, &root_dir)); + &default_title + }); + let meta_title = format!("{} | {}", page_title, meta_title); + zlog::trace!(logger => "Updating {:?}", pretty_path(&file, &root_dir)); + let contents = contents.replace("#description#", meta_description); + let contents = TITLE_REGEX + .replace(&contents, |_: ®ex::Captures| { + format!("{}", meta_title) + }) + .to_string(); + // let contents = contents.replace("#title#", &meta_title); + std::fs::write(file, contents)?; + } + return Ok(()); + + fn pretty_path<'a>( + path: &'a std::path::PathBuf, + root: &'a std::path::PathBuf, + ) -> &'a std::path::Path { + &path.strip_prefix(&root).unwrap_or(&path) + } + const TITLE_REGEX: std::cell::LazyCell = + std::cell::LazyCell::new(|| Regex::new(r"\s*(.*?)\s*").unwrap()); + fn extract_title_from_page(contents: &str, pretty_path: &std::path::Path) -> String { + let title_tag_contents = &TITLE_REGEX + .captures(&contents) + .with_context(|| format!("Failed to find title in {:?}", pretty_path)) + .expect("Page has element")[1]; + let title = title_tag_contents + .trim() + .strip_suffix("- Zed") + .unwrap_or(title_tag_contents) + .trim() + .to_string(); + title + } +} diff --git a/crates/zlog/src/sink.rs b/crates/zlog/src/sink.rs index acf0469c77..17aa08026e 100644 --- a/crates/zlog/src/sink.rs +++ b/crates/zlog/src/sink.rs @@ -21,6 +21,8 @@ const ANSI_MAGENTA: &str = "\x1b[35m"; /// Whether stdout output is enabled. static mut ENABLED_SINKS_STDOUT: bool = false; +/// Whether stderr output is enabled. +static mut ENABLED_SINKS_STDERR: bool = false; /// Is Some(file) if file output is enabled. static ENABLED_SINKS_FILE: Mutex<Option<std::fs::File>> = Mutex::new(None); @@ -45,6 +47,12 @@ pub fn init_output_stdout() { } } +pub fn init_output_stderr() { + unsafe { + ENABLED_SINKS_STDERR = true; + } +} + pub fn init_output_file( path: &'static PathBuf, path_rotate: Option<&'static PathBuf>, @@ -115,6 +123,21 @@ pub fn submit(record: Record) { }, record.message ); + } else if unsafe { ENABLED_SINKS_STDERR } { + let mut stdout = std::io::stderr().lock(); + _ = writeln!( + &mut stdout, + "{} {ANSI_BOLD}{}{}{ANSI_RESET} {} {}", + chrono::Local::now().format("%Y-%m-%dT%H:%M:%S%:z"), + LEVEL_ANSI_COLORS[record.level as usize], + LEVEL_OUTPUT_STRINGS[record.level as usize], + SourceFmt { + scope: record.scope, + module_path: record.module_path, + ansi: true, + }, + record.message + ); } let mut file = ENABLED_SINKS_FILE.lock().unwrap_or_else(|handle| { ENABLED_SINKS_FILE.clear_poison(); diff --git a/crates/zlog/src/zlog.rs b/crates/zlog/src/zlog.rs index 570c82314c..5b40278f3f 100644 --- a/crates/zlog/src/zlog.rs +++ b/crates/zlog/src/zlog.rs @@ -5,7 +5,7 @@ mod env_config; pub mod filter; pub mod sink; -pub use sink::{flush, init_output_file, init_output_stdout}; +pub use sink::{flush, init_output_file, init_output_stderr, init_output_stdout}; pub const SCOPE_DEPTH_MAX: usize = 4; diff --git a/docs/README.md b/docs/README.md index 55993c9e36..a225903674 100644 --- a/docs/README.md +++ b/docs/README.md @@ -69,3 +69,64 @@ Templates are just functions that modify the source of the docs pages (usually w - Template Trait: crates/docs_preprocessor/src/templates.rs - Example template: crates/docs_preprocessor/src/templates/keybinding.rs - Client-side plugins: docs/theme/plugins.js + +## Postprocessor + +A postprocessor is implemented as a sub-command of `docs_preprocessor` that wraps the builtin `html` renderer and applies post-processing to the `html` files, to add support for page-specific title and meta description values. + +An example of the syntax can be found in `git.md`, as well as below + +```md +--- +title: Some more detailed title for this page +description: A page-specific description +--- + +# Editor +``` + +The above will be transformed into (with non-relevant tags removed) + +```html +<head> + <title>Editor | Some more detailed title for this page + + + +

Editor

+ +``` + +If no front-matter is provided, or If one or both keys aren't provided, the title and description will be set based on the `default-title` and `default-description` keys in `book.toml` respectively. + +### Implementation details + +Unfortunately, `mdbook` does not support post-processing like it does pre-processing, and only supports defining one description to put in the meta tag per book rather than per file. So in order to apply post-processing (necessary to modify the html head tags) the global book description is set to a marker value `#description#` and the html renderer is replaced with a sub-command of `docs_preprocessor` that wraps the builtin `html` renderer and applies post-processing to the `html` files, replacing the marker value and the `(.*)` with the contents of the front-matter if there is one. + +### Known limitations + +The front-matter parsing is extremely simple, which avoids needing to take on an additional dependency, or implement full yaml parsing. + +- Double quotes and multi-line values are not supported, i.e. Keys and values must be entirely on the same line, with no double quotes around the value. + +The following will not work: + +```md +--- +title: Some + Multi-line + Title +--- +``` + +And neither will: + +```md +--- +title: "Some title" +--- +``` + +- The front-matter must be at the top of the file, with only white-space preceding it + +- The contents of the title and description will not be html-escaped. They should be simple ascii text with no unicode or emoji characters diff --git a/docs/book.toml b/docs/book.toml index 518fbec819..60ddc5ac51 100644 --- a/docs/book.toml +++ b/docs/book.toml @@ -6,13 +6,27 @@ src = "src" title = "Zed" site-url = "/docs/" -[output.html] +[build] +extra-watch-dirs = ["../crates/docs_preprocessor"] + +# zed-html is a "custom" renderer that just wraps the +# builtin mdbook html renderer, and applies post-processing +# as post-processing is not possible with mdbook in the same way +# pre-processing is +# The config is passed directly to the html renderer, so all config +# options that apply to html apply to zed-html +[output.zed-html] +command = "cargo run -p docs_preprocessor -- postprocess" +# Set here instead of above as we only use it replace the `#description#` we set in the template +# when no front-matter is provided value +default-description = "Learn how to use and customize Zed, the fast, collaborative code editor. Official docs on features, configuration, AI tools, and workflows." +default-title = "Zed Code Editor Documentation" no-section-label = true preferred-dark-theme = "dark" additional-css = ["theme/page-toc.css", "theme/plugins.css", "theme/highlight.css"] additional-js = ["theme/page-toc.js", "theme/plugins.js"] -[output.html.print] +[output.zed-html.print] enable = false # Redirects for `/docs` pages. @@ -24,7 +38,7 @@ enable = false # The destination URLs are interpreted relative to `https://zed.dev`. # - Redirects to other docs pages should end in `.html` # - You can link to pages on the Zed site by omitting the `/docs` in front of it. -[output.html.redirect] +[output.zed-html.redirect] # AI "/ai.html" = "/docs/ai/overview.html" "/assistant-panel.html" = "/docs/ai/agent-panel.html" diff --git a/docs/src/git.md b/docs/src/git.md index 5b5c8a3b15..cccbad9b2e 100644 --- a/docs/src/git.md +++ b/docs/src/git.md @@ -1,3 +1,8 @@ +--- +description: Zed is a text editor that supports lots of Git features +title: Zed Editor Git integration documentation +--- + # Git Zed currently offers a set of fundamental Git features, with support coming in the future for more advanced ones, like conflict resolution tools, line by line staging, and more. diff --git a/docs/theme/index.hbs b/docs/theme/index.hbs index 8ab4f21cf1..4339a02d17 100644 --- a/docs/theme/index.hbs +++ b/docs/theme/index.hbs @@ -15,7 +15,7 @@ {{> head}} - +