theme_importer: Improve syntax token matching (#3549)

This PR improves the approach we use to match syntax tokens between Zed
and VS Code in the `theme_importer`.

We now use the list of scopes assigned to each Zed syntax token to rank
the possible candidates in the VS Code and then pick the candidate with
the highest rank.

So far this has proved to provide better colors across the board, but
we'll continue to refine the matching over time.

Release Notes:

- N/A

crates/theme_importer/src/vscode/converter.rs

@@ -275,20 +275,31 @@ impl VsCodeThemeConverter {
         let mut highlight_styles = IndexMap::new();
 
         for syntax_token in ZedSyntaxToken::iter() {
-            let multimatch_scopes = syntax_token.to_vscode();
+            let best_match = syntax_token
+                .find_best_token_color_match(&self.theme.token_colors)
+                .or_else(|| {
+                    syntax_token.fallbacks().iter().find_map(|fallback| {
+                        fallback.find_best_token_color_match(&self.theme.token_colors)
+                    })
+                });
 
-            let token_color = self.theme.token_colors.iter().find(|token_color| {
-                token_color
-                    .scope
-                    .as_ref()
-                    .map(|scope| scope.multimatch(&multimatch_scopes))
-                    .unwrap_or(false)
-            });
-
-            let Some(token_color) = token_color else {
+            let Some(token_color) = best_match else {
+                log::warn!("No matching token color found for '{syntax_token}'");
                 continue;
             };
 
+            log::info!(
+                "Matched '{syntax_token}' to '{}'",
+                token_color
+                    .name
+                    .clone()
+                    .or_else(|| token_color
+                        .scope
+                        .as_ref()
+                        .map(|scope| format!("{:?}", scope)))
+                    .unwrap_or_else(|| "no identifier".to_string())
+            );
+
             let highlight_style = UserHighlightStyle {
                 color: token_color
                     .settings

crates/theme_importer/src/vscode/syntax.rs

@@ -1,3 +1,4 @@
+use indexmap::IndexMap;
 use serde::Deserialize;
 use strum::EnumIter;
 
@@ -8,19 +9,9 @@ pub enum VsCodeTokenScope {
     Many(Vec<String>),
 }
 
-impl VsCodeTokenScope {
-    pub fn multimatch(&self, matches: &[&'static str]) -> bool {
-        match self {
-            VsCodeTokenScope::One(scope) => matches.iter().any(|&s| s == scope),
-            VsCodeTokenScope::Many(scopes) => {
-                matches.iter().any(|s| scopes.contains(&s.to_string()))
-            }
-        }
-    }
-}
-
 #[derive(Debug, Deserialize)]
 pub struct VsCodeTokenColor {
+    pub name: Option<String>,
     pub scope: Option<VsCodeTokenScope>,
     pub settings: VsCodeTokenColorSettings,
 }
@@ -127,6 +118,60 @@ impl std::fmt::Display for ZedSyntaxToken {
 }
 
 impl ZedSyntaxToken {
+    pub fn find_best_token_color_match<'a>(
+        &self,
+        token_colors: &'a [VsCodeTokenColor],
+    ) -> Option<&'a VsCodeTokenColor> {
+        let mut ranked_matches = IndexMap::new();
+
+        for (ix, token_color) in token_colors.iter().enumerate() {
+            if token_color.settings.foreground.is_none() {
+                continue;
+            }
+
+            let Some(rank) = self.rank_match(token_color) else {
+                continue;
+            };
+
+            if rank > 0 {
+                ranked_matches.insert(ix, rank);
+            }
+        }
+
+        ranked_matches
+            .into_iter()
+            .max_by_key(|(_, rank)| *rank)
+            .map(|(ix, _)| &token_colors[ix])
+    }
+
+    fn rank_match(&self, token_color: &VsCodeTokenColor) -> Option<u32> {
+        let candidate_scopes = match token_color.scope.as_ref()? {
+            VsCodeTokenScope::One(scope) => vec![scope],
+            VsCodeTokenScope::Many(scopes) => scopes.iter().collect(),
+        }
+        .iter()
+        .map(|scope| scope.as_str())
+        .collect::<Vec<_>>();
+
+        let mut matches = 0;
+
+        for scope in self.to_vscode() {
+            if candidate_scopes.contains(&scope) {
+                matches += 1;
+            }
+        }
+
+        Some(matches)
+    }
+
+    pub fn fallbacks(&self) -> &[Self] {
+        match self {
+            ZedSyntaxToken::CommentDoc => &[ZedSyntaxToken::Comment],
+            ZedSyntaxToken::VariableSpecial => &[ZedSyntaxToken::Variable],
+            _ => &[],
+        }
+    }
+
     pub fn to_vscode(&self) -> Vec<&'static str> {
         match self {
             ZedSyntaxToken::Attribute => vec!["entity.other.attribute-name"],
@@ -150,7 +195,15 @@ impl ZedSyntaxToken {
                 "variable.function",
                 "support.function",
             ],
-            ZedSyntaxToken::Keyword => vec!["keyword"],
+            ZedSyntaxToken::Hint => vec![],
+            ZedSyntaxToken::Keyword => vec![
+                "keyword",
+                "keyword.control",
+                "keyword.control.fun",
+                "keyword.other.fn.rust",
+                "punctuation.accessor",
+                "entity.name.tag",
+            ],
             ZedSyntaxToken::Label => vec![
                 "label",
                 "entity.name",
@@ -161,7 +214,9 @@ impl ZedSyntaxToken {
             ZedSyntaxToken::LinkUri => vec!["markup.underline.link", "string.other.link"],
             ZedSyntaxToken::Number => vec!["constant.numeric", "number"],
             ZedSyntaxToken::Operator => vec!["operator", "keyword.operator"],
+            ZedSyntaxToken::Predictive => vec![],
             ZedSyntaxToken::Preproc => vec!["preproc"],
+            ZedSyntaxToken::Primary => vec![],
             ZedSyntaxToken::Property => vec![
                 "variable.member",
                 "support.type.property-name",
@@ -202,11 +257,20 @@ impl ZedSyntaxToken {
             ZedSyntaxToken::Tag => vec!["tag", "entity.name.tag", "meta.tag.sgml"],
             ZedSyntaxToken::TextLiteral => vec!["text.literal", "string"],
             ZedSyntaxToken::Title => vec!["title", "entity.name"],
-            ZedSyntaxToken::Type => vec!["entity.name.type", "support.type", "support.class"],
+            ZedSyntaxToken::Type => vec![
+                "entity.name.type",
+                "entity.name.type.primitive",
+                "entity.name.type.numeric",
+                "keyword.type",
+                "support.type",
+                "support.type.primitive",
+                "support.class",
+            ],
             ZedSyntaxToken::Variable => vec![
                 "variable",
                 "variable.language",
                 "variable.member",
+                "variable.parameter",
                 "variable.parameter.function-call",
             ],
             ZedSyntaxToken::VariableSpecial => vec![
@@ -216,7 +280,6 @@ impl ZedSyntaxToken {
                 "variable.language",
             ],
             ZedSyntaxToken::Variant => vec!["variant"],
-            _ => vec![],
         }
     }
 }