Better highlightign API

crates/ra_ide/src/lib.rs

@@ -74,7 +74,9 @@ pub use crate::{
     runnables::{Runnable, RunnableKind, TestId},
     source_change::{FileSystemEdit, SourceChange, SourceFileEdit},
     ssr::SsrError,
-    syntax_highlighting::{HighlightTag, HighlightedRange},
+    syntax_highlighting::{
+        Highlight, HighlightModifier, HighlightModifiers, HighlightTag, HighlightedRange,
+    },
 };
 
 pub use hir::Documentation;

crates/ra_ide/src/syntax_highlighting.rs

@@ -1,6 +1,6 @@
 //! FIXME: write short doc here
 
-mod highlight_tag;
+mod highlight;
 
 use hir::{Name, Semantics};
 use ra_db::SourceDatabase;
@@ -17,12 +17,12 @@ use rustc_hash::FxHashMap;
 
 use crate::{references::classify_name_ref, FileId};
 
-pub use highlight_tag::HighlightTag;
+pub use highlight::{Highlight, HighlightModifier, HighlightModifiers, HighlightTag};
 
 #[derive(Debug)]
 pub struct HighlightedRange {
     pub range: TextRange,
-    pub tag: HighlightTag,
+    pub highlight: Highlight,
     pub binding_hash: Option<u64>,
 }
 
@@ -79,33 +79,33 @@ pub(crate) fn highlight(
                         if let Some(range) = highlight_macro(node) {
                             res.push(HighlightedRange {
                                 range,
-                                tag: HighlightTag::MACRO,
+                                highlight: HighlightTag::Macro.into(),
                                 binding_hash: None,
                             });
                         }
                     }
                     _ if in_macro_call.is_some() => {
                         if let Some(token) = node.as_token() {
-                            if let Some((tag, binding_hash)) = highlight_token_tree(
+                            if let Some((highlight, binding_hash)) = highlight_token_tree(
                                 &sema,
                                 &mut bindings_shadow_count,
                                 token.clone(),
                             ) {
                                 res.push(HighlightedRange {
                                     range: node.text_range(),
-                                    tag,
+                                    highlight,
                                     binding_hash,
                                 });
                             }
                         }
                     }
                     _ => {
-                        if let Some((tag, binding_hash)) =
+                        if let Some((highlight, binding_hash)) =
                             highlight_node(&sema, &mut bindings_shadow_count, node.clone())
                         {
                             res.push(HighlightedRange {
                                 range: node.text_range(),
-                                tag,
+                                highlight,
                                 binding_hash,
                             });
                         }
@@ -150,7 +150,7 @@ fn highlight_token_tree(
     sema: &Semantics<RootDatabase>,
     bindings_shadow_count: &mut FxHashMap<Name, u32>,
     token: SyntaxToken,
-) -> Option<(HighlightTag, Option<u64>)> {
+) -> Option<(Highlight, Option<u64>)> {
     if token.parent().kind() != TOKEN_TREE {
         return None;
     }
@@ -171,19 +171,21 @@ fn highlight_node(
     sema: &Semantics<RootDatabase>,
     bindings_shadow_count: &mut FxHashMap<Name, u32>,
     node: SyntaxElement,
-) -> Option<(HighlightTag, Option<u64>)> {
+) -> Option<(Highlight, Option<u64>)> {
     let db = sema.db;
     let mut binding_hash = None;
-    let tag = match node.kind() {
+    let highlight: Highlight = match node.kind() {
         FN_DEF => {
             bindings_shadow_count.clear();
             return None;
         }
-        COMMENT => HighlightTag::LITERAL_COMMENT,
+        COMMENT => HighlightTag::Comment.into(),
-        STRING | RAW_STRING | RAW_BYTE_STRING | BYTE_STRING => HighlightTag::LITERAL_STRING,
+        STRING | RAW_STRING | RAW_BYTE_STRING | BYTE_STRING => HighlightTag::LiteralString.into(),
-        ATTR => HighlightTag::LITERAL_ATTRIBUTE,
+        ATTR => HighlightTag::Attribute.into(),
         // Special-case field init shorthand
-        NAME_REF if node.parent().and_then(ast::RecordField::cast).is_some() => HighlightTag::FIELD,
+        NAME_REF if node.parent().and_then(ast::RecordField::cast).is_some() => {
+            HighlightTag::Field.into()
+        }
         NAME_REF if node.ancestors().any(|it| it.kind() == ATTR) => return None,
         NAME_REF => {
             let name_ref = node.as_node().cloned().and_then(ast::NameRef::cast).unwrap();
@@ -217,26 +219,30 @@ fn highlight_node(
 
             match name_kind {
                 Some(name_kind) => highlight_name(db, name_kind),
-                None => name.syntax().parent().map_or(HighlightTag::FUNCTION, |x| match x.kind() {
+                None => name.syntax().parent().map_or(HighlightTag::Function.into(), |x| {
-                    STRUCT_DEF | ENUM_DEF | TRAIT_DEF | TYPE_ALIAS_DEF => HighlightTag::TYPE,
+                    match x.kind() {
-                    TYPE_PARAM => HighlightTag::TYPE_PARAM,
+                        STRUCT_DEF | ENUM_DEF | TRAIT_DEF | TYPE_ALIAS_DEF => {
-                    RECORD_FIELD_DEF => HighlightTag::FIELD,
+                            HighlightTag::Type.into()
-                    _ => HighlightTag::FUNCTION,
+                        }
+                        TYPE_PARAM => HighlightTag::TypeParam.into(),
+                        RECORD_FIELD_DEF => HighlightTag::Field.into(),
+                        _ => HighlightTag::Function.into(),
+                    }
                 }),
             }
         }
-        INT_NUMBER | FLOAT_NUMBER => HighlightTag::LITERAL_NUMERIC,
+        INT_NUMBER | FLOAT_NUMBER => HighlightTag::LiteralNumeric.into(),
-        BYTE => HighlightTag::LITERAL_BYTE,
+        BYTE => HighlightTag::LiteralByte.into(),
-        CHAR => HighlightTag::LITERAL_CHAR,
+        CHAR => HighlightTag::LiteralChar.into(),
-        LIFETIME => HighlightTag::TYPE_LIFETIME,
+        LIFETIME => HighlightTag::TypeLifetime.into(),
-        T![unsafe] => HighlightTag::KEYWORD_UNSAFE,
+        T![unsafe] => HighlightTag::Keyword | HighlightModifier::Unsafe,
-        k if is_control_keyword(k) => HighlightTag::KEYWORD_CONTROL,
+        k if is_control_keyword(k) => HighlightTag::Keyword | HighlightModifier::Control,
-        k if k.is_keyword() => HighlightTag::KEYWORD,
+        k if k.is_keyword() => HighlightTag::Keyword.into(),
 
         _ => return None,
     };
 
-    return Some((tag, binding_hash));
+    return Some((highlight, binding_hash));
 
     fn calc_binding_hash(name: &Name, shadow_count: u32) -> u64 {
         fn hash<T: std::hash::Hash + std::fmt::Debug>(x: T) -> u64 {
@@ -293,7 +299,11 @@ pub(crate) fn highlight_as_html(db: &RootDatabase, file_id: FileId, rainbow: boo
         if ranges.is_empty() {
             buf.push_str(&text);
         } else {
-            let classes = ranges.iter().map(|x| x.tag.to_string()).collect::<Vec<_>>().join(" ");
+            let classes = ranges
+                .iter()
+                .map(|it| it.highlight.to_string().replace('.', " "))
+                .collect::<Vec<_>>()
+                .join(" ");
             let binding_hash = ranges.first().and_then(|x| x.binding_hash);
             let color = match (rainbow, binding_hash) {
                 (true, Some(hash)) => format!(
@@ -310,29 +320,32 @@ pub(crate) fn highlight_as_html(db: &RootDatabase, file_id: FileId, rainbow: boo
     buf
 }
 
-fn highlight_name(db: &RootDatabase, def: NameDefinition) -> HighlightTag {
+fn highlight_name(db: &RootDatabase, def: NameDefinition) -> Highlight {
     match def {
-        NameDefinition::Macro(_) => HighlightTag::MACRO,
+        NameDefinition::Macro(_) => HighlightTag::Macro,
-        NameDefinition::StructField(_) => HighlightTag::FIELD,
+        NameDefinition::StructField(_) => HighlightTag::Field,
-        NameDefinition::ModuleDef(hir::ModuleDef::Module(_)) => HighlightTag::MODULE,
+        NameDefinition::ModuleDef(hir::ModuleDef::Module(_)) => HighlightTag::Module,
-        NameDefinition::ModuleDef(hir::ModuleDef::Function(_)) => HighlightTag::FUNCTION,
+        NameDefinition::ModuleDef(hir::ModuleDef::Function(_)) => HighlightTag::Function,
-        NameDefinition::ModuleDef(hir::ModuleDef::Adt(_)) => HighlightTag::TYPE,
+        NameDefinition::ModuleDef(hir::ModuleDef::Adt(_)) => HighlightTag::Type,
-        NameDefinition::ModuleDef(hir::ModuleDef::EnumVariant(_)) => HighlightTag::CONSTANT,
+        NameDefinition::ModuleDef(hir::ModuleDef::EnumVariant(_)) => HighlightTag::Constant,
-        NameDefinition::ModuleDef(hir::ModuleDef::Const(_)) => HighlightTag::CONSTANT,
+        NameDefinition::ModuleDef(hir::ModuleDef::Const(_)) => HighlightTag::Constant,
-        NameDefinition::ModuleDef(hir::ModuleDef::Static(_)) => HighlightTag::CONSTANT,
+        NameDefinition::ModuleDef(hir::ModuleDef::Static(_)) => HighlightTag::Constant,
-        NameDefinition::ModuleDef(hir::ModuleDef::Trait(_)) => HighlightTag::TYPE,
+        NameDefinition::ModuleDef(hir::ModuleDef::Trait(_)) => HighlightTag::Type,
-        NameDefinition::ModuleDef(hir::ModuleDef::TypeAlias(_)) => HighlightTag::TYPE,
+        NameDefinition::ModuleDef(hir::ModuleDef::TypeAlias(_)) => HighlightTag::Type,
-        NameDefinition::ModuleDef(hir::ModuleDef::BuiltinType(_)) => HighlightTag::TYPE_BUILTIN,
+        NameDefinition::ModuleDef(hir::ModuleDef::BuiltinType(_)) => {
-        NameDefinition::SelfType(_) => HighlightTag::TYPE_SELF,
+            return HighlightTag::Type | HighlightModifier::Builtin
-        NameDefinition::TypeParam(_) => HighlightTag::TYPE_PARAM,
+        }
+        NameDefinition::SelfType(_) => HighlightTag::TypeSelf,
+        NameDefinition::TypeParam(_) => HighlightTag::TypeParam,
         NameDefinition::Local(local) => {
+            let mut h = Highlight::new(HighlightTag::Variable);
             if local.is_mut(db) || local.ty(db).is_mutable_reference() {
-                HighlightTag::VARIABLE_MUT
+                h |= HighlightModifier::Mutable;
-            } else {
-                HighlightTag::VARIABLE
             }
+            return h;
         }
     }
+    .into()
 }
 
 //FIXME: like, real html escaping
@@ -498,6 +511,6 @@ fn bar() {
             })
             .unwrap();
 
-        assert_eq!(&highlights[0].tag.to_string(), "field");
+        assert_eq!(&highlights[0].highlight.to_string(), "field");
     }
 }

crates/ra_ide/src/syntax_highlighting/highlight.rs

@@ -0,0 +1,163 @@
+//! Defines token tags we use for syntax highlighting.
+//! A tag is not unlike a CSS class.
+
+use std::{fmt, ops};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct Highlight {
+    pub tag: HighlightTag,
+    pub modifiers: HighlightModifiers,
+}
+
+#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub struct HighlightModifiers(u32);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum HighlightTag {
+    Field,
+    Function,
+    Module,
+    Constant,
+    Macro,
+    Variable,
+
+    Type,
+    TypeSelf,
+    TypeParam,
+    TypeLifetime,
+
+    LiteralByte,
+    LiteralNumeric,
+    LiteralChar,
+
+    Comment,
+    LiteralString,
+    Attribute,
+
+    Keyword,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[repr(u8)]
+pub enum HighlightModifier {
+    Mutable = 0,
+    Unsafe,
+    /// Used with keywords like `if` and `break`.
+    Control,
+    Builtin,
+}
+
+impl HighlightTag {
+    fn as_str(self) -> &'static str {
+        match self {
+            HighlightTag::Field => "field",
+            HighlightTag::Function => "function",
+            HighlightTag::Module => "module",
+            HighlightTag::Constant => "constant",
+            HighlightTag::Macro => "macro",
+            HighlightTag::Variable => "variable",
+            HighlightTag::Type => "type",
+            HighlightTag::TypeSelf => "type.self",
+            HighlightTag::TypeParam => "type.param",
+            HighlightTag::TypeLifetime => "type.lifetime",
+            HighlightTag::LiteralByte => "literal.byte",
+            HighlightTag::LiteralNumeric => "literal.numeric",
+            HighlightTag::LiteralChar => "literal.char",
+            HighlightTag::Comment => "comment",
+            HighlightTag::LiteralString => "string",
+            HighlightTag::Attribute => "attribute",
+            HighlightTag::Keyword => "keyword",
+        }
+    }
+}
+
+impl fmt::Display for HighlightTag {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self.as_str(), f)
+    }
+}
+
+impl HighlightModifier {
+    const ALL: &'static [HighlightModifier] = &[
+        HighlightModifier::Mutable,
+        HighlightModifier::Unsafe,
+        HighlightModifier::Control,
+        HighlightModifier::Builtin,
+    ];
+
+    fn as_str(self) -> &'static str {
+        match self {
+            HighlightModifier::Mutable => "mutable",
+            HighlightModifier::Unsafe => "unsafe",
+            HighlightModifier::Control => "control",
+            HighlightModifier::Builtin => "builtin",
+        }
+    }
+
+    fn mask(self) -> u32 {
+        1 << (self as u32)
+    }
+}
+
+impl fmt::Display for HighlightModifier {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self.as_str(), f)
+    }
+}
+
+impl fmt::Display for Highlight {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.tag)?;
+        for modifier in self.modifiers.iter() {
+            write!(f, ".{}", modifier)?
+        }
+        Ok(())
+    }
+}
+
+impl From<HighlightTag> for Highlight {
+    fn from(tag: HighlightTag) -> Highlight {
+        Highlight::new(tag)
+    }
+}
+
+impl Highlight {
+    pub(crate) fn new(tag: HighlightTag) -> Highlight {
+        Highlight { tag, modifiers: HighlightModifiers::default() }
+    }
+}
+
+impl ops::BitOr<HighlightModifier> for HighlightTag {
+    type Output = Highlight;
+
+    fn bitor(self, rhs: HighlightModifier) -> Highlight {
+        Highlight::new(self) | rhs
+    }
+}
+
+impl ops::BitOrAssign<HighlightModifier> for HighlightModifiers {
+    fn bitor_assign(&mut self, rhs: HighlightModifier) {
+        self.0 |= rhs.mask();
+    }
+}
+
+impl ops::BitOrAssign<HighlightModifier> for Highlight {
+    fn bitor_assign(&mut self, rhs: HighlightModifier) {
+        self.modifiers |= rhs;
+    }
+}
+
+impl ops::BitOr<HighlightModifier> for Highlight {
+    type Output = Highlight;
+
+    fn bitor(mut self, rhs: HighlightModifier) -> Highlight {
+        self |= rhs;
+        self
+    }
+}
+
+impl HighlightModifiers {
+    pub fn iter(self) -> impl Iterator<Item = HighlightModifier> {
+        HighlightModifier::ALL.iter().copied().filter(move |it| self.0 & it.mask() == it.mask())
+    }
+}

crates/ra_ide/src/syntax_highlighting/highlight_tag.rs

@@ -1,43 +0,0 @@
-//! Defines token tags we use for syntax highlighting.
-//! A tag is not unlike a CSS class.
-
-use std::fmt;
-
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub struct HighlightTag(&'static str);
-
-impl fmt::Display for HighlightTag {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Display::fmt(self.0, f)
-    }
-}
-
-#[rustfmt::skip]
-impl HighlightTag {
-    pub const FIELD: HighlightTag              = HighlightTag("field");
-    pub const FUNCTION: HighlightTag           = HighlightTag("function");
-    pub const MODULE: HighlightTag             = HighlightTag("module");
-    pub const CONSTANT: HighlightTag           = HighlightTag("constant");
-    pub const MACRO: HighlightTag              = HighlightTag("macro");
-
-    pub const VARIABLE: HighlightTag           = HighlightTag("variable");
-    pub const VARIABLE_MUT: HighlightTag       = HighlightTag("variable.mut");
-
-    pub const TYPE: HighlightTag               = HighlightTag("type");
-    pub const TYPE_BUILTIN: HighlightTag       = HighlightTag("type.builtin");
-    pub const TYPE_SELF: HighlightTag          = HighlightTag("type.self");
-    pub const TYPE_PARAM: HighlightTag         = HighlightTag("type.param");
-    pub const TYPE_LIFETIME: HighlightTag      = HighlightTag("type.lifetime");
-
-    pub const LITERAL_BYTE: HighlightTag       = HighlightTag("literal.byte");
-    pub const LITERAL_NUMERIC: HighlightTag    = HighlightTag("literal.numeric");
-    pub const LITERAL_CHAR: HighlightTag       = HighlightTag("literal.char");
-
-    pub const LITERAL_COMMENT: HighlightTag    = HighlightTag("comment");
-    pub const LITERAL_STRING: HighlightTag     = HighlightTag("string");
-    pub const LITERAL_ATTRIBUTE: HighlightTag  = HighlightTag("attribute");
-
-    pub const KEYWORD: HighlightTag            = HighlightTag("keyword");
-    pub const KEYWORD_UNSAFE: HighlightTag     = HighlightTag("keyword.unsafe");
-    pub const KEYWORD_CONTROL: HighlightTag    = HighlightTag("keyword.control");
-}

crates/rust-analyzer/src/caps.rs

@@ -63,11 +63,8 @@ pub fn server_capabilities() -> ServerCapabilities {
         semantic_tokens_provider: Some(
             SemanticTokensOptions {
                 legend: SemanticTokensLegend {
-                    token_types: semantic_tokens::supported_token_types().iter().cloned().collect(),
+                    token_types: semantic_tokens::SUPPORTED_TYPES.iter().cloned().collect(),
-                    token_modifiers: semantic_tokens::supported_token_modifiers()
+                    token_modifiers: semantic_tokens::SUPPORTED_MODIFIERS.iter().cloned().collect(),
-                        .iter()
-                        .cloned()
-                        .collect(),
                 },
 
                 document_provider: Some(SemanticTokensDocumentProvider::Bool(true)),

crates/rust-analyzer/src/conv.rs

@@ -10,14 +10,21 @@ use lsp_types::{
 };
 use ra_ide::{
     translate_offset_with_edit, CompletionItem, CompletionItemKind, FileId, FilePosition,
-    FileRange, FileSystemEdit, Fold, FoldKind, HighlightTag, InsertTextFormat, LineCol, LineIndex,
+    FileRange, FileSystemEdit, Fold, FoldKind, Highlight, HighlightModifier, HighlightTag,
-    NavigationTarget, RangeInfo, ReferenceAccess, Severity, SourceChange, SourceFileEdit,
+    InsertTextFormat, LineCol, LineIndex, NavigationTarget, RangeInfo, ReferenceAccess, Severity,
+    SourceChange, SourceFileEdit,
 };
 use ra_syntax::{SyntaxKind, TextRange, TextUnit};
 use ra_text_edit::{AtomTextEdit, TextEdit};
 use ra_vfs::LineEndings;
 
-use crate::{req, semantic_tokens, world::WorldSnapshot, Result};
+use crate::{
+    req,
+    semantic_tokens::{self, ModifierSet, BUILTIN, CONTROL, MUTABLE, UNSAFE},
+    world::WorldSnapshot,
+    Result,
+};
+use semantic_tokens::ATTRIBUTE;
 
 pub trait Conv {
     type Output;
@@ -303,74 +310,52 @@ impl ConvWith<&FoldConvCtx<'_>> for Fold {
     }
 }
 
-impl Conv for HighlightTag {
+impl Conv for Highlight {
-    type Output = (SemanticTokenType, Vec<SemanticTokenModifier>);
-
-    fn conv(self) -> (SemanticTokenType, Vec<SemanticTokenModifier>) {
-        let token_type: SemanticTokenType = match self {
-            HighlightTag::FIELD => SemanticTokenType::MEMBER,
-            HighlightTag::FUNCTION => SemanticTokenType::FUNCTION,
-            HighlightTag::MODULE => SemanticTokenType::NAMESPACE,
-            HighlightTag::CONSTANT => {
-                return (
-                    SemanticTokenType::VARIABLE,
-                    vec![SemanticTokenModifier::STATIC, SemanticTokenModifier::READONLY],
-                )
-            }
-            HighlightTag::MACRO => SemanticTokenType::MACRO,
-
-            HighlightTag::VARIABLE => {
-                return (SemanticTokenType::VARIABLE, vec![SemanticTokenModifier::READONLY])
-            }
-            HighlightTag::VARIABLE_MUT => SemanticTokenType::VARIABLE,
-
-            HighlightTag::TYPE => SemanticTokenType::TYPE,
-            HighlightTag::TYPE_BUILTIN => SemanticTokenType::TYPE,
-            HighlightTag::TYPE_SELF => {
-                return (SemanticTokenType::TYPE, vec![SemanticTokenModifier::REFERENCE])
-            }
-            HighlightTag::TYPE_PARAM => SemanticTokenType::TYPE_PARAMETER,
-            HighlightTag::TYPE_LIFETIME => {
-                return (SemanticTokenType::LABEL, vec![SemanticTokenModifier::REFERENCE])
-            }
-
-            HighlightTag::LITERAL_BYTE => SemanticTokenType::NUMBER,
-            HighlightTag::LITERAL_NUMERIC => SemanticTokenType::NUMBER,
-            HighlightTag::LITERAL_CHAR => SemanticTokenType::NUMBER,
-
-            HighlightTag::LITERAL_COMMENT => {
-                return (SemanticTokenType::COMMENT, vec![SemanticTokenModifier::DOCUMENTATION])
-            }
-
-            HighlightTag::LITERAL_STRING => SemanticTokenType::STRING,
-            HighlightTag::LITERAL_ATTRIBUTE => SemanticTokenType::KEYWORD,
-
-            HighlightTag::KEYWORD => SemanticTokenType::KEYWORD,
-            HighlightTag::KEYWORD_UNSAFE => SemanticTokenType::KEYWORD,
-            HighlightTag::KEYWORD_CONTROL => SemanticTokenType::KEYWORD,
-            unknown => panic!("Unknown semantic token: {}", unknown),
-        };
-
-        (token_type, vec![])
-    }
-}
-
-impl Conv for (SemanticTokenType, Vec<SemanticTokenModifier>) {
     type Output = (u32, u32);
 
     fn conv(self) -> Self::Output {
-        let token_index =
+        let mut mods = ModifierSet::default();
-            semantic_tokens::supported_token_types().iter().position(|it| *it == self.0).unwrap();
+        let type_ = match self.tag {
-        let mut token_modifier_bitset = 0;
+            HighlightTag::Field => SemanticTokenType::MEMBER,
-        for modifier in self.1.iter() {
+            HighlightTag::Function => SemanticTokenType::FUNCTION,
-            let modifier_index = semantic_tokens::supported_token_modifiers()
+            HighlightTag::Module => SemanticTokenType::NAMESPACE,
-                .iter()
+            HighlightTag::Constant => {
-                .position(|it| it == modifier)
+                mods |= SemanticTokenModifier::STATIC;
-                .unwrap();
+                mods |= SemanticTokenModifier::READONLY;
-            token_modifier_bitset |= 1 << modifier_index;
+                SemanticTokenType::VARIABLE
+            }
+            HighlightTag::Macro => SemanticTokenType::MACRO,
+            HighlightTag::Variable => SemanticTokenType::VARIABLE,
+            HighlightTag::Type => SemanticTokenType::TYPE,
+            HighlightTag::TypeSelf => {
+                mods |= SemanticTokenModifier::REFERENCE;
+                SemanticTokenType::TYPE
+            }
+            HighlightTag::TypeParam => SemanticTokenType::TYPE_PARAMETER,
+            HighlightTag::TypeLifetime => {
+                mods |= SemanticTokenModifier::REFERENCE;
+                SemanticTokenType::LABEL
+            }
+            HighlightTag::LiteralByte => SemanticTokenType::NUMBER,
+            HighlightTag::LiteralNumeric => SemanticTokenType::NUMBER,
+            HighlightTag::LiteralChar => SemanticTokenType::NUMBER,
+            HighlightTag::Comment => SemanticTokenType::COMMENT,
+            HighlightTag::LiteralString => SemanticTokenType::STRING,
+            HighlightTag::Attribute => ATTRIBUTE,
+            HighlightTag::Keyword => SemanticTokenType::KEYWORD,
+        };
+
+        for modifier in self.modifiers.iter() {
+            let modifier = match modifier {
+                HighlightModifier::Mutable => MUTABLE,
+                HighlightModifier::Unsafe => UNSAFE,
+                HighlightModifier::Control => CONTROL,
+                HighlightModifier::Builtin => BUILTIN,
+            };
+            mods |= modifier;
         }
 
-        (token_index as u32, token_modifier_bitset as u32)
+        (semantic_tokens::type_index(type_), mods.0)
     }
 }
 

crates/rust-analyzer/src/main_loop/handlers.rs

@@ -16,9 +16,9 @@ use lsp_types::{
     CodeAction, CodeActionOrCommand, CodeActionResponse, CodeLens, Command, CompletionItem,
     Diagnostic, DocumentFormattingParams, DocumentHighlight, DocumentSymbol, FoldingRange,
     FoldingRangeParams, Hover, HoverContents, Location, MarkupContent, MarkupKind, Position,
-    PrepareRenameResponse, Range, RenameParams, SemanticTokenModifier, SemanticTokenType,
+    PrepareRenameResponse, Range, RenameParams, SemanticTokens, SemanticTokensParams,
-    SemanticTokens, SemanticTokensParams, SemanticTokensRangeParams, SemanticTokensRangeResult,
+    SemanticTokensRangeParams, SemanticTokensRangeResult, SemanticTokensResult, SymbolInformation,
-    SemanticTokensResult, SymbolInformation, TextDocumentIdentifier, TextEdit, WorkspaceEdit,
+    TextDocumentIdentifier, TextEdit, WorkspaceEdit,
 };
 use ra_ide::{
     AssistId, FileId, FilePosition, FileRange, Query, RangeInfo, Runnable, RunnableKind,
@@ -954,7 +954,7 @@ fn highlight(world: &WorldSnapshot, file_id: FileId) -> Result<Vec<Decoration>>
         .into_iter()
         .map(|h| Decoration {
             range: h.range.conv_with(&line_index),
-            tag: h.tag.to_string(),
+            tag: h.highlight.to_string(),
             binding_hash: h.binding_hash.map(|x| x.to_string()),
         })
         .collect();
@@ -1082,10 +1082,9 @@ pub fn handle_semantic_tokens(
 
     let mut builder = SemanticTokensBuilder::default();
 
-    for h in world.analysis().highlight(file_id)?.into_iter() {
+    for highlight_range in world.analysis().highlight(file_id)?.into_iter() {
-        let type_and_modifiers: (SemanticTokenType, Vec<SemanticTokenModifier>) = h.tag.conv();
+        let (token_type, token_modifiers) = highlight_range.highlight.conv();
-        let (token_type, token_modifiers) = type_and_modifiers.conv();
+        builder.push(highlight_range.range.conv_with(&line_index), token_type, token_modifiers);
-        builder.push(h.range.conv_with(&line_index), token_type, token_modifiers);
     }
 
     let tokens = SemanticTokens { data: builder.build(), ..Default::default() };
@@ -1104,10 +1103,9 @@ pub fn handle_semantic_tokens_range(
 
     let mut builder = SemanticTokensBuilder::default();
 
-    for h in world.analysis().highlight_range(frange)?.into_iter() {
+    for highlight_range in world.analysis().highlight_range(frange)?.into_iter() {
-        let type_and_modifiers: (SemanticTokenType, Vec<SemanticTokenModifier>) = h.tag.conv();
+        let (token_type, token_modifiers) = highlight_range.highlight.conv();
-        let (token_type, token_modifiers) = type_and_modifiers.conv();
+        builder.push(highlight_range.range.conv_with(&line_index), token_type, token_modifiers);
-        builder.push(h.range.conv_with(&line_index), token_type, token_modifiers);
     }
 
     let tokens = SemanticTokens { data: builder.build(), ..Default::default() };

crates/rust-analyzer/src/semantic_tokens.rs

@@ -1,8 +1,17 @@
 //! Semantic Tokens helpers
 
+use std::ops;
+
 use lsp_types::{Range, SemanticToken, SemanticTokenModifier, SemanticTokenType};
 
-const SUPPORTED_TYPES: &[SemanticTokenType] = &[
+pub(crate) const ATTRIBUTE: SemanticTokenType = SemanticTokenType::new("attribute");
+
+pub(crate) const MUTABLE: SemanticTokenModifier = SemanticTokenModifier::new("mutable");
+pub(crate) const UNSAFE: SemanticTokenModifier = SemanticTokenModifier::new("unsafe");
+pub(crate) const CONTROL: SemanticTokenModifier = SemanticTokenModifier::new("control");
+pub(crate) const BUILTIN: SemanticTokenModifier = SemanticTokenModifier::new("builtin");
+
+pub(crate) const SUPPORTED_TYPES: &[SemanticTokenType] = &[
     SemanticTokenType::COMMENT,
     SemanticTokenType::KEYWORD,
     SemanticTokenType::STRING,
@@ -23,9 +32,10 @@ const SUPPORTED_TYPES: &[SemanticTokenType] = &[
     SemanticTokenType::VARIABLE,
     SemanticTokenType::PARAMETER,
     SemanticTokenType::LABEL,
+    ATTRIBUTE,
 ];
 
-const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
+pub(crate) const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
     SemanticTokenModifier::DOCUMENTATION,
     SemanticTokenModifier::DECLARATION,
     SemanticTokenModifier::DEFINITION,
@@ -36,16 +46,20 @@ const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
     SemanticTokenModifier::ASYNC,
     SemanticTokenModifier::VOLATILE,
     SemanticTokenModifier::READONLY,
+    MUTABLE,
+    UNSAFE,
+    CONTROL,
+    BUILTIN,
 ];
 
-/// Token types that the server supports
+#[derive(Default)]
-pub(crate) fn supported_token_types() -> &'static [SemanticTokenType] {
+pub(crate) struct ModifierSet(pub(crate) u32);
-    SUPPORTED_TYPES
-}
 
-/// Token modifiers that the server supports
+impl ops::BitOrAssign<SemanticTokenModifier> for ModifierSet {
-pub(crate) fn supported_token_modifiers() -> &'static [SemanticTokenModifier] {
+    fn bitor_assign(&mut self, rhs: SemanticTokenModifier) {
-    SUPPORTED_MODIFIERS
+        let idx = SUPPORTED_MODIFIERS.iter().position(|it| it == &rhs).unwrap();
+        self.0 |= 1 << idx;
+    }
 }
 
 /// Tokens are encoded relative to each other.
@@ -92,3 +106,7 @@ impl SemanticTokensBuilder {
         self.data
     }
 }
+
+pub fn type_index(type_: SemanticTokenType) -> u32 {
+    SUPPORTED_TYPES.iter().position(|it| *it == type_).unwrap() as u32
+}

editors/code/package.json

@@ -380,6 +380,25 @@
                 }
             }
         ],
+        "semanticTokenTypes": [
+            {
+                "id": "attribute"
+            }
+        ],
+        "semanticTokenModifiers": [
+            {
+                "id": "mutable"
+            },
+            {
+                "id": "unsafe"
+            },
+            {
+                "id": "control"
+            },
+            {
+                "id": "builtin"
+            }
+        ],
         "semanticTokenStyleDefaults": [
             {
                 "selector": "*.mutable",