add ast::tokens

crates/ra_assists/src/add_missing_impl_members.rs

@@ -5,7 +5,7 @@ use crate::{Assist, AssistId, AssistCtx};
 use hir::Resolver;
 use hir::db::HirDatabase;
 use ra_syntax::{SmolStr, SyntaxKind, TextRange, TextUnit, TreeArc};
-use ra_syntax::ast::{self, AstNode, FnDef, ImplItem, ImplItemKind, NameOwner};
+use ra_syntax::ast::{self, AstNode, AstToken, FnDef, ImplItem, ImplItemKind, NameOwner};
 use ra_db::FilePosition;
 use ra_fmt::{leading_indent, reindent};
 

crates/ra_assists/src/inline_local_variable.rs

@@ -1,14 +1,9 @@
 use hir::{
     db::HirDatabase,
-    source_binder::function_from_child_node
+    source_binder::function_from_child_node,
 };
 use ra_syntax::{
-    ast::{
-        self,
-        AstNode,
-        PatKind,
-        ExprKind
-    },
+    ast::{self, AstNode, AstToken, PatKind, ExprKind},
     TextRange,
 };
 

crates/ra_fmt/src/lib.rs

@@ -2,9 +2,8 @@
 //!
 use itertools::Itertools;
 use ra_syntax::{
-    AstNode,
     SyntaxNode, SyntaxKind::*, SyntaxToken, SyntaxKind,
-    ast,
+    ast::{self, AstNode, AstToken},
     algo::generate,
 };
 

crates/ra_ide_api/src/extend_selection.rs

@@ -1,9 +1,9 @@
 use ra_db::SourceDatabase;
 use ra_syntax::{
-    Direction, SyntaxNode, TextRange, TextUnit, AstNode, SyntaxElement,
+    Direction, SyntaxNode, TextRange, TextUnit, SyntaxElement,
     algo::{find_covering_element, find_token_at_offset, TokenAtOffset},
     SyntaxKind::*, SyntaxToken,
-    ast::Comment,
+    ast::{self, AstNode, AstToken},
 };
 
 use crate::{FileRange, db::RootDatabase};
@@ -55,7 +55,7 @@ fn try_extend_selection(root: &SyntaxNode, range: TextRange) -> Option<TextRange
             if token.range() != range {
                 return Some(token.range());
             }
-            if let Some(comment) = Comment::cast(token) {
+            if let Some(comment) = ast::Comment::cast(token) {
                 if let Some(range) = extend_comments(comment) {
                     return Some(range);
                 }
@@ -176,7 +176,7 @@ fn extend_list_item(node: &SyntaxNode) -> Option<TextRange> {
     None
 }
 
-fn extend_comments(comment: Comment) -> Option<TextRange> {
+fn extend_comments(comment: ast::Comment) -> Option<TextRange> {
     let prev = adj_comments(comment, Direction::Prev);
     let next = adj_comments(comment, Direction::Next);
     if prev != next {
@@ -186,14 +186,14 @@ fn extend_comments(comment: Comment) -> Option<TextRange> {
     }
 }
 
-fn adj_comments(comment: Comment, dir: Direction) -> Comment {
+fn adj_comments(comment: ast::Comment, dir: Direction) -> ast::Comment {
     let mut res = comment;
     for element in comment.syntax().siblings_with_tokens(dir) {
         let token = match element.as_token() {
             None => break,
             Some(token) => token,
         };
-        if let Some(c) = Comment::cast(token) {
+        if let Some(c) = ast::Comment::cast(token) {
             res = c
         } else if token.kind() != WHITESPACE || token.text().contains("\n\n") {
             break;

crates/ra_ide_api/src/folding_ranges.rs

@@ -1,9 +1,9 @@
 use rustc_hash::FxHashSet;
 
 use ra_syntax::{
-    AstNode, SourceFile, SyntaxNode, TextRange, Direction, SyntaxElement,
+    SourceFile, SyntaxNode, TextRange, Direction, SyntaxElement,
     SyntaxKind::{self, *},
-    ast::{self, VisibilityOwner, Comment},
+    ast::{self, AstNode, AstToken, VisibilityOwner},
 };
 
 #[derive(Debug, PartialEq, Eq)]
@@ -139,8 +139,8 @@ fn contiguous_range_for_group_unless<'a>(
 }
 
 fn contiguous_range_for_comment<'a>(
-    first: Comment<'a>,
-    visited: &mut FxHashSet<Comment<'a>>,
+    first: ast::Comment<'a>,
+    visited: &mut FxHashSet<ast::Comment<'a>>,
 ) -> Option<TextRange> {
     visited.insert(first);
 
@@ -157,7 +157,7 @@ fn contiguous_range_for_comment<'a>(
                         continue;
                     }
                 }
-                if let Some(c) = Comment::cast(token) {
+                if let Some(c) = ast::Comment::cast(token) {
                     if c.flavor() == group_flavor {
                         visited.insert(c);
                         last = c;

crates/ra_ide_api/src/join_lines.rs

@@ -1,9 +1,9 @@
 use itertools::Itertools;
 use ra_syntax::{
-    SourceFile, TextRange, TextUnit, AstNode, SyntaxNode, SyntaxElement, SyntaxToken,
+    SourceFile, TextRange, TextUnit, SyntaxNode, SyntaxElement, SyntaxToken,
     SyntaxKind::{self, WHITESPACE, COMMA, R_CURLY, R_PAREN, R_BRACK},
     algo::{find_covering_element, non_trivia_sibling},
-    ast,
+    ast::{self, AstNode, AstToken},
     Direction,
 };
 use ra_fmt::{

crates/ra_ide_api/src/typing.rs

@@ -2,7 +2,7 @@ use ra_syntax::{
     AstNode, SourceFile, SyntaxKind::*,
     TextUnit, TextRange, SyntaxToken,
     algo::{find_node_at_offset, find_token_at_offset, TokenAtOffset},
-    ast::{self},
+    ast::{self, AstToken},
 };
 use ra_fmt::leading_indent;
 use ra_text_edit::{TextEdit, TextEditBuilder};

crates/ra_syntax/src/ast.rs

@@ -1,6 +1,7 @@
 //! Abstract Syntax Tree, layered on top of untyped `SyntaxNode`s
 mod generated;
 mod traits;
+mod tokens;
 
 use std::marker::PhantomData;
 
@@ -15,6 +16,7 @@ use crate::{
 pub use self::{
     generated::*,
     traits::*,
+    tokens::*,
 };
 
 /// The main trait to go from untyped `SyntaxNode`  to a typed ast. The
@@ -49,6 +51,16 @@ impl<'a, N: AstNode + 'a> Iterator for AstChildren<'a, N> {
     }
 }
 
+pub trait AstToken<'a> {
+    fn cast(token: SyntaxToken<'a>) -> Option<Self>
+    where
+        Self: Sized;
+    fn syntax(&self) -> SyntaxToken<'a>;
+    fn text(&self) -> &'a SmolStr {
+        self.syntax().text()
+    }
+}
+
 impl Attr {
     pub fn is_inner(&self) -> bool {
         let tt = match self.value() {
@@ -96,100 +108,6 @@ impl Attr {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Comment<'a>(SyntaxToken<'a>);
-
-impl<'a> Comment<'a> {
-    pub fn cast(token: SyntaxToken<'a>) -> Option<Self> {
-        if token.kind() == COMMENT {
-            Some(Comment(token))
-        } else {
-            None
-        }
-    }
-
-    pub fn syntax(&self) -> SyntaxToken<'a> {
-        self.0
-    }
-
-    pub fn text(&self) -> &'a SmolStr {
-        self.0.text()
-    }
-
-    pub fn flavor(&self) -> CommentFlavor {
-        let text = self.text();
-        if text.starts_with("///") {
-            CommentFlavor::Doc
-        } else if text.starts_with("//!") {
-            CommentFlavor::ModuleDoc
-        } else if text.starts_with("//") {
-            CommentFlavor::Line
-        } else {
-            CommentFlavor::Multiline
-        }
-    }
-
-    pub fn is_doc_comment(&self) -> bool {
-        self.flavor().is_doc_comment()
-    }
-
-    pub fn prefix(&self) -> &'static str {
-        self.flavor().prefix()
-    }
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub enum CommentFlavor {
-    Line,
-    Doc,
-    ModuleDoc,
-    Multiline,
-}
-
-impl CommentFlavor {
-    pub fn prefix(&self) -> &'static str {
-        use self::CommentFlavor::*;
-        match *self {
-            Line => "//",
-            Doc => "///",
-            ModuleDoc => "//!",
-            Multiline => "/*",
-        }
-    }
-
-    pub fn is_doc_comment(&self) -> bool {
-        match self {
-            CommentFlavor::Doc | CommentFlavor::ModuleDoc => true,
-            _ => false,
-        }
-    }
-}
-
-pub struct Whitespace<'a>(SyntaxToken<'a>);
-
-impl<'a> Whitespace<'a> {
-    pub fn cast(token: SyntaxToken<'a>) -> Option<Self> {
-        if token.kind() == WHITESPACE {
-            Some(Whitespace(token))
-        } else {
-            None
-        }
-    }
-
-    pub fn syntax(&self) -> SyntaxToken<'a> {
-        self.0
-    }
-
-    pub fn text(&self) -> &'a SmolStr {
-        self.0.text()
-    }
-
-    pub fn spans_multiple_lines(&self) -> bool {
-        let text = self.text();
-        text.find('\n').map_or(false, |idx| text[idx + 1..].contains('\n'))
-    }
-}
-
 impl Name {
     pub fn text(&self) -> &SmolStr {
         let ident = self.syntax().first_child_or_token().unwrap().as_token().unwrap();

crates/ra_syntax/src/ast/tokens.rs

@@ -0,0 +1,93 @@
+use crate::{
+    SyntaxToken,
+    SyntaxKind::{COMMENT, WHITESPACE},
+    ast::AstToken,
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Comment<'a>(SyntaxToken<'a>);
+
+impl<'a> AstToken<'a> for Comment<'a> {
+    fn cast(token: SyntaxToken<'a>) -> Option<Self> {
+        if token.kind() == COMMENT {
+            Some(Comment(token))
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> SyntaxToken<'a> {
+        self.0
+    }
+}
+
+impl<'a> Comment<'a> {
+    pub fn flavor(&self) -> CommentFlavor {
+        let text = self.text();
+        if text.starts_with("///") {
+            CommentFlavor::Doc
+        } else if text.starts_with("//!") {
+            CommentFlavor::ModuleDoc
+        } else if text.starts_with("//") {
+            CommentFlavor::Line
+        } else {
+            CommentFlavor::Multiline
+        }
+    }
+
+    pub fn is_doc_comment(&self) -> bool {
+        self.flavor().is_doc_comment()
+    }
+
+    pub fn prefix(&self) -> &'static str {
+        self.flavor().prefix()
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum CommentFlavor {
+    Line,
+    Doc,
+    ModuleDoc,
+    Multiline,
+}
+
+impl CommentFlavor {
+    pub fn prefix(&self) -> &'static str {
+        use self::CommentFlavor::*;
+        match *self {
+            Line => "//",
+            Doc => "///",
+            ModuleDoc => "//!",
+            Multiline => "/*",
+        }
+    }
+
+    pub fn is_doc_comment(&self) -> bool {
+        match self {
+            CommentFlavor::Doc | CommentFlavor::ModuleDoc => true,
+            _ => false,
+        }
+    }
+}
+
+pub struct Whitespace<'a>(SyntaxToken<'a>);
+
+impl<'a> AstToken<'a> for Whitespace<'a> {
+    fn cast(token: SyntaxToken<'a>) -> Option<Self> {
+        if token.kind() == WHITESPACE {
+            Some(Whitespace(token))
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> SyntaxToken<'a> {
+        self.0
+    }
+}
+
+impl<'a> Whitespace<'a> {
+    pub fn spans_multiple_lines(&self) -> bool {
+        let text = self.text();
+        text.find('\n').map_or(false, |idx| text[idx + 1..].contains('\n'))
+    }
+}

crates/ra_syntax/src/ast/traits.rs

@@ -2,7 +2,7 @@ use itertools::Itertools;
 
 use crate::{
     syntax_node::{SyntaxNodeChildren, SyntaxElementChildren},
-    ast::{self, child_opt, children, AstNode, AstChildren},
+    ast::{self, child_opt, children, AstNode, AstToken, AstChildren},
 };
 
 pub trait TypeAscriptionOwner: AstNode {