[PERF-EXPERIMENT] Keep two spans per token

Author: petrochenkov

compiler/rustc_ast/src/attr/mod.rs

@@ -407,6 +407,7 @@ impl MetaItem {
             Some(TokenTree::Token(Token {
                 kind: kind @ (token::Ident(..) | token::ModSep),
                 span,
+                ..
             })) => 'arm: {
                 let mut segments = if let token::Ident(name, _) = kind {
                     if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
@@ -420,8 +421,9 @@ impl MetaItem {
                     vec![PathSegment::path_root(span)]
                 };
                 loop {
-                    if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
-                        tokens.next().map(TokenTree::uninterpolate)
+                    if let Some(TokenTree::Token(Token {
+                        kind: token::Ident(name, _), span, ..
+                    })) = tokens.next().map(TokenTree::uninterpolate)
                     {
                         segments.push(PathSegment::from_ident(Ident::new(name, span)));
                     } else {

compiler/rustc_ast/src/mut_visit.rs

@@ -719,13 +719,14 @@ pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis
 // In practice the ident part is not actually used by specific visitors right now,
 // but there's a test below checking that it works.
 pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
-    let Token { kind, span } = t;
+    let Token { kind, span, span2 } = t;
     match kind {
         token::Ident(name, _) | token::Lifetime(name) => {
             let mut ident = Ident::new(*name, *span);
             vis.visit_ident(&mut ident);
             *name = ident.name;
             *span = ident.span;
+            vis.visit_span(span2);
             return; // Avoid visiting the span for the second time.
         }
         token::Interpolated(nt) => {
@@ -735,6 +736,7 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
         _ => {}
     }
     vis.visit_span(span);
+    vis.visit_span(span2);
 }
 
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.

compiler/rustc_ast/src/token.rs

@@ -248,6 +248,7 @@ rustc_data_structures::static_assert_size!(TokenKind, 16);
 pub struct Token {
     pub kind: TokenKind,
     pub span: Span,
+    pub span2: Span,
 }
 
 impl TokenKind {
@@ -306,7 +307,7 @@ impl TokenKind {
 
 impl Token {
     pub fn new(kind: TokenKind, span: Span) -> Self {
-        Token { kind, span }
+        Token { kind, span, span2: span }
     }
 
     /// Some token that will be thrown away later.

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -95,7 +95,7 @@ type NamedMatchVec = SmallVec<[NamedMatch; 1]>;
 
 // This type is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(NamedMatchVec, 48);
+rustc_data_structures::static_assert_size!(NamedMatchVec, 56);
 
 #[derive(Clone)]
 enum MatcherKind<'tt> {

compiler/rustc_expand/src/mbe/quoted.rs

@@ -57,7 +57,9 @@ pub(super) fn parse(
         match tree {
             TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
                 let span = match trees.next() {
-                    Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
+                    Some(tokenstream::TokenTree::Token(Token {
+                        kind: token::Colon, span, ..
+                    })) => {
                         match trees.next() {
                             Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
                                 Some((frag, _)) => {
@@ -149,7 +151,7 @@ fn parse_tree(
     // Depending on what `tree` is, we could be parsing different parts of a macro
     match tree {
         // `tree` is a `$` token. Look at the next token in `trees`
-        tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
+        tokenstream::TokenTree::Token(Token { kind: token::Dollar, span, .. }) => {
             // FIXME: Handle `None`-delimited groups in a more systematic way
             // during parsing.
             let mut next = outer_trees.next();
@@ -168,7 +170,7 @@ fn parse_tree(
                         if delim != token::Paren {
                             span_dollar_dollar_or_metavar_in_the_lhs_err(
                                 sess,
-                                &Token { kind: token::OpenDelim(delim), span: delim_span.entire() },
+                                &Token::new(token::OpenDelim(delim), delim_span.entire()),
                             );
                         }
                     } else {
@@ -237,11 +239,13 @@ fn parse_tree(
                 }
 
                 // `tree` is followed by another `$`. This is an escaped `$`.
-                Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => {
+                Some(tokenstream::TokenTree::Token(Token {
+                    kind: token::Dollar, span, ..
+                })) => {
                     if parsing_patterns {
                         span_dollar_dollar_or_metavar_in_the_lhs_err(
                             sess,
-                            &Token { kind: token::Dollar, span },
+                            &Token::new(token::Dollar, span),
                         );
                     } else {
                         maybe_emit_macro_metavar_expr_feature(features, sess, span);

compiler/rustc_expand/src/proc_macro_server.rs

@@ -59,7 +59,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
         use rustc_ast::token::*;
 
         let joint = spacing == Joint;
-        let Token { kind, span } = match tree {
+        let Token { kind, span, .. } = match tree {
             tokenstream::TokenTree::Delimited(span, delim, tts) => {
                 let delimiter = Delimiter::from_internal(delim);
                 return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false });

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -96,7 +96,7 @@ struct LazyTokenStreamImpl {
 }
 
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 144);
+rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 152);
 
 impl CreateTokenStream for LazyTokenStreamImpl {
     fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
@@ -414,10 +414,10 @@ fn make_token_stream(
     let mut token_and_spacing = iter.next();
     while let Some((token, spacing)) = token_and_spacing {
         match token {
-            FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
+            FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span, .. }) => {
                 stack.push(FrameData { open: span, open_delim: delim, inner: vec![] });
             }
-            FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
+            FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span, .. }) => {
                 // HACK: If we encounter a mismatched `None` delimiter at the top
                 // level, just ignore it.
                 if matches!(delim, DelimToken::NoDelim)

compiler/rustc_parse/src/parser/mod.rs

@@ -299,7 +299,7 @@ impl TokenCursor {
     #[inline(always)]
     fn inlined_next_desugared(&mut self) -> (Token, Spacing) {
         let (data, attr_style, sp) = match self.inlined_next() {
-            (Token { kind: token::DocComment(_, attr_style, data), span }, _) => {
+            (Token { kind: token::DocComment(_, attr_style, data), span, .. }, _) => {
                 (data, attr_style, span)
             }
             tok => return tok,

src/tools/rustfmt/src/macros.rs

@@ -683,10 +683,9 @@ struct MacroArgParser {
 fn last_tok(tt: &TokenTree) -> Token {
     match *tt {
         TokenTree::Token(ref t) => t.clone(),
-        TokenTree::Delimited(delim_span, delim, _) => Token {
-            kind: TokenKind::CloseDelim(delim),
-            span: delim_span.close,
-        },
+        TokenTree::Delimited(delim_span, delim, _) => {
+            Token::new(TokenKind::CloseDelim(delim), delim_span.close)
+        }
     }
 }
 
@@ -695,14 +694,8 @@ impl MacroArgParser {
         MacroArgParser {
             buf: String::new(),
             is_meta_var: false,
-            last_tok: Token {
-                kind: TokenKind::Eof,
-                span: DUMMY_SP,
-            },
-            start_tok: Token {
-                kind: TokenKind::Eof,
-                span: DUMMY_SP,
-            },
+            last_tok: Token::new(TokenKind::Eof, DUMMY_SP),
+            start_tok: Token::new(TokenKind::Eof, DUMMY_SP),
             result: vec![],
         }
     }
@@ -862,6 +855,7 @@ impl MacroArgParser {
                 TokenTree::Token(Token {
                     kind: TokenKind::Dollar,
                     span,
+                    ..
                 }) => {
                     // We always want to add a separator before meta variables.
                     if !self.buf.is_empty() {
@@ -870,10 +864,7 @@ impl MacroArgParser {
 
                     // Start keeping the name of this metavariable in the buffer.
                     self.is_meta_var = true;
-                    self.start_tok = Token {
-                        kind: TokenKind::Dollar,
-                        span,
-                    };
+                    self.start_tok = Token::new(TokenKind::Dollar, span);
                 }
                 TokenTree::Token(Token {
                     kind: TokenKind::Colon,
@@ -1150,6 +1141,7 @@ impl MacroParser {
         if let Some(TokenTree::Token(Token {
             kind: TokenKind::Semi,
             span,
+            ..
         })) = self.toks.look_ahead(0)
         {
             hi = span.hi();