Auto merge of #23331 - eddyb:attr-lookahead, r=nikomatsakis

Most of the changes are cleanup facilitated by straight-forward attribute handling.
This is a minor [breaking-change] for users of `quote_stmt!` (returns `Option<P<Stmt>>` now) and some of the public methods in `Parser` (a few `Vec<Attribute>` arguments/returns were removed).

r? @nikomatsakis

Author: bors

src/libsyntax/ext/quote.rs

@@ -361,8 +361,7 @@ pub mod rt {
             parse::parse_stmt_from_source_str("<quote expansion>".to_string(),
                                               s,
                                               self.cfg(),
-                                              Vec::new(),
-                                              self.parse_sess())
+                                              self.parse_sess()).expect("parse error")
         }
 
         fn parse_expr(&self, s: String) -> P<ast::Expr> {
@@ -407,16 +406,15 @@ pub fn expand_quote_expr<'cx>(cx: &'cx mut ExtCtxt,
                               sp: Span,
                               tts: &[ast::TokenTree])
                               -> Box<base::MacResult+'cx> {
-    let expanded = expand_parse_call(cx, sp, "parse_expr", Vec::new(), tts);
+    let expanded = expand_parse_call(cx, sp, "parse_expr", vec!(), tts);
     base::MacEager::expr(expanded)
 }
 
 pub fn expand_quote_item<'cx>(cx: &mut ExtCtxt,
                               sp: Span,
                               tts: &[ast::TokenTree])
                               -> Box<base::MacResult+'cx> {
-    let expanded = expand_parse_call(cx, sp, "parse_item_with_outer_attributes",
-                                    vec!(), tts);
+    let expanded = expand_parse_call(cx, sp, "parse_item", vec!(), tts);
     base::MacEager::expr(expanded)
 }
 
@@ -448,9 +446,7 @@ pub fn expand_quote_stmt(cx: &mut ExtCtxt,
                          sp: Span,
                          tts: &[ast::TokenTree])
                          -> Box<base::MacResult+'static> {
-    let e_attrs = cx.expr_vec_ng(sp);
-    let expanded = expand_parse_call(cx, sp, "parse_stmt",
-                                    vec!(e_attrs), tts);
+    let expanded = expand_parse_call(cx, sp, "parse_stmt", vec!(), tts);
     base::MacEager::expr(expanded)
 }
 

src/libsyntax/ext/source_util.rs

@@ -115,7 +115,7 @@ pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree
                       -> Option<SmallVector<P<ast::Item>>> {
             let mut ret = SmallVector::zero();
             while self.p.token != token::Eof {
-                match self.p.parse_item_with_outer_attributes() {
+                match self.p.parse_item() {
                     Some(item) => ret.push(item),
                     None => self.p.span_fatal(
                         self.p.span,

src/libsyntax/ext/tt/macro_parser.rs

@@ -521,12 +521,15 @@ pub fn parse_nt(p: &mut Parser, sp: Span, name: &str) -> Nonterminal {
     // check at the beginning and the parser checks after each bump
     p.check_unknown_macro_variable();
     match name {
-      "item" => match p.parse_item(Vec::new()) {
+      "item" => match p.parse_item() {
         Some(i) => token::NtItem(i),
         None => p.fatal("expected an item keyword")
       },
       "block" => token::NtBlock(p.parse_block()),
-      "stmt" => token::NtStmt(p.parse_stmt(Vec::new())),
+      "stmt" => match p.parse_stmt() {
+        Some(s) => token::NtStmt(s),
+        None => p.fatal("expected a statement")
+      },
       "pat" => token::NtPat(p.parse_pat()),
       "expr" => token::NtExpr(p.parse_expr()),
       "ty" => token::NtTy(p.parse_ty()),

src/libsyntax/ext/tt/macro_rules.rs

@@ -17,7 +17,6 @@ use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use ext::tt::macro_parser::{parse, parse_or_else};
 use parse::lexer::new_tt_reader;
 use parse::parser::Parser;
-use parse::attr::ParserAttr;
 use parse::token::{self, special_idents, gensym_ident, NtTT, Token};
 use parse::token::Token::*;
 use print;
@@ -68,15 +67,8 @@ impl<'a> MacResult for ParserAnyMacro<'a> {
     }
     fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> {
         let mut ret = SmallVector::zero();
-        loop {
-            let mut parser = self.parser.borrow_mut();
-            // so... do outer attributes attached to the macro invocation
-            // just disappear? This question applies to make_impl_items, as
-            // well.
-            match parser.parse_item_with_outer_attributes() {
-                Some(item) => ret.push(item),
-                None => break
-            }
+        while let Some(item) = self.parser.borrow_mut().parse_item() {
+            ret.push(item);
         }
         self.ensure_complete_parse(false);
         Some(ret)
@@ -89,18 +81,17 @@ impl<'a> MacResult for ParserAnyMacro<'a> {
             let mut parser = self.parser.borrow_mut();
             match parser.token {
                 token::Eof => break,
-                _ => ret.push(parser.parse_impl_item_with_outer_attributes())
+                _ => ret.push(parser.parse_impl_item())
             }
         }
         self.ensure_complete_parse(false);
         Some(ret)
     }
 
     fn make_stmt(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Stmt>> {
-        let attrs = self.parser.borrow_mut().parse_outer_attributes();
-        let ret = self.parser.borrow_mut().parse_stmt(attrs);
+        let ret = self.parser.borrow_mut().parse_stmt();
         self.ensure_complete_parse(true);
-        Some(ret)
+        ret
     }
 }
 

src/libsyntax/parse/attr.rs

@@ -19,9 +19,8 @@ use ptr::P;
 /// A parser that can parse attributes.
 pub trait ParserAttr {
     fn parse_outer_attributes(&mut self) -> Vec<ast::Attribute>;
+    fn parse_inner_attributes(&mut self) -> Vec<ast::Attribute>;
     fn parse_attribute(&mut self, permit_inner: bool) -> ast::Attribute;
-    fn parse_inner_attrs_and_next(&mut self)
-                                  -> (Vec<ast::Attribute>, Vec<ast::Attribute>);
     fn parse_meta_item(&mut self) -> P<ast::MetaItem>;
     fn parse_meta_seq(&mut self) -> Vec<P<ast::MetaItem>>;
     fn parse_optional_meta(&mut self) -> Vec<P<ast::MetaItem>>;
@@ -118,45 +117,40 @@ impl<'a> ParserAttr for Parser<'a> {
 
     /// Parse attributes that appear after the opening of an item. These should
     /// be preceded by an exclamation mark, but we accept and warn about one
-    /// terminated by a semicolon. In addition to a vector of inner attributes,
-    /// this function also returns a vector that may contain the first outer
-    /// attribute of the next item (since we can't know whether the attribute
-    /// is an inner attribute of the containing item or an outer attribute of
-    /// the first contained item until we see the semi).
-
-    /// matches inner_attrs* outer_attr?
-    /// you can make the 'next' field an Option, but the result is going to be
-    /// more useful as a vector.
-    fn parse_inner_attrs_and_next(&mut self)
-                                  -> (Vec<ast::Attribute> , Vec<ast::Attribute> ) {
-        let mut inner_attrs: Vec<ast::Attribute> = Vec::new();
-        let mut next_outer_attrs: Vec<ast::Attribute> = Vec::new();
+    /// terminated by a semicolon.
+
+    /// matches inner_attrs*
+    fn parse_inner_attributes(&mut self) -> Vec<ast::Attribute> {
+        let mut attrs: Vec<ast::Attribute> = vec![];
         loop {
-            let attr = match self.token {
+            match self.token {
                 token::Pound => {
-                    self.parse_attribute(true)
+                    // Don't even try to parse if it's not an inner attribute.
+                    if !self.look_ahead(1, |t| t == &token::Not) {
+                        break;
+                    }
+
+                    let attr = self.parse_attribute(true);
+                    assert!(attr.node.style == ast::AttrInner);
+                    attrs.push(attr);
                 }
                 token::DocComment(s) => {
                     // we need to get the position of this token before we bump.
                     let Span { lo, hi, .. } = self.span;
-                    self.bump();
-                    attr::mk_sugared_doc_attr(attr::mk_attr_id(),
-                                              self.id_to_interned_str(s.ident()),
-                                              lo,
-                                              hi)
-                }
-                _ => {
-                    break;
+                    let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(),
+                                                         self.id_to_interned_str(s.ident()),
+                                                         lo, hi);
+                    if attr.node.style == ast::AttrInner {
+                        attrs.push(attr);
+                        self.bump();
+                    } else {
+                        break;
+                    }
                 }
-            };
-            if attr.node.style == ast::AttrInner {
-                inner_attrs.push(attr);
-            } else {
-                next_outer_attrs.push(attr);
-                break;
+                _ => break
             }
         }
-        (inner_attrs, next_outer_attrs)
+        attrs
     }
 
     /// matches meta_item = IDENT

src/libsyntax/parse/mod.rs

@@ -96,9 +96,7 @@ pub fn parse_crate_attrs_from_file(
     cfg: ast::CrateConfig,
     sess: &ParseSess
 ) -> Vec<ast::Attribute> {
-    let mut parser = new_parser_from_file(sess, cfg, input);
-    let (inner, _) = parser.parse_inner_attrs_and_next();
-    inner
+    new_parser_from_file(sess, cfg, input).parse_inner_attributes()
 }
 
 pub fn parse_crate_from_source_str(name: String,
@@ -122,8 +120,7 @@ pub fn parse_crate_attrs_from_source_str(name: String,
                                            cfg,
                                            name,
                                            source);
-    let (inner, _) = maybe_aborted(p.parse_inner_attrs_and_next(),p);
-    inner
+    maybe_aborted(p.parse_inner_attributes(), p)
 }
 
 pub fn parse_expr_from_source_str(name: String,
@@ -141,7 +138,7 @@ pub fn parse_item_from_source_str(name: String,
                                   sess: &ParseSess)
                                   -> Option<P<ast::Item>> {
     let mut p = new_parser_from_source_str(sess, cfg, name, source);
-    maybe_aborted(p.parse_item_with_outer_attributes(),p)
+    maybe_aborted(p.parse_item(),p)
 }
 
 pub fn parse_meta_from_source_str(name: String,
@@ -156,16 +153,15 @@ pub fn parse_meta_from_source_str(name: String,
 pub fn parse_stmt_from_source_str(name: String,
                                   source: String,
                                   cfg: ast::CrateConfig,
-                                  attrs: Vec<ast::Attribute> ,
                                   sess: &ParseSess)
-                                  -> P<ast::Stmt> {
+                                  -> Option<P<ast::Stmt>> {
     let mut p = new_parser_from_source_str(
         sess,
         cfg,
         name,
         source
     );
-    maybe_aborted(p.parse_stmt(attrs),p)
+    maybe_aborted(p.parse_stmt(), p)
 }
 
 // Note: keep in sync with `with_hygiene::parse_tts_from_source_str`