Repository: alexcrichton/proc-macro2 (rust)

Issue:
Fallback handling of negative integer literals is different to proc_macro
This crate's fallback implementation of `From<TokenTree>` for `TokenStream` treats negative integer literals as one token, however `rustc`'s implementation treats negative integer literals as an alone `-` followed by the positive integer literal.

### How to Reproduce

1. Make a simple proc-macro crate, with this code:
```rust
use std::iter;
use proc_macro2::{TokenStream, TokenTree, Literal};

#[proc_macro]
pub fn proc_macro_test(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
    //proc_macro2::fallback::force();

    let int: i32 = -3;
    let mut tokens = TokenStream::new();
    tokens.extend(iter::once(TokenTree::Literal(Literal::i32_suffixed(int))));
    dbg!(&tokens);

    input
}
```
2. Run that proc macro in another crate. With the commented line commented it will output two separate tokens, but with it uncommented it will output one negative literal token.

Pull request description:
Match libproc_macro behavior of negative literals
Fixes #235.

Patch:
diff --git a/build.rs b/build.rs
index 89e2ab3..153e13f 100644
--- a/build.rs
+++ b/build.rs
@@ -61,6 +61,10 @@ fn main() {
         println!("cargo:rustc-cfg=span_locations");
     }
 
+    if version.minor < 39 {
+        println!("cargo:rustc-cfg=no_bind_by_move_pattern_guard");
+    }
+
     if version.minor >= 45 {
         println!("cargo:rustc-cfg=hygiene");
     }
diff --git a/src/fallback.rs b/src/fallback.rs
index 4d102ef..2a06430 100644
--- a/src/fallback.rs
+++ b/src/fallback.rs
@@ -49,6 +49,49 @@ impl TokenStream {
     fn take_inner(&mut self) -> Vec<TokenTree> {
         mem::replace(&mut self.inner, Vec::new())
     }
+
+    fn push_token(&mut self, token: TokenTree) {
+        // https://github.com/alexcrichton/proc-macro2/issues/235
+        match token {
+            #[cfg(not(no_bind_by_move_pattern_guard))]
+            TokenTree::Literal(crate::Literal {
+                #[cfg(wrap_proc_macro)]
+                    inner: crate::imp::Literal::Fallback(literal),
+                #[cfg(not(wrap_proc_macro))]
+                    inner: literal,
+                ..
+            }) if literal.text.starts_with('-') => {
+                push_negative_literal(self, literal);
+            }
+            #[cfg(no_bind_by_move_pattern_guard)]
+            TokenTree::Literal(crate::Literal {
+                #[cfg(wrap_proc_macro)]
+                    inner: crate::imp::Literal::Fallback(literal),
+                #[cfg(not(wrap_proc_macro))]
+                    inner: literal,
+                ..
+            }) => {
+                if literal.text.starts_with('-') {
+                    push_negative_literal(self, literal);
+                } else {
+                    self.inner
+                        .push(TokenTree::Literal(crate::Literal::_new_stable(literal)));
+                }
+            }
+            _ => self.inner.push(token),
+        }
+
+        #[cold]
+        fn push_negative_literal(stream: &mut TokenStream, mut literal: Literal) {
+            literal.text.remove(0);
+            let mut punct = crate::Punct::new('-', Spacing::Alone);
+            punct.set_span(crate::Span::_new_stable(literal.span));
+            stream.inner.push(TokenTree::Punct(punct));
+            stream
+                .inner
+                .push(TokenTree::Literal(crate::Literal::_new_stable(literal)));
+        }
+    }
 }
 
 // Nonrecursive to prevent stack overflow.
@@ -172,19 +215,17 @@ impl From<TokenStream> for proc_macro::TokenStream {
 
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
-        TokenStream { inner: vec![tree] }
+        let mut stream = TokenStream::new();
+        stream.push_token(tree);
+        stream
     }
 }
 
 impl FromIterator<TokenTree> for TokenStream {
-    fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
-        let mut v = Vec::new();
-
-        for token in streams {
-            v.push(token);
-        }
-
-        TokenStream { inner: v }
+    fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
+        let mut stream = TokenStream::new();
+        stream.extend(tokens);
+        stream
     }
 }
 
@@ -201,8 +242,8 @@ impl FromIterator<TokenStream> for TokenStream {
 }
 
 impl Extend<TokenTree> for TokenStream {
-    fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
-        self.inner.extend(streams);
+    fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
+        tokens.into_iter().for_each(|token| self.push_token(token));
     }
 }
 


Test patch:
diff --git a/tests/test.rs b/tests/test.rs
index a0133f6..39e0c78 100644
--- a/tests/test.rs
+++ b/tests/test.rs
@@ -117,6 +117,27 @@ fn literal_suffix() {
     assert_eq!(token_count("b'b'b"), 1);
 }
 
+#[test]
+fn literal_iter_negative() {
+    let negative_literal = Literal::i32_suffixed(-3);
+    let tokens = TokenStream::from(TokenTree::Literal(negative_literal));
+    let mut iter = tokens.into_iter();
+    match iter.next().unwrap() {
+        TokenTree::Punct(punct) => {
+            assert_eq!(punct.as_char(), '-');
+            assert_eq!(punct.spacing(), Spacing::Alone);
+        }
+        unexpected => panic!("unexpected token {:?}", unexpected),
+    }
+    match iter.next().unwrap() {
+        TokenTree::Literal(literal) => {
+            assert_eq!(literal.to_string(), "3i32");
+        }
+        unexpected => panic!("unexpected token {:?}", unexpected),
+    }
+    assert!(iter.next().is_none());
+}
+
 #[test]
 fn roundtrip() {
     fn roundtrip(p: &str) {