Repository: ron-rs/ron (rust)

Issue:
Inconsistencies in floating point parsing
Grammar taken from [current master](https://github.com/ron-rs/ron/blob/3ec0dc09f9b557d73790b9916b0d5cc4cc0f4c3d/docs/grammar.md). There are a couple inconcistencies between the grammar, the current implementation, and from how rust does it.

1. Exponential notation. The grammar allows `1.0e1`, but disallows `1e1` and bizzarely disallows sign: `1.0e+1` and `1.0e-1`. But the implementation accepts all those forms, and so does rust.

2. Underscores are disallowed in floats, and this is consistent in both the grammar and the implementation; but it differs from rust and is a bit unexpected. 

Here's small repro for everything:
```rust
type R<T> = ron::Result<T>;

fn main() {
    let x1: R<i64> = ron::from_str("1_0");
    println!("{:?}", &x1);
    // >>> Ok(10)

    let x2: R<f64> = ron::from_str("1_0.1_0");
    println!("{:?}", &x2);
    // >>> Err(Error { code: TrailingCharacters, position: Position { line: 1, col: 2 } })

    let x3: R<f64> = ron::from_str("1_0.10");
    println!("{:?}", &x3);
    // >>> Err(Error { code: TrailingCharacters, position: Position { line: 1, col: 2 } })

    let x4: R<f64> = ron::from_str("10.1_0");
    println!("{:?}", &x4);
    // >>> Err(Error { code: TrailingCharacters, position: Position { line: 1, col: 5 } })

    let x5: R<f64> = ron::from_str("1.0e-1");
    println!("{:?}", &x5);
    // >>> Ok(10.1)

    let y1: R<f64> = ron::from_str("1.0e1");
    println!("{:?}", &y1);
    // >>> Ok(10.0)

    let y2: R<f64> = ron::from_str("1e1");
    println!("{:?}", &y2);
    // >>> Ok(10.0)

    let y3: R<f64> = ron::from_str("1.0e1.0");
    println!("{:?}", &y3);
    // >>> Err(Error { code: ExpectedFloat, position: Position { line: 1, col: 1 } })

    let y4: R<f64> = ron::from_str("1.0e+1");
    println!("{:?}", &y4);
    // >>> Ok(10.0)

    let y5: R<f64> = ron::from_str("1.0e-1");
    println!("{:?}", &y5);
    // >>> Ok(0.1)
}
// y3 being Err is alright I believe, but the error message is weird
```

Pull request:
Fix #370 with better float EBNF and +inf -NaN +NaN parsing
Fixes #370

Fixes the RON grammar for floats to be equivalent to `FromStr`'s grammar (which we use internally):
```ebnf
float = ["+" | "-"], ("inf" | "NaN" | float_num);
float_num = (float_int | float_std | float_frac), [float_exp];
float_int = digit, { digit };
float_std = digit, { digit }, ".", {digit};
float_frac = ".", digit, {digit};
float_exp = ("e" | "E"), ["+" | "-"], digit, {digit};
```

Consequently, RON now also accepts `+inf`, `-NaN`, and `+NaN`.

In comparison with `rustc`, `FromStr` still accepts floats without a leading digit, e.g. `.3` - these continue to work. The other way around, `rustc` accepts underscores in floats while `FromStr` and all other parsers I checked do not. I've added a special error `FloatUnderscore` to lint against this case.

* [x] I've included my change in `CHANGELOG.md`


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fc2a753a..53fc8bb7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix issue [#337](https://github.com/ron-rs/ron/issues/337) by removing `decimal_floats` PrettyConfig option and unconditional decimals in floats ([#363](https://github.com/ron-rs/ron/pull/363))
 - Fix issue [#203](https://github.com/ron-rs/ron/issues/203) with full de error positioning ([#356](https://github.com/ron-rs/ron/pull/356))
 - Fix issue [#367](https://github.com/ron-rs/ron/issues/367) with eager implicit some ([#368](https://github.com/ron-rs/ron/pull/368))
+- Fix issue [#370](https://github.com/ron-rs/ron/issues/370) with `FromStr`-equivalent float EBNF and `Error::FloatUnderscore` ([#371](https://github.com/ron-rs/ron/pull/371))
 
 ## [0.7.0] - 2021-10-22
 
diff --git a/docs/grammar.md b/docs/grammar.md
index d52d7339..61b195f0 100644
--- a/docs/grammar.md
+++ b/docs/grammar.md
@@ -50,10 +50,12 @@ hex_digit = "A" | "a" | "B" | "b" | "C" | "c" | "D" | "d" | "E" | "e" | "F" | "f
 unsigned = (["0", ("b" | "o")], digit, { digit | '_' } |
              "0x", (digit | hex_digit), { digit | hex_digit | '_' });
 signed = ["+" | "-"], unsigned;
-float = float_std | float_frac;
-float_std = ["+" | "-"], digit, { digit }, ".", {digit}, [float_exp];
-float_frac = ".", digit, {digit}, [float_exp];
-float_exp = ("e" | "E"), digit, {digit};
+float = ["+" | "-"], ("inf" | "NaN" | float_num);
+float_num = (float_int | float_std | float_frac), [float_exp];
+float_int = digit, { digit };
+float_std = digit, { digit }, ".", {digit};
+float_frac = ".", digit, {digit};
+float_exp = ("e" | "E"), ["+" | "-"], digit, {digit};
 ```
 
 ## String
diff --git a/src/error.rs b/src/error.rs
index e7d279d6..29fbe878 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -27,6 +27,7 @@ pub enum Error {
     ExpectedComma,
     ExpectedChar,
     ExpectedFloat,
+    FloatUnderscore,
     ExpectedInteger,
     ExpectedOption,
     ExpectedOptionEnd,
@@ -86,6 +87,7 @@ impl fmt::Display for Error {
             Error::ExpectedComma => f.write_str("Expected comma"),
             Error::ExpectedChar => f.write_str("Expected char"),
             Error::ExpectedFloat => f.write_str("Expected float"),
+            Error::FloatUnderscore => f.write_str("Unexpected underscore in float"),
             Error::ExpectedInteger => f.write_str("Expected integer"),
             Error::ExpectedOption => f.write_str("Expected option"),
             Error::ExpectedOptionEnd => f.write_str("Expected closing `)`"),
diff --git a/src/parse.rs b/src/parse.rs
index 3f9953a0..36b17ca0 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -12,7 +12,7 @@ use crate::{
 
 // We have the following char categories.
 const INT_CHAR: u8 = 1 << 0; // [0-9A-Fa-f_]
-const FLOAT_CHAR: u8 = 1 << 1; // [0-9\.Ee+-]
+const FLOAT_CHAR: u8 = 1 << 1; // [0-9\.Ee+-_]
 const IDENT_FIRST_CHAR: u8 = 1 << 2; // [A-Za-z_]
 const IDENT_OTHER_CHAR: u8 = 1 << 3; // [A-Za-z_0-9]
 const IDENT_RAW_CHAR: u8 = 1 << 4; // [A-Za-z_0-9\.+-]
@@ -21,7 +21,7 @@ const WHITESPACE_CHAR: u8 = 1 << 5; // [\n\t\r ]
 // We encode each char as belonging to some number of these categories.
 const DIGIT: u8 = INT_CHAR | FLOAT_CHAR | IDENT_OTHER_CHAR | IDENT_RAW_CHAR; // [0-9]
 const ABCDF: u8 = INT_CHAR | IDENT_FIRST_CHAR | IDENT_OTHER_CHAR | IDENT_RAW_CHAR; // [ABCDFabcdf]
-const UNDER: u8 = INT_CHAR | IDENT_FIRST_CHAR | IDENT_OTHER_CHAR | IDENT_RAW_CHAR; // [_]
+const UNDER: u8 = INT_CHAR | FLOAT_CHAR | IDENT_FIRST_CHAR | IDENT_OTHER_CHAR | IDENT_RAW_CHAR; // [_]
 const E____: u8 = INT_CHAR | FLOAT_CHAR | IDENT_FIRST_CHAR | IDENT_OTHER_CHAR | IDENT_RAW_CHAR; // [Ee]
 const G2Z__: u8 = IDENT_FIRST_CHAR | IDENT_OTHER_CHAR | IDENT_RAW_CHAR; // [G-Zg-z]
 const PUNCT: u8 = FLOAT_CHAR | IDENT_RAW_CHAR; // [\.+-]
@@ -558,7 +558,7 @@ impl<'a> Bytes<'a> {
     where
         T: FromStr,
     {
-        for literal in &["inf", "-inf", "NaN"] {
+        for literal in &["inf", "+inf", "-inf", "NaN", "+NaN", "-NaN"] {
             if self.consume_ident(literal) {
                 return FromStr::from_str(literal).map_err(|_| unreachable!()); // must not fail
             }
@@ -566,6 +566,13 @@ impl<'a> Bytes<'a> {
 
         let num_bytes = self.next_bytes_contained_in(is_float_char);
 
+        // Since `rustc` allows `1_0.0_1`, lint against underscores in floats
+        if let Some(err_bytes) = self.bytes[0..num_bytes].iter().position(|b| *b == b'_') {
+            let _ = self.advance(err_bytes);
+
+            return Err(Error::FloatUnderscore);
+        }
+
         let s = unsafe { from_utf8_unchecked(&self.bytes[0..num_bytes]) };
         let res = FromStr::from_str(s).map_err(|_| Error::ExpectedFloat);
 


Test patch:
diff --git a/tests/370_float_parsing.rs b/tests/370_float_parsing.rs
new file mode 100644
index 00000000..786d5ae3
--- /dev/null
+++ b/tests/370_float_parsing.rs
@@ -0,0 +1,66 @@
+use ron::{
+    de::{Position, SpannedError},
+    Error,
+};
+
+#[test]
+fn test_float_literal_parsing() {
+    assert_eq!(ron::from_str("inf"), Ok(f64::INFINITY));
+    assert_eq!(ron::from_str("+inf"), Ok(f64::INFINITY));
+    assert_eq!(ron::from_str("-inf"), Ok(f64::NEG_INFINITY));
+
+    assert!(ron::from_str::<f64>("NaN").unwrap().is_nan());
+    assert!(ron::from_str::<f64>("+NaN").unwrap().is_nan());
+    assert!(ron::from_str::<f64>("-NaN").unwrap().is_nan());
+
+    assert_eq!(ron::from_str("1"), Ok(1.0_f64));
+    assert_eq!(ron::from_str("+1"), Ok(1.0_f64));
+    assert_eq!(ron::from_str("-1"), Ok(-1.0_f64));
+    assert_eq!(ron::from_str("1e3"), Ok(1000.0_f64));
+    assert_eq!(ron::from_str("1e+1"), Ok(10.0_f64));
+    assert_eq!(ron::from_str("7E-1"), Ok(0.7_f64));
+
+    assert_eq!(ron::from_str("1."), Ok(1.0_f64));
+    assert_eq!(ron::from_str("+1.1"), Ok(1.1_f64));
+    assert_eq!(ron::from_str("-1.42"), Ok(-1.42_f64));
+    assert_eq!(ron::from_str("-1.5e3"), Ok(-1500.0_f64));
+    assert_eq!(ron::from_str("1.e+1"), Ok(10.0_f64));
+    assert_eq!(ron::from_str("7.4E-1"), Ok(0.74_f64));
+
+    assert_eq!(ron::from_str(".1"), Ok(0.1_f64));
+    assert_eq!(ron::from_str("+.1"), Ok(0.1_f64));
+    assert_eq!(ron::from_str("-.42"), Ok(-0.42_f64));
+    assert_eq!(ron::from_str("-.5e3"), Ok(-500.0_f64));
+    assert_eq!(ron::from_str(".3e+1"), Ok(3.0_f64));
+    assert_eq!(ron::from_str(".4E-1"), Ok(0.04_f64));
+
+    assert_eq!(
+        ron::from_str::<f64>("1_0.1_0"),
+        Err(SpannedError {
+            code: Error::FloatUnderscore,
+            position: Position { line: 1, col: 2 },
+        })
+    );
+    assert_eq!(
+        ron::from_str::<f64>("1_0.10"),
+        Err(SpannedError {
+            code: Error::FloatUnderscore,
+            position: Position { line: 1, col: 2 },
+        })
+    );
+    assert_eq!(
+        ron::from_str::<f64>("10.1_0"),
+        Err(SpannedError {
+            code: Error::FloatUnderscore,
+            position: Position { line: 1, col: 5 },
+        })
+    );
+
+    assert_eq!(
+        ron::from_str::<f64>("1.0e1.0"),
+        Err(SpannedError {
+            code: Error::ExpectedFloat,
+            position: Position { line: 1, col: 8 },
+        })
+    );
+}