Repository: apollographql/apollo-rs
Pull request #591 (2023-07-10 21:18:28)

Create EOF token with empty data
tl;dr Make consuming the token stream's data produce an identical string to the original input of the lexer.

I was working on a streaming processor using the streaming lexer and noticed that the `Token::Eof` has `"EOF"` as a string hard-coded as it's data. This doesn't appear to be used anywhere (though is maybe a breaking change?) and the tests still pass without this.

The formatter for `Debug` prefixes the location with `EOF` so the debugging output is not effected. The practical effect of this is that __consuming a token stream's data does not produce the same output as the original input.__




Patch:
diff --git a/crates/apollo-parser/src/lexer/mod.rs b/crates/apollo-parser/src/lexer/mod.rs
index 6147e80..6c7802c 100644
--- a/crates/apollo-parser/src/lexer/mod.rs
+++ b/crates/apollo-parser/src/lexer/mod.rs
@@ -113,7 +113,7 @@ impl<'a> Iterator for Lexer<'a> {
         }
 
         if self.input.is_empty() {
-            let mut eof = Token::new(TokenKind::Eof, "EOF");
+            let mut eof = Token::new(TokenKind::Eof, "");
             eof.index = self.index;
 
             self.finished = true;
@@ -149,7 +149,7 @@ impl<'a> Cursor<'a> {
         let mut state = State::Start;
         let mut token = Token {
             kind: TokenKind::Eof,
-            data: "EOF",
+            data: "",
             index: self.index(),
         };
 
@@ -654,4 +654,21 @@ type Query {
             ],
         );
     }
+
+    #[test]
+    fn stream_produces_original_input() {
+        let schema = r#"
+type Query {
+    name: String
+    format: String = "Y-m-d\\TH:i:sP"
+}
+        "#;
+
+        let lexer = Lexer::new(schema);
+        let processed_schema = lexer
+            .into_iter()
+            .fold(String::new(), |acc, token| acc + token.unwrap().data());
+
+        assert_eq!(schema, processed_schema);
+    }
 }
diff --git a/crates/apollo-parser/src/parser/mod.rs b/crates/apollo-parser/src/parser/mod.rs
index e5298d1..ae13cc0 100644
--- a/crates/apollo-parser/src/parser/mod.rs
+++ b/crates/apollo-parser/src/parser/mod.rs
@@ -271,10 +271,11 @@ impl<'a> Parser<'a> {
             return;
         }
 
-        let message = format!("expected {kind:?}, got {data}");
         let err = if is_eof {
+            let message = format!("expected {kind:?}, got EOF");
             Error::eof(message, index)
         } else {
+            let message = format!("expected {kind:?}, got {data}");
             Error::with_loc(message, data, index)
         };
 


Test patch:
diff --git a/crates/apollo-parser/test_data/parser/err/0041_operation_definition_with_missing_selection_set.txt b/crates/apollo-parser/test_data/parser/err/0041_operation_definition_with_missing_selection_set.txt
index 4913f56..189b192 100644
--- a/crates/apollo-parser/test_data/parser/err/0041_operation_definition_with_missing_selection_set.txt
+++ b/crates/apollo-parser/test_data/parser/err/0041_operation_definition_with_missing_selection_set.txt
@@ -1,10 +1,10 @@
-- DOCUMENT@0..19
-    - OPERATION_DEFINITION@0..19
+- DOCUMENT@0..16
+    - OPERATION_DEFINITION@0..16
         - OPERATION_TYPE@0..5
             - query_KW@0..5 "query"
         - WHITESPACE@5..6 " "
         - NAME@6..16
             - IDENT@6..16 "__typename"
-        - ERROR@16..19 "EOF"
+        - ERROR@16..16 ""
 - ERROR@16:16 "expected a Selection Set" EOF
 recursion limit: 4096, high: 0
\ No newline at end of file