Repository: prql/prql
Pull request #439 (2022-04-30 07:40:31)

Multicharacter quotes for strings
Closes https://github.com/prql/prql/issues/383

The best way of seeing the impact is the docs https://github.com/prql/prql/compare/r-strings?expand=1#diff-e4ccfc02f32624d7e442e065aede37f48083d4202418c0c92fcb2d37c734968c

This isn't actually an "R" string, since I don't think we need those at this point, given we don't interpret escapes now.

One thing I realize — we don't escape `'` characters when we convert to SQL, which we should do; I'll open a different issue for that


Patch:
diff --git a/prql-compiler/src/parser.rs b/prql-compiler/src/parser.rs
index 6caeafbb..35b2b105 100644
--- a/prql-compiler/src/parser.rs
+++ b/prql-compiler/src/parser.rs
@@ -170,7 +170,7 @@ fn ast_of_parse_tree(pairs: Pairs<Rule>) -> Result<Vec<Node>> {
                     })
                 }
                 Rule::ident => Item::Ident(pair.as_str().to_string()),
-                Rule::string_literal => Item::String(
+                Rule::string => Item::String(
                     // Put the string_inner (which doesn't have quotes) into the
                     // String item.
                     pair.into_inner().into_only()?.as_str().to_string(),
@@ -253,7 +253,7 @@ fn ast_of_interpolate_items(pair: Pair<Rule>) -> Result<Vec<InterpolateItem>> {
     pair.into_inner()
         .map(|x| {
             Ok(match x.as_rule() {
-                Rule::interpolate_string => InterpolateItem::String(x.as_str().to_string()),
+                Rule::interpolate_string_inner => InterpolateItem::String(x.as_str().to_string()),
                 _ => InterpolateItem::Expr(Box::new(
                     ast_of_parse_tree(x.into_inner())?.into_expr().into(),
                 )),
@@ -275,10 +275,10 @@ mod test {
 
     #[test]
     fn test_parse_string() -> Result<()> {
-        assert_debug_snapshot!(parse_tree_of_str(r#"" U S A ""#, Rule::string_literal)?, @r###"
+        assert_debug_snapshot!(parse_tree_of_str(r#"" U S A ""#, Rule::string)?, @r###"
         [
             Pair {
-                rule: string_literal,
+                rule: string,
                 span: Span {
                     str: "\" U S A \"",
                     start: 0,
@@ -298,32 +298,32 @@ mod test {
             },
         ]
         "###);
-        let double_quoted_ast = ast_of_string(r#"" U S A ""#, Rule::string_literal)?;
+        let double_quoted_ast = ast_of_string(r#"" U S A ""#, Rule::string)?;
         assert_yaml_snapshot!(double_quoted_ast, @r###"
         ---
         String: " U S A "
         "###);
 
-        let single_quoted_ast = ast_of_string(r#"' U S A '"#, Rule::string_literal)?;
+        let single_quoted_ast = ast_of_string(r#"' U S A '"#, Rule::string)?;
         assert_eq!(single_quoted_ast, double_quoted_ast);
 
         // Single quotes within double quotes should produce a string containing
         // the single quotes (and vice versa).
-        assert_yaml_snapshot!(ast_of_string(r#""' U S A '""#, Rule::string_literal)? , @r###"
+        assert_yaml_snapshot!(ast_of_string(r#""' U S A '""#, Rule::string)? , @r###"
         ---
         String: "' U S A '"
         "###);
-        assert_yaml_snapshot!(ast_of_string(r#"'" U S A "'"#, Rule::string_literal)? , @r###"
+        assert_yaml_snapshot!(ast_of_string(r#"'" U S A "'"#, Rule::string)? , @r###"
         ---
         String: "\" U S A \""
         "###);
 
-        assert!(ast_of_string(r#"" U S A"#, Rule::string_literal).is_err());
-        assert!(ast_of_string(r#"" U S A '"#, Rule::string_literal).is_err());
+        assert!(ast_of_string(r#"" U S A"#, Rule::string).is_err());
+        assert!(ast_of_string(r#"" U S A '"#, Rule::string).is_err());
 
         // Escapes get passed through (the insta snapshot has them escaped I
         // think, which isn't that clear, so repeated below).
-        let escaped_string = ast_of_string(r#"" \U S A ""#, Rule::string_literal)?;
+        let escaped_string = ast_of_string(r#"" \U S A ""#, Rule::string)?;
         assert_yaml_snapshot!(escaped_string, @r###"
         ---
         String: " \\U S A "
@@ -332,16 +332,44 @@ mod test {
 
         // Currently we don't allow escaping closing quotes — because it's not
         // trivial to do in pest, and I'm not sure it's a great idea either — we
-        // should arguably encourage r-strings. (Though no objection if someone
-        // wants to implement it, this test is recording current behavior rather
-        // than maintaining a contract).
-        let escaped_quotes = ast_of_string(r#"" Canada \""#, Rule::string_literal)?;
+        // should arguably encourage multiline-strings. (Though no objection if
+        // someone wants to implement it, this test is recording current
+        // behavior rather than maintaining a contract).
+        let escaped_quotes = ast_of_string(r#"" Canada \""#, Rule::string)?;
         assert_yaml_snapshot!(escaped_quotes, @r###"
         ---
         String: " Canada \\"
         "###);
         assert_eq!(escaped_quotes.item.as_string().unwrap(), r#" Canada \"#);
 
+        let multi_double = ast_of_string(
+            r#""""
+''
+Canada
+"
+
+""""#,
+            Rule::string,
+        )?;
+        assert_yaml_snapshot!(multi_double, @r###"
+        ---
+        String: "\n''\nCanada\n\"\n\n"
+        "###);
+
+        let multi_single = ast_of_string(
+            r#"'''
+Canada
+"
+"""
+
+'''"#,
+            Rule::string,
+        )?;
+        assert_yaml_snapshot!(multi_single, @r###"
+        ---
+        String: "\nCanada\n\"\n\"\"\"\n\n"
+        "###);
+
         Ok(())
     }
 
@@ -1378,4 +1406,27 @@ select [
                     named_args: {}
         "###);
     }
+
+    #[test]
+    fn test_multiline_string() {
+        assert_yaml_snapshot!(parse(r###"
+        derive x: r#"r-string test"#
+        "###).unwrap(), @r###"
+        ---
+        version: ~
+        dialect: Generic
+        nodes:
+          - Pipeline:
+              value: ~
+              functions:
+                - FuncCall:
+                    name: derive
+                    args:
+                      - NamedExpr:
+                          name: x
+                          expr:
+                            Ident: r
+                    named_args: {}
+        "### )
+    }
 }
diff --git a/prql-compiler/src/prql.pest b/prql-compiler/src/prql.pest
index 933ceecd..9a3a7b50 100644
--- a/prql-compiler/src/prql.pest
+++ b/prql-compiler/src/prql.pest
@@ -72,7 +72,7 @@ expr_add = !{ expr_mul ~ (operator_add ~ expr_mul)* }
 expr_mul = !{ term ~ (operator_mul ~ term)* }
 
 term = _{ ( s_string | f_string | ident | nested_expr | nested_pipeline | list | range | literal ) }
-literal = _{ interval | number | string_literal }
+literal = _{ interval | number | string}
 list = { "[" ~ (NEWLINE* ~ named_expr_call ~ ("," ~ NEWLINE* ~ named_expr_call )* ~ ","?)? ~ NEWLINE* ~ "]" }
 nested_expr = _{ "(" ~ expr_call ~ ")" }
 nested_pipeline = { "(" ~ expr_call? ~ pipe ~ pipeline ~ NEWLINE* ~ ")" }
@@ -82,9 +82,9 @@ nested_pipeline = { "(" ~ expr_call? ~ pipe ~ pipeline ~ NEWLINE* ~ ")" }
 // https://pest.rs/book/examples/rust/literals.html
 
 // We need to have a non-silent rule which contains the quotes
-// — `string_literal` in this case — because of
+// — `string` in this case — because of
 // https://github.com/pest-parser/pest/issues/583. Then when converting to AST,
-// we only keep the `string_inner` and discard the `string_literal` given it contains
+// we only keep the `string_inner` and discard the `string` given it contains
 // the quotes.
 //
 // TODO: I'm still a bit unclear how preceeding and trailing spaces are working
@@ -93,9 +93,11 @@ nested_pipeline = { "(" ~ expr_call? ~ pipe ~ pipeline ~ NEWLINE* ~ ")" }
 // atomic operator. We have some rudimentary tests for these.
 
 quote = _{ "\"" | "'" }
+multi_quote = _{ "\""{3,} | "'"{3,} }
 // PEEK refers to the opening quote; either `"` or `'`.
 string_inner = { ( !( PEEK ) ~ ANY )+ }
-string_literal = ${ PUSH(quote) ~ string_inner ~ POP }
+// Either > 3 quotes, or just one. Currently both of those can be multiline.
+string= ${ ( PUSH(multi_quote) | PUSH(quote) ) ~ string_inner ~ POP }
 
 number = ${ ( ASCII_DIGIT )+ ~ ("." ~ ( ASCII_DIGIT )+)? }
 
@@ -112,9 +114,10 @@ operator_add = ${ "+" | "-" }
 operator_compare = ${ "==" | "!=" | "=" | ">" | "<" | ">=" | "<=" }
 operator_logical = ${ ("and" | "or" | "not") ~ &WHITESPACE }
 
-s_string = ${ "s" ~ PUSH(quote) ~ ( interpolate_string | ( "{" ~ expr_call ~ "}" ))* ~ POP }
-f_string = ${ "f" ~ PUSH(quote) ~ ( interpolate_string | ( "{" ~ expr_call ~ "}" ))* ~ POP }
-interpolate_string = { ( !( PEEK | "{" ) ~ ANY )+ }
+// TODO: allow a raw s-string / f-string (but not an s-f string...)
+s_string = ${ "s" ~ PUSH(quote) ~ ( interpolate_string_inner | ( "{" ~ expr_call ~ "}" ))* ~ POP }
+f_string = ${ "f" ~ PUSH(quote) ~ ( interpolate_string_inner | ( "{" ~ expr_call ~ "}" ))* ~ POP }
+interpolate_string_inner = { ( !( PEEK | "{" ) ~ ANY )+ }
 
 interval_kind = { "microseconds" | "milliseconds" | "seconds" | "minutes" | "hours" | "days" | "weeks" | "months" | "years" }
 interval = ${ number ~ interval_kind }
diff --git a/reference/src/SUMMARY.md b/reference/src/SUMMARY.md
index 1ac2c393..3ee51972 100644
--- a/reference/src/SUMMARY.md
+++ b/reference/src/SUMMARY.md
@@ -4,14 +4,14 @@
 - [Queries](./queries/README.md)
   - [Pipelines](./queries/pipelines.md)
 - [Language features](./language-features/README.md)
-
+  - [Strings](./language-features/strings.md)
   - [F-Strings](./language-features/f-strings.md)
+  - [S-Strings](./language-features/s-strings.md)
   - [Ranges](./language-features/ranges.md)
     <!-- - - Dates — `"2021-01-01"` -> `@2021-01-01`? And `DATE_TRUNC(foo_date, YEAR)` -> `foo_date.year`? Or -> `foo_date | as year`? Or `foo_date | to year`? -->
   - [Dates](./language-features/dates.md)
     <!--   - Regex — `REGEX_MATCH(foo, "\\w{3}")` -> `foo ~ r"\w{3}"`? Or -> `regex foo r"\w{3}"`? -->
   - [Regex]()
-  - [S-Strings](./language-features/s-strings.md)
 
 - [Transforms](./transforms.md)
 - [Syntax](./syntax.md)
diff --git a/reference/src/language-features/strings.md b/reference/src/language-features/strings.md
new file mode 100644
index 00000000..24a4783f
--- /dev/null
+++ b/reference/src/language-features/strings.md
@@ -0,0 +1,28 @@
+# Strings
+
+Strings in PRQL can use either single or double quotes:
+
+```prql
+derive x: "hello world"
+```
+
+```prql
+derive x: 'hello world'
+```
+
+To quote a string containing quotes, either use the "other" type of quote, or
+use three-or-more quotes, and close with the same number.
+
+```prql
+derive x: '"hello world"'
+```
+
+```prql
+derive x: """I said "hello world"!"""
+```
+
+Currently PRQL does not adjust escape characters during the compilation process.
+
+Currently PRQL allows multiline strings with either a single character or
+multiple character quotes. This may change for strings using a single character
+quote in future versions.


Test patch:
diff --git a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree-4.snap b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree-4.snap
index 99ae899b..e81aa39f 100644
--- a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree-4.snap
+++ b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree-4.snap
@@ -107,7 +107,7 @@ expression: "parse_tree_of_str(r#\"    filter country = \"USA\"\"#, Rule::pipeli
                                                         },
                                                         inner: [
                                                             Pair {
-                                                                rule: string_literal,
+                                                                rule: string,
                                                                 span: Span {
                                                                     str: "\"USA\"",
                                                                     start: 21,
diff --git a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree.snap b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree.snap
index 6af3ce1f..6a904f86 100644
--- a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree.snap
+++ b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_into_parse_tree.snap
@@ -74,7 +74,7 @@ expression: "parse_tree_of_str(r#\"country = \"USA\"\"#, Rule::expr)?"
                                 },
                                 inner: [
                                     Pair {
-                                        rule: string_literal,
+                                        rule: string,
                                         span: Span {
                                             str: "\"USA\"",
                                             start: 10,
diff --git a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-2.snap b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-2.snap
index e364e6aa..d87e6da7 100644
--- a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-2.snap
+++ b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-2.snap
@@ -1,6 +1,6 @@
 ---
 source: prql-compiler/src/parser.rs
-expression: "parse_tree_of_str(r#\"\n    from employees\n    filter country = \"USA\"\n    \"#.trim(),\n                  Rule::pipeline)?"
+expression: "parse_tree_of_str(r#\"\n    from employees\n    filter country = \"USA\"\n    \"#.trim(),\n        Rule::pipeline)?"
 ---
 [
     Pair {
@@ -185,7 +185,7 @@ expression: "parse_tree_of_str(r#\"\n    from employees\n    filter country = \"
                                                         },
                                                         inner: [
                                                             Pair {
-                                                                rule: string_literal,
+                                                                rule: string,
                                                                 span: Span {
                                                                     str: "\"USA\"",
                                                                     start: 36,
diff --git a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-3.snap b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-3.snap
index 94e22be9..0f7aa828 100644
--- a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-3.snap
+++ b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_pipeline_parse_tree-3.snap
@@ -1,6 +1,6 @@
 ---
 source: prql-compiler/src/parser.rs
-expression: "parse_tree_of_str(r#\"\nfrom employees\nfilter country = \"USA\"                           # Each line transforms the previous result.\nderive [                                         # This adds columns / variables.\n  gross_salary: salary + payroll_tax,\n  gross_cost  : gross_salary + benefits_cost    # Variables can use other variables.\n]\nfilter gross_cost > 0\ngroup [title, country] (\n    aggregate [                                  # `by` are the columns to group by.\n        average salary,                          # These are aggregation calcs run on each group.\n        sum     salary,\n        average gross_salary,\n        sum     gross_salary,\n        average gross_cost,\n        sum_gross_cost: sum gross_cost,\n        count: count,\n    ]\n)\nsort sum_gross_cost\nfilter count > 200\ntake 20\n    \"#.trim(),\n                  Rule::pipeline)?"
+expression: "parse_tree_of_str(r#\"\nfrom employees\nfilter country = \"USA\"                           # Each line transforms the previous result.\nderive [                                         # This adds columns / variables.\n  gross_salary: salary + payroll_tax,\n  gross_cost  : gross_salary + benefits_cost    # Variables can use other variables.\n]\nfilter gross_cost > 0\ngroup [title, country] (\n    aggregate [                                  # `by` are the columns to group by.\n        average salary,                          # These are aggregation calcs run on each group.\n        sum     salary,\n        average gross_salary,\n        sum     gross_salary,\n        average gross_cost,\n        sum_gross_cost: sum gross_cost,\n        count: count,\n    ]\n)\nsort sum_gross_cost\nfilter count > 200\ntake 20\n    \"#.trim(),\n        Rule::pipeline)?"
 ---
 [
     Pair {
@@ -185,7 +185,7 @@ expression: "parse_tree_of_str(r#\"\nfrom employees\nfilter country = \"USA\"
                                                         },
                                                         inner: [
                                                             Pair {
-                                                                rule: string_literal,
+                                                                rule: string,
                                                                 span: Span {
                                                                     str: "\"USA\"",
                                                                     start: 32,
diff --git a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_s_string.snap b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_s_string.snap
index aa60f4cd..0933a9e8 100644
--- a/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_s_string.snap
+++ b/prql-compiler/src/snapshots/prql_compiler__parser__test__parse_s_string.snap
@@ -44,7 +44,7 @@ expression: "parse_tree_of_str(r#\"s\"SUM({col})\"\"#, Rule::expr_call)?"
                                         },
                                         inner: [
                                             Pair {
-                                                rule: interpolate_string,
+                                                rule: interpolate_string_inner,
                                                 span: Span {
                                                     str: "SUM(",
                                                     start: 2,
@@ -102,7 +102,7 @@ expression: "parse_tree_of_str(r#\"s\"SUM({col})\"\"#, Rule::expr_call)?"
                                                 ],
                                             },
                                             Pair {
-                                                rule: interpolate_string,
+                                                rule: interpolate_string_inner,
                                                 span: Span {
                                                     str: ")",
                                                     start: 11,
diff --git a/prql-compiler/src/snapshots/prql_compiler__translator__test__prql_to_sql_2.snap b/prql-compiler/src/snapshots/prql_compiler__translator__test__prql_to_sql_2.snap
deleted file mode 100644
index 76578aea..00000000
--- a/prql-compiler/src/snapshots/prql_compiler__translator__test__prql_to_sql_2.snap
+++ /dev/null
@@ -1,28 +0,0 @@
----
-source: prql-compiler/src/translator.rs
-expression: sql
----
-SELECT
-  title,
-  country,
-  AVG(salary),
-  SUM(salary),
-  AVG(salary + payroll_tax),
-  SUM(salary + payroll_tax),
-  AVG(salary + payroll_tax + benefits_cost),
-  SUM(salary + payroll_tax + benefits_cost) AS sum_gross_cost,
-  COUNT(*) AS ct
-FROM
-  employees
-WHERE
-  country = 'USA'
-  AND salary + payroll_tax + benefits_cost > 0
-GROUP BY
-  title,
-  country
-HAVING
-  COUNT(*) > 200
-ORDER BY
-  sum_gross_cost
-LIMIT
-  20
diff --git a/reference/tests/examples/language-features/strings-0.prql b/reference/tests/examples/language-features/strings-0.prql
new file mode 100644
index 00000000..089c7d19
--- /dev/null
+++ b/reference/tests/examples/language-features/strings-0.prql
@@ -0,0 +1,1 @@
+derive x: "hello world"
diff --git a/reference/tests/examples/language-features/strings-1.prql b/reference/tests/examples/language-features/strings-1.prql
new file mode 100644
index 00000000..7841f16c
--- /dev/null
+++ b/reference/tests/examples/language-features/strings-1.prql
@@ -0,0 +1,1 @@
+derive x: 'hello world'
diff --git a/reference/tests/examples/language-features/strings-2.prql b/reference/tests/examples/language-features/strings-2.prql
new file mode 100644
index 00000000..8cc42365
--- /dev/null
+++ b/reference/tests/examples/language-features/strings-2.prql
@@ -0,0 +1,1 @@
+derive x: '"hello world"'
diff --git a/reference/tests/examples/language-features/strings-3.prql b/reference/tests/examples/language-features/strings-3.prql
new file mode 100644
index 00000000..24f3351a
--- /dev/null
+++ b/reference/tests/examples/language-features/strings-3.prql
@@ -0,0 +1,1 @@
+derive x: """I said "hello world"!"""
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@strings-0.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@strings-0.prql.snap
new file mode 100644
index 00000000..38282067
--- /dev/null
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@strings-0.prql.snap
@@ -0,0 +1,7 @@
+---
+source: reference/tests/snapshot.rs
+expression: sql
+input_file: reference/tests/examples/language-features/strings-0.prql
+---
+SELECT
+  'hello world' AS x
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@strings-1.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@strings-1.prql.snap
new file mode 100644
index 00000000..4be09f0c
--- /dev/null
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@strings-1.prql.snap
@@ -0,0 +1,7 @@
+---
+source: reference/tests/snapshot.rs
+expression: sql
+input_file: reference/tests/examples/language-features/strings-1.prql
+---
+SELECT
+  'hello world' AS x
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@strings-2.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@strings-2.prql.snap
new file mode 100644
index 00000000..370820cf
--- /dev/null
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@strings-2.prql.snap
@@ -0,0 +1,7 @@
+---
+source: reference/tests/snapshot.rs
+expression: sql
+input_file: reference/tests/examples/language-features/strings-2.prql
+---
+SELECT
+  '"hello world"' AS x
diff --git a/reference/tests/snapshots/snapshot__run_reference_examples@strings-3.prql.snap b/reference/tests/snapshots/snapshot__run_reference_examples@strings-3.prql.snap
new file mode 100644
index 00000000..bdc5d4ff
--- /dev/null
+++ b/reference/tests/snapshots/snapshot__run_reference_examples@strings-3.prql.snap
@@ -0,0 +1,7 @@
+---
+source: reference/tests/snapshot.rs
+expression: sql
+input_file: reference/tests/examples/language-features/strings-3.prql
+---
+SELECT
+  'I said "hello world"!' AS x