Repository: tweag/nickel
Pull request #1016 (2022-12-22 15:45:14)

Symbolic strings with prefixes
This PR changes the syntax and semantics of symbolic strings. Instead of `s%"Hello, world!"%` a symbolic string now takes a prefix: `foo-s%"Hello, world!"%`. The intention is that consumers in Nickel library code can enforce different namespacing for symbolic strings.

Any valid identifier can precede `-s%` and the parser will interpret the result as a symbolic string. The identifier is converted to a string and, e.g., `foo-s%"Hello %{1}"` evaluates to the record
```
{
  tag = `SymbolicString,
  prefix = "foo",
  fragments = ["Hello ", 1]
}
```


Patch:
diff --git a/lsp/nls/src/requests/completion.rs b/lsp/nls/src/requests/completion.rs
index 1930d4f0..8b81cb37 100644
--- a/lsp/nls/src/requests/completion.rs
+++ b/lsp/nls/src/requests/completion.rs
@@ -561,7 +561,8 @@ fn get_completion_identifiers(
                     .get_in_scope(item, &server.lin_cache)
                     .iter()
                     .filter_map(|i| match i.kind {
-                        TermKind::Declaration(ident, _, _) => Some(IdentWithType {
+                        TermKind::Declaration(ident, _, _)
+                        | TermKind::RecordField { ident, .. } => Some(IdentWithType {
                             ident,
                             item: Some(item.clone()),
                             ty: ty.clone(),
diff --git a/src/parser/grammar.lalrpop b/src/parser/grammar.lalrpop
index 33c8d2d1..2f578b7a 100644
--- a/src/parser/grammar.lalrpop
+++ b/src/parser/grammar.lalrpop
@@ -46,7 +46,7 @@ use lalrpop_util::ErrorRecovery;
 use super::{
     ExtendedTerm,
     utils::*,
-    lexer::{Token, NormalToken, StringToken, MultiStringToken},
+    lexer::{Token, NormalToken, StringToken, MultiStringToken, SymbolicStringStart},
     error::ParseError,
     uniterm::*,
 };
@@ -60,7 +60,7 @@ use crate::{
     term::{
         BinaryOp, RichTerm, Term, UnaryOp, StrChunk, MetaValue,
         MergePriority, Contract, NAryOp, record::RecordAttrs, SharedTerm,
-        NumeralPriority, array::Array, make as mk_term,
+        NumeralPriority, array::Array, make as mk_term
     },
     types::{
         Types, TypeF, EnumRows, EnumRowsF, RecordRows, RecordRowsF,
@@ -511,12 +511,17 @@ StrChunks: RichTerm = {
             chunks
         };
 
-        if start == StringStartDelimiter::Symbolic {
+        if let StringStartDelimiter::Symbolic(prefix) = start {
             let terms = chunks.into_iter().map(|chunk| match chunk {
                 StrChunk::Literal(l) => Term::Str(l).into(),
                 StrChunk::Expr(e, _) => e,
             }).collect();
-            RichTerm::from(Term::Array(terms, Default::default()))
+
+            RichTerm::from(build_record([
+                (FieldPathElem::Ident("tag".into()), RichTerm::from(Term::Enum("SymbolicString".into()))),
+                (FieldPathElem::Ident("prefix".into()), RichTerm::from(Term::Str(prefix.to_owned()))),
+                (FieldPathElem::Ident("fragments".into()), RichTerm::from(Term::Array(terms, Default::default())))
+            ], Default::default()))
         } else {
             let mut chunks = chunks;
             chunks.reverse();
@@ -525,10 +530,10 @@ StrChunks: RichTerm = {
     },
 };
 
-StringStart : StringStartDelimiter = {
+StringStart : StringStartDelimiter<'input> = {
     "\"" => StringStartDelimiter::Standard,
     "m%\"" => StringStartDelimiter::Multiline,
-    "s%\"" => StringStartDelimiter::Symbolic,
+    "symbolic string start" => StringStartDelimiter::Symbolic(<>.0),
 };
 
 StringEnd : StringEndDelimiter = {
@@ -909,7 +914,8 @@ extern {
         "\"" => Token::Normal(NormalToken::DoubleQuote),
         "\"%" => Token::MultiStr(MultiStringToken::End),
         "m%\"" => Token::Normal(NormalToken::MultiStringStart(<usize>)),
-        "s%\"" => Token::Normal(NormalToken::SymbolicStringStart(<usize>)),
+        "symbolic string start" => Token::Normal(NormalToken::SymbolicStringStart(
+          SymbolicStringStart{prefix: <&'input str>, length: <usize>})),
 
         "Num" => Token::Normal(NormalToken::Num),
         "Dyn" => Token::Normal(NormalToken::Dyn),
diff --git a/src/parser/lexer.rs b/src/parser/lexer.rs
index 89ece5a0..2bf5c3e5 100644
--- a/src/parser/lexer.rs
+++ b/src/parser/lexer.rs
@@ -6,10 +6,10 @@
 //! following string:
 //!
 //! ```text
-//! "hello, I have 1 + ${ {a = "40"}.a } + 1 bananas."
+//! "hello, I have 1 + %{ {a = "40"}.a } + 1 bananas."
 //! ```
 //!
-//! Once the `${` token is encountered, the lexer has to switch back to lexing expressions as
+//! Once the `%{` token is encountered, the lexer has to switch back to lexing expressions as
 //! usual. But at the end of the interpolated expression, `+ 1 bananas.` needs to be parsed as a
 //! string again, and not as normal program tokens. Since the interpolated expression is arbitrary,
 //! it can contains nested `{` and `}` (as here, with records) and strings which themselves have
@@ -23,7 +23,7 @@
 //! decide if a closing brace `}` belongs to the expression or is actually the closing brace of the
 //! interpolated expression, indicating that we should switch back to string mode.
 //!
-//! When entering a string, the `Str` mode is entered. When a `${` is encountered in a string,
+//! When entering a string, the `Str` mode is entered. When a `%{` is encountered in a string,
 //! starting an interpolated expression, the normal mode is pushed. At each starting `{` in normal
 //! mode, the brace counter is incremented. At each closing '}', it is decremented. When it reaches
 //! `0`, this is the end of the current interpolated expressions, and we leave the normal mode and
@@ -33,10 +33,22 @@ use crate::parser::error::{LexicalError, ParseError};
 use logos::Logos;
 use std::ops::Range;
 
+fn symbolic_string_prefix_and_length<'input>(
+    lex: &mut logos::Lexer<'input, NormalToken<'input>>,
+) -> SymbolicStringStart<'input> {
+    let slice = lex.slice();
+    let (prefix, postfix) = slice
+        .rsplit_once('-')
+        .expect("The logos regexp ensures this succeeds");
+    SymbolicStringStart {
+        prefix,
+        length: postfix.len(),
+    }
+}
+
 // **IMPORTANT**
 // When adding or removing tokens that might be parsed as identifiers,
-// please update the KEYWORDS array
-// list
+// please update the [KEYWORDS] array
 /// The tokens in normal mode.
 #[derive(Logos, Debug, PartialEq, Clone)]
 pub enum NormalToken<'input> {
@@ -160,8 +172,8 @@ pub enum NormalToken<'input> {
     Underscore,
     #[regex("m(%+)\"", |lex| lex.slice().len())]
     MultiStringStart(usize),
-    #[regex("s(%+)\"", |lex| lex.slice().len())]
-    SymbolicStringStart(usize),
+    #[regex("[a-zA-Z][_a-zA-Z0-9-']*-s(%+)\"", symbolic_string_prefix_and_length)]
+    SymbolicStringStart(SymbolicStringStart<'input>),
 
     #[token("%tag%")]
     Tag,
@@ -327,6 +339,15 @@ pub const KEYWORDS: &[&str] = &[
     "priority", "force",
 ];
 
+#[derive(Debug, Clone, PartialEq)]
+pub struct SymbolicStringStart<'input> {
+    /// The prefix for the symbolic string, e.g. `nix-s%""%` has prefix `"nix"`
+    pub prefix: &'input str,
+    /// The length of the string delimiter, excluding `prefix` and `-`. E.g. `nix-s%%""%` has
+    /// `length` 4, the length of `s%%"`
+    pub length: usize,
+}
+
 /// The tokens in string mode.
 #[derive(Logos, Debug, PartialEq, Eq, Clone)]
 pub enum StringToken<'input> {
@@ -611,7 +632,9 @@ impl<'input> Iterator for Lexer<'input> {
             }
             Some(Normal(
                 NormalToken::MultiStringStart(delim_size)
-                | NormalToken::SymbolicStringStart(delim_size),
+                | NormalToken::SymbolicStringStart(SymbolicStringStart {
+                    length: delim_size, ..
+                }),
             )) => {
                 // for interpolation & closing delimeters we only care about
                 // the number of `%`s (plus the opening `"` or `{`) so we
diff --git a/src/parser/utils.rs b/src/parser/utils.rs
index 5cde50db..88e3d434 100644
--- a/src/parser/utils.rs
+++ b/src/parser/utils.rs
@@ -26,26 +26,29 @@ use crate::{
 /// Distinguish between the standard string opening delimiter `"`, the multi-line string
 /// opening delimter `m%"`, and the symbolic string opening delimiter `s%"`.
 #[derive(Copy, Clone, Eq, PartialEq, Debug)]
-pub enum StringStartDelimiter {
+pub enum StringStartDelimiter<'input> {
     Standard,
     Multiline,
-    Symbolic,
+    Symbolic(&'input str),
 }
 
-impl StringStartDelimiter {
+impl StringStartDelimiter<'_> {
     pub fn is_closed_by(&self, close: &StringEndDelimiter) -> bool {
         matches!(
             (self, close),
             (StringStartDelimiter::Standard, StringEndDelimiter::Standard)
                 | (StringStartDelimiter::Multiline, StringEndDelimiter::Special)
-                | (StringStartDelimiter::Symbolic, StringEndDelimiter::Special)
+                | (
+                    StringStartDelimiter::Symbolic(_),
+                    StringEndDelimiter::Special
+                )
         )
     }
 
     pub fn needs_strip_indent(&self) -> bool {
         match self {
             StringStartDelimiter::Standard => false,
-            StringStartDelimiter::Multiline | StringStartDelimiter::Symbolic => true,
+            StringStartDelimiter::Multiline | StringStartDelimiter::Symbolic(_) => true,
         }
     }
 }


Test patch:
diff --git a/src/parser/tests.rs b/src/parser/tests.rs
index f43acbc8..9b3d6e37 100644
--- a/src/parser/tests.rs
+++ b/src/parser/tests.rs
@@ -1,9 +1,13 @@
-use super::lexer::{Lexer, MultiStringToken, NormalToken, StringToken, Token};
+use std::rc::Rc;
+
+use super::lexer::{Lexer, MultiStringToken, NormalToken, StringToken, SymbolicStringStart, Token};
+use super::utils::{build_record, FieldPathElem};
 use crate::error::ParseError;
 use crate::identifier::Ident;
 use crate::parser::error::ParseError as InternalParseError;
-use crate::term::make as mk_term;
+use crate::term::array::Array;
 use crate::term::Term::*;
+use crate::term::{make as mk_term, Term};
 use crate::term::{record, BinaryOp, RichTerm, StrChunk, UnaryOp};
 use crate::{mk_app, mk_match};
 use assert_matches::assert_matches;
@@ -34,6 +38,30 @@ fn mk_single_chunk(s: &str) -> RichTerm {
     StrChunks(vec![StrChunk::Literal(String::from(s))]).into()
 }
 
+fn mk_symbolic_single_chunk(prefix: &str, s: &str) -> RichTerm {
+    build_record(
+        [
+            (
+                FieldPathElem::Ident("tag".into()),
+                RichTerm::from(Term::Enum("SymbolicString".into())),
+            ),
+            (
+                FieldPathElem::Ident("prefix".into()),
+                RichTerm::from(Term::Str(prefix.to_owned())),
+            ),
+            (
+                FieldPathElem::Ident("fragments".into()),
+                RichTerm::from(Array(
+                    Array::new(Rc::new([Str(String::from(s)).into()])),
+                    Default::default(),
+                )),
+            ),
+        ],
+        Default::default(),
+    )
+    .into()
+}
+
 #[test]
 fn numbers() {
     assert_eq!(parse_without_pos("22"), Num(22.0).into());
@@ -74,6 +102,14 @@ fn strings() {
     )
 }
 
+#[test]
+fn symbolic_strings() {
+    assert_eq!(
+        parse_without_pos(r#"foo-s%"hello world"%"#),
+        mk_symbolic_single_chunk("foo", "hello world"),
+    );
+}
+
 #[test]
 fn plus() {
     assert_eq!(
@@ -354,17 +390,23 @@ fn string_lexing() {
         ),
         (
             "empty symbolic string lexes like multi-line str",
-            r#"s%""%"#,
+            r#"foo-s%""%"#,
             vec![
-                Token::Normal(NormalToken::SymbolicStringStart(3)),
+                Token::Normal(NormalToken::SymbolicStringStart(SymbolicStringStart {
+                    prefix: "foo",
+                    length: 3,
+                })),
                 Token::MultiStr(MultiStringToken::End),
             ],
         ),
         (
             "symbolic string with interpolation",
-            r#"s%"text %{ 1 } etc."%"#,
+            r#"foo-s%"text %{ 1 } etc."%"#,
             vec![
-                Token::Normal(NormalToken::SymbolicStringStart(3)),
+                Token::Normal(NormalToken::SymbolicStringStart(SymbolicStringStart {
+                    prefix: "foo",
+                    length: 3,
+                })),
                 Token::MultiStr(MultiStringToken::Literal("text ")),
                 Token::MultiStr(MultiStringToken::Interpolation),
                 Token::Normal(NormalToken::NumLiteral(1.0)),
@@ -373,6 +415,17 @@ fn string_lexing() {
                 Token::MultiStr(MultiStringToken::End),
             ],
         ),
+        (
+            "empty symbolic string with tag",
+            r#"tf-s%""%"#,
+            vec![
+                Token::Normal(NormalToken::SymbolicStringStart(SymbolicStringStart {
+                    prefix: "tf",
+                    length: 3,
+                })),
+                Token::MultiStr(MultiStringToken::End),
+            ],
+        ),
     ] {
         assert_eq!(
             lex_without_pos(input),
diff --git a/tests/integration/pass/symbolic-strings.ncl b/tests/integration/pass/symbolic-strings.ncl
index 9e4d7e7a..4895789e 100644
--- a/tests/integration/pass/symbolic-strings.ncl
+++ b/tests/integration/pass/symbolic-strings.ncl
@@ -1,37 +1,37 @@
 let {check, ..} = import "lib/assert.ncl" in
-
+let sym = fun prefix_ fragments_ => { tag = `SymbolicString, prefix = prefix_, fragments = fragments_ } in
 [
   # Static symbolic string
-  s%"hello, world"% == ["hello, world"],
+  foo-s%"hello, world"% == sym "foo" ["hello, world"] ,
   # Interpolating a string
   let s = "test" in
-  s%"This is a %{s}"% == ["This is a ", "test"],
+  foo-s%"This is a %{s}"% == sym "foo" ["This is a ", "test"],
   # Interpolating an interpolated string
   let f = "f" in
-  s%"abc %{"de%{f}"}"% == ["abc ", "def"],
+  foo-s%"abc %{"de%{f}"}"% == sym "foo" ["abc ", "def"],
   # Interpolating a number
-  s%"num: %{100}"% == ["num: ", 100],
+  foo-s%"num: %{100}"% == sym "foo" ["num: ", 100],
   # Interpolating a bool
-  s%"bool: %{true}"% == ["bool: ", true],
+  foo-s%"bool: %{true}"% == sym "foo" ["bool: ", true],
   # Interpolating an array
-  s%"array: %{[true, 1, "yes"]}"% == ["array: ", [true, 1, "yes"]],
+  foo-s%"array: %{[true, 1, "yes"]}"% == sym "foo" ["array: ", [true, 1, "yes"]],
   # Interpolating a record
   let r = { a = 1, b = false } in
-  s%"record: %{r}"% == ["record: ", r],
+  foo-s%"record: %{r}"% == sym "foo" ["record: ", r],
   # Interpolating multiple values
   let str = "some string" in
   let num = 999.999 in
   let bool = false in
   let array = ["an", "array", 100] in
   let record = { a = 1, simple = "yes", record = true } in
-  let actual = s%"
+  let actual = foo-s%"
      1. %{str}
      2. %{num}
      3. %{bool}
      4. %{array}
      5. %{record}"%
   in
-  let expected = [
+  let expected = sym "foo" [
     "1. ", str,
     "\n2. ", num,
     "\n3. ", bool,