Repository: Behat/Gherkin (php)

Issue:
CucumberNDJsonAstLoader does not parse tables and pystrings for tests
From https://github.com/Behat/Gherkin/issues/319#issuecomment-2876628868 -
currently we are not comparing `arguments` (TableNode and PyString) in the cucumber CompatibilityTest.

This is because because [CucumberNDJsonAstLoader](https://github.com/Behat/Gherkin/blob/master/src/Loader/CucumberNDJsonAstLoader.php#L139) doesn't attempt to load/parse step arguments. 

It looks like those are present in the NDJson (as either `docString` or `dataTable` keys) so I'm not sure whether there was a problem parsing them originally, or whether it was just felt that it was not required.

If possible, we should parse those arguments and compare them to ensure that our parser treats these the same as cucumber/gherkin.

Pull request description:
test: Compare step arguments when checking gherkin parity
Originally, the NDJson loader did not attempt to create PyString / TableNode objects from the cucumber/gherkin AST. As a result, the CompatibilityTest just removed arguments our parser had included with any StepNodes, before comparing to the cucumber result.

This commit instead hydrates this testdata from the gherkin AST. This allows us to check parity of parsing multiline step arguments.

Fixes #320

Patch:
diff --git a/src/Loader/CucumberNDJsonAstLoader.php b/src/Loader/CucumberNDJsonAstLoader.php
index 70c2695..d19ab18 100644
--- a/src/Loader/CucumberNDJsonAstLoader.php
+++ b/src/Loader/CucumberNDJsonAstLoader.php
@@ -15,9 +15,11 @@ use Behat\Gherkin\Node\BackgroundNode;
 use Behat\Gherkin\Node\ExampleTableNode;
 use Behat\Gherkin\Node\FeatureNode;
 use Behat\Gherkin\Node\OutlineNode;
+use Behat\Gherkin\Node\PyStringNode;
 use Behat\Gherkin\Node\ScenarioInterface;
 use Behat\Gherkin\Node\ScenarioNode;
 use Behat\Gherkin\Node\StepNode;
+use Behat\Gherkin\Node\TableNode;
 
 /**
  * Loads a feature from cucumber's protobuf JSON format.
@@ -31,12 +33,16 @@ class CucumberNDJsonAstLoader implements LoaderInterface
 
     public function load($resource)
     {
-        return array_values(array_filter(array_map(
-            static function ($line) use ($resource) {
-                return self::getFeature(json_decode($line, true), $resource);
-            },
-            file($resource)
-        )));
+        return array_values(
+            array_filter(
+                array_map(
+                    static function ($line) use ($resource) {
+                        return self::getFeature(json_decode($line, true), $resource);
+                    },
+                    file($resource)
+                )
+            )
+        );
     }
 
     /**
@@ -143,7 +149,7 @@ class CucumberNDJsonAstLoader implements LoaderInterface
             static fn (array $item) => new StepNode(
                 trim($item['keyword']),
                 $item['text'],
-                [],
+                self::getStepArguments($item),
                 $item['location']['line'],
                 trim($item['keyword'])
             ),
@@ -151,6 +157,28 @@ class CucumberNDJsonAstLoader implements LoaderInterface
         );
     }
 
+    private static function getStepArguments(array $step): array
+    {
+        $args = [];
+
+        if (isset($step['docString'])) {
+            $args[] = new PyStringNode(
+                explode("\n", $step['docString']['content']),
+                $step['docString']['location']['line'],
+            );
+        }
+
+        if (isset($step['dataTable'])) {
+            $table = [];
+            foreach ($step['dataTable']['rows'] as $row) {
+                $table[$row['location']['line']] = array_column($row['cells'], 'value');
+            }
+            $args[] = new TableNode($table);
+        }
+
+        return $args;
+    }
+
     /**
      * @return ExampleTableNode[]
      */


Test patch:
diff --git a/tests/Cucumber/CompatibilityTest.php b/tests/Cucumber/CompatibilityTest.php
index 4352c32..7fe8d15 100644
--- a/tests/Cucumber/CompatibilityTest.php
+++ b/tests/Cucumber/CompatibilityTest.php
@@ -44,6 +44,9 @@ class CompatibilityTest extends TestCase
      */
     private array $notParsingCorrectly = [
         'complex_background.feature' => 'Rule keyword not supported',
+        'docstrings.feature' => 'Escaped delimiters in docstrings are not unescaped',
+        'datatables_with_new_lines.feature' => 'Escaped newlines in table cells are not unescaped',
+        'escaped_pipes.feature' => 'Escaped newlines in table cells are not unescaped',
         'rule.feature' => 'Rule keyword not supported',
         'rule_with_tag.feature' => 'Rule keyword not supported',
         'tags.feature' => 'Rule keyword not supported',
@@ -226,9 +229,7 @@ class CompatibilityTest extends TestCase
         return new StepNode(
             $stepNode->getKeyword(),
             $stepNode->getText(),
-            // CucumberNDJsonParser does not currently parse tables / pystrings attached to a step
-            // See https://github.com/Behat/Gherkin/issues/320
-            [],
+            $stepNode->getArguments(),
             $stepNode->getLine(),
             // We cannot compare the keywordsType property on a StepNode because this concept
             // is specific to Behat/Gherkin and there is no equivalent value in the cucumber/gherkin
diff --git a/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature
new file mode 100644
index 0000000..dabd50d
--- /dev/null
+++ b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature
@@ -0,0 +1,50 @@
+# TEMPORARY until we resume parsing the official `docstrings.feature`
+Feature: DocString variations
+
+  Scenario: minimalistic
+    Given a simple DocString
+      """
+      first line (no indent)
+        second line (indented with two spaces)
+
+      third line was empty
+      """
+    Given a DocString with content type
+      """xml
+      <foo>
+        <bar />
+      </foo>
+      """
+    And a DocString with wrong indentation
+      """
+    wrongly indented line
+      """
+    And a DocString with alternative separator
+      ```
+      first line
+      second line
+      ```
+    And a DocString with normal separator inside
+      ```
+      first line
+      """
+      third line
+      ```
+    And a DocString with alternative separator inside
+      """
+      first line
+      ```
+      third line
+      """
+    #And a DocString with escaped separator inside
+    #  """
+    #  first line
+    #  \"\"\"
+    #  third line
+    #  """
+    #And a DocString with an escaped alternative separator inside
+    #  ```
+    #  first line
+    #  \`\`\`
+    #  third line
+    #  ```
\ No newline at end of file
diff --git a/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.ast.ndjson b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.ast.ndjson
new file mode 100644
index 0000000..2a0e6a6
--- /dev/null
+++ b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.ast.ndjson
@@ -0,0 +1,1 @@
+{"gherkinDocument":{"comments":[{"location":{"column":1,"line":1},"text":"# TEMPORARY until we resume parsing the official `docstrings.feature`"},{"location":{"column":1,"line":39},"text":"    #And a DocString with escaped separator inside"},{"location":{"column":1,"line":40},"text":"    #  \"\"\""},{"location":{"column":1,"line":41},"text":"    #  first line"},{"location":{"column":1,"line":42},"text":"    #  \\\"\\\"\\\""},{"location":{"column":1,"line":43},"text":"    #  third line"},{"location":{"column":1,"line":44},"text":"    #  \"\"\""},{"location":{"column":1,"line":45},"text":"    #And a DocString with an escaped alternative separator inside"},{"location":{"column":1,"line":46},"text":"    #  ```"},{"location":{"column":1,"line":47},"text":"    #  first line"},{"location":{"column":1,"line":48},"text":"    #  \\`\\`\\`"},{"location":{"column":1,"line":49},"text":"    #  third line"},{"location":{"column":1,"line":50},"text":"    #  ```"}],"feature":{"children":[{"scenario":{"description":"","examples":[],"id":"6","keyword":"Scenario","location":{"column":3,"line":4},"name":"minimalistic","steps":[{"docString":{"content":"first line (no indent)\n  second line (indented with two spaces)\n\nthird line was empty","delimiter":"\"\"\"","location":{"column":7,"line":6}},"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":5},"text":"a simple DocString"},{"docString":{"content":"<foo>\n  <bar />\n</foo>","delimiter":"\"\"\"","location":{"column":7,"line":13},"mediaType":"xml"},"id":"1","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":12},"text":"a DocString with content type"},{"docString":{"content":"wrongly indented line","delimiter":"\"\"\"","location":{"column":7,"line":19}},"id":"2","keyword":"And ","keywordType":"Conjunction","location":{"column":5,"line":18},"text":"a DocString with wrong indentation"},{"docString":{"content":"first line\nsecond line","delimiter":"```","location":{"column":7,"line":23}},"id":"3","keyword":"And ","keywordType":"Conjunction","location":{"column":5,"line":22},"text":"a DocString with alternative separator"},{"docString":{"content":"first line\n\"\"\"\nthird line","delimiter":"```","location":{"column":7,"line":28}},"id":"4","keyword":"And ","keywordType":"Conjunction","location":{"column":5,"line":27},"text":"a DocString with normal separator inside"},{"docString":{"content":"first line\n```\nthird line","delimiter":"\"\"\"","location":{"column":7,"line":34}},"id":"5","keyword":"And ","keywordType":"Conjunction","location":{"column":5,"line":33},"text":"a DocString with alternative separator inside"}],"tags":[]}}],"description":"","keyword":"Feature","language":"en","location":{"column":1,"line":2},"name":"DocString variations","tags":[]},"uri":"../good/tmp_simple_docstrings.feature"}}
diff --git a/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.pickles.ndjson b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.pickles.ndjson
new file mode 100644
index 0000000..ecda0c8
--- /dev/null
+++ b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.pickles.ndjson
@@ -0,0 +1,1 @@
+{"pickle":{"astNodeIds":["6"],"id":"13","language":"en","name":"minimalistic","steps":[{"argument":{"docString":{"content":"first line (no indent)\n  second line (indented with two spaces)\n\nthird line was empty"}},"astNodeIds":["0"],"id":"7","text":"a simple DocString","type":"Context"},{"argument":{"docString":{"content":"<foo>\n  <bar />\n</foo>","mediaType":"xml"}},"astNodeIds":["1"],"id":"8","text":"a DocString with content type","type":"Context"},{"argument":{"docString":{"content":"wrongly indented line"}},"astNodeIds":["2"],"id":"9","text":"a DocString with wrong indentation","type":"Context"},{"argument":{"docString":{"content":"first line\nsecond line"}},"astNodeIds":["3"],"id":"10","text":"a DocString with alternative separator","type":"Context"},{"argument":{"docString":{"content":"first line\n\"\"\"\nthird line"}},"astNodeIds":["4"],"id":"11","text":"a DocString with normal separator inside","type":"Context"},{"argument":{"docString":{"content":"first line\n```\nthird line"}},"astNodeIds":["5"],"id":"12","text":"a DocString with alternative separator inside","type":"Context"}],"tags":[],"uri":"../good/tmp_simple_docstrings.feature"}}
diff --git a/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.source.ndjson b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.source.ndjson
new file mode 100644
index 0000000..7eae689
--- /dev/null
+++ b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.source.ndjson
@@ -0,0 +1,1 @@
+{"source":{"data":"# TEMPORARY until we resume parsing the official `docstrings.feature`\nFeature: DocString variations\n\n  Scenario: minimalistic\n    Given a simple DocString\n      \"\"\"\n      first line (no indent)\n        second line (indented with two spaces)\n\n      third line was empty\n      \"\"\"\n    Given a DocString with content type\n      \"\"\"xml\n      <foo>\n        <bar />\n      </foo>\n      \"\"\"\n    And a DocString with wrong indentation\n      \"\"\"\n    wrongly indented line\n      \"\"\"\n    And a DocString with alternative separator\n      ```\n      first line\n      second line\n      ```\n    And a DocString with normal separator inside\n      ```\n      first line\n      \"\"\"\n      third line\n      ```\n    And a DocString with alternative separator inside\n      \"\"\"\n      first line\n      ```\n      third line\n      \"\"\"\n    #And a DocString with escaped separator inside\n    #  \"\"\"\n    #  first line\n    #  \\\"\\\"\\\"\n    #  third line\n    #  \"\"\"\n    #And a DocString with an escaped alternative separator inside\n    #  ```\n    #  first line\n    #  \\`\\`\\`\n    #  third line\n    #  ```","mediaType":"text/x.cucumber.gherkin+plain","uri":"../good/tmp_simple_docstrings.feature"}}
diff --git a/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.tokens b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.tokens
new file mode 100644
index 0000000..e105db3
--- /dev/null
+++ b/tests/Cucumber/extra_testdata/good/tmp_simple_docstrings.feature.tokens
@@ -0,0 +1,51 @@
+(1:1)Comment:/# TEMPORARY until we resume parsing the official `docstrings.feature`/
+(2:1)FeatureLine:()Feature/DocString variations/
+(3:1)Empty://
+(4:3)ScenarioLine:()Scenario/minimalistic/
+(5:5)StepLine:(Context)Given /a simple DocString/
+(6:7)DocStringSeparator:()"""//
+(7:1)Other:/first line (no indent)/
+(8:1)Other:/  second line (indented with two spaces)/
+(9:1)Other://
+(10:1)Other:/third line was empty/
+(11:7)DocStringSeparator:()"""//
+(12:5)StepLine:(Context)Given /a DocString with content type/
+(13:7)DocStringSeparator:()"""/xml/
+(14:1)Other:/<foo>/
+(15:1)Other:/  <bar />/
+(16:1)Other:/</foo>/
+(17:7)DocStringSeparator:()"""//
+(18:5)StepLine:(Conjunction)And /a DocString with wrong indentation/
+(19:7)DocStringSeparator:()"""//
+(20:1)Other:/wrongly indented line/
+(21:7)DocStringSeparator:()"""//
+(22:5)StepLine:(Conjunction)And /a DocString with alternative separator/
+(23:7)DocStringSeparator:()```//
+(24:1)Other:/first line/
+(25:1)Other:/second line/
+(26:7)DocStringSeparator:()```//
+(27:5)StepLine:(Conjunction)And /a DocString with normal separator inside/
+(28:7)DocStringSeparator:()```//
+(29:1)Other:/first line/
+(30:1)Other:/"""/
+(31:1)Other:/third line/
+(32:7)DocStringSeparator:()```//
+(33:5)StepLine:(Conjunction)And /a DocString with alternative separator inside/
+(34:7)DocStringSeparator:()"""//
+(35:1)Other:/first line/
+(36:1)Other:/```/
+(37:1)Other:/third line/
+(38:7)DocStringSeparator:()"""//
+(39:1)Comment:/    #And a DocString with escaped separator inside/
+(40:1)Comment:/    #  """/
+(41:1)Comment:/    #  first line/
+(42:1)Comment:/    #  \"\"\"/
+(43:1)Comment:/    #  third line/
+(44:1)Comment:/    #  """/
+(45:1)Comment:/    #And a DocString with an escaped alternative separator inside/
+(46:1)Comment:/    #  ```/
+(47:1)Comment:/    #  first line/
+(48:1)Comment:/    #  \`\`\`/
+(49:1)Comment:/    #  third line/
+(50:1)Comment:/    #  ```/
+EOF