Repository: w3c/webidl2.js (js)

Issue:
Comment order parsing bug 
Give the following input (top level):

```
/* first */
// second
```

The parser is returning:
```
// second
/* first */
```

As json:

```JS
[
  {
    "type": "line-comment",
    "value": " second"
  },
  {
    "type": "multiline-comment",
    "value": " first "
  },
  {
    "type": "ws",
    "value": "\n"
  }
]```



Pull request description:
Remove m postfix from all_ws()
Fixes #107.

The `m` prefix causes the regex to run for every line, which causes the bug.


Patch:
diff --git a/README.md b/README.md
index 66949da..8fcdbc8 100644
--- a/README.md
+++ b/README.md
@@ -136,6 +136,7 @@ Where the fields are as follows:
 * `union`: Boolean indicating whether this is a union type or not.
 
 ### Interface
+
 Interfaces look like this:
 
 ```JS
@@ -167,7 +168,36 @@ The fields are as follows:
   sense.
 * `extAttrs`: A list of [extended attributes](#extended-attributes).
 
+### Interface mixins
+
+Interfaces mixins look like this:
+
+```JS
+{
+  "type": "interface mixin",
+  "name": "Animal",
+  "partial": false,
+  "members": [...],
+  "extAttrs": [...]
+}, {
+  "type": "interface mixin",
+  "name": "Human",
+  "partial": false,
+  "members": [...],
+  "extAttrs": [...]
+}
+```
+
+The fields are as follows:
+
+* `type`: Always "interface mixin".
+* `name`: The name of the interface mixin.
+* `partial`: A boolean indicating whether it's a partial interface mixin.
+* `members`: An array of interface members (attributes, operations, etc.). Empty if there are none.
+* `extAttrs`: A list of [extended attributes](#extended-attributes).
+
 ### Namespace
+
 Namespaces look like this:
 
 ```JS
@@ -350,6 +380,26 @@ The fields are as follows:
 * `implements`: The interface that is being implemented by the target.
 * `extAttrs`: A list of [extended attributes](#extended-attributes).
 
+### Includes
+
+An includes definition looks like this:
+
+```JS
+{
+  "type": "includes",
+  "target": "Node",
+  "includes": "EventTarget",
+  "extAttrs": []
+}
+```
+
+The fields are as follows:
+
+* `type`: Always "includes".
+* `target`: The interface that includes an interface mixin.
+* `includes`: The interface mixin that is being included by the target.
+* `extAttrs`: A list of [extended attributes](#extended-attributes).
+
 ### Operation Member
 
 An operation looks like this:
diff --git a/lib/webidl2.js b/lib/webidl2.js
index 2fd2671..29da7ba 100644
--- a/lib/webidl2.js
+++ b/lib/webidl2.js
@@ -1,8 +1,6 @@
 "use strict";
 
-// Foo
-
-(function() {
+(() => {
   function tokenise(str) {
     const tokens = [];
     const re = {
@@ -54,6 +52,15 @@
     const STR = "string";
     const OTHER = "other";
 
+    const EMPTY_OPERATION = Object.freeze({
+      type: "operation",
+      getter: false,
+      setter: false,
+      deleter: false,
+      "static": false,
+      stringifier: false
+    });
+
     function error(str) {
       let tok = "";
       let numTokens = 0;
@@ -108,7 +115,7 @@
           let w = t.value;
           const re = {
             "ws": /^([\t\n\r ]+)/,
-            "line-comment": /^\/\/(.*)\n?/m,
+            "line-comment": /^\/\/(.*)\r?\n?/,
             "multiline-comment": /^\/\*((?:.|\n|\r)*?)\*\//
           };
           const wsTypes = [];
@@ -481,8 +488,7 @@
       all_ws();
       const tok = consume(ID, "interface");
       if (tok) {
-        tokens.unshift(tok);
-        ret = interface_();
+        ret = interface_rest();
         ret.type = "callback interface";
         return ret;
       }
@@ -512,13 +518,6 @@
         inherit: false,
         readonly: false
       };
-      if (consume(ID, "static")) {
-        ret["static"] = true;
-        grabbed.push(last_token);
-      } else if (consume(ID, "stringifier")) {
-        ret.stringifier = true;
-        grabbed.push(last_token);
-      }
       const w = all_ws();
       if (w) grabbed.push(w);
       if (consume(ID, "inherit")) {
@@ -569,14 +568,7 @@
 
     function operation(store) {
       all_ws(store, "pea");
-      const ret = {
-        type: "operation",
-        getter: false,
-        setter: false,
-        deleter: false,
-        "static": false,
-        stringifier: false
-      };
+      const ret = Object.assign({}, EMPTY_OPERATION);
       while (true) {
         all_ws();
         if (consume(ID, "getter")) ret.getter = true;
@@ -590,19 +582,6 @@
         operation_rest(ret, store);
         return ret;
       }
-      if (consume(ID, "static")) {
-        ret["static"] = true;
-        ret.idlType = return_type();
-        operation_rest(ret, store);
-        return ret;
-      } else if (consume(ID, "stringifier")) {
-        ret.stringifier = true; -
-        all_ws();
-        if (consume(OTHER, ";")) return ret;
-        ret.idlType = return_type();
-        operation_rest(ret, store);
-        return ret;
-      }
       ret.idlType = return_type();
       all_ws();
       if (consume(ID, "iterator")) {
@@ -624,6 +603,27 @@
       }
     };
 
+    function static_member(store) {
+      all_ws(store, "pea");
+      if (!consume(ID, "static")) return;
+      all_ws();
+      return noninherited_attribute(store, "static") ||
+        regular_operation(store, "static") ||
+        error("No body in static member");
+    }
+
+    function stringifier(store) {
+      all_ws(store, "pea");
+      if (!consume(ID, "stringifier")) return;
+      all_ws();
+      if (consume(OTHER, ";")) {
+        return Object.assign({}, EMPTY_OPERATION, { stringifier: true });
+      }
+      return noninherited_attribute(store, "stringifier") ||
+        regular_operation(store, "stringifier") ||
+        error("Unterminated stringifier");
+    }
+
     function identifiers(arr) {
       while (true) {
         all_ws();
@@ -697,9 +697,7 @@
       return ret;
     };
 
-    function interface_(isPartial, store) {
-      all_ws(isPartial ? null : store, "pea");
-      if (!consume(ID, "interface")) return;
+    function interface_rest(isPartial, store) {
       all_ws();
       const name = consume(ID) || error("No name for interface");
       const mems = [];
@@ -728,6 +726,8 @@
           continue;
         }
         const mem = (opt.allowNestedTypedefs && typedef(store ? mems : null)) ||
+          static_member(store ? mems : null) ||
+          stringifier(store ? mems : null) ||
           iterable(store ? mems : null) ||
           attribute(store ? mems : null) ||
           operation(store ? mems : null) ||
@@ -737,6 +737,52 @@
       }
     };
 
+    function mixin_rest(isPartial, store) {
+      all_ws();
+      if (!consume(ID, "mixin")) return;
+      all_ws();
+      const name = consume(ID) || error("No name for interface mixin");
+      const mems = [];
+      const ret = {
+        type: "interface mixin",
+        name: isPartial ? name.value : sanitize_name(name.value, "interface mixin"),
+        partial: false,
+        members: mems
+      };
+      all_ws();
+      consume(OTHER, "{") || error("Bodyless interface mixin");
+      while (true) {
+        all_ws(store ? mems : null);
+        if (consume(OTHER, "}")) {
+          all_ws();
+          consume(OTHER, ";") || error("Missing semicolon after interface mixin");
+          return ret;
+        }
+        const ea = extended_attrs(store ? mems : null);
+        all_ws();
+        const cnt = const_(store ? mems : null);
+        if (cnt) {
+          cnt.extAttrs = ea;
+          ret.members.push(cnt);
+          continue;
+        }
+        const mem = stringifier(store ? mems : null) ||
+          noninherited_attribute(store ? mems : null) ||
+          regular_operation(store ? mems : null) ||
+          error("Unknown member");
+        mem.extAttrs = ea;
+        ret.members.push(mem);
+      }
+    }
+
+    function interface_(isPartial, store) {
+      all_ws(isPartial ? null : store, "pea");
+      if (!consume(ID, "interface")) return;
+      return mixin_rest(isPartial, store) ||
+        interface_rest(isPartial, store) ||
+        error("Interface has no proper body");
+    }
+
     function namespace(isPartial, store) {
       all_ws(isPartial ? null : store, "pea");
       if (!consume(ID, "namespace")) return;
@@ -761,14 +807,14 @@
         const ea = extended_attrs(store ? mems : null);
         all_ws();
         const mem = noninherited_attribute(store ? mems : null) ||
-          nonspecial_operation(store ? mems : null) ||
+          regular_operation(store ? mems : null) ||
           error("Unknown member");
         mem.extAttrs = ea;
         ret.members.push(mem);
       }
     }
 
-    function noninherited_attribute(store) {
+    function noninherited_attribute(store, prefix) {
       const w = all_ws(store, "pea");
       const grabbed = [];
       const ret = {
@@ -778,6 +824,9 @@
         inherit: false,
         readonly: false
       };
+      if (prefix) {
+        ret[prefix] = true;
+      }
       if (w) grabbed.push(w);
       if (consume(ID, "readonly")) {
         ret.readonly = true;
@@ -792,16 +841,12 @@
       return rest;
     }
 
-    function nonspecial_operation(store) {
+    function regular_operation(store, prefix) {
       all_ws(store, "pea");
-      const ret = {
-        type: "operation",
-        getter: false,
-        setter: false,
-        deleter: false,
-        "static": false,
-        stringifier: false
-      };
+      const ret = Object.assign({}, EMPTY_OPERATION);
+      if (prefix) {
+        ret[prefix] = true;
+      }
       ret.idlType = return_type();
       return operation_rest(ret, store);
     }
@@ -863,7 +908,7 @@
       }
     };
 
-    const enum_ = function(store) {
+    function enum_(store) {
       all_ws(store, "pea");
       if (!consume(ID, "enum")) return;
       all_ws();
@@ -886,7 +931,7 @@
         }
         const val = consume(STR) || error("Unexpected value in enum");
         val.value = val.value.replace(/"/g, "");
-        ret.values.push( val );
+        ret.values.push(val);
         all_ws(store ? vals : null);
         if (consume(OTHER, ",")) {
           if (store) vals.push({ type: "," });
@@ -937,6 +982,29 @@
       }
     };
 
+    function includes(store) {
+      all_ws(store, "pea");
+      const target = consume(ID);
+      if (!target) return;
+      const w = all_ws();
+      if (consume(ID, "includes")) {
+        const ret = {
+          type: "includes",
+          target: target.value
+        };
+        all_ws();
+        const imp = consume(ID) || error("Incomplete includes statement");
+        ret["includes"] = imp.value;
+        all_ws();
+        consume(OTHER, ";") || error("No terminating ; for includes statement");
+        return ret;
+      } else {
+        // rollback
+        tokens.unshift(w);
+        tokens.unshift(target);
+      }
+    };
+
     function definition(store) {
       return callback(store) ||
         interface_(false, store) ||
@@ -945,6 +1013,7 @@
         enum_(store) ||
         typedef(store) ||
         implements_(store) ||
+        includes(store) ||
         namespace(false, store);
     };
 
@@ -983,4 +1052,4 @@
   } else {
     (self || window).WebIDL2 = obj;
   }
-}());
+})();
diff --git a/package.json b/package.json
index b764abd..969464b 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "webidl2",
   "description": "A WebIDL Parser",
-  "version": "7.0.0",
+  "version": "8.0.0",
   "contributors": [
     "Robin Berjon <robin@berjon.com> (https://berjon.com)",
     "Marcos Caceres <marcos@marcosc.com> (https://marcosc.com)",


Test patch:
diff --git a/test/syntax.js b/test/syntax.js
index a7767af..0838f35 100644
--- a/test/syntax.js
+++ b/test/syntax.js
@@ -26,7 +26,7 @@ describe("Parses all of the IDLs to produce the correct ASTs", () => {
         if (fs.existsSync(optFile))
           opt = JSON.parse(fs.readFileSync(optFile, "utf8"));
         const diff = jdp.diff(JSON.parse(fs.readFileSync(json, "utf8")),
-          wp.parse(fs.readFileSync(idl, "utf8"), opt));
+          wp.parse(fs.readFileSync(idl, "utf8").replace(/\r\n/g, "\n"), opt));
         if (diff && debug) console.log(JSON.stringify(diff, null, 4));
         expect(diff).toBe(undefined);
       }
diff --git a/test/syntax/idl/linecomment.widl b/test/syntax/idl/linecomment.widl
new file mode 100644
index 0000000..5d31bdb
--- /dev/null
+++ b/test/syntax/idl/linecomment.widl
@@ -0,0 +1,2 @@
+/* first */
+// second
diff --git a/test/syntax/idl/mixin.widl b/test/syntax/idl/mixin.widl
new file mode 100644
index 0000000..7c37a6e
--- /dev/null
+++ b/test/syntax/idl/mixin.widl
@@ -0,0 +1,12 @@
+// Extracted from https://heycam.github.io/webidl/#using-mixins-and-partials on 2017-11-02
+
+interface mixin GlobalCrypto {
+  readonly attribute Crypto crypto;
+};
+
+Window includes GlobalCrypto;
+WorkerGlobalScope includes GlobalCrypto;
+
+partial interface mixin WindowOrWorkerGlobalScope {
+  readonly attribute Crypto crypto;
+};
diff --git a/test/syntax/json/linecomment.json b/test/syntax/json/linecomment.json
new file mode 100644
index 0000000..11fd14f
--- /dev/null
+++ b/test/syntax/json/linecomment.json
@@ -0,0 +1,14 @@
+[
+    {
+        "type": "multiline-comment",
+        "value": " first "
+    },
+    {
+        "type": "ws",
+        "value": "\n"
+    },
+    {
+        "type": "line-comment",
+        "value": " second"
+    }
+]
diff --git a/test/syntax/json/mixin.json b/test/syntax/json/mixin.json
new file mode 100644
index 0000000..a2afae3
--- /dev/null
+++ b/test/syntax/json/mixin.json
@@ -0,0 +1,62 @@
+[
+    {
+        "type": "interface mixin",
+        "name": "GlobalCrypto",
+        "partial": false,
+        "members": [
+            {
+                "type": "attribute",
+                "static": false,
+                "stringifier": false,
+                "inherit": false,
+                "readonly": true,
+                "idlType": {
+                    "sequence": false,
+                    "generic": null,
+                    "nullable": false,
+                    "union": false,
+                    "idlType": "Crypto"
+                },
+                "name": "crypto",
+                "extAttrs": []
+            }
+        ],
+        "extAttrs": []
+    },
+    {
+        "type": "includes",
+        "target": "Window",
+        "includes": "GlobalCrypto",
+        "extAttrs": []
+    },
+    {
+        "type": "includes",
+        "target": "WorkerGlobalScope",
+        "includes": "GlobalCrypto",
+        "extAttrs": []
+    },
+    {
+        "type": "interface mixin",
+        "name": "WindowOrWorkerGlobalScope",
+        "partial": true,
+        "members": [
+            {
+                "type": "attribute",
+                "static": false,
+                "stringifier": false,
+                "inherit": false,
+                "readonly": true,
+                "idlType": {
+                    "sequence": false,
+                    "generic": null,
+                    "nullable": false,
+                    "union": false,
+                    "idlType": "Crypto"
+                },
+                "name": "crypto",
+                "extAttrs": []
+            }
+        ],
+        "extAttrs": []
+    }
+]
diff --git a/test/syntax/opt/linecomment.json b/test/syntax/opt/linecomment.json
new file mode 100644
index 0000000..fbcdbf4
--- /dev/null
+++ b/test/syntax/opt/linecomment.json
@@ -0,0 +1,3 @@
+{
+    "ws": true
+}