Repository: DmitrySoshnikov/regexp-tree
Pull request #96 (2017-11-06 07:49:57)

Handle codePoints in checkClassRange()
This is a fix for #92.

I added a `codePoint` key on `Char` tokens, so that we can compare all syntaxes the same way.

I had to handle a special case for `control` chars, which seem to not be allowed in character ranges.

I also handled `meta` character classes short forms (`\w`, `\s`, etc.) using `codePoint` with the value `NaN`. I'm not really happy with this. In fact, something like `[a-\s]` should not even be parsed as a character class in the first place, but rather as three separate chars.


Golmote: Ogod what have I done! Is it ok to add this property on Chars? I'll have to fix all tests... but I'll wait for your opinion first. ^^
DmitrySoshnikov: This looks great, thanks! We should update the documentation in the main [README.md](https://github.com/DmitrySoshnikov/regexp-tree#char) (specification for the `Char` node, adding this `codePoint` property, and also mention it in all examples). We should also probably add `symbol` property to simple chars as well for consistency.

Patch:
diff --git a/README.md b/README.md
index b17ab2b..9b33516 100644
--- a/README.md
+++ b/README.md
@@ -100,12 +100,16 @@ Which produces an AST node corresponding to this regular expression:
     left: {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     right: {
       type: 'Char',
       value: 'b',
-      kind: 'simple'
+      symbol: 'b',
+      kind: 'simple',
+      codePoint: 98
     }
   },
   flags: 'i',
@@ -201,7 +205,9 @@ This attaches `loc` object to each AST node:
       {
         type: 'Char',
         value: 'a',
+        symbol: 'a',
         kind: 'simple',
+        codePoint: 97,
         loc: {
           start: {
             line: 1,
@@ -218,7 +224,9 @@ This attaches `loc` object to each AST node:
       {
         type: 'Char',
         value: 'b',
+        symbol: 'b',
         kind: 'simple',
+        codePoint: 98,
         loc: {
           start: {
             line: 1,
@@ -370,6 +378,8 @@ module.exports = {
   Char({node}) {
     if (node.kind === 'simple' && node.value === 'a') {
       node.value = 'b';
+      node.symbol = 'b';
+      node.codePoint = 98;
     }
   },
 };
@@ -404,7 +414,9 @@ const re = regexpTree.generate({
   body: {
     type: 'Char',
     value: 'a',
+    symbol: 'a',
     kind: 'simple',
+    codePoint: 97
   },
   flags: 'i',
 });
@@ -705,7 +717,9 @@ Node:
 {
   type: 'Char',
   value: 'z',
-  kind: 'simple'
+  symbol: 'z',
+  kind: 'simple',
+  codePoint: 122
 }
 ```
 
@@ -723,7 +737,9 @@ The same value, `escaped` flag is added:
 {
   type: 'Char',
   value: 'z',
+  symbol: 'z',
   kind: 'simple',
+  codePoint: 122,
   escaped: true
 }
 ```
@@ -745,7 +761,9 @@ OK, node:
 {
   type: 'Char',
   value: '*',
+  symbol: '*',
   kind: 'simple',
+  codePoint: 42,
   escaped: true
 }
 ```
@@ -766,11 +784,15 @@ Node:
 {
   type: 'Char',
   value: '\\n',
+  symbol: '\n',
   kind: 'meta',
+  codePoint: 10
 }
 ```
 
-Among other meta character are: `\f`, `\r`, `\n`, `\t`, `\v`, `\0`, `[\b]` (backspace char), `\s`, `\S`, `\w`, `\W`, `\d`, `\D`.
+Among other meta character are: `.`, `\f`, `\r`, `\n`, `\t`, `\v`, `\0`, `[\b]` (backspace char), `\s`, `\S`, `\w`, `\W`, `\d`, `\D`.
+
+> NOTE: Meta characters representing ranges (like `.`, `\s`, etc.) have `undefined` value for `symbol` and `NaN` for `codePoint`.
 
 > NOTE: `\b` and `\B` are parsed as `Assertion` node type, not `Char`.
 
@@ -788,7 +810,9 @@ Node:
 {
   type: 'Char',
   value: '\\cx',
+  symbol: undefined,
   kind: 'control',
+  codePoint: NaN
 }
 ```
 
@@ -806,7 +830,9 @@ Node:
 {
   type: 'Char',
   value: '\\x3B',
+  symbol: ';',
   kind: 'hex',
+  codePoint: 59
 }
 ```
 
@@ -824,7 +850,9 @@ Node:
 {
   type: 'Char',
   value: '\\42',
+  symbol: '*',
   kind: 'decimal',
+  codePoint: 42
 }
 ```
 
@@ -842,7 +870,9 @@ Node:
 {
   type: 'Char',
   value: '\\073',
+  symbol: ';',
   kind: 'oct',
+  codePoint: 59
 }
 ```
 
@@ -861,7 +891,9 @@ Node:
 {
   type: 'Char',
   value: '\\u003B',
+  symbol: ';',
   kind: 'unicode',
+  codePoint: 59
 }
 ```
 
@@ -886,12 +918,16 @@ A node:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     {
       type: 'Char',
       value: '*',
-      kind: 'simple'
+      symbol: '*',
+      kind: 'simple',
+      codePoint: 42
     }
   ]
 }
@@ -917,12 +953,16 @@ An AST node is the same, just `negative` property is added:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     {
       type: 'Char',
       value: 'b',
-      kind: 'simple'
+      symbol: 'b',
+      kind: 'simple',
+      codePoint: 98
     }
   ]
 }
@@ -947,12 +987,16 @@ A node:
       from: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 97
       },
       to: {
         type: 'Char',
         value: 'z',
-        kind: 'simple'
+        symbol: 'z',
+        kind: 'simple',
+        codePoint: 122
       }
     }
   ]
@@ -994,17 +1038,23 @@ A node:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     {
       type: 'Char',
       value: 'b',
-      kind: 'simple'
+      symbol: 'b',
+      kind: 'simple',
+      codePoint: 98
     },
     {
       type: 'Char',
       value: 'c',
-      kind: 'simple'
+      symbol: 'c',
+      kind: 'simple',
+      codePoint: 99
     }
   ]
 }
@@ -1038,12 +1088,16 @@ A node:
   left: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   right: {
     type: 'Char',
     value: 'b',
-    kind: 'simple'
+    symbol: 'b',
+    kind: 'simple',
+    codePoint: 98
   }
 }
 ```
@@ -1078,12 +1132,16 @@ A node:
           {
             type: 'Char',
             value: 'a',
-            kind: 'simple'
+            symbol: 'a',
+            kind: 'simple',
+            codePoint: 97
           },
           {
             type: 'Char',
             value: 'b',
-            kind: 'simple'
+            symbol: 'b',
+            kind: 'simple',
+            codePoint: 98
           }
         ]
       }
@@ -1091,7 +1149,9 @@ A node:
     {
       type: 'Char',
       value: 'c',
-      kind: 'simple'
+      symbol: 'c',
+      kind: 'simple',
+      codePoint: 99
     }
   ]
 }
@@ -1135,7 +1195,9 @@ We have the following node (the `name` property with value `foo` is added):
   expression: {
     type: 'Char',
     value: 'x',
-    kind: 'simple'
+    symbol: 'x',
+    kind: 'simple',
+    codePoint: 120
   }
 }
 ```
@@ -1165,12 +1227,16 @@ The same node, the `capturing` flag is `false`:
           {
             type: 'Char',
             value: 'a',
-            kind: 'simple'
+            symbol: 'a',
+            kind: 'simple',
+            codePoint: 97
           },
           {
             type: 'Char',
             value: 'b',
-            kind: 'simple'
+            symbol: 'b',
+            kind: 'simple',
+            codePoint: 98
           }
         ]
       }
@@ -1178,7 +1244,9 @@ The same node, the `capturing` flag is `false`:
     {
       type: 'Char',
       value: 'c',
-      kind: 'simple'
+      symbol: 'c',
+      kind: 'simple',
+      codePoint: 99
     }
   ]
 }
@@ -1210,12 +1278,16 @@ A node:
           {
             type: 'Char',
             value: 'a',
-            kind: 'simple'
+            symbol: 'a',
+            kind: 'simple',
+            codePoint: 97
           },
           {
             type: 'Char',
             value: 'b',
-            kind: 'simple'
+            symbol: 'b',
+            kind: 'simple',
+            codePoint: 98
           }
         ]
       }
@@ -1252,7 +1324,9 @@ A node:
       expression: {
         type: 'Char',
         value: 'w',
-        kind: 'simple'
+        symbol: 'w',
+        kind: 'simple',
+        codePoint: 119
       }
     },
     {
@@ -1291,7 +1365,9 @@ Node:
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1317,7 +1393,9 @@ Node:
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1344,7 +1422,9 @@ Node:
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1372,7 +1452,9 @@ The type of the quantifier is `Range`, and `from`, and `to` properties have the
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1400,7 +1482,9 @@ An AST node for such range doesn't contain `to` property:
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1430,7 +1514,9 @@ An AST node for a closed range:
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1462,7 +1548,9 @@ Node:
   expression: {
     type: 'Char',
     value: 'a',
-    kind: 'simple'
+    symbol: 'a',
+    kind: 'simple',
+    codePoint: 97
   },
   quantifier: {
     type: 'Quantifier',
@@ -1509,7 +1597,9 @@ The node:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     }
   ]
 }
@@ -1538,7 +1628,9 @@ A node:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     {
       type: 'Assertion',
@@ -1576,7 +1668,9 @@ A node:
     {
       type: 'Char',
       value: 'x',
-      kind: 'simple'
+      symbol: 'x',
+      kind: 'simple',
+      codePoint: 120
     },
     {
       type: 'Assertion',
@@ -1601,7 +1695,9 @@ A node is the same:
     {
       type: 'Char',
       value: 'x',
-      kind: 'simple'
+      symbol: 'x',
+      kind: 'simple',
+      codePoint: 120
     },
     {
       type: 'Assertion',
@@ -1632,7 +1728,9 @@ A node:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     {
       type: 'Assertion',
@@ -1640,7 +1738,9 @@ A node:
       assertion: {
         type: 'Char',
         value: 'b',
-        kind: 'simple'
+        symbol: 'b',
+        kind: 'simple',
+        codePoint: 98
       }
     }
   ]
@@ -1664,7 +1764,9 @@ A node is similar, just `negative` flag is added:
     {
       type: 'Char',
       value: 'a',
-      kind: 'simple'
+      symbol: 'a',
+      kind: 'simple',
+      codePoint: 97
     },
     {
       type: 'Assertion',
@@ -1673,7 +1775,9 @@ A node is similar, just `negative` flag is added:
       assertion: {
         type: 'Char',
         value: 'b',
-        kind: 'simple'
+        symbol: 'b',
+        kind: 'simple',
+        codePoint: 98
       }
     }
   ]
@@ -1706,13 +1810,17 @@ A node:
       assertion: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 97
       }
     },
     {
       type: 'Char',
       value: 'b',
-      kind: 'simple'
+      symbol: 'b',
+      kind: 'simple',
+      codePoint: 98
     },
   ]
 }
@@ -1739,13 +1847,17 @@ A node:
       assertion: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 97
       }
     },
     {
       type: 'Char',
       value: 'b',
-      kind: 'simple'
+      symbol: 'b',
+      kind: 'simple',
+      codePoint: 98
     },
   ]
 }
diff --git a/package.json b/package.json
index f803af0..f390196 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "regexp-tree",
-  "version": "0.0.60",
+  "version": "0.0.59",
   "license": "MIT",
   "description": "Regular Expressions parser in JavaScript",
   "repository": "DmitrySoshnikov/regexp-tree",
diff --git a/src/optimizer/transforms/char-escape-unescape-transform.js b/src/optimizer/transforms/char-escape-unescape-transform.js
index 5a2df62..3198459 100644
--- a/src/optimizer/transforms/char-escape-unescape-transform.js
+++ b/src/optimizer/transforms/char-escape-unescape-transform.js
@@ -51,5 +51,5 @@ function preservesInCharClass(value) {
 // into `a{2}`. TODO: more sophisticated analisys.
 
 function preservesEscape(value) {
-  return /[*\[\]()+?^$.\/\\\{\}\|]/.test(value);
+  return /[*\[\]()+?^$.\/\\\{\}]/.test(value);
 }
\ No newline at end of file
diff --git a/src/parser/README.md b/src/parser/README.md
index 87065b8..019393e 100644
--- a/src/parser/README.md
+++ b/src/parser/README.md
@@ -23,12 +23,16 @@ Result:
           from: {
             type: 'Char',
             value: 'a',
-            kind: 'simple'
+            symbol: 'a',
+            kind: 'simple',
+            codePoint: 97
           },
           to: {
             type: 'Char',
             value: 'z',
-            kind: 'simple'
+            symbol: 'z',
+            kind: 'simple',
+            codePoint: 122
           }
         }
       ]
diff --git a/src/parser/generated/regexp-tree.js b/src/parser/generated/regexp-tree.js
index f777f99..3918614 100644
--- a/src/parser/generated/regexp-tree.js
+++ b/src/parser/generated/regexp-tree.js
@@ -271,7 +271,7 @@ const productions = [[-1,1,(_1,_1loc) => { __loc = yyloc(_1loc, _1loc);__ = _1 }
 [19,1,(_1,_1loc) => { __loc = yyloc(_1loc, _1loc); __ = [_1]  }],
 [19,2,(_1,_2,_1loc,_2loc) => { __loc = yyloc(_1loc, _2loc); __ = [_1].concat(_2)  }],
 [19,4,(_1,_2,_3,_4,_1loc,_2loc,_3loc,_4loc) => { __loc = yyloc(_1loc, _4loc);
-      checkClassRange(_1.value, _3.value);
+      checkClassRange(_1, _3);
 
       __ = [
         Node({
@@ -288,7 +288,7 @@ const productions = [[-1,1,(_1,_1loc) => { __loc = yyloc(_1loc, _1loc);__ = _1 }
 [20,1,(_1,_1loc) => { __loc = yyloc(_1loc, _1loc);__ = _1 }],
 [20,2,(_1,_2,_1loc,_2loc) => { __loc = yyloc(_1loc, _2loc); __ = [_1].concat(_2)  }],
 [20,4,(_1,_2,_3,_4,_1loc,_2loc,_3loc,_4loc) => { __loc = yyloc(_1loc, _4loc);
-      checkClassRange(_1.value, _3.value);
+      checkClassRange(_1, _3);
 
       __ = [
         Node({
@@ -850,8 +850,8 @@ function getRange(text) {
  * Checks class range
  */
 function checkClassRange(from, to) {
-  if (from > to) {
-    throw new SyntaxError(`Range ${from}-${to} out of order in character class`);
+  if (from.kind === 'control' || to.kind === 'control' || (!isNaN(from.codePoint) && !isNaN(to.codePoint) && from.codePoint > to.codePoint)) {
+    throw new SyntaxError(`Range ${from.value}-${to.value} out of order in character class`);
   }
 }
 
@@ -860,27 +860,70 @@ function checkClassRange(from, to) {
  */
 function Char(value, kind, loc) {
   let symbol;
+  let codePoint;
 
   switch (kind) {
     case 'decimal': {
-      const code = Number(value.slice(1));
-      symbol = String.fromCodePoint(code);
+      codePoint = Number(value.slice(1));
+      symbol = String.fromCodePoint(codePoint);
       break;
     }
     case 'oct': {
-      const code = parseInt(value.slice(1), 8);
-      symbol = String.fromCodePoint(code);
+      codePoint = parseInt(value.slice(1), 8);
+      symbol = String.fromCodePoint(codePoint);
       break;
     }
     case 'hex':
     case 'unicode': {
       const hex = value.slice(2).replace('{', '');
-      const code = parseInt(hex, 16);
-      if (code > 0x10ffff) {
+      codePoint = parseInt(hex, 16);
+      if (codePoint > 0x10ffff) {
         throw new SyntaxError(`Bad character escape sequence: ${value}`);
       }
 
-      symbol = String.fromCodePoint(code);
+      symbol = String.fromCodePoint(codePoint);
+      break;
+    }
+    case 'meta': {
+      switch (value) {
+        case '\\t':
+          symbol = '\t';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\n':
+          symbol = '\n';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\r':
+          symbol = '\r';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\v':
+          symbol = '\v';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\f':
+          symbol = '\f';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\b':
+          symbol = '\b';
+          codePoint = symbol.codePointAt(0);
+        case '\\0':
+          symbol = '\0';
+          codePoint = 0;
+        case '.':
+          symbol = '.';
+          codePoint = NaN;
+          break;
+        default:
+          codePoint = NaN;
+      }
+      break;
+    }
+    case 'simple': {
+      symbol = value;
+      codePoint = symbol.codePointAt(0);
       break;
     }
   }
@@ -890,6 +933,7 @@ function Char(value, kind, loc) {
     value,
     kind,
     symbol,
+    codePoint,
   }, loc);
 }
 
diff --git a/src/parser/regexp.bnf b/src/parser/regexp.bnf
index 476dbae..2381b61 100644
--- a/src/parser/regexp.bnf
+++ b/src/parser/regexp.bnf
@@ -178,8 +178,8 @@ function getRange(text) {
  * Checks class range
  */
 function checkClassRange(from, to) {
-  if (from > to) {
-    throw new SyntaxError(`Range ${from}-${to} out of order in character class`);
+  if (from.kind === 'control' || to.kind === 'control' || (!isNaN(from.codePoint) && !isNaN(to.codePoint) && from.codePoint > to.codePoint)) {
+    throw new SyntaxError(`Range ${from.value}-${to.value} out of order in character class`);
   }
 }
 
@@ -188,27 +188,70 @@ function checkClassRange(from, to) {
  */
 function Char(value, kind, loc) {
   let symbol;
+  let codePoint;
 
   switch (kind) {
     case 'decimal': {
-      const code = Number(value.slice(1));
-      symbol = String.fromCodePoint(code);
+      codePoint = Number(value.slice(1));
+      symbol = String.fromCodePoint(codePoint);
       break;
     }
     case 'oct': {
-      const code = parseInt(value.slice(1), 8);
-      symbol = String.fromCodePoint(code);
+      codePoint = parseInt(value.slice(1), 8);
+      symbol = String.fromCodePoint(codePoint);
       break;
     }
     case 'hex':
     case 'unicode': {
       const hex = value.slice(2).replace('{', '');
-      const code = parseInt(hex, 16);
-      if (code > 0x10ffff) {
+      codePoint = parseInt(hex, 16);
+      if (codePoint > 0x10ffff) {
         throw new SyntaxError(`Bad character escape sequence: ${value}`);
       }
 
-      symbol = String.fromCodePoint(code);
+      symbol = String.fromCodePoint(codePoint);
+      break;
+    }
+    case 'meta': {
+      switch (value) {
+        case '\\t':
+          symbol = '\t';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\n':
+          symbol = '\n';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\r':
+          symbol = '\r';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\v':
+          symbol = '\v';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\f':
+          symbol = '\f';
+          codePoint = symbol.codePointAt(0);
+          break;
+        case '\\b':
+          symbol = '\b';
+          codePoint = symbol.codePointAt(0);
+        case '\\0':
+          symbol = '\0';
+          codePoint = 0;
+        case '.':
+          symbol = '.';
+          codePoint = NaN;
+          break;
+        default:
+          codePoint = NaN;
+      }
+      break;
+    }
+    case 'simple': {
+      symbol = value;
+      codePoint = symbol.codePointAt(0);
       break;
     }
   }
@@ -218,6 +261,7 @@ function Char(value, kind, loc) {
     value,
     kind,
     symbol,
+    codePoint,
   }, loc);
 }
 
@@ -731,7 +775,7 @@ NonemptyClassRanges
 
   | ClassAtom DASH ClassAtom ClassRanges
     {
-      checkClassRange($1.value, $3.value);
+      checkClassRange($1, $3);
 
       $$ = [
         Node({
@@ -754,7 +798,7 @@ NonemptyClassRangesNoDash
 
   | ClassAtomNoDash DASH ClassAtom ClassRanges
     {
-      checkClassRange($1.value, $3.value);
+      checkClassRange($1, $3);
 
       $$ = [
         Node({


Test patch:
diff --git a/__tests__/integration-test.js b/__tests__/integration-test.js
index f1a9d35..a9ca3ad 100644
--- a/__tests__/integration-test.js
+++ b/__tests__/integration-test.js
@@ -47,7 +47,9 @@ describe('regexp-tree', () => {
       body: {
         type: 'Char',
         value: 'a',
+        symbol: 'a',
         kind: 'simple',
+        codePoint: 'a'.codePointAt(0)
       },
       flags: 'i',
     });
diff --git a/src/optimizer/transforms/__tests__/char-escape-unescape-transform-test.js b/src/optimizer/transforms/__tests__/char-escape-unescape-transform-test.js
index 90bc69d..17544d5 100644
--- a/src/optimizer/transforms/__tests__/char-escape-unescape-transform-test.js
+++ b/src/optimizer/transforms/__tests__/char-escape-unescape-transform-test.js
@@ -18,10 +18,10 @@ describe('\e -> e', () => {
   });
 
   it('preserve escape', () => {
-    const re = transform(/\*\^\$\(\)\[\{\}\|/, [
+    const re = transform(/\*\^\$\(\)\[\{\}/, [
       charUnescape,
     ]);
-    expect(re.toString()).toBe(/\*\^\$\(\)\[\{\}\|/.toString());
+    expect(re.toString()).toBe(/\*\^\$\(\)\[\{\}/.toString());
   });
 
   it('char class', () => {
diff --git a/src/parser/__tests__/parser-basic-test.js b/src/parser/__tests__/parser-basic-test.js
index d70f625..3ad5991 100644
--- a/src/parser/__tests__/parser-basic-test.js
+++ b/src/parser/__tests__/parser-basic-test.js
@@ -17,7 +17,9 @@ describe('basic', () => {
       body: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 'a'.codePointAt(0),
       },
       flags: '',
     });
@@ -32,14 +34,18 @@ describe('basic', () => {
           {
             type: 'Char',
             value: '(',
+            symbol: '(',
             kind: 'simple',
             escaped: true,
+            codePoint: '('.codePointAt(0)
           },
           {
             type: 'Char',
             value: ')',
+            symbol: ')',
             kind: 'simple',
             escaped: true,
+            codePoint: ')'.codePointAt(0)
           },
         ]
       },
@@ -55,12 +61,16 @@ describe('basic', () => {
         left: {
           type: 'Char',
           value: 'a',
+          symbol: 'a',
           kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
         right: {
           type: 'Char',
           value: 'b',
+          symbol: 'b',
           kind: 'simple',
+          codePoint: 'b'.codePointAt(0)
         }
       },
       flags: '',
@@ -76,12 +86,16 @@ describe('basic', () => {
           {
             type: 'Char',
             value: 'a',
+            symbol: 'a',
             kind: 'simple',
+            codePoint: 'a'.codePointAt(0)
           },
           {
             type: 'Char',
             value: 'b',
+            symbol: 'b',
             kind: 'simple',
+            codePoint: 'b'.codePointAt(0)
           }
         ],
       },
@@ -100,18 +114,23 @@ describe('basic', () => {
             from: {
               type: 'Char',
               value: 'a',
-              kind: 'simple'
+              symbol: 'a',
+              kind: 'simple',
+              codePoint: 'a'.codePointAt(0)
             },
             to: {
               type: 'Char',
               value: 'z',
-              kind: 'simple'
+              symbol: 'z',
+              kind: 'simple',
+              codePoint: 'z'.codePointAt(0)
             }
           },
           {
             type: 'Char',
             value: '\\d',
-            kind: 'meta'
+            kind: 'meta',
+            codePoint: NaN
           }
         ]
       },
@@ -149,147 +168,205 @@ describe('basic', () => {
           {
             type: 'Char',
             kind: 'simple',
-            value: '!'
+            value: '!',
+            symbol: '!',
+            codePoint: '!'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '"'
+            value: '"',
+            symbol: '"',
+            codePoint: '"'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '#'
+            value: '#',
+            symbol: '#',
+            codePoint: '#'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '$'
+            value: '$',
+            symbol: '$',
+            codePoint: '$'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '%'
+            value: '%',
+            symbol: '%',
+            codePoint: '%'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '&'
+            value: '&',
+            symbol: '&',
+            codePoint: '&'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '\''
+            value: '\'',
+            symbol: '\'',
+            codePoint: '\''.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '('
+            value: '(',
+            symbol: '(',
+            codePoint: '('.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: ')'
+            value: ')',
+            symbol: ')',
+            codePoint: ')'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '*'
+            value: '*',
+            symbol: '*',
+            codePoint: '*'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '+'
+            value: '+',
+            symbol: '+',
+            codePoint: '+'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: ','
+            value: ',',
+            symbol: ',',
+            codePoint: ','.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '.'
+            value: '.',
+            symbol: '.',
+            codePoint: '.'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '/'
+            value: '/',
+            symbol: '/',
+            codePoint: '/'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: ':'
+            value: ':',
+            symbol: ':',
+            codePoint: ':'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: ';'
+            value: ';',
+            symbol: ';',
+            codePoint: ';'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '<'
+            value: '<',
+            symbol: '<',
+            codePoint: '<'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '='
+            value: '=',
+            symbol: '=',
+            codePoint: '='.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '>'
+            value: '>',
+            symbol: '>',
+            codePoint: '>'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '?'
+            value: '?',
+            symbol: '?',
+            codePoint: '?'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '@'
+            value: '@',
+            symbol: '@',
+            codePoint: '@'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '^'
+            value: '^',
+            symbol: '^',
+            codePoint: '^'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '_'
+            value: '_',
+            symbol: '_',
+            codePoint: '_'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '`'
+            value: '`',
+            symbol: '`',
+            codePoint: '`'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '{'
+            value: '{',
+            symbol: '{',
+            codePoint: '{'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '|'
+            value: '|',
+            symbol: '|',
+            codePoint: '|'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '}'
+            value: '}',
+            symbol: '}',
+            codePoint: '}'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '~'
+            value: '~',
+            symbol: '~',
+            codePoint: '~'.codePointAt(0)
           },
           {
             type: 'Char',
             kind: 'simple',
-            value: '-'
+            value: '-',
+            symbol: '-',
+            codePoint: '-'.codePointAt(0)
           }
         ]
       },
@@ -315,7 +392,9 @@ describe('basic', () => {
             expression: {
               type: 'Char',
               value: 'a',
-              kind: 'simple'
+              symbol: 'a',
+              kind: 'simple',
+              codePoint: 'a'.codePointAt(0)
             }
           },
           {
@@ -331,7 +410,9 @@ describe('basic', () => {
             expression: {
               type: 'Char',
               value: 'b',
-              kind: 'simple'
+              symbol: 'b',
+              kind: 'simple',
+              codePoint: 'b'.codePointAt(0)
             }
           }
         ]
@@ -386,7 +467,9 @@ describe('basic', () => {
         expression: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -403,7 +486,9 @@ describe('basic', () => {
         expression: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -417,7 +502,9 @@ describe('basic', () => {
         expression: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -456,7 +543,9 @@ describe('basic', () => {
         assertion: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -471,7 +560,9 @@ describe('basic', () => {
         assertion: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -514,7 +605,9 @@ describe('basic', () => {
         assertion: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -529,7 +622,9 @@ describe('basic', () => {
         assertion: {
           type: 'Char',
           value: 'a',
-          kind: 'simple'
+          symbol: 'a',
+          kind: 'simple',
+          codePoint: 'a'.codePointAt(0)
         },
       },
       flags: '',
@@ -549,7 +644,9 @@ describe('basic', () => {
             expression: {
               type: 'Char',
               value: 'a',
-              kind: 'simple'
+              symbol: 'a',
+              kind: 'simple',
+              codePoint: 'a'.codePointAt(0)
             }
           },
           {
@@ -563,6 +660,7 @@ describe('basic', () => {
             value: '\\2',
             kind: 'decimal',
             symbol: String.fromCodePoint(2),
+            codePoint: 2
           }
         ]
       },
@@ -585,7 +683,9 @@ describe('basic', () => {
             expression: {
               type: 'Char',
               value: 'y',
-              kind: 'simple'
+              symbol: 'y',
+              kind: 'simple',
+              codePoint: 'y'.codePointAt(0)
             }
           },
           {
@@ -597,23 +697,31 @@ describe('basic', () => {
           {
             type: 'Char',
             value: 'k',
+            symbol: 'k',
             kind: 'simple',
             escaped: true,
+            codePoint: 'k'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '<',
+            symbol: '<',
             kind: 'simple',
+            codePoint: '<'.codePointAt(0)
           },
           {
             type: 'Char',
             value: 'z',
+            symbol: 'z',
             kind: 'simple',
+            codePoint: 'z'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '>',
+            symbol: '>',
             kind: 'simple',
+            codePoint: '>'.codePointAt(0)
           },
         ]
       },
@@ -631,45 +739,59 @@ describe('basic', () => {
           {
             type: 'Char',
             value: 'k',
+            symbol: 'k',
             kind: 'simple',
             escaped: true,
+            codePoint: 'k'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '<',
+            symbol: '<',
             kind: 'simple',
+            codePoint: '<'.codePointAt(0)
           },
           {
             type: 'Char',
             value: 'a',
+            symbol: 'a',
             kind: 'simple',
+            codePoint: 'a'.codePointAt(0)
           },
           {
             type: 'Char',
             value: 'b',
+            symbol: 'b',
             kind: 'simple',
+            codePoint: 'b'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '\\u003B',
             kind: 'unicode',
             symbol: String.fromCodePoint(0x003B),
+            codePoint: 0x003B
           },
           {
             type: 'Char',
             value: '\\u{003B}',
             kind: 'unicode',
             symbol: String.fromCodePoint(0x003B),
+            codePoint: 0x003B
           },
           {
             type: 'Char',
             value: 'c',
+            symbol: 'c',
             kind: 'simple',
+            codePoint: 'c'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '>',
+            symbol: '>',
             kind: 'simple',
+            codePoint: '>'.codePointAt(0)
           },
         ]
       },
@@ -690,35 +812,46 @@ describe('basic', () => {
             expression: {
               type: 'Char',
               value: 'a',
-              kind: 'simple'
+              symbol: 'a',
+              kind: 'simple',
+              codePoint: 'a'.codePointAt(0)
             }
           },
           {
             type: 'Char',
             value: '\\1',
             symbol: String.fromCodePoint(1),
-            kind: 'decimal'
+            kind: 'decimal',
+            codePoint: 1
           },
           {
             type: 'Char',
             value: 'k',
+            symbol: 'k',
             kind: 'simple',
             escaped: true,
+            codePoint: 'k'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '<',
+            symbol: '<',
             kind: 'simple',
+            codePoint: '<'.codePointAt(0)
           },
           {
             type: 'Char',
             value: 'z',
+            symbol: 'z',
             kind: 'simple',
+            codePoint: 'z'.codePointAt(0)
           },
           {
             type: 'Char',
             value: '>',
+            symbol: '>',
             kind: 'simple',
+            codePoint: '>'.codePointAt(0)
           },
         ]
       },
@@ -770,6 +903,7 @@ describe('basic', () => {
       value: '\\u003B',
       symbol: String.fromCodePoint(0x003b),
       kind: 'unicode',
+      codePoint: 0x003b
     });
 
     // Using `u` flag, 1 digit.
@@ -778,6 +912,7 @@ describe('basic', () => {
       value: '\\u{9}',
       symbol: String.fromCodePoint(9),
       kind: 'unicode',
+      codePoint: 9
     });
 
     // Using `u` flag, 6 digits, 10FFFF is max.
@@ -786,6 +921,7 @@ describe('basic', () => {
       value: '\\u{10FFFF}',
       symbol: String.fromCodePoint(0x10ffff),
       kind: 'unicode',
+      codePoint: 0x10ffff
     });
 
     // Using `u` flag, leading zeros.
@@ -794,6 +930,7 @@ describe('basic', () => {
       value: '\\u{000001D306}',
       symbol: String.fromCodePoint(0x000001d306),
       kind: 'unicode',
+      codePoint: 0x000001d306
     });
 
     // TODO: without `u` flag \u{1234} should be parsed NOT as
@@ -824,7 +961,9 @@ describe('basic', () => {
       body: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 'a'.codePointAt(0)
       },
       flags: 'gimuy',
     });
@@ -837,7 +976,9 @@ describe('basic', () => {
       body: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 'a'.codePointAt(0)
       },
       flags: 'gimuy',
     });
@@ -851,6 +992,7 @@ describe('basic', () => {
         value: '\\x33',
         kind: 'hex',
         symbol: String.fromCodePoint(0x33),
+        codePoint: 0x33
       },
       flags: '',
     });
@@ -864,6 +1006,7 @@ describe('basic', () => {
         value: '\\99',
         kind: 'decimal',
         symbol: String.fromCodePoint(99),
+        codePoint: 99
       },
       flags: '',
     });
@@ -876,7 +1019,9 @@ describe('basic', () => {
       body: {
         type: 'Char',
         value: 'a',
-        kind: 'simple'
+        symbol: 'a',
+        kind: 'simple',
+        codePoint: 'a'.codePointAt(0)
       },
       flags: 's',
     });
diff --git a/src/parser/__tests__/parser-extended-test.js b/src/parser/__tests__/parser-extended-test.js
index ceb3324..115aa74 100644
--- a/src/parser/__tests__/parser-extended-test.js
+++ b/src/parser/__tests__/parser-extended-test.js
@@ -34,7 +34,9 @@ describe('extended', () => {
               expression: {
                 type: 'Char',
                 kind: 'simple',
-                value: 'd'
+                value: 'd',
+                symbol: 'd',
+                codePoint: 'd'.codePointAt(0)
               },
               quantifier: {
                 type: 'Quantifier',
@@ -48,7 +50,9 @@ describe('extended', () => {
           {
             type: 'Char',
             kind: 'simple',
-            value: '-'
+            value: '-',
+            symbol: '-',
+            codePoint: '-'.codePointAt(0)
           },
           {
             type: 'Group',
@@ -60,7 +64,9 @@ describe('extended', () => {
               expression: {
                 type: 'Char',
                 kind: 'simple',
-                value: 'd'
+                value: 'd',
+                symbol: 'd',
+                codePoint: 'd'.codePointAt(0)
               },
               quantifier: {
                 type: 'Quantifier',
@@ -74,7 +80,9 @@ describe('extended', () => {
           {
             type: 'Char',
             kind: 'simple',
-            value: '-'
+            value: '-',
+            symbol: '-',
+            codePoint: '-'.codePointAt(0)
           },
           {
             type: 'Group',
@@ -86,7 +94,9 @@ describe('extended', () => {
               expression: {
                 kind: 'simple',
                 type: 'Char',
-                value: 'd'
+                value: 'd',
+                symbol: 'd',
+                codePoint: 'd'.codePointAt(0)
               },
               quantifier: {
                 type: 'Quantifier',
@@ -114,14 +124,18 @@ describe('extended', () => {
           {
             type: 'Char',
             value: ' ',
+            symbol: ' ',
             kind: 'simple',
             escaped: true,
+            codePoint: ' '.codePointAt(0)
           },
           {
             type: 'Char',
             value: '#',
+            symbol: '#',
             kind: 'simple',
             escaped: true,
+            codePoint: '#'.codePointAt(0)
           },
         ],
       },
@@ -140,12 +154,16 @@ describe('extended', () => {
           {
             type: 'Char',
             value: ' ',
+            symbol: ' ',
             kind: 'simple',
+            codePoint: ' '.codePointAt(0)
           },
           {
             type: 'Char',
             value: '#',
+            symbol: '#',
             kind: 'simple',
+            codePoint: '#'.codePointAt(0)
           },
         ],
       },
diff --git a/src/transform/__tests__/transform-basic-test.js b/src/transform/__tests__/transform-basic-test.js
index a46a708..cdeddfe 100644
--- a/src/transform/__tests__/transform-basic-test.js
+++ b/src/transform/__tests__/transform-basic-test.js
@@ -45,7 +45,9 @@ describe('transform-basic', () => {
         expression: {
           type: 'Char',
           value: 'a',
+          symbol: 'a',
           kind: 'simple',
+          codePoint: 'a'.codePointAt(0),
           loc: {
             source: 'a',
             start: {
diff --git a/src/traverse/__tests__/node-path-test.js b/src/traverse/__tests__/node-path-test.js
index 1213bfc..94bdbbd 100644
--- a/src/traverse/__tests__/node-path-test.js
+++ b/src/traverse/__tests__/node-path-test.js
@@ -54,7 +54,9 @@ describe('NodePath', () => {
           {
             type: 'Char',
             value: 'a',
+            symbol: 'a',
             kind: 'simple',
+            codePoint: 'a'.codePointAt(0)
           },
           // No 'b' char.
         ]
@@ -231,7 +233,9 @@ describe('NodePath', () => {
           const xNode = {
             type: 'Char',
             value: 'x',
+            symbol: 'x',
             kind: 'simple',
+            codePoint: 'x'.codePointAt(0)
           };
 
           const parentPath = path.parentPath;
@@ -245,7 +249,9 @@ describe('NodePath', () => {
           const yNode = {
             type: 'Char',
             value: 'y',
+            symbol: 'y',
             kind: 'simple',
+            codePoint: 'y'.codePointAt(0)
           };
 
           parentPath.insertChildAt(yNode, 3);
@@ -267,7 +273,9 @@ describe('NodePath', () => {
           const zNode = {
             type: 'Char',
             value: 'z',
+            symbol: 'z',
             kind: 'simple',
+            codePoint: 'z'.codePointAt(0)
           };
           path.parentPath.insertChildAt(zNode, 4);
 
@@ -293,7 +301,9 @@ describe('NodePath', () => {
     const cNode = {
       type: 'Char',
       value: 'c',
+      symbol: 'c',
       kind: 'simple',
+      codePoint: 'c'.codePointAt(0)
     };
 
     bCharPath.replace(cNode);
@@ -307,7 +317,9 @@ describe('NodePath', () => {
           {
             type: 'Char',
             value: 'a',
+            symbol: 'a',
             kind: 'simple',
+            codePoint: 'a'.codePointAt(0)
           },
           // 'b' replaced with 'c'
           cNode
@@ -387,7 +399,9 @@ describe('NodePath', () => {
     const cNode = {
       type: 'Char',
       value: 'c',
+      symbol: 'c',
       kind: 'simple',
+      codePoint: 'c'.codePointAt(0)
     };
 
     const groupNode = {
@@ -420,7 +434,9 @@ describe('NodePath', () => {
     const dNode = {
       type: 'Char',
       value: 'd',
+      symbol: 'd',
       kind: 'simple',
+      codePoint: 'd'.codePointAt(0)
     };
     alterPath.appendChild(dNode);
 
@@ -546,7 +562,9 @@ describe('NodePath', () => {
     const node =  {
       type: 'Char',
       value: 'a',
+      symbol: 'a',
       kind: 'simple',
+      codePoint: 'a'.codePointAt(0)
     };
 
     const path = NodePath.getForNode(
diff --git a/src/traverse/__tests__/traverse-basic-test.js b/src/traverse/__tests__/traverse-basic-test.js
index 3fa015e..5c6b3c0 100644
--- a/src/traverse/__tests__/traverse-basic-test.js
+++ b/src/traverse/__tests__/traverse-basic-test.js
@@ -301,7 +301,9 @@ describe('traverse-basic', () => {
       body: {
         type: 'Char',
         value: '.',
+        symbol: '.',
         kind: 'meta',
+        codePoint: NaN
       },
       flags: '',
     });