Repository: mozilla/nunjucks (js)

Issue:
Raw is broken, gets confused with "fake" variable declarations
Raw blocks break when there's something that looks like a variable declaration but isn't, for example `{{ x }` (it opens like a variable but doesn't end like one).

### Sample

```html
{% raw %}
<code>
{{ i_am_a_dumb_variable }}

{{ i_am_an_imposter_i_kill_nunjucks }
</code>
{% endraw %}
```

### Stacktrace

```
/Users/aaron/git/nunjucks/src/environment.js:425
            else throw err;
                       ^
Template render error: (/Users/aaron/git/nunjucks/tests/raw.html)
  expected endraw, got end of file
    at Object.exports.prettifyError (/Users/aaron/git/nunjucks/src/lib.js:34:15)
    at Obj.extend.render (/Users/aaron/git/nunjucks/src/environment.js:423:27)
    at /Users/aaron/git/nunjucks/src/environment.js:270:35
    at createTemplate (/Users/aaron/git/nunjucks/src/environment.js:194:25)
    at handle (/Users/aaron/git/nunjucks/src/environment.js:206:25)
    at /Users/aaron/git/nunjucks/src/environment.js:223:21
    at next (/Users/aaron/git/nunjucks/src/lib.js:198:13)
    at Object.exports.asyncIter (/Users/aaron/git/nunjucks/src/lib.js:205:5)
    at Obj.extend.getTemplate (/Users/aaron/git/nunjucks/src/environment.js:202:17)
    at Obj.extend.render (/Users/aaron/git/nunjucks/src/environment.js:262:14)
```

This obviously isn't the expected behavior. I'm looking into the parser's raw parsing, but maybe someone more familiar with the codebase knows precisely where it messes up.

Pull request:
Fix/raw
This PR fixes #447 

### Summary

* [x] Add unit tests for broken blocks in raw
* [x] Implement a fix in `src/parser.js`

### Bonus
* [x] Add tests for raw blocks in raw blocks
* [x] Support raw blocks in raw blocks

Ideally I think this should be fixed in `src/lexer.js` at the Tokenizer level. (Maybe I'm wrong on that, someone with a better knowledge of the codebase should know).

Patch:
diff --git a/src/lexer.js b/src/lexer.js
index 0ef3f2b..dc70b3d 100644
--- a/src/lexer.js
+++ b/src/lexer.js
@@ -385,6 +385,18 @@ Tokenizer.prototype._extractMatching = function (breakOnMatch, charString) {
     return '';
 };
 
+Tokenizer.prototype._extractRegex = function(regex) {
+    var matches = this.currentStr().match(regex);
+    if(!matches) {
+        return null;
+    }
+
+    // Move forward whatever was matched
+    this.forwardN(matches[0].length);
+
+    return matches;
+};
+
 Tokenizer.prototype.is_finished = function() {
     return this.index >= this.len;
 };
@@ -407,6 +419,12 @@ Tokenizer.prototype.forward = function() {
     }
 };
 
+Tokenizer.prototype.backN = function(n) {
+    for(var i=0; i<n; i++) {
+        this.back();
+    }
+};
+
 Tokenizer.prototype.back = function() {
     this.index--;
 
@@ -426,6 +444,7 @@ Tokenizer.prototype.back = function() {
     }
 };
 
+// current returns current character
 Tokenizer.prototype.current = function() {
     if(!this.is_finished()) {
         return this.str.charAt(this.index);
@@ -433,6 +452,14 @@ Tokenizer.prototype.current = function() {
     return '';
 };
 
+// currentStr returns what's left of the unparsed string
+Tokenizer.prototype.currentStr = function() {
+    if(!this.is_finished()) {
+        return this.str.substr(this.index);
+    }
+    return '';
+};
+
 Tokenizer.prototype.previous = function() {
     return this.str.charAt(this.index-1);
 };
diff --git a/src/parser.js b/src/parser.js
index f0af7da..0b6b76c 100644
--- a/src/parser.js
+++ b/src/parser.js
@@ -124,6 +124,8 @@ var Parser = Object.extend({
         else {
             this.fail('expected block end in ' + name + ' statement');
         }
+
+        return tok;
     },
 
     advanceAfterVariableEnd: function() {
@@ -547,60 +549,46 @@ var Parser = Object.extend({
     },
 
     parseRaw: function() {
-        this.advanceAfterBlockEnd();
+        var blockRegex = /([\s\S]*){%\s*(.*?)\s*(?=%})%}/;
+        var rawLevel = 1;
         var str = '';
-        var begun = this.peekToken();
-
-        while(1) {
-            // Passing true gives us all the whitespace tokens as
-            // well, which are usually ignored.
-            var tok = this.nextToken(true);
+        var matches = null;
 
-            if(!tok) {
-                this.fail('expected endraw, got end of file');
-            }
+        // Skip opening raw token
+        // Keep this token to track line and column numbers
+        var begun = this.advanceAfterBlockEnd();
 
-            if(tok.type === lexer.TOKEN_BLOCK_START) {
-                // We need to look for the `endraw` block statement,
-                // which involves a lookahead so carefully keep track
-                // of whitespace
-                var ws = null;
-                var name = this.nextToken(true);
+        // Exit when there's nothing to match
+        // or when we've found the matching "endraw" block
+        while((matches = this.tokens._extractRegex(blockRegex)) && rawLevel > 0) {
+            var all = matches[0];
+            var pre = matches[1];
+            var blockName = matches[2];
 
-                if(name.type === lexer.TOKEN_WHITESPACE) {
-                    ws = name;
-                    name = this.nextToken();
-                }
+            // Adjust rawlevel
+            if(blockName === 'raw') {
 
-                if(name.type === lexer.TOKEN_SYMBOL &&
-                   name.value === 'endraw') {
-                    this.advanceAfterBlockEnd(name.value);
-                    break;
-                }
-                else {
-                    str += tok.value;
-                    if(ws) {
-                        str += ws.value;
-                    }
-                    str += name.value;
-                }
+                rawLevel += 1;
+            } else if(blockName === 'endraw') {
+                rawLevel -= 1;
             }
-            else if(tok.type === lexer.TOKEN_STRING) {
-                str += '"' + tok.value + '"';
-            }
-            else {
-                str += tok.value;
+
+            // Add to str
+            if(rawLevel === 0) {
+                // We want to exclude the last "endraw"
+                str += pre;
+                // Move tokenizer to beginning of endraw block
+                this.tokens.backN(all.length - pre.length);
+            } else {
+                str += all;
             }
         }
 
-
-        var output = new nodes.Output(
+        return new nodes.Output(
             begun.lineno,
             begun.colno,
             [new nodes.TemplateData(begun.lineno, begun.colno, str)]
         );
-
-        return output;
     },
 
     parsePostfix: function(node) {


Test patch:
diff --git a/tests/parser.js b/tests/parser.js
index eb047e6..0962cf1 100644
--- a/tests/parser.js
+++ b/tests/parser.js
@@ -412,6 +412,42 @@
                     [nodes.TemplateData, 'hello {{ {% %} }}']]]);
         });
 
+        it('should parse raw with broken variables', function() {
+            isAST(parser.parse('{% raw %}{{ x }{% endraw %}'),
+                  [nodes.Root,
+                   [nodes.Output,
+                    [nodes.TemplateData, '{{ x }']]]);
+        });
+
+        it('should parse raw with broken blocks', function() {
+            isAST(parser.parse('{% raw %}{% if i_am_stupid }Still do your job well{% endraw %}'),
+                  [nodes.Root,
+                   [nodes.Output,
+                    [nodes.TemplateData, '{% if i_am_stupid }Still do your job well']]]);
+        });
+
+        it('should parse raw with pure text', function() {
+            isAST(parser.parse('{% raw %}abc{% endraw %}'),
+                  [nodes.Root,
+                   [nodes.Output,
+                    [nodes.TemplateData, 'abc']]]);
+        });
+
+
+        it('should parse raw with raw blocks', function() {
+            isAST(parser.parse('{% raw %}{% raw %}{{ x }{% endraw %}{% endraw %}'),
+                  [nodes.Root,
+                   [nodes.Output,
+                    [nodes.TemplateData, '{% raw %}{{ x }{% endraw %}']]]);
+        });
+
+        it('should parse raw with comment blocks', function() {
+          isAST(parser.parse('{% raw %}{# test {% endraw %}'),
+                  [nodes.Root,
+                   [nodes.Output,
+                    [nodes.TemplateData, '{# test ']]]);
+        });
+
         it('should parse keyword and non-keyword arguments', function() {
             isAST(parser.parse('{{ foo("bar", falalalala, baz="foobar") }}'),
                   [nodes.Root,