Repository: alecthomas/chroma
Pull request #362 (2020-05-14 03:01:00)

feat: support HTTP/2 in HTTP lexer
This adds support for HTTP/2 lexing in the HTTP lexer, which previously was hard-coded to only support HTTP/1 or HTTP/1.1.

Before the change you get uncolored output like this after fetching e.g. `https://google.com/foo`:

```
HTTP/2.0 404 Not Found
Alt-Svc: h3-27=":443"; ma=2592000,h3-25=":443"; ma=2592000,h3-T050=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q049=":443"; ma=2592000,h3-Q048=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Content-Length: 1564
Content-Type: text/html; charset=UTF-8
Date: Wed, 13 May 2020 23:21:08 GMT
Referrer-Policy: no-referrer

<!DOCTYPE html>
<html lang=en>
  <meta charset=utf-8>
  <meta name=viewport content="initial-scale=1, minimum-scale=1, width=device-width">
  <title>Error 404 (Not Found)!!1</title>
  <style>
    *{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px}* > body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}
  </style>
  <a href=//www.google.com/><span id=logo aria-label=Google></span></a>
  <p><b>404.</b> <ins>That’s an error.</ins>
  <p>The requested URL <code>/foo</code> was not found on this server.  <ins>That’s all we know.</ins>
```

Afterward you get:

<img width="911" alt="Screen Shot 2020-05-13 at 7 59 08 PM" src="https://user-images.githubusercontent.com/106826/81887558-3a87ef00-9554-11ea-9eb2-5919e1de314a.png">

These responses were serialized using the standard library `httputil.DumpResponse`, which represents an HTTP/2 request/response as text like you would see with HTTP/1.

Included with this PR is a small test to ensure we don't accidentally break in the future.

alecthomas: Nice!

Patch:
diff --git a/lexers/h/http.go b/lexers/h/http.go
index 0a1264c..135ec73 100644
--- a/lexers/h/http.go
+++ b/lexers/h/http.go
@@ -19,8 +19,8 @@ var HTTP = internal.Register(httpBodyContentTypeLexer(MustNewLexer(
 	},
 	Rules{
 		"root": {
-			{`(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH|CONNECT)( +)([^ ]+)( +)(HTTP)(/)(1\.[01])(\r?\n|\Z)`, ByGroups(NameFunction, Text, NameNamespace, Text, KeywordReserved, Operator, LiteralNumber, Text), Push("headers")},
-			{`(HTTP)(/)(1\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|\Z)`, ByGroups(KeywordReserved, Operator, LiteralNumber, Text, LiteralNumber, Text, NameException, Text), Push("headers")},
+			{`(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|PATCH|CONNECT)( +)([^ ]+)( +)(HTTP)(/)([12]\.[01])(\r?\n|\Z)`, ByGroups(NameFunction, Text, NameNamespace, Text, KeywordReserved, Operator, LiteralNumber, Text), Push("headers")},
+			{`(HTTP)(/)([12]\.[01])( +)(\d{3})( +)([^\r\n]+)(\r?\n|\Z)`, ByGroups(KeywordReserved, Operator, LiteralNumber, Text, LiteralNumber, Text, NameException, Text), Push("headers")},
 		},
 		"headers": {
 			{`([^\s:]+)( *)(:)( *)([^\r\n]+)(\r?\n|\Z)`, EmitterFunc(httpHeaderBlock), nil},


Test patch:
diff --git a/lexers/lexers_test.go b/lexers/lexers_test.go
index 34496dd..a54db51 100644
--- a/lexers/lexers_test.go
+++ b/lexers/lexers_test.go
@@ -55,7 +55,8 @@ func TestLexers(t *testing.T) {
 			continue
 		}
 
-		lexer := lexers.Get(strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())))
+		base := strings.Split(strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())), "-")[0]
+		lexer := lexers.Get(base)
 		assert.NotNil(t, lexer)
 
 		filename := filepath.Join("testdata", file.Name())
diff --git a/lexers/testdata/http-2.actual b/lexers/testdata/http-2.actual
new file mode 100644
index 0000000..0627400
--- /dev/null
+++ b/lexers/testdata/http-2.actual
@@ -0,0 +1,5 @@
+GET /foo HTTP/2.0
+Content-Type: application/json
+User-Agent: foo
+
+{"hello": "world"}
diff --git a/lexers/testdata/http-2.expected b/lexers/testdata/http-2.expected
new file mode 100644
index 0000000..c1a1c90
--- /dev/null
+++ b/lexers/testdata/http-2.expected
@@ -0,0 +1,27 @@
+[
+  {"type":"NameFunction","value":"GET"},
+  {"type":"Text","value":" "},
+  {"type":"NameNamespace","value":"/foo"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordReserved","value":"HTTP"},
+  {"type":"Operator","value":"/"},
+  {"type":"LiteralNumber","value":"2.0"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"Content-Type"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"Literal","value":"application/json"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"User-Agent"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"Literal","value":"foo"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"NameTag","value":"\"hello\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"world\""},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]