Repository: alecthomas/chroma
Pull request #933 (2024-02-23 18:33:27)

Add Lexer for NDISASM
The Netwide Disassembler outputs text that is assembly prefixed with memory offsets and opcode bytes.


Patch:
diff --git a/bin/.esbuild-0.20.0.pkg b/bin/.esbuild-0.20.1.pkg
similarity index 100%
rename from bin/.esbuild-0.20.0.pkg
rename to bin/.esbuild-0.20.1.pkg
diff --git a/bin/esbuild b/bin/esbuild
index 9130c8a..a083bbf 120000
--- a/bin/esbuild
+++ b/bin/esbuild
@@ -1,1 +1,1 @@
-.esbuild-0.20.0.pkg
\ No newline at end of file
+.esbuild-0.20.1.pkg
\ No newline at end of file
diff --git a/formatters/html/html.go b/formatters/html/html.go
index 6b21a29..3d97bef 100644
--- a/formatters/html/html.go
+++ b/formatters/html/html.go
@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/alecthomas/chroma/v2"
 )
@@ -133,6 +134,7 @@ func New(options ...Option) *Formatter {
 		baseLineNumber: 1,
 		preWrapper:     defaultPreWrapper,
 	}
+	f.styleCache = newStyleCache(f)
 	for _, option := range options {
 		option(f)
 	}
@@ -189,6 +191,7 @@ var (
 
 // Formatter that generates HTML.
 type Formatter struct {
+	styleCache            *styleCache
 	standalone            bool
 	prefix                string
 	Classes               bool // Exported field to detect when classes are being used
@@ -221,12 +224,7 @@ func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Ite
 //
 // OTOH we need to be super careful about correct escaping...
 func (f *Formatter) writeHTML(w io.Writer, style *chroma.Style, tokens []chroma.Token) (err error) { // nolint: gocyclo
-	css := f.styleToCSS(style)
-	if !f.Classes {
-		for t, style := range css {
-			css[t] = compressStyle(style)
-		}
-	}
+	css := f.styleCache.get(style)
 	if f.standalone {
 		fmt.Fprint(w, "<html>\n")
 		if f.Classes {
@@ -420,7 +418,7 @@ func (f *Formatter) tabWidthStyle() string {
 
 // WriteCSS writes CSS style definitions (without any surrounding HTML).
 func (f *Formatter) WriteCSS(w io.Writer, style *chroma.Style) error {
-	css := f.styleToCSS(style)
+	css := f.styleCache.get(style)
 	// Special-case background as it is mapped to the outer ".chroma" class.
 	if _, err := fmt.Fprintf(w, "/* %s */ .%sbg { %s }\n", chroma.Background, f.prefix, css[chroma.Background]); err != nil {
 		return err
@@ -563,3 +561,60 @@ func compressStyle(s string) string {
 	}
 	return strings.Join(out, ";")
 }
+
+const styleCacheLimit = 16
+
+type styleCacheEntry struct {
+	style *chroma.Style
+	cache map[chroma.TokenType]string
+}
+
+type styleCache struct {
+	mu sync.Mutex
+	// LRU cache of compiled (and possibly compressed) styles. This is a slice
+	// because the cache size is small, and a slice is sufficiently fast for
+	// small N.
+	cache []styleCacheEntry
+	f     *Formatter
+}
+
+func newStyleCache(f *Formatter) *styleCache {
+	return &styleCache{f: f}
+}
+
+func (l *styleCache) get(style *chroma.Style) map[chroma.TokenType]string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	// Look for an existing entry.
+	for i := len(l.cache) - 1; i >= 0; i-- {
+		entry := l.cache[i]
+		if entry.style == style {
+			// Top of the cache, no need to adjust the order.
+			if i == len(l.cache)-1 {
+				return entry.cache
+			}
+			// Move this entry to the end of the LRU
+			copy(l.cache[i:], l.cache[i+1:])
+			l.cache[len(l.cache)-1] = entry
+			return entry.cache
+		}
+	}
+
+	// No entry, create one.
+	cached := l.f.styleToCSS(style)
+	if !l.f.Classes {
+		for t, style := range cached {
+			cached[t] = compressStyle(style)
+		}
+	}
+	for t, style := range cached {
+		cached[t] = compressStyle(style)
+	}
+	// Evict the oldest entry.
+	if len(l.cache) >= styleCacheLimit {
+		l.cache = l.cache[0:copy(l.cache, l.cache[1:])]
+	}
+	l.cache = append(l.cache, styleCacheEntry{style: style, cache: cached})
+	return cached
+}
diff --git a/go.mod b/go.mod
index 66a16ee..5bc94e5 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,9 @@ module github.com/alecthomas/chroma/v2
 go 1.19
 
 require (
-	github.com/alecthomas/assert/v2 v2.5.0
-	github.com/alecthomas/repr v0.3.0
-	github.com/dlclark/regexp2 v1.10.0
+	github.com/alecthomas/assert/v2 v2.6.0
+	github.com/alecthomas/repr v0.4.0
+	github.com/dlclark/regexp2 v1.11.0
 )
 
 require github.com/hexops/gotextdiff v1.0.3 // indirect
diff --git a/go.sum b/go.sum
index 9ff2563..131b1fc 100644
--- a/go.sum
+++ b/go.sum
@@ -4,11 +4,17 @@ github.com/alecthomas/assert/v2 v2.4.1 h1:mwPZod/d35nlaCppr6sFP0rbCL05WH9fIo7lvs
 github.com/alecthomas/assert/v2 v2.4.1/go.mod h1:fw5suVxB+wfYJ3291t0hRTqtGzFYdSwstnRQdaQx2DM=
 github.com/alecthomas/assert/v2 v2.5.0 h1:OJKYg53BQx06/bMRBSPDCO49CbCDNiUQXwdoNrt6x5w=
 github.com/alecthomas/assert/v2 v2.5.0/go.mod h1:fw5suVxB+wfYJ3291t0hRTqtGzFYdSwstnRQdaQx2DM=
+github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
+github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8=
 github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
+github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
 github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
diff --git a/lexers/embedded/cue.xml b/lexers/embedded/cue.xml
index 16d7387..d6c3ea7 100644
--- a/lexers/embedded/cue.xml
+++ b/lexers/embedded/cue.xml
@@ -49,7 +49,7 @@
       <rule pattern="(true|false|null|_)\b">
         <token type="KeywordConstant"/>
       </rule>
-      <rule pattern="[_a-zA-Z]\w*">
+      <rule pattern="#?[_a-zA-Z$]\w*">
         <token type="Name"/>
       </rule>
     </state>
diff --git a/lexers/embedded/ndisasm.xml b/lexers/embedded/ndisasm.xml
new file mode 100644
index 0000000..74d443b
--- /dev/null
+++ b/lexers/embedded/ndisasm.xml
@@ -0,0 +1,123 @@
+<lexer>
+  <config>
+    <name>NDISASM</name>
+    <alias>ndisasm</alias>
+    <mime_type>text/x-disasm</mime_type>
+    <case_insensitive>true</case_insensitive>
+    <priority>0.5</priority> <!-- Lower than NASM -->
+  </config>
+  <rules>
+    <state name="root">
+        <rule pattern="^[0-9A-Za-z]+">
+            <token type="CommentSpecial"/>
+            <push state="offset"/>
+        </rule>
+    </state>
+    <state name="offset">
+        <rule pattern="[0-9A-Za-z]+">
+            <token type="CommentSpecial"/>
+            <push state="assembly"/>
+        </rule>
+        <rule>
+            <include state="whitespace"/>
+        </rule>
+    </state>
+    <state name="punctuation">
+      <rule pattern="[,():\[\]]+">
+        <token type="Punctuation"/>
+      </rule>
+      <rule pattern="[&amp;|^&lt;&gt;+*/%~-]+">
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="[$]+">
+        <token type="KeywordConstant"/>
+      </rule>
+      <rule pattern="seg|wrt|strict">
+        <token type="OperatorWord"/>
+      </rule>
+      <rule pattern="byte|[dq]?word">
+        <token type="KeywordType"/>
+      </rule>
+    </state>
+    <state name="assembly">
+      <rule>
+        <include state="whitespace"/>
+      </rule>
+      <rule pattern="[a-z$._?][\w$.?#@~]*:">
+        <token type="NameLabel"/>
+      </rule>
+      <rule pattern="([a-z$._?][\w$.?#@~]*)(\s+)(equ)">
+        <bygroups>
+          <token type="NameConstant"/>
+          <token type="KeywordDeclaration"/>
+          <token type="KeywordDeclaration"/>
+        </bygroups>
+        <push state="instruction-args"/>
+      </rule>
+      <rule pattern="BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|EXPORT|LIBRARY|MODULE">
+        <token type="Keyword"/>
+        <push state="instruction-args"/>
+      </rule>
+      <rule pattern="(?:res|d)[bwdqt]|times">
+        <token type="KeywordDeclaration"/>
+        <push state="instruction-args"/>
+      </rule>
+      <rule pattern="[a-z$._?][\w$.?#@~]*">
+        <token type="NameFunction"/>
+        <push state="instruction-args"/>
+      </rule>
+      <rule pattern="[\r\n]+">
+        <token type="Text"/>
+        <pop depth="2"/>
+      </rule>
+    </state>
+    <state name="instruction-args">
+      <rule pattern="&#34;(\\&#34;|[^&#34;\n])*&#34;|&#39;(\\&#39;|[^&#39;\n])*&#39;|`(\\`|[^`\n])*`">
+        <token type="LiteralString"/>
+      </rule>
+      <rule pattern="(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)">
+        <token type="LiteralNumberHex"/>
+      </rule>
+      <rule pattern="[0-7]+q">
+        <token type="LiteralNumberOct"/>
+      </rule>
+      <rule pattern="[01]+b">
+        <token type="LiteralNumberBin"/>
+      </rule>
+      <rule pattern="[0-9]+\.e?[0-9]+">
+        <token type="LiteralNumberFloat"/>
+      </rule>
+      <rule pattern="[0-9]+">
+        <token type="LiteralNumberInteger"/>
+      </rule>
+      <rule>
+        <include state="punctuation"/>
+      </rule>
+      <rule pattern="r[0-9][0-5]?[bwd]|[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]">
+        <token type="NameBuiltin"/>
+      </rule>
+      <rule pattern="[a-z$._?][\w$.?#@~]*">
+        <token type="NameVariable"/>
+      </rule>
+      <rule pattern="[\r\n]+">
+        <token type="Text"/>
+        <pop depth="3"/>
+      </rule>
+      <rule>
+        <include state="whitespace"/>
+      </rule>
+    </state>
+    <state name="whitespace">
+      <rule pattern="\n">
+        <token type="Text"/>
+        <pop depth="2"/>
+      </rule>
+      <rule pattern="[ \t]+">
+        <token type="Text"/>
+      </rule>
+      <rule pattern=";.*">
+        <token type="CommentSingle"/>
+      </rule>
+    </state>
+  </rules>
+</lexer>
\ No newline at end of file


Test patch:
diff --git a/formatters/html/html_test.go b/formatters/html/html_test.go
index 14c6468..2524524 100644
--- a/formatters/html/html_test.go
+++ b/formatters/html/html_test.go
@@ -222,7 +222,7 @@ func TestTableLinkeableLineNumbers(t *testing.T) {
 
 	assert.Contains(t, buf.String(), `id="line1"><a class="lnlinks" href="#line1">1</a>`)
 	assert.Contains(t, buf.String(), `id="line5"><a class="lnlinks" href="#line5">5</a>`)
-	assert.Contains(t, buf.String(), `/* LineLink */ .chroma .lnlinks { outline: none; text-decoration: none; color: inherit }`, buf.String())
+	assert.Contains(t, buf.String(), `/* LineLink */ .chroma .lnlinks { outline:none;text-decoration:none;color:inherit }`, buf.String())
 }
 
 func TestTableLineNumberSpacing(t *testing.T) {
@@ -351,8 +351,7 @@ func TestReconfigureOptions(t *testing.T) {
 }
 
 func TestWriteCssWithAllClasses(t *testing.T) {
-	formatter := New()
-	formatter.allClasses = true
+	formatter := New(WithAllClasses(true))
 
 	var buf bytes.Buffer
 	err := formatter.WriteCSS(&buf, styles.Fallback)
@@ -360,3 +359,17 @@ func TestWriteCssWithAllClasses(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotContains(t, buf.String(), ".chroma . {", "Generated css doesn't contain invalid css")
 }
+
+func TestStyleCache(t *testing.T) {
+	f := New()
+
+	assert.True(t, len(styles.Registry) > styleCacheLimit)
+
+	for _, style := range styles.Registry {
+		var buf bytes.Buffer
+		err := f.WriteCSS(&buf, style)
+		assert.NoError(t, err)
+	}
+
+	assert.Equal(t, styleCacheLimit, len(f.styleCache.cache))
+}
diff --git a/lexers/testdata/cue.actual b/lexers/testdata/cue.actual
index f6a35e7..830682b 100644
--- a/lexers/testdata/cue.actual
+++ b/lexers/testdata/cue.actual
@@ -29,3 +29,8 @@ A: close({
     field2: string
 })
 
+// Verify that various field types work
+_hidden: int
+regular: int
+$id: int
+#definition: int
diff --git a/lexers/testdata/cue.expected b/lexers/testdata/cue.expected
index 23204ac..c7c0dce 100644
--- a/lexers/testdata/cue.expected
+++ b/lexers/testdata/cue.expected
@@ -104,5 +104,27 @@
   {"type":"KeywordType","value":"string"},
   {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":"})"},
-  {"type":"Text","value":"\n\n"}
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// Verify that various field types work"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"_hidden"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"regular"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"$id"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"#definition"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":"\n"}
 ]
diff --git a/lexers/testdata/ndisasm.actual b/lexers/testdata/ndisasm.actual
new file mode 100644
index 0000000..911bc0d
--- /dev/null
+++ b/lexers/testdata/ndisasm.actual
@@ -0,0 +1,12 @@
+00000000  B013              mov al,0x13
+00000002  CD10              int 0x10
+00000004  B9027D            mov cx,0x7d02
+00000007  51                push cx
+00000008  6800A0            push word 0xa000
+0000000B  07                pop es
+0000000C  B00F              mov al,0xf
+0000000E  F3AA              rep stosb
+00000010  59                pop cx
+00000011  B004              mov al,0x4
+00000013  F3AA              rep stosb
+00000015  CD16              int 0x16
diff --git a/lexers/testdata/ndisasm.expected b/lexers/testdata/ndisasm.expected
new file mode 100644
index 0000000..13127a8
--- /dev/null
+++ b/lexers/testdata/ndisasm.expected
@@ -0,0 +1,108 @@
+[
+  {"type":"CommentSpecial","value":"00000000"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"B013"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"mov"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"al"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberHex","value":"0x13"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000002"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"CD10"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x10"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000004"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"B9027D"},
+  {"type":"Text","value":"            "},
+  {"type":"NameFunction","value":"mov"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"cx"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberHex","value":"0x7d02"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000007"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"51"},
+  {"type":"Text","value":"                "},
+  {"type":"NameFunction","value":"push"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"cx"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000008"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"6800A0"},
+  {"type":"Text","value":"            "},
+  {"type":"NameFunction","value":"push"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"word"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0xa000"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"0000000B"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"07"},
+  {"type":"Text","value":"                "},
+  {"type":"NameFunction","value":"pop"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"es"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"0000000C"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"B00F"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"mov"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"al"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberHex","value":"0xf"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"0000000E"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"F3AA"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"rep"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"stosb"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000010"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"59"},
+  {"type":"Text","value":"                "},
+  {"type":"NameFunction","value":"pop"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"cx"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000011"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"B004"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"mov"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"al"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberHex","value":"0x4"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000013"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"F3AA"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"rep"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"stosb"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"00000015"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSpecial","value":"CD16"},
+  {"type":"Text","value":"              "},
+  {"type":"NameFunction","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberHex","value":"0x16"},
+  {"type":"Text","value":"\n"}
+]