Repository: gabriel-vasile/mimetype (go)

Issue:
Html file with utf-8 byte order mark is misclassified as text/plain
**Attach the file for which the detection is inaccurate**
https://heyworth.tech/iqsYhUZZr8qH5jMr/html.utf8bom.html

**Expected MIME type**
`text/html; charset=utf-8`

**Returned MIME type**
`text/plain; charset=utf-8`

**Version of the library you are using**
`v1.4.0` and latest fdfefdd3015f86bc69bc442644b8241c72a30120 head

**Output of `go version`**
`go version go1.18 linux/amd64`



Pull request:
add html utf8 bom signature
closes #267

Patch:
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index a42be1f..5f3698c 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -6,13 +6,21 @@ on:
   pull_request:
     branches: [master]
 
+permissions:
+  contents: read
+
+env:
+  CGO_ENABLED: 0
+
 jobs:
   CodeQL-Build:
     runs-on: ubuntu-latest
+    permissions:
+      security-events: write
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v2.4.0
+        uses: actions/checkout@v3.0.1
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2.1.6
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index fe5dc51..b6fd740 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -1,12 +1,18 @@
+name: Run Tests
 on:
   push:
     branches:
       - master
   pull_request:
 
-name: run tests
+permissions:
+  contents: read
+
 jobs:
   lint:
+    strategy:
+      matrix:
+        go-version: ["1.18.x"]
     runs-on: ubuntu-latest
     steps:
     - name: Install Go
@@ -14,17 +20,18 @@ jobs:
       with:
         go-version: 1.16
     - name: Checkout code
-      uses: actions/checkout@v2.4.0
+      uses: actions/checkout@v3.0.1
     - name: Run linters
-      uses: golangci/golangci-lint-action@v2.5.2
+      uses: golangci/golangci-lint-action@v3.1.0
       with:
         version: "v1.37.1"
+        go-version: ${{ matrix.go-version }}
 
   test:
     strategy:
       matrix:
-        go-version: ["1.12.0", "^1.17.6"]
-        platform: [ubuntu-latest, macos-latest, windows-latest]
+        go-version: ["1.12.0", "1.18.x"]
+        platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
     - name: Install Go
@@ -33,7 +40,7 @@ jobs:
       with:
         go-version: ${{ matrix.go-version }}
     - name: Checkout code
-      uses: actions/checkout@v2.4.0
+      uses: actions/checkout@v3.0.1
     - name: Run tests
       run: go test -race ./...
 
@@ -46,7 +53,7 @@ jobs:
       with:
         go-version: 1.16
     - name: Checkout code
-      uses: actions/checkout@v2.4.0
+      uses: actions/checkout@v3.0.1
     - name: Calc coverage
       run: |
         go test -v -covermode=count -coverprofile=coverage.out
diff --git a/internal/charset/charset.go b/internal/charset/charset.go
index b0e1388..0647f73 100644
--- a/internal/charset/charset.go
+++ b/internal/charset/charset.go
@@ -157,9 +157,14 @@ func fromXML(content []byte) string {
 	return strings.ToLower(xmlEncoding(string(t.Inst)))
 }
 
-// FromHTML returns the charset of an HTML document. It relies on the meta tag
-// <meta charset="UTF-8"> and falls back on the plain text content.
+// FromHTML returns the charset of an HTML document. It first looks if a BOM is
+// present and if so uses it to determine the charset. If no BOM is present,
+// it relies on the meta tag <meta charset="UTF-8"> and falls back on the
+// plain text content.
 func FromHTML(content []byte) string {
+	if cset := FromBOM(content); cset != "" {
+		return cset
+	}
 	if cset := fromHTML(content); cset != "" {
 		return cset
 	}
diff --git a/internal/magic/magic.go b/internal/magic/magic.go
index abab040..466058f 100644
--- a/internal/magic/magic.go
+++ b/internal/magic/magic.go
@@ -104,7 +104,14 @@ func xmlCheck(sig xmlSig, raw []byte) bool {
 // matches the raw input.
 func markup(sigs ...[]byte) Detector {
 	return func(raw []byte, limit uint32) bool {
-		raw = trimLWS(raw)
+		if bytes.HasPrefix(raw, []byte{0xEF, 0xBB, 0xBF}) {
+			// We skip the UTF-8 BOM if present to ensure we correctly
+			// process any leading whitespace. The presence of the BOM
+			// is taken into account during charset detection in charset.go.
+			raw = trimLWS(raw[3:])
+		} else {
+			raw = trimLWS(raw)
+		}
 		if len(raw) == 0 {
 			return false
 		}


Test patch:
diff --git a/internal/charset/charset_test.go b/internal/charset/charset_test.go
index d9b8603..387c1ec 100644
--- a/internal/charset/charset_test.go
+++ b/internal/charset/charset_test.go
@@ -24,6 +24,29 @@ const htmlDoc = `<!DOCTYPE html>
     <div class="container footer">さ</div>
   </body>
 </html>`
+const htmlDocWithIncorrectCharset = `<!DOCTYPE html>
+<!--
+Some comment
+
+-->
+<html dir="ltr" mozdisallowselectionprint>
+  <head>
+    <meta charset="ISO-8859-16">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+    <meta name="some name" content="notranslate">
+    <title>test</title>
+
+
+    <link rel="stylesheet" href="html.utf8bom.css">
+
+
+
+  </head>
+
+  <body tabindex="1">
+    <div id="printContainer"></div>
+  </body>
+</html>`
 
 func TestFromXML(t *testing.T) {
 	charset := FromXML([]byte(xmlDoc))
@@ -39,6 +62,13 @@ func TestFromHTML(t *testing.T) {
 	}
 }
 
+func TestFromHTMLWithBOM(t *testing.T) {
+	charset := FromHTML(append([]byte{0xEF, 0xBB, 0xBF}, []byte(htmlDocWithIncorrectCharset)...))
+	if charset != "utf-8" {
+		t.Errorf("expected: utf-8; got: %s", charset)
+	}
+}
+
 func TestFromPlain(t *testing.T) {
 	tcases := []struct {
 		raw     []byte
diff --git a/mimetype_test.go b/mimetype_test.go
index 2b92693..0632536 100644
--- a/mimetype_test.go
+++ b/mimetype_test.go
@@ -18,83 +18,86 @@ const testDataDir = "testdata"
 
 // test files sorted by the file name in alphabetical order.
 var files = map[string]string{
-	"3g2.3g2":            "video/3gpp2",
-	"3gp.3gp":            "video/3gpp",
-	"3mf.3mf":            "application/vnd.ms-package.3dmanufacturing-3dmodel+xml",
-	"7z.7z":              "application/x-7z-compressed",
-	"a.a":                "application/x-archive",
-	"aac.aac":            "audio/aac",
-	"aaf.aaf":            "application/octet-stream",
-	"accdb.accdb":        "application/x-msaccess",
-	"aiff.aiff":          "audio/aiff",
-	"amf.amf":            "application/x-amf",
-	"amr.amr":            "audio/amr",
-	"ape.ape":            "audio/ape",
-	"apng.png":           "image/vnd.mozilla.apng",
-	"asf.asf":            "video/x-ms-asf",
-	"atom.atom":          "application/atom+xml",
-	"au.au":              "audio/basic",
-	"avi.avi":            "video/x-msvideo",
-	"avif.avif":          "image/avif",
-	"avifsequence.avif":  "image/avif",
-	"bmp.bmp":            "image/bmp",
-	"bpg.bpg":            "image/bpg",
-	"bz2.bz2":            "application/x-bzip2",
-	"cab.cab":            "application/vnd.ms-cab-compressed",
-	"class.class":        "application/x-java-applet",
-	"crx.crx":            "application/x-chrome-extension",
-	"csv.csv":            "text/csv",
-	"cpio.cpio":          "application/x-cpio",
-	"dae.dae":            "model/vnd.collada+xml",
-	"dbf.dbf":            "application/x-dbf",
-	"dcm.dcm":            "application/dicom",
-	"deb.deb":            "application/vnd.debian.binary-package",
-	"djvu.djvu":          "image/vnd.djvu",
-	"doc.doc":            "application/msword",
-	"docx.1.docx":        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-	"docx.docx":          "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-	"drpm.rpm":           "application/x-rpm",
-	"dwg.1.dwg":          "image/vnd.dwg",
-	"dwg.dwg":            "image/vnd.dwg",
-	"eot.eot":            "application/vnd.ms-fontobject",
-	"epub.epub":          "application/epub+zip",
-	"exe.exe":            "application/vnd.microsoft.portable-executable",
-	"fdf.fdf":            "application/vnd.fdf",
-	"fits.fits":          "application/fits",
-	"flac.flac":          "audio/flac",
-	"flv.flv":            "video/x-flv",
-	"gbr.gbr":            "image/x-gimp-gbr",
-	"geojson.1.geojson":  "application/geo+json",
-	"geojson.geojson":    "application/geo+json",
-	"gif.gif":            "image/gif",
-	"glb.glb":            "model/gltf-binary",
-	"gml.gml":            "application/gml+xml",
-	"gpx.gpx":            "application/gpx+xml",
-	"gz.gz":              "application/gzip",
-	"har.har":            "application/json",
-	"hdr.hdr":            "image/vnd.radiance",
-	"heic.single.heic":   "image/heic",
-	"heif.heif":          "image/heif",
-	"html.html":          "text/html; charset=utf-8",
-	"html.iso88591.html": "text/html; charset=iso-8859-1",
-	"html.svg.html":      "text/html; charset=utf-8",
-	"html.usascii.html":  "text/html; charset=us-ascii",
-	"html.utf8.html":     "text/html; charset=utf-8",
-	"html.withbr.html":   "text/html; charset=utf-8",
-	"ico.ico":            "image/x-icon",
-	"ics.dos.ics":        "text/calendar",
-	"ics.ics":            "text/calendar",
-	"iso88591.txt":       "text/plain; charset=iso-8859-1",
-	"jar.jar":            "application/jar",
-	"jp2.jp2":            "image/jp2",
-	"jpf.jpf":            "image/jpx",
-	"jpg.jpg":            "image/jpeg",
-	"jpm.jpm":            "image/jpm",
-	"jxl.jxl":            "image/jxl",
-	"xpm.xpm":            "image/x-xpixmap",
-	"js.js":              "application/javascript",
-	"json.json":          "application/json",
-	"json.lowascii.json": "application/json",
+	"3g2.3g2":                 "video/3gpp2",
+	"3gp.3gp":                 "video/3gpp",
+	"3mf.3mf":                 "application/vnd.ms-package.3dmanufacturing-3dmodel+xml",
+	"7z.7z":                   "application/x-7z-compressed",
+	"a.a":                     "application/x-archive",
+	"aac.aac":                 "audio/aac",
+	"aaf.aaf":                 "application/octet-stream",
+	"accdb.accdb":             "application/x-msaccess",
+	"aiff.aiff":               "audio/aiff",
+	"amf.amf":                 "application/x-amf",
+	"amr.amr":                 "audio/amr",
+	"ape.ape":                 "audio/ape",
+	"apng.png":                "image/vnd.mozilla.apng",
+	"asf.asf":                 "video/x-ms-asf",
+	"atom.atom":               "application/atom+xml",
+	"au.au":                   "audio/basic",
+	"avi.avi":                 "video/x-msvideo",
+	"avif.avif":               "image/avif",
+	"avifsequence.avif":       "image/avif",
+	"bmp.bmp":                 "image/bmp",
+	"bpg.bpg":                 "image/bpg",
+	"bz2.bz2":                 "application/x-bzip2",
+	"cab.cab":                 "application/vnd.ms-cab-compressed",
+	"class.class":             "application/x-java-applet",
+	"crx.crx":                 "application/x-chrome-extension",
+	"csv.csv":                 "text/csv",
+	"cpio.cpio":               "application/x-cpio",
+	"dae.dae":                 "model/vnd.collada+xml",
+	"dbf.dbf":                 "application/x-dbf",
+	"dcm.dcm":                 "application/dicom",
+	"deb.deb":                 "application/vnd.debian.binary-package",
+	"djvu.djvu":               "image/vnd.djvu",
+	"doc.doc":                 "application/msword",
+	"docx.1.docx":             "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	"docx.docx":               "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	"drpm.rpm":                "application/x-rpm",
+	"dwg.1.dwg":               "image/vnd.dwg",
+	"dwg.dwg":                 "image/vnd.dwg",
+	"eot.eot":                 "application/vnd.ms-fontobject",
+	"epub.epub":               "application/epub+zip",
+	"exe.exe":                 "application/vnd.microsoft.portable-executable",
+	"fdf.fdf":                 "application/vnd.fdf",
+	"fits.fits":               "application/fits",
+	"flac.flac":               "audio/flac",
+	"flv.flv":                 "video/x-flv",
+	"gbr.gbr":                 "image/x-gimp-gbr",
+	"geojson.1.geojson":       "application/geo+json",
+	"geojson.geojson":         "application/geo+json",
+	"gif.gif":                 "image/gif",
+	"glb.glb":                 "model/gltf-binary",
+	"gml.gml":                 "application/gml+xml",
+	"gpx.gpx":                 "application/gpx+xml",
+	"gz.gz":                   "application/gzip",
+	"har.har":                 "application/json",
+	"hdr.hdr":                 "image/vnd.radiance",
+	"heic.single.heic":        "image/heic",
+	"heif.heif":               "image/heif",
+	"html.html":               "text/html; charset=utf-8",
+	"html.iso88591.html":      "text/html; charset=iso-8859-1",
+	"html.svg.html":           "text/html; charset=utf-8",
+	"html.usascii.html":       "text/html; charset=us-ascii",
+	"html.utf8.html":          "text/html; charset=utf-8",
+	"html.utf8bom.html":       "text/html; charset=utf-8",
+	"html.utf8bomws.html":     "text/html; charset=utf-8",
+	"html.utf8bomdetect.html": "text/html; charset=utf-8",
+	"html.withbr.html":        "text/html; charset=utf-8",
+	"ico.ico":                 "image/x-icon",
+	"ics.dos.ics":             "text/calendar",
+	"ics.ics":                 "text/calendar",
+	"iso88591.txt":            "text/plain; charset=iso-8859-1",
+	"jar.jar":                 "application/jar",
+	"jp2.jp2":                 "image/jp2",
+	"jpf.jpf":                 "image/jpx",
+	"jpg.jpg":                 "image/jpeg",
+	"jpm.jpm":                 "image/jpm",
+	"jxl.jxl":                 "image/jxl",
+	"xpm.xpm":                 "image/x-xpixmap",
+	"js.js":                   "application/javascript",
+	"json.json":               "application/json",
+	"json.lowascii.json":      "application/json",
 	// json.{int,float,string}.txt contain a single JSON value. They are valid JSON
 	// documents, but they should not be detected as application/json. This mimics
 	// the behaviour of the file utility and seems the correct thing to do.
diff --git a/testdata/html.utf8bom.html b/testdata/html.utf8bom.html
new file mode 100644
index 0000000..3c47ba3
--- /dev/null
+++ b/testdata/html.utf8bom.html
@@ -0,0 +1,23 @@
+﻿<!DOCTYPE html>
+<!--
+Some comment
+
+-->
+<html dir="ltr" mozdisallowselectionprint>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+    <meta name="some name" content="notranslate">
+    <title>test</title>
+
+
+    <link rel="stylesheet" href="html.utf8bom.css">
+
+
+
+  </head>
+
+  <body tabindex="1">
+    <div id="printContainer"></div>
+  </body>
+</html>
diff --git a/testdata/html.utf8bomdetect.html b/testdata/html.utf8bomdetect.html
new file mode 100644
index 0000000..c776772
--- /dev/null
+++ b/testdata/html.utf8bomdetect.html
@@ -0,0 +1,24 @@
+﻿    
+<!DOCTYPE html>
+<!--
+Some comment
+
+-->
+<html dir="ltr" mozdisallowselectionprint>
+  <head>
+    <meta charset="ISO-8859-16">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+    <meta name="some name" content="notranslate">
+    <title>test</title>
+
+
+    <link rel="stylesheet" href="html.utf8bom.css">
+
+
+
+  </head>
+
+  <body tabindex="1">
+    <div id="printContainer"></div>
+  </body>
+</html>
diff --git a/testdata/html.utf8bomws.html b/testdata/html.utf8bomws.html
new file mode 100644
index 0000000..69fa3f3
--- /dev/null
+++ b/testdata/html.utf8bomws.html
@@ -0,0 +1,24 @@
+﻿    
+<!DOCTYPE html>
+<!--
+Some comment
+
+-->
+<html dir="ltr" mozdisallowselectionprint>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+    <meta name="some name" content="notranslate">
+    <title>test</title>
+
+
+    <link rel="stylesheet" href="html.utf8bom.css">
+
+
+
+  </head>
+
+  <body tabindex="1">
+    <div id="printContainer"></div>
+  </body>
+</html>