Repository: jhillyerd/enmime
Pull request #229 (2022-01-27 20:55:28)

Remove <!DOCTYPE tag in bad content types
We came across a bad content type, which contained a doctype tag. This Change ignores it.

jhillyerd: I feel this is fixup is overly specific; it may make more sense to strip out any HTML tag, not just doctype.

But more concerning to me is that the Index into the string does not respect quoted values, which could further mangle a properly quoted string.
jhillyerd: A unit test to show what happens when the match is inside a string may be enough to prove we don't do too much harm, I'm not certain.

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b7d1ed..3a29869 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,17 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## [Unreleased]
 
 
+## [0.9.3] - 2022-01-29
+
+### Added
+- Support for more charsets (#230)
+- fixMangledMediaType now removes extra content-type parts (#225)
+
+### Fixed
+- Fix new lines (ie in filenames) in mediatype.Parse (#224)
+- Fix crash in QPCleaner, when line is too long and buffer is almost full (#220)
+
+
 ## [0.9.2] - 2021-08-21
 
 ### Added
@@ -218,7 +229,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Initial implementation of MIME encoding, using `enmime.MailBuilder`
 
 
-[Unreleased]: https://github.com/jhillyerd/enmime/compare/v0.9.2...master
+[Unreleased]: https://github.com/jhillyerd/enmime/compare/v0.9.3...master
+[0.9.3]:      https://github.com/jhillyerd/enmime/compare/v0.9.2...v0.9.3
 [0.9.2]:      https://github.com/jhillyerd/enmime/compare/v0.9.1...v0.9.2
 [0.9.1]:      https://github.com/jhillyerd/enmime/compare/v0.9.0...v0.9.1
 [0.9.0]:      https://github.com/jhillyerd/enmime/compare/v0.8.4...v0.9.0
diff --git a/internal/coding/charsets.go b/internal/coding/charsets.go
index 7d635d8..7745bc4 100644
--- a/internal/coding/charsets.go
+++ b/internal/coding/charsets.go
@@ -248,12 +248,40 @@ var encodings = map[string]struct {
 	"iso646-us":           {charmap.Windows1252, "windows-1252"}, // ISO646 isn't us-ascii but 1991 version is.
 	"iso: western":        {charmap.Windows1252, "windows-1252"}, // same as iso-8859-1
 	"we8iso8859p1":        {charmap.Windows1252, "windows-1252"}, // same as iso-8859-1
-	"iso=8859-1":          {charmap.Windows1252, "windows-1252"}, // same as iso-8859-1
 	"cp936":               {simplifiedchinese.GBK, "gbk"},        // same as gb2312
 	"cp850":               {charmap.CodePage850, "cp850"},
 	"cp-850":              {charmap.CodePage850, "cp850"},
 	"ibm850":              {charmap.CodePage850, "cp850"},
 	"136":                 {traditionalchinese.Big5, "big5"}, // same as chinese big5
+	"cp932":               {japanese.ShiftJIS, "shift_jis"},
+	"8859-1":              {charmap.Windows1252, "windows-1252"},
+	"8859_1":              {charmap.Windows1252, "windows-1252"},
+	"8859-2":              {charmap.ISO8859_2, "iso-8859-2"},
+	"8859_2":              {charmap.ISO8859_2, "iso-8859-2"},
+	"8859-3":              {charmap.ISO8859_3, "iso-8859-3"},
+	"8859_3":              {charmap.ISO8859_3, "iso-8859-3"},
+	"8859-4":              {charmap.ISO8859_4, "iso-8859-4"},
+	"8859_4":              {charmap.ISO8859_4, "iso-8859-4"},
+	"8859-5":              {charmap.ISO8859_5, "iso-8859-5"},
+	"8859_5":              {charmap.ISO8859_5, "iso-8859-5"},
+	"8859-6":              {charmap.ISO8859_6, "iso-8859-6"},
+	"8859_6":              {charmap.ISO8859_6, "iso-8859-6"},
+	"8859-7":              {charmap.ISO8859_7, "iso-8859-7"},
+	"8859_7":              {charmap.ISO8859_7, "iso-8859-7"},
+	"8859-8":              {charmap.ISO8859_8, "iso-8859-8"},
+	"8859_8":              {charmap.ISO8859_8, "iso-8859-8"},
+	"8859-10":             {charmap.ISO8859_10, "iso-8859-10"},
+	"8859_10":             {charmap.ISO8859_10, "iso-8859-10"},
+	"8859-13":             {charmap.ISO8859_13, "iso-8859-13"},
+	"8859_13":             {charmap.ISO8859_13, "iso-8859-13"},
+	"8859-14":             {charmap.ISO8859_14, "iso-8859-14"},
+	"8859_14":             {charmap.ISO8859_14, "iso-8859-14"},
+	"8859-15":             {charmap.ISO8859_15, "iso-8859-15"},
+	"8859_15":             {charmap.ISO8859_15, "iso-8859-15"},
+	"8859-16":             {charmap.ISO8859_16, "iso-8859-16"},
+	"8859_16":             {charmap.ISO8859_16, "iso-8859-16"},
+	"utf8mb4":             {encoding.Nop, "utf-8"}, // emojis, but golang can handle it directly
+	"238":                 {charmap.Windows1250, "windows-1250"},
 }
 
 var metaTagCharsetRegexp = regexp.MustCompile(
diff --git a/mediatype/mediatype.go b/mediatype/mediatype.go
index 85a87cc..7ad75f4 100644
--- a/mediatype/mediatype.go
+++ b/mediatype/mediatype.go
@@ -42,7 +42,7 @@ const (
 //   * Newline characters
 func Parse(ctype string) (mtype string, params map[string]string, invalidParams []string, err error) {
 	mtype, params, err = mime.ParseMediaType(
-		fixNewlines(fixUnescapedQuotes(fixUnquotedSpecials(fixMangledMediaType(ctype, ';')))))
+		fixNewlines(fixUnescapedQuotes(fixUnquotedSpecials(fixMangledMediaType(fixUnexpectedTag(ctype), ';')))))
 	if err != nil {
 		if err.Error() == "mime: no media type" {
 			return "", nil, nil, nil
@@ -484,3 +484,12 @@ func fixNewlines(value string) string {
 	value = strings.ReplaceAll(value, "\r", "")
 	return value
 }
+
+// fixUnexpectedTag removes an unexpected <!DOCTYPE tag
+func fixUnexpectedTag(value string) string {
+	dtIdx := strings.Index(value, "<")
+	if dtIdx > -1 {
+		return value[0:dtIdx]
+	}
+	return value
+}
diff --git a/part.go b/part.go
index 6c93634..7bf56cc 100644
--- a/part.go
+++ b/part.go
@@ -231,7 +231,8 @@ func (p *Part) convertFromStatedCharset(r io.Reader) io.Reader {
 	// Try to parse charset again here to see if we can salvage some badly formed
 	// ones like charset="charset=utf-8".
 	charsetp := strings.Split(p.Charset, "=")
-	if strings.ToLower(charsetp[0]) == "charset" && len(charsetp) > 1 {
+	if strings.EqualFold(charsetp[0], "charset") && len(charsetp) > 1 ||
+		strings.EqualFold(charsetp[0], "iso") && len(charsetp) > 1 {
 		p.Charset = charsetp[1]
 		reader, err = coding.NewCharsetReader(p.Charset, r)
 		if err != nil {


Test patch:
diff --git a/mediatype/mediatype_test.go b/mediatype/mediatype_test.go
index 37e252c..e221b1f 100644
--- a/mediatype/mediatype_test.go
+++ b/mediatype/mediatype_test.go
@@ -476,6 +476,18 @@ func TestParseMediaType(t *testing.T) {
 			mtype:  "application/octet-stream",
 			params: map[string]string{"name": "Outlook-Logo  Desc"},
 		},
+		{
+			label:  "ignore tag",
+			input:  `text/html; charset="iso-8859-1"<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">`,
+			mtype:  "text/html",
+			params: map[string]string{"charset": "iso-8859-1"},
+		},
+		{
+			label:  "ignore tag in string",
+			input:  `text/html; charset="iso-8859-1<!DOCTYPE ... ">"`,
+			mtype:  "text/html",
+			params: map[string]string{"charset": "iso-8859-1"},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.label, func(t *testing.T) {