Repository: jhillyerd/enmime
Pull request #317 (2024-01-03 09:27:13)

feat: remove tspecials from type/subtype
According to the RFC, content type and subtype cannot contain special characters and any such character will fail parsing. Removing the characters from the type/subtype can help successfully parsing the content type that contains some extra garbage.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/64848928/badge)](https://coveralls.io/builds/64848928)

coverage: 86.239% (+0.03%) from 86.213%
when pulling **d3cb93071e69aab43327843c15ec8f04acbaab41 on milankonir:feat-filter-tspecials**
into **ed0394eda018ea5b8f6e81f9e71e7ab68c5b7a91 on jhillyerd:main**.


Patch:
diff --git a/encode.go b/encode.go
index 6a23662..5a6c522 100644
--- a/encode.go
+++ b/encode.go
@@ -154,7 +154,7 @@ func (p *Part) setupMIMEHeaders() transferEncoding {
 		param := make(map[string]string)
 		setParamValue(param, hpFilename, fileName)
 		if !p.FileModDate.IsZero() {
-			setParamValue(param, hpModDate, p.FileModDate.Format(time.RFC822))
+			setParamValue(param, hpModDate, p.FileModDate.UTC().Format(time.RFC822))
 		}
 		if mt := mime.FormatMediaType(p.Disposition, param); mt != "" {
 			p.Disposition = mt
@@ -170,15 +170,19 @@ func (p *Part) encodeHeader(b *bufio.Writer) error {
 	for k := range p.Header {
 		keys = append(keys, k)
 	}
+	rawContent := p.parser != nil && p.parser.rawContent
+
 	sort.Strings(keys)
 	for _, k := range keys {
 		for _, v := range p.Header[k] {
 			encv := v
-			switch selectTransferEncoding([]byte(v), true) {
-			case teBase64:
-				encv = mime.BEncoding.Encode(utf8, v)
-			case teQuoted:
-				encv = mime.QEncoding.Encode(utf8, v)
+			if !rawContent {
+				switch selectTransferEncoding([]byte(v), true) {
+				case teBase64:
+					encv = mime.BEncoding.Encode(utf8, v)
+				case teQuoted:
+					encv = mime.QEncoding.Encode(utf8, v)
+				}
 			}
 			// _ used to prevent early wrapping
 			wb := stringutil.Wrap(76, k, ":_", encv, "\r\n")
diff --git a/header.go b/header.go
index 5fc10db..21744c5 100644
--- a/header.go
+++ b/header.go
@@ -113,6 +113,8 @@ func ParseAddressList(list string) ([]*mail.Address, error) {
 //   - Missing ';' between content-type and media parameters
 //   - Repeating media parameters
 //   - Unquoted values in media parameters containing 'tspecials' characters
+//
+// Deprecated: Use mediaType.Parse instead
 func ParseMediaType(ctype string) (mtype string, params map[string]string, invalidParams []string,
 	err error) {
 	// Export of internal function.
diff --git a/internal/textproto/reader.go b/internal/textproto/reader.go
index 010396d..a0ae11e 100644
--- a/internal/textproto/reader.go
+++ b/internal/textproto/reader.go
@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"net/textproto"
 	"strconv"
 	"strings"
 	"sync"
@@ -198,20 +199,20 @@ func (r *Reader) readCodeLine(expectCode int) (code int, continued bool, message
 
 func parseCodeLine(line string, expectCode int) (code int, continued bool, message string, err error) {
 	if len(line) < 4 || line[3] != ' ' && line[3] != '-' {
-		err = ProtocolError("short response: " + line)
+		err = textproto.ProtocolError("short response: " + line)
 		return
 	}
 	continued = line[3] == '-'
 	code, err = strconv.Atoi(line[0:3])
 	if err != nil || code < 100 {
-		err = ProtocolError("invalid response code: " + line)
+		err = textproto.ProtocolError("invalid response code: " + line)
 		return
 	}
 	message = line[4:]
 	if 1 <= expectCode && expectCode < 10 && code/100 != expectCode ||
 		10 <= expectCode && expectCode < 100 && code/10 != expectCode ||
 		100 <= expectCode && expectCode < 1000 && code != expectCode {
-		err = &Error{code, message}
+		err = &textproto.Error{Code: code, Msg: message}
 	}
 	return
 }
@@ -236,7 +237,7 @@ func parseCodeLine(line string, expectCode int) (code int, continued bool, messa
 func (r *Reader) ReadCodeLine(expectCode int) (code int, message string, err error) {
 	code, continued, message, err := r.readCodeLine(expectCode)
 	if err == nil && continued {
-		err = ProtocolError("unexpected multi-line response: " + message)
+		err = textproto.ProtocolError("unexpected multi-line response: " + message)
 	}
 	return
 }
@@ -288,7 +289,7 @@ func (r *Reader) ReadResponse(expectCode int) (code int, message string, err err
 	}
 	if err != nil && multi && message != "" {
 		// replace one line error message with all lines (full message)
-		err = &Error{code, message}
+		err = &textproto.Error{Code: code, Msg: message}
 	}
 	return
 }
@@ -506,7 +507,7 @@ func readMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) {
 		if err != nil {
 			return m, err
 		}
-		return m, ProtocolError("malformed MIME header initial line: " + string(line))
+		return m, textproto.ProtocolError("malformed MIME header initial line: " + string(line))
 	}
 
 	for {
@@ -518,15 +519,15 @@ func readMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) {
 		// Key ends at first colon.
 		k, v, ok := bytes.Cut(kv, colon)
 		if !ok {
-			return m, ProtocolError("malformed MIME header line: " + string(kv))
+			return m, textproto.ProtocolError("malformed MIME header line: " + string(kv))
 		}
 		key, ok := canonicalMIMEHeaderKey(k)
 		if !ok {
-			return m, ProtocolError("malformed MIME header line: " + string(kv))
+			return m, textproto.ProtocolError("malformed MIME header line: " + string(kv))
 		}
 		for _, c := range v {
 			if !validHeaderValueByte(c) {
-				return m, ProtocolError("malformed MIME header line: " + string(kv))
+				return m, textproto.ProtocolError("malformed MIME header line: " + string(kv))
 			}
 		}
 
@@ -578,7 +579,7 @@ func noValidation(_ []byte) error { return nil }
 // contain a colon.
 func mustHaveFieldNameColon(line []byte) error {
 	if bytes.IndexByte(line, ':') < 0 {
-		return ProtocolError(fmt.Sprintf("malformed MIME header: missing colon: %q", line))
+		return textproto.ProtocolError(fmt.Sprintf("malformed MIME header: missing colon: %q", line))
 	}
 	return nil
 }
diff --git a/internal/textproto/reader_email.go b/internal/textproto/reader_email.go
index 0cc8e36..0b49509 100644
--- a/internal/textproto/reader_email.go
+++ b/internal/textproto/reader_email.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"math"
+	"net/textproto"
 )
 
 // ReadEmailMIMEHeader reads a MIME-style header from r.
@@ -32,7 +33,7 @@ func readEmailMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) {
 		if err != nil {
 			return m, err
 		}
-		return m, ProtocolError("malformed MIME header initial line: " + string(line))
+		return m, textproto.ProtocolError("malformed MIME header initial line: " + string(line))
 	}
 
 	for {
@@ -44,11 +45,11 @@ func readEmailMIMEHeader(r *Reader, lim int64) (MIMEHeader, error) {
 		// Key ends at first colon.
 		k, v, ok := bytes.Cut(kv, colon)
 		if !ok {
-			return m, ProtocolError("malformed MIME header line: " + string(kv))
+			return m, textproto.ProtocolError("malformed MIME header line: " + string(kv))
 		}
 		key, ok := canonicalEmailMIMEHeaderKey(k)
 		if !ok {
-			return m, ProtocolError("malformed MIME header line: " + string(kv))
+			return m, textproto.ProtocolError("malformed MIME header line: " + string(kv))
 		}
 		// for _, c := range v {
 		// 	if !validHeaderValueByte(c) {
diff --git a/internal/textproto/textproto.go b/internal/textproto/textproto.go
index 70038d5..0d92bf2 100644
--- a/internal/textproto/textproto.go
+++ b/internal/textproto/textproto.go
@@ -26,29 +26,10 @@ package textproto
 
 import (
 	"bufio"
-	"fmt"
 	"io"
 	"net"
 )
 
-// An Error represents a numeric error response from a server.
-type Error struct {
-	Code int
-	Msg  string
-}
-
-func (e *Error) Error() string {
-	return fmt.Sprintf("%03d %s", e.Code, e.Msg)
-}
-
-// A ProtocolError describes a protocol violation such
-// as an invalid response or a hung-up connection.
-type ProtocolError string
-
-func (p ProtocolError) Error() string {
-	return string(p)
-}
-
 // A Conn represents a textual network protocol connection.
 // It consists of a Reader and Writer to manage I/O
 // and a Pipeline to sequence concurrent requests on the connection.
diff --git a/mediatype/mediatype.go b/mediatype/mediatype.go
index e9e44c8..94b9b23 100644
--- a/mediatype/mediatype.go
+++ b/mediatype/mediatype.go
@@ -30,6 +30,10 @@ const (
 	utf8 = "utf-8"
 )
 
+type MediaTypeParseOptions struct {
+	StripMediaTypeInvalidCharacters bool
+}
+
 // Parse is a more tolerant implementation of Go's mime.ParseMediaType function.
 //
 // Tolerances accounted for:
@@ -38,8 +42,13 @@ const (
 //   - Unquoted values in media parameters containing 'tspecials' characters
 //   - Newline characters
 func Parse(ctype string) (mtype string, params map[string]string, invalidParams []string, err error) {
+	return ParseWithOptions(ctype, MediaTypeParseOptions{})
+}
+
+// ParseWithOptions parses media-type with additional options controlling the parsing behavior.
+func ParseWithOptions(ctype string, options MediaTypeParseOptions) (mtype string, params map[string]string, invalidParams []string, err error) {
 	mtype, params, err = mime.ParseMediaType(
-		fixNewlines(fixUnescapedQuotes(fixUnquotedSpecials(fixMangledMediaType(removeTrailingHTMLTags(ctype), ';')))))
+		fixNewlines(fixUnescapedQuotes(fixUnquotedSpecials(fixMangledMediaType(removeTrailingHTMLTags(ctype), ';', options)))))
 	if err != nil {
 		if err.Error() == "mime: no media type" {
 			return "", nil, nil, nil
@@ -63,7 +72,7 @@ func Parse(ctype string) (mtype string, params map[string]string, invalidParams
 
 // fixMangledMediaType is used to insert ; separators into media type strings that lack them, and
 // remove repeated parameters.
-func fixMangledMediaType(mtype string, sep rune) string {
+func fixMangledMediaType(mtype string, sep rune, options MediaTypeParseOptions) string {
 	strsep := string([]rune{sep})
 	if mtype == "" {
 		return ""
@@ -84,6 +93,10 @@ func fixMangledMediaType(mtype string, sep rune) string {
 				// The content type is completely missing. Put in a placeholder.
 				p = ctPlaceholder
 			}
+			// Remove invalid characters (specials)
+			if options.StripMediaTypeInvalidCharacters {
+				p = removeTypeSpecials(p)
+			}
 			// Check for missing token after slash.
 			if strings.HasSuffix(p, "/") {
 				switch p {
@@ -525,3 +538,11 @@ loop:
 
 	return value
 }
+
+func removeTypeSpecials(value string) string {
+	for _, r := range []string{"(", ")", "<", ">", "@", ",", ":", "\\", "\"", "[", "]", "?", "="} {
+		value = strings.ReplaceAll(value, r, "")
+	}
+
+	return value
+}
diff --git a/options.go b/options.go
index ea9afe4..0a0175e 100644
--- a/options.go
+++ b/options.go
@@ -76,3 +76,15 @@ type parseMediaTypeOption CustomParseMediaType
 func (o parseMediaTypeOption) apply(p *Parser) {
 	p.customParseMediaType = CustomParseMediaType(o)
 }
+
+type stripMediaTypeInvalidCharactersOption bool
+
+func (o stripMediaTypeInvalidCharactersOption) apply(p *Parser) {
+	p.stripMediaTypeInvalidCharacters = bool(o)
+}
+
+// StripMediaTypeInvalidCharacters sets stripMediaTypeInvalidCharacters option. If true, invalid characters
+// will be removed from media type during parsing.
+func StripMediaTypeInvalidCharacters(stripMediaTypeInvalidCharacters bool) Option {
+	return stripMediaTypeInvalidCharactersOption(stripMediaTypeInvalidCharacters)
+}
diff --git a/parser.go b/parser.go
index 4894fbf..a2bf422 100644
--- a/parser.go
+++ b/parser.go
@@ -25,6 +25,7 @@ type Parser struct {
 	skipMalformedParts              bool
 	rawContent                      bool
 	customParseMediaType            CustomParseMediaType
+	stripMediaTypeInvalidCharacters bool
 }
 
 // defaultParser is a Parser with default configuration.
diff --git a/part.go b/part.go
index 564c34a..5aa9e37 100644
--- a/part.go
+++ b/part.go
@@ -15,6 +15,7 @@ import (
 	"github.com/gogs/chardet"
 	"github.com/jhillyerd/enmime/internal/coding"
 	inttp "github.com/jhillyerd/enmime/internal/textproto"
+	"github.com/jhillyerd/enmime/mediatype"
 	"github.com/pkg/errors"
 )
 
@@ -328,7 +329,7 @@ func (p *Part) decodeContent(r io.Reader, readPartErrorPolicy ReadPartErrorPolic
 // parses media type using custom or default media type parser
 func (p *Part) parseMediaType(ctype string) (mtype string, params map[string]string, invalidParams []string, err error) {
 	if p.parser == nil || p.parser.customParseMediaType == nil {
-		return ParseMediaType(ctype)
+		return mediatype.ParseWithOptions(ctype, mediatype.MediaTypeParseOptions{StripMediaTypeInvalidCharacters: p.parser.stripMediaTypeInvalidCharacters})
 	}
 
 	return p.parser.customParseMediaType(ctype)


Test patch:
diff --git a/encode_test.go b/encode_test.go
index 3e89d74..3a33ba7 100644
--- a/encode_test.go
+++ b/encode_test.go
@@ -425,3 +425,19 @@ func TestParseRawContentTextOptionFalse(t *testing.T) {
 	}
 	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-text-option-false.raw.golden")
 }
+
+// TestRawContentUTF8Headers verifies plain-text headers are unmodified with the rawContent parser option.
+func TestRawContentUTF8Headers(t *testing.T) {
+	r := test.OpenTestData("encode", "utf8-to.raw.golden")
+	p := enmime.NewParser(enmime.RawContent(true))
+	e, err := p.ReadEnvelope(r)
+	if err != nil {
+		t.Fatal("Failed to parse MIME:", err)
+	}
+	b := &bytes.Buffer{}
+	if err := e.Root.Encode(b); err != nil {
+		t.Fatal(err)
+	}
+
+	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "utf8-to.raw.golden")
+}
diff --git a/inspect_test.go b/inspect_test.go
index 58f22ad..e65a83b 100644
--- a/inspect_test.go
+++ b/inspect_test.go
@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"io"
 	"net/mail"
+	"net/textproto"
 	"strings"
 	"testing"
 
 	"github.com/jhillyerd/enmime"
 	"github.com/jhillyerd/enmime/internal/test"
-	"github.com/jhillyerd/enmime/internal/textproto"
 )
 
 func TestDecodeRFC2047(t *testing.T) {
diff --git a/internal/textproto/reader_test.go b/internal/textproto/reader_test.go
index 2c7d2aa..f6c728d 100644
--- a/internal/textproto/reader_test.go
+++ b/internal/textproto/reader_test.go
@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"io"
 	"net"
+	"net/textproto"
 	"reflect"
 	"strings"
 	"sync"
@@ -69,7 +70,7 @@ func TestReadCodeLine(t *testing.T) {
 	if code != 345 || msg != "no way" || err == nil {
 		t.Fatalf("Line 3: %d, %s, %v", code, msg, err)
 	}
-	if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg {
+	if e, ok := err.(*textproto.Error); !ok || e.Code != code || e.Msg != msg {
 		t.Fatalf("Line 3: wrong error %v\n", err)
 	}
 	code, msg, err = r.ReadCodeLine(1)
diff --git a/mediatype/mediatype_test.go b/mediatype/mediatype_test.go
index fa1fd12..0969c23 100644
--- a/mediatype/mediatype_test.go
+++ b/mediatype/mediatype_test.go
@@ -6,9 +6,10 @@ import (
 
 func TestFixMangledMediaType(t *testing.T) {
 	testCases := []struct {
-		input string
-		sep   rune
-		want  string
+		input   string
+		sep     rune
+		want    string
+		options MediaTypeParseOptions
 	}{
 		{
 			input: "",
@@ -48,9 +49,9 @@ func TestFixMangledMediaType(t *testing.T) {
 		},
 		{
 			// Removes empty parameters in the middle
-			input: `Content-Type: text/html; =""; charset=""`,
+			input: `text/html; =""; charset=""`,
 			sep:   ';',
-			want:  `Content-Type: text/html; charset=""`,
+			want:  `text/html; charset=""`,
 		},
 		{
 			input: "application/octet-stream;=?UTF-8?B?bmFtZT0iw7DCn8KUwoo=?=You've got a new voice miss call.msg",
@@ -133,10 +134,23 @@ func TestFixMangledMediaType(t *testing.T) {
 			sep:   ';',
 			want:  `application/pdf; name=1337.pdf`,
 		},
+		// invalid media type characters
+		{
+			input: `text/html>`,
+			sep:   ';',
+			want:  `text/html>`,
+		},
+		// invalid media type characters with stripping invalid characters sanitation enabled
+		{
+			input:   `text/html>`,
+			sep:     ';',
+			want:    `text/html`,
+			options: MediaTypeParseOptions{StripMediaTypeInvalidCharacters: true},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.input, func(t *testing.T) {
-			got := fixMangledMediaType(tc.input, tc.sep)
+			got := fixMangledMediaType(tc.input, tc.sep, tc.options)
 			if got != tc.want {
 				t.Errorf("got %q, want %q", got, tc.want)
 			}
diff --git a/part_test.go b/part_test.go
index 8df05c7..8a7774e 100644
--- a/part_test.go
+++ b/part_test.go
@@ -1302,3 +1302,19 @@ func TestChardetSuccess(t *testing.T) {
 		test.ComparePart(t, p, wantp)
 	})
 }
+
+func TestCtypeInvalidCharacters(t *testing.T) {
+	r := test.OpenTestData("parts", "ctype-invalid-characters.raw")
+	parser := enmime.NewParser(enmime.StripMediaTypeInvalidCharacters(true))
+	p, err := parser.ReadParts(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wantp := &enmime.Part{
+		PartID:      "0",
+		ContentType: "text/plain",
+	}
+
+	test.ComparePart(t, p, wantp)
+}
diff --git a/testdata/encode/part-default-headers.golden b/testdata/encode/part-default-headers.golden
index 59686cb..6fca8c4 100644
--- a/testdata/encode/part-default-headers.golden
+++ b/testdata/encode/part-default-headers.golden
@@ -1,5 +1,5 @@
 Content-Disposition: attachment; filename=stuff.zip; modification-date="01
- Feb 03 04:05 GMT"
+ Feb 03 04:05 UTC"
 Content-Id: <mycontentid>
 Content-Transfer-Encoding: base64
 Content-Type: application/zip; boundary=enmime-abcdefg0123456789;
diff --git a/testdata/encode/part-file-mod-date.golden b/testdata/encode/part-file-mod-date.golden
index 691fdb1..556bd6b 100644
--- a/testdata/encode/part-file-mod-date.golden
+++ b/testdata/encode/part-file-mod-date.golden
@@ -1,4 +1,4 @@
-Content-Disposition: inline; modification-date="01 Feb 03 04:05 GMT"
+Content-Disposition: inline; modification-date="01 Feb 03 04:05 UTC"
 Content-Transfer-Encoding: quoted-printable
 Content-Type: text/plain; charset=utf-8
 
diff --git a/testdata/encode/part-quoted-headers.golden b/testdata/encode/part-quoted-headers.golden
index d3278ba..910a854 100644
--- a/testdata/encode/part-quoted-headers.golden
+++ b/testdata/encode/part-quoted-headers.golden
@@ -1,6 +1,6 @@
 Content-Disposition: attachment;
  filename="=?utf-8?b?w6FydsOtenTFsXLFkSAieCIgdMO8a8O2cmbDunLDs2fDqXAuemlw?=";
- modification-date="01 Feb 03 04:05 GMT"
+ modification-date="01 Feb 03 04:05 UTC"
 Content-Id: <mycontentid>
 Content-Transfer-Encoding: base64
 Content-Type: application/zip; boundary=enmime-abcdefg0123456789;
diff --git a/testdata/encode/part-quoted-printable-headers.golden b/testdata/encode/part-quoted-printable-headers.golden
index d8dd302..73cc5b1 100644
--- a/testdata/encode/part-quoted-printable-headers.golden
+++ b/testdata/encode/part-quoted-printable-headers.golden
@@ -1,6 +1,6 @@
 Content-Disposition: attachment;
  filename="=?utf-8?b?w6FydsOtenTFsXLFkSAieCIgdMO8a8O2cmbDunLDs2fDqXAuemlw?=";
- modification-date="01 Feb 03 04:05 GMT"
+ modification-date="01 Feb 03 04:05 UTC"
 Content-Id: <mycontentid>
 Content-Transfer-Encoding: base64
 Content-Type: application/zip; boundary=enmime-abcdefg0123456789;
diff --git a/testdata/encode/utf8-to.raw.golden b/testdata/encode/utf8-to.raw.golden
new file mode 100644
index 0000000..dc26e83
--- /dev/null
+++ b/testdata/encode/utf8-to.raw.golden
@@ -0,0 +1,7 @@
+Content-Type: text/plain; charset=utf-8
+From: Weiß foo@bar.com
+Mime-Version: 1.0
+Subject: test utf8 to
+To: Jøhn Døe <bar@baz.com>
+
+This is a test mailing
diff --git a/testdata/parts/ctype-invalid-characters.raw b/testdata/parts/ctype-invalid-characters.raw
new file mode 100644
index 0000000..111b55f
--- /dev/null
+++ b/testdata/parts/ctype-invalid-characters.raw
@@ -0,0 +1,4 @@
+Content-Type: text/plain>
+
+Test of text/plain section with invalid content type
+