Repository: jhillyerd/enmime
Pull request #310 (2023-11-08 15:16:21)

feat: don't auto detect charset when using raw content parser option
When the content is sent as raw, the charset can sometimes return a wrong value.
In this text example with Japanese characters, the charset content returning was iso-8859-1.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/63832378/badge)](https://coveralls.io/builds/63832378)

coverage: 86.199%. remained the same
when pulling **6b2685595dbf820396c9eead13c98ca5ecd261dd on jerjako:feat/dont-auto-detect-charset-for-raw-content**
into **c35a936602fe4948e765fcf775d7639257aa73e1 on jhillyerd:main**.


Patch:
diff --git a/part.go b/part.go
index 4649fe5..564c34a 100644
--- a/part.go
+++ b/part.go
@@ -298,7 +298,7 @@ func (p *Part) decodeContent(r io.Reader, readPartErrorPolicy ReadPartErrorPolic
 			encoding)
 	}
 	// Build charset decoding reader.
-	if validEncoding && strings.HasPrefix(p.ContentType, "text/") {
+	if validEncoding && strings.HasPrefix(p.ContentType, "text/") && !p.parser.rawContent {
 		var err error
 		contentReader, err = p.convertFromDetectedCharset(contentReader, readPartErrorPolicy)
 		if err != nil {


Test patch:
diff --git a/encode_test.go b/encode_test.go
index ecc054a..3e89d74 100644
--- a/encode_test.go
+++ b/encode_test.go
@@ -365,8 +365,9 @@ func TestEncodePartContentNonAsciiText(t *testing.T) {
 	}
 }
 
-func TestParseRawContentOptionTrue(t *testing.T) {
-	r := test.OpenTestData("encode", "parser-raw-content-option.raw")
+// TestParseRawContentHTMLOptionTrue tests the RawContent Parser option with an HTML part only if the content isn't changed.
+func TestParseRawContentHTMLOptionTrue(t *testing.T) {
+	r := test.OpenTestData("encode", "parser-raw-content-html-option.raw")
 	p := enmime.NewParser(enmime.RawContent(true))
 	e, err := p.ReadEnvelope(r)
 	if err != nil {
@@ -376,11 +377,12 @@ func TestParseRawContentOptionTrue(t *testing.T) {
 	if err := e.Root.Encode(b); err != nil {
 		t.Fatal(err)
 	}
-	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-option-true.raw.golden")
+	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-html-option-true.raw.golden")
 }
 
-func TestParseRawContentOptionFalse(t *testing.T) {
-	r := test.OpenTestData("encode", "parser-raw-content-option.raw")
+// TestParseRawContentHTMLOptionFalse tests without the RawContent Parser option and an HTML part only if the content is normally parsed.
+func TestParseRawContentHTMLOptionFalse(t *testing.T) {
+	r := test.OpenTestData("encode", "parser-raw-content-html-option.raw")
 	p := enmime.NewParser()
 	e, err := p.ReadEnvelope(r)
 	if err != nil {
@@ -390,5 +392,36 @@ func TestParseRawContentOptionFalse(t *testing.T) {
 	if err := e.Root.Encode(b); err != nil {
 		t.Fatal(err)
 	}
-	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-option-false.raw.golden")
+	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-html-option-false.raw.golden")
+}
+
+// TestParseRawContentTextOptionTrue tests the RawContent Parser option with a TEXT part only if the content isn't changed.
+// This test uses Japanese characters to also ensure that the charset isn't altered.
+func TestParseRawContentTextOptionTrue(t *testing.T) {
+	r := test.OpenTestData("encode", "parser-raw-content-text-option.raw")
+	p := enmime.NewParser(enmime.RawContent(true))
+	e, err := p.ReadEnvelope(r)
+	if err != nil {
+		t.Fatal("Failed to parse MIME:", err)
+	}
+	b := &bytes.Buffer{}
+	if err := e.Root.Encode(b); err != nil {
+		t.Fatal(err)
+	}
+	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-text-option-true.raw.golden")
+}
+
+// TestParseRawContentTextOptionFalse tests without the RawContent Parser option and a TEXT part only if the content is normally parsed.
+func TestParseRawContentTextOptionFalse(t *testing.T) {
+	r := test.OpenTestData("encode", "parser-raw-content-text-option.raw")
+	p := enmime.NewParser()
+	e, err := p.ReadEnvelope(r)
+	if err != nil {
+		t.Fatal("Failed to parse MIME:", err)
+	}
+	b := &bytes.Buffer{}
+	if err := e.Root.Encode(b); err != nil {
+		t.Fatal(err)
+	}
+	test.DiffGolden(t, b.Bytes(), "testdata", "encode", "parser-raw-content-text-option-false.raw.golden")
 }
diff --git a/testdata/encode/parser-raw-content-option-false.raw.golden b/testdata/encode/parser-raw-content-html-option-false.raw.golden
similarity index 100%
rename from testdata/encode/parser-raw-content-option-false.raw.golden
rename to testdata/encode/parser-raw-content-html-option-false.raw.golden
diff --git a/testdata/encode/parser-raw-content-option-true.raw.golden b/testdata/encode/parser-raw-content-html-option-true.raw.golden
similarity index 100%
rename from testdata/encode/parser-raw-content-option-true.raw.golden
rename to testdata/encode/parser-raw-content-html-option-true.raw.golden
diff --git a/testdata/encode/parser-raw-content-option.raw b/testdata/encode/parser-raw-content-html-option.raw
similarity index 100%
rename from testdata/encode/parser-raw-content-option.raw
rename to testdata/encode/parser-raw-content-html-option.raw
diff --git a/testdata/encode/parser-raw-content-text-option-false.raw.golden b/testdata/encode/parser-raw-content-text-option-false.raw.golden
new file mode 100644
index 0000000..0d614a9
--- /dev/null
+++ b/testdata/encode/parser-raw-content-text-option-false.raw.golden
@@ -0,0 +1,8 @@
+Content-Transfer-Encoding: base64
+Content-Type: text/plain; charset=utf-8
+From: me@me.com
+Mime-Version: 1.0
+Subject: Test subject
+To: you@you.com
+
+Cgrmt7vku5jjg5XjgqHjgqTjg6vlkI3jg4bjgrnjg4gNCg==
diff --git a/testdata/encode/parser-raw-content-text-option-true.raw.golden b/testdata/encode/parser-raw-content-text-option-true.raw.golden
new file mode 100644
index 0000000..6052874
--- /dev/null
+++ b/testdata/encode/parser-raw-content-text-option-true.raw.golden
@@ -0,0 +1,9 @@
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=utf-8
+From: me@me.com
+Mime-Version: 1.0
+Subject: Test subject
+To: you@you.com
+
+=0A=0A=E6=B7=BB=E4=BB=98=E3=83=95=E3=82=A1=E3=
+=82=A4=E3=83=AB=E5=90=8D=E3=83=86=E3=82=B9=E3=83=88
diff --git a/testdata/encode/parser-raw-content-text-option.raw b/testdata/encode/parser-raw-content-text-option.raw
new file mode 100644
index 0000000..6052874
--- /dev/null
+++ b/testdata/encode/parser-raw-content-text-option.raw
@@ -0,0 +1,9 @@
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=utf-8
+From: me@me.com
+Mime-Version: 1.0
+Subject: Test subject
+To: you@you.com
+
+=0A=0A=E6=B7=BB=E4=BB=98=E3=83=95=E3=82=A1=E3=
+=82=A4=E3=83=AB=E5=90=8D=E3=83=86=E3=82=B9=E3=83=88