Repository: jhillyerd/enmime
Pull request #343 (2024-08-31 20:12:34)

feat: add option to control minimum charset detect length
Allows for changing from the default 100 chars.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/69540080/badge)](https://coveralls.io/builds/69540080)

coverage: 86.489% (+0.04%) from 86.446%
when pulling **49b69d07aaf9adb2896138fd55ad39bd08cdafed on minchardet**
into **a9fae7acb4311055c3721367ce970a6b629bdeea on main**.


Patch:
diff --git a/options.go b/options.go
index 8b3104a..eddd66a 100644
--- a/options.go
+++ b/options.go
@@ -112,3 +112,16 @@ func (o disableCharacterDetectionOption) apply(p *Parser) {
 func DisableCharacterDetection(disableCharacterDetection bool) Option {
 	return disableCharacterDetectionOption(disableCharacterDetection)
 }
+
+type minCharsetDetectRunesOption int
+
+func (o minCharsetDetectRunesOption) apply(p *Parser) {
+	p.minCharsetDetectRunes = int(o)
+}
+
+// MinCharsetDetectRunes sets the minimum length of a MIME part before enmime will attempt to
+// detect its character set.  The shorter the text, the more likely an incorrect character set
+// will be chosen.  The default is 100.
+func MinCharsetDetectRunes(minCharsetDetectRunes int) Option {
+	return minCharsetDetectRunesOption(minCharsetDetectRunes)
+}
diff --git a/parser.go b/parser.go
index 9e57124..87e0ea9 100644
--- a/parser.go
+++ b/parser.go
@@ -16,8 +16,7 @@ func AllowCorruptTextPartErrorPolicy(p *Part, err error) bool {
 // CustomParseMediaType parses media type. See ParseMediaType for more details
 type CustomParseMediaType func(ctype string) (mtype string, params map[string]string, invalidParams []string, err error)
 
-// Parser parses MIME.
-// Default parser is a valid one.
+// Parser parses MIME.  Create with NewParser to inherit recommended defaults.
 type Parser struct {
 	maxStoredPartErrors             *int // TODO: Pointer until global var removed.
 	multipartWOBoundaryAsSinglePart bool
@@ -28,14 +27,18 @@ type Parser struct {
 	stripMediaTypeInvalidCharacters bool
 	disableTextConversion           bool
 	disableCharacterDetection       bool
+	minCharsetDetectRunes           int
 }
 
 // defaultParser is a Parser with default configuration.
-var defaultParser = Parser{}
+var defaultParser = *NewParser()
 
 // NewParser creates new parser with given options.
 func NewParser(ops ...Option) *Parser {
-	p := Parser{}
+	// Construct parser with default options.
+	p := Parser{
+		minCharsetDetectRunes: 100,
+	}
 
 	for _, o := range ops {
 		o.apply(&p)
diff --git a/part.go b/part.go
index 579f015..afeeb92 100644
--- a/part.go
+++ b/part.go
@@ -21,7 +21,6 @@ import (
 
 const (
 	minCharsetConfidence = 85
-	minCharsetRuneLength = 100
 )
 
 // Part represents a node in the MIME multipart tree.  The Content-Type, Disposition and File Name
@@ -199,6 +198,14 @@ func (p *Part) convertFromDetectedCharset(r io.Reader, readPartErrorPolicy ReadP
 		return nil, errors.WithStack(err)
 	}
 
+	// Restore r.
+	r = bytes.NewReader(buf)
+
+	if p.parser.disableCharacterDetection && p.Charset != "" {
+		// Charset detection is disabled, use declared charset.
+		return p.convertFromStatedCharset(r), nil
+	}
+
 	cs, err := cd.DetectBest(buf)
 	switch err {
 	case nil:
@@ -207,11 +214,7 @@ func (p *Part) convertFromDetectedCharset(r io.Reader, readPartErrorPolicy ReadP
 		p.addWarning(ErrorCharsetDeclaration, "charset could not be detected: %v", err)
 	}
 
-	// Restore r.
-	r = bytes.NewReader(buf)
-
-	if (p.parser.disableCharacterDetection && p.Charset != "") ||
-		(cs == nil || cs.Confidence < minCharsetConfidence || len(bytes.Runes(buf)) < minCharsetRuneLength) {
+	if cs == nil || cs.Confidence < minCharsetConfidence || len(bytes.Runes(buf)) < p.parser.minCharsetDetectRunes {
 		// Low confidence or not enough characters, use declared character set.
 		return p.convertFromStatedCharset(r), nil
 	}


Test patch:
diff --git a/part_test.go b/part_test.go
index 4be0d59..857c239 100644
--- a/part_test.go
+++ b/part_test.go
@@ -1343,3 +1343,30 @@ func TestDisableCharacterDetectionPart(t *testing.T) {
 
 	test.ComparePart(t, p, wantp)
 }
+
+func TestCharacterDetectionRunes(t *testing.T) {
+	var wantp *enmime.Part
+
+	// chardet considers this test file to be ISO-8859-1.
+	r := test.OpenTestData("parts", "chardet-detection.raw")
+
+	// Set minimum higher than test file length.
+	parser := enmime.NewParser(enmime.MinCharsetDetectRunes(150))
+	p, err := parser.ReadParts(r)
+
+	// Examine root
+	if err != nil {
+		t.Fatalf("Unexpected parse error: %+v", err)
+	}
+	if p == nil {
+		t.Fatal("Root node should not be nil")
+	}
+
+	wantp = &enmime.Part{
+		ContentType: "text/plain",
+		PartID:      "0",
+		Charset:     "utf-8",
+	}
+
+	test.ComparePart(t, p, wantp)
+}