Repository: jhillyerd/enmime
Pull request #133 (2019-08-30 22:40:00)

added public HumanHeadersOnly package-level function
#64 Proposed Solution

Obviously the function name is up for debate, but this is the general flow
I noticed that the main lib doesn't yet support recursive rfc2047 encoded header values, so that will need to get incorporated into the ReadEnvelope flow as well

requaos: Preliminary benchmarks results put this ~%40 faster than calling ReadEnvelope for just those three headers. In this function we still read the whole `[]byte` when it can just halt and return when the header/body boundary is reached
coveralls: 
[![Coverage Status](https://coveralls.io/builds/25531025/badge)](https://coveralls.io/builds/25531025)

Coverage decreased (-0.07%) to 87.818% when pulling **27d39c3a40b7df4342984ae5a316b15dbac3faba on requaos:requaos/humanheadersonly** into **51ed44ec39b1f7e7ff1a7ef9f539b3d57131b983 on jhillyerd:develop**.

requaos: Much faster now
```
BenchmarkHumanHeadersOnly-4   	  100000	     19347 ns/op
BenchmarkReadEnvelope-4       	    1000	   1213698 ns/op
requaos: Once this is merged, we will be able to incorporate the recursive rfc2047 into normal flow and resolve #134 with a refactor of `rfc2047AttributeName` like this:
```
func rfc2047AttributeName(name string) string {
	if !strings.Contains(s, "=?") && !strings.Contains(s, "?=") {
		// the RFC2047 chunk is either malformed or is not an RFC2047 chunk
		return name
	}
	
	// Copy the string so we can return the original if we encounter any issues.
	s := name

	result := &strings.Builder{}

	_, err := result.WriteString(rfc2047decode(s))
	if err != nil {
		return name
	}

	keyValuePair := strings.SplitAfter(result.String(), "=")
	if len(keyValuePair) < 2 {
		return result.String()
	}
	// Add quotes as needed
	if !strings.HasPrefix(keyValuePair[1], "\"") {
		keyValuePair[1] = fmt.Sprintf("\"%s", keyValuePair[1])
	}
	if !strings.HasSuffix(keyValuePair[1], "\"") {
		keyValuePair[1] = fmt.Sprintf("%s\"", keyValuePair[1])
	}
	return strings.Join(keyValuePair, "")
}
requaos: @jhillyerd If everything looks good here, feel free to merge
jhillyerd: Thanks!

Patch:
diff --git a/inspect.go b/inspect.go
new file mode 100644
index 0000000..c149353
--- /dev/null
+++ b/inspect.go
@@ -0,0 +1,172 @@
+package enmime
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"mime"
+	"net/textproto"
+	"strings"
+
+	"github.com/jhillyerd/enmime/internal/coding"
+	"github.com/pkg/errors"
+)
+
+var defaultHeadersList = []string{
+	"From",
+	"To",
+	"Sender",
+	"CC",
+	"BCC",
+	"Subject",
+	"Date",
+}
+
+// DecodeHeaders returns a limited selection of mime headers for use by user agents
+// Default header list:
+//   "Date", "Subject", "Sender", "From", "To", "CC" and "BCC"
+//
+// Additional headers provided will be formatted canonically:
+//   h, err := enmime.DecodeHeaders(b, "content-type", "user-agent")
+func DecodeHeaders(b []byte, addtlHeaders ...string) (textproto.MIMEHeader, error) {
+	b = ensureHeaderBoundary(b)
+	tr := textproto.NewReader(bufio.NewReader(bytes.NewReader(b)))
+	headers, err := tr.ReadMIMEHeader()
+	switch errors.Cause(err) {
+	case nil, io.EOF:
+	// carry on, io.EOF is expected
+	default:
+		return nil, err
+	}
+	headerList := defaultHeadersList
+	headerList = append(headerList, addtlHeaders...)
+	res := map[string][]string{}
+	for _, header := range headerList {
+		h := textproto.CanonicalMIMEHeaderKey(header)
+		res[h] = make([]string, 0, len(headers[h]))
+		for _, value := range headers[h] {
+			res[h] = append(res[h], rfc2047decode(value))
+		}
+	}
+
+	return res, nil
+}
+
+// ensureHeaderBoundary scans through an rfc822 document to ensure the boundary between headers and body exists
+func ensureHeaderBoundary(b []byte) []byte {
+	slice := bytes.SplitAfter(b, []byte{'\r', '\n'})
+	dest := make([]byte, 0, len(b)+2)
+	headers := true
+	for _, v := range slice {
+		if headers && (bytes.Contains(v, []byte{':'}) || bytes.HasPrefix(v, []byte{' '}) || bytes.HasPrefix(v, []byte{'\t'})) {
+			dest = append(dest, v...)
+			continue
+		}
+		if headers {
+			headers = false
+			if !bytes.Equal(v, []byte{'\r', '\n'}) {
+				dest = append(dest, append([]byte{'\r', '\n'}, v...)...)
+				continue
+			}
+		}
+		dest = append(dest, v...)
+	}
+
+	return dest
+}
+
+// rfc2047decode returns a decoded string if the input uses RFC2047 encoding, otherwise it will return the input.
+// RFC2047 Example:
+//     `=?UTF-8?B?bmFtZT0iw7DCn8KUwoo=?=`
+func rfc2047decode(s string) string {
+	s = strings.Map(func(r rune) rune {
+		if r == '\n' || r == '\r' {
+			return ' '
+		}
+		return r
+	}, s)
+	var err error
+	for {
+		s, err = rfc2047recurse(s)
+		switch err {
+		case nil:
+			continue
+		default:
+			return s
+		}
+	}
+}
+
+// rfc2047recurse is called for if the value contains content encoded in RFC2047 format and decodes it.
+// RFC2047 Example:
+//     `=?UTF-8?B?bmFtZT0iw7DCn8KUwoo=?=`
+func rfc2047recurse(s string) (string, error) {
+	us := strings.ToUpper(s)
+	if !strings.Contains(us, "?Q?") && !strings.Contains(us, "?B?") {
+		return s, io.EOF
+	}
+
+	val, err := decodeHeaderWithError(s)
+	if err != nil {
+		return val, err
+	}
+	if val == s {
+		val, err = decodeHeaderWithError(fixRFC2047String(val))
+		if err != nil {
+			return val, err
+		}
+		if val == s {
+			return val, io.EOF
+		}
+	}
+
+	return val, nil
+}
+
+// decodeHeaderWithError decodes a single line (per RFC 2047) using Golang's mime.WordDecoder
+func decodeHeaderWithError(input string) (string, error) {
+	dec := new(mime.WordDecoder)
+	dec.CharsetReader = coding.NewCharsetReader
+	header, err := dec.DecodeHeader(input)
+	if err != nil {
+		return input, err
+	}
+	return header, nil
+}
+
+// fixRFC2047String removes the following characters from charset and encoding segments of an RFC2047 string:
+//  '\n', '\r' and ' '
+func fixRFC2047String(s string) string {
+	inString := false
+	isWithinTerminatingEqualSigns := false
+	questionMarkCount := 0
+	sb := &strings.Builder{}
+	for _, v := range s {
+		switch v {
+		case '=':
+			if questionMarkCount == 3 {
+				inString = false
+			} else {
+				isWithinTerminatingEqualSigns = true
+			}
+			sb.WriteRune(v)
+		case '?':
+			if isWithinTerminatingEqualSigns {
+				inString = true
+			} else {
+				questionMarkCount++
+			}
+			isWithinTerminatingEqualSigns = false
+			sb.WriteRune(v)
+		case '\n', '\r', ' ':
+			if !inString {
+				sb.WriteRune(v)
+			}
+			isWithinTerminatingEqualSigns = false
+		default:
+			isWithinTerminatingEqualSigns = false
+			sb.WriteRune(v)
+		}
+	}
+	return sb.String()
+}


Test patch:
diff --git a/inspect_test.go b/inspect_test.go
new file mode 100644
index 0000000..862ecfe
--- /dev/null
+++ b/inspect_test.go
@@ -0,0 +1,97 @@
+package enmime_test
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/mail"
+	"strings"
+	"testing"
+
+	"github.com/jhillyerd/enmime"
+	"github.com/jhillyerd/enmime/internal/test"
+)
+
+func TestDecodeHeaders(t *testing.T) {
+	t.Run("rfc2047 sample", func(t *testing.T) {
+		r := test.OpenTestData("mail", "qp-utf8-header.raw")
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			t.Errorf("%+v", err)
+		}
+		h, err := enmime.DecodeHeaders(b)
+		if err != nil {
+			t.Errorf("%+v", err)
+		}
+		if !strings.Contains(h.Get("To"), "Mirosław Marczak") {
+			t.Errorf("Error decoding RFC2047 header value")
+		}
+	})
+
+	t.Run("rfc2047 recursive sample", func(t *testing.T) {
+		r := test.OpenTestData("mail", "qp-utf8-header-recursed.raw")
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			t.Errorf("%+v", err)
+		}
+		h, err := enmime.DecodeHeaders(b)
+		if err != nil {
+			t.Errorf("%+v", err)
+		}
+		if !strings.Contains(h.Get("From"), "WirelessCaller (203) 402-5984 WirelessCaller (203) 402-5984 WirelessCaller (203) 402-5984") {
+			t.Errorf("Error decoding recursive RFC2047 header value")
+		}
+	})
+}
+
+func BenchmarkHumanHeadersOnly(b *testing.B) {
+	r := test.OpenTestData("mail", "qp-utf8-header.raw")
+	eml, err := ioutil.ReadAll(r)
+	if err != nil {
+		b.Fatal(err)
+	}
+	for i := 0; i < b.N; i++ {
+		h, err := enmime.DecodeHeaders(eml)
+		if err != nil {
+			b.Fatal(err)
+		}
+		_, err = mail.ParseAddressList(h.Get("From"))
+		if err != nil {
+			b.Fatal(err)
+		}
+		_, err = mail.ParseAddressList(h.Get("To"))
+		if err != nil {
+			b.Fatal(err)
+		}
+		_ = h.Get("Subject")
+	}
+}
+
+func BenchmarkReadEnvelope(b *testing.B) {
+	r := test.OpenTestData("mail", "qp-utf8-header.raw")
+	eml, err := ioutil.ReadAll(r)
+	if err != nil {
+		b.Fatal(err)
+	}
+	reusedReader := bytes.NewReader(eml)
+	for i := 0; i < b.N; i++ {
+		env, err := enmime.ReadEnvelope(reusedReader)
+		if err != nil {
+			b.Fatal(err)
+		}
+		_, err = env.AddressList("From")
+		if err != nil {
+			b.Fatal(err)
+		}
+		_, err = env.AddressList("To")
+		if err != nil {
+			b.Fatal(err)
+		}
+		env.GetHeader("Subject")
+		// reset reader for next run
+		_, err = reusedReader.Seek(0, io.SeekStart)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
diff --git a/part_test.go b/part_test.go
index 580a046..bc46a7a 100644
--- a/part_test.go
+++ b/part_test.go
@@ -868,7 +868,6 @@ func TestNoClosingBoundary(t *testing.T) {
 		Charset:     "UTF-8",
 	}
 	test.ComparePart(t, p.FirstChild, wantp)
-	t.Log(string(p.FirstChild.Content))
 
 	expected := "Missing Boundary"
 	hasCorrectError := false
diff --git a/testdata/mail/qp-utf8-header-recursed.raw b/testdata/mail/qp-utf8-header-recursed.raw
new file mode 100644
index 0000000..7015c96
--- /dev/null
+++ b/testdata/mail/qp-utf8-header-recursed.raw
@@ -0,0 +1,83 @@
+Message-ID: <5081A889.3020108@jamehi03lx.noa.com>
+Date: Fri, 19 Oct 2012 12:22:49 -0700
+From: James Hillyerd <jamehi03@jamehi03lx.noa.com>, =?UTF-8? Q?=3D=3Futf-8=3FQ=3FWirelessCaller=5F=3D28203=3D29=5F4?=
+ =?UTF-8?Q?02-5984=5FWirelessCaller=5F=3D28203=3D29=5F402-=3F?=
+ =?UTF-8?Q?=3D=0A_=3D=3Futf-8=3FQ=3F5984=5FWirelessCaller=5F=3D?=
+ =?UTF-8?Q?28203=3D29=5F402-5984=3F=3D?= <PIRARD@vm1.ulg.ac.be>
+Sender: =?ISO-8859-1?Q?Andr=E9?= Pirard <PIRARD@vm1.ulg.ac.be>
+User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:16.0) Gecko/20121010 Thunderbird/16.0.1
+MIME-Version: 1.0
+To: =?UTF-8?Q?Miros=C5=82aw_Marczak?= <marczak@inbucket.com>
+Subject: =?utf-8?q?MIME_UTF8_Test_=c2=a2?= More Text
+Content-Type: multipart/alternative;
+ boundary="------------020203040006070307010003"
+
+This is a multi-part message in MIME format.
+--------------020203040006070307010003
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam =
+venenatis ante fermentum justo varius hendrerit. Sed adipiscing =
+adipiscing nisi at placerat. Sed luctus neque pellentesque sem laoreet =
+dapibus. Class aptent taciti sociosqu ad litora torquent per conubia =
+nostra, per inceptos himenaeos. In eu scelerisque nibh. Fusce faucibus, =
+nisl vel tincidunt sodales, quam est condimentum lorem, vitae semper =
+lacus nisl vel lacus. Vestibulum vitae iaculis urna. Donec pellentesque =
+pellentesque ipsum sit amet suscipit. Ut aliquam vestibulum justo sit =
+amet congue. Mauris nisl lacus, varius a ultrices id, suscipit mattis =
+libero.
+
+Donec iaculis dapibus purus in accumsan. Cras faucibus, orci dictum =
+molestie congue, neque magna adipiscing lacus, ut laoreet sapien ante et =
+mi. Suspendisse potenti. Nullam sed dui magna. Nullam vel purus augue, =
+imperdiet vehicula purus. Lorem ipsum dolor sit amet, consectetur =
+adipiscing elit. In aliquet, ante at tincidunt ultricies, arcu dolor =
+rutrum odio, quis tempus lacus leo a quam. Fusce hendrerit, urna sed =
+elementum pulvinar, dolor sem imperdiet arcu, ut ornare erat metus sit =
+amet diam. Duis sagittis libero ut metus vulputate dictum. Etiam eget =
+augue dolor, in lacinia felis. Nulla facilisi. Proin sollicitudin =
+laoreet vehicula. Praesent non nibh odio.
+
+
+--------------020203040006070307010003
+Content-Type: text/html; charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+
+<html>
+  <head>
+
+    <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1">
+  </head>
+  <body bgcolor="#FFFFFF" text="#000000">
+    <div id="lipsum">
+      <p>
+        Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam
+        venenatis ante fermentum justo varius hendrerit. Sed adipiscing
+        adipiscing nisi at placerat. Sed luctus neque pellentesque sem
+        laoreet dapibus. Class aptent taciti sociosqu ad litora torquent
+        per conubia nostra, per inceptos himenaeos. In eu scelerisque
+        nibh. Fusce faucibus, nisl vel tincidunt sodales, quam est
+        condimentum lorem, vitae semper lacus nisl vel lacus. Vestibulum
+        vitae iaculis urna. Donec pellentesque pellentesque ipsum sit
+        amet suscipit. Ut aliquam vestibulum justo sit amet congue.
+        Mauris nisl lacus, varius a ultrices id, suscipit mattis libero.
+      </p>
+      <p>
+        Donec iaculis dapibus purus in accumsan. Cras faucibus, orci
+        dictum molestie congue, neque magna adipiscing lacus, ut laoreet
+        sapien ante et mi. Suspendisse potenti. Nullam sed dui magna.
+        Nullam vel purus augue, imperdiet vehicula purus. Lorem ipsum
+        dolor sit amet, consectetur adipiscing elit. In aliquet, ante at
+        tincidunt ultricies, arcu dolor rutrum odio, quis tempus lacus
+        leo a quam. Fusce hendrerit, urna sed elementum pulvinar, dolor
+        sem imperdiet arcu, ut ornare erat metus sit amet diam. Duis
+        sagittis libero ut metus vulputate dictum. Etiam eget augue
+        dolor, in lacinia felis. Nulla facilisi. Proin sollicitudin
+        laoreet vehicula. Praesent non nibh odio.
+      </p>
+    </div>
+  </body>
+</html>
+
+--------------020203040006070307010003--