Repository: moov-io/ach
Pull request #1388 (2024-03-01 21:30:16)

fix: parse large files without newlines
Fixes: https://github.com/moov-io/ach/issues/1381 

codecov-commenter: ## [Codecov](https://app.codecov.io/gh/moov-io/ach/pull/1388?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=moov-io) Report
> Merging [#1388](https://app.codecov.io/gh/moov-io/ach/pull/1388?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=moov-io) (592854c) into [master](https://app.codecov.io/gh/moov-io/ach/commit/2ee78c0eb69d3af1246a23dff18faa4b51eebaeb?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=moov-io) (2ee78c0) will **decrease** coverage by `0.22%`.
> The diff coverage is `85.71%`.

> :exclamation: Current head 592854c differs from pull request most recent head 23a7ad4. Consider uploading reports for the commit 23a7ad4 to get more accurate results

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #1388      +/-   ##
==========================================
- Coverage   84.53%   84.31%   -0.22%     
==========================================
  Files          82       82              
  Lines        8184     8212      +28     
==========================================
+ Hits         6918     6924       +6     
- Misses        896      918      +22     
  Partials      370      370              
```

</details>

Patch:
diff --git a/reader.go b/reader.go
index 596c9164..2f7ff6ce 100644
--- a/reader.go
+++ b/reader.go
@@ -19,11 +19,13 @@ package ach
 
 import (
 	"bufio"
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 
 	"github.com/moov-io/base"
@@ -165,6 +167,8 @@ func (r *Reader) SetMaxLines(max int) {
 	r.maxLines = max
 }
 
+const lineLength = 94
+
 // Read reads each line in the underlying io.Reader and returns a File and any errors encountered.
 //
 // Read enforces ACH formatting rules and the first character of each line determines which parser is used.
@@ -179,14 +183,60 @@ func (r *Reader) Read() (File, error) {
 	if r.scanner == nil {
 		return r.File, errors.New("nil scanner")
 	}
+	// r.scanner.Split(scanLines)
+	r.scanner.Split(bufio.ScanRunes)
+
+	// Accumulate the current line
+	var currentLineRuneCount int
+	var currentLine bytes.Buffer
+	currentLine.Grow(lineLength)
+
 	for r.scanner.Scan() {
-		line := r.scanner.Text()
+		char := r.scanner.Text()
+		switch char {
+		case "\n", "\r":
+			// Skip accumulating the newline, but parse the line
+			if currentLineRuneCount > 0 {
+				goto fullLine
+			}
+		default:
+			currentLineRuneCount += 1
+			currentLine.WriteString(char)
+		}
+
+		if currentLineRuneCount < lineLength {
+			continue // next rune
+		}
+
+		// We have a full line to parse
+	fullLine:
 		r.lineNum++
 		if r.lineNum > r.maxLines {
 			r.errors.Add(ErrFileTooLong)
 			return r.File, r.errors
 		}
-		err := r.readLine(line)
+
+		// skip the buffered line if it's blank
+		line := currentLine.String()
+		if !blankLine(line) {
+			// hand off the line to be parsed
+			err := r.readLine(line)
+			if err != nil {
+				r.errors.Add(err)
+			}
+		}
+
+		// reset the read buffer
+		currentLine.Reset()
+		currentLineRuneCount = 0
+	}
+	if err := r.scanner.Err(); err != nil {
+		return r.File, err
+	}
+
+	// Flush anything that's left over after the scanner completes
+	if currentLineRuneCount > 0 {
+		err := r.readLine(currentLine.String())
 		if err != nil {
 			r.errors.Add(err)
 		}
@@ -236,6 +286,15 @@ func (r *Reader) Read() (File, error) {
 	return r.File, r.errors
 }
 
+func blankLine(line string) bool {
+	for _, r := range line {
+		if !unicode.IsSpace(r) {
+			return false
+		}
+	}
+	return true
+}
+
 func (r *Reader) readLine(line string) error {
 	lineLength := utf8.RuneCountInString(line)
 	switch {
@@ -249,7 +308,7 @@ func (r *Reader) readLine(line string) error {
 				lineLength,
 			)
 			return r.parseError(err)
-		} else if err := r.processFixedWidthFile(&line); err != nil {
+		} else if err := r.processFixedWidthFile(line); err != nil {
 			return err
 		}
 	case lineLength != RecordLength:
@@ -278,20 +337,20 @@ func (r *Reader) readLine(line string) error {
 }
 
 func trimSpacesFromLongLine(s string) string {
-	return strings.TrimSuffix(s[:94], " ")
+	return strings.TrimSuffix(s[:lineLength], " ")
 }
 
 func rightPadShortLine(s string) (string, error) {
 	if n := len(s); n > RecordLength {
 		return s, NewRecordWrongLengthErr(n)
 	}
-	return s + strings.Repeat(" ", 94-len(s)), nil
+	return s + strings.Repeat(" ", lineLength-len(s)), nil
 }
 
-func (r *Reader) processFixedWidthFile(line *string) error {
+func (r *Reader) processFixedWidthFile(line string) error {
 	// It should be safe to parse this byte by byte since ACH files are ASCII only.
 	record := ""
-	for i, c := range *line {
+	for i, c := range line {
 		record = record + string(c)
 		if i > 0 && (i+1)%RecordLength == 0 {
 			r.line = record


Test patch:
diff --git a/reader_test.go b/reader_test.go
index d9082f6b..44a72728 100644
--- a/reader_test.go
+++ b/reader_test.go
@@ -327,7 +327,7 @@ func TestPPDDebitFixedLengthRead__InvalidLength(t *testing.T) {
 		t.Errorf("expected error")
 	}
 
-	wantErr := "1 extra character(s) in ACH file: must be 470 but found 471"
+	wantErr := "ach.RecordWrongLengthErr must be 94 characters and found 1"
 	if err != nil && !strings.Contains(err.Error(), wantErr) {
 		t.Errorf("want: %v, got: %v", wantErr, err)
 	}
diff --git a/test/issues/issue1381_test.go b/test/issues/issue1381_test.go
new file mode 100644
index 00000000..70c4a56f
--- /dev/null
+++ b/test/issues/issue1381_test.go
@@ -0,0 +1,109 @@
+// Licensed to The Moov Authors under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. The Moov Authors licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package issues
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/moov-io/ach"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIssue1381(t *testing.T) {
+	dir := t.TempDir()
+	filename := filepath.Join(dir, "input.ach")
+
+	entryCount := 1000
+	makeLargeFileWithoutBreaks(t, filename, entryCount)
+
+	input, err := os.Open(filename)
+	require.NoError(t, err)
+	t.Cleanup(func() { input.Close() })
+
+	file, err := ach.NewReader(input).Read()
+	require.NoError(t, err)
+	require.NotNil(t, file)
+
+	require.Len(t, file.Batches, 1)
+	require.Len(t, file.Batches[0].GetEntries(), entryCount)
+}
+
+func makeLargeFileWithoutBreaks(t *testing.T, where string, entryCount int) {
+	t.Helper()
+
+	fd, err := os.Create(where)
+	require.NoError(t, err)
+	defer fd.Close()
+
+	file := ach.NewFile()
+	file.Header = ach.NewFileHeader()
+	file.Header.ImmediateDestination = "121042882"
+	file.Header.ImmediateOrigin = "91012980"
+	file.Header.FileCreationDate = time.Now().AddDate(0, 0, 1).Format("060102") // YYMMDD
+	file.Header.ImmediateDestinationName = "Federal Reserve Bank"
+	file.Header.ImmediateOriginName = "My Bank Name"
+
+	bh := ach.NewBatchHeader()
+	bh.ServiceClassCode = ach.MixedDebitsAndCredits
+	bh.CompanyName = "WhlBodyFit"
+	bh.CompanyIdentification = "MOOVTESTER"
+	bh.StandardEntryClassCode = ach.CCD
+	bh.CompanyEntryDescription = "PMTCHECK"
+	bh.EffectiveEntryDate = "230411"
+	bh.SettlementDate = "102"
+	bh.ODFIIdentification = "05310055"
+
+	batch, err := ach.NewBatch(bh)
+	require.NoError(t, err)
+
+	for i := 0; i < entryCount; i++ {
+		entry := ach.NewEntryDetail()
+		entry.TransactionCode = ach.CheckingCredit
+		entry.RDFIIdentification = "12345678"
+		entry.CheckDigit = "0"
+		entry.DFIAccountNumber = "2345532322345"
+		entry.Amount = 10001
+		entry.IndividualName = "Whole Body Fitness"
+		entry.SetTraceNumber(bh.ODFIIdentification, i)
+		entry.Category = ach.CategoryForward
+		batch.AddEntry(entry)
+	}
+	require.NoError(t, batch.Create())
+
+	file.AddBatch(batch)
+	require.NoError(t, file.Create())
+
+	// Now write it all to the
+	fd.WriteString(file.Header.String())
+	fd.WriteString(file.Batches[0].GetHeader().String())
+
+	entries := file.Batches[0].GetEntries()
+	for i := range entries {
+		fd.WriteString(entries[i].String())
+	}
+
+	fd.WriteString(file.Batches[0].GetControl().String())
+	fd.WriteString(file.Control.String())
+
+	err = fd.Sync()
+	require.NoError(t, err)
+}
diff --git a/test/testdata/ppd-debit-fixedLengthInvalid.ach b/test/testdata/ppd-debit-fixedLengthInvalid.ach
index 39258b66..98907fcd 100644
--- a/test/testdata/ppd-debit-fixedLengthInvalid.ach
+++ b/test/testdata/ppd-debit-fixedLengthInvalid.ach
@@ -1,1 +1,1 @@
-1101 076401251 0764012510807291511A094101achdestname            companyname                    5225companyname                         origid    PPDCHECKPAYMT000002080730   107640125000000162705320001912345            0000010500c-1            Bachman Eric          DD007640125565529182250000010005320001000000010500000000000000origid                             0764012500000019000001000001000000010005320001000000010500000000000000                                       
+1101 076401251 0764012510807291511A094101achdestname            companyname                    5225companyname                         origid    PPDCHECKPAYMT000002080730   107640125000000162705320001912345            0000010500c-1            Bachman Eric          DD007640125565529182250000010005320001000000010500000000000000origid                             0764012500000019000001000001000000010005320001000000010500000000000000999999999999999999999999999999999999999