Repository: moov-io/ach
Pull request #1178 (2023-02-20 02:34:28)

Enhance FlattenBatches operation
A few changes to the behavior of the `FlattenBatches` function of `File`:
- **Don't combine two batches if they share any trace numbers.** The NACHA spec indicates that trace numbers must be unique within a batch but not within the file, so currently we (correctly) see a validation error if two entries with the same trace number have similar batch headers and we call `FlattenBatches`. Instead, leave batches with similar headers separate if combining them would result in a trace number collision.
- **Make an effort to preserve original batch ordering.** Currently the way batch headers are loaded into a map as part of flattening means the batch order of the resulting file is random. Instead, have flattened batches ordered based on the lowest batch number of their contained batches. This makes the resulting file more closely resemble the original and makes the output of the flattening process deterministic, which is easier to test.

Because trace number collision prevention made the `FlattenBatches` code a bit more complicated:
- **Add more comprehensive tests.** Leave all the existing flatten tests intact, but also confirm that final entry groupings look exactly the way we expect, and that every entry detail is represented in the new file with the same associated header information.
- **Move batch flattening code to a dedicated file.** This allows us to use an interface for IAT and "other" batches, so the actual flattening logic is represented once, without making a big mess. Also makes for a convenient place to put the now-obnoxious batch flattening tests. But if this feels against the style of the repo happy to move this code back to `file.go`.

Two small fixture changes:
- `test/testdata/flattenADVBatchesMultipleBatchHeaders.ach` give each entry a different `Amount` so that they are easier to distinguish between the original and flattened files.
- `test/testdata/flattenIATBatchesMultipleBatchHeaders.ach` there were only 3 different trace numbers represented across 12 different entries, which made for a boring flattening.


Patch:
diff --git a/file.go b/file.go
index 8cd2c7a5..175d4246 100644
--- a/file.go
+++ b/file.go
@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/moov-io/base"
@@ -1194,109 +1193,6 @@ func segmentFileBatchAddADVEntry(creditBatch Batcher, debitBatch Batcher, entry
 	}
 }
 
-// FlattenBatches flattens the file's batches by consolidating batches with the same BatchHeader data into one Batch.
-// Entries within each flattened batch will be sorted by their TraceNumber field.
-func (f *File) FlattenBatches() (*File, error) {
-	out := NewFile()
-
-	// Helper method to fetch the batch header without the trace number
-	getHeader := func(batchHeader fmt.Stringer) string {
-		return batchHeader.String()[:87]
-	}
-
-	batchesByHeader := make(map[string]Batcher)
-	if f.Batches != nil {
-		for _, b := range f.Batches {
-			bhKey := getHeader(b.GetHeader())
-			_, found := batchesByHeader[bhKey]
-			if !found {
-				newBatch, err := NewBatch(b.GetHeader())
-				if err != nil {
-					return nil, err
-				}
-
-				batchesByHeader[bhKey] = newBatch
-				out.AddBatch(newBatch)
-			}
-
-			newBatch := batchesByHeader[bhKey]
-			if newBatch.GetHeader().StandardEntryClassCode == "ADV" {
-				for _, e := range b.GetADVEntries() {
-					newBatch.AddADVEntry(e)
-				}
-			} else {
-				for _, e := range b.GetEntries() {
-					newBatch.AddEntry(e)
-				}
-			}
-		}
-	}
-	for i := range out.Batches {
-		bh := out.Batches[i].GetHeader()
-		if bh.StandardEntryClassCode != "ADV" {
-			batch, _ := NewBatch(bh)
-			entries := sortEntriesByTraceNumber(out.Batches[i].GetEntries())
-			for i := range entries {
-				batch.AddEntry(entries[i])
-			}
-			out.Batches[i] = batch
-			if err := batch.Create(); err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	IATBatchesByHeader := make(map[string]*IATBatch)
-	if f.IATBatches != nil {
-		for _, b := range f.IATBatches {
-			bhKey := getHeader(b.GetHeader())
-			_, found := IATBatchesByHeader[bhKey]
-			if !found {
-				newBatch := NewIATBatch(b.GetHeader())
-				IATBatchesByHeader[bhKey] = &newBatch
-				out.AddIATBatch(newBatch)
-			}
-
-			newBatch := IATBatchesByHeader[bhKey]
-			for _, e := range b.GetEntries() {
-				newBatch.AddEntry(e)
-			}
-		}
-	}
-
-	// Set batches to valid state
-	wg := sync.WaitGroup{}
-
-	wg.Add(len(batchesByHeader))
-	for _, b := range batchesByHeader {
-		go func(b Batcher) {
-			defer wg.Done()
-			_ = b.Create()
-		}(b)
-	}
-
-	wg.Add(len(IATBatchesByHeader))
-	for _, b := range IATBatchesByHeader {
-		go func(b IATBatch) {
-			defer wg.Done()
-			_ = b.Create()
-		}(*b)
-	}
-
-	wg.Wait()
-
-	// Add FileHeaderData.
-	f.addFileHeaderData(out)
-
-	if err := out.Create(); err != nil {
-		return nil, err
-	}
-	if err := out.Validate(); err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
 // Validates that the batch numbers are ascending
 func (f *File) isSequenceAscending() error {
 	lastSeq := 0
diff --git a/file_flattener.go b/file_flattener.go
new file mode 100644
index 00000000..42cf76ac
--- /dev/null
+++ b/file_flattener.go
@@ -0,0 +1,275 @@
+// Licensed to The Moov Authors under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. The Moov Authors licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package ach
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+)
+
+var (
+	ErrFlattenChangedEntryCount   = errors.New("Flatten operation changed entry and addenda count")
+	ErrFlattenChangedDebitAmount  = errors.New("Flatten operation changed total debit entry amount")
+	ErrFlattenChangedCreditAmount = errors.New("Flatten operation changed total credit entry amount")
+)
+
+// Flatten returns a flattened version of a File, where batches with similar batch headers are consolidated.
+//
+// Two batches are eligible to be combined if:
+//   - their headers match, excluding the batch number (which isn't used in return matching and reflects
+//     the final composition of the file.)
+//   - they don't contain any entries with common trace numbers, since trace numbers must be unique
+//     within a batch.
+func Flatten(originalFile *File) (*File, error) {
+	var originalBatches []mergeable
+
+	// Convert batches and IAT batches to "mergeables" for consistent flattening logic
+	for i := range originalFile.Batches {
+		originalBatches = append(originalBatches, mergeableBatcher{originalFile.Batches[i], nil})
+	}
+	for i := range originalFile.IATBatches {
+		originalBatches = append(originalBatches, mergeableIATBatch{&originalFile.IATBatches[i], nil})
+	}
+
+	// Considering bigger batches first allows for the least number of flattened batches
+	sort.Slice(originalBatches, func(i, j int) bool {
+		return originalBatches[i].GetEntryCount() < originalBatches[j].GetEntryCount()
+	})
+
+	// Merge each original batch into a new batch
+	newBatchesByHeader := map[string][]mergeable{}
+	for i := range originalBatches {
+		batch := originalBatches[i]
+
+		var batchToMergeWith mergeable
+
+		batchesWithMatchingHeader, found := newBatchesByHeader[batch.GetHeaderSignature()]
+		if found {
+			for _, batchWithMatchingHeader := range batchesWithMatchingHeader {
+				if canMerge(batch, batchWithMatchingHeader) {
+					batchToMergeWith = batchWithMatchingHeader
+					break
+				}
+			}
+		}
+
+		if batchToMergeWith == nil {
+			newBatchesByHeader[batch.GetHeaderSignature()] = append(newBatchesByHeader[batch.GetHeaderSignature()], batch.Copy())
+		} else {
+			batchToMergeWith.Consume(batch)
+		}
+	}
+
+	// Create a new file containing each of our new batches
+	newFile := originalFile.addFileHeaderData(NewFile())
+	var allBatches []mergeable
+	for i := range newBatchesByHeader {
+		allBatches = append(allBatches, newBatchesByHeader[i]...)
+	}
+
+	// Sort batches by original batch number to roughly maintain batch order in the flattened file
+	sort.Slice(allBatches, func(i int, j int) bool { return allBatches[i].GetBatchNumber() < allBatches[j].GetBatchNumber() })
+
+	for i := range allBatches {
+		allBatches[i].AddToFile(newFile)
+	}
+
+	if err := newFile.Create(); err != nil {
+		return nil, err
+	}
+	if err := newFile.Validate(); err != nil {
+		return nil, err
+	}
+
+	// Sanity checks; this is kind of a scary operation!
+	if originalFile.Control.EntryAddendaCount != newFile.Control.EntryAddendaCount {
+		return nil, askForBugReports(ErrFlattenChangedEntryCount)
+	}
+	if originalFile.Control.TotalDebitEntryDollarAmountInFile != newFile.Control.TotalDebitEntryDollarAmountInFile {
+		return nil, askForBugReports(ErrFlattenChangedDebitAmount)
+	}
+	if originalFile.Control.TotalCreditEntryDollarAmountInFile != newFile.Control.TotalCreditEntryDollarAmountInFile {
+		return nil, askForBugReports(ErrFlattenChangedCreditAmount)
+	}
+
+	return newFile, nil
+}
+
+// FlattenBatches flattens the file's batches by consolidating batches with the same BatchHeader data into one Batch.
+// Entries within each flattened batch will be sorted by their TraceNumber field.
+func (f *File) FlattenBatches() (*File, error) {
+	return Flatten(f)
+}
+
+// Determine if two batches can be combined (ie, have the same header and no common trace numbers)
+func canMerge(a mergeable, b mergeable) bool {
+	traceNumbers := b.GetTraceNumbers()
+	for traceNumber := range a.GetTraceNumbers() {
+		_, found := traceNumbers[traceNumber]
+		if found {
+			return false
+		}
+	}
+
+	return a.GetHeaderSignature() == b.GetHeaderSignature()
+}
+
+// Represents either a "normal" batch or an IAT batch
+type mergeable interface {
+	GetHeaderSignature() string
+	GetTraceNumbers() map[string]bool
+	Consume(mergeable)
+	GetBatch() interface{}
+	GetBatchNumber() int
+	Copy() mergeable
+	GetEntryCount() int
+	AddToFile(*File)
+}
+
+type mergeableBatcher struct {
+	batcher      Batcher
+	traceNumbers map[string]bool
+}
+
+// Batch header excluding the batch number, which isn't important to preserve
+func (b mergeableBatcher) GetHeaderSignature() string { return b.batcher.GetHeader().String()[:87] }
+func (b mergeableBatcher) GetBatch() interface{}      { return b.batcher }
+func (b mergeableBatcher) GetEntryCount() int         { return len(b.batcher.GetEntries()) }
+func (b mergeableBatcher) GetBatchNumber() int        { return b.batcher.GetHeader().BatchNumber }
+
+func (b mergeableBatcher) GetTraceNumbers() map[string]bool {
+	if b.traceNumbers != nil {
+		return b.traceNumbers
+	}
+
+	b.traceNumbers = map[string]bool{}
+	for _, entry := range b.batcher.GetEntries() {
+		b.traceNumbers[entry.TraceNumber] = true
+	}
+
+	return b.traceNumbers
+}
+
+func (m mergeableBatcher) Consume(mergeableToConsume mergeable) {
+	batcherToConsume, ok := mergeableToConsume.GetBatch().(Batcher)
+	if !ok {
+		panic(fmt.Sprintf("cannot consume %T - incompatible batch types", mergeableToConsume))
+	}
+
+	// Keep the lower of the two batch numbers, to roughly maintain batch order in the flattened file
+	if batcherToConsume.GetHeader().BatchNumber < m.batcher.GetHeader().BatchNumber {
+		m.batcher.GetHeader().BatchNumber = batcherToConsume.GetHeader().BatchNumber
+	}
+
+	entries := batcherToConsume.GetEntries()
+	for i := range entries {
+		m.batcher.AddEntry(entries[i])
+	}
+	advEntries := batcherToConsume.GetADVEntries()
+	for i := range advEntries {
+		m.batcher.AddADVEntry(advEntries[i])
+	}
+}
+
+func (m mergeableBatcher) Copy() mergeable {
+	newBatcher, _ := NewBatch(m.batcher.GetHeader())
+	newMergeable := mergeableBatcher{newBatcher, nil}
+	newMergeable.Consume(m)
+
+	return newMergeable
+}
+
+func (m mergeableBatcher) AddToFile(file *File) {
+	// Sort entries by trace number
+	sort.Slice(m.batcher.GetEntries(), func(i, j int) bool {
+		return m.batcher.GetEntries()[i].TraceNumber < m.batcher.GetEntries()[j].TraceNumber
+	})
+
+	err := m.batcher.Create()
+	if err != nil {
+		panic(askForBugReports(fmt.Errorf("mergeableBatcher - AddToFile: %v", err)))
+	}
+
+	m.batcher.GetHeader().BatchNumber = 0
+
+	file.AddBatch(m.batcher)
+}
+
+type mergeableIATBatch struct {
+	iatBatch     *IATBatch
+	traceNumbers map[string]bool
+}
+
+// Batch header excluding the batch number, which isn't important to preserve
+func (b mergeableIATBatch) GetHeaderSignature() string { return b.iatBatch.Header.String()[:87] }
+func (b mergeableIATBatch) GetBatch() interface{}      { return *b.iatBatch }
+func (b mergeableIATBatch) GetEntryCount() int         { return len(b.iatBatch.Entries) }
+func (b mergeableIATBatch) GetBatchNumber() int        { return b.iatBatch.Header.BatchNumber }
+
+func (b mergeableIATBatch) GetTraceNumbers() map[string]bool {
+	if b.traceNumbers != nil {
+		return b.traceNumbers
+	}
+
+	b.traceNumbers = map[string]bool{}
+	for _, entry := range b.iatBatch.Entries {
+		b.traceNumbers[entry.TraceNumber] = true
+	}
+
+	return b.traceNumbers
+}
+
+func (m mergeableIATBatch) Consume(mergeableToConsume mergeable) {
+	batchToConsume, ok := mergeableToConsume.GetBatch().(IATBatch)
+	if !ok {
+		panic(fmt.Sprintf("IAT cannot consume %T - incompatible batch types", mergeableToConsume))
+	}
+
+	// Keep the lower of the two batch numbers, to roughly maintain batch order in the flattened file
+	if batchToConsume.Header.BatchNumber < m.iatBatch.Header.BatchNumber {
+		m.iatBatch.Header.BatchNumber = batchToConsume.Header.BatchNumber
+	}
+
+	for _, entry := range batchToConsume.Entries {
+		m.iatBatch.AddEntry(entry)
+	}
+}
+
+func (m mergeableIATBatch) Copy() mergeable {
+	newIATBatch := NewIATBatch(m.iatBatch.Header)
+	newMergeable := mergeableIATBatch{&newIATBatch, nil}
+	newMergeable.Consume(m)
+
+	return newMergeable
+}
+
+func (m mergeableIATBatch) AddToFile(file *File) {
+	// Sort entries by trace number
+	sort.Slice(m.iatBatch.Entries, func(i, j int) bool {
+		return m.iatBatch.Entries[i].TraceNumber < m.iatBatch.Entries[j].TraceNumber
+	})
+
+	err := m.iatBatch.Create()
+	if err != nil {
+		panic(askForBugReports(fmt.Errorf("mergeableIATBatch - AddToFile: %v", err)))
+	}
+	m.iatBatch.Header.BatchNumber = 0
+
+	file.AddIATBatch(*m.iatBatch)
+}
diff --git a/oss.go b/oss.go
new file mode 100644
index 00000000..af12418d
--- /dev/null
+++ b/oss.go
@@ -0,0 +1,43 @@
+// Licensed to The Moov Authors under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. The Moov Authors licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package ach
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	ErrPleaseReportBug = errors.New("Please report this error to github.com/moov-io/ach/issues")
+)
+
+type reportableError struct {
+	err error
+}
+
+func (e *reportableError) Unwrap() error {
+	return e.err
+}
+
+func (e *reportableError) Error() string {
+	return fmt.Sprintf("%v: %v", ErrPleaseReportBug, e.err)
+}
+
+func askForBugReports(err error) error {
+	return &reportableError{err: err}
+}


Test patch:
diff --git a/file_flattener_test.go b/file_flattener_test.go
new file mode 100644
index 00000000..b818a789
--- /dev/null
+++ b/file_flattener_test.go
@@ -0,0 +1,203 @@
+// Licensed to The Moov Authors under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. The Moov Authors licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package ach
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestFlattenFile(t *testing.T) {
+	// Represent a file as groupings of trace numbers (or, hackily, amounts for ADV entries)
+	getFileStructure := func(file *File) [][]string {
+		var fileStructure [][]string
+		for _, batcher := range file.Batches {
+			var batchStructure []string
+			for _, entry := range batcher.GetEntries() {
+				batchStructure = append(batchStructure, entry.TraceNumber)
+			}
+			for _, entry := range batcher.GetADVEntries() {
+				batchStructure = append(batchStructure, fmt.Sprintf("%d", entry.Amount))
+			}
+			fileStructure = append(fileStructure, batchStructure)
+		}
+		for _, batch := range file.IATBatches {
+			var batchStructure []string
+			for _, entry := range batch.Entries {
+				batchStructure = append(batchStructure, entry.TraceNumber)
+			}
+			fileStructure = append(fileStructure, batchStructure)
+		}
+
+		return fileStructure
+	}
+
+	testCases := []struct {
+		label string
+
+		inputFixtureName string
+
+		inputStructure     [][]string
+		flattenedStructure [][]string
+	}{
+		{
+			label:            "Single batch",
+			inputFixtureName: "flattenBatchesOneBatchHeader.ach",
+			inputStructure: [][]string{
+				{"121042880000001", "121042880000002", "121042880000003"},
+			},
+			flattenedStructure: [][]string{
+				{"121042880000001", "121042880000002", "121042880000003"},
+			},
+		},
+		{
+			label:            "Multiple batches (no trace collision)",
+			inputFixtureName: "flattenBatchesMultipleBatchHeaders.ach",
+			inputStructure: [][]string{
+				{"121042880000001", "121042880000002", "121042880000003"},
+				{"121042880000004", "121042880000005", "121042880000006"},
+				{"121042880000007", "121042880000008", "121042880000009"},
+				{"121042880000010", "121042880000011", "121042880000012"},
+			},
+			flattenedStructure: [][]string{
+				{"121042880000001", "121042880000002", "121042880000003", "121042880000004", "121042880000005", "121042880000006"},
+				{"121042880000007", "121042880000008", "121042880000009"},
+				{"121042880000010", "121042880000011", "121042880000012"},
+			},
+		},
+		{
+			label:            "Trace number collision",
+			inputFixtureName: "flattenBatchesTraceNumberCollision.ach",
+			inputStructure: [][]string{
+				{"121042880000001", "121042880000002", "121042880000022"},
+				{"121042880000001", "121042880000002", "121042880000003"},
+				{"121042880000004", "121042880000005", "121042880000006"},
+				{"121042880000007", "121042880000008", "121042880000009"},
+				{"121042880000010", "121042880000011", "121042880000012"},
+			},
+			flattenedStructure: [][]string{
+				{"121042880000001", "121042880000002", "121042880000004", "121042880000005", "121042880000006", "121042880000022"},
+				{"121042880000001", "121042880000002", "121042880000003"},
+				{"121042880000007", "121042880000008", "121042880000009"},
+				{"121042880000010", "121042880000011", "121042880000012"},
+			},
+		},
+		{
+			label:            "IAT batches",
+			inputFixtureName: "flattenIATBatchesMultipleBatchHeaders.ach",
+			inputStructure: [][]string{
+				{"231380100000001", "231380100000002", "231380100000003"},
+				{"231380100000004", "231380100000005", "231380100000006"},
+				{"231380100000007", "231380100000008", "231380100000009"},
+				{"231380100000010", "231380100000011", "231380100000012"},
+			},
+			flattenedStructure: [][]string{
+				{"231380100000001", "231380100000002", "231380100000003", "231380100000004", "231380100000005", "231380100000006"},
+				{"231380100000007", "231380100000008", "231380100000009", "231380100000010", "231380100000011", "231380100000012"},
+			},
+		},
+		{
+			label:            "ADV batches",
+			inputFixtureName: "flattenADVBatchesMultipleBatchHeaders.ach",
+			inputStructure: [][]string{
+				{"50001", "50002", "50003"},
+				{"50004", "50005", "50006"},
+				{"50007", "50008", "50009"},
+				{"50010", "50011", "50012"},
+			},
+			flattenedStructure: [][]string{
+				{"50001", "50002", "50003", "50004", "50005", "50006"},
+				{"50007", "50008", "50009", "50010", "50011", "50012"},
+			},
+		},
+		{
+			label:            "Micro deposits",
+			inputFixtureName: "two-micro-deposits.ach",
+			inputStructure: [][]string{
+				{"121042886829038", "121042886829039", "121042886829040"},
+				{"121042889211556", "121042889211557", "121042889211558"},
+			},
+			flattenedStructure: [][]string{
+				{"121042886829038", "121042886829039", "121042886829040", "121042889211556", "121042889211557", "121042889211558"},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.label, func(t *testing.T) {
+			inputRaw, _ := os.Open(filepath.Join("test", "testdata", testCase.inputFixtureName))
+			inputFile, _ := NewReader(inputRaw).Read()
+
+			flattenedFile, err := Flatten(&inputFile)
+			if err != nil {
+				panic(err)
+			}
+
+			inputFileStructureJson, _ := json.MarshalIndent(getFileStructure(&inputFile), "", "  ")
+			flattenedFileStructureJson, _ := json.MarshalIndent(getFileStructure(flattenedFile), "", "  ")
+			expectedInputFileStructureJson, _ := json.MarshalIndent(testCase.inputStructure, "", "  ")
+			expectedFlattenedFileStructureJson, _ := json.MarshalIndent(testCase.flattenedStructure, "", "  ")
+
+			if string(inputFileStructureJson) != string(expectedInputFileStructureJson) {
+				t.Fatalf(
+					"Input file structure doesn't match expectation. \n\nActual:\n%s\n\nExpected:\n%s",
+					string(inputFileStructureJson),
+					string(expectedInputFileStructureJson),
+				)
+			}
+
+			if string(flattenedFileStructureJson) != string(expectedFlattenedFileStructureJson) {
+				t.Fatalf(
+					"Flattened file structure doesn't match expectation. \n\nActual:\n%s\n\nExpected:\n%s",
+					string(flattenedFileStructureJson),
+					string(expectedFlattenedFileStructureJson),
+				)
+			}
+
+			// Ensure that every entry from the input file is present in the flattened file, and is associated with the same batch header
+			getEntrySignatures := func(file File) []string {
+				var entrySignatureSet []string
+				for _, batcher := range file.Batches {
+					for _, entry := range batcher.GetEntries() {
+						entrySignatureSet = append(entrySignatureSet, fmt.Sprintf("%s|%s", batcher.GetHeader().String()[4:87], entry.String()))
+					}
+					for _, entry := range batcher.GetADVEntries() {
+						entrySignatureSet = append(entrySignatureSet, fmt.Sprintf("%s|%s", batcher.GetHeader().String()[4:87], entry.String()[:90]))
+					}
+				}
+				for _, batch := range file.IATBatches {
+					for _, entry := range batch.Entries {
+						entrySignatureSet = append(entrySignatureSet, fmt.Sprintf("%s|%s", batch.Header.String()[4:87], entry.String()))
+					}
+				}
+
+				sort.Strings(entrySignatureSet)
+				return entrySignatureSet
+			}
+
+			if strings.Join(getEntrySignatures(inputFile), ", ") != strings.Join(getEntrySignatures(*flattenedFile), ", ") {
+				t.Fatal("Discrepancy between input and flattened file entry signatures")
+			}
+		})
+	}
+}
diff --git a/oss_test.go b/oss_test.go
new file mode 100644
index 00000000..0b85fd9a
--- /dev/null
+++ b/oss_test.go
@@ -0,0 +1,37 @@
+// Licensed to The Moov Authors under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. The Moov Authors licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package ach
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOSS(t *testing.T) {
+	err := askForBugReports(ErrFlattenChangedEntryCount)
+	require.Equal(t,
+		"Please report this error to github.com/moov-io/ach/issues: Flatten operation changed entry and addenda count",
+		fmt.Sprintf("%v", err),
+	)
+
+	err = errors.Unwrap(err)
+	require.True(t, errors.Is(err, ErrFlattenChangedEntryCount))
+}
diff --git a/test/testdata/flattenADVBatchesMultipleBatchHeaders.ach b/test/testdata/flattenADVBatchesMultipleBatchHeaders.ach
index ccc5104a..269da92e 100644
--- a/test/testdata/flattenADVBatchesMultipleBatchHeaders.ach
+++ b/test/testdata/flattenADVBatchesMultipleBatchHeaders.ach
@@ -1,24 +1,24 @@
 101 23138010401210428821909051411A094101Federal Reserve Bank   My Bank Name                   
 5280Company Name, In                    121042882 ADVAccounting      190906   0121042880000001
-681231380104744-5678-99    00000005000012104288211131 Name                    0011000010500001
-681231380104744-5678-99    00000005000012104288211131 Name                    0011000010500002
-681231380104744-5678-99    00000005000012104288211131 Name                    0011000010500003
-828000000300694140300000000000000000000000000000000000150000Company Name, Inc  121042880000001
+681231380104744-5678-99    00000005000112104288211131 Name                    0011000010500001
+681231380104744-5678-99    00000005000212104288211131 Name                    0011000010500002
+681231380104744-5678-99    00000005000312104288211131 Name                    0011000010500003
+828000000300694140300000000000000000000000000000000000150006Company Name, Inc  121042880000001
 5280Company Name, In                    121042882 ADVAccounting      190906   0121042880000002
-681231380104744-5678-99    00000005000012104288211131 Name                    0011000010500001
-681231380104744-5678-99    00000005000012104288211131 Name                    0011000010500002
-681231380104744-5678-99    00000005000012104288211131 Name                    0011000010500003
-828000000300694140300000000000000000000000000000000000150000Company Name, Inc  121042880000002
+681231380104744-5678-99    00000005000412104288211131 Name                    0011000010500001
+681231380104744-5678-99    00000005000512104288211131 Name                    0011000010500002
+681231380104744-5678-99    00000005000612104288211131 Name                    0011000010500003
+828000000300694140300000000000000000000000000000000000150015Company Name, Inc  121042880000002
 5280Company Incorpor                    121042882 ADVAccounting      190906   0031202080000003
-681231380104744-5678-99    00000005000003120208411135 Name                    0011000010500001
-681231380104744-5678-99    00000005000003120208411135 Name                    0011000010500002
-681231380104744-5678-99    00000005000003120208411135 Name                    0011000010500003
-828000000300694140300000000000000000000000000000000000150000Company Incorporate031202080000003
+681231380104744-5678-99    00000005000703120208411135 Name                    0011000010500001
+681231380104744-5678-99    00000005000803120208411135 Name                    0011000010500002
+681231380104744-5678-99    00000005000903120208411135 Name                    0011000010500003
+828000000300694140300000000000000000000000000000000000150024Company Incorporate031202080000003
 5280Company Incorpor                    121042882 ADVAccounting      190906   0031202080000004
-681231380104744-5678-99    00000005000003120208411135 Name                    0011000010500001
-681231380104744-5678-99    00000005000003120208411135 Name                    0011000010500002
-681231380104744-5678-99    00000005000003120208411135 Name                    0011000010500003
-828000000300694140300000000000000000000000000000000000150000Company Incorporate031202080000004
+681231380104744-5678-99    00000005001003120208411135 Name                    0011000010500001
+681231380104744-5678-99    00000005001103120208411135 Name                    0011000010500002
+681231380104744-5678-99    00000005001203120208411135 Name                    0011000010500003
+828000000300694140300000000000000000000000000000000000150033Company Incorporate031202080000004
 90000040000030000001202776561200000000000000000000000000000000000600000                       
 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
@@ -27,4 +27,4 @@
 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
-9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
\ No newline at end of file
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
diff --git a/test/testdata/flattenBatchesTraceNumberCollision.ach b/test/testdata/flattenBatchesTraceNumberCollision.ach
new file mode 100644
index 00000000..94a2a71d
--- /dev/null
+++ b/test/testdata/flattenBatchesTraceNumberCollision.ach
@@ -0,0 +1,50 @@
+101 231380104 1210428821908261725A094101Citadel                Wells Fargo                    
+5200Wells Fargo                         121042882 PPDTrans. Des      190827   1121042880000001
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000001
+705bonus pay for amazing work on #OSS                                              00010000001
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000002
+705bonus pay for amazing work on #OSS                                              00010000002
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000022
+705bonus pay for amazing work on #OSS                                              00010000022
+82000000060069414030000000000000000000300000121042882                          121042880000001
+5200Wells Fargo                         121042882 PPDTrans. Des      190827   1121042880000002
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000001
+705bonus pay for amazing work on #OSS                                              00010000001
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000002
+705bonus pay for amazing work on #OSS                                              00010000002
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000003
+705bonus pay for amazing work on #OSS                                              00010000003
+82000000060069414030000000000000000000300000121042882                          121042880000002
+5200Wells Fargo                         121042882 PPDTrans. Des      190827   1121042880000003
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000004
+705bonus pay for amazing work on #OSS                                              00010000004
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000005
+705bonus pay for amazing work on #OSS                                              00010000005
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000006
+705bonus pay for amazing work on #OSS                                              00010000006
+82000000060069414030000000000000000000300000121042882                          121042880000003
+5200Wells Bank                          121042882 PPDTrans. Des      190827   1121042880000004
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000007
+705bonus pay for amazing work on #OSS                                              00010000007
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000008
+705bonus pay for amazing work on #OSS                                              00010000008
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000009
+705bonus pay for amazing work on #OSS                                              00010000009
+82000000060069414030000000000000000000300000121042882                          121042880000004
+5200Wells                               121042882 PPDTrans. Des      190827   1121042880000005
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000010
+705bonus pay for amazing work on #OSS                                              00010000010
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000011
+705bonus pay for amazing work on #OSS                                              00010000011
+62223138010481967038518      0000100000#83738AB#      Steven Tander           1121042880000012
+705bonus pay for amazing work on #OSS                                              00010000012
+82000000060069414030000000000000000000300000121042882                          121042880000005
+9000005000005000000300347070150000000000000000001500000                                       
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
+9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999
diff --git a/test/testdata/flattenIATBatchesMultipleBatchHeaders.ach b/test/testdata/flattenIATBatchesMultipleBatchHeaders.ach
index 3c9736fd..f2e58df0 100644
--- a/test/testdata/flattenIATBatchesMultipleBatchHeaders.ach
+++ b/test/testdata/flattenIATBatchesMultipleBatchHeaders.ach
@@ -1,4 +1,4 @@
-101 12104288202313801041909051150A094101Bank                   My Bank Name                   
+101 121042882 2313801041909051150A094101Bank                   My Bank Name                   
 5200                FF3               US123456789 IATTRADEPAYMTCADUSD190906   0231380100000001
 6271210428820007             0000100000123456789                              1231380100000001
 710ANN000000000000100000928383-23938          BEK Enterprises                          0000001
@@ -32,99 +32,99 @@
 718Bank of France                     01456456456987987                   FR       00010000003
 82000000300036312864000000300000000000000000                                   231380100000001
 5200                FF3               US123456789 IATTRADEPAYMTCADUSD190906   0231380100000002
-6271210428820007             0000100000123456789                              1231380100000001
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000001
-711BEK Solutions                      15 West Place Street                             0000001
-712JacobsTown*PA\                     US*19305\                                        0000001
-713Wells Fargo                        01231380104                         US           0000001
-714Citadel Bank                       01121042882                         CA           0000001
-7159874654932139872121 Front Street                                                    0000001
-716LetterTown*AB\                     CA*80014\                                        0000001
-717This is an international payment                                                00010000001
-718Bank of France                     01456456456987987                   FR       00010000001
-6271210428820007             0000100000123456789                              1231380100000002
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000002
-711BEK Solutions                      15 West Place Street                             0000002
-712JacobsTown*PA\                     US*19305\                                        0000002
-713Wells Fargo                        01231380104                         US           0000002
-714Citadel Bank                       01121042882                         CA           0000002
-7159874654932139872121 Front Street                                                    0000002
-716LetterTown*AB\                     CA*80014\                                        0000002
-717This is an international payment                                                00010000002
-718Bank of France                     01456456456987987                   FR       00010000002
-6271210428820007             0000100000123456789                              1231380100000003
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000003
-711BEK Solutions                      15 West Place Street                             0000003
-712JacobsTown*PA\                     US*19305\                                        0000003
-713Wells Fargo                        01231380104                         US           0000003
-714Citadel Bank                       01121042882                         CA           0000003
-7159874654932139872121 Front Street                                                    0000003
-716LetterTown*AB\                     CA*80014\                                        0000003
-717This is an international payment                                                00010000003
-718Bank of France                     01456456456987987                   FR       00010000003
+6271210428820007             0000100000123456789                              1231380100000004
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000004
+711BEK Solutions                      15 West Place Street                             0000004
+712JacobsTown*PA\                     US*19305\                                        0000004
+713Wells Fargo                        01231380104                         US           0000004
+714Citadel Bank                       01121042882                         CA           0000004
+7159874654932139872121 Front Street                                                    0000004
+716LetterTown*AB\                     CA*80014\                                        0000004
+717This is an international payment                                                00010000004
+718Bank of France                     01456456456987987                   FR       00010000004
+6271210428820007             0000100000123456789                              1231380100000005
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000005
+711BEK Solutions                      15 West Place Street                             0000005
+712JacobsTown*PA\                     US*19305\                                        0000005
+713Wells Fargo                        01231380104                         US           0000005
+714Citadel Bank                       01121042882                         CA           0000005
+7159874654932139872121 Front Street                                                    0000005
+716LetterTown*AB\                     CA*80014\                                        0000005
+717This is an international payment                                                00010000005
+718Bank of France                     01456456456987987                   FR       00010000005
+6271210428820007             0000100000123456789                              1231380100000006
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000006
+711BEK Solutions                      15 West Place Street                             0000006
+712JacobsTown*PA\                     US*19305\                                        0000006
+713Wells Fargo                        01231380104                         US           0000006
+714Citadel Bank                       01121042882                         CA           0000006
+7159874654932139872121 Front Street                                                    0000006
+716LetterTown*AB\                     CA*80014\                                        0000006
+717This is an international payment                                                00010000006
+718Bank of France                     01456456456987987                   FR       00010000006
 82000000300036312864000000300000000000000000                                   231380100000002
 5200                FF3               US987654321 IATPAYMENT   CADUSD190906   0231380100000003
-6271210428820007             0000100000123456789                              1231380100000001
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000001
-711BEK Solutions                      15 West Place Street                             0000001
-712JacobsTown*PA\                     US*19305\                                        0000001
-713Wells Fargo                        01231380104                         US           0000001
-714Citadel Bank                       01121042882                         CA           0000001
-7159874654932139872121 Front Street                                                    0000001
-716LetterTown*AB\                     CA*80014\                                        0000001
-717This is an international payment                                                00010000001
-718Bank of France                     01456456456987987                   FR       00010000001
-6271210428820007             0000100000123456789                              1231380100000002
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000002
-711BEK Solutions                      15 West Place Street                             0000002
-712JacobsTown*PA\                     US*19305\                                        0000002
-713Wells Fargo                        01231380104                         US           0000002
-714Citadel Bank                       01121042882                         CA           0000002
-7159874654932139872121 Front Street                                                    0000002
-716LetterTown*AB\                     CA*80014\                                        0000002
-717This is an international payment                                                00010000002
-718Bank of France                     01456456456987987                   FR       00010000002
-6271210428820007             0000100000123456789                              1231380100000003
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000003
-711BEK Solutions                      15 West Place Street                             0000003
-712JacobsTown*PA\                     US*19305\                                        0000003
-713Wells Fargo                        01231380104                         US           0000003
-714Citadel Bank                       01121042882                         CA           0000003
-7159874654932139872121 Front Street                                                    0000003
-716LetterTown*AB\                     CA*80014\                                        0000003
-717This is an international payment                                                00010000003
-718Bank of France                     01456456456987987                   FR       00010000003
+6271210428820007             0000100000123456789                              1231380100000007
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000007
+711BEK Solutions                      15 West Place Street                             0000007
+712JacobsTown*PA\                     US*19305\                                        0000007
+713Wells Fargo                        01231380104                         US           0000007
+714Citadel Bank                       01121042882                         CA           0000007
+7159874654932139872121 Front Street                                                    0000007
+716LetterTown*AB\                     CA*80014\                                        0000007
+717This is an international payment                                                00010000007
+718Bank of France                     01456456456987987                   FR       00010000007
+6271210428820007             0000100000123456789                              1231380100000008
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000008
+711BEK Solutions                      15 West Place Street                             0000008
+712JacobsTown*PA\                     US*19305\                                        0000008
+713Wells Fargo                        01231380104                         US           0000008
+714Citadel Bank                       01121042882                         CA           0000008
+7159874654932139872121 Front Street                                                    0000008
+716LetterTown*AB\                     CA*80014\                                        0000008
+717This is an international payment                                                00010000008
+718Bank of France                     01456456456987987                   FR       00010000008
+6271210428820007             0000100000123456789                              1231380100000009
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000009
+711BEK Solutions                      15 West Place Street                             0000009
+712JacobsTown*PA\                     US*19305\                                        0000009
+713Wells Fargo                        01231380104                         US           0000009
+714Citadel Bank                       01121042882                         CA           0000009
+7159874654932139872121 Front Street                                                    0000009
+716LetterTown*AB\                     CA*80014\                                        0000009
+717This is an international payment                                                00010000009
+718Bank of France                     01456456456987987                   FR       00010000009
 82000000300036312864000000300000000000000000                                   231380100000003
 5200                FF3               US987654321 IATPAYMENT   CADUSD190906   0231380100000004
-6271210428820007             0000100000123456789                              1231380100000001
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000001
-711BEK Solutions                      15 West Place Street                             0000001
-712JacobsTown*PA\                     US*19305\                                        0000001
-713Wells Fargo                        01231380104                         US           0000001
-714Citadel Bank                       01121042882                         CA           0000001
-7159874654932139872121 Front Street                                                    0000001
-716LetterTown*AB\                     CA*80014\                                        0000001
-717This is an international payment                                                00010000001
-718Bank of France                     01456456456987987                   FR       00010000001
-6271210428820007             0000100000123456789                              1231380100000002
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000002
-711BEK Solutions                      15 West Place Street                             0000002
-712JacobsTown*PA\                     US*19305\                                        0000002
-713Wells Fargo                        01231380104                         US           0000002
-714Citadel Bank                       01121042882                         CA           0000002
-7159874654932139872121 Front Street                                                    0000002
-716LetterTown*AB\                     CA*80014\                                        0000002
-717This is an international payment                                                00010000002
-718Bank of France                     01456456456987987                   FR       00010000002
-6271210428820007             0000100000123456789                              1231380100000003
-710ANN000000000000100000928383-23938          BEK Enterprises                          0000003
-711BEK Solutions                      15 West Place Street                             0000003
-712JacobsTown*PA\                     US*19305\                                        0000003
-713Wells Fargo                        01231380104                         US           0000003
-714Citadel Bank                       01121042882                         CA           0000003
-7159874654932139872121 Front Street                                                    0000003
-716LetterTown*AB\                     CA*80014\                                        0000003
-717This is an international payment                                                00010000003
-718Bank of France                     01456456456987987                   FR       00010000003
+6271210428820007             0000100000123456789                              1231380100000010
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000010
+711BEK Solutions                      15 West Place Street                             0000010
+712JacobsTown*PA\                     US*19305\                                        0000010
+713Wells Fargo                        01231380104                         US           0000010
+714Citadel Bank                       01121042882                         CA           0000010
+7159874654932139872121 Front Street                                                    0000010
+716LetterTown*AB\                     CA*80014\                                        0000010
+717This is an international payment                                                00010000010
+718Bank of France                     01456456456987987                   FR       00010000010
+6271210428820007             0000100000123456789                              1231380100000011
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000011
+711BEK Solutions                      15 West Place Street                             0000011
+712JacobsTown*PA\                     US*19305\                                        0000011
+713Wells Fargo                        01231380104                         US           0000011
+714Citadel Bank                       01121042882                         CA           0000011
+7159874654932139872121 Front Street                                                    0000011
+716LetterTown*AB\                     CA*80014\                                        0000011
+717This is an international payment                                                00010000011
+718Bank of France                     01456456456987987                   FR       00010000011
+6271210428820007             0000100000123456789                              1231380100000012
+710ANN000000000000100000928383-23938          BEK Enterprises                          0000012
+711BEK Solutions                      15 West Place Street                             0000012
+712JacobsTown*PA\                     US*19305\                                        0000012
+713Wells Fargo                        01231380104                         US           0000012
+714Citadel Bank                       01121042882                         CA           0000012
+7159874654932139872121 Front Street                                                    0000012
+716LetterTown*AB\                     CA*80014\                                        0000012
+717This is an international payment                                                00010000012
+718Bank of France                     01456456456987987                   FR       00010000012
 82000000300036312864000000300000000000000000                                   231380100000004
-9000004000013000001200145251456000001200000000000000000                                       
\ No newline at end of file
+9000004000013000001200145251456000001200000000000000000