Repository: xlate/staedi (java)

Issue:
Exception caused by 80 character delimited X12
**Describe the bug**
Attempting to parse the included X12 EDI 210 which is 80 character delimited is causing the following exception:
```
io.xlate.edi.internal.stream.tokenization.EDIException: EDIE003 - Invalid processing stateInvalid header segment at offset 105

	at io.xlate.edi.internal.stream.tokenization.Lexer.error(Lexer.java:403)
	at io.xlate.edi.internal.stream.tokenization.Lexer.dialectConfirmed(Lexer.java:395)
	at io.xlate.edi.internal.stream.tokenization.Lexer.parse(Lexer.java:225)
	at io.xlate.edi.internal.stream.StaEDIStreamReader.nextEvent(StaEDIStreamReader.java:155)
	at io.xlate.edi.internal.stream.StaEDIStreamReader.next(StaEDIStreamReader.java:187)
	at io.xlate.edi.internal.stream.StaEDIStreamReader.nextTag(StaEDIStreamReader.java:205)
```

**To Reproduce**
```
    EDIInputFactory factory = EDIInputFactory.newFactory();
    try (EDIStreamReader reader = factory.createEDIStreamReader(ediInputStream)) {
      while (reader.hasNext()) {
        reader.nextTag();
       }
     }
```
With the attached data file used in the `ediInputStream`

**Additional context**
[80char-edi210.txt](https://github.com/xlate/staedi/files/5311925/80char-edi210.txt)



Pull request:
Option to ignore extraneous input characters
Fixes #128 

Patch:
diff --git a/pom.xml b/pom.xml
index 9ce976d..59b6034 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
 
   <groupId>io.xlate</groupId>
   <artifactId>staedi</artifactId>
-  <version>1.12.1-SNAPSHOT</version>
+  <version>1.13.0-SNAPSHOT</version>
 
   <name>StAEDI : Streaming API for EDI for Java</name>
   <description>Streaming API for EDI for Java</description>
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java
index a5b0947..4fcc6c3 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java
@@ -38,6 +38,7 @@ public class StaEDIInputFactory extends EDIInputFactory {
         supportedProperties.add(EDI_VALIDATE_CONTROL_STRUCTURE);
         supportedProperties.add(EDI_VALIDATE_CONTROL_CODE_VALUES);
         supportedProperties.add(XML_DECLARE_TRANSACTION_XMLNS);
+        supportedProperties.add(EDI_IGNORE_EXTRANEOUS_CHARACTERS);
     }
 
     @Override
@@ -52,13 +53,14 @@ public class StaEDIInputFactory extends EDIInputFactory {
 
     @Override
     public EDIStreamReader createEDIStreamReader(InputStream stream, Schema schema) {
+        Objects.requireNonNull(stream, "stream must not be null");
         return new StaEDIStreamReader(stream, StandardCharsets.UTF_8, schema, properties, getErrorReporter());
     }
 
     @SuppressWarnings("resource")
     @Override
     public EDIStreamReader createEDIStreamReader(InputStream stream, String encoding, Schema schema) throws EDIStreamException {
-        Objects.requireNonNull(stream);
+        Objects.requireNonNull(stream, "stream must not be null");
 
         if (Charset.isSupported(encoding)) {
             return new StaEDIStreamReader(stream, Charset.forName(encoding), schema, properties, getErrorReporter());
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java
index efb0136..a561bee 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java
@@ -69,7 +69,7 @@ public class StaEDIStreamReader implements EDIStreamReader {
         this.properties = new HashMap<>(properties);
         this.reporter = reporter;
         this.proxy = new ProxyEventHandler(location, this.controlSchema);
-        this.lexer = new Lexer(stream, charset, proxy, location);
+        this.lexer = new Lexer(stream, charset, proxy, location, ignoreExtraneousCharacters());
     }
 
     private void ensureOpen() {
@@ -472,6 +472,10 @@ public class StaEDIStreamReader implements EDIStreamReader {
         return getBooleanProperty(EDIInputFactory.EDI_VALIDATE_CONTROL_STRUCTURE, true);
     }
 
+    boolean ignoreExtraneousCharacters() {
+        return getBooleanProperty(EDIInputFactory.EDI_IGNORE_EXTRANEOUS_CHARACTERS, false);
+    }
+
     boolean getBooleanProperty(String propertyName, boolean defaultValue) {
         Object property = properties.get(propertyName);
 
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
index c8278fb..086db38 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
@@ -174,10 +174,16 @@ public class CharacterSet {
 
     private final CharacterClass[] list;
     private final Map<Integer, CharacterClass> auxilary;
+    private final boolean extraneousIgnored;
 
     public CharacterSet() {
-        list = Arrays.copyOf(prototype, prototype.length);
-        auxilary = new TreeMap<>();
+        this(false);
+    }
+
+    public CharacterSet(boolean extraneousIgnored) {
+        this.list = Arrays.copyOf(prototype, prototype.length);
+        this.auxilary = new TreeMap<>();
+        this.extraneousIgnored = extraneousIgnored;
     }
 
     public CharacterClass getClass(int character) {
@@ -209,6 +215,17 @@ public class CharacterSet {
         }
     }
 
+    public boolean isIgnored(int character) {
+        switch (getClass(character)) {
+        case CONTROL:
+        case INVALID:
+        case WHITESPACE:
+            return extraneousIgnored;
+        default:
+            return false;
+        }
+    }
+
     public boolean isCharacterClass(int character, CharacterClass clazz) {
         return getClass(character).equals(clazz);
     }
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
index 838a5f9..e095b55 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
@@ -146,17 +146,25 @@ public class EDIFACTDialect extends Dialect {
 
     @Override
     public boolean appendHeader(CharacterSet characters, char value) {
+        boolean proceed = true;
+
         if (++index == 0) {
             header = new StringBuilder();
         }
 
-        header.append(value);
-
         if (UNB.equals(headerTag)) {
-            return processInterchangeHeader(characters, value);
+            if (characters.isIgnored(value)) {
+                index--;
+            } else {
+                header.append(value);
+                proceed = processInterchangeHeader(characters, value);
+            }
+        } else {
+            header.append(value);
+            proceed = processServiceStringAdvice(characters, value);
         }
 
-        return processServiceStringAdvice(characters, value);
+        return proceed;
     }
 
     boolean processInterchangeHeader(CharacterSet characters, char value) {
@@ -177,6 +185,8 @@ public class EDIFACTDialect extends Dialect {
     }
 
     boolean processServiceStringAdvice(CharacterSet characters, char value) {
+        boolean proceed = true;
+
         switch (index) {
         case 3:
             componentDelimiter = value;
@@ -206,10 +216,12 @@ public class EDIFACTDialect extends Dialect {
         }
 
         if (index > EDIFACT_UNA_LENGTH) {
-            if (unbStart > -1 && (index - unbStart) > 3) {
+            if (characters.isIgnored(value)) {
+                header.deleteCharAt(index--);
+            } else if (isIndexBeyondUNBFirstElement()) {
                 if (value == elementDelimiter) {
                     rejected = !initialize(characters);
-                    return isConfirmed();
+                    proceed = isConfirmed();
                 }
             } else if (value == 'B') {
                 CharSequence un = header.subSequence(index - 2, index);
@@ -218,15 +230,23 @@ public class EDIFACTDialect extends Dialect {
                     unbStart = index - 2;
                 } else {
                     // Some other segment / element?
-                    return false;
+                    proceed = false;
                 }
-            } else if (unbStart < 0 && value == elementDelimiter) {
+            } else if (isUnexpectedSegmentDetected(value)) {
                 // Some other segment / element?
-                return false;
+                proceed = false;
             }
         }
 
-        return true;
+        return proceed;
+    }
+
+    boolean isIndexBeyondUNBFirstElement() {
+        return unbStart > -1 && (index - unbStart) > 3;
+    }
+
+    boolean isUnexpectedSegmentDetected(int value) {
+        return unbStart < 0 && value == elementDelimiter;
     }
 
     void setCharacterClass(CharacterSet characters, CharacterClass charClass, char value, boolean allowSpace) {
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
index 9b29b63..d6bcdda 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
@@ -62,8 +62,7 @@ public class Lexer {
     private ByteBuffer readByteBuf = ByteBuffer.allocate(4);
 
     private final StaEDIStreamLocation location;
-
-    private CharacterSet characters = new CharacterSet();
+    private final CharacterSet characters;
     private CharBuffer buffer = CharBuffer.allocate(4096);
     private Dialect dialect;
 
@@ -79,7 +78,7 @@ public class Lexer {
     private Notifier en;
     private Notifier bn;
 
-    public Lexer(InputStream stream, Charset charset, EventHandler handler, StaEDIStreamLocation location) {
+    public Lexer(InputStream stream, Charset charset, EventHandler handler, StaEDIStreamLocation location, boolean extraneousIgnored) {
         if (stream.markSupported()) {
             this.stream = stream;
         } else {
@@ -89,6 +88,7 @@ public class Lexer {
         this.decoder = charset.newDecoder();
 
         this.location = location;
+        this.characters = new CharacterSet(extraneousIgnored);
 
         isn = (notifyState, start, length) -> {
             handler.interchangeBegin(dialect);
@@ -206,13 +206,17 @@ public class Lexer {
             case TRAILER_TAG_N:
             case TRAILER_TAG_Z:
             case ELEMENT_DATA:
-            case ELEMENT_INVALID_DATA:
             case TRAILER_ELEMENT_DATA:
                 buffer.put((char) input);
                 break;
-            case HEADER_TAG_1:
-            case HEADER_TAG_2:
-            case HEADER_TAG_3:
+            case ELEMENT_INVALID_DATA:
+                if (!characters.isIgnored(input)) {
+                    buffer.put((char) input);
+                }
+                break;
+            case HEADER_TAG_1: // U - When UNA is present
+            case HEADER_TAG_2: // N - When UNA is present
+            case HEADER_TAG_3: // B - When UNA is present
                 handleStateHeaderTag(input);
                 break;
             case DATA_RELEASE:
@@ -222,6 +226,7 @@ public class Lexer {
                 handleStateElementDataBinary();
                 break;
             case INTERCHANGE_CANDIDATE:
+                // ISA, UNA, or UNB was found
                 handleStateInterchangeCandidate(input);
                 break;
             case HEADER_DATA:
@@ -272,7 +277,9 @@ public class Lexer {
                 eventsReady = nextEvent();
                 break;
             default:
-                if (clazz != CharacterClass.INVALID) {
+                if (characters.isIgnored(input)) {
+                    state = previous;
+                } else if (clazz != CharacterClass.INVALID) {
                     throw invalidStateError();
                 } else {
                     throw error(EDIException.INVALID_CHARACTER);
@@ -380,7 +387,7 @@ public class Lexer {
         case RELEASE_CHARACTER:
             break;
         default:
-            if (dialect.getDecimalMark() != input) {
+            if (dialect.getDecimalMark() != input && !characters.isIgnored(input)) {
                 buffer.put((char) input);
             }
             break;
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/X12Dialect.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/X12Dialect.java
index df4b4fd..53883ed 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/X12Dialect.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/X12Dialect.java
@@ -148,31 +148,36 @@ public class X12Dialect extends Dialect {
     public boolean appendHeader(CharacterSet characters, char value) {
         index++;
 
-        if (index < X12_ISA_LENGTH) {
-            switch (index) {
-            case 0:
-                header = new char[X12_ISA_LENGTH];
-                break;
-            case X12_ELEMENT_OFFSET:
-                elementDelimiter = value;
-                characters.setClass(elementDelimiter, CharacterClass.ELEMENT_DELIMITER);
-                break;
-            default:
-                break;
+        switch (index) {
+        case 0:
+            header = new char[X12_ISA_LENGTH];
+            break;
+        case X12_ELEMENT_OFFSET:
+            elementDelimiter = value;
+            characters.setClass(elementDelimiter, CharacterClass.ELEMENT_DELIMITER);
+            break;
+        case X12_REPEAT_OFFSET:
+        case X12_COMPONENT_OFFSET:
+        case X12_SEGMENT_OFFSET:
+            break;
+        default:
+            if (characters.isIgnored(value)) {
+                // Discard control character if not used as a delimiter
+                index--;
+                return true;
             }
+            break;
+        }
 
-            header[index] = value;
+        header[index] = value;
+        boolean proceed = true;
 
-            if (index == X12_ISA_LENGTH - 1) {
-                rejected = !initialize(characters);
-                return isConfirmed();
-            }
-        } else {
-            rejected = true;
-            return false;
+        if (index == X12_SEGMENT_OFFSET) {
+            rejected = !initialize(characters);
+            proceed = isConfirmed();
         }
 
-        return true;
+        return proceed;
     }
 
     void clearTransactionVersion() {
diff --git a/src/main/java/io/xlate/edi/stream/EDIInputFactory.java b/src/main/java/io/xlate/edi/stream/EDIInputFactory.java
index f41d1a5..b923790 100644
--- a/src/main/java/io/xlate/edi/stream/EDIInputFactory.java
+++ b/src/main/java/io/xlate/edi/stream/EDIInputFactory.java
@@ -39,6 +39,14 @@ public abstract class EDIInputFactory extends PropertySupport {
      */
     public static final String XML_DECLARE_TRANSACTION_XMLNS = "io.xlate.edi.stream.XML_DECLARE_TRANSACTION_XMLNS";
 
+    /**
+     * When set to true, non-graphical, control characters will be ignored in the EDI
+     * input stream. This includes characters ranging from 0x00 through 0x1F and 0x7F.
+     *
+     * @since 1.13
+     */
+    public static final String EDI_IGNORE_EXTRANEOUS_CHARACTERS = "io.xlate.edi.stream.EDI_IGNORE_EXTRANEOUS_CHARACTERS";
+
     /**
      * Create a new instance of the factory. This static method creates a new
      * factory instance.


Test patch:
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderExtraneousCharsTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderExtraneousCharsTest.java
new file mode 100644
index 0000000..9672fdd
--- /dev/null
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderExtraneousCharsTest.java
@@ -0,0 +1,118 @@
+/*******************************************************************************
+ * Copyright 2020 xlate.io LLC, http://www.xlate.io
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ ******************************************************************************/
+package io.xlate.edi.internal.stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import io.xlate.edi.stream.EDIInputFactory;
+import io.xlate.edi.stream.EDIStreamReader;
+
+class StaEDIStreamReaderExtraneousCharsTest {
+
+    Logger LOGGER = Logger.getGlobal();
+
+    @SuppressWarnings("unchecked")
+    List<Object>[] readFully(EDIInputFactory factory, String resource) throws IOException {
+        EDIStreamReader reader = factory.createEDIStreamReader(getClass().getResourceAsStream(resource));
+        List<Object> expected = new ArrayList<>();
+        List<Object> unexpected = new ArrayList<>();
+
+        try {
+            while (reader.hasNext()) {
+                switch (reader.next()) {
+                case START_INTERCHANGE:
+                    expected.add(reader.getEventType());
+                    break;
+                case SEGMENT_ERROR:
+                case ELEMENT_OCCURRENCE_ERROR:
+                case ELEMENT_DATA_ERROR:
+                    LOGGER.log(Level.WARNING,
+                               () -> reader.getErrorType() + ", " + reader.getLocation() + ", data: [" + reader.getText() + "]");
+                    unexpected.add(reader.getErrorType());
+                    break;
+                case START_GROUP:
+                case START_TRANSACTION:
+                case START_LOOP:
+                case START_SEGMENT:
+                case END_GROUP:
+                case END_TRANSACTION:
+                case END_LOOP:
+                case END_SEGMENT:
+                case ELEMENT_DATA:
+                    expected.add(reader.getEventType() + " [" + reader.getText() + "]");
+                    break;
+                default:
+                    expected.add(reader.getEventType() + ", " + reader.getLocation());
+                    break;
+                }
+            }
+        } catch (Exception e) {
+            unexpected.add(e);
+            e.printStackTrace();
+        } finally {
+            reader.close();
+        }
+
+        return new List[] { expected, unexpected };
+    }
+
+    /**
+     * Original issue: https://github.com/xlate/staedi/issues/128
+     *
+     * @throws Exception
+     */
+    @ParameterizedTest
+    @ValueSource(
+            strings = {
+                        "/x12/issue128/ts210_80char.edi",
+                        "/EDIFACT/issue128/wrapped_invoic_d97b_una.edi",
+                        "/EDIFACT/issue128/wrapped_invoic_d97b.edi"
+            })
+    void testExtraneousCharactersIgnoredWithoutError(String resourceName) throws Exception {
+        EDIInputFactory factory = EDIInputFactory.newFactory();
+        factory.setProperty(EDIInputFactory.EDI_IGNORE_EXTRANEOUS_CHARACTERS, "true");
+        List<Object>[] results = readFully(factory, resourceName);
+        List<Object> unexpected = results[1];
+        assertEquals(0, unexpected.size(), () -> "Expected none, but got: " + unexpected);
+    }
+
+    @ParameterizedTest
+    @CsvSource({
+        "/EDIFACT/issue128/wrapped_invoic_d97b_una.edi, /EDIFACT/invoic_d97b_una.edi",
+        "/EDIFACT/issue128/wrapped_invoic_d97b.edi, /EDIFACT/invoic_d97b.edi"
+    })
+    void testExtraneousCharactersRemovedMatchesOriginal(String wrapped, String original) throws Exception {
+        EDIInputFactory factory = EDIInputFactory.newFactory();
+        List<Object>[] results1 = readFully(factory, original);
+
+        factory.setProperty(EDIInputFactory.EDI_IGNORE_EXTRANEOUS_CHARACTERS, "true");
+        List<Object>[] results2 = readFully(factory, wrapped);
+
+        assertEquals(0, results1[1].size(), () -> "Expected none, but got: " + results1[1]);
+        assertEquals(results1[0], results2[0]);
+        assertEquals(results1[1], results2[1]);
+    }
+}
diff --git a/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java b/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
index 3e7b07c..98441c2 100644
--- a/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
@@ -201,7 +201,7 @@ class LexerTest {
         element = null;
 
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, handler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, handler, location, false);
 
         lexer.parse();
         assertTrue(interchangeStarted, "Interchange not started");
@@ -222,7 +222,7 @@ class LexerTest {
         element = null;
 
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, handler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, handler, location, false);
 
         lexer.parse();
         assertTrue(interchangeStarted, "Interchange not started");
@@ -239,7 +239,7 @@ class LexerTest {
         InputStream stream = getClass().getResourceAsStream("/x12/simple997.edi");
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         String last;
         int s = -1;
 
@@ -267,7 +267,7 @@ class LexerTest {
         InputStream stream = getClass().getResourceAsStream("/EDIFACT/invoic_d97b_una.edi");
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         String last;
         int s = -1;
 
@@ -294,7 +294,7 @@ class LexerTest {
         InputStream stream = getClass().getResourceAsStream("/EDIFACT/invoic_d97b.edi");
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         String last;
         int s = -1;
 
@@ -321,7 +321,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream("ISA*00?          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~".getBytes());
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         EDIException thrown = assertThrows(EDIException.class, lexer::parse);
         assertTrue(thrown.getMessage().contains("EDIE003"));
     }
@@ -333,7 +333,7 @@ class LexerTest {
                 + "TA\u0008").getBytes()); // Backspace char in segment tag
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         for (int i = 0; i < 19; i++) {
             lexer.parse(); // Interchange start through end of ISA
         }
@@ -347,7 +347,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream("ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~".getBytes());
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         for (int i = 0; i < 19; i++) {
             lexer.parse(); // Interchange start through end of ISA
         }
@@ -365,7 +365,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream(data2);
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location, false);
         for (int i = 0; i < 20; i++) {
             lexer.parse(); // Interchange start through end of ISA + GS start tag
         }
@@ -379,7 +379,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream("ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~😀".getBytes());
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, StandardCharsets.US_ASCII, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.US_ASCII, eventHandler, location, false);
         for (int i = 0; i < 19; i++) {
             lexer.parse(); // Interchange start through end of ISA
         }
diff --git a/src/test/resources/EDIFACT/issue128/wrapped_invoic_d97b.edi b/src/test/resources/EDIFACT/issue128/wrapped_invoic_d97b.edi
new file mode 100644
index 0000000..8f08a5b
--- /dev/null
+++ b/src/test/resources/EDIFACT/issue128/wrapped_invoic_d97b.edi
@@ -0,0 +1,11 @@
+UNB+UNOA:3+005435656:1+006415160:1+060515:1434+000
+00000000778'UNH+00000000000117+INVOIC:D:97B:UN'BGM
++380+342459+9'DTM+3:20060515:102'RFF+ON:521052'NAD
++BY+792820524::16++CUMMINS MID-RANGE ENGINE PLANT'
+NAD+SE+005435656::16++BÜTTNER WIDGET COMPANY'CUX+1
+:USD'LIN+1++157870:IN'IMD+F++:::WIDGET'QTY+47:1020
+:EA'ALI+US'MOA+203:1202.58'PRI+INV:1.179'LIN+2++15
+7871:IN'IMD+F++:::DIFFERENT WIDGET'QTY+47:20:EA'AL
+I+JP'MOA+203:410'PRI+INV:20.5'UNS+S'MOA+39:2137.58
+'ALC+C+ABG'MOA+8:525'UNT+24+00000000000117'UNZ+1+0
+0000000000778'
\ No newline at end of file
diff --git a/src/test/resources/EDIFACT/issue128/wrapped_invoic_d97b_una.edi b/src/test/resources/EDIFACT/issue128/wrapped_invoic_d97b_una.edi
new file mode 100644
index 0000000..8d393ab
--- /dev/null
+++ b/src/test/resources/EDIFACT/issue128/wrapped_invoic_d97b_una.edi
@@ -0,0 +1,35 @@
+UNA=*.? ~UNB*UN
+OA=3*005435656=
+1*006?415160=1*
+060515=1434*000
+00000000778~UNH
+*00000000000117
+*INVOIC=D=97B=U
+N~BGM*380*34245
+9*9~DTM*3=20060
+515=102~RFF*ON=
+521052~NAD*BY*7
+92820524==16**C
+UMMINS MID-RANG
+E ENGINE PLANT~
+NAD*SE*00543565
+6==16**GENERAL 
+WIDGET COMPANY~
+CUX*1=USD~LIN*1
+**157870=IN~IMD
+*F**===WIDGET~Q
+TY*47=1020=EA~A
+LI*US~MOA*203=1
+202.58~PRI*INV=
+1.179~LIN*2**15
+7871=IN~IMD*F**
+===DIFFERENT WI
+DGET~QTY*47=20=
+EA~ALI*JP~MOA*2
+03=410~PRI*INV=
+20.5~UNS*S~MOA*
+39=2137.58~ALC*
+C*ABG~MOA*8=525
+~UNT*23*0000000
+0000117~UNZ*1*0
+0000000000778~
\ No newline at end of file
diff --git a/src/test/resources/x12/issue128/ts210_80char.edi b/src/test/resources/x12/issue128/ts210_80char.edi
new file mode 100644
index 0000000..0f319c4
--- /dev/null
+++ b/src/test/resources/x12/issue128/ts210_80char.edi
@@ -0,0 +1,11 @@
+ISA*00*          *00*          *ZZ*DDDD           *ZZ*XXXXXX         *200918*022
+4*U*00401*000026003*0*P*>~GS*IM*DDDD*XXXXXX*20180918*0224*2619*X*004010~ST*210*1
+305~B3**699751522*134982*PP**20180914*12417**20180915*035*DDDD*20180914~N9*PO*72
+1295~N9*PO*721295~N9*PO*109483~N1*SH*UNDERFLEX GRP 7~N3*251 INDUSTRIAL DR EXT~N4
+*WILKESBORO*NC*286977576*US~N1*CN*GAME FARMS~N3*791 PECAN ST~N4*DUDLEY*NC*283335
+369*US~N1*BT*UNDERFLEX NURRELL~N2*C/O XXXXXX~N3*PO BOX 2545~N4*FAIRFAX*VA*220302
+545*US~LX*1~L5*1*PLT  PL FILM/SHEET 15-30P FOOD GRADE  156830-3 CLA~L5*1*SS 775~
+L0*1***1340*G***1*PLT**L~L1*1*41.93*PH*56186~L7*1*DDDD*0912****77.5~LX*2~L5*2*XP
+O LOGISTICS DISCOUNT SAVES YOU~L1*2***-44949****DSC~L7*2*DDDD*0912~LX*3~L5*3*FSC
+ FUEL SURCHARGE 10.50%~L1*3*11.8*FR*1180****FUE~L7*3*DDDD*110~L3*1340*G***12417*
+*****1*L~SE*31*1305~GE*1*2619~IEA*1*000026003~
\ No newline at end of file