Repository: xlate/staedi (java)

Issue:
Ignore repetition separator
PADIS EDIFACT from IATA says ([source](https://www.iata.org/contentassets/18a5fdb2dc144d619a8c10dc1472ae80/pnrgov20edifact20implementation20guide2016_1.pdf)):

- Repetition separator is not used in PNRGOV messages and therefore the default * separator does not need to be released.

So, our clients are sending files like:
```
UNA:+.?*'
...
LTS+0/O/SS/NT6604 Y 21FEB 5 LPADSS LK4 1200 1415/NN *1A/E*  /NT/ES/C/I/CAB Y//0////'
...
UNZ+1+0001'
```
Where those asterisks are content, not separators :( 

¿Would it be possible to support a setting to ignore the repetition separator?


Pull request description:
Allow EDIFACT element repetition only in version 4 and above
Fixes #77 

- Suppress output of repetition character prior to version 4, per
EDIFACT standard

Patch:
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java
index 7b7e2b3..004a979 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java
@@ -78,6 +78,7 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
     private final Map<String, Object> properties;
     private final EDIOutputErrorReporter reporter;
     private Dialect dialect;
+    CharBuffer unconfirmedBuffer = CharBuffer.allocate(500);
 
     private final StaEDIStreamLocation location;
     private Schema controlSchema;
@@ -138,9 +139,9 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
         segmentTerminator = getDelimiter(properties, Delimiters.SEGMENT, dialect::getSegmentTerminator);
         dataElementSeparator = getDelimiter(properties, Delimiters.DATA_ELEMENT, dialect::getDataElementSeparator);
         componentElementSeparator = getDelimiter(properties, Delimiters.COMPONENT_ELEMENT, dialect::getComponentElementSeparator);
-        repetitionSeparator = getDelimiter(properties, Delimiters.REPETITION, dialect::getRepetitionSeparator);
         decimalMark = getDelimiter(properties, Delimiters.DECIMAL, dialect::getDecimalMark);
         releaseIndicator = getDelimiter(properties, Delimiters.RELEASE, dialect::getReleaseIndicator);
+        repetitionSeparator = getDelimiter(properties, Delimiters.REPETITION, dialect::getRepetitionSeparator);
 
         String lineSeparator = System.getProperty("line.separator");
 
@@ -163,11 +164,11 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
                      .anyMatch(properties::containsKey);
     }
 
-    static char getDelimiter(Map<String, Object> properties, String key, Supplier<Character> defaultSupplier) {
-        if (properties.containsKey(key)) {
+    char getDelimiter(Map<String, Object> properties, String key, Supplier<Character> dialectSupplier) {
+        if (properties.containsKey(key) && !dialect.isConfirmed()) {
             return (char) properties.get(key);
         }
-        return defaultSupplier.get();
+        return dialectSupplier.get();
     }
 
     static void putDelimiter(String key, char value, Map<String, Character> delimiters) {
@@ -302,44 +303,67 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
         state = state.transition(clazz);
 
         switch (state) {
-        case HEADER_TAG_I:
+        case HEADER_TAG_I: // I(SA)
+        case HEADER_TAG_U: // U(NA) or U(NB)
+            unconfirmedBuffer.clear();
+            writeHeader(output);
+            break;
         case HEADER_TAG_S:
-        case HEADER_TAG_U:
         case HEADER_TAG_N:
-            //case HEADER_TAG_A:
         case INTERCHANGE_CANDIDATE:
         case HEADER_DATA:
         case HEADER_ELEMENT_END:
         case HEADER_COMPONENT_END:
-            if (dialect.appendHeader(characters, (char) output)) {
-                if (dialect.isConfirmed()) {
-                    // Set up the delimiters again once the dialect has confirmed them
-                    setupDelimiters();
-
-                    switch (state) {
-                    case HEADER_DATA:
-                        state = State.TAG_SEARCH;
-                        break;
-                    case HEADER_ELEMENT_END:
-                        state = State.ELEMENT_END;
-                        break;
-                    case HEADER_COMPONENT_END:
-                        state = State.COMPONENT_END;
-                        break;
-                    default:
-                        throw new IllegalStateException("Confirmed at state " + state);
-                    }
-                }
-            } else {
-                throw new EDIStreamException(String.format("Unexpected header character: 0x%04X [%s]", output, (char) output), location);
-            }
+            writeHeader(output);
             break;
         case INVALID:
             throw new EDIException(String.format("Invalid state: %s; output 0x%04X", state, output));
         default:
+            writeOutput(output);
             break;
         }
+    }
+
+    void writeHeader(int output) throws EDIStreamException {
+        if (!dialect.appendHeader(characters, (char) output)) {
+            throw new EDIStreamException(String.format("Unexpected header character: 0x%04X [%s]", output, (char) output), location);
+        }
+
+        unconfirmedBuffer.append((char) output);
+
+        if (dialect.isConfirmed()) {
+            // Set up the delimiters again once the dialect has confirmed them
+            setupDelimiters();
+
+            // Switching to non-header states to proceed after dialect is confirmed
+            switch (state) {
+            case HEADER_DATA:
+                state = State.TAG_SEARCH;
+                break;
+            case HEADER_ELEMENT_END:
+                state = State.ELEMENT_END;
+                break;
+            case HEADER_COMPONENT_END:
+                state = State.COMPONENT_END;
+                break;
+            default:
+                throw new IllegalStateException("Confirmed at state " + state);
+            }
+
+            unconfirmedBuffer.flip();
+
+            if (EDIFACTDialect.UNA.equals(dialect.getHeaderTag())) {
+                // Overlay the UNA segment repetition separator now that it has be confirmed
+                unconfirmedBuffer.put(7, this.repetitionSeparator > 0 ? this.repetitionSeparator : ' ');
+            }
+
+            while (unconfirmedBuffer.hasRemaining()) {
+                writeOutput(unconfirmedBuffer.get());
+            }
+        }
+    }
 
+    void writeOutput(int output) throws EDIStreamException {
         try {
             location.incrementOffset(output);
             writer.write(output);
@@ -422,6 +446,7 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
         write(this.dataElementSeparator);
         write(this.decimalMark);
         write(this.releaseIndicator);
+        // This will be re-written once the dialect version is detected
         write(this.repetitionSeparator);
     }
 
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
index 207e942..4e18714 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
@@ -24,6 +24,14 @@ public class EDIFACTDialect extends Dialect {
     public static final String UNB = "UNB";
 
     private static final String[] EMPTY = new String[0];
+
+    static final char DFLT_SEGMENT_TERMINATOR = '\'';
+    static final char DFLT_DATA_ELEMENT_SEPARATOR = '+';
+    static final char DFLT_COMPONENT_ELEMENT_SEPARATOR = ':';
+    static final char DFLT_REPETITION_SEPARATOR = '*';
+    static final char DFLT_RELEASE_CHARACTER = '?';
+    static final char DFLT_DECIMAL_MARK = '.';
+
     private static final int EDIFACT_UNA_LENGTH = 9;
 
     private String headerTag;
@@ -42,12 +50,12 @@ public class EDIFACTDialect extends Dialect {
     private String transactionVersionString;
 
     EDIFACTDialect() {
-        componentDelimiter = ':';
-        elementDelimiter = '+';
-        decimalMark = '.';
-        releaseIndicator = '?';
-        elementRepeater = '*';
-        segmentDelimiter = '\'';
+        componentDelimiter = DFLT_COMPONENT_ELEMENT_SEPARATOR;
+        elementDelimiter = DFLT_DATA_ELEMENT_SEPARATOR;
+        decimalMark = DFLT_DECIMAL_MARK;
+        releaseIndicator = DFLT_RELEASE_CHARACTER;
+        elementRepeater = DFLT_REPETITION_SEPARATOR;
+        segmentDelimiter = DFLT_SEGMENT_TERMINATOR;
 
         clearTransactionVersion();
     }
@@ -65,19 +73,22 @@ public class EDIFACTDialect extends Dialect {
     boolean initialize(CharacterSet characters) {
         String[] parsedVersion = parseVersion();
 
-        if (parsedVersion.length > 0) {
+        if (parsedVersion.length > 1) {
             this.version = parsedVersion;
+            final String syntaxVersion = this.version[1];
 
             characters.setClass(componentDelimiter, CharacterClass.COMPONENT_DELIMITER);
             characters.setClass(elementDelimiter, CharacterClass.ELEMENT_DELIMITER);
 
-            if (releaseIndicator != ' ') {
+            if (syntaxVersion.compareTo("4") >= 0 || releaseIndicator != ' ') {
+                // Must not be blank for version 4 and above, may be blank before version 4 if not used
                 characters.setClass(releaseIndicator, CharacterClass.RELEASE_CHARACTER);
             } else {
                 releaseIndicator = '\0';
             }
 
-            if (elementRepeater != ' ') {
+            if (syntaxVersion.compareTo("4") >= 0) {
+                // Must not be blank for version 4 and above
                 characters.setClass(elementRepeater, CharacterClass.ELEMENT_REPEATER);
             } else {
                 elementRepeater = '\0';
@@ -151,14 +162,13 @@ public class EDIFACTDialect extends Dialect {
     boolean processInterchangeHeader(CharacterSet characters, char value) {
         if (index == 0) {
             characters.setClass(componentDelimiter, CharacterClass.COMPONENT_DELIMITER);
-            characters.setClass(releaseIndicator, CharacterClass.RELEASE_CHARACTER);
-            characters.setClass(elementRepeater, CharacterClass.ELEMENT_REPEATER);
-            return true;
         } else if (index == 3) {
+            /*
+             * Do not set the element delimiter until after the segment tag has been passed
+             * to prevent triggering an "element data" event prematurely.
+             */
             characters.setClass(elementDelimiter, CharacterClass.ELEMENT_DELIMITER);
-            return true;
         } else if (segmentDelimiter == value) {
-            characters.setClass(segmentDelimiter, CharacterClass.SEGMENT_DELIMITER);
             rejected = !initialize(characters);
             return isConfirmed();
         }
@@ -185,7 +195,7 @@ public class EDIFACTDialect extends Dialect {
             break;
         case 7:
             elementRepeater = value;
-            setCharacterClass(characters, CharacterClass.ELEMENT_REPEATER, value, false);
+            // Do not set the character class until initialize() is executed
             break;
         case 8:
             segmentDelimiter = value;


Test patch:
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
index 1e186ba..a5bc0bf 100644
--- a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
@@ -161,7 +161,7 @@ class StaEDIStreamReaderTest implements ConstantsTest {
         expected.put(Delimiters.SEGMENT, '\'');
         expected.put(Delimiters.DATA_ELEMENT, '+');
         expected.put(Delimiters.COMPONENT_ELEMENT, ':');
-        expected.put(Delimiters.REPETITION, '*');
+        // expected.put(Delimiters.REPETITION, '*'); Input is version 3. No repetition character until version 4
         expected.put(Delimiters.RELEASE, '?');
         expected.put(Delimiters.DECIMAL, '.');
 
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
index fade85f..cd9ba7c 100644
--- a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
@@ -78,6 +78,13 @@ class StaEDIStreamWriterTest {
         writer.writeEndSegment();
     }
 
+    static void unconfirmedBufferEquals(String expected, EDIStreamWriter writer) {
+        StaEDIStreamWriter writerImpl = (StaEDIStreamWriter) writer;
+        writerImpl.unconfirmedBuffer.mark();
+        writerImpl.unconfirmedBuffer.flip();
+        assertEquals(expected, writerImpl.unconfirmedBuffer.toString());
+    }
+
     @Test
     void testGetProperty() {
         EDIOutputFactory factory = EDIOutputFactory.newFactory();
@@ -178,7 +185,7 @@ class StaEDIStreamWriterTest {
         writer.startInterchange();
         writer.writeStartSegment("ISA");
         writer.flush();
-        assertEquals("ISA", stream.toString());
+        unconfirmedBufferEquals("ISA", writer);
     }
 
     @Test
@@ -224,7 +231,7 @@ class StaEDIStreamWriterTest {
         writer.writeStartElement().writeElementData("E1").endElement();
         writer.writeEndSegment();
         writer.flush();
-        assertEquals("ISA*E1~", stream.toString());
+        unconfirmedBufferEquals("ISA*E1~", writer);
     }
 
     @Test
@@ -252,7 +259,7 @@ class StaEDIStreamWriterTest {
               .writeStartElement()
               .endElement();
         writer.flush();
-        assertEquals("ISA****", stream.toString());
+        unconfirmedBufferEquals("ISA****", writer);
     }
 
     @Test
@@ -355,7 +362,7 @@ class StaEDIStreamWriterTest {
               .startComponent()
               .writeEndSegment();
         writer.flush();
-        assertEquals("ISA*:~", stream.toString());
+        unconfirmedBufferEquals("ISA*:~", writer);
     }
 
     @Test
@@ -375,7 +382,8 @@ class StaEDIStreamWriterTest {
               .writeEmptyElement()
               .writeEndSegment();
         writer.flush();
-        assertEquals("ISA~", stream.toString());
+
+        unconfirmedBufferEquals("ISA~", writer);
     }
 
     @Test
@@ -422,7 +430,7 @@ class StaEDIStreamWriterTest {
         writer.writeEmptyElement();
         writer.writeEndSegment();
         writer.flush();
-        assertEquals("ISA****~", stream.toString());
+        unconfirmedBufferEquals("ISA****~", writer);
     }
 
     @Test
@@ -439,7 +447,7 @@ class StaEDIStreamWriterTest {
         writer.writeEmptyComponent();
         writer.writeEndSegment();
         writer.flush();
-        assertEquals("ISA*:::~", stream.toString());
+        unconfirmedBufferEquals("ISA*:::~", writer);
     }
 
     @Test
@@ -458,7 +466,7 @@ class StaEDIStreamWriterTest {
         writer.endElement();
         writer.writeEndSegment();
         writer.flush();
-        assertEquals("ISA~", stream.toString());
+        unconfirmedBufferEquals("ISA~", writer);
     }
 
     @Test
@@ -518,7 +526,7 @@ class StaEDIStreamWriterTest {
         writer.writeEndSegment();
         writer.flush();
 
-        assertEquals("ISA**:LAST*:::LAST*:SECOND::LAST***LAST~", stream.toString());
+        unconfirmedBufferEquals("ISA**:LAST*:::LAST*:SECOND::LAST***LAST~", writer);
     }
 
     @Test
@@ -532,7 +540,7 @@ class StaEDIStreamWriterTest {
         writer.writeElementData("TEST-ELEMENT");
         writer.writeEndSegment();
         writer.flush();
-        assertEquals("ISA*TEST-ELEMENT~", stream.toString());
+        unconfirmedBufferEquals("ISA*TEST-ELEMENT~", writer);
     }
 
     @Test
@@ -558,7 +566,7 @@ class StaEDIStreamWriterTest {
         writer.writeElementData(new char[] { 'C', 'H', 'A', 'R', 'S' }, 0, 5);
         writer.writeEndSegment();
         writer.flush();
-        assertEquals("ISA*CHARS~", stream.toString());
+        unconfirmedBufferEquals("ISA*CHARS~", writer);
     }
 
     @Test
@@ -929,6 +937,7 @@ class StaEDIStreamWriterTest {
 
         EDIOutputFactory outputFactory = EDIOutputFactory.newFactory();
         outputFactory.setProperty(EDIOutputFactory.PRETTY_PRINT, true);
+        outputFactory.setProperty(Delimiters.REPETITION, '*');
         ByteArrayOutputStream result = new ByteArrayOutputStream(16384);
         EDIStreamWriter writer = null;
 
@@ -1710,7 +1719,53 @@ class StaEDIStreamWriterTest {
     }
 
     @Test
-    void testGetDelimitersEDIFACT_defaults() throws EDIStreamException {
+    void testGetDelimitersEDIFACT_defaults_v4() throws EDIStreamException {
+        EDIOutputFactory factory = EDIOutputFactory.newFactory();
+        ByteArrayOutputStream stream = new ByteArrayOutputStream(4096);
+        EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
+
+        writer.startInterchange();
+        // UNB+UNOA:4+005435656:1+006415160:1+060515:1434+00000000000778'
+        writer.writeStartSegment("UNB");
+
+        writer.writeStartElement();
+        writer.writeComponent("UNOA");
+        writer.writeComponent("4");
+        writer.endElement();
+
+        writer.writeStartElement();
+        writer.writeComponent("005435656");
+        writer.writeComponent("1");
+        writer.endElement();
+
+        writer.writeStartElement();
+        writer.writeComponent("006415160");
+        writer.writeComponent("1");
+        writer.endElement();
+
+        writer.writeStartElement();
+        writer.writeComponent("20060515");
+        writer.writeComponent("1434");
+        writer.endElement();
+
+        writer.writeElement("00000000000778");
+        writer.writeEndSegment();
+
+        Map<String, Character> delimiters = writer.getDelimiters();
+        assertEquals('\'', delimiters.get(Delimiters.SEGMENT));
+        assertEquals('+', delimiters.get(Delimiters.DATA_ELEMENT));
+        assertEquals(':', delimiters.get(Delimiters.COMPONENT_ELEMENT));
+        assertEquals('.', delimiters.get(Delimiters.DECIMAL));
+        assertEquals('?', delimiters.get(Delimiters.RELEASE));
+        assertEquals('*', delimiters.get(Delimiters.REPETITION));
+
+        writer.flush();
+        assertEquals("UNB+UNOA:4+005435656:1+006415160:1+20060515:1434+00000000000778'",
+                     new String(stream.toByteArray()));
+    }
+
+    @Test
+    void testGetDelimitersEDIFACT_defaults_v3() throws EDIStreamException {
         EDIOutputFactory factory = EDIOutputFactory.newFactory();
         ByteArrayOutputStream stream = new ByteArrayOutputStream(4096);
         EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
@@ -1748,7 +1803,7 @@ class StaEDIStreamWriterTest {
         assertEquals(':', delimiters.get(Delimiters.COMPONENT_ELEMENT));
         assertEquals('.', delimiters.get(Delimiters.DECIMAL));
         assertEquals('?', delimiters.get(Delimiters.RELEASE));
-        assertEquals('*', delimiters.get(Delimiters.REPETITION));
+        assertNull(delimiters.get(Delimiters.REPETITION)); // Not introduced until v4
 
         writer.flush();
         assertEquals("UNB+UNOA:3+005435656:1+006415160:1+060515:1434+00000000000778'",
@@ -1756,14 +1811,14 @@ class StaEDIStreamWriterTest {
     }
 
     @Test
-    void testGetDelimitersEDIFACT_customSegment() throws EDIStreamException {
+    void testGetDelimitersEDIFACT_noRelease_v3() throws EDIStreamException {
         EDIOutputFactory factory = EDIOutputFactory.newFactory();
+        factory.setProperty(Delimiters.RELEASE, ' ');
         ByteArrayOutputStream stream = new ByteArrayOutputStream(4096);
-        factory.setProperty(Delimiters.SEGMENT, '~');
         EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
 
         writer.startInterchange();
-        // UNB+UNOA:3+005435656:1+006415160:1+060515:1434+00000000000778~
+        // UNB+UNOA:3+005435656:1+006415160:1+060515:1434+00000000000778'
         writer.writeStartSegment("UNB");
 
         writer.writeStartElement();
@@ -1789,6 +1844,53 @@ class StaEDIStreamWriterTest {
         writer.writeElement("00000000000778");
         writer.writeEndSegment();
 
+        Map<String, Character> delimiters = writer.getDelimiters();
+        assertEquals('\'', delimiters.get(Delimiters.SEGMENT));
+        assertEquals('+', delimiters.get(Delimiters.DATA_ELEMENT));
+        assertEquals(':', delimiters.get(Delimiters.COMPONENT_ELEMENT));
+        assertEquals('.', delimiters.get(Delimiters.DECIMAL));
+        assertNull(delimiters.get(Delimiters.RELEASE));
+        assertNull(delimiters.get(Delimiters.REPETITION)); // Not introduced until v4
+
+        writer.flush();
+        assertEquals("UNA:+.  'UNB+UNOA:3+005435656:1+006415160:1+060515:1434+00000000000778'",
+                     new String(stream.toByteArray()));
+    }
+
+    @Test
+    void testGetDelimitersEDIFACT_customSegment() throws EDIStreamException {
+        EDIOutputFactory factory = EDIOutputFactory.newFactory();
+        ByteArrayOutputStream stream = new ByteArrayOutputStream(4096);
+        factory.setProperty(Delimiters.SEGMENT, '~');
+        EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
+
+        writer.startInterchange();
+        // UNB+UNOA:4+005435656:1+006415160:1+20060515:1434+00000000000778~
+        writer.writeStartSegment("UNB");
+
+        writer.writeStartElement();
+        writer.writeComponent("UNOA");
+        writer.writeComponent("4");
+        writer.endElement();
+
+        writer.writeStartElement();
+        writer.writeComponent("005435656");
+        writer.writeComponent("1");
+        writer.endElement();
+
+        writer.writeStartElement();
+        writer.writeComponent("006415160");
+        writer.writeComponent("1");
+        writer.endElement();
+
+        writer.writeStartElement();
+        writer.writeComponent("20060515");
+        writer.writeComponent("1434");
+        writer.endElement();
+
+        writer.writeElement("00000000000778");
+        writer.writeEndSegment();
+
         Map<String, Character> delimiters = writer.getDelimiters();
         assertEquals('~', delimiters.get(Delimiters.SEGMENT));
         assertEquals('+', delimiters.get(Delimiters.DATA_ELEMENT));
@@ -1798,7 +1900,7 @@ class StaEDIStreamWriterTest {
         assertEquals('*', delimiters.get(Delimiters.REPETITION));
 
         writer.flush();
-        assertEquals("UNA:+.?*~UNB+UNOA:3+005435656:1+006415160:1+060515:1434+00000000000778~",
+        assertEquals("UNA:+.?*~UNB+UNOA:4+005435656:1+006415160:1+20060515:1434+00000000000778~",
                      new String(stream.toByteArray()));
     }
 
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIXMLStreamWriterTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIXMLStreamWriterTest.java
index a34b368..8db9963 100644
--- a/src/test/java/io/xlate/edi/internal/stream/StaEDIXMLStreamWriterTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIXMLStreamWriterTest.java
@@ -1,6 +1,5 @@
 package io.xlate.edi.internal.stream;
 
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -17,9 +16,9 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
+import io.xlate.edi.stream.EDINamespaces;
 import io.xlate.edi.stream.EDIOutputFactory;
 import io.xlate.edi.stream.EDIStreamWriter;
-import io.xlate.edi.stream.EDINamespaces;
 
 class StaEDIXMLStreamWriterTest {
 
@@ -56,6 +55,13 @@ class StaEDIXMLStreamWriterTest {
 
     }
 
+    static void unconfirmedBufferEquals(String expected, EDIStreamWriter writer) {
+        StaEDIStreamWriter writerImpl = (StaEDIStreamWriter) writer;
+        writerImpl.unconfirmedBuffer.mark();
+        writerImpl.unconfirmedBuffer.flip();
+        assertEquals(expected, writerImpl.unconfirmedBuffer.toString());
+    }
+
     @Test
     void testWriteStartElementString() throws XMLStreamException {
         it.setPrefix("l", EDINamespaces.LOOPS);
@@ -65,7 +71,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("s:ISA");
         it.flush();
         it.close();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -76,7 +82,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("l:INTERCHANGE");
         it.writeStartElement("ISA");
         it.flush();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -102,7 +108,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement(EDINamespaces.LOOPS, "INTERCHANGE");
         it.writeStartElement(EDINamespaces.SEGMENTS, "ISA");
         it.flush();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -113,7 +119,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("e", "ISA01", EDINamespaces.ELEMENTS); // Test
         it.writeCharacters("00");
         it.flush();
-        assertArrayEquals("ISA*00".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA*00", ediWriter);
     }
 
     @Test
@@ -125,7 +131,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("e", "ISA02", EDINamespaces.ELEMENTS);
         it.writeCharacters("          ");
         it.flush();
-        assertArrayEquals("ISA**          ".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA**          ", ediWriter);
     }
 
     @Test
@@ -137,7 +143,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("e", "ISA02", EDINamespaces.ELEMENTS);
         it.writeCharacters("          ");
         it.flush();
-        assertArrayEquals("ISA**          ".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA**          ", ediWriter);
     }
 
     @Test
@@ -149,7 +155,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("e", "ISA02", EDINamespaces.ELEMENTS);
         it.writeCharacters("          ");
         it.flush();
-        assertArrayEquals("ISA**          ".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA**          ", ediWriter);
     }
 
     @Test
@@ -162,7 +168,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("e", "ISA02", EDINamespaces.ELEMENTS);
         it.writeCharacters("          ");
         it.flush();
-        assertArrayEquals("ISA**          ".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA**          ", ediWriter);
     }
 
     @Test
@@ -185,7 +191,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeCharacters("00");
         it.writeEndElement(); // Test
         it.flush();
-        assertArrayEquals("ISA*00".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA*00", ediWriter);
     }
 
     @Test
@@ -237,7 +243,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("s:ISA");
         it.flush();
         it.close();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -249,7 +255,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeStartElement("s:ISA");
         it.flush();
         it.close();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -259,7 +265,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeCData(" \t\n\r"); // Test
         it.writeStartElement("s", "ISA", EDINamespaces.SEGMENTS);
         it.flush();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -269,7 +275,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeCharacters(" \t\n\r"); // Test
         it.writeStartElement("s", "ISA", EDINamespaces.SEGMENTS);
         it.flush();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
@@ -287,7 +293,7 @@ class StaEDIXMLStreamWriterTest {
         it.writeCharacters(" \t \n \r  OUT OF INDEX BOUNDS".toCharArray(), 2, 6); // Test
         it.writeStartElement("s", "ISA", EDINamespaces.SEGMENTS);
         it.flush();
-        assertArrayEquals("ISA".getBytes(), stream.toByteArray());
+        unconfirmedBufferEquals("ISA", ediWriter);
     }
 
     @Test
diff --git a/src/test/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialectTest.java b/src/test/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialectTest.java
index 06904d8..b6dc173 100644
--- a/src/test/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialectTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialectTest.java
@@ -67,7 +67,7 @@ class EDIFACTDialectTest {
     }
 
     @Test
-    void testBlankReleaseChar() throws EDIException {
+    void testBlankReleaseCharPreVersion4() throws EDIException {
         EDIFACTDialect edifact = (EDIFACTDialect) DialectFactory.getDialect("UNA".toCharArray(), 0, 3);
         CharacterSet characters = new CharacterSet();
         "UNA:+. *'UNB+UNOA:1+111111111:1+222222222:1+200726:1455+1'".chars().forEach(c -> edifact.appendHeader(characters, (char) c));
@@ -77,10 +77,25 @@ class EDIFACTDialectTest {
         assertEquals('+', edifact.getDataElementSeparator());
         assertEquals(':', edifact.getComponentElementSeparator());
         assertEquals('.', edifact.getDecimalMark());
-        assertEquals('*', edifact.getRepetitionSeparator());
+        assertEquals('\0', edifact.getRepetitionSeparator());
         assertEquals('\0', edifact.getReleaseIndicator());
     }
 
+    @Test
+    void testBlankReleaseCharVersion4() throws EDIException {
+        EDIFACTDialect edifact = (EDIFACTDialect) DialectFactory.getDialect("UNA".toCharArray(), 0, 3);
+        CharacterSet characters = new CharacterSet();
+        "UNA:+. *'UNB+UNOA:4+111111111:1+222222222:1+200726:1455+1'".chars().forEach(c -> edifact.appendHeader(characters, (char) c));
+
+        assertTrue(edifact.initialize(characters));
+        assertEquals('\'', edifact.getSegmentTerminator());
+        assertEquals('+', edifact.getDataElementSeparator());
+        assertEquals(':', edifact.getComponentElementSeparator());
+        assertEquals('.', edifact.getDecimalMark());
+        assertEquals('*', edifact.getRepetitionSeparator());
+        assertEquals(' ', edifact.getReleaseIndicator());
+    }
+
     @Test
     void testBlankVersionUNA() throws EDIException {
         EDIFACTDialect edifact = (EDIFACTDialect) DialectFactory.getDialect("UNA".toCharArray(), 0, 3);
diff --git a/src/test/resources/EDIFACT/invoic_d97b_una.edi b/src/test/resources/EDIFACT/invoic_d97b_una.edi
index f124964..8be3f9e 100644
--- a/src/test/resources/EDIFACT/invoic_d97b_una.edi
+++ b/src/test/resources/EDIFACT/invoic_d97b_una.edi
@@ -1,4 +1,4 @@
-UNA=*.?^~
+UNA=*.? ~
 UNB*UNOA=3*005435656=1*006?415160=1*060515=1434*00000000000778~
 UNH*00000000000117*INVOIC=D=97B=UN~
 BGM*380*342459*9~
diff --git a/src/test/resources/EDIFACT/pnrgov.edi b/src/test/resources/EDIFACT/pnrgov.edi
index 2e0802e..524dd18 100644
--- a/src/test/resources/EDIFACT/pnrgov.edi
+++ b/src/test/resources/EDIFACT/pnrgov.edi
@@ -1,4 +1,4 @@
-UNA:+.\*'
+UNA:+.\ '
 UNB+IATA:1+1A+KRC+130527:0649+0003'
 UNH+1+PNRGOV:11:1:IA+270513/0649/SQ/602'
 MSG+:22'
@@ -36,7 +36,7 @@ TRI++SIN-169:::3'
 TIF+BELT:I+SOPHY:IN:3'
 SSD+011D++++J'
 LTS+0/O/NM/BELT/ISABELLE MRS(ADT)(INF/SOPHY/20SEP12)'
-LTS+0/O/SS/SQ 602 D 27MAY 1 SINICN LK1 1430 2205/NN \*1A/E\* /SQ/SG/C/I/CAB J//1///// /Y 1625/B 153//AY 1838/EY 1685/SINICN/D'
+LTS+0/O/SS/SQ 602 D 27MAY 1 SINICN LK1 1430 2205/NN *1A/E* /SQ/SG/C/I/CAB J//1///// /Y 1625/B 153//AY 1838/EY 1685/SINICN/D'
 LTS+0/O/SR/SSR INFTSQNN1 BELT/SOPHY 20SEP12/SQ 602 D 27MAY SINICN/BELT/ISABELLE MRS(ADT)(INF/SOPHY/20SEP12)'
 LTS+0/O/SR/SSR FQTVSQHK/ SQ8794285757 S/KFES/BELT/ISABELLE MRS(ADT)(INF/SOPHY/20SEP12)'
 LTS+0/O/SR/SSR FQTSSQHK1 SQ8794285757 S/KFES/SQ 602 D 27MAY SINICN/BELT/ISABELLE MRS(ADT)(INF/SOPHY/20SEP12)'
@@ -67,7 +67,7 @@ LTS+11/A/SK/SK LKPX SQ HK1 Z8OWIE-P1/SQ 602 D 27MAY SINICN/BELT/ISABELLE MRS(ADT
 LTS+11/Z/ISSBOE CR-SINSQ01W0 32391122 GS 1916VV/RO-9CB39093 MUCPI2SQ1 00000000 25MAY0313Z'
 LTS+12/Z/MRS BELT CR-SINSQ01W0 32391122 GS 1916VV/RO-9CB39093 MUCPI2SQ1 00000000 25MAY0315Z'
 LTS+13/Z/1AINV RM SQ 250315 CR-1AINV RM SQ 0000 25MAY0315Z'
-LTS+14/A/7/RX \*\*\*\*\*\*\*\* ATTN SINKKXH SINKDXH SINKNXH PLS ASSIST PAX WITH TKT 618-2402058077 AND 618-2402241436 TO BE SEATED TOGETHER FLT'
+LTS+14/A/7/RX ******** ATTN SINKKXH SINKDXH SINKNXH PLS ASSIST PAX WITH TKT 618-2402058077 AND 618-2402241436 TO BE SEATED TOGETHER FLT'
 LTS+14/A/7/RX SQ602 D SIN - ICN 27MAY13 14\:30 ON BSCT SEAT X MANY THANKS SINRRRSQ'
 LTS+14/A/7/RX MS BELT CALLED TO UPDATE SEAT RQST X ADDED SEAT / MEAL X TELEX SENT X NIL SEATS AND MEALS X MOBILE CONTACT NUMBER UPDATED'
 LTS+14/A/7/RX X VIAN / 8\:48 IST 25/5/2013'