Repository: xlate/staedi (java)

Issue:
Non-ASCII input and output are not properly handled
Reading element `BÜTTNER WIDGET COMPANY` from an EDI stream results in the Java String `BÃTTNER WIDGET COMPANY`.

Writing element `BÜTTNER` results in `B�TTNER` being written to the output.

Pull request:
Decode input characters and encode output using provided Charset
Fixes #33 

Calling `flush()` (or `close()`) on the `EDIStreamWriter` is critical with this change due to the buffer used by `OutputStreamWriter`.

Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ac675bb..5d32832 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -72,9 +72,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          # Disabling shallow clone is recommended for improving relevancy of reporting
-          fetch-depth: 0
+      - run: |
+          git fetch --prune --unshallow --tags
 
       - uses: actions/setup-java@v1.3.0
         with:
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java
index 653ae62..b2d7475 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIInputFactory.java
@@ -16,9 +16,9 @@
 package io.xlate.edi.internal.stream;
 
 import java.io.InputStream;
-import java.util.HashSet;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
-import java.util.Set;
 
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
@@ -32,18 +32,12 @@ import io.xlate.edi.stream.EDIStreamReader;
 
 public class StaEDIInputFactory extends EDIInputFactory {
 
-    private static final String DEFAULT_ENCODING = "US-ASCII";
-
-    private final Set<String> supportedCharsets;
     private EDIReporter reporter;
 
     public StaEDIInputFactory() {
         supportedProperties.add(EDI_VALIDATE_CONTROL_STRUCTURE);
         supportedProperties.add(EDI_VALIDATE_CONTROL_CODE_VALUES);
         supportedProperties.add(XML_DECLARE_TRANSACTION_XMLNS);
-
-        supportedCharsets = new HashSet<>();
-        supportedCharsets.add(DEFAULT_ENCODING);
     }
 
     @Override
@@ -58,7 +52,7 @@ public class StaEDIInputFactory extends EDIInputFactory {
 
     @Override
     public EDIStreamReader createEDIStreamReader(InputStream stream, Schema schema) {
-        return new StaEDIStreamReader(stream, DEFAULT_ENCODING, schema, properties, getEDIReporter());
+        return new StaEDIStreamReader(stream, StandardCharsets.UTF_8, schema, properties, getEDIReporter());
     }
 
     @SuppressWarnings("resource")
@@ -66,8 +60,8 @@ public class StaEDIInputFactory extends EDIInputFactory {
     public EDIStreamReader createEDIStreamReader(InputStream stream, String encoding, Schema schema) throws EDIStreamException {
         Objects.requireNonNull(stream);
 
-        if (supportedCharsets.contains(encoding)) {
-            return new StaEDIStreamReader(stream, encoding, schema, properties, getEDIReporter());
+        if (Charset.isSupported(encoding)) {
+            return new StaEDIStreamReader(stream, Charset.forName(encoding), schema, properties, getEDIReporter());
         }
 
         throw new EDIStreamException("Unsupported encoding: " + encoding);
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIOutputFactory.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIOutputFactory.java
index 50e97a5..196ebfc 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIOutputFactory.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIOutputFactory.java
@@ -16,8 +16,8 @@
 package io.xlate.edi.internal.stream;
 
 import java.io.OutputStream;
-import java.util.HashSet;
-import java.util.Set;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 
 import javax.xml.stream.XMLStreamWriter;
 
@@ -28,14 +28,7 @@ import io.xlate.edi.stream.EDIStreamWriter;
 
 public class StaEDIOutputFactory extends EDIOutputFactory {
 
-    private static final String DEFAULT_ENCODING = "US-ASCII";
-
-    private final Set<String> supportedCharsets;
-
     public StaEDIOutputFactory() {
-        supportedCharsets = new HashSet<>();
-        supportedCharsets.add(DEFAULT_ENCODING);
-
         supportedProperties.add(EDIStreamConstants.Delimiters.SEGMENT);
         supportedProperties.add(EDIStreamConstants.Delimiters.DATA_ELEMENT);
         supportedProperties.add(EDIStreamConstants.Delimiters.COMPONENT_ELEMENT);
@@ -49,13 +42,13 @@ public class StaEDIOutputFactory extends EDIOutputFactory {
 
     @Override
     public EDIStreamWriter createEDIStreamWriter(OutputStream stream) {
-        return new StaEDIStreamWriter(stream, DEFAULT_ENCODING, properties);
+        return new StaEDIStreamWriter(stream, StandardCharsets.UTF_8, properties);
     }
 
     @Override
     public EDIStreamWriter createEDIStreamWriter(OutputStream stream, String encoding) throws EDIStreamException {
-        if (supportedCharsets.contains(encoding)) {
-            return new StaEDIStreamWriter(stream, encoding, properties);
+        if (Charset.isSupported(encoding)) {
+            return new StaEDIStreamWriter(stream, Charset.forName(encoding), properties);
         }
         throw new EDIStreamException("Unsupported encoding: " + encoding);
     }
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java
index 762ac36..99714e8 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamReader.java
@@ -18,6 +18,7 @@ package io.xlate.edi.internal.stream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.CharBuffer;
+import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -45,9 +46,6 @@ public class StaEDIStreamReader implements EDIStreamReader {
 
     private static final Logger LOGGER = Logger.getLogger(StaEDIStreamReader.class.getName());
 
-    private final InputStream stream;
-    @SuppressWarnings("unused")
-    private final String encoding;
     private Schema controlSchema;
     private final Map<String, Object> properties;
     private final EDIReporter reporter;
@@ -60,17 +58,16 @@ public class StaEDIStreamReader implements EDIStreamReader {
 
     public StaEDIStreamReader(
             InputStream stream,
-            String encoding,
+            Charset charset,
             Schema schema,
             Map<String, Object> properties,
             EDIReporter reporter) {
-        this.stream = stream;
-        this.encoding = encoding;
+
         this.controlSchema = schema;
         this.properties = new HashMap<>(properties);
         this.reporter = reporter;
         this.proxy = new ProxyEventHandler(location, this.controlSchema);
-        this.lexer = new Lexer(this.stream, proxy, location);
+        this.lexer = new Lexer(stream, charset, proxy, location);
     }
 
     private void ensureOpen() {
diff --git a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java
index 2e836b7..a8c9247 100644
--- a/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java
+++ b/src/main/java/io/xlate/edi/internal/stream/StaEDIStreamWriter.java
@@ -18,8 +18,10 @@ package io.xlate.edi.internal.stream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -66,8 +68,7 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
     private CharacterSet characters = new CharacterSet();
 
     private final OutputStream stream;
-    @SuppressWarnings("unused")
-    private final String encoding;
+    private final OutputStreamWriter writer;
     private final Map<String, Object> properties;
     private Dialect dialect;
 
@@ -92,9 +93,9 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
     private final boolean prettyPrint;
     private final String lineSeparator;
 
-    public StaEDIStreamWriter(OutputStream stream, String encoding, Map<String, Object> properties) {
+    public StaEDIStreamWriter(OutputStream stream, Charset charset, Map<String, Object> properties) {
         this.stream = stream;
-        this.encoding = encoding;
+        this.writer = new OutputStreamWriter(stream, charset);
         this.properties = new HashMap<>(properties);
         this.prettyPrint = property(EDIOutputFactory.PRETTY_PRINT, Boolean::valueOf);
 
@@ -182,6 +183,7 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
     @Override
     public void flush() throws EDIStreamException {
         try {
+            writer.flush();
             stream.flush();
         } catch (IOException e) {
             throw new EDIStreamException("Exception flushing output stream", location, e);
@@ -268,7 +270,7 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
 
         try {
             location.incrementOffset(output);
-            stream.write(output);
+            writer.write(output);
         } catch (IOException e) {
             throw new EDIStreamException("Exception to output stream", location, e);
         }
@@ -284,9 +286,9 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
 
     @Override
     public EDIStreamWriter endInterchange() throws EDIStreamException {
-
         ensureLevel(LEVEL_INTERCHANGE);
         level = LEVEL_INITIAL;
+        flush();
         return this;
     }
 
@@ -540,6 +542,8 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
         int output;
 
         try {
+            flush(); // Write `Writer` buffers to stream before writing binary
+
             while ((output = binaryStream.read()) != -1) {
                 location.incrementOffset(output);
                 stream.write(output);
@@ -557,8 +561,15 @@ public class StaEDIStreamWriter implements EDIStreamWriter, ElementDataHandler,
         ensureState(State.ELEMENT_DATA_BINARY);
         ensureArgs(binary.length, start, end);
 
-        for (int i = start; i < end; i++) {
-            write(binary[i]);
+        try {
+            flush(); // Write `Writer` buffers to stream before writing binary
+
+            for (int i = start; i < end; i++) {
+                location.incrementOffset(binary[i]);
+                stream.write(binary[i]);
+            }
+        } catch (IOException e) {
+            throw new EDIStreamException("Exception writing binary element data", location, e);
         }
 
         return this;
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
index 8395458..db9d4a6 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
@@ -171,13 +171,15 @@ public class CharacterSet {
     };
 
     private final CharacterClass[] list;
+    private final int size;
 
     public CharacterSet() {
         list = Arrays.copyOf(prototype, prototype.length);
+        size = list.length;
     }
 
     public CharacterClass getClass(int character) {
-        return (character < 128) ? list[character] : _OTHER;
+        return (character < size) ? list[character] : _OTHER;
     }
 
     public void reset() {
@@ -185,7 +187,7 @@ public class CharacterSet {
     }
 
     public void setClass(int character, CharacterClass clazz) {
-        if (character < 128) {
+        if (character < size) {
             list[character] = clazz;
         } else {
             throw new ArrayIndexOutOfBoundsException(character);
@@ -213,7 +215,7 @@ public class CharacterSet {
     }
 
     public boolean isDelimiter(int character) {
-        if (character < 128) {
+        if (character < size) {
             switch (list[character]) {
             case ELEMENT_DELIMITER:
             case ELEMENT_REPEATER:
@@ -229,13 +231,13 @@ public class CharacterSet {
     }
 
     public boolean isRelease(int character) {
-        if (character >= 128) {
+        if (character >= size) {
             return false;
         }
         return list[character] == CharacterClass.RELEASE_CHARACTER;
     }
 
     public static boolean isValid(int character) {
-        return character >= 128 || prototype[character] != _INVLD;
+        return character >= prototype.length || prototype[character] != _INVLD;
     }
 }
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIException.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIException.java
index 8be5838..24190a3 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIException.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIException.java
@@ -59,7 +59,7 @@ public class EDIException extends EDIStreamException {
     }
 
     public EDIException(Integer id, String message) {
-        super(exceptionMessages.get(id) + message);
+        super(exceptionMessages.get(id) + ' ' + message);
     }
 
     public EDIException(Throwable cause) {
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
index 196f5fb..1ab7232 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
@@ -18,7 +18,11 @@ package io.xlate.edi.internal.stream.tokenization;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
 import java.util.ArrayDeque;
 import java.util.Deque;
 import java.util.logging.Logger;
@@ -51,6 +55,11 @@ public class Lexer {
     private final Deque<Integer> lengthQueue = new ArrayDeque<>(20);
 
     private final InputStream stream;
+    private CharsetDecoder decoder;
+    private char[] readChar = new char[1];
+    private CharBuffer readCharBuf = CharBuffer.wrap(readChar);
+    private ByteBuffer readByteBuf = ByteBuffer.allocate(4);
+
     private final StaEDIStreamLocation location;
 
     private CharacterSet characters = new CharacterSet();
@@ -69,13 +78,15 @@ public class Lexer {
     private Notifier en;
     private Notifier bn;
 
-    public Lexer(InputStream stream, EventHandler handler, StaEDIStreamLocation location) {
+    public Lexer(InputStream stream, Charset charset, EventHandler handler, StaEDIStreamLocation location) {
         if (stream.markSupported()) {
             this.stream = stream;
         } else {
             this.stream = new BufferedInputStream(stream);
         }
 
+        this.decoder = charset.newDecoder();
+
         this.location = location;
 
         isn = (notifyState, start, length) -> {
@@ -163,7 +174,7 @@ public class Lexer {
         int input = 0;
         boolean eventsReady = false;
 
-        while (!eventsReady && (input = stream.read()) > -1) {
+        while (!eventsReady && (input = readCharacter()) > -1) {
             location.incrementOffset(input);
 
             CharacterClass clazz = characters.getClass(input);
@@ -276,6 +287,60 @@ public class Lexer {
         }
     }
 
+    int readCharacter() throws IOException {
+        int next = stream.read();
+
+        if (next < 0) {
+            return -1;
+        }
+
+        boolean endOfInput = false;
+        boolean complete = false;
+        int position = 0;
+
+        readCharBuf.clear();
+        readByteBuf.clear();
+        readByteBuf.put((byte) next);
+
+        do {
+            readByteBuf.flip();
+            CoderResult cr = decoder.decode(readByteBuf, readCharBuf, endOfInput);
+
+            if (!cr.isUnderflow()) {
+                cr.throwException();
+            }
+
+            if (endOfInput) {
+                complete = true;
+            } else if (readCharBuf.position() > 0) {
+                // Single character successfully written to the CharBuffer
+                complete = true;
+            } else {
+                next = stream.read();
+
+                if (next < 0) {
+                    endOfInput = true;
+                    decoder.reset();
+                } else {
+                    readByteBuf.limit(readByteBuf.capacity());
+                    readByteBuf.position(++position);
+                    readByteBuf.put((byte) next);
+                }
+            }
+        } while (!complete);
+
+        if (endOfInput) {
+            decoder.reset();
+        }
+
+        if (readCharBuf.position() == 0 && endOfInput) {
+            // Nothing was written to the CharBuffer
+            return -1;
+        }
+
+        return readChar[0];
+    }
+
     void handleStateHeaderTag(int input) {
         buffer.put((char) input);
         dialect.appendHeader(characters, (char) input);
@@ -292,7 +357,6 @@ public class Lexer {
     }
 
     void handleStateInterchangeCandidate(int input) throws EDIException {
-        stream.mark(500);
         buffer.put((char) input);
         final char[] header = buffer.array();
         final int length = buffer.position();
@@ -317,13 +381,12 @@ public class Lexer {
         }
     }
 
-    private boolean dialectConfirmed(State confirmed) throws IOException, EDIException {
+    private boolean dialectConfirmed(State confirmed) throws EDIException {
         if (dialect.isConfirmed()) {
             state = confirmed;
             nextEvent();
             return true;
         } else if (dialect.isRejected()) {
-            stream.reset();
             buffer.clear();
             clearQueues();
             dialect = null;
diff --git a/src/main/java/io/xlate/edi/stream/EDIInputFactory.java b/src/main/java/io/xlate/edi/stream/EDIInputFactory.java
index 5b61005..3eaacbc 100644
--- a/src/main/java/io/xlate/edi/stream/EDIInputFactory.java
+++ b/src/main/java/io/xlate/edi/stream/EDIInputFactory.java
@@ -27,15 +27,15 @@ public abstract class EDIInputFactory extends PropertySupport {
     public static final String EDI_VALIDATE_CONTROL_STRUCTURE = "io.xlate.edi.stream.EDI_VALIDATE_CONTROL_STRUCTURE";
 
     /**
-     * When set to false, enumerated code values of control structure elements will be ignore. Element size and type
-     * validation will still occur.
+     * When set to false, enumerated code values of control structure elements
+     * will be ignore. Element size and type validation will still occur.
      */
     public static final String EDI_VALIDATE_CONTROL_CODE_VALUES = "io.xlate.edi.stream.EDI_VALIDATE_CONTROL_CODE_VALUES";
 
     /**
-     * When set to true, XMLStreamReader instances created from an EDIInputFactory
-     * will generate XMLNS attributes on the TRANSACTION element in addition to the
-     * INTERCHANGE element.
+     * When set to true, XMLStreamReader instances created from an
+     * EDIInputFactory will generate XMLNS attributes on the TRANSACTION element
+     * in addition to the INTERCHANGE element.
      */
     public static final String XML_DECLARE_TRANSACTION_XMLNS = "io.xlate.edi.stream.XML_DECLARE_TRANSACTION_XMLNS";
 
@@ -64,18 +64,21 @@ public abstract class EDIInputFactory extends PropertySupport {
 
     /**
      * Creates a new {@link EDIStreamReader} using the given {@link InputStream}
-     * and encoding.
+     * and encoding. The encoding must be a valid
+     * {@link java.nio.charset.Charset Charset}.
      *
      * @param stream
      *            {@link InputStream} from which the EDI data will be read
      * @param encoding
-     *            character encoding of the stream
+     *            character encoding of the stream, must be a valid
+     *            {@link java.nio.charset.Charset Charset}.
      * @return a new {@link EDIStreamReader} which reads from the stream
      * @throws EDIStreamException
      *             when encoding is not supported
      */
     public abstract EDIStreamReader createEDIStreamReader(InputStream stream,
-                                                          String encoding) throws EDIStreamException;
+                                                          String encoding)
+            throws EDIStreamException;
 
     /**
      * Creates a new {@link EDIStreamReader} using the given {@link InputStream}
@@ -97,7 +100,8 @@ public abstract class EDIInputFactory extends PropertySupport {
     /**
      * Creates a new {@link EDIStreamReader} using the given {@link InputStream}
      * and encoding which uses the {@link Schema} for validation of the input's
-     * control structures (interchange, group, transaction).
+     * control structures (interchange, group, transaction). The encoding must
+     * be a valid {@link java.nio.charset.Charset Charset}.
      *
      * Note that a separate schema for validation of messages/transactions may
      * be passed directly to the {@link EDIStreamReader} once the type and
@@ -106,7 +110,8 @@ public abstract class EDIInputFactory extends PropertySupport {
      * @param stream
      *            {@link InputStream} from which the EDI data will be read
      * @param encoding
-     *            character encoding of the stream
+     *            character encoding of the stream, must be a valid
+     *            {@link java.nio.charset.Charset Charset}.
      * @param schema
      *            {@link Schema} for control structure validation
      * @return a new {@link EDIStreamReader} which reads from the stream
@@ -115,7 +120,8 @@ public abstract class EDIInputFactory extends PropertySupport {
      */
     public abstract EDIStreamReader createEDIStreamReader(InputStream stream,
                                                           String encoding,
-                                                          Schema schema) throws EDIStreamException;
+                                                          Schema schema)
+            throws EDIStreamException;
 
     /**
      * Creates a new {@link EDIStreamReader} by wrapping the given reader with
diff --git a/src/main/java/io/xlate/edi/stream/EDIOutputFactory.java b/src/main/java/io/xlate/edi/stream/EDIOutputFactory.java
index 3112135..21ce1ac 100644
--- a/src/main/java/io/xlate/edi/stream/EDIOutputFactory.java
+++ b/src/main/java/io/xlate/edi/stream/EDIOutputFactory.java
@@ -40,7 +40,7 @@ public abstract class EDIOutputFactory extends PropertySupport {
      * Create a new EDIStreamWriter that writes to a stream
      *
      * @param stream
-     *            - the stream to write to
+     *            {@link OutputStream} to which the EDI data will be written
      * @return the writer instance
      */
     public abstract EDIStreamWriter createEDIStreamWriter(OutputStream stream);
@@ -49,24 +49,29 @@ public abstract class EDIOutputFactory extends PropertySupport {
      * Create a new EDIStreamWriter that writes to a stream
      *
      * @param stream
-     *            - the stream to write to
+     *            {@link OutputStream} to which the EDI data will be written
      * @param encoding
-     *            - the encoding to use
+     *            character encoding of the stream, must be a valid
+     *            {@link java.nio.charset.Charset Charset}.
      * @return the writer instance
      * @throws EDIStreamException
-     *  when encoding is not supported
+     *             when encoding is not supported
      */
     public abstract EDIStreamWriter createEDIStreamWriter(OutputStream stream,
-                                                          String encoding) throws EDIStreamException;
+                                                          String encoding)
+            throws EDIStreamException;
 
     /**
-     * Creates a new {@link XMLStreamWriter} that uses the given writer as its output.
+     * Creates a new {@link XMLStreamWriter} that uses the given writer as its
+     * output.
      *
-     * XML Elements written to the writer must use the namespaces declared by the constants
-     * in {@link EDINamespaces}. The sequence of elements is critical and must align with
-     * the structure of the intended EDI output to be written via the given EDI writer.
+     * XML Elements written to the writer must use the namespaces declared by
+     * the constants in {@link EDINamespaces}. The sequence of elements is
+     * critical and must align with the structure of the intended EDI output to
+     * be written via the given EDI writer.
      *
-     * @param writer the writer used to generate EDI output using the XML writer
+     * @param writer
+     *            the writer used to generate EDI output using the XML writer
      * @return a new {@link XMLStreamWriter}
      */
     public abstract XMLStreamWriter createXMLStreamWriter(EDIStreamWriter writer);


Test patch:
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
index bac5a13..808695d 100644
--- a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
@@ -180,6 +180,32 @@ class StaEDIStreamReaderTest implements ConstantsTest {
         assertEquals(1, delimiterExceptions, "Unexpected exceptions");
     }
 
+    @Test
+    void testAlternateEncodingEDIFACT() throws EDIStreamException {
+        EDIInputFactory factory = EDIInputFactory.newFactory();
+        InputStream stream = getClass().getResourceAsStream("/EDIFACT/invoic_d97b.edi");
+        EDIStreamReader reader = factory.createEDIStreamReader(stream);
+        int matches = 0;
+
+        while (reader.hasNext()) {
+            switch (reader.next()) {
+            case ELEMENT_DATA:
+                Location location = reader.getLocation();
+                if ("NAD".equals(location.getSegmentTag())
+                        && location.getSegmentPosition() == 7
+                        && location.getElementPosition() == 4) {
+                    assertEquals("BÜTTNER WIDGET COMPANY", reader.getText());
+                    matches++;
+                }
+                break;
+            default:
+                break;
+            }
+        }
+
+        assertEquals(1, matches);
+    }
+
     @Test
     void testNext() throws EDIStreamException {
         EDIInputFactory factory = EDIInputFactory.newFactory();
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
index 402913e..a4ee024 100644
--- a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
@@ -24,7 +24,6 @@ import static org.junit.jupiter.api.Assertions.fail;
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -158,6 +157,7 @@ class StaEDIStreamWriterTest {
         EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
         writer.startInterchange();
         writer.writeStartSegment("ISA");
+        writer.flush();
         assertEquals("ISA", stream.toString());
     }
 
@@ -203,6 +203,7 @@ class StaEDIStreamWriterTest {
         writer.writeStartSegment("ISA");
         writer.writeStartElement().writeElementData("E1").endElement();
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("ISA*E1~", stream.toString());
     }
 
@@ -230,6 +231,7 @@ class StaEDIStreamWriterTest {
               .endElement()
               .writeStartElement()
               .endElement();
+        writer.flush();
         assertEquals("ISA****", stream.toString());
     }
 
@@ -301,9 +303,11 @@ class StaEDIStreamWriterTest {
         EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
         writer.startInterchange();
         writeHeader(writer);
+        writer.flush();
         stream.reset();
         writer.writeStartSegment("BIN");
         writer.writeStartElementBinary().writeEndSegment();
+        writer.flush();
         assertEquals("BIN*~", stream.toString());
     }
 
@@ -330,6 +334,7 @@ class StaEDIStreamWriterTest {
               .endComponent()
               .startComponent()
               .writeEndSegment();
+        writer.flush();
         assertEquals("ISA*:~", stream.toString());
     }
 
@@ -352,6 +357,7 @@ class StaEDIStreamWriterTest {
         EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
         writer.startInterchange();
         writeHeader(writer);
+        writer.flush();
         stream.reset();
         writer.writeStartSegment("SEG");
         writer.writeStartElement()
@@ -359,6 +365,7 @@ class StaEDIStreamWriterTest {
               .writeRepeatElement()
               .writeElementData("R2")
               .writeEndSegment();
+        writer.flush();
         assertEquals("SEG*R1^R2~", stream.toString());
     }
 
@@ -374,6 +381,7 @@ class StaEDIStreamWriterTest {
         writer.writeEmptyElement();
         writer.writeEmptyElement();
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("ISA****~", stream.toString());
     }
 
@@ -390,6 +398,7 @@ class StaEDIStreamWriterTest {
         writer.writeEmptyComponent();
         writer.writeEmptyComponent();
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("ISA*:::~", stream.toString());
     }
 
@@ -403,6 +412,7 @@ class StaEDIStreamWriterTest {
         writer.writeStartElement();
         writer.writeElementData("TEST-ELEMENT");
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("ISA*TEST-ELEMENT~", stream.toString());
     }
 
@@ -428,6 +438,7 @@ class StaEDIStreamWriterTest {
         writer.writeStartElement();
         writer.writeElementData(new char[] { 'C', 'H', 'A', 'R', 'S' }, 0, 5);
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("ISA*CHARS~", stream.toString());
     }
 
@@ -467,6 +478,7 @@ class StaEDIStreamWriterTest {
         InputStream binaryStream = new ByteArrayInputStream(binary);
         writer.startInterchange();
         writeHeader(writer);
+        writer.flush();
         stream.reset();
         writer.writeStartSegment("BIN");
         writer.writeStartElement();
@@ -476,6 +488,7 @@ class StaEDIStreamWriterTest {
         writer.writeBinaryData(binaryStream);
         writer.endElement();
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("BIN*8*\n\u0000\u0001\u0002\u0003\u0004\u0005\t~", stream.toString());
     }
 
@@ -510,6 +523,7 @@ class StaEDIStreamWriterTest {
         byte[] binary = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A' };
         writer.startInterchange();
         writeHeader(writer);
+        writer.flush();
         stream.reset();
         writer.writeStartSegment("BIN");
         writer.writeStartElement();
@@ -519,6 +533,7 @@ class StaEDIStreamWriterTest {
         writer.writeBinaryData(binary, 0, binary.length);
         writer.endElement();
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("BIN*11*0123456789A~", stream.toString());
     }
 
@@ -531,6 +546,7 @@ class StaEDIStreamWriterTest {
         ByteBuffer buffer = ByteBuffer.wrap(binary);
         writer.startInterchange();
         writeHeader(writer);
+        writer.flush();
         stream.reset();
         writer.writeStartSegment("BIN");
         writer.writeStartElement();
@@ -540,6 +556,7 @@ class StaEDIStreamWriterTest {
         writer.writeBinaryData(buffer);
         writer.endElement();
         writer.writeEndSegment();
+        writer.flush();
         assertEquals("BIN*14*BUSTMYBUFFERS\n~", stream.toString());
     }
 
@@ -547,19 +564,17 @@ class StaEDIStreamWriterTest {
     void testInputEquivalenceX12() throws Exception {
         EDIInputFactory inputFactory = EDIInputFactory.newFactory();
         final ByteArrayOutputStream expected = new ByteArrayOutputStream(16384);
+        final InputStream delegate = getClass().getResourceAsStream("/x12/sample275_with_HL7_valid_BIN01.edi");
 
         InputStream source = new InputStream() {
-            final InputStream delegate;
-            {
-                delegate = getClass().getResourceAsStream("/x12/sample275_with_HL7_valid_BIN01.edi");
-            }
-
             @Override
             public int read() throws IOException {
                 int value = delegate.read();
 
                 if (value != -1) {
                     expected.write(value);
+                    System.out.write(value);
+                    System.out.flush();
                     return value;
                 }
 
@@ -971,6 +986,7 @@ class StaEDIStreamWriterTest {
             reader.close();
         }
 
+        System.out.println(expected.toString());
         assertEquals(expected.toString().trim(), result.toString().trim());
     }
 
@@ -1040,7 +1056,7 @@ class StaEDIStreamWriterTest {
         Schema transaction = schemaFactory.createSchema(getClass().getResource("/x12/EDISchema999.xml"));
         final InputStream delegate = new BufferedInputStream(getClass().getResourceAsStream("/x12/simple999.edi"));
 
-        InputStream source = new FilterInputStream(delegate) {
+        InputStream source = new InputStream() {
             @Override
             public int read() throws IOException {
                 int value = delegate.read();
@@ -1205,4 +1221,22 @@ class StaEDIStreamWriterTest {
         writer.writeStartSegment("ISA");
         assertEquals(EDIStreamConstants.Standards.X12, writer.getStandard());
     }
+
+    @Test
+    void testWriteAlternateEncodedElement() throws EDIStreamException {
+        EDIOutputFactory factory = EDIOutputFactory.newFactory();
+        ByteArrayOutputStream stream = new ByteArrayOutputStream(4096);
+        EDIStreamWriter writer = factory.createEDIStreamWriter(stream);
+
+        writer.startInterchange();
+        writeHeader(writer);
+        writer.flush();
+        stream.reset();
+
+        writer.writeStartSegment("SEG");
+        writer.writeElement("BÜTTNER")
+              .writeEndSegment();
+        writer.flush();
+        assertEquals("SEG*BÜTTNER~", stream.toString());
+    }
 }
diff --git a/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java b/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java
new file mode 100644
index 0000000..2fb0882
--- /dev/null
+++ b/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java
@@ -0,0 +1,77 @@
+package io.xlate.edi.internal.stream.tokenization;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+class CharacterSetTest {
+
+    @Test
+    void testGetClassAscii() {
+        CharacterSet target = new CharacterSet();
+        assertEquals(CharacterClass.ALPHANUMERIC, target.getClass('X'));
+    }
+
+    @Test
+    void testGetClassUTF8() {
+        CharacterSet target = new CharacterSet();
+        assertEquals(CharacterClass.OTHER, target.getClass('£'));
+    }
+
+    @Test
+    void testSetClassAscii() {
+        CharacterSet target = new CharacterSet();
+        target.setClass('\\', CharacterClass.RELEASE_CHARACTER);
+        assertTrue(target.isRelease('\\'));
+    }
+
+    @Test
+    void testSetClassUTF8() {
+        CharacterSet target = new CharacterSet();
+        Throwable thrown = assertThrows(ArrayIndexOutOfBoundsException.class,
+                                        () -> target.setClass('£', CharacterClass.RELEASE_CHARACTER));
+        assertEquals("Array index out of range: " + ((int) '£'), thrown.getMessage());
+    }
+
+    @Test
+    void testGetDelimiterValidClass() {
+        CharacterSet target = new CharacterSet();
+        target.setClass('`', CharacterClass.ELEMENT_REPEATER);
+        assertEquals('`', target.getDelimiter(CharacterClass.ELEMENT_REPEATER));
+    }
+
+    @Test
+    void testGetDelimiterInvalidClass() {
+        CharacterSet target = new CharacterSet();
+        target.setClass('X', CharacterClass.ALPHANUMERIC);
+        Throwable thrown = assertThrows(IllegalArgumentException.class,
+                                        () -> target.getDelimiter(CharacterClass.ALPHANUMERIC));
+        assertEquals("Nondelimiter class: " + CharacterClass.ALPHANUMERIC, thrown.getMessage());
+    }
+
+    @Test
+    void testIsReleaseAscii() {
+        CharacterSet target = new CharacterSet();
+        target.setClass('\\', CharacterClass.RELEASE_CHARACTER);
+        assertTrue(target.isRelease('\\'));
+    }
+
+    @Test
+    void testIsReleaseUTF8() {
+        CharacterSet target = new CharacterSet();
+        assertFalse(target.isRelease('£'));
+    }
+
+    @Test
+    void testIsValidAscii() {
+        assertTrue(CharacterSet.isValid('A'));
+    }
+
+    @Test
+    void testIsValidUTF8() {
+        assertTrue(CharacterSet.isValid('ü'));
+    }
+}
diff --git a/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java b/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
index 3d40464..899f88f 100644
--- a/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
@@ -23,6 +23,8 @@ import static org.junit.jupiter.api.Assertions.fail;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.MalformedInputException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -196,7 +198,7 @@ class LexerTest {
         element = null;
 
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, handler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, handler, location);
 
         lexer.parse();
         assertTrue(interchangeStarted, "Interchange not started");
@@ -217,7 +219,7 @@ class LexerTest {
         element = null;
 
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, handler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, handler, location);
 
         lexer.parse();
         assertTrue(interchangeStarted, "Interchange not started");
@@ -234,7 +236,7 @@ class LexerTest {
         InputStream stream = getClass().getResourceAsStream("/x12/simple997.edi");
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
         String last;
         int s = -1;
 
@@ -262,7 +264,7 @@ class LexerTest {
         InputStream stream = getClass().getResourceAsStream("/EDIFACT/invoic_d97b_una.edi");
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
         String last;
         int s = -1;
 
@@ -289,7 +291,7 @@ class LexerTest {
         InputStream stream = getClass().getResourceAsStream("/EDIFACT/invoic_d97b.edi");
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
         String last;
         int s = -1;
 
@@ -316,7 +318,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream("ISA*00?          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~".getBytes());
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
         EDIException thrown = assertThrows(EDIException.class, lexer::parse);
         assertTrue(thrown.getMessage().contains("EDIE003"));
     }
@@ -326,7 +328,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream("ISA*00*\u0008         *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~".getBytes());
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
         EDIException thrown = assertThrows(EDIException.class, lexer::parse);
         assertTrue(thrown.getMessage().contains("EDIE004"));
     }
@@ -336,7 +338,7 @@ class LexerTest {
         InputStream stream = new ByteArrayInputStream("ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~".getBytes());
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
-        final Lexer lexer = new Lexer(stream, eventHandler, location);
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
         for (int i = 0; i < 19; i++) {
             lexer.parse(); // Interchange start through end of ISA
         }
@@ -344,4 +346,35 @@ class LexerTest {
         EDIException thrown = assertThrows(EDIException.class, lexer::parse);
         assertTrue(thrown.getMessage().contains("EDIE005"));
     }
+
+    @Test
+    void testIncompleteInputTextEndingWithDoubleByteChar() throws Exception {
+        byte[] data1 = "ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~GS*".getBytes();
+        byte[] data2 = new byte[data1.length + 1];
+        System.arraycopy(data1, 0, data2, 0, data1.length);
+        data2[data2.length - 1] = (byte) 195;
+        InputStream stream = new ByteArrayInputStream(data2);
+        TestLexerEventHandler eventHandler = new TestLexerEventHandler();
+        final StaEDIStreamLocation location = new StaEDIStreamLocation();
+        final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        for (int i = 0; i < 20; i++) {
+            lexer.parse(); // Interchange start through end of ISA + GS start tag
+        }
+
+        EDIException thrown = assertThrows(EDIException.class, lexer::parse);
+        assertTrue(thrown.getMessage().contains("EDIE005"));
+    }
+
+    @Test
+    void testUnmappabledCharacter() throws EDIException, IOException {
+        InputStream stream = new ByteArrayInputStream("ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~😀".getBytes());
+        TestLexerEventHandler eventHandler = new TestLexerEventHandler();
+        final StaEDIStreamLocation location = new StaEDIStreamLocation();
+        final Lexer lexer = new Lexer(stream, StandardCharsets.US_ASCII, eventHandler, location);
+        for (int i = 0; i < 19; i++) {
+            lexer.parse(); // Interchange start through end of ISA
+        }
+        MalformedInputException thrown = assertThrows(MalformedInputException.class, lexer::parse);
+        assertEquals("Input length = 1", thrown.getMessage());
+    }
 }
diff --git a/src/test/resources/EDIFACT/invoic_d97b.edi b/src/test/resources/EDIFACT/invoic_d97b.edi
index 08f5f52..9af2c13 100644
--- a/src/test/resources/EDIFACT/invoic_d97b.edi
+++ b/src/test/resources/EDIFACT/invoic_d97b.edi
@@ -4,7 +4,7 @@ BGM+380+342459+9'
 DTM+3:20060515:102'
 RFF+ON:521052'
 NAD+BY+792820524::16++CUMMINS MID-RANGE ENGINE PLANT'
-NAD+SE+005435656::16++GENERAL WIDGET COMPANY'
+NAD+SE+005435656::16++BÜTTNER WIDGET COMPANY'
 CUX+1:USD'
 LIN+1++157870:IN'
 IMD+F++:::WIDGET'