Repository: xlate/staedi (java)

Issue:
Delimiters outside ASCII range fails to parse
**Describe the bug**
Attempting to parse the included EDI, I'm getting the following Exception:

    java.lang.ArrayIndexOutOfBoundsException: Array index out of range: 8230

	at io.xlate.edi.internal.stream.tokenization.CharacterSet.setClass(CharacterSet.java:193)
	at io.xlate.edi.internal.stream.tokenization.X12Dialect.initialize(X12Dialect.java:95)
	at io.xlate.edi.internal.stream.tokenization.X12Dialect.appendHeader(X12Dialect.java:167)
	at io.xlate.edi.internal.stream.tokenization.Lexer.handleStateHeaderData(Lexer.java:370)
	at io.xlate.edi.internal.stream.tokenization.Lexer.parse(Lexer.java:229)
	at io.xlate.edi.internal.stream.StaEDIStreamReader.nextEvent(StaEDIStreamReader.java:155)
	at io.xlate.edi.internal.stream.StaEDIStreamReader.next(StaEDIStreamReader.java:187)


**To Reproduce**
```
EDIInputFactory factory = EDIInputFactory.newFactory();
EDIStreamReader reader = factory.createEDIStreamReader(ediInputStream);

while (reader.hasNext()) {
  EDIStreamEvent ediStreamEvent = reader.next();
  switch (ediStreamEvent) {
    case START_INTERCHANGE:
      String[] version = reader.getVersion();
      break;
  }
}
```
With the follow as the data used in the `ediInputStream`
```
ISA*00*          *00*          *ZZ*XXXX           *ZZ*DDDDDD         *200910*1930*U*00400*000075776*0*P*>…
GS*QM*XXXX*DDDDDD*20200910*1930*75776*X*004010…
ST*214*757760001…
B10*0542550913*7727019*XXXX…
L11*900169*PO…
N1*SH*HEATCO INC…
N3*50 HEATCO CT NW…
N4*CARTERSVILLE*GA*30120…
N1*CN*ADDISON HVAC LLC…
N3*7050 OVERLAND RD…
N4*ORLANDO*FL*32810…
N1*BT*TRANSAVER FREIGHT SE…
N3*108 WASHINGTON S…
N4*MANLIUS*NY*13104…
LX*1…
AT7*AG*NS***20200911*1159*ET…
MS2*XXXX*902577*CV…
LX*2…
AT7*AF*NS***20200910*1135*ET…
MS2*XXXX*902577*CV…
AT8*G*L*2085*5…
SE*20*757760001…
GE*1*75776…
IEA*1*000075776…
```

@MikeEdgar mentioned this might/probably is a bug, "the parser should automatically detect the delimiters, but it's possible there is an issue if it's outside the ASCII range"

Pull request description:
Support non-ASCII delimiter characters
Fixes #109 

Patch:
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
index 6393160..c8278fb 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/CharacterSet.java
@@ -16,6 +16,8 @@
 package io.xlate.edi.internal.stream.tokenization;
 
 import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
 
 public class CharacterSet {
 
@@ -171,70 +173,44 @@ public class CharacterSet {
     };
 
     private final CharacterClass[] list;
-    private final int size;
+    private final Map<Integer, CharacterClass> auxilary;
 
     public CharacterSet() {
         list = Arrays.copyOf(prototype, prototype.length);
-        size = list.length;
+        auxilary = new TreeMap<>();
     }
 
     public CharacterClass getClass(int character) {
-        return (character < size) ? list[character] : _OTHER;
+        return (character < list.length) ? list[character] : auxilary.getOrDefault(character, _OTHER);
     }
 
     public void reset() {
         System.arraycopy(prototype, 0, list, 0, prototype.length);
+        auxilary.clear();
     }
 
     public void setClass(int character, CharacterClass clazz) {
-        if (character < size) {
+        if (character < list.length) {
             list[character] = clazz;
         } else {
-            throw new ArrayIndexOutOfBoundsException(character);
+            auxilary.put(character, clazz);
         }
     }
 
-    public int getDelimiter(CharacterClass clazz) {
-        switch (clazz) {
+    public boolean isDelimiter(int character) {
+        switch (getClass(character)) {
         case ELEMENT_DELIMITER:
         case ELEMENT_REPEATER:
         case SEGMENT_DELIMITER:
         case COMPONENT_DELIMITER:
-            break;
+            return true;
         default:
-            throw new IllegalArgumentException("Nondelimiter class: " + clazz);
-        }
-
-        for (int i = 0, l = list.length; i < l; i++) {
-            if (list[i] == clazz) {
-                return i;
-            }
-        }
-
-        return -1;
-    }
-
-    public boolean isDelimiter(int character) {
-        if (character < size) {
-            switch (list[character]) {
-            case ELEMENT_DELIMITER:
-            case ELEMENT_REPEATER:
-            case SEGMENT_DELIMITER:
-            case COMPONENT_DELIMITER:
-                return true;
-            default:
-                return false;
-            }
+            return false;
         }
-
-        return false;
     }
 
-    public boolean isRelease(int character) {
-        if (character >= size) {
-            return false;
-        }
-        return list[character] == CharacterClass.RELEASE_CHARACTER;
+    public boolean isCharacterClass(int character, CharacterClass clazz) {
+        return getClass(character).equals(clazz);
     }
 
     public static boolean isValid(int character) {
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
index 977a261..9b29b63 100644
--- a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
+++ b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
@@ -369,13 +369,21 @@ public class Lexer {
     void handleStateHeaderData(int input) throws EDIException {
         dialect.appendHeader(characters, (char) input);
 
-        if (characters.isDelimiter(input)) {
-            if (characters.getDelimiter(CharacterClass.SEGMENT_DELIMITER) == input) {
-                closeSegment();
-                state = State.HEADER_TAG_SEARCH;
+        switch (characters.getClass(input)) {
+        case SEGMENT_DELIMITER:
+            closeSegment();
+            state = State.HEADER_TAG_SEARCH;
+            break;
+        case ELEMENT_DELIMITER:
+        case ELEMENT_REPEATER:
+        case COMPONENT_DELIMITER:
+        case RELEASE_CHARACTER:
+            break;
+        default:
+            if (dialect.getDecimalMark() != input) {
+                buffer.put((char) input);
             }
-        } else if (!characters.isRelease(input) && dialect.getDecimalMark() != input) {
-            buffer.put((char) input);
+            break;
         }
     }
 


Test patch:
diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
index cd9ba7c..9dd24f1 100644
--- a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
@@ -693,6 +693,34 @@ class StaEDIStreamWriterTest {
         final ByteArrayOutputStream expected = new ByteArrayOutputStream(16384);
         final InputStream delegate = getClass().getResourceAsStream("/x12/sample275_with_HL7_valid_BIN01.edi");
 
+        InputStream source = new InputStream() {
+            @Override
+            public int read() throws IOException {
+                int value = delegate.read();
+
+                if (value != -1) {
+                    expected.write(value);
+                    System.out.write(value);
+                    System.out.flush();
+                    return value;
+                }
+
+                return -1;
+            }
+        };
+        EDIStreamReader rawReader = inputFactory.createEDIStreamReader(source);
+        EDIStreamReader reader = inputFactory.createFilteredReader(rawReader, r -> true); // Accept all events
+        ByteArrayOutputStream result = new ByteArrayOutputStream(16384);
+        writeFromReader(reader, result);
+        assertEquals(expected.toString().trim(), result.toString().trim());
+    }
+
+    @Test
+    void testInputEquivalenceX12_NonAsciiSegmentTerminator() throws Exception {
+        EDIInputFactory inputFactory = EDIInputFactory.newFactory();
+        final ByteArrayOutputStream expected = new ByteArrayOutputStream(16384);
+        final InputStream delegate = getClass().getResourceAsStream("/x12/issue109/ts214_ellipses_segterm.edi");
+
         InputStream source = new InputStream() {
             @Override
             public int read() throws IOException {
@@ -709,12 +737,15 @@ class StaEDIStreamWriterTest {
             }
         };
         EDIStreamReader reader = inputFactory.createEDIStreamReader(source);
+        ByteArrayOutputStream result = new ByteArrayOutputStream(16384);
+        writeFromReader(reader, result);
+        assertEquals(expected.toString().trim(), result.toString().trim());
+    }
 
+    void writeFromReader(EDIStreamReader reader, OutputStream result) throws Exception {
         EDIOutputFactory outputFactory = EDIOutputFactory.newFactory();
         outputFactory.setProperty(EDIOutputFactory.PRETTY_PRINT, true);
-        ByteArrayOutputStream result = new ByteArrayOutputStream(16384);
-        EDIStreamWriter writer = outputFactory.createEDIStreamWriter(result);
-
+        EDIStreamWriter writer = null;
         EDIStreamEvent event;
         String tag = null;
         boolean composite = false;
@@ -725,6 +756,10 @@ class StaEDIStreamWriterTest {
 
                 switch (event) {
                 case START_INTERCHANGE:
+                    for (Map.Entry<String, Character> delim : reader.getDelimiters().entrySet()) {
+                        outputFactory.setProperty(delim.getKey(), delim.getValue());
+                    }
+                    writer = outputFactory.createEDIStreamWriter(result);
                     writer.startInterchange();
                     break;
                 case END_INTERCHANGE:
@@ -789,8 +824,6 @@ class StaEDIStreamWriterTest {
         } finally {
             reader.close();
         }
-
-        assertEquals(expected.toString().trim(), result.toString().trim());
     }
 
     @Test
diff --git a/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java b/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java
index 2fb0882..2f04264 100644
--- a/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java
+++ b/src/test/java/io/xlate/edi/internal/stream/tokenization/CharacterSetTest.java
@@ -1,8 +1,7 @@
 package io.xlate.edi.internal.stream.tokenization;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.junit.jupiter.api.Test;
@@ -25,44 +24,34 @@ class CharacterSetTest {
     void testSetClassAscii() {
         CharacterSet target = new CharacterSet();
         target.setClass('\\', CharacterClass.RELEASE_CHARACTER);
-        assertTrue(target.isRelease('\\'));
+        assertTrue(target.isCharacterClass('\\', CharacterClass.RELEASE_CHARACTER));
     }
 
     @Test
     void testSetClassUTF8() {
         CharacterSet target = new CharacterSet();
-        Throwable thrown = assertThrows(ArrayIndexOutOfBoundsException.class,
-                                        () -> target.setClass('£', CharacterClass.RELEASE_CHARACTER));
-        assertEquals("Array index out of range: " + ((int) '£'), thrown.getMessage());
+        target.setClass('£', CharacterClass.RELEASE_CHARACTER);
+        assertTrue(target.isCharacterClass('£', CharacterClass.RELEASE_CHARACTER));
     }
 
     @Test
     void testGetDelimiterValidClass() {
         CharacterSet target = new CharacterSet();
         target.setClass('`', CharacterClass.ELEMENT_REPEATER);
-        assertEquals('`', target.getDelimiter(CharacterClass.ELEMENT_REPEATER));
-    }
-
-    @Test
-    void testGetDelimiterInvalidClass() {
-        CharacterSet target = new CharacterSet();
-        target.setClass('X', CharacterClass.ALPHANUMERIC);
-        Throwable thrown = assertThrows(IllegalArgumentException.class,
-                                        () -> target.getDelimiter(CharacterClass.ALPHANUMERIC));
-        assertEquals("Nondelimiter class: " + CharacterClass.ALPHANUMERIC, thrown.getMessage());
+        assertEquals(CharacterClass.ELEMENT_REPEATER, target.getClass('`'));
     }
 
     @Test
     void testIsReleaseAscii() {
         CharacterSet target = new CharacterSet();
         target.setClass('\\', CharacterClass.RELEASE_CHARACTER);
-        assertTrue(target.isRelease('\\'));
+        assertEquals(CharacterClass.RELEASE_CHARACTER, target.getClass('\\'));
     }
 
     @Test
     void testIsReleaseUTF8() {
         CharacterSet target = new CharacterSet();
-        assertFalse(target.isRelease('£'));
+        assertNotEquals(CharacterClass.RELEASE_CHARACTER, target.getClass('£'));
     }
 
     @Test
diff --git a/src/test/resources/logging.properties b/src/test/resources/logging.properties
index 7e703bd..b2327fb 100644
--- a/src/test/resources/logging.properties
+++ b/src/test/resources/logging.properties
@@ -4,4 +4,4 @@ java.util.logging.ConsoleHandler.level=FINEST
 java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
 java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$-7s] %5$s %n
 
-io.xlate.edi.level=FINE
+io.xlate.edi.level=FINER
diff --git a/src/test/resources/x12/issue109/ts214_ellipses_segterm.edi b/src/test/resources/x12/issue109/ts214_ellipses_segterm.edi
new file mode 100644
index 0000000..670f3f4
--- /dev/null
+++ b/src/test/resources/x12/issue109/ts214_ellipses_segterm.edi
@@ -0,0 +1,24 @@
+ISA*00*          *00*          *ZZ*XXXX           *ZZ*DDDDDD         *200910*1930*U*00400*000075776*0*P*>…
+GS*QM*XXXX*DDDDDD*20200910*1930*75776*X*004010…
+ST*214*757760001…
+B10*0542550913*7727019*XXXX…
+L11*900169*PO…
+N1*SH*HEATCO INC…
+N3*50 HEATCO CT NW…
+N4*CARTERSVILLE*GA*30120…
+N1*CN*ADDISON HVAC LLC…
+N3*7050 OVERLAND RD…
+N4*ORLANDO*FL*32810…
+N1*BT*TRANSAVER FREIGHT SE…
+N3*108 WASHINGTON S…
+N4*MANLIUS*NY*13104…
+LX*1…
+AT7*AG*NS***20200911*1159*ET…
+MS2*XXXX*902577*CV…
+LX*2…
+AT7*AF*NS***20200910*1135*ET…
+MS2*XXXX*902577*CV…
+AT8*G*L*2085*5…
+SE*20*757760001…
+GE*1*75776…
+IEA*1*000075776…
\ No newline at end of file