asinger/tnetstringsj — Issue #3: Character encodings with >2 byte characters are encoded incorrectly
Author: markkuv
Created: 2011-12-22 11:19:44+00:00
State: open

The byteLength() method assumes that characters > 127 are always 2 bytes long. This is not correct for all encodings. For example, ISO 8859-1 (Latin-1) encodes characters 128-255 as a single byte, and UTF-8 encodes characters above U+07FF with 3-6 bytes.

The fix is not to use byteLength() but to encode using the given charset and use the resulting byte length instead. Patch below.

```
--- TNetstring.java	Thu Dec 22 11:50:15 2011
+++ TNetstring-patched2.java	Thu Dec 22 13:00:37 2011
@@ -277,11 +277,7 @@

static byte[] getBytes(CharSequence s, Charset charset) {
if(!ASCII.equals(charset)) {
- try {
- return String.valueOf(s).getBytes(charset.name());
- } catch (final UnsupportedEncodingException e) {
- throw new IllegalArgumentException(e);
- }
+ return String.valueOf(s).getBytes(charset);
}
return asciiBytes(s);
}
@@ -304,8 +300,8 @@
}

public static byte[] dump(final CharSequence data, final Charset charset) {
- return concat(asciiBytes(byteLength(data) + ":"),
- getBytes(data, charset), COMMA_BYTES);
+ byte[] dataBytes = getBytes(data, charset);
+ return concat(asciiBytes(dataBytes.length + ":"), dataBytes, COMMA_BYTES);
}

private static final byte[] TRUE_BYTES = asciiBytes("4:true!");
@@ -426,8 +422,8 @@
}

public static byte[] dump(final char data, final Charset charset) {
- return concat(asciiBytes(byteLength(data) + ":"),
- getBytes(String.valueOf(data), charset), COMMA_BYTES);
+ byte[] dataBytes = getBytes(String.valueOf(data), charset);
+ return concat(asciiBytes(dataBytes.length + ":"), dataBytes, COMMA_BYTES);
}

public static byte[] dump(final char[] data, final Charset charset) {
@@ -559,19 +555,6 @@
else if (data instanceof char[]) return dump((char[]) data, charsetForStrings);
else if (data instanceof boolean[]) return dump((boolean[]) data);
throw new IllegalArgumentException("Can't serialize a " + data.getClass().getName());
- }
-
- /** java String.length() does not always equal byte length! */
- private static int byteLength(final CharSequence s) {
- int result = 0;
- for (int i = 0, len = s.length(); i < len; i++) {
- result += byteLength(s.charAt(i));
- }
- return result;
- }
-
- private static int byteLength(final char c) {
- return c > 127 ? 2 : 1;
}

private static byte[] concat(final byte[]... arrays) {
```