Repository: Microsoft/thrifty (kotlin)

Issue:
[BUG] non-ascii characters are broken in Simple JSON protocol
non-ascii characters in string are broken when encoding struct to json using SimpleJsonProtocol.

```
   "autocomplete_suggestion_display_strings": [
      "San Francisco, California",
      "Beijing � z\/�1P",
      "Montreal � 1|�Plus?�",
      "New York � S�",
      "\u0005� � S�",
      "�� � S�",
```
After search, I find that the cause is casting java character directly to byte in this method:
```
    @Override
    public void writeString(String str) throws IOException {
        writeContext().beforeWrite();

        int len = str.length();
        UnsafeByteArrayOutputStream baos = new UnsafeByteArrayOutputStream(len);

...
            } else {
                baos.write(c);
                           ^ this method accepts a char but directly cast to byte and store into a bytearray
            }
        }

...
    }
```

```
public synchronized void write(int b) {
        ensureCapacity(count + 1);
        buf[count] = (byte) b;
                       ^ direct cast happens here
        count += 1;
    }
```

I suggest to use a utf8 encoder to encode char to bytes

Pull request:
Fixes #316

Patch:
diff --git a/thrifty-runtime/src/main/java/com/microsoft/thrifty/protocol/SimpleJsonProtocol.java b/thrifty-runtime/src/main/java/com/microsoft/thrifty/protocol/SimpleJsonProtocol.java
index 245c5a3e3..86e2dd34d 100644
--- a/thrifty-runtime/src/main/java/com/microsoft/thrifty/protocol/SimpleJsonProtocol.java
+++ b/thrifty-runtime/src/main/java/com/microsoft/thrifty/protocol/SimpleJsonProtocol.java
@@ -25,6 +25,7 @@
 import okio.ByteString;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.net.ProtocolException;
 import java.nio.charset.Charset;
 import java.util.ArrayDeque;
@@ -117,7 +118,7 @@ void onPop() throws IOException {
         }
     }
 
-    private static final byte[][] ESCAPES;
+    private static final char[][] ESCAPES;
     private static final Charset UTF8;
 
     private static final byte[] TRUE = { 't', 'r', 'u', 'e' };
@@ -131,21 +132,21 @@ void onPop() throws IOException {
     private static final byte[] RBRACE = { '}' };
 
     static {
-        ESCAPES = new byte[128][];
+        ESCAPES = new char[128][];
 
         UTF8 = Charset.forName("UTF-8");
         for (int i = 0; i < 32; ++i) {
             // Control chars must be escaped
-            ESCAPES[i] = String.format("\\u%04x", i).getBytes(UTF8);
+            ESCAPES[i] = String.format("\\u%04x", i).toCharArray();
         }
 
-        ESCAPES['\\'] = new byte[] { '\\', '\\' };
-        ESCAPES['\"'] = new byte[] { '\\', '"' };
-        ESCAPES['\b'] = new byte[] { '\\', 'b' };
-        ESCAPES['\f'] = new byte[] { '\\', 'f' };
-        ESCAPES['\r'] = new byte[] { '\\', 'r' };
-        ESCAPES['\n'] = new byte[] { '\\', 'n' };
-        ESCAPES['\t'] = new byte[] { '\\', 't' };
+        ESCAPES['\\'] = new char[] { '\\', '\\' };
+        ESCAPES['\"'] = new char[] { '\\', '"' };
+        ESCAPES['\b'] = new char[] { '\\', 'b' };
+        ESCAPES['\f'] = new char[] { '\\', 'f' };
+        ESCAPES['\r'] = new char[] { '\\', 'r' };
+        ESCAPES['\n'] = new char[] { '\\', 'n' };
+        ESCAPES['\t'] = new char[] { '\\', 't' };
     }
 
     private final WriteContext defaultWriteContext = new WriteContext() {
@@ -296,22 +297,24 @@ public void writeString(String str) throws IOException {
 
         int len = str.length();
         UnsafeByteArrayOutputStream baos = new UnsafeByteArrayOutputStream(len);
+        OutputStreamWriter writer = new OutputStreamWriter(baos, UTF8);
 
-        baos.write('"');
+        writer.write('"');
         for (int i = 0; i < len; ++i) {
             char c = str.charAt(i);
             if (c < 128) {
-                byte[] maybeEscape = ESCAPES[c];
+                char[] maybeEscape = ESCAPES[c];
                 if (maybeEscape != null) {
-                    baos.write(maybeEscape);
+                    writer.write(maybeEscape);
                 } else {
-                    baos.write(c);
+                    writer.write(c);
                 }
             } else {
-                baos.write(c);
+                writer.write(c);
             }
         }
-        baos.write('"');
+        writer.write('"');
+        writer.flush();
 
         transport.write(baos.getBuffer(), 0, baos.size());
     }


Test patch:
diff --git a/thrifty-runtime/src/test/java/com/microsoft/thrifty/protocol/SimpleJsonProtocolTest.java b/thrifty-runtime/src/test/java/com/microsoft/thrifty/protocol/SimpleJsonProtocolTest.java
index 22bcc681d..cc0fdab01 100644
--- a/thrifty-runtime/src/test/java/com/microsoft/thrifty/protocol/SimpleJsonProtocolTest.java
+++ b/thrifty-runtime/src/test/java/com/microsoft/thrifty/protocol/SimpleJsonProtocolTest.java
@@ -24,22 +24,14 @@
 import com.microsoft.thrifty.transport.BufferTransport;
 import okio.Buffer;
 import okio.ByteString;
-import org.junit.Before;
 import org.junit.Test;
 
 import static com.google.common.truth.Truth.assertThat;
 
 public class SimpleJsonProtocolTest {
-    private Buffer buffer;
-    private BufferTransport transport;
-    private SimpleJsonProtocol protocol;
-
-    @Before
-    public void setup() {
-        buffer = new Buffer();
-        transport = new BufferTransport(buffer);
-        protocol = new SimpleJsonProtocol(transport);
-    }
+    private Buffer buffer = new Buffer();
+    private BufferTransport transport = new BufferTransport(buffer);
+    private SimpleJsonProtocol protocol = new SimpleJsonProtocol(transport);
 
     @Test
     public void emptyJsonString() throws Exception {
@@ -177,4 +169,10 @@ public void b64BinaryOutputMode() throws Exception {
 
         assertThat(buffer.readUtf8()).isEqualTo("\"Zm9vYmFy\"");
     }
+
+    @Test
+    public void nonAsciiCharacters() throws Exception {
+        protocol.writeString("测试");
+        assertThat(buffer.readUtf8()).isEqualTo("\"测试\"");
+    }
 }
\ No newline at end of file