Repository: square/retrofit
Issue: Force UTF8 everywhere. Blow-up otherwise.

It will please @swankjesse.


Issue comments:
#127 


Pull request review comments:
Rather than doing a loop with two splits, you might be better off with a single regex. Perhaps a case-insensitive pattern like `(?i)\Wcharset=([^\s]+)`

(From Guava's tests, content types look like this: `application/atom+xml; a=1; a=2; b=3`)

Guava's tests suggest that charset can be in upper case.

Should pull in tests. I'm slacking.

oooh, and the RFCs:
http://www.ietf.org/rfc/rfc2045.txt
http://www.ietf.org/rfc/rfc2046

Eep, Guava also handles this nastiness:
  text/plain; charset="utf-8"
  text/plain; charset="\u\tf-\8"

IOException ?

nit: something like \W at the beginning as to not match say `text/plain; notthecharset=ascii`.

dig

test that you can have another parameter after the charset?


Patch that resolved the issue:
diff --git a/retrofit/src/main/java/retrofit/http/GsonConverter.java b/retrofit/src/main/java/retrofit/http/GsonConverter.java
index 01213902b6..6dbb992be9 100644
--- a/retrofit/src/main/java/retrofit/http/GsonConverter.java
+++ b/retrofit/src/main/java/retrofit/http/GsonConverter.java
@@ -12,13 +12,14 @@
 import retrofit.io.MimeType;
 import retrofit.io.TypedBytes;
 
+import static retrofit.http.RestAdapter.UTF_8;
+
 /**
  * A {@link Converter} which uses GSON for serialization and deserialization of entities.
  *
  * @author Jake Wharton (jw@squareup.com)
  */
 public class GsonConverter implements Converter {
-  private static final String ENCODING = "UTF-8"; // TODO use actual encoding
   private static final MimeType JSON = new MimeType("application/json", "json");
 
   private final Gson gson;
@@ -29,7 +30,7 @@ public GsonConverter(Gson gson) {
 
   @Override public Object to(byte[] body, Type type) throws ConversionException {
     try {
-      InputStreamReader isr = new InputStreamReader(new ByteArrayInputStream(body), ENCODING);
+      InputStreamReader isr = new InputStreamReader(new ByteArrayInputStream(body), UTF_8);
       return gson.fromJson(isr, type);
     } catch (IOException e) {
       throw new ConversionException(e);
@@ -47,9 +48,9 @@ private static class JsonTypedBytes implements TypedBytes {
 
     JsonTypedBytes(Gson gson, Object object) {
       try {
-        jsonBytes = gson.toJson(object).getBytes(ENCODING);
+        jsonBytes = gson.toJson(object).getBytes(UTF_8);
       } catch (UnsupportedEncodingException e) {
-        throw new IllegalArgumentException(ENCODING + " doesn't exist!?");
+        throw new IllegalStateException(UTF_8 + " encoding does not exist.");
       }
     }
 
diff --git a/retrofit/src/main/java/retrofit/http/HttpMethodType.java b/retrofit/src/main/java/retrofit/http/HttpMethodType.java
index 91be554021..79e755c98e 100644
--- a/retrofit/src/main/java/retrofit/http/HttpMethodType.java
+++ b/retrofit/src/main/java/retrofit/http/HttpMethodType.java
@@ -144,7 +144,6 @@ private static void addParams(HttpEntityEnclosingRequestBase request,
           request.setEntity(entity);
         } else {
           List<NameValuePair> paramList = builder.getParamList(true);
-          // TODO: Use specified encoding. (See CallbackResponseHandler et al)
           request.setEntity(new UrlEncodedFormEntity(paramList, HTTP.UTF_8));
         }
       } catch (UnsupportedEncodingException e) {
diff --git a/retrofit/src/main/java/retrofit/http/RestAdapter.java b/retrofit/src/main/java/retrofit/http/RestAdapter.java
index ba81a910b6..d82e3663c3 100644
--- a/retrofit/src/main/java/retrofit/http/RestAdapter.java
+++ b/retrofit/src/main/java/retrofit/http/RestAdapter.java
@@ -22,6 +22,7 @@
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
 import retrofit.http.HttpProfiler.RequestInformation;
 
@@ -35,6 +36,7 @@ public class RestAdapter {
   private static final Logger LOGGER = Logger.getLogger(RestAdapter.class.getName());
   private static final int LOG_CHUNK_SIZE = 4000;
   static final String THREAD_PREFIX = "Retrofit-";
+  static final String UTF_8 = "UTF-8";
 
   private final Server server;
   private final Provider<HttpClient> httpClientProvider;
@@ -175,7 +177,15 @@ private Object invokeRequest(Method method, Object[] args, boolean isSynchronous
           headers = new Header[realHeaders.length];
           for (int i = 0; i < realHeaders.length; i++) {
             org.apache.http.Header realHeader = realHeaders[i];
-            headers[i] = new Header(realHeader.getName(), realHeader.getValue());
+            String headerName = realHeader.getName();
+            String headerValue = realHeader.getValue();
+
+            if (HTTP.CONTENT_TYPE.equalsIgnoreCase(headerName) //
+                && !UTF_8.equalsIgnoreCase(Utils.parseCharset(headerValue))) {
+              throw new IOException("Only UTF-8 charset supported.");
+            }
+
+            headers[i] = new Header(headerName, headerValue);
           }
         }
 
@@ -200,7 +210,7 @@ private Object invokeRequest(Method method, Object[] args, boolean isSynchronous
   private static void logResponseBody(String url, byte[] body, int statusCode, long elapsedTime)
       throws UnsupportedEncodingException {
     LOGGER.fine("---- HTTP " + statusCode + " from " + url + " (" + elapsedTime + "ms)");
-    String bodyString = new String(body, "UTF-8");
+    String bodyString = new String(body, UTF_8);
     for (int i = 0; i < body.length; i += LOG_CHUNK_SIZE) {
       int end = Math.min(bodyString.length(), i + LOG_CHUNK_SIZE);
       LOGGER.fine(bodyString.substring(i, end));
@@ -271,7 +281,7 @@ static Type getResponseObjectType(Method method, boolean isSynchronousInvocation
           String.format("Last parameter of %s must be a Class or ParameterizedType", method));
     }
     if (Callback.class.isAssignableFrom(callbackClass)) {
-      callbackType = Types.getGenericSupertype(callbackType, callbackClass, Callback.class);
+      callbackType = Utils.getGenericSupertype(callbackType, callbackClass, Callback.class);
       if (callbackType instanceof ParameterizedType) {
         Type[] types = ((ParameterizedType) callbackType).getActualTypeArguments();
         for (int i = 0; i < types.length; i++) {
diff --git a/retrofit/src/main/java/retrofit/http/Types.java b/retrofit/src/main/java/retrofit/http/Utils.java
similarity index 71%
rename from retrofit/src/main/java/retrofit/http/Types.java
rename to retrofit/src/main/java/retrofit/http/Utils.java
index 548d630153..60cc067931 100644
--- a/retrofit/src/main/java/retrofit/http/Types.java
+++ b/retrofit/src/main/java/retrofit/http/Utils.java
@@ -1,18 +1,22 @@
-// Copyright 2008 Google, Inc.
+// Copyright 2012 Square, Inc.
 package retrofit.http;
 
 import java.lang.reflect.Type;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
+import static retrofit.http.RestAdapter.UTF_8;
+
+final class Utils {
+  private static final Pattern CHARSET = Pattern.compile("\\Wcharset=([^\\s;]+)", CASE_INSENSITIVE);
 
-/**
- * Helper methods for dealing with generic types via reflection copied from Guice's {@code
- * MoreTypes} class.
- */
-class Types {
   /**
    * Returns the generic supertype for {@code supertype}. For example, given a class
    * {@code IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>}
    * and the result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.
    */
+  // Copied from Guice's {@code MoreTypes} class. Copyright 2006 Google, Inc.
   static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {
     if (toResolve == rawType) {
       return context;
@@ -46,4 +50,12 @@ static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResol
     // we can't resolve this further
     return toResolve;
   }
+
+  static String parseCharset(String headerValue) {
+    Matcher match = CHARSET.matcher(headerValue);
+    if (match.find()) {
+      return match.group(1).replaceAll("[\"\\\\]", "");
+    }
+    return UTF_8;
+  }
 }
\ No newline at end of file


Test patch:
diff --git a/retrofit/src/test/java/retrofit/http/UtilsTest.java b/retrofit/src/test/java/retrofit/http/UtilsTest.java
new file mode 100644
index 0000000000..e529f7f033
--- /dev/null
+++ b/retrofit/src/test/java/retrofit/http/UtilsTest.java
@@ -0,0 +1,24 @@
+// Copyright 2012 Square, Inc.
+package retrofit.http;
+
+import org.junit.Test;
+
+import static org.fest.assertions.api.Assertions.assertThat;
+import static retrofit.http.RestAdapter.UTF_8;
+import static retrofit.http.Utils.parseCharset;
+
+public class UtilsTest {
+  @Test public void testCharsetParsing() {
+    assertThat(parseCharset("text/plain;charset=utf-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; charset=utf-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain;  charset=utf-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; \tcharset=utf-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; \r\n\tcharset=utf-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; CHARSET=utf-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; charset=UTF-8")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; charset=\"\\u\\tf-\\8\"")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; charset=\"utf-8\"")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; charset=utf-8; other=thing")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain; notthecharset=utf-16;")).isEqualToIgnoringCase(UTF_8);
+  }
+}