Repository: square/retrofit
Issue: Relax UTF-8 requirements or make configurable

In RestAdapter.invokeRequest(RestMethodInfo methodDetails, Object[] args)

this check (line ~186)

``` java
        for (Header header : headers) {
          if (HTTP.CONTENT_TYPE.equalsIgnoreCase(header.getName()) //
              && !UTF_8.equalsIgnoreCase(Utils.parseCharset(header.getValue()))) {
            throw new IOException("Only UTF-8 charset supported.");
          }
        }
```

Naturally, bombs out on sites that don't "behave properly", like rottentomatoes api.  Can this check be made more flexible or removed altogether?


Issue comments:
We force the implementations of `Client` to parse the "Content-Type" header in order to create a `TypedInput` for the response. This `TypedInput` is handed directly to the response converter which is responsible for decoding the bytes in accordance to the MIME type and encoding. Having the iteration here is not only redundant, but unnecessary.

It should be the responsibility of the converter to determine whether or not a MIME type is invalid. We probably shouldn't be enforcing this by default since we don't live in a perfect world of well-behaved servers. Theoretically you could deal with a server that sends down ASCII text documents and your converter turns them into `String`s which should be a perfectly valid use-case.

@brh could you report this as a bug against Rotton Tomatoes? We're never going to get out of this charset hell until everyone goes to UTF-8.

HA!  There are a few defects or requests I have saw on their page that are several years old and they have not gotten any reply at all.  Will look into it though.


Pull request review comments:

Patch that resolved the issue:
diff --git a/retrofit/src/main/java/retrofit/http/GsonConverter.java b/retrofit/src/main/java/retrofit/http/GsonConverter.java
index ae032b3e68..a2372291bb 100644
--- a/retrofit/src/main/java/retrofit/http/GsonConverter.java
+++ b/retrofit/src/main/java/retrofit/http/GsonConverter.java
@@ -11,8 +11,6 @@
 import retrofit.http.mime.TypedInput;
 import retrofit.http.mime.TypedOutput;
 
-import static retrofit.http.RestAdapter.UTF_8;
-
 /**
  * A {@link Converter} which uses GSON for serialization and deserialization of entities.
  *
@@ -26,9 +24,10 @@ public GsonConverter(Gson gson) {
   }
 
   @Override public Object fromBody(TypedInput body, Type type) throws ConversionException {
+    String charset = Utils.parseCharset(body.mimeType());
     InputStreamReader isr = null;
     try {
-      isr = new InputStreamReader(body.in(), UTF_8);
+      isr = new InputStreamReader(body.in(), charset);
       return gson.fromJson(isr, type);
     } catch (IOException e) {
       throw new ConversionException(e);
@@ -46,7 +45,7 @@ public GsonConverter(Gson gson) {
 
   @Override public TypedOutput toBody(Object object) {
     try {
-      return new JsonTypedOutput(gson.toJson(object).getBytes(UTF_8));
+      return new JsonTypedOutput(gson.toJson(object).getBytes("UTF-8"));
     } catch (UnsupportedEncodingException e) {
       throw new AssertionError(e);
     }
diff --git a/retrofit/src/main/java/retrofit/http/RequestBuilder.java b/retrofit/src/main/java/retrofit/http/RequestBuilder.java
index d73adcdd8b..a7b8495690 100644
--- a/retrofit/src/main/java/retrofit/http/RequestBuilder.java
+++ b/retrofit/src/main/java/retrofit/http/RequestBuilder.java
@@ -11,7 +11,6 @@
 import retrofit.http.mime.TypedOutput;
 import retrofit.http.mime.TypedString;
 
-import static retrofit.http.RestAdapter.UTF_8;
 import static retrofit.http.RestMethodInfo.NO_SINGLE_ENTITY;
 
 /**
@@ -156,7 +155,7 @@ Request build() {
 
   private static String getUrlEncodedValue(Parameter found) {
     try {
-      return URLEncoder.encode(String.valueOf(found.getValue()), UTF_8);
+      return URLEncoder.encode(String.valueOf(found.getValue()), "UTF-8");
     } catch (UnsupportedEncodingException e) {
       throw new AssertionError(e);
     }
diff --git a/retrofit/src/main/java/retrofit/http/RestAdapter.java b/retrofit/src/main/java/retrofit/http/RestAdapter.java
index 3a0503674e..fd10dff55b 100644
--- a/retrofit/src/main/java/retrofit/http/RestAdapter.java
+++ b/retrofit/src/main/java/retrofit/http/RestAdapter.java
@@ -8,7 +8,6 @@
 import java.lang.reflect.Proxy;
 import java.lang.reflect.Type;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
@@ -34,7 +33,6 @@ public class RestAdapter {
   private static final Logger LOGGER = Logger.getLogger(RestAdapter.class.getName());
   private static final int LOG_CHUNK_SIZE = 4000;
   static final String THREAD_PREFIX = "Retrofit-";
-  static final String UTF_8 = "UTF-8";
 
   private final Server server;
   private final Client.Provider clientProvider;
@@ -183,14 +181,6 @@ private Object invokeRequest(RestMethodInfo methodDetails, Object[] args) {
           body = logResponse(url, response.getStatus(), body, elapsedTime);
         }
 
-        List<Header> headers = response.getHeaders();
-        for (Header header : headers) {
-          if ("Content-Type".equalsIgnoreCase(header.getName()) //
-              && !UTF_8.equalsIgnoreCase(Utils.parseCharset(header.getValue()))) {
-            throw new IOException("Only UTF-8 charset supported.");
-          }
-        }
-
         Type type = methodDetails.type;
         if (statusCode >= 200 && statusCode < 300) { // 2XX == successful request
           if (type.equals(Response.class)) {
@@ -230,7 +220,8 @@ private static TypedInput logResponse(String url, int statusCode, TypedInput bod
     LOGGER.fine("<--- HTTP " + statusCode + " " + url + " (" + elapsedTime + "ms)");
 
     byte[] bodyBytes = Utils.streamToBytes(body.in());
-    String bodyString = new String(bodyBytes, UTF_8);
+    String bodyCharset = Utils.parseCharset(body.mimeType());
+    String bodyString = new String(bodyBytes, bodyCharset);
     for (int i = 0; i < bodyString.length(); i += LOG_CHUNK_SIZE) {
       int end = Math.min(bodyString.length(), i + LOG_CHUNK_SIZE);
       LOGGER.fine(bodyString.substring(i, end));
diff --git a/retrofit/src/main/java/retrofit/http/Utils.java b/retrofit/src/main/java/retrofit/http/Utils.java
index c5d89a6494..7e4789f6c2 100644
--- a/retrofit/src/main/java/retrofit/http/Utils.java
+++ b/retrofit/src/main/java/retrofit/http/Utils.java
@@ -11,9 +11,8 @@
 import java.util.regex.Pattern;
 
 import static java.util.regex.Pattern.CASE_INSENSITIVE;
-import static retrofit.http.RestAdapter.UTF_8;
 
-final class Utils {
+public final class Utils {
   private static final Pattern CHARSET = Pattern.compile("\\Wcharset=([^\\s;]+)", CASE_INSENSITIVE);
   private static final int BUFFER_SIZE = 0x1000;
 
@@ -75,12 +74,12 @@ static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResol
     return toResolve;
   }
 
-  static String parseCharset(String mimeType) {
+  public static String parseCharset(String mimeType) {
     Matcher match = CHARSET.matcher(mimeType);
     if (match.find()) {
       return match.group(1).replaceAll("[\"\\\\]", "");
     }
-    return UTF_8;
+    return "UTF-8";
   }
 
   static class SynchronousExecutor implements Executor {
@@ -88,4 +87,8 @@ static class SynchronousExecutor implements Executor {
       runnable.run();
     }
   }
+
+  public Utils() {
+    // No instances.
+  }
 }
\ No newline at end of file


Test patch:
diff --git a/retrofit/src/test/java/retrofit/http/UtilsTest.java b/retrofit/src/test/java/retrofit/http/UtilsTest.java
index a81347f377..5176f02309 100644
--- a/retrofit/src/test/java/retrofit/http/UtilsTest.java
+++ b/retrofit/src/test/java/retrofit/http/UtilsTest.java
@@ -4,21 +4,20 @@
 import org.junit.Test;
 
 import static org.fest.assertions.api.Assertions.assertThat;
-import static retrofit.http.RestAdapter.UTF_8;
 import static retrofit.http.Utils.parseCharset;
 
 public class UtilsTest {
   @Test public void charsetParsing() {
-    assertThat(parseCharset("text/plain;charset=utf-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; charset=utf-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain;  charset=utf-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; \tcharset=utf-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; \r\n\tcharset=utf-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; CHARSET=utf-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; charset=UTF-8")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; charset=\"\\u\\tf-\\8\"")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; charset=\"utf-8\"")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; charset=utf-8; other=thing")).isEqualToIgnoringCase(UTF_8);
-    assertThat(parseCharset("text/plain; notthecharset=utf-16;")).isEqualToIgnoringCase(UTF_8);
+    assertThat(parseCharset("text/plain;charset=utf-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; charset=utf-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain;  charset=utf-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; \tcharset=utf-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; \r\n\tcharset=utf-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; CHARSET=utf-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; charset=UTF-8")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; charset=\"\\u\\tf-\\8\"")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; charset=\"utf-8\"")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain;charset=utf-8;other=thing")).isEqualToIgnoringCase("UTF-8");
+    assertThat(parseCharset("text/plain; notthecharset=utf-16;")).isEqualToIgnoringCase("UTF-8");
   }
 }