Repository: OpenRefine/OpenRefine (java)

Issue:
wikibase: Test failure when run on non-English locales
When running the test suite with a French locale, I get:

```
[ERROR] Failures: 
[ERROR]   PreviewWikibaseSchemaCommandTest.testWarningData:147 expected [country] but found [pays]
```

This is a symptom of the fact that https://github.com/OpenRefine/OpenRefine/pull/6895 (cc @sunilnatraj) picks the language in which to fetch the labels based on the system locale.

Instead, we should change the command to take this language from the HTTP request so that it can match the language used in the UI (which is often not the same thing as the system locale).

As a temporary workaround, one can [change the locale used by Maven](https://stackoverflow.com/questions/8901880/set-surefire-plugin-locale-in-maven-pom-file) to get the test suite to pass.

Pull request:
wikibase: Test failure when run on non-English locales
Fixes #6911 

Changes proposed in this pull request:
- Set language before test as the test validates label which is language specific
- 
- 


Patch:
diff --git a/main/src/com/google/refine/importers/JsonImporter.java b/main/src/com/google/refine/importers/JsonImporter.java
index 1d8505dd9..ce8e0367e 100644
--- a/main/src/com/google/refine/importers/JsonImporter.java
+++ b/main/src/com/google/refine/importers/JsonImporter.java
@@ -44,6 +44,7 @@ import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonParser.NumberType;
 import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.StreamReadFeature;
 import com.fasterxml.jackson.core.json.JsonReadFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -92,7 +93,7 @@ public class JsonImporter extends TreeImportingParserBase {
                 ObjectNode firstFileRecord = fileRecords.get(0);
                 File file = ImportingUtilities.getFile(job, firstFileRecord);
                 JsonFactory factory = JsonFactory.builder().enable(JsonReadFeature.ALLOW_JAVA_COMMENTS)
-                        .enable(JsonReadFeature.ALLOW_YAML_COMMENTS).build();
+                        .enable(JsonReadFeature.ALLOW_YAML_COMMENTS).enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build();
                 JsonParser parser = factory.createParser(file);
                 parser.enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS);
 
@@ -103,6 +104,7 @@ public class JsonImporter extends TreeImportingParserBase {
                 }
             } catch (IOException e) {
                 logger.error("Error generating parser UI initialization data for JSON file", e);
+                JSONUtilities.safePut(options, "error", e.toString());
             }
         }
 
@@ -141,64 +143,52 @@ public class JsonImporter extends TreeImportingParserBase {
         return null;
     }
 
-    final static private JsonNode parseForPreview(JsonParser parser, PreviewParsingState state) {
-        try {
-            JsonToken token = parser.nextToken();
-            state.tokenCount++;
-            return parseForPreview(parser, state, token);
-        } catch (IOException e) {
-            return null;
-        }
+    final static private JsonNode parseForPreview(JsonParser parser, PreviewParsingState state) throws IOException {
+        JsonToken token = parser.nextToken();
+        state.tokenCount++;
+        return parseForPreview(parser, state, token);
     }
 
-    final static private ObjectNode parseObjectForPreview(JsonParser parser, PreviewParsingState state) {
+    final static private ObjectNode parseObjectForPreview(JsonParser parser, PreviewParsingState state) throws IOException {
         ObjectNode result = ParsingUtilities.mapper.createObjectNode();
         loop: while (state.tokenCount < PREVIEW_PARSING_LIMIT) {
-            try {
-                JsonToken token = parser.nextToken();
-                if (token == null) {
-                    break;
-                }
-                state.tokenCount++;
-
-                switch (token) {
-                    case FIELD_NAME:
-                        String fieldName = parser.getText();
-                        JsonNode fieldValue = parseForPreview(parser, state);
-                        JSONUtilities.safePut(result, fieldName, fieldValue);
-                        break;
-                    case END_OBJECT:
-                        break loop;
-                    default:
-                        break loop;
-                }
-            } catch (IOException e) {
+            JsonToken token = parser.nextToken();
+            if (token == null) {
                 break;
             }
+            state.tokenCount++;
+
+            switch (token) {
+                case FIELD_NAME:
+                    String fieldName = parser.getText();
+                    JsonNode fieldValue = parseForPreview(parser, state);
+                    JSONUtilities.safePut(result, fieldName, fieldValue);
+                    break;
+                case END_OBJECT:
+                    break loop;
+                default:
+                    break loop;
+            }
         }
         return result;
     }
 
-    final static private ArrayNode parseArrayForPreview(JsonParser parser, PreviewParsingState state) {
+    final static private ArrayNode parseArrayForPreview(JsonParser parser, PreviewParsingState state) throws IOException {
         ArrayNode result = ParsingUtilities.mapper.createArrayNode();
         loop: while (state.tokenCount < PREVIEW_PARSING_LIMIT) {
-            try {
-                JsonToken token = parser.nextToken();
-                if (token == null) {
-                    break;
-                }
-                state.tokenCount++;
-
-                switch (token) {
-                    case END_ARRAY:
-                        break loop;
-                    default:
-                        JsonNode element = parseForPreview(parser, state, token);
-                        result.add(element);
-                }
-            } catch (IOException e) {
+            JsonToken token = parser.nextToken();
+            if (token == null) {
                 break;
             }
+            state.tokenCount++;
+
+            switch (token) {
+                case END_ARRAY:
+                    break loop;
+                default:
+                    JsonNode element = parseForPreview(parser, state, token);
+                    result.add(element);
+            }
         }
         return result;
     }


Test patch:
diff --git a/extensions/wikibase/tests/src/org/openrefine/wikibase/commands/PreviewWikibaseSchemaCommandTest.java b/extensions/wikibase/tests/src/org/openrefine/wikibase/commands/PreviewWikibaseSchemaCommandTest.java
index e45f0e019..0b9220d15 100644
--- a/extensions/wikibase/tests/src/org/openrefine/wikibase/commands/PreviewWikibaseSchemaCommandTest.java
+++ b/extensions/wikibase/tests/src/org/openrefine/wikibase/commands/PreviewWikibaseSchemaCommandTest.java
@@ -30,6 +30,7 @@ import static org.testng.Assert.assertEquals;
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.Locale;
 
 import javax.servlet.ServletException;
 
@@ -40,6 +41,7 @@ import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
+import com.google.refine.util.LocaleUtils;
 import com.google.refine.util.ParsingUtilities;
 
 import org.openrefine.wikibase.testing.TestingData;
@@ -48,6 +50,8 @@ import org.openrefine.wikibase.utils.EntityCacheStub;
 
 public class PreviewWikibaseSchemaCommandTest extends SchemaCommandTest {
 
+    private String localeSetting;
+
     @BeforeMethod
     public void SetUp() {
         command = new PreviewWikibaseSchemaCommand();
@@ -121,6 +125,17 @@ public class PreviewWikibaseSchemaCommandTest extends SchemaCommandTest {
                 "{\"code\":\"error\",\"message\":\"Wikibase manifest could not be parsed. Error message: invalid manifest format\"}");
     }
 
+    @BeforeMethod
+    public void setLocale() {
+        localeSetting = Locale.getDefault().getLanguage();
+        LocaleUtils.setLocale("en");
+    }
+
+    @AfterMethod
+    public void unsetLocale() {
+        LocaleUtils.setLocale(localeSetting);
+    }
+
     @Test
     public void testWarningData() throws Exception {
 
diff --git a/main/tests/server/src/com/google/refine/importers/JsonImporterTests.java b/main/tests/server/src/com/google/refine/importers/JsonImporterTests.java
index 43dacc72d..726e1a4ce 100644
--- a/main/tests/server/src/com/google/refine/importers/JsonImporterTests.java
+++ b/main/tests/server/src/com/google/refine/importers/JsonImporterTests.java
@@ -33,6 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.importers;
 
+import static org.testng.Assert.assertEquals;
+
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
@@ -40,6 +42,7 @@ import java.io.InputStream;
 import java.io.Serializable;
 import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -109,7 +112,7 @@ public class JsonImporterTests extends ImporterTest {
         Row row = project.rows.get(0);
         Assert.assertNotNull(row);
         Assert.assertNotNull(row.getCell(1));
-        Assert.assertEquals(row.getCell(1).value, "Author 1, The");
+        assertEquals(row.getCell(1).value, "Author 1, The");
     }
 
     @Test
@@ -120,14 +123,27 @@ public class JsonImporterTests extends ImporterTest {
         Row row = project.rows.get(0);
         Assert.assertNotNull(row);
         Assert.assertNotNull(row.getCell(1));
-        Assert.assertEquals(row.getCell(1).value, "Author 1, The");
+        assertEquals(row.getCell(1).value, "Author 1, The");
     }
 
     @Test
-    public void canThrowError() {
+    public void canThrowError() throws IOException {
         String errJSON = getSampleWithError();
+        ObjectNode fileRecord = ParsingUtilities.mapper.createObjectNode();
+        JSONUtilities.safePut(fileRecord, "origin", "clipboard");
+        JSONUtilities.safePut(fileRecord, "declaredEncoding", StandardCharsets.UTF_8.toString());
+        JSONUtilities.safePut(fileRecord, "declaredMimeType", "text/json");
+        JSONUtilities.safePut(fileRecord, "format", "text/json");
+        JSONUtilities.safePut(fileRecord, "fileName", "(clipboard)");
+        File jsonFile = new File(job.getRawDataDir(), "test.json");
+        Files.write(jsonFile.toPath(), errJSON.getBytes(StandardCharsets.UTF_8));
+        JSONUtilities.safePut(fileRecord, "location", "test.json");
         ObjectNode options = SUT.createParserUIInitializationData(
-                job, new LinkedList<>(), "text/json");
+                job, List.of(fileRecord), "text/json");
+        assertEquals(options.get("error").asText(),
+                "com.fasterxml.jackson.core.JsonParseException: Unexpected character (';' (code 59)): was expecting comma to separate Object entries\n"
+                        +
+                        " at [Source: (File); line: 1, column: 11]");
         ArrayNode path = ParsingUtilities.mapper.createArrayNode();
         JSONUtilities.append(path, JsonImporter.ANONYMOUS);
         JSONUtilities.safePut(options, "recordPath", path);
@@ -137,7 +153,7 @@ public class JsonImporterTests extends ImporterTest {
 
         inputStream = new ByteArrayInputStream(errJSON.getBytes(StandardCharsets.UTF_8));
         ImportColumnGroup rootColumnGroup = new ImportColumnGroup();
-        List<Exception> exceptions = new ArrayList<Exception>();
+        List<Exception> exceptions = new ArrayList<>();
 
         SUT.parseOneFile(
                 project,
@@ -150,7 +166,7 @@ public class JsonImporterTests extends ImporterTest {
                 options,
                 exceptions);
         Assert.assertFalse(exceptions.isEmpty());
-        Assert.assertEquals("Unexpected character (';' (code 59)): was expecting comma to separate Object entries",
+        assertEquals("Unexpected character (';' (code 59)): was expecting comma to separate Object entries",
                 exceptions.get(0).getMessage());
     }
 
@@ -369,8 +385,8 @@ public class JsonImporterTests extends ImporterTest {
             }
             i++;
             if (i == 3) {
-                Assert.assertEquals(Token.Value, token);
-                Assert.assertEquals("field", parser.getFieldName());
+                assertEquals(Token.Value, token);
+                assertEquals("field", parser.getFieldName());
             }
         }
 
@@ -384,8 +400,8 @@ public class JsonImporterTests extends ImporterTest {
             }
             i++;
             if (i == 3) {
-                Assert.assertEquals(Token.StartEntity, token);
-                Assert.assertEquals(parser.getFieldName(), "field");
+                assertEquals(Token.StartEntity, token);
+                assertEquals(parser.getFieldName(), "field");
             }
         }
 
@@ -399,16 +415,16 @@ public class JsonImporterTests extends ImporterTest {
             }
             i++;
             if (i == 3) {
-                Assert.assertEquals(token, Token.StartEntity);
-                Assert.assertEquals(parser.getFieldName(), "field");
+                assertEquals(token, Token.StartEntity);
+                assertEquals(parser.getFieldName(), "field");
             }
             if (i == 4) {
-                Assert.assertEquals(token, Token.StartEntity);
-                Assert.assertEquals(parser.getFieldName(), JsonImporter.ANONYMOUS);
+                assertEquals(token, Token.StartEntity);
+                assertEquals(parser.getFieldName(), JsonImporter.ANONYMOUS);
             }
             if (i == 6) {
-                Assert.assertEquals(token, Token.StartEntity);
-                Assert.assertEquals(parser.getFieldName(), JsonImporter.ANONYMOUS);
+                assertEquals(token, Token.StartEntity);
+                assertEquals(parser.getFieldName(), JsonImporter.ANONYMOUS);
             }
         }
     }
@@ -428,9 +444,9 @@ public class JsonImporterTests extends ImporterTest {
             }
             i++;
             if (i == 3) {
-                Assert.assertEquals(Token.Value, token);
-                Assert.assertEquals("\tfield", parser.getFieldName());
-                Assert.assertEquals("\tvalue", parser.getFieldValue());
+                assertEquals(Token.Value, token);
+                assertEquals("\tfield", parser.getFieldName());
+                assertEquals("\tvalue", parser.getFieldValue());
             }
         }
     }