Repository: doccano/doccano

Issue:
Metadata column repeated when exported as csv
Hi I have recently come across a bug when you export data as csv
<environment.-->
*   Operating System:MacOS 10.14
*   Python Version Used: 3.9.5
*   Doccano installed through pip3 install     doccano 

I have created a DocumentClassification project and have imported some json data. 

The json data is in the format of 

```bash
{"text":"The ravioli was excellent" , "hidden":"The FOOD was excellent"} 
```

When these sentences are imported, the "hidden" : "The FOOD was excellent" becomes part of the Metadata. I have quite a few of these sentences and have labelled them with my own labels 

The issue is when I export the dataset as csv, the Metadata column repeats. For example if I have 10 labelled sentences, the Metadata column is repeated 10 times per row of data in excel. 


Discussion:

Resolving patch:
diff --git a/backend/api/views/download/data.py b/backend/api/views/download/data.py
--- a/backend/api/views/download/data.py
+++ b/backend/api/views/download/data.py
@@ -1,3 +1,4 @@
+import json
 from typing import Any, Dict, List
 
 
@@ -16,4 +17,10 @@ def __init__(self,
         self.metadata = metadata
 
     def __str__(self):
-        return f'{self.data}\t{self.label}'
+        return json.dumps({
+            'id': self.id,
+            'data': self.data,
+            'label': self.label,
+            'user': self.user,
+            'metadata': self.metadata
+        })
diff --git a/backend/api/views/download/writer.py b/backend/api/views/download/writer.py
--- a/backend/api/views/download/writer.py
+++ b/backend/api/views/download/writer.py
@@ -90,7 +90,7 @@ def create_line(self, record) -> Dict:
 
     def create_header(self, records: List[Record]) -> Iterable[str]:
         header = ['id', 'data', 'label']
-        header += list(itertools.chain(*[r.metadata.keys() for r in records]))
+        header += sorted(set(itertools.chain(*[r.metadata.keys() for r in records])))
         return header
 
 


Test patch:
diff --git a/backend/api/tests/download/__init__.py b/backend/api/tests/download/__init__.py
new file mode 100644
diff --git a/backend/api/tests/download/test_writer.py b/backend/api/tests/download/test_writer.py
new file mode 100644
--- /dev/null
+++ b/backend/api/tests/download/test_writer.py
@@ -0,0 +1,53 @@
+import unittest
+from unittest.mock import call, patch
+
+from ...views.download.data import Record
+from ...views.download.writer import CsvWriter
+
+
+class TestCSVWriter(unittest.TestCase):
+
+    def setUp(self):
+        self.records = [
+            Record(id=0, data='exampleA', label=['labelA'], user='admin', metadata={'hidden': 'secretA'}),
+            Record(id=1, data='exampleB', label=['labelB'], user='admin', metadata={'hidden': 'secretB'}),
+            Record(id=2, data='exampleC', label=['labelC'], user='admin', metadata={'meta': 'secretC'})
+        ]
+
+    def test_create_header(self):
+        writer = CsvWriter('.')
+        header = writer.create_header(self.records)
+        expected = ['id', 'data', 'label', 'hidden', 'meta']
+        self.assertEqual(header, expected)
+
+    def test_create_line(self):
+        writer = CsvWriter('.')
+        record = self.records[0]
+        line = writer.create_line(record)
+        expected = {
+            'id': record.id,
+            'data': record.data,
+            'label': record.label[0],
+            'hidden': 'secretA'
+        }
+        self.assertEqual(line, expected)
+
+    @patch('os.remove')
+    @patch('zipfile.ZipFile')
+    @patch('csv.DictWriter.writerow')
+    @patch('builtins.open')
+    def test_dump(self, mock_open_file, csv_io, zip_io, mock_remove_file):
+        writer = CsvWriter('.')
+        writer.write(self.records)
+
+        self.assertEqual(mock_open_file.call_count, 1)
+        mock_open_file.assert_called_with('./admin.csv', mode='a', encoding='utf-8')
+
+        self.assertEqual(csv_io.call_count, len(self.records) + 1)  # +1 is for a header
+        calls = [
+            call({'id': 'id', 'data': 'data', 'label': 'label', 'hidden': 'hidden', 'meta': 'meta'}),
+            call({'id': 0, 'data': 'exampleA', 'label': 'labelA', 'hidden': 'secretA'}),
+            call({'id': 1, 'data': 'exampleB', 'label': 'labelB', 'hidden': 'secretB'}),
+            call({'id': 2, 'data': 'exampleC', 'label': 'labelC', 'meta': 'secretC'})
+        ]
+        csv_io.assert_has_calls(calls)