Repository: fossasia/open-event-server

Issue:
Export CSV files of Attendees with random, duplicate and unsorted data
There are several issues with the exported CSV file of attendees and orders. To replicate the issue download a CSV from the attendee or order overview.

![Screenshot from 2023-06-26 13-50-37](https://github.com/fossasia/open-event-frontend/assets/1583873/c791b410-32a8-47c2-9150-cb0c8006708c)

Then check the CSVs of attendees and orders. They have random and duplicate data.

1. Firstly in a perfect way the columns would be ordered according to the order of questions on the web front-end.
2. Secondly, the columns with the data on Email consent, photo consent and other columns appear several times
3. Thirdly specifically the following columns are in the wrong place:
* Column AF "Email" should be behind column "O (Zipcode)
* Column P "áa" seems to have some random information
* Column Q to AE should be coming after column BG
4. The answer in the forms shows "True" or "False". Normal users do not know what this means. Just show the answer in the default language, e.g. "Yes".

[attendees-dc360898141111eea9e90242c0a87005.csv](https://github.com/fossasia/open-event-frontend/files/11868359/attendees-dc360898141111eea9e90242c0a87005.csv)



Discussion:

Resolving patch:
diff --git a/app/api/helpers/csv_jobs_util.py b/app/api/helpers/csv_jobs_util.py
--- a/app/api/helpers/csv_jobs_util.py
+++ b/app/api/helpers/csv_jobs_util.py
@@ -57,72 +57,65 @@ def export_orders_csv(orders):
     return rows
 
 
-def export_attendees_csv(attendees, custom_forms):
+def export_attendees_csv(attendees, custom_forms, attendee_form_dict):
+    return_dict_list = []
 
-    headers = [
-        'Order#',
-        'Order Date',
-        'Status',
-        'Payment Type',
-        'Payment Mode',
-        'Ticket Name',
-        'Ticket Price',
-        'Ticket Type',
-        'Tax ID',
-        'Address',
-        'Company',
-        'Country',
-        'State',
-        'City',
-        'Zipcode',
-    ]
-
-    for fields in custom_forms:
-        headers.append(fields.name)
-
-    rows = [headers]
     for attendee in attendees:
-        column = [
-            str(attendee.order.get_invoice_number()) if attendee.order else '-',
-            str(attendee.order.created_at.strftime('%B %-d, %Y %H:%M %z'))
+        data = {
+            'Order#': str(attendee.order.get_invoice_number()) if attendee.order else '-',
+            'Order Date': str(attendee.order.created_at.strftime('%B %-d, %Y %H:%M %z'))
             if attendee.order and attendee.order.created_at
             else '-',
-            str(attendee.order.status)
+            'Status': str(attendee.order.status)
             if attendee.order and attendee.order.status
             else '-',
-            str(attendee.order.paid_via)
+            'Payment Type': str(attendee.order.paid_via)
             if attendee.order and attendee.order.paid_via
             else '',
-            str(attendee.order.payment_mode)
+            'Payment Mode':  str(attendee.order.payment_mode)
             if attendee.order and attendee.order.payment_mode
             else '',
-            str(attendee.ticket.name) if attendee.ticket and attendee.ticket.name else '',
-            str(attendee.ticket.price)
+            'Ticket Name':  str(attendee.ticket.name)
+            if attendee.ticket and attendee.ticket.name else '',
+            'Ticket Price': str(attendee.ticket.price)
             if attendee.ticket and attendee.ticket.price
             else '0',
-            str(attendee.ticket.type) if attendee.ticket and attendee.ticket.type else '',
-            str(attendee.order.tax_business_info)
+            'Ticket Type': str(attendee.ticket.type)
+            if attendee.ticket and attendee.ticket.type else '',
+            'Tax ID': str(attendee.order.tax_business_info)
             if attendee.order.tax_business_info
             else '',
-            str(attendee.order.address) if attendee.order.address else '',
-            str(attendee.order.company) if attendee.order.company else '',
-            str(attendee.order.country) if attendee.order.country else '',
-            str(attendee.order.state) if attendee.order.state else '',
-            str(attendee.order.city) if attendee.order.city else '',
-            str(attendee.order.zipcode) if attendee.order.zipcode else '',
-        ]
+            'Address': str(attendee.order.address) if attendee.order.address else '',
+            'Company':  str(attendee.order.company) if attendee.order.company else '',
+            'Country': str(attendee.order.country) if attendee.order.country else '',
+            'State':  str(attendee.order.state) if attendee.order.state else '',
+            'City': str(attendee.order.city) if attendee.order.city else '',
+            'Zipcode': str(attendee.order.zipcode) if attendee.order.zipcode else '',
+        }
+
         for field in custom_forms:
+            # keys don't match up, for keys like
+            # acceptVideoRecording vs accept_video_recording ..
+            key_mapping = {}
+
+            for k in attendee_form_dict.keys():
+                key_mapping[k.replace("_", "").lower()] = k
+
+            field_raw = field.identifier.replace("_", "").lower()
+            key = key_mapping.get(field_raw)
+            converted_header = attendee_form_dict.get(key)
             if field.is_complex:
                 fields_dict = attendee.complex_field_values
-                column.append(
-                    fields_dict.get(field.identifier, '') if fields_dict else ''
-                )
+                data[converted_header] = fields_dict.get(
+                    field.identifier, '') if fields_dict else ''
             else:
-                column.append(getattr(attendee, field.identifier, ''))
+                dict_value = (getattr(attendee, field.identifier, ''))
+                dict_value = "Yes" if str(dict_value) == "True" \
+                    else "No" if str(dict_value) == "False" else dict_value
+                data[converted_header] = dict_value
+        return_dict_list.append(data)
 
-        rows.append(column)
-
-    return rows
+    return return_dict_list
 
 
 def export_sessions_csv(sessions):
diff --git a/app/api/helpers/tasks.py b/app/api/helpers/tasks.py
--- a/app/api/helpers/tasks.py
+++ b/app/api/helpers/tasks.py
@@ -39,7 +39,7 @@
 from app.api.imports import import_event_task_base
 from app.instance import create_app
 from app.models import db
-from app.models.custom_form import CustomForms
+from app.models.custom_form import CustomForms, ATTENDEE_CUSTOM_FORM
 from app.models.discount_code import DiscountCode
 from app.models.event import Event
 from app.models.exhibitor import Exhibitor
@@ -52,8 +52,8 @@
 from app.models.user import User
 from app.models.user_follow_group import UserFollowGroup
 from app.settings import get_settings
-
 from .import_helpers import update_import_job
+from app.api.helpers.csv_jobs_util import export_attendees_csv
 
 """
 Define all API v2 celery tasks here
@@ -541,20 +541,43 @@ def export_attendees_csv_task(self, event_id):
     custom_forms = db.session.query(CustomForms).filter_by(
         event_id=event_id, form=CustomForms.TYPE.ATTENDEE, is_included=True
     )
+
+    field_headers = list(ATTENDEE_CUSTOM_FORM.keys())
+
+    def custom_form_validation(cf_orm, field_headers):
+        # set() is O(1) in membership testing
+        field_headers_set = set(field_headers)
+        forms_result = [None] * len(field_headers_set)
+
+        for row in cf_orm:
+            if row.field_identifier in field_headers_set:
+
+                field_headers_set.discard(row.field_identifier)
+                index_append = field_headers.index(row.field_identifier)
+                # forms_result.append(row)
+                forms_result.insert(index_append, row)
+
+        forms_result = [e for e in forms_result if e is not None]
+        return forms_result
+
     try:
+        custom_forms = custom_form_validation(custom_forms, field_headers)
         filedir = os.path.join(current_app.config.get('BASE_DIR'), 'static/uploads/temp/')
         if not os.path.isdir(filedir):
             os.makedirs(filedir)
         filename = f"attendees-{uuid.uuid1().hex}.csv"
         file_path = os.path.join(filedir, filename)
 
+        dict_list = export_attendees_csv(attendees, custom_forms, ATTENDEE_CUSTOM_FORM)
+        csv_headers = list(dict_list[0].keys())
+
         with open(file_path, "w") as temp_file:
-            writer = csv.writer(temp_file)
-            from app.api.helpers.csv_jobs_util import export_attendees_csv
+            writer = csv.DictWriter(temp_file, fieldnames=csv_headers)
+            writer.writeheader()
 
-            content = export_attendees_csv(attendees, custom_forms)
-            for row in content:
+            for row in dict_list:
                 writer.writerow(row)
+
         attendees_csv_file = UploadedFile(file_path=file_path, filename=filename)
         attendees_csv_url = upload(
             attendees_csv_file,


Test patch:
diff --git a/tests/all/integration/api/helpers/test_csv_jobs_util.py b/tests/all/integration/api/helpers/test_csv_jobs_util.py
--- a/tests/all/integration/api/helpers/test_csv_jobs_util.py
+++ b/tests/all/integration/api/helpers/test_csv_jobs_util.py
@@ -11,6 +11,7 @@
 from tests.factories.order import OrderFactory
 from tests.factories.session import SessionSubFactory
 from tests.factories.speaker import SpeakerFactory
+from app.models.custom_form import ATTENDEE_CUSTOM_FORM
 
 
 class TestExportCSV(OpenEventTestCase):
@@ -32,8 +33,10 @@ def test_export_attendees_csv(self):
             test_order = OrderFactory(created_at=datetime.now())
             test_attendee.order = test_order
             custom_forms = CustomFormFactory()
-            field_data = export_attendees_csv([test_attendee], [custom_forms])
-            assert field_data[1][8] == 'tax id'
+            field_data = export_attendees_csv(
+                [test_attendee], [custom_forms], ATTENDEE_CUSTOM_FORM)
+            # new export_attendees_csv will return list of dictionary for csv_writer
+            assert field_data[0].get("Tax ID") == "tax id"
 
     def _test_export_session_csv(self, test_session=None):
         with self.app.test_request_context():