Repository: e-valuation/EvaP

Issue:
Openpyxl `max_column` computation bottlenecks importers
The importers currently take seconds up to minutes to run. There's no inherent reason for that, all processing logic inside of them is designed to have O(n) runtime with O(1) database queries. A quick profiling run of the import view shows that we spend most time in openpyxl's `worksheet.py` file in [`max_column`](https://foss.heptapod.net/openpyxl/openpyxl/-/blob/branch/3.0/openpyxl/worksheet/worksheet.py#L365-L375) and a generator expression creating the set `cols`:
```python3
    @property
    def max_column(self):
        """The maximum column index containing data (1-based)

        :type: int
        """
        max_col = 1
        if self._cells:
            cols = set(c[1] for c in self._cells)  ##### line 373 #####
            max_col = max(cols)
        return max_col
```

![photo_5289897819194377466_y](https://user-images.githubusercontent.com/13838962/209836150-f999e9a0-36fc-475e-857e-b00c64dd34a9.jpg)

I feel like figuring out how many columns a 200kB (12 x 6000 cells) excel file has should be possible in less than 60 seconds, even in a debug-profiling run.
They have an open issue here: https://foss.heptapod.net/openpyxl/openpyxl/-/issues/1763

If I read the output above right, we have 3000 accesses to max_column, just reusing the computed value might be enough for now.


Discussion:

Resolving patch:
diff --git a/evap/staff/importers/base.py b/evap/staff/importers/base.py
--- a/evap/staff/importers/base.py
+++ b/evap/staff/importers/base.py
@@ -216,10 +216,11 @@ def map(self, file_content: bytes):
                     )
                 )
 
-            for row_index in range(self.skip_first_n_rows, sheet.max_row):
-                # We use 0-based indexing, openpyxl uses 1-based indexing.
-                row = [cell.value for cell in sheet[row_index + 1]]
-                location = ExcelFileLocation(sheet.title, row_index)
+            # openpyxl uses 1-based indexing.
+            for row_number, row in enumerate(
+                sheet.iter_rows(min_row=self.skip_first_n_rows + 1, values_only=True), start=self.skip_first_n_rows
+            ):
+                location = ExcelFileLocation(sheet.title, row_number)
 
                 if not all(isinstance(cell, str) or cell is None for cell in row):
                     self.importer_log.add_error(
diff --git a/evap/staff/importers/enrollment.py b/evap/staff/importers/enrollment.py
--- a/evap/staff/importers/enrollment.py
+++ b/evap/staff/importers/enrollment.py
@@ -321,7 +321,7 @@ class NameEnCollisionException(Exception):
         """Course with same name_en, but different name_de exists"""
 
     def __init__(self, semester: Semester):
-        courses = Course.objects.filter(semester=semester).prefetch_related("responsibles", "evaluations")
+        courses = Course.objects.filter(semester=semester).prefetch_related("type", "responsibles", "evaluations")
 
         assert ("semester", "name_de") in Course._meta.unique_together
         self.courses_by_name_de = {course.name_de: course for course in courses}
@@ -339,9 +339,10 @@ def get_merge_hindrances(course_data: CourseData, merge_candidate: Course) -> Li
         if len(responsibles) != 1 or responsibles[0].email != course_data.responsible_email:
             hindrances.append(_("the responsibles of the course do not match"))
 
-        if len(merge_candidate.evaluations.all()) != 1:
+        merge_candidate_evaluations = merge_candidate.evaluations.all()
+        if len(merge_candidate_evaluations) != 1:
             hindrances.append(_("the existing course does not have exactly one evaluation"))
-        elif merge_candidate.evaluations.get().wait_for_grade_upload_before_publishing != course_data.is_graded:
+        elif merge_candidate_evaluations[0].wait_for_grade_upload_before_publishing != course_data.is_graded:
             hindrances.append(_("the evaluation of the existing course has a mismatching grading specification"))
 
         return hindrances


Test patch:
diff --git a/evap/staff/tests/test_importers.py b/evap/staff/tests/test_importers.py
--- a/evap/staff/tests/test_importers.py
+++ b/evap/staff/tests/test_importers.py
@@ -1,4 +1,5 @@
 from copy import deepcopy
+from dataclasses import dataclass
 from datetime import date, datetime
 from unittest.mock import patch
 
@@ -9,10 +10,35 @@
 
 import evap.staff.fixtures.excel_files_test_data as excel_data
 from evap.evaluation.models import Contribution, Course, CourseType, Degree, Evaluation, Semester, UserProfile
-from evap.staff.importers import ImporterLogEntry, import_enrollments, import_persons_from_evaluation, import_users
+from evap.staff.importers import (
+    ImporterLog,
+    ImporterLogEntry,
+    import_enrollments,
+    import_persons_from_evaluation,
+    import_users,
+)
+from evap.staff.importers.base import ExcelFileLocation, ExcelFileRowMapper, InputRow
 from evap.staff.tools import ImportType, user_edit_link
 
 
+class TestExcelFileRowMapper(TestCase):
+    @dataclass
+    class SingleColumnInputRow(InputRow):
+        column_count = 1
+        location: ExcelFileLocation
+        value: str
+
+    def test_skip_first_n_rows_handled_correctly(self):
+        workbook_data = {"SheetName": [[str(i)] for i in range(10)]}
+        workbook_file_contents = excel_data.create_memory_excel_file(workbook_data)
+
+        mapper = ExcelFileRowMapper(skip_first_n_rows=3, row_cls=self.SingleColumnInputRow, importer_log=ImporterLog())
+        rows = mapper.map(workbook_file_contents)
+
+        self.assertEqual(rows[0].location, ExcelFileLocation("SheetName", 3))
+        self.assertEqual(rows[0].value, "3")
+
+
 class TestUserImport(TestCase):
     # valid user import tested in tests/test_views.py, TestUserImportView