Repository: pretix/pretix

Issue:
Database error when invoice sequence has gaps
There seems to be a problem when customers from web.de create tickets (needs to be debugged). There are no invoices for these tickets, and an attempt to create one resulted in the following problem (full log on request by mail):

Internal Server Error: /<user>/<event>/order/<id>/l2ycrynl25ra2x9j/invoice

IntegrityError at /<user>/<event>/order/<id>/l2ycrynl25ra2x9j/invoice
(1062, "Duplicate entry '1-L-2019--00007' for key 'pretixbase_invoice_organizer_id_prefix_invoice_no_8da728d2_uniq'")

Traceback:

```
File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/utils.py" in _execute
  85.                 return self.cursor.execute(sql, params)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/mysql/base.py" in execute
  71.             return self.cursor.execute(query, args)

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in execute
  250.             self.errorhandler(self, exc, value)

File "/usr/lib64/python3.6/site-packages/MySQLdb/connections.py" in defaulterrorhandler
  50.         raise errorvalue

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in execute
  247.             res = self._query(query)

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in _query
  412.         rowcount = self._do_query(q)

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in _do_query
  375.         db.query(q)

File "/usr/lib64/python3.6/site-packages/MySQLdb/connections.py" in query
  276.             _mysql.connection.query(self, query)

The above exception ((1062, "Duplicate entry '1-L-2019--00007' for key 'pretixbase_invoice_organizer_id_prefix_invoice_no_8da728d2_uniq'")) was the direct cause of the following exception:

File "/home/pretix/.local/lib/python3.6/site-packages/django/core/handlers/exception.py" in inner
  34.             response = get_response(request)

File "/home/pretix/.local/lib/python3.6/site-packages/django/core/handlers/base.py" in _get_response
  126.                 response = self.process_exception_by_middleware(e, request)

File "/home/pretix/.local/lib/python3.6/site-packages/django/core/handlers/base.py" in _get_response
  124.                 response = wrapped_callback(request, *callback_args, **callback_kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/django/views/generic/base.py" in view
  68.             return self.dispatch(request, *args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/django/utils/decorators.py" in _wrapper
  45.         return bound_method(*args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/django/views/decorators/clickjacking.py" in wrapped_view
  50.         resp = view_func(*args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/pretix/presale/views/order.py" in dispatch
  496.         return super().dispatch(request, *args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/pretix/presale/views/robots.py" in dispatch
  7.         resp = super().dispatch(request, *args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/django/views/generic/base.py" in dispatch
  88.         return handler(request, *args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/pretix/presale/views/order.py" in post
  511.             i = generate_invoice(self.order)

File "/home/pretix/.local/lib/python3.6/site-packages/pretix/base/services/invoices.py" in generate_invoice
  215.     invoice = build_invoice(invoice)

File "/usr/lib64/python3.6/contextlib.py" in inner
  52.                 return func(*args, **kwds)

File "/home/pretix/.local/lib/python3.6/site-packages/pretix/base/services/invoices.py" in build_invoice
  99.         invoice.save()

File "/home/pretix/.local/lib/python3.6/site-packages/pretix/base/models/invoices.py" in save
  125.                         return super().save(*args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/base.py" in save
  718.                        force_update=force_update, update_fields=update_fields)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/base.py" in save_base
  748.             updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/base.py" in _save_table
  831.             result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/base.py" in _do_insert
  869.                                using=using, raw=raw)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/manager.py" in manager_method
  82.                 return getattr(self.get_queryset(), name)(*args, **kwargs)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/query.py" in _insert
  1136.         return query.get_compiler(using=using).execute_sql(return_id)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/models/sql/compiler.py" in execute_sql
  1289.                 cursor.execute(sql, params)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/utils.py" in execute
  68.         return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/utils.py" in _execute_with_wrappers
  77.         return executor(sql, params, many, context)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/utils.py" in _execute
  85.                 return self.cursor.execute(sql, params)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/utils.py" in __exit__
  89.                 raise dj_exc_value.with_traceback(traceback) from exc_value

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/utils.py" in _execute
  85.                 return self.cursor.execute(sql, params)

File "/home/pretix/.local/lib/python3.6/site-packages/django/db/backends/mysql/base.py" in execute
  71.             return self.cursor.execute(query, args)

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in execute
  250.             self.errorhandler(self, exc, value)

File "/usr/lib64/python3.6/site-packages/MySQLdb/connections.py" in defaulterrorhandler
  50.         raise errorvalue

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in execute
  247.             res = self._query(query)

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in _query
  412.         rowcount = self._do_query(q)

File "/usr/lib64/python3.6/site-packages/MySQLdb/cursors.py" in _do_query
  375.         db.query(q)

File "/usr/lib64/python3.6/site-packages/MySQLdb/connections.py" in query
  276.             _mysql.connection.query(self, query)
```


Discussion:
The question is: what is the question?
I have no idea, I have never seen this error before.

* I have not even the faintest idea how any of this could be related to users having a web.de email address, except if that triggers an SMTP error due to something with your mailserver (maybe web.de blocks you?) and that prevents any follow-up tasks. Is there anything related to mail in the log files?
* The exception shown here clearly states that the system tried to use an invoice number twice, I have never seen that before except in very rare race conditions, so if that is reproducible, I don't see why. Can you share the exact setting of your invoice number prefix, as well as maybe a list of all invoice numbers already in the pretixbase_invoice database table?
That web.de thing may just be coincidence. When I create a new order I can do everything just fine, but when I do the last step of the order I get 2 messages in the frontend: "an unexpected error has happened" and "your chart is empty". The pretix.log shows the same error over and over again:

```
pretix.base.tasks tasks Unexpected exception: IntegrityError('(1062, "Duplicate entry \'1-L-2019--00007\' for key \'pretixbase_invoice_organizer_id_prefix_invoice_no_8da728d2_uniq\'")',)
```

So I guess there is something wrong in the database, but I fail to see what. I guess I have to delete a duplicate entry somewhere or so. Any pointer where to look?
When I look into data/media/invoices I see things like that:

```
L-2019-00006-8....-*.pdf
L-2019-00006-W....-*.pdf
L-2019-00006-X....-*.pdf
L-2019-00007-8....-*.pdf
L-2019-00007-R....-*.pdf
L-2019-00008-8....-*.pdf
L-2019-00008-B....-*.pdf
L-2019-00009-8....-*.pdf
```

So the same PDF and bill number exists for different invoice ids, which does not look right to me.
How is the invoice number generated after all? I guess there is something wrong in the database so the next number it calculates is one that is already used, and then things break.

When I try to create an invoice in the backend I get the message "You cannot generate an invoice for this order" without further comment.
I guess it's this one:

```python
    def _get_numeric_invoice_number(self):
        numeric_invoices = Invoice.objects.filter(
            event__organizer=self.event.organizer,
            prefix=self.prefix,
        ).exclude(invoice_no__contains='-')
        return self._to_numeric_invoice_number(numeric_invoices.count() + 1)
```

If for whatever reason that number already exists, e.g. because one is somehow missing from the database, then this problem is not catched, but will cause probably exactly the headache I'm currently seeing.
I can confirm that at least the immediate error is gone when I replace the "+1" with a number that skips over all gaps in the invoice numbering. The code should probably find the biggest number already used and add one to those number for stability.
Hmm, interesting. Could it be that you are *missing* invoice numbers in between? Manually deleting invoices seems like the only reason this ``.count()`` approach can go wrong.

We should indeed evaluate if we can use ``Max()`` here instead. I believe there is a problem with it, otherwise we would have chosen it (because we use Max in similar places), probably because the invoice number is a string, not an integer, and databases are not well-behaved in sorting that.

Resolving patch:
diff --git a/src/pretix/base/models/invoices.py b/src/pretix/base/models/invoices.py
--- a/src/pretix/base/models/invoices.py
+++ b/src/pretix/base/models/invoices.py
@@ -2,6 +2,8 @@
 from decimal import Decimal
 
 from django.db import DatabaseError, models, transaction
+from django.db.models import Max
+from django.db.models.functions import Cast
 from django.utils import timezone
 from django.utils.crypto import get_random_string
 from django.utils.functional import cached_property
@@ -124,8 +126,12 @@ def _get_numeric_invoice_number(self):
         numeric_invoices = Invoice.objects.filter(
             event__organizer=self.event.organizer,
             prefix=self.prefix,
-        ).exclude(invoice_no__contains='-')
-        return self._to_numeric_invoice_number(numeric_invoices.count() + 1)
+        ).exclude(invoice_no__contains='-').annotate(
+            numeric_number=Cast('invoice_no', models.IntegerField())
+        ).aggregate(
+            max=Max('numeric_number')
+        )['max'] or 0
+        return self._to_numeric_invoice_number(numeric_invoices + 1)
 
     def _get_invoice_number_from_order(self):
         return '{order}-{count}'.format(
@@ -183,7 +189,7 @@ def canceled(self):
 
     class Meta:
         unique_together = ('organizer', 'prefix', 'invoice_no')
-        ordering = ('invoice_no',)
+        ordering = ('date', 'invoice_no',)
 
 
 class InvoiceLine(models.Model):


Test patch:
diff --git a/src/tests/base/test_invoices.py b/src/tests/base/test_invoices.py
--- a/src/tests/base/test_invoices.py
+++ b/src/tests/base/test_invoices.py
@@ -294,20 +294,27 @@ def test_invoice_numbers(env):
 
     event.settings.set('invoice_numbers_consecutive', True)
     inv5 = generate_invoice(order)
+    inv6 = generate_invoice(order)
+    inv7 = generate_invoice(order)
+    Invoice.objects.filter(pk=inv6.pk).delete()  # This should never ever happen, but what if it happens anyway?
+    inv8 = generate_invoice(order)
     inv23 = generate_invoice(order2)
 
-    # expected behaviour for switching between numbering formats
+    # expected behaviour for switching between numbering formats or dealing with gaps
     assert inv1.invoice_no == '00001'
     assert inv2.invoice_no == '00002'
     assert inv3.invoice_no == '{}-3'.format(order.code)
     assert inv4.invoice_no == '{}-4'.format(order.code)
     assert inv5.invoice_no == '00003'
+    assert inv6.invoice_no == '00004'
+    assert inv7.invoice_no == '00005'
+    assert inv8.invoice_no == '00006'
 
     # test that separate orders are counted separately in this mode
     assert inv21.invoice_no == '{}-1'.format(order2.code)
     assert inv22.invoice_no == '{}-2'.format(order2.code)
     # but consecutively in this mode
-    assert inv23.invoice_no == '00004'
+    assert inv23.invoice_no == '00007'
 
     # test Invoice.number, too
     assert inv1.number == '{}-00001'.format(event.slug.upper())