Repository: getsentry/sentry

Issue:
Infinite loop in sqlquerycount for MySQL
```
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/debug/utils/patch_context.py", line 27, in wrapped 
May 05 14:36:41 sentry-1-001 sentry[39317]: return self.callback(self.func, *args, **kwargs) 
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/utils/performance/sqlquerycount.py", line 57, in cursor 
May 05 14:36:41 sentry-1-001 sentry[39317]: result = func(self, *args, **kwargs) 
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/debug/utils/patch_context.py", line 27, in wrapped 
May 05 14:36:41 sentry-1-001 sentry[39317]: return self.callback(self.func, *args, **kwargs) 
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/utils/performance/sqlquerycount.py", line 57, in cursor 
May 05 14:36:41 sentry-1-001 sentry[39317]: result = func(self, *args, **kwargs)

(patch_context.py)

22 def patch(self): 
23 func = getattr(self.target, self.attr) 
24 
25 def wrapped(*args, **kwargs): 
26 __traceback_hide__ = True # NOQA 
27 return self.callback(self.func, *args, **kwargs) 
28 
29 wrapped.__name__ = func.__name__ 
30 if hasattr(func, '__doc__'): 
31 wrapped.__doc__ = func.__doc__ 
32 if hasattr(func, '__module__'): 
33 wrapped.__module__ = func.__module__

(sqlquerycount.py)

53 
54 
55 def get_cursor_wrapper(state): 
56 def cursor(func, self, *args, **kwargs): 
57 result = func(self, *args, **kwargs) 
58 
59 return CursorWrapper(result, self, state) 
60 return cursor 
61
```

```
May 05 14:36:41 sentry-1-001 sentry[39317]: [ERROR] maximum recursion depth exceeded in __instancecheck__
May 05 14:36:41 sentry-1-001 sentry[39317]: Traceback (most recent call last):
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/options/store.py", line 165, in get_store
May 05 14:36:41 sentry-1-001 sentry[39317]: value = self.model.objects.get(key=key.name).value
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/manager.py", line 151, in get
May 05 14:36:41 sentry-1-001 sentry[39317]: return self.get_queryset().get(*args, **kwargs)
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/query.py", line 304, in get
May 05 14:36:41 sentry-1-001 sentry[39317]: num = len(clone)
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/query.py", line 77, in __len__
May 05 14:36:41 sentry-1-001 sentry[39317]: self._fetch_all()
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/query.py", line 857, in _fetch_all
May 05 14:36:41 sentry-1-001 sentry[39317]: self._result_cache = list(self.iterator())
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/query.py", line 220, in iterator
May 05 14:36:41 sentry-1-001 sentry[39317]: for row in compiler.results_iter():
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/sql/compiler.py", line 713, in results_iter
May 05 14:36:41 sentry-1-001 sentry[39317]: for rows in self.execute_sql(MULTI):
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/../django/db/models/sql/compiler.py", line 785, in execute_sql
May 05 14:36:41 sentry-1-001 sentry[39317]: cursor = self.connection.cursor()
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/debug/utils/patch_context.py", line 27, in wrapped
May 05 14:36:41 sentry-1-001 sentry[39317]: return self.callback(self.func, *args, **kwargs)
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/utils/performance/sqlquerycount.py", line 57, in cursor
May 05 14:36:41 sentry-1-001 sentry[39317]: result = func(self, *args, **kwargs)
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/debug/utils/patch_context.py", line 27, in wrapped
May 05 14:36:41 sentry-1-001 sentry[39317]: return self.callback(self.func, *args, **kwargs)
May 05 14:36:41 sentry-1-001 sentry[39317]: File "/opt/sentry/venv/lib/python2.7/site-packages/sentry/utils/performance/sqlquerycount.py", line 57, in cursor
May 05 14:36:41 sentry-1-001 sentry[39317]: result = func(self, *args, **kwargs)
```

@mitsuhiko @dcramer 



Discussion:
Looking at this a bit it seems like this is a MySQL unrelated issue. The patcher is currently not thread safe so what might happen is that two things attempt to patch and then it recursively calls itself instead of the original function. I need to verify that this is actually what happened there but it would appear that this is very much possible.


Resolving patch:
diff --git a/src/sentry/debug/utils/patch_context.py b/src/sentry/debug/utils/patch_context.py
--- a/src/sentry/debug/utils/patch_context.py
+++ b/src/sentry/debug/utils/patch_context.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+from threading import Lock
 from sentry.utils.imports import import_string
 
 
@@ -7,10 +8,12 @@ class PatchContext(object):
     def __init__(self, target, callback):
         target, attr = target.rsplit('.', 1)
         target = import_string(target)
-        self.func = getattr(target, attr)
         self.target = target
         self.attr = attr
         self.callback = callback
+        self._lock = Lock()
+        with self._lock:
+            self.func = getattr(target, attr)
 
     def __enter__(self):
         self.patch()
@@ -20,19 +23,21 @@ def __exit__(self, exc_type, exc_value, traceback):
         self.unpatch()
 
     def patch(self):
-        func = getattr(self.target, self.attr)
+        with self._lock:
+            func = getattr(self.target, self.attr)
 
-        def wrapped(*args, **kwargs):
-            __traceback_hide__ = True  # NOQA
-            return self.callback(self.func, *args, **kwargs)
+            def wrapped(*args, **kwargs):
+                __traceback_hide__ = True  # NOQA
+                return self.callback(self.func, *args, **kwargs)
 
-        wrapped.__name__ = func.__name__
-        if hasattr(func, '__doc__'):
-            wrapped.__doc__ = func.__doc__
-        if hasattr(func, '__module__'):
-            wrapped.__module__ = func.__module__
+            wrapped.__name__ = func.__name__
+            if hasattr(func, '__doc__'):
+                wrapped.__doc__ = func.__doc__
+            if hasattr(func, '__module__'):
+                wrapped.__module__ = func.__module__
 
-        setattr(self.target, self.attr, wrapped)
+            setattr(self.target, self.attr, wrapped)
 
     def unpatch(self):
-        setattr(self.target, self.attr, self.func)
+        with self._lock:
+            setattr(self.target, self.attr, self.func)


Test patch: