Repository: ResonantGeoData/ResonantGeoData

Issue:
Potential DB Connection Leak
There appears to be a DB connection leak - it seems the connection might not be cleaned up when a server error happens. I can reproduce this specifically when tile serving.

For example, take the following diff and open the detail view of any raster. Then pan around and zoom in and out for a min or so until the connection errors happen. You'll notice that every tile request errors (that's intentional from this diff).

```diff
diff --git a/django-rgd-imagery/rgd_imagery/templates/rgd_imagery/rastermeta_detail.html b/django-rgd-imagery/rgd_imagery/templates/rgd_imagery/rastermeta_detail.html
index 97c8d29..1f5612a 100644
--- a/django-rgd-imagery/rgd_imagery/templates/rgd_imagery/rastermeta_detail.html
+++ b/django-rgd-imagery/rgd_imagery/templates/rgd_imagery/rastermeta_detail.html
@@ -92,13 +92,13 @@
     // Make sure regions outside the extent of the raster do not load null tiles
     //   this relieves strain on the tile server
     const dataset_bb = extents.extent;
-    tileLayer.isValid = (index) => {
-      const tileBounds = tileLayer.gcsTileBounds(index);
-      return tileBounds.left <= dataset_bb.xmax &&
-        tileBounds.right >= dataset_bb.xmin &&
-        tileBounds.top >= dataset_bb.ymin &&
-        tileBounds.bottom <= dataset_bb.ymax;
-    }
+    // tileLayer.isValid = (index) => {
+    //   const tileBounds = tileLayer.gcsTileBounds(index);
+    //   return tileBounds.left <= dataset_bb.xmax &&
+    //     tileBounds.right >= dataset_bb.xmin &&
+    //     tileBounds.top >= dataset_bb.ymin &&
+    //     tileBounds.bottom <= dataset_bb.ymax;
+    // }
     tileLayer.idleAfter = 'all';
 
     function updateBandThumbnail () {
@@ -115,8 +115,8 @@
           band = 0
         }
         tileLayer.visible(true);
-        tileLayer.url(`${host}/api/image_process/imagery/${image_id}/tiles/{z}/{x}/{y}.png?projection=EPSG:3857&band=${band}`)
-        thumbnail.src = `${host}/api/image_process/imagery/${image_id}/thumbnail?band=${band}`
+        tileLayer.url(`${host}/api/image_process/imagery/${image_id}/tiles/{z}/{x}/{y}.png?projection=EPSG:3857&band=${band}&palette=bane`)
+        thumbnail.src = `${host}/api/image_process/imagery/${image_id}/thumbnail?band=1`
         thumbnail.hidden = false
       } else {
         tileLayer.visible(false);

```

Perhaps this is only an issue in a Development configuration? I'm not sure how django handles cleaning up the connections in a dev server which is multithreaded (?) vs prod which is multi-processes (?).

@mcovalt, would you help look into this and see if it is indeed an problem we should address?

```
django.db.utils.OperationalError: FATAL:  sorry, too many clients already
```

Stack trace

<details>

```
resonantgeodata-django-1     |            ERROR    Internal Server Error:                            log.py:224
resonantgeodata-django-1     |                     /api/image_process/imagery/11/tiles/5/13/10.png
resonantgeodata-django-1     |                     Traceback (most recent call last):
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/sessions/backends/base.py", line
resonantgeodata-django-1     |                     233, in _get_session
resonantgeodata-django-1     |                         return self._session_cache
resonantgeodata-django-1     |                     AttributeError: 'SessionStore' object has no
resonantgeodata-django-1     |                     attribute '_session_cache'
resonantgeodata-django-1     |
resonantgeodata-django-1     |                     During handling of the above exception, another
resonantgeodata-django-1     |                     exception occurred:
resonantgeodata-django-1     |
resonantgeodata-django-1     |                     Traceback (most recent call last):
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 219, in
resonantgeodata-django-1     |                     ensure_connection
resonantgeodata-django-1     |                         self.connect()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/asyncio.py", line 33, in inner
resonantgeodata-django-1     |                         return func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 200, in
resonantgeodata-django-1     |                     connect
resonantgeodata-django-1     |                         self.connection =
resonantgeodata-django-1     |                     self.get_new_connection(conn_params)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/asyncio.py", line 33, in inner
resonantgeodata-django-1     |                         return func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/postgresql/base.py", line 187,
resonantgeodata-django-1     |                     in get_new_connection
resonantgeodata-django-1     |                         connection = Database.connect(**conn_params)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/ps
resonantgeodata-django-1     |                     ycopg2/__init__.py", line 122, in connect
resonantgeodata-django-1     |                         conn = _connect(dsn,
resonantgeodata-django-1     |                     connection_factory=connection_factory, **kwasync)
resonantgeodata-django-1     |                     psycopg2.OperationalError: FATAL:  sorry, too
resonantgeodata-django-1     |                     many clients already
resonantgeodata-django-1     |
resonantgeodata-django-1     |
resonantgeodata-django-1     |                     The above exception was the direct cause of the
resonantgeodata-django-1     |                     following exception:
resonantgeodata-django-1     |
resonantgeodata-django-1     |                     Traceback (most recent call last):
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/core/handlers/exception.py", line 47, in
resonantgeodata-django-1     |                     inner
resonantgeodata-django-1     |                         response = get_response(request)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/core/handlers/base.py", line 181, in
resonantgeodata-django-1     |                     _get_response
resonantgeodata-django-1     |                         response = wrapped_callback(request,
resonantgeodata-django-1     |                     *callback_args, **callback_kwargs)
resonantgeodata-postgres-1   | 2021-11-21 04:08:10.509 UTC [327] FATAL:  sorry, too many clients already
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/views/decorators/csrf.py", line 54, in
resonantgeodata-django-1     |                     wrapped_view
resonantgeodata-django-1     |                         return view_func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/views/generic/base.py", line 70, in view
resonantgeodata-django-1     |                         return self.dispatch(request, *args,
resonantgeodata-django-1     |                     **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/views.py", line 509, in dispatch
resonantgeodata-django-1     |                         response = self.handle_exception(exc)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/views.py", line 469, in
resonantgeodata-django-1     |                     handle_exception
resonantgeodata-django-1     |                         self.raise_uncaught_exception(exc)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/views.py", line 480, in
resonantgeodata-django-1     |                     raise_uncaught_exception
resonantgeodata-django-1     |                         raise exc
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/views.py", line 497, in dispatch
resonantgeodata-django-1     |                         self.initial(request, *args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/views.py", line 414, in initial
resonantgeodata-django-1     |                         self.perform_authentication(request)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/views.py", line 324, in
resonantgeodata-django-1     |                     perform_authentication
resonantgeodata-django-1     |                         request.user
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/request.py", line 227, in user
resonantgeodata-django-1     |                         self._authenticate()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/request.py", line 380, in
resonantgeodata-django-1     |                     _authenticate
resonantgeodata-django-1     |                         user_auth_tuple =
resonantgeodata-django-1     |                     authenticator.authenticate(self)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/re
resonantgeodata-django-1     |                     st_framework/authentication.py", line 127, in
resonantgeodata-django-1     |                     authenticate
resonantgeodata-django-1     |                         if not user or not user.is_active:
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/functional.py", line 246, in inner
resonantgeodata-django-1     |                         self._setup()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/functional.py", line 382, in _setup
resonantgeodata-django-1     |                         self._wrapped = self._setupfunc()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/auth/middleware.py", line 23, in
resonantgeodata-django-1     |                     <lambda>
resonantgeodata-django-1     |                         request.user = SimpleLazyObject(lambda:
resonantgeodata-django-1     |                     get_user(request))
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/auth/middleware.py", line 11, in
resonantgeodata-django-1     |                     get_user
resonantgeodata-django-1     |                         request._cached_user = auth.get_user(request)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/auth/__init__.py", line 177, in
resonantgeodata-django-1     |                     get_user
resonantgeodata-django-1     |                         user_id = _get_user_session_key(request)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/auth/__init__.py", line 60, in
resonantgeodata-django-1     |                     _get_user_session_key
resonantgeodata-django-1     |                         return get_user_model()._meta.pk.to_python(re
resonantgeodata-django-1     |                     quest.session[SESSION_KEY])
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/sessions/backends/base.py", line 65,
resonantgeodata-django-1     |                     in __getitem__
resonantgeodata-django-1     |                         return self._session[key]
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/sessions/backends/base.py", line
resonantgeodata-django-1     |                     238, in _get_session
resonantgeodata-django-1     |                         self._session_cache = self.load()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/sessions/backends/db.py", line 43,
resonantgeodata-django-1     |                     in load
resonantgeodata-django-1     |                         s = self._get_session_from_db()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/contrib/sessions/backends/db.py", line 32,
resonantgeodata-django-1     |                     in _get_session_from_db
resonantgeodata-django-1     |                         return self.model.objects.get(
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/models/manager.py", line 85, in
resonantgeodata-django-1     |                     manager_method
resonantgeodata-django-1     |                         return getattr(self.get_queryset(),
resonantgeodata-django-1     |                     name)(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/models/query.py", line 431, in get
resonantgeodata-django-1     |                         num = len(clone)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/models/query.py", line 262, in __len__
resonantgeodata-django-1     |                         self._fetch_all()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/models/query.py", line 1324, in
resonantgeodata-django-1     |                     _fetch_all
resonantgeodata-django-1     |                         self._result_cache =
resonantgeodata-django-1     |                     list(self._iterable_class(self))
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/models/query.py", line 51, in __iter__
resonantgeodata-django-1     |                         results = compiler.execute_sql(chunked_fetch=
resonantgeodata-django-1     |                     self.chunked_fetch, chunk_size=self.chunk_size)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/models/sql/compiler.py", line 1173, in
resonantgeodata-django-1     |                     execute_sql
resonantgeodata-django-1     |                         cursor = self.connection.cursor()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/de
resonantgeodata-django-1     |                     bug_toolbar/panels/sql/tracking.py", line 54, in
resonantgeodata-django-1     |                     cursor
resonantgeodata-django-1     |                         connection._djdt_cursor(*args, **kwargs),
resonantgeodata-django-1     |                     connection, panel
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/asyncio.py", line 33, in inner
resonantgeodata-django-1     |                         return func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 259, in
resonantgeodata-django-1     |                     cursor
resonantgeodata-django-1     |                         return self._cursor()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 235, in
resonantgeodata-django-1     |                     _cursor
resonantgeodata-django-1     |                         self.ensure_connection()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/asyncio.py", line 33, in inner
resonantgeodata-django-1     |                         return func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 219, in
resonantgeodata-django-1     |                     ensure_connection
resonantgeodata-django-1     |                         self.connect()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/utils.py", line 90, in __exit__
resonantgeodata-django-1     |                         raise dj_exc_value.with_traceback(traceback)
resonantgeodata-django-1     |                     from exc_value
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 219, in
resonantgeodata-django-1     |                     ensure_connection
resonantgeodata-django-1     |                         self.connect()
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/asyncio.py", line 33, in inner
resonantgeodata-django-1     |                         return func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/base/base.py", line 200, in
resonantgeodata-django-1     |                     connect
resonantgeodata-django-1     |                         self.connection =
resonantgeodata-django-1     |                     self.get_new_connection(conn_params)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/utils/asyncio.py", line 33, in inner
resonantgeodata-django-1     |                         return func(*args, **kwargs)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/dj
resonantgeodata-django-1     |                     ango/db/backends/postgresql/base.py", line 187,
resonantgeodata-django-1     |                     in get_new_connection
resonantgeodata-django-1     |                         connection = Database.connect(**conn_params)
resonantgeodata-django-1     |                       File "/usr/local/lib/python3.8/site-packages/ps
resonantgeodata-django-1     |                     ycopg2/__init__.py", line 122, in connect
resonantgeodata-django-1     |                         conn = _connect(dsn,
resonantgeodata-django-1     |                     connection_factory=connection_factory, **kwasync)
resonantgeodata-django-1     |                     django.db.utils.OperationalError: FATAL:  sorry,
resonantgeodata-django-1     |                     too many clients already
resonantgeodata-django-1     |
````

</details>


Discussion:
FWIW you can simulate the prod environment by running the server using `gunicorn` instead of `manage.py runserver`.
By default, Django opens a new connection to PostgreSQL for each request/response cycle. PostgreSQL handles each new connection by forking a new process. Therefore, there's strict limits are usually set on how many concurrent connections are allowed.

The tile server is going to handle the most amount of concurrent requests in our app for sure. Here we see that we are hitting that limit. It looks like that limit is being reached with the session middleware. We could bypass the session issue entirely by letting [memcached handle the sessions as opposed to PostgreSQL](https://docs.djangoproject.com/en/3.2/topics/http/sessions/#using-cached-sessions) (which we should do regardless), but we're also hitting the database in the tile view to get image metadata.

A few options:
1. try to cache everything we can in the tile views
2. deploy with a connection pooler (e.g. [pgagroal](https://agroal.github.io/pgagroal/), [pgBouncer](http://www.pgbouncer.org/), [odyssey](https://github.com/yandex/odyssey), [Pgpool-II](https://pgpool.net/mediawiki/index.php/Main_Page), etc.)
3. try to use an alternative to psycopg2 with better connection pooling capabilities builtin (e.g. [the newly released psycopg3](https://www.psycopg.org/psycopg3/docs/), [asyncpg](https://github.com/MagicStack/asyncpg), etc.)

Option 2 is the most popular solution, but can get annoying. It increases deployment complexity and has nuances that increase application complexity (transactions can get weird when multiplexed). I think we need to do number 1 regardless. That will likely fix the issue. I think we'll also need a better connection pooler and for that I like the idea of number 3 since it doesn't add a new service into the mix.
[Looks like we can't upgrade to psycopg3 quite yet](https://www.psycopg.org/articles/2021/08/02/psycopg3-django-driver/). We'll need to deploy with a connection pooler. I'll look into it.

Resolving patch:
diff --git a/django-rgd-imagery/rgd_imagery/rest/tiles.py b/django-rgd-imagery/rgd_imagery/rest/tiles.py
--- a/django-rgd-imagery/rgd_imagery/rest/tiles.py
+++ b/django-rgd-imagery/rgd_imagery/rest/tiles.py
@@ -1,5 +1,6 @@
 import json
 
+from django.core.cache import cache
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from django.utils.decorators import method_decorator
@@ -19,8 +20,19 @@
 class BaseTileView(BaseRestViewMixin, APIView):
     def get_tile_source(self, request: Request, pk: int) -> FileTileSource:
         """Return the built tile source."""
-        image_entry = get_object_or_404(Image, pk=pk)
-        self.check_object_permissions(request, image_entry)
+        # get image_entry from cache
+        image_cache_key = f'large_image_tile:image_{pk}'
+        if (image_entry := cache.get(image_cache_key, None)) is None:
+            image_entry = get_object_or_404(Image.objects.select_related('file'), pk=pk)
+            cache.set(image_cache_key, image_entry, CACHE_TIMEOUT)
+
+        # check authentication
+        sentinel = object()
+        auth_cache_key = f'large_image_tile:image_{pk}:user_{request.user.pk}'
+        if cache.get(auth_cache_key, sentinel) is sentinel:
+            self.check_object_permissions(request, image_entry)
+            cache.set(auth_cache_key, None, CACHE_TIMEOUT)
+
         projection = request.query_params.get('projection', None)
         band = int(request.query_params.get('band', 0))
         style = None
diff --git a/django-rgd/rgd/configuration/configuration.py b/django-rgd/rgd/configuration/configuration.py
--- a/django-rgd/rgd/configuration/configuration.py
+++ b/django-rgd/rgd/configuration/configuration.py
@@ -46,6 +46,7 @@ class MemachedMixin(ConfigMixin):
     MEMCACHED_USERNAME = values.Value(default=None)
     MEMCACHED_PASSWORD = values.Value(default=None)
     MEMCACHED_BINARY = values.Value(default=True)
+    SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'
 
     @classmethod
     def post_setup(cls):


Test patch:
diff --git a/django-rgd-imagery/tests/test_tiles.py b/django-rgd-imagery/tests/test_tiles.py
--- a/django-rgd-imagery/tests/test_tiles.py
+++ b/django-rgd-imagery/tests/test_tiles.py
@@ -29,3 +29,12 @@ def test_thumbnail(admin_api_client, geotiff_image_entry):
     )
     assert response.status_code == 200
     assert response['Content-Type'] == 'image/png'
+
+
+@pytest.mark.django_db(transaction=True)
+def test_cache(admin_api_client, geotiff_image_entry, django_assert_num_queries):
+    # cache a response
+    admin_api_client.get(f'/api/image_process/imagery/{geotiff_image_entry.pk}/tiles/1/0/0.png')
+    # ensure no new queries are made for a different tile request made by same user on same image
+    with django_assert_num_queries(0):
+        admin_api_client.get(f'/api/image_process/imagery/{geotiff_image_entry.pk}/tiles/1/1/0.png')