Repository: buildbot/buildbot

Issue:
optimize number of database queries
buildbot seems to be running database queries in a loop fetching data for individual builders (resulting in large number of queries). This would affect performance in case of large number of builders. I believe it would be more efficient to use a single database query to fetch data for all the builders and then filter it in python.

I noticed following in postgresql logs (which indicate queries being run in a loop for each builder):

2017-06-23 21:19:52.221 GMT 24738 localhost(56312) LOG:  statement: SELECT tags.name 
    FROM tags JOIN builders_tags ON tags.id = builders_tags.tagid 
    WHERE builders_tags.builderid = 218
2017-06-23 21:19:52.223 GMT 24738 localhost(56312) LOG:  statement: SELECT tags.name 
    FROM tags JOIN builders_tags ON tags.id = builders_tags.tagid 
    WHERE builders_tags.builderid = 219
2017-06-23 21:19:52.226 GMT 24738 localhost(56312) LOG:  statement: SELECT tags.name 
    FROM tags JOIN builders_tags ON tags.id = builders_tags.tagid 
    WHERE builders_tags.builderid = 220
2017-06-23 21:19:52.235 GMT 24738 localhost(56312) LOG:  statement: SELECT tags.name 
    FROM tags JOIN builders_tags ON tags.id = builders_tags.tagid 
    WHERE builders_tags.builderid = 221



Discussion:
Indeed, there is a performance issue in https://github.com/buildbot/buildbot/blob/master/master/buildbot/db/builders.py#L138

Even if we fetch the builder with a single query, then we fetch the list of tags with lots of queries. This is indeed inefficient. would you like to work on fixing this?

Resolving patch:
diff --git a/master/buildbot/db/builders.py b/master/buildbot/db/builders.py
--- a/master/buildbot/db/builders.py
+++ b/master/buildbot/db/builders.py
@@ -16,6 +16,8 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+from collections import defaultdict
+
 import sqlalchemy as sa
 
 from twisted.internet import defer
@@ -104,6 +106,9 @@ def getBuilders(self, masterid=None, _builderid=None):
         def thd(conn):
             bldr_tbl = self.db.model.builders
             bm_tbl = self.db.model.builder_masters
+            builders_tags_tbl = self.db.model.builders_tags
+            tags_tbl = self.db.model.tags
+
             j = bldr_tbl.outerjoin(bm_tbl)
             # if we want to filter by masterid, we must join to builder_masters
             # again, so we can still get the full set of masters for each
@@ -123,31 +128,24 @@ def thd(conn):
             if _builderid is not None:
                 q = q.where(bldr_tbl.c.id == _builderid)
 
+            # build up a intermediate builder id -> tag names map (fixes performance issue #3396)
+            bldr_id_to_tags = defaultdict(list)
+            bldr_q = sa.select([builders_tags_tbl.c.builderid, tags_tbl.c.name])
+            bldr_q = bldr_q.select_from(tags_tbl.join(builders_tags_tbl))
+
+            for bldr_id, tag in conn.execute(bldr_q).fetchall():
+                bldr_id_to_tags[bldr_id].append(tag)
+
             # now group those by builderid, aggregating by masterid
             rv = []
             last = None
             for row in conn.execute(q).fetchall():
                 # pylint: disable=unsubscriptable-object
                 if not last or row['id'] != last['id']:
-                    last = self._thd_row2dict(conn, row)
+                    last = dict(id=row.id, name=row.name, masterids=[], description=row.description,
+                                tags=bldr_id_to_tags[row.id])
                     rv.append(last)
                 if row['masterid']:
                     last['masterids'].append(row['masterid'])
             return rv
         return self.db.pool.do(thd)
-
-    def _thd_row2dict(self, conn, row):
-        # get tags
-        builders_tags = self.db.model.builders_tags
-        tags = self.db.model.tags
-        from_clause = tags
-        from_clause = from_clause.join(builders_tags)
-        q = sa.select([tags.c.name],
-                      (builders_tags.c.builderid == row.id)).select_from(from_clause)
-
-        tags = [r.name for r in
-                conn.execute(q).fetchall()]
-
-        return dict(id=row.id, name=row.name, masterids=[],
-                    description=row.description,
-                    tags=tags)


Test patch: