Repository: buildbot/buildbot

Issue:
test: buildbot.test.integration.test_telegram_bot is flaky
Fails due to database errors:
```
[ERROR]
Traceback (most recent call last):
  File "/buildbot/buildbot-job/build/master/buildbot/reporters/words.py", line 1057, in _load_channels_state
    data = yield self.master.db.state.getState(objectid, attr, ())
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/buildbot/buildbot-job/build/master/buildbot/db/pool.py", line 229, in do
    args, kwargs)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/twisted/python/threadpool.py", line 250, in inContext
    result = inContext.theWork()
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/twisted/python/threadpool.py", line 266, in <lambda>
    inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
  File "/buildbot/buildbot-job/build/master/buildbot/db/pool.py", line 190, in __thd
    rv = callable(arg, *args, **kwargs)
  File "/buildbot/buildbot-job/build/master/buildbot/db/state.py", line 98, in thd
    return self.thdGetState(conn, objectid, name, default=default)
  File "/buildbot/buildbot-job/build/master/buildbot/db/state.py", line 107, in thdGetState
    res = conn.execute(q)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute
    return meth(self, multiparams, params)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
    distilled_params,
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1253, in _execute_context
    e, statement, parameters, cursor, context
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1473, in _handle_dbapi_exception
    util.raise_from_cause(sqlalchemy_exception, exc_info)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1249, in _execute_context
    cursor, statement, parameters, context
  File "/buildbot/buildbot-job/build/sandbox/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 580, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: object_state
[SQL: SELECT object_state.value_json 
FROM object_state 
WHERE object_state.objectid = ? AND object_state.name = ?]
[parameters: (2, 'missing_workers')]
(Background on this error at: http://sqlalche.me/e/e3q8)
```

https://buildbot.buildbot.net/#/builders/23/builds/1559/steps/8/logs/stdio

https://buildbot.buildbot.net/#/builders/59/builds/86/steps/10/logs/stdio

https://buildbot.buildbot.net/#/builders/66/builds/81/steps/8/logs/stdio


Discussion:

Resolving patch:
diff --git a/master/buildbot/reporters/words.py b/master/buildbot/reporters/words.py
--- a/master/buildbot/reporters/words.py
+++ b/master/buildbot/reporters/words.py
@@ -349,20 +349,22 @@ def notify_for_finished(self, build):
 
         return False
 
+    @defer.inlineCallbacks
     def workerMissing(self, worker):
         self.missing_workers.add(worker['workerid'])
         if self.notify_for('worker'):
             self.send(("Worker `{name}` is missing. It was seen last on "
                        "{last_connection}.").format(**worker))
-        self.bot.saveMissingWorkers()
+        yield self.bot.saveMissingWorkers()
 
+    @defer.inlineCallbacks
     def workerConnected(self, worker):
         workerid = worker['workerid']
         if workerid in self.missing_workers:
             self.missing_workers.remove(workerid)
             if self.notify_for('worker'):
                 self.send("Worker `{name}` is back online.".format(**worker))
-            self.bot.saveMissingWorkers()
+            yield self.bot.saveMissingWorkers()
 
 
 class Contact:


Test patch:
diff --git a/master/buildbot/test/integration/test_telegram_bot.py b/master/buildbot/test/integration/test_telegram_bot.py
--- a/master/buildbot/test/integration/test_telegram_bot.py
+++ b/master/buildbot/test/integration/test_telegram_bot.py
@@ -33,7 +33,6 @@
 from buildbot.test.fake import httpclientservice as fakehttpclientservice
 from buildbot.test.util import db
 from buildbot.test.util import www
-from buildbot.test.util.decorators import flaky
 from buildbot.util import bytes2unicode
 from buildbot.util import unicode2bytes
 from buildbot.www import auth
@@ -99,6 +98,7 @@ def setUp(self):
         master.mq = mqconnector.MQConnector()
         yield master.mq.setServiceParent(master)
         yield master.mq.setup()
+        yield master.mq.startService()
 
         master.config.www = dict(
             port='tcp:0:interface=127.0.0.1',
@@ -148,9 +148,9 @@ def send_message(chat, message, **kwargs):
     def tearDown(self):
         if self.master:
             yield self.master.www.stopService()
+            yield self.master.mq.stopService()
         yield self.tearDownRealDatabaseWithConnector()
 
-    @flaky(issueNumber=5120)
     @defer.inlineCallbacks
     def testWebhook(self):
         payload = unicode2bytes(json.dumps({
@@ -179,7 +179,6 @@ def testWebhook(self):
         self.assertIn('123456789', self.sent_messages[0][1])
         self.assertIn('-12345678', self.sent_messages[1][1])
 
-    @flaky(issueNumber=5120)
     @defer.inlineCallbacks
     def testReconfig(self):
         tb = self.master.config.services['TelegramBot']
@@ -188,7 +187,6 @@ def testReconfig(self):
             chat_ids=[-123456], notify_events=['problem']
         )
 
-    @flaky(issueNumber=5120)
     @defer.inlineCallbacks
     def testLoadState(self):
         tboid = yield self.master.db.state.getObjectId(
@@ -206,7 +204,6 @@ def testLoadState(self):
         self.assertEquals(c.channel.notify_events, {'started', 'finished'})
         self.assertEquals(c.channel.missing_workers, {12})
 
-    @flaky(issueNumber=5120)
     @defer.inlineCallbacks
     def testSaveState(self):
         tb = self.master.config.services['TelegramBot']
@@ -234,7 +231,6 @@ def testSaveState(self):
         self.assertIn([99, ['cancelled']], notify_events)
         self.assertIn([99, [13]], missing_workers)
 
-    @flaky(issueNumber=5120)
     @defer.inlineCallbacks
     def testMissingWorker(self):
         yield self.insertTestData([fakedb.Worker(id=1, name='local1')])