Repository: evennia/evennia

Issue:
Crash from File I/O Error on reload
#### Brief summary of issue / Description of requested feature:
Earlier today, a staff member tried to `@reload` the server. Unlike the server hang seen in #1128 this didn't start the reload process at all, but immediately resulted in seeing repeated `... Portal has lost connection to Server.` messages until Evennia stop was called, with a repeated traceback in the console.

```
2017-04-14T10:08:52-0400 [AMPProtocol,0,127.0.0.1] Unhandled Error
Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/log.py", line 101, in callWithLogger
    return callWithContext({"system": lp}, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/log.py", line 84, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
    return func(*args,**kw)
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 597, in _doReadOrWrite
    why = selectable.doRead()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 208, in doRead
    return self._dataReceived(data)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 214, in _dataReceived
    rval = self.protocol.dataReceived(data)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 2411, in dataReceived
    return Int16StringReceiver.dataReceived(self, data)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/basic.py", line 765, in dataReceived
    self.stringReceived(packet)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/basic.py", line 877, in stringReceived
    self.state = statehandler(string)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 2464, in proto_key
    self.boxReceiver.ampBoxReceived(self._currentBox)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 1063, in ampBoxReceived
    self._answerReceived(box)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 989, in _answerReceived
    question = self._outstandingRequests.pop(box[ANSWER])
exceptions.KeyError: '3433'

2017-04-14T10:08:53-0400 [evennia.server.amp.AmpServerFactory] Unhandled Error
Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/log.py", line 84, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
    return func(*args,**kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 597, in _doReadOrWrite
    why = selectable.doRead()
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 1072, in doRead
    transport = self.transport(skt, protocol, addr, self, s, self.reactor)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 789, in __init__
    self.startReading()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/abstract.py", line 434, in startReading
    self.reactor.addReader(self)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 271, in addReader
    EPOLLIN, EPOLLOUT)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 255, in _add
    self._poller.modify(fd, flags)
exceptions.IOError: [Errno 2] No such file or directory

2017-04-14T10:08:54-0400 [evennia.server.amp.AmpServerFactory] Unhandled Error
Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/log.py", line 84, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
    return func(*args,**kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 597, in _doReadOrWrite
    why = selectable.doRead()
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 1072, in doRead
    transport = self.transport(skt, protocol, addr, self, s, self.reactor)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 789, in __init__
    self.startReading()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/abstract.py", line 434, in startReading
    self.reactor.addReader(self)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 271, in addReader
    EPOLLIN, EPOLLOUT)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 255, in _add
    self._poller.modify(fd, flags)
exceptions.IOError: [Errno 2] No such file or directory
```
The first error it printed is the same from #1299 .The last error repeated every few seconds until the server was stopped. 

#### Steps to reproduce the issue / Reasons for adding feature:

1. Using `@reload` command

#### Error output / Expected result of feature
Above traceback.


#### Extra information, such as Evennia revision/repo/branch, operating system and ideas for how to solve / implement:
My guess is attempting to write to a log file at the same time the server was restarting.


<bountysource-plugin>

---
Want to back this issue? **[Post a bounty on it!](https://www.bountysource.com/issues/44146972-crash-from-file-i-o-error-on-reload?utm_campaign=plugin&utm_content=tracker%2F522101&utm_medium=issues&utm_source=github)** We accept bounties via [Bountysource](https://www.bountysource.com/?utm_campaign=plugin&utm_content=tracker%2F522101&utm_medium=issues&utm_source=github).
</bountysource-plugin>


Discussion:
My first thought when seeing this would something with the channel log - the channel log file is written to in a separate thread, using an continuously held-open file handle for minimal overhead.  Nothing in the traceback appears to point out the channel logger specifically, but it is the only file I know of that Evennia is actually writing to (except for the database file and if that was missing you would be in a whole other level of hurt).
This happened again, though interestingly it was about a minute after a `@reload`, not immediately. I noticed the repeating traceback in console coincided with '...Portal has lost connection to server' that came up every few seconds. So my guess is that something in the logging caused the AMP bridge to lose connection, and every time it tried to reestablish connection the same error was repeated. Trying to shut down the server from console yielded a different error:
```
(arx) tehom@arxhost:~/arx/arx$ evennia2017-06-04T19:14:11-0400 [-] Unhandled Error
Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/application/app.py", line 395, in startReactor
    self.config, oldstdout, oldstderr, self.profiler, reactor)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/application/app.py", line 310, in runReactorWithLogging
    reactor.run()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1195, in run
    self.mainLoop()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1204, in mainLoop
    self.runUntilCurrent()
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/base.py", line 825, in runUntilCurrent
    call.func(*call.args, **call.kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 393, in callback
    self._startRunCallbacks(result)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 494, in _startRunCallbacks
    raise AlreadyCalledError
twisted.internet.defer.AlreadyCalledError: 

2017-06-04T19:14:11-0400 [twisted.internet.defer#critical] Unhandled error in Deferred:

Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 306, in addCallbacks
    self._runCallbacks()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 587, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/tehom/arx/evennia/evennia/server/server.py", line 178, in <lambda>
    d.addCallback(lambda _: self.shutdown(_reactor_stopping=True))
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1331, in unwindGenerator
    return _inlineCallbacks(None, gen, Deferred())
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1185, in _inlineCallbacks
    result = g.send(result)
  File "/home/tehom/arx/evennia/evennia/server/server.py", line 382, in shutdown
    for p in PlayerDB.get_all_cached_instances()]
  File "/home/tehom/arx/evennia/evennia/players/players.py", line 313, in unpuppet_all
    self.unpuppet_object(self.sessions.all())
  File "/home/tehom/arx/evennia/evennia/players/players.py", line 303, in unpuppet_object
    obj.at_post_unpuppet(self, session=session)
  File "/home/tehom/arx/arx/typeclasses/characters.py", line 734, in at_post_unpuppet
    super(Character, self).at_post_unpuppet(player, session)
  File "/home/tehom/arx/evennia/evennia/objects/objects.py", line 1616, in at_post_unpuppet
    self.location.for_contents(message, exclude=[self], from_obj=self)
  File "/home/tehom/arx/evennia/evennia/objects/objects.py", line 548, in for_contents
    func(obj, **kwargs)
  File "/home/tehom/arx/evennia/evennia/objects/objects.py", line 1615, in message
    obj.msg("%s has left the game." % self.get_display_name(obj), from_obj=from_obj)
  File "/home/tehom/arx/arx/typeclasses/mixins.py", line 611, in msg
    super(MsgMixins, self).msg(text, from_obj, session, options, **kwargs)
  File "/home/tehom/arx/evennia/evennia/objects/objects.py", line 526, in msg
    session.data_out(text=text, **kwargs)
  File "/home/tehom/arx/evennia/evennia/server/serversession.py", line 378, in data_out
    self.sessionhandler.data_out(self, **kwargs)
  File "/home/tehom/arx/evennia/evennia/server/sessionhandler.py", line 704, in data_out
    **kwargs)
  File "/home/tehom/arx/evennia/evennia/server/amp.py", line 485, in send_MsgServer2Portal
    return self.send_data(MsgServer2Portal, session.sessid, **kwargs)
  File "/home/tehom/arx/evennia/evennia/server/amp.py", line 421, in send_data
    packed_data=dumps((sessid, kwargs))
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 971, in callRemote
    return co._doCommand(self)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 2002, in _doCommand
    self.requiresAnswer)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 902, in _sendBoxCommand
    box._sendTo(self.boxSender)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/protocols/amp.py", line 723, in _sendTo
    proto.sendBox(self)
exceptions.AttributeError: 'NoneType' object has no attribute 'sendBox'
```

Because of that exception on trying to shut down, the server had to be killed with -9.
This is an interesting bug to try to read up on. https://github.com/pixelated/pixelated-user-agent/issues/884 has a few references to this - some people suggest switching the reactor type, and that it might be memory related in some way.

Since this happened after I switched to using logfiles for channels, and it happens usually shortly after the server starts up, my theory is that it's an error in a filehandle being opened: someone tries to access a channel's log for the first time, either by using channel history or writing to the channel log, and that causes this crash. Now, as to why that happens, I have no idea: you'd think if it was just a normal IOError, we'd catch the exception - unless it prevents any log file from working, so trying to log the error raises the exception again? Perhaps that's why the error keeps on being raised, if every attempt to reconnect would write to the log file, and so it keeps repeating.
Had this happen again while all the staff were asleep, though the behavior was kind of interesting. The tracebcacks happened once every few seconds for 5 minutes or so, then the portal recovered, and then the tracebacks continued less frequently (usually around 10-50 seconds) for another hours and a half.

The other weird thing is that although the traceback remained the same, the source of the unhandled error would change:
```
2017-08-23T05:07:24-0400 [evennia.server.amp.AmpServerFactory] Unhandled Error
Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/log.py", line 84, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 118, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/python/context.py", line 81, in callWithContext
    return func(*args,**kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/posixbase.py", line 597, in _doReadOrWrite
    why = selectable.doRead()
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 1072, in doRead
    transport = self.transport(skt, protocol, addr, self, s, self.reactor)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 789, in __init__
    self.startReading()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/abstract.py", line 434, in startReading
    self.reactor.addReader(self)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 271, in addReader
    EPOLLIN, EPOLLOUT)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 255, in _add
    self._poller.modify(fd, flags)
exceptions.IOError: [Errno 2] No such file or directory
```
to `2017-08-23T05:07:25-0400 [evennia.server.webserver.Website] Unhandled Error` with the same traceback. It'd alternate between them occasionally, with Website less frequently than AmpServerFactory.

The portal then recovered here with a different traceback:
```
2017-08-23T05:12:12-0400 [twisted.internet.defer#critical] Unhandled error in Deferred:

Traceback (most recent call last):
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 587, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/base.py", line 254, in _checkTimeout
    userDeferred.callback(result)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 393, in callback
    self._startRunCallbacks(result)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
    self._runCallbacks()
--- <exception caught here> ---
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 587, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 433, in _setRealAddress
    self.doConnect()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/tcp.py", line 572, in doConnect
    self.startReading()
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/abstract.py", line 434, in startReading
    self.reactor.addReader(self)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 271, in addReader
    EPOLLIN, EPOLLOUT)
  File "/home/tehom/arx/local/lib/python2.7/site-packages/twisted/internet/epollreactor.py", line 255, in _add
    self._poller.modify(fd, flags)
exceptions.IOError: [Errno 2] No such file or directory
2017-08-23T05:12:14-0400 [evennia.server.amp.AmpServerFactory] AMPProtocol connection established (HOST:IPv4Address(TCP, '127.0.0.1', 5000) PEER:IPv4Address(TCP, '127.0.0.1', 34308))
```
And then for the next hour or so, the logging error occurred irregularly, usually between 9-40 seconds, but only for the Website, not the Ampserver.
Since a suggestion I read was that the error might be caused by not having enough memory to open a file descriptor, I decided to check the memory profiles in htop. The portal had been running for close to 600 hours or so (the server had been running somewhere between 10-20), and was using around 6 gb of memory out of 8 available, so that could have been what caused it. I have no idea what the portal is using so much, even over such a long period of time. Still, stopping the portal once a week is probably a good idea going forward for live games.
Woah, the portal should not be hogging that much memory; it suggests a memory leak. Just to check: You are not running with DEBUG=True or somesuch I hope?
Yeah, I leave DEBUG=False. Occasionally I've turned it on with `@py from django.conf import settings;settings.DEBUG = True` briefly in order to diagnose something that would be lost with a `@reload`, but I always leave it False in settings.
And it's not that the portal was activated with DEBUG=True, which was later turned off? A memory usahe like that os highly unexpected from the portal.
@TehomCD Any Update on this with Evennia 0.7. It's unfortunately very hard to understand what could be going on; but it's worrying the Portal would be able to take that much memory out of DEBUG=True mode. One possibility would be that Portal + Server might have been started with DEBUG=True and then when the setting was changed, only the Server was reloaded, keeping the Portal in DEBUG mode accidentally? 
I'm still trying to get a few changes in that are blocking me from switching production to 0.7, so I'll let you know if the memory usage still occurs after I do that. I don't think the portal has ever been started with `DEBUG=True`: I've only set that to True with a `@py` call while leaving settings unchanged, so the Portal should have always had DEBUG as False when starting. With luck, it might have been related to the portal having a database connection, which you resolved in 0.7. But I'll let you know.
Just a quick note that this is still happening in 0.7, unfortunately. Still very rare and random, at least. Weirdly, the last time it happened, the AMP bridge was able to reestablish connection on its own after trying for 10 minutes or so.
I did finally find reference to it in Twisted's issues. Unfortunately, it was reported 5 years ago and hasn't been fixed yet, and just seems to be something about the epoll reactor under heavy usage. At least it seems pretty rare - it's much more rare for us than it was to the people who originally reported it seeing it every 3 days. If it gets to be more of an issue, maybe could try switching to the select reactor. 

https://twistedmatrix.com/trac/ticket/6346

Resolving patch:
diff --git a/evennia/utils/logger.py b/evennia/utils/logger.py
--- a/evennia/utils/logger.py
+++ b/evennia/utils/logger.py
@@ -59,6 +59,22 @@ def timeformat(when=None):
         tz_sign, tz_hour, tz_mins)
 
 
+def log_msg(msg):
+    """
+    Wrapper around log.msg call to catch any exceptions that might
+    occur in logging. If an exception is raised, we'll print to
+    stdout instead.
+
+    Args:
+        msg: The message that was passed to log.msg
+
+    """
+    try:
+        log.msg(msg)
+    except Exception:
+        print("Exception raised while writing message to log. Original message: %s" % msg)
+
+
 def log_trace(errmsg=None):
     """
     Log a traceback to the log. This should be called from within an
@@ -80,9 +96,9 @@ def log_trace(errmsg=None):
             except Exception as e:
                 errmsg = str(e)
             for line in errmsg.splitlines():
-                log.msg('[EE] %s' % line)
+                log_msg('[EE] %s' % line)
     except Exception:
-        log.msg('[EE] %s' % errmsg)
+        log_msg('[EE] %s' % errmsg)
 
 
 log_tracemsg = log_trace
@@ -101,7 +117,7 @@ def log_err(errmsg):
     except Exception as e:
         errmsg = str(e)
     for line in errmsg.splitlines():
-        log.msg('[EE] %s' % line)
+        log_msg('[EE] %s' % line)
 
 
     # log.err('ERROR: %s' % (errmsg,))
@@ -121,7 +137,7 @@ def log_warn(warnmsg):
     except Exception as e:
         warnmsg = str(e)
     for line in warnmsg.splitlines():
-        log.msg('[WW] %s' % line)
+        log_msg('[WW] %s' % line)
 
 
     # log.msg('WARNING: %s' % (warnmsg,))
@@ -139,7 +155,7 @@ def log_info(infomsg):
     except Exception as e:
         infomsg = str(e)
     for line in infomsg.splitlines():
-        log.msg('[..] %s' % line)
+        log_msg('[..] %s' % line)
 
 
 log_infomsg = log_info
@@ -157,7 +173,7 @@ def log_dep(depmsg):
     except Exception as e:
         depmsg = str(e)
     for line in depmsg.splitlines():
-        log.msg('[DP] %s' % line)
+        log_msg('[DP] %s' % line)
 
 
 log_depmsg = log_dep


Test patch: