Repository: cowrie/cowrie

Issue:
Could not accept new connection (EMFILE)
Today, I've seen the following messages on my honeypots:

```
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
2016-12-01 09:02:14+0300 [cowrie.telnet.transport.HoneyPotTelnetFactory] Could not accept new connection (EMFILE)
```

I'm not 100% sure that it was caused by original cowrie, since I have some patches in place. So, first of all, I'll be appreciated if you guys check your instances and tell me if you ever experienced such errors. Different honeypots got this error both with Telnet and SSH.

One honeypot even has a traceback:

```
2016-12-01 08:41:36+0300 [-] Unhandled error in Deferred:
2016-12-01 08:41:36+0300 [-] Unhandled Error
        Traceback (most recent call last):
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1079, in connectionFailed
            self.factory.clientConnectionFailed(self, reason)
          File "/usr/local/lib/python2.7/site-packages/twisted/web/client.py", line 457, in clientConnectionFailed
            self.deferred.errback(reason)
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 434, in errback
            self._startRunCallbacks(fail)
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
            self._runCallbacks()
        --- <exception caught here> ---
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 588, in _runCallbacks
            current.result = callback(current.result, *args, **kw)
          File "/home/cowrie/cowrie/cowrie/commands/wget.py", line 233, in error
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 338, in resume
            self.runCommand()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 330, in runCommand
            self.protocol.call_command(pp, cmdclass, *cmd_array[0]['rargs'])
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 335, in call_command
            HoneyPotBaseProtocol.call_command(self, pp, cmd, *args)
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 192, in call_command
            obj.start()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 94, in start
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 338, in resume
            self.runCommand()
            self.factory.clientConnectionFailed(self, reason)
          File "/usr/local/lib/python2.7/site-packages/twisted/web/client.py", line 457, in clientConnectionFailed
            self.deferred.errback(reason)
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 434, in errback
            self._startRunCallbacks(fail)
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 501, in _startRunCallbacks
            self._runCallbacks()
        --- <exception caught here> ---
          File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 588, in _runCallbacks
            current.result = callback(current.result, *args, **kw)
          File "/home/cowrie/cowrie/cowrie/commands/wget.py", line 233, in error
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 338, in resume
            self.runCommand()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 330, in runCommand
            self.protocol.call_command(pp, cmdclass, *cmd_array[0]['rargs'])
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 335, in call_command
            HoneyPotBaseProtocol.call_command(self, pp, cmd, *args)
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 192, in call_command
            obj.start()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 94, in start
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 338, in resume
            self.runCommand()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 330, in runCommand
            self.protocol.call_command(pp, cmdclass, *cmd_array[0]['rargs'])
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 335, in call_command
            HoneyPotBaseProtocol.call_command(self, pp, cmd, *args)
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 192, in call_command
            obj.start()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 94, in start
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 338, in resume
            self.runCommand()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 330, in runCommand
            self.protocol.call_command(pp, cmdclass, *cmd_array[0]['rargs'])
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 335, in call_command
            HoneyPotBaseProtocol.call_command(self, pp, cmd, *args)
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 192, in call_command
            obj.start()
          File "/home/cowrie/cowrie/cowrie/commands/tftp.py", line 146, in start
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 337, in resume
            self.protocol.setInsertMode()
          File "/usr/local/lib/python2.7/site-packages/twisted/conch/recvline.py", line 173, in setInsertMode
            self.terminal.setModes([insults.modes.IRM])
          File "/usr/local/lib/python2.7/site-packages/twisted/conch/insults/insults.py", line 628, in setModes
            self.write('\x1b[%sh' % (';'.join(map(str, modes)),))
          File "/home/cowrie/cowrie/cowrie/insults/insults.py", line 92, in write
            self.protocol.call_command(pp, cmdclass, *cmd_array[0]['rargs'])
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 335, in call_command
            HoneyPotBaseProtocol.call_command(self, pp, cmd, *args)
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 192, in call_command
            obj.start()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 94, in start
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 338, in resume
            self.runCommand()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 330, in runCommand
            self.protocol.call_command(pp, cmdclass, *cmd_array[0]['rargs'])
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 335, in call_command
            HoneyPotBaseProtocol.call_command(self, pp, cmd, *args)
          File "/home/cowrie/cowrie/cowrie/core/protocol.py", line 192, in call_command
            obj.start()
          File "/home/cowrie/cowrie/cowrie/commands/tftp.py", line 146, in start
            self.exit()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 109, in exit
            self.protocol.cmdstack[-1].resume()
          File "/home/cowrie/cowrie/cowrie/core/honeypot.py", line 337, in resume
            self.protocol.setInsertMode()
          File "/usr/local/lib/python2.7/site-packages/twisted/conch/recvline.py", line 173, in setInsertMode
            self.terminal.setModes([insults.modes.IRM])
          File "/usr/local/lib/python2.7/site-packages/twisted/conch/insults/insults.py", line 628, in setModes
            self.write('\x1b[%sh' % (';'.join(map(str, modes)),))
          File "/home/cowrie/cowrie/cowrie/insults/insults.py", line 92, in write
            ttylog.TYPE_OUTPUT, time.time(), bytes)
          File "/home/cowrie/cowrie/cowrie/core/ttylog.py", line 39, in ttylog_write
            with open(logfile, 'ab') as f:
        exceptions.IOError: [Errno 24] Too many open files: 'log/tty/20161201-074642-None-3340i.log'

```


Discussion:
Could be that the server is running out of file descriptors. The "exceptions.IOError: [Errno 24] Too many open files: 'log/tty/20161201-074642-None-3340i.log'" provides another hint that it was unable to write new file. Increase maximum number of file descriptor of the server might temporarily solves this problem. In my own cowrie instance, I sometimes noticed that connections (about 2000 of them) tied to cowrie process were left hanging in CLOSE_WAIT status for a long time. I had to restart cowrie process to get rid of them. 
I've just checked the open descriptors and looks like cowrie doesn't drop connection under some circumstances, because some descriptors (about 50% of all descriptors) are open for more than a day now:

lrwx------ 1 cowrie cowrie 64 Dec  4 11:39 91 -> socket:[1259944917]
l-wx------ 1 cowrie cowrie 64 Dec  4 11:39 92 -> /home/cowrie/cowrie/dl/20161202224203_tftp1_sh
lrwx------ 1 cowrie cowrie 64 Dec  4 11:39 93 -> socket:[1247497450]
l-wx------ 1 cowrie cowrie 64 Dec  4 11:39 94 -> /home/cowrie/cowrie/dl/20161202205345_tftp1_sh
lrwx------ 1 cowrie cowrie 64 Dec  4 11:39 95 -> socket:[1247501472]
l-wx------ 1 cowrie cowrie 64 Dec  4 11:39 96 -> /home/cowrie/cowrie/dl/20161202205410_tftp2_sh
lrwx------ 1 cowrie cowrie 64 Dec  4 11:39 97 -> socket:[1248238996]
l-wx------ 1 cowrie cowrie 64 Dec  4 11:39 98 -> /home/cowrie/cowrie/dl/20161202224228_tftp2_sh

> I had to restart cowrie process to get rid of them.

Probably, it would be nice to schedule a daily restart to prevent such things.
There is a timeout for each session, and if sessions are hanging, the session timeout should be fixed first..
Also it appears the files stay open that are downloaded (_sh files). Those should have been closed. 
I've reproduced the issue. 2-3 days of uptime is enough to get this error on almost all of my honeypots.
I could confirm that the problem related only to tftp-downloaded files. Probably, the client doesn't close the file handle for some reasons. I'll try to track the problem...

Also, I have a couple of hang (?) connections. During stopping process, cowrie logged the following:

2016-12-07 07:56:00+0300 [HoneyPotSSHTransport,505,attackers_ip] avatar admin logging out
2016-12-07 07:56:00+0300 [HoneyPotSSHTransport,505,attackers_ip] connection lost
2016-12-07 07:56:00+0300 [HoneyPotSSHTransport,505,attackers_ip] Connection lost after 66706 seconds
2016-12-07 07:56:01+0300 [HoneyPotSSHTransport,144,attackers_ip] avatar admin logging out
2016-12-07 07:56:01+0300 [HoneyPotSSHTransport,144,attackers_ip] connection lost
2016-12-07 07:56:01+0300 [HoneyPotSSHTransport,144,attackers_ip] Connection lost after 126256 seconds
2016-12-07 07:56:01+0300 [HoneyPotSSHTransport,504,attackers_ip] avatar admin logging out
2016-12-07 07:56:01+0300 [HoneyPotSSHTransport,504,attackers_ip] connection lost
2016-12-07 07:56:01+0300 [HoneyPotSSHTransport,504,attackers_ip] Connection lost after 67007 seconds

I've checked this sessions - it was direct-tcp forward requests. Probably connection timeout doesn't affect this requests, does it? 

Regarding the tftp_sh file descriptor being left open, I believe it was due to the connection from either side was not terminated properly. As of now, I currently observes this on my cowrie pot :

```
2016-12-13T16:49:49+0000 [twisted.internet.defer#critical] Unhandled error in Deferred:
2016-12-13T16:49:49+0000 [twisted.internet.defer#critical]
        Traceback (most recent call last):
        Failure: twisted.internet.error.ConnectionLost: Connection to the other side was lost in a non-clean fashion.

2016-12-13T16:49:49+0000 [twisted.internet.defer#critical] Unhandled error in Deferred:
2016-12-13T16:49:49+0000 [twisted.internet.defer#critical]
        Traceback (most recent call last):
        Failure: twisted.internet.error.ConnectionLost: Connection to the other side was lost in a non-clean fashion.
2016-12-13T16:49:49+0000 [CowrieTelnetTransport,3522,<removed>] Closing TTY Log: log/tty/20161213-164935-None-3522i.log after 13 seconds
2016-12-13T16:49:49+0000 [CowrieTelnetTransport,3522,<removed>] honeypot terminal protocol connection lost [Failure instance: Traceback (failure with no frames): <class 'twisted.
internet.error.ConnectionLost'>: Connection to the other side was lost in a non-clean fashion.
```

The TFTP connections are left hanging open. These could contribute to the increase of open files over the time :
```
# netstat -uanlp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
udp        0      0 0.0.0.0:54921           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:39318           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:44057           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:55712           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:46115           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:48170           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:55850           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:44974           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:56117           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:57149           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:53186           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:46918           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:51786           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:37326           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:36439           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:52567           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:47191           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:45022           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:33384           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:53999           0.0.0.0:*                           24010/python2
udp        0      0 0.0.0.0:45940           0.0.0.0:*                           24010/python2
```

But, the files the tftp files that are downloaded seems to be closed accordingly. 
The cause of the described problem is file handle leakage in tftpy when it raises `TftpTimeout` in `TftpContextClientDownload`.

The quick fix will be to patch `TftpContextClientDownload` class in cowrie-env/lib/python2.7/site-packages/tftpy/TftpContexts.py in order to close file handle before raising the exception:

```
    def start(self):
        """Initiate the download."""
        log.info("Sending tftp download request to %s" % self.host)
        log.info("    filename -> %s" % self.file_to_transfer)
        log.info("    options -> %s" % self.options)

        self.metrics.start_time = time.time()
        log.debug("Set metrics.start_time to %s", self.metrics.start_time)

        # FIXME: put this in a sendRRQ method?
        pkt = TftpPacketRRQ()
        pkt.filename = self.file_to_transfer
        pkt.mode = "octet" # FIXME - shouldn't hardcode this
        pkt.options = self.options
        self.sock.sendto(pkt.encode().buffer, (self.host, self.port))
        self.next_block = 1
        self.last_pkt = pkt

        self.state = TftpStateSentRRQ(self)

        while self.state:
            try:
                log.debug("State is %s", self.state)
                self.cycle()
            except TftpTimeout, err:
                log.error(str(err))
                self.retry_count += 1
                if self.retry_count >= TIMEOUT_RETRIES:
                    log.debug("hit max retries, giving up")
                    # begining of the patch
                    if self.fileobj and not self.fileobj.closed:
                        self.fileobj.close()
                    # end of the patch
                    raise
                else:
                    log.warn("resending last packet")
                    self.state.resendLast()
```

Resolving patch:
diff --git a/cowrie/commands/tftp.py b/cowrie/commands/tftp.py
--- a/cowrie/commands/tftp.py
+++ b/cowrie/commands/tftp.py
@@ -48,7 +48,6 @@ def makeTftpRetrieval(self):
         """
         """
         progresshook = Progress(self).progresshook
-        tclient = tftpy.TftpClient(self.hostname, int(self.port))
         cfg = self.protocol.cfg
 
         if cfg.has_option('honeypot', 'download_limit_size'):
@@ -63,51 +62,52 @@ def makeTftpRetrieval(self):
                      re.sub('[^A-Za-z0-9]', '_', self.file_to_get))
         self.safeoutfile = os.path.join(self.download_path, tmp_fname)
 
-        try:
-            tclient.download(self.file_to_get, self.safeoutfile, progresshook)
-            self.file_to_get = self.fs.resolve_path(self.file_to_get, self.protocol.cwd)
-            self.fs.mkfile(self.file_to_get, 0, 0, tclient.context.metrics.bytes, 33188)
-            self.fs.update_realfile(self.fs.getfile(self.file_to_get), self.safeoutfile)
-
-            if os.path.exists(self.safeoutfile):
-
-                if os.path.getsize(self.safeoutfile) == 0:
-                    os.remove(self.safeoutfile)
-                    self.safeoutfile = None
-                    return
-
-                with open(self.safeoutfile, 'rb') as f:
-                    shasum = hashlib.sha256(f.read()).hexdigest()
-                    hash_path = os.path.join(self.download_path, shasum)
-
-                    # If we have content already, delete temp file
-                    if not os.path.exists(hash_path):
-                        os.rename(self.safeoutfile, hash_path)
-                    else:
-                        os.remove(self.safeoutfile)
-                        log.msg("Not storing duplicate content " + shasum)
-
-                    log.msg(eventid='cowrie.session.file_download',
-                            format='Downloaded tftpFile (%(url)s) with SHA-256 %(shasum)s to %(outfile)s',
-                            url=self.file_to_get,
-                            outfile=hash_path,
-                            shasum=shasum)
-
-                    # Link friendly name to hash
-                    os.symlink(shasum, self.safeoutfile)
-
-                    # Update the honeyfs to point to downloaded file
-                    f = self.fs.getfile(self.file_to_get)
-                    f[A_REALFILE] = hash_path
-
-        except tftpy.TftpException, err:
-            if os.path.exists(self.safeoutfile):
-                if os.path.getsize(self.safeoutfile) == 0:
-                    os.remove(self.safeoutfile)
-            return
+        with tftpy.TftpClient(self.hostname, int(self.port)) as tclient:
+            try:
+                tclient.download(self.file_to_get, self.safeoutfile, progresshook)
+                self.file_to_get = self.fs.resolve_path(self.file_to_get, self.protocol.cwd)
+                if hasattr(tclient.context, 'metrics'):
+                    self.fs.mkfile(self.file_to_get, 0, 0, tclient.context.metrics.bytes, 33188)
+                else:
+                    self.fs.mkfile(self.file_to_get, 0, 0, 0, 33188)
+                self.fs.update_realfile(self.fs.getfile(self.file_to_get), self.safeoutfile)
+            except tftpy.TftpException, err:
+                pass
+
+        if os.path.exists(self.safeoutfile):
+
+            if os.path.getsize(self.safeoutfile) == 0:
+                os.remove(self.safeoutfile)
+                self.safeoutfile = None
+                return
+
+            with open(self.safeoutfile, 'rb') as f:
+                shasum = hashlib.sha256(f.read()).hexdigest()
+                hash_path = os.path.join(self.download_path, shasum)
+
+            # If we have content already, delete temp file
+            if not os.path.exists(hash_path):
+                os.rename(self.safeoutfile, hash_path)
+            else:
+                os.remove(self.safeoutfile)
+                log.msg("Not storing duplicate content " + shasum)
+
+            url = 'tftp://%s/%s' % (self.hostname, self.file_to_get.strip('/'))
+
+            log.msg(eventid='cowrie.session.file_download',
+                    format='Downloaded tftpFile (%(url)s) with SHA-256 %(shasum)s to %(outfile)s',
+                    url=url,
+                    outfile=hash_path,
+                    shasum=shasum)
+
+            # Link friendly name to hash
+            os.symlink(shasum, self.safeoutfile)
+
+            # Update the honeyfs to point to downloaded file
+            f = self.fs.getfile(self.file_to_get)
+            f[A_REALFILE] = hash_path
+
 
-        except KeyboardInterrupt:
-            pass
 
 
     def start(self):


Test patch: