Repository: cowrie/cowrie

Issue:
Unable to WGET from an HTTPS website
**Describe the bug**
When trying to use wget to fetch a file from an HTTPS source, the emulator gets stuck. In the logs I can see that the the code is failing to locate `CertificationOptions` from `twisted.internet.ssl`. That class doesn't exist as per current documentation.

**To Reproduce**
Steps to reproduce the behavior:
1. Run the agent (in my case through docker-cowrie)
2. Successfully connect to the honeypot via SSH
3. Run `wget https://raw.githubusercontent.com/cowrie/cowrie/master/README.rst`
4. The execution hangs indefinitely...

**Expected behavior**
The execution of wget should complete without issues.

**Server (please complete the following information):**
 - OS: Debian Buster (via Docker using the docker-cowrie repo)

**Additional context**
The Twisted documentation has an entry for `CertificateOptions` (see [here](https://twistedmatrix.com/documents/current/api/twisted.internet.ssl.CertificateOptions.html)), so I tried manually editing the problematic lines (see [curl.py](https://github.com/cowrie/cowrie/blob/a5ff09f02dfa0f5d8a875f38e6dd536697561672/src/cowrie/commands/curl.py#L268) and [wget.py](https://github.com/cowrie/cowrie/blob/a5ff09f02dfa0f5d8a875f38e6dd536697561672/src/cowrie/commands/wget.py#L160)) and it worked.
Unfortunately, I don't really have the time and means to fork the project and setup the development environment to make a PR at this time...


Discussion:

Resolving patch:
diff --git a/src/cowrie/commands/curl.py b/src/cowrie/commands/curl.py
--- a/src/cowrie/commands/curl.py
+++ b/src/cowrie/commands/curl.py
@@ -7,8 +7,6 @@
 import os
 import time
 
-from OpenSSL import SSL
-
 from twisted.internet import reactor, ssl
 from twisted.python import compat, log
 from twisted.web import client
@@ -19,77 +17,7 @@
 
 commands = {}
 
-
-class command_curl(HoneyPotCommand):
-    """
-    curl command
-    """
-    limit_size = CowrieConfig().getint('honeypot', 'download_limit_size', fallback=0)
-    download_path = CowrieConfig().get('honeypot', 'download_path')
-
-    def start(self):
-        try:
-            optlist, args = getopt.getopt(self.args, 'sho:O', ['help', 'manual', 'silent'])
-        except getopt.GetoptError as err:
-            # TODO: should be 'unknown' instead of 'not recognized'
-            self.write("curl: {}\n".format(err))
-            self.write("curl: try 'curl --help' or 'curl --manual' for more information\n")
-            self.exit()
-            return
-
-        for opt in optlist:
-            if opt[0] == '-h' or opt[0] == '--help':
-                self.curl_help()
-                return
-            elif opt[0] == '-s' or opt[0] == '--silent':
-                self.silent = True
-
-        if len(args):
-            if args[0] is not None:
-                url = str(args[0]).strip()
-        else:
-            self.write("curl: try 'curl --help' or 'curl --manual' for more information\n")
-            self.exit()
-            return
-
-        if '://' not in url:
-            url = 'http://' + url
-        urldata = compat.urllib_parse.urlparse(url)
-
-        outfile = None
-        for opt in optlist:
-            if opt[0] == '-o':
-                outfile = opt[1]
-            if opt[0] == '-O':
-                outfile = urldata.path.split('/')[-1]
-                if outfile is None or not len(outfile.strip()) or not urldata.path.count('/'):
-                    self.write('curl: Remote file name has no length!\n')
-                    self.exit()
-                    return
-
-        if outfile:
-            outfile = self.fs.resolve_path(outfile, self.protocol.cwd)
-            path = os.path.dirname(outfile)
-            if not path or \
-                    not self.fs.exists(path) or \
-                    not self.fs.isdir(path):
-                self.write('curl: %s: Cannot open: No such file or directory\n' % outfile)
-                self.exit()
-                return
-
-        url = url.encode('ascii')
-        self.url = url
-
-        self.artifactFile = Artifact(outfile)
-        # HTTPDownloader will close() the file object so need to preserve the name
-
-        self.deferred = self.download(url, outfile, self.artifactFile)
-        if self.deferred:
-            self.deferred.addCallback(self.success, outfile)
-            self.deferred.addErrback(self.error, url)
-
-    def curl_help(self):
-        self.write("""Usage: curl [options...] <url>
+CURL_HELP = """Usage: curl [options...] <url>
 Options: (H) means HTTP/HTTPS only, (F) means FTP only
      --anyauth       Pick "any" authentication method (H)
  -a, --append        Append to target file when uploading (F/SFTP)
@@ -242,8 +170,78 @@ def curl_help(self):
  -V, --version       Show version number and quit
  -w, --write-out FORMAT  What to output after completion
      --xattr        Store metadata in extended file attributes
- -q                 If used as the first parameter disables .curlrc\n""")
-        self.exit()
+ -q                 If used as the first parameter disables .curlrc
+ """
+
+
+class command_curl(HoneyPotCommand):
+    """
+    curl command
+    """
+    limit_size = CowrieConfig().getint('honeypot', 'download_limit_size', fallback=0)
+    download_path = CowrieConfig().get('honeypot', 'download_path')
+
+    def start(self):
+        try:
+            optlist, args = getopt.getopt(self.args, 'sho:O', ['help', 'manual', 'silent'])
+        except getopt.GetoptError as err:
+            # TODO: should be 'unknown' instead of 'not recognized'
+            self.write("curl: {}\n".format(err))
+            self.write("curl: try 'curl --help' or 'curl --manual' for more information\n")
+            self.exit()
+            return
+
+        for opt in optlist:
+            if opt[0] == '-h' or opt[0] == '--help':
+                self.write(CURL_HELP)
+                self.exit()
+                return
+            elif opt[0] == '-s' or opt[0] == '--silent':
+                self.silent = True
+
+        if len(args):
+            if args[0] is not None:
+                url = str(args[0]).strip()
+        else:
+            self.write("curl: try 'curl --help' or 'curl --manual' for more information\n")
+            self.exit()
+            return
+
+        if '://' not in url:
+            url = 'http://' + url
+        urldata = compat.urllib_parse.urlparse(url)
+
+        outfile = None
+        for opt in optlist:
+            if opt[0] == '-o':
+                outfile = opt[1]
+            if opt[0] == '-O':
+                outfile = urldata.path.split('/')[-1]
+                if outfile is None or not len(outfile.strip()) or not urldata.path.count('/'):
+                    self.write('curl: Remote file name has no length!\n')
+                    self.exit()
+                    return
+
+        if outfile:
+            outfile = self.fs.resolve_path(outfile, self.protocol.cwd)
+            path = os.path.dirname(outfile)
+            if not path or \
+                    not self.fs.exists(path) or \
+                    not self.fs.isdir(path):
+                self.write('curl: %s: Cannot open: No such file or directory\n' % outfile)
+                self.exit()
+                return
+
+        url = url.encode('ascii')
+        self.url = url
+
+        self.artifactFile = Artifact(outfile)
+        # HTTPDownloader will close() the file object so need to preserve the name
+
+        self.deferred = self.download(url, outfile, self.artifactFile)
+        if self.deferred:
+            self.deferred.addCallback(self.success, outfile)
+            self.deferred.addErrback(self.error, url)
 
     def download(self, url, fakeoutfile, outputfile, *args, **kwargs):
         try:
@@ -265,11 +263,10 @@ def download(self, url, fakeoutfile, outputfile, *args, **kwargs):
             out_addr = (CowrieConfig().get('honeypot', 'out_addr'), 0)
 
         if scheme == 'https':
-            contextFactory = ssl.CertificateOptions(method=SSL.SSLv23_METHOD)
-            reactor.connectSSL(host, port, factory, contextFactory, bindAddress=out_addr)
+            context_factory = ssl.optionsForClientTLS(hostname=host)
+            self.connection = reactor.connectSSL(host, port, factory, context_factory, bindAddress=out_addr)
         else:  # Can only be http
-            self.connection = reactor.connectTCP(
-                host, port, factory, bindAddress=out_addr)
+            self.connection = reactor.connectTCP(host, port, factory, bindAddress=out_addr)
 
         return factory.deferred
 
@@ -320,7 +317,7 @@ class HTTPProgressDownloader(client.HTTPDownloader):
     lastupdate = 0
 
     def __init__(self, curl, fakeoutfile, url, outfile, headers=None):
-        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=b'curl/7.38.0')
+        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=b'curl/7.38.0', followRedirect=False)
         self.status = None
         self.curl = curl
         self.fakeoutfile = fakeoutfile
diff --git a/src/cowrie/commands/wget.py b/src/cowrie/commands/wget.py
--- a/src/cowrie/commands/wget.py
+++ b/src/cowrie/commands/wget.py
@@ -7,8 +7,6 @@
 import os
 import time
 
-from OpenSSL import SSL
-
 from twisted.internet import reactor, ssl
 from twisted.python import compat, log
 from twisted.web import client
@@ -77,20 +75,20 @@ def start(self):
             self.exit()
             return
 
-        outfile = None
+        self.outfile = None
         self.quiet = False
         for opt in optlist:
             if opt[0] == '-O':
-                outfile = opt[1]
+                self.outfile = opt[1]
             if opt[0] == '-q':
                 self.quiet = True
 
         # for some reason getopt doesn't recognize "-O -"
         # use try..except for the case if passed command is malformed
         try:
-            if not outfile:
+            if not self.outfile:
                 if '-O' in args:
-                    outfile = args[args.index('-O') + 1]
+                    self.outfile = args[args.index('-O') + 1]
         except Exception:
             pass
 
@@ -99,38 +97,32 @@ def start(self):
 
         urldata = compat.urllib_parse.urlparse(url)
 
-        url = url.encode('utf8')
+        self.url = url.encode('utf8')
 
-        if outfile is None:
-            outfile = urldata.path.split('/')[-1]
-            if not len(outfile.strip()) or not urldata.path.count('/'):
-                outfile = 'index.html'
+        if self.outfile is None:
+            self.outfile = urldata.path.split('/')[-1]
+            if not len(self.outfile.strip()) or not urldata.path.count('/'):
+                self.outfile = 'index.html'
 
-        if outfile != '-':
-            outfile = self.fs.resolve_path(outfile, self.protocol.cwd)
-            path = os.path.dirname(outfile)
+        if self.outfile != '-':
+            self.outfile = self.fs.resolve_path(self.outfile, self.protocol.cwd)
+            path = os.path.dirname(self.outfile)
             if not path or not self.fs.exists(path) or not self.fs.isdir(path):
-                self.errorWrite('wget: %s: Cannot open: No such file or directory\n' % outfile)
+                self.errorWrite('wget: %s: Cannot open: No such file or directory\n' % self.outfile)
                 self.exit()
                 return
 
-        self.url = url
-
-        self.artifactFile = Artifact(outfile)
-        # HTTPDownloader will close() the file object so need to preserve the name
-
-        d = self.download(url, outfile, self.artifactFile)
-        if d:
-            d.addCallback(self.success, outfile)
-            d.addErrback(self.error, url)
+        self.deferred = self.download(self.url, self.outfile)
+        if self.deferred:
+            self.deferred.addCallback(self.success)
+            self.deferred.addErrback(self.error, self.url)
         else:
             self.exit()
 
-    def download(self, url, fakeoutfile, outputfile, *args, **kwargs):
+    def download(self, url, fakeoutfile, *args, **kwargs):
         """
         url - URL to download
         fakeoutfile - file in guest's fs that attacker wants content to be downloaded to
-        outputfile - file in host's fs that will hold content of the downloaded file
         """
         try:
             parsed = compat.urllib_parse.urlparse(url)
@@ -145,20 +137,25 @@ def download(self, url, fakeoutfile, outputfile, *args, **kwargs):
             self.errorWrite('%s: Unsupported scheme.\n' % (url,))
             return None
 
+        # File in host's fs that will hold content of the downloaded file
+        # HTTPDownloader will close() the file object so need to preserve the name
+        self.artifactFile = Artifact(self.outfile)
+
         if not self.quiet:
             self.errorWrite('--%s--  %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), url.decode('utf8')))
             self.errorWrite('Connecting to %s:%d... connected.\n' % (host, port))
             self.errorWrite('HTTP request sent, awaiting response... ')
 
-        factory = HTTPProgressDownloader(self, fakeoutfile, url, outputfile, *args, **kwargs)
+        factory = HTTPProgressDownloader(self, fakeoutfile, url, self.artifactFile, *args, **kwargs)
 
         out_addr = None
         if CowrieConfig().has_option('honeypot', 'out_addr'):
             out_addr = (CowrieConfig().get('honeypot', 'out_addr'), 0)
 
         if scheme == b'https':
-            contextFactory = ssl.CertificateOptions(method=SSL.SSLv23_METHOD)
-            self.connection = reactor.connectSSL(host, port, factory, contextFactory, bindAddress=out_addr)
+            context_factory = ssl.optionsForClientTLS(hostname=host)
+            self.connection = reactor.connectSSL(host, port, factory, context_factory, bindAddress=out_addr)
+
         elif scheme == b'http':
             self.connection = reactor.connectTCP(host, port, factory, bindAddress=out_addr)
         else:
@@ -170,7 +167,7 @@ def handle_CTRL_C(self):
         self.errorWrite('^C\n')
         self.connection.transport.loseConnection()
 
-    def success(self, data, outfile):
+    def success(self, data):
         if not os.path.isfile(self.artifactFile.shasumFilename):
             log.msg("there's no file " + self.artifactFile.shasumFilename)
             self.exit()
@@ -189,9 +186,9 @@ def success(self, data, outfile):
                                   shasum=self.artifactFile.shasum)
 
         # Update honeyfs to point to downloaded file or write to screen
-        if outfile != '-':
-            self.fs.update_realfile(self.fs.getfile(outfile), self.artifactFile.shasumFilename)
-            self.fs.chown(outfile, self.protocol.user.uid, self.protocol.user.gid)
+        if self.outfile != '-':
+            self.fs.update_realfile(self.fs.getfile(self.outfile), self.artifactFile.shasumFilename)
+            self.fs.chown(self.outfile, self.protocol.user.uid, self.protocol.user.gid)
         else:
             with open(self.artifactFile.shasumFilename, 'rb') as f:
                 self.writeBytes(f.read())
@@ -199,31 +196,45 @@ def success(self, data, outfile):
         self.exit()
 
     def error(self, error, url):
-        if hasattr(error, 'getErrorMessage'):  # exceptions
-            errorMessage = error.getErrorMessage()
-            self.errorWrite(errorMessage + '\n')
-            # Real wget also adds this:
-        if hasattr(error, 'webStatus') and error.webStatus and hasattr(error, 'webMessage'):  # exceptions
-            self.errorWrite('{} ERROR {}: {}\n'.format(time.strftime('%Y-%m-%d %T'), error.webStatus.decode(),
-                                                       error.webMessage.decode('utf8')))
+        # we need to handle 301 redirects separately
+        if hasattr(error, 'webStatus') and error.webStatus.decode() == '301':
+            self.errorWrite('{} {}\n'.format(error.webStatus.decode(), error.webMessage.decode()))
+            https_url = error.getErrorMessage().replace('301 Moved Permanently to ', '')
+            self.errorWrite('Location {} [following]\n'.format(https_url))
+
+            # do the download again with the https URL
+            self.deferred = self.download(https_url.encode('utf8'), self.outfile)
+            if self.deferred:
+                self.deferred.addCallback(self.success)
+                self.deferred.addErrback(self.error, https_url)
+            else:
+                self.exit()
         else:
-            self.errorWrite('{} ERROR 404: Not Found.\n'.format(time.strftime('%Y-%m-%d %T')))
+            if hasattr(error, 'getErrorMessage'):  # exceptions
+                errorMessage = error.getErrorMessage()
+                self.errorWrite(errorMessage + '\n')
+                # Real wget also adds this:
+            if hasattr(error, 'webStatus') and error.webStatus and hasattr(error, 'webMessage'):  # exceptions
+                self.errorWrite('{} ERROR {}: {}\n'.format(time.strftime('%Y-%m-%d %T'), error.webStatus.decode(),
+                                                           error.webMessage.decode('utf8')))
+            else:
+                self.errorWrite('{} ERROR 404: Not Found.\n'.format(time.strftime('%Y-%m-%d %T')))
 
-        # prevent cowrie from crashing if the terminal have been already destroyed
-        try:
-            self.protocol.logDispatch(eventid='cowrie.session.file_download.failed',
-                                      format='Attempt to download file(s) from URL (%(url)s) failed',
-                                      url=self.url)
-        except Exception:
-            pass
+            # prevent cowrie from crashing if the terminal have been already destroyed
+            try:
+                self.protocol.logDispatch(eventid='cowrie.session.file_download.failed',
+                                          format='Attempt to download file(s) from URL (%(url)s) failed',
+                                          url=self.url)
+            except Exception:
+                pass
 
-        self.exit()
+            self.exit()
 
 
 # From http://code.activestate.com/recipes/525493/
 class HTTPProgressDownloader(client.HTTPDownloader):
     def __init__(self, wget, fakeoutfile, url, outfile, headers=None):
-        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=b'Wget/1.11.4')
+        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=b'Wget/1.11.4', followRedirect=False)
         self.status = None
         self.wget = wget
         self.fakeoutfile = fakeoutfile


Test patch: