Repository: obspy/obspy

Issue:
decryption of arclink download: Unicode error
I get a UnicodeDecodeError when downloading encrypted data with the arclink client. It occurs when the key and the iv are created.

I use 
ObsPy 1.0.0
Python 2.7.11
Anaconda 2.4.1
Ubuntu 14.04

The M2Crypto package I have installed is
m2crypto 0.23.0  py27_1  anaconda-cluster

Here is the code I used to download

``` python
from obspy import UTCDateTime
from obspy.clients.arclink.client import Client
client = Client(user='sens-schoenfelder@gfz-potsdam.de',debug=True)
t = UTCDateTime(2015, 6, 17, 0, 0,0)  
st = client.get_waveforms("XN", "NEP05", "*", "HHZ", t,t+2,compressed=False)
```

To check the delivery form the datacenter I commented out lines 375 and 376 form arclink/client.py `data = decryptor.update(data); data += decryptor.final()`

and saved the un-decrypted stream in file_stream.openssl after  execution of 

``` python
self.save_waveforms(file_stream, network, station, location, channel,
                            starttime, endtime, format=format,
                            compressed=compressed, route=route)
```

on line 444.

This can be decrypted correctly with openssl

```
openssl des-cbc -pass pass:'my_password' -in file_stream.openssl -out stream.mseed -d
```

The UnicodeError does not depend on my password. But if anyone looks into this I can give you the password.

here are the files:
encrypted: https://www.dropbox.com/s/dijvkb3wgg5z3ie/stream.mseed.openssl?dl=0
decrypted: https://www.dropbox.com/s/go7v3p5aye7wz3o/stream.mseed?dl=0

And finally here is the full output:

```
Connected to webdc.eu:18002
>>> HELLO

... ArcLink v1.2 (2013.070)
... GFZ
>>> USER sens-schoenfelder@gfz-potsdam.de

... OK
>>> INSTITUTION Anonymous

... OK
>>> REQUEST ROUTING 

>>> 2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05

>>> END

... OK
... 34994076
>>> STATUS 34994076

... <?xml version="1.0"?>
<arclink>
  <request id="34994076" type="ROUTING" label="" args="" encrypted="false" size="0" ready="false" error="false" message="">
    <volume id="UNSET" status="UNSET" size="0" message="">
      <line content="2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05" status="UNSET" size="0" message=""/>
    </volume>
  </request>
</arclink>
END
>>> STATUS 34994076

... <?xml version="1.0"?>
<arclink>
  <request id="34994076" type="ROUTING" label="" args="" encrypted="false" size="390" ready="true" error="false" message="">
    <volume id="routing" dcid="GFZ" status="OK" size="390" encrypted="false" message="">
      <line content="2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05" status="OK" size="0" message=""/>
    </volume>
  </request>
</arclink>
END
>>> DOWNLOAD 34994076

<?xml version="1.0" encoding="utf-8"?>
<ns0:routing xmlns:ns0="http://geofon.gfz-potsdam.de/ns/Routing/1.0/">
    <ns0:route locationCode="" networkCode="XN" publicID="Route#20150305195044.937296.9213" stationCode="" streamCode="">
        <ns0:arclink address="eida.gfz-potsdam.de:18002" end="2020-12-31T00:00:00.0000Z" priority="1" start="2015-01-01T00:00:00.0000Z" />
    </ns0:route>
</ns0:routing>
>>> PURGE 34994076

>>> BYE


Requesting eida.gfz-potsdam.de:18002
>>> HELLO

... ArcLink v1.2 (2013.070)
... GFZ
>>> USER sens-schoenfelder@gfz-potsdam.de

... OK
>>> INSTITUTION Anonymous

... OK
>>> REQUEST WAVEFORM format=MSEED

>>> 2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05 HHZ *

>>> END

... OK
... 34994078
>>> STATUS 34994078

... <?xml version="1.0"?>
<arclink>
  <request id="34994078" type="WAVEFORM" label="" args="format=MSEED" encrypted="false" size="0" ready="false" error="false" message="">
    <volume id="UNSET" status="UNSET" size="0" message="">
      <line content="2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05 HHZ *" status="UNSET" size="0" message=""/>
    </volume>
  </request>
</arclink>
END
>>> STATUS 34994078

... <?xml version="1.0"?>
<arclink>
  <request id="34994078" type="WAVEFORM" label="" args="format=MSEED" encrypted="true" size="24" ready="false" error="false" message="">
    <volume id="UNSET" status="UNSET" size="0" message="">
      <line content="2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05 HHZ *" status="UNSET" size="0" message=""/>
    </volume>
  </request>
</arclink>
END
>>> STATUS 34994078

... <?xml version="1.0"?>
<arclink>
  <request id="34994078" type="WAVEFORM" label="" args="format=MSEED" encrypted="true" size="37527576" ready="false" error="false" message="">
    <volume id="GFZ" dcid="GFZ" status="OK" size="37527576" encrypted="true" message="">
      <line content="2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05 HHZ *" status="OK" size="37527552" message=""/>
    </volume>
  </request>
</arclink>
END
>>> STATUS 34994078

... <?xml version="1.0"?>
<arclink>
  <request id="34994078" type="WAVEFORM" label="" args="format=MSEED" encrypted="true" size="37527576" ready="true" error="false" message="">
    <volume id="GFZ" dcid="GFZ" status="OK" size="37527576" encrypted="true" message="">
      <line content="2015,6,16,23,59,59,0 2015,6,17,0,0,3,0 XN NEP05 HHZ *" status="OK" size="37527552" message=""/>
    </volume>
  </request>
</arclink>
END
>>> DOWNLOAD 34994078

37527576 bytes of data read
>>> PURGE 34994078

>>> BYE

Traceback (most recent call last):
  File "encrypt_check.py", line 7, in <module>
    st = client.get_waveforms("XN", "NEP05", "*", "HHZ", t,t+2,compressed=False)
  File "/home/chris/lib/anaconda2/lib/python2.7/site-packages/obspy/clients/arclink/client.py", line 442, in get_waveforms
    compressed=compressed, route=route)
  File "/home/chris/lib/anaconda2/lib/python2.7/site-packages/obspy/clients/arclink/client.py", line 566, in save_waveforms
    data = self._fetch(rtype, rdata, route=route)
  File "/home/chris/lib/anaconda2/lib/python2.7/site-packages/obspy/clients/arclink/client.py", line 261, in _fetch
    return self._request(request_type, request_data)
  File "/home/chris/lib/anaconda2/lib/python2.7/site-packages/obspy/clients/arclink/client.py", line 373, in _request
    data = decryptor.update(data)
  File "/home/chris/lib/anaconda2/lib/python2.7/site-packages/obspy/clients/arclink/decrypt.py", line 43, in update
    [key, iv] = self._get_key_iv(self._password, chunk[8:16])
  File "/home/chris/lib/anaconda2/lib/python2.7/site-packages/obspy/clients/arclink/decrypt.py", line 71, in _get_key_iv
    key += chunk[0:i]
UnicodeDecodeError: 'ascii' codec can't decode byte 0x97 in position 3: ordinal not in range(128)
```



Discussion:
see also #1314

Please send me your pw to barsch@egu.eu - I'll take a look this evening

I read the #1314 and I don't think it is related. The password properly arrives in _get_key_iv as a unicode string. u'my_password'

could be - but using your credentials I am maybe able to solve this too - will see ...

Thanks! Works fine.


Resolving patch:
diff --git a/obspy/clients/arclink/client.py b/obspy/clients/arclink/client.py
--- a/obspy/clients/arclink/client.py
+++ b/obspy/clients/arclink/client.py
@@ -182,7 +182,7 @@ def _write_ln(self, buffer):
         b_buffer = (buffer + '\r\n').encode()
         self._client.write(b_buffer)
         if self.debug:
-            print(b'>>> ' + b_buffer)
+            print('>>> ' + buffer)
 
     def _read_ln(self, value=b''):
         line = self._client.read_until(value + b'\r\n', self.timeout)
@@ -191,7 +191,7 @@ def _read_ln(self, value=b''):
             msg = "Timeout waiting for expected %s, got %s"
             raise ArcLinkException(msg % (value, line.decode()))
         if self.debug:
-            print(b'... ' + line)
+            print('... ' + line.decode())
         return line
 
     def _hello(self):
diff --git a/obspy/clients/arclink/decrypt.py b/obspy/clients/arclink/decrypt.py
--- a/obspy/clients/arclink/decrypt.py
+++ b/obspy/clients/arclink/decrypt.py
@@ -14,19 +14,27 @@
                         unicode_literals)
 from future.builtins import *  # NOQA
 
+import hashlib
+
 try:
     from M2Crypto import EVP
     hasM2Crypto = True
 except ImportError:
     hasM2Crypto = False
 
+try:
+    from Crypto.Cipher import DES
+    hasPyCrypto = True
+except ImportError:
+    hasPyCrypto = False
+
 
 class SSLWrapper:
     """
     """
     def __init__(self, password):
-        if not hasM2Crypto:
-            raise Exception("Module M2Crypto was not found on this system.")
+        if not (hasM2Crypto or hasPyCrypto):
+            raise ImportError("Module M2Crypto or PyCrypto is needed.")
         self._cypher = None
         self._password = None
         if password is None:
@@ -38,33 +46,41 @@ def update(self, chunk):
         if self._cypher is None:
             if len(chunk) < 16:
                 raise Exception('Invalid first chunk (Size < 16).')
-            if chunk[0:8] != "Salted__":
+            if chunk[0:8] != b"Salted__":
                 raise Exception('Invalid first chunk (expected: Salted__')
             [key, iv] = self._get_key_iv(self._password, chunk[8:16])
-            self._cypher = EVP.Cipher('des_cbc', key, iv, 0)
             chunk = chunk[16:]
-        if len(chunk) > 0:
-            return self._cypher.update(chunk)
-        else:
-            return ''
+            if len(chunk) <= 0:
+                return ''
+            if hasM2Crypto:
+                self._cypher = EVP.Cipher('des_cbc', key, iv, 0)
+                return self._cypher.update(chunk)
+            else:
+                self._cypher = DES.new(key, DES.MODE_CBC, iv)
+                return self._cypher.decrypt(chunk)
 
     def final(self):
         if self._cypher is None:
             raise Exception('Wrapper has not started yet.')
-        return self._cypher.final()
+        if hasM2Crypto:
+            return self._cypher.final()
+        else:
+            return b""
 
     def _get_key_iv(self, password, salt=None, size=8):
+        # make sure password is a string and not unicode
+        password = password.encode('utf-8')
         chunk = None
-        key = ""
-        iv = ""
+        key = b""
+        iv = b""
         while True:
-            hash = EVP.MessageDigest('md5')
+            hash = hashlib.md5()
             if (chunk is not None):
                 hash.update(chunk)
             hash.update(password)
             if (salt is not None):
                 hash.update(salt)
-            chunk = hash.final()
+            chunk = hash.digest()
             i = 0
             if len(key) < size:
                 i = min(size - len(key), len(chunk))


Test patch:
diff --git a/obspy/clients/arclink/tests/test_client.py b/obspy/clients/arclink/tests/test_client.py
--- a/obspy/clients/arclink/tests/test_client.py
+++ b/obspy/clients/arclink/tests/test_client.py
@@ -573,7 +573,7 @@ def test_save_response(self):
         with NamedTemporaryFile() as tf:
             tempfile = tf.name
             # Dataless SEED
-            client.saveResponse(tempfile, 'BW', 'MANZ', '', 'EHZ', start, end)
+            client.save_response(tempfile, 'BW', 'MANZ', '', 'EHZ', start, end)
             with open(tempfile, 'rb') as fp:
                 self.assertEqual(fp.read(8), b"000001V ")
 
@@ -583,7 +583,7 @@ def test_save_response(self):
         start = UTCDateTime(2008, 1, 1)
         end = start + 1
         # Dataless SEED
-        client.saveResponse(file_object, 'BW', 'MANZ', '', 'EHZ', start, end)
+        client.save_response(file_object, 'BW', 'MANZ', '', 'EHZ', start, end)
         file_object.seek(0, 0)
         self.assertEqual(file_object.read(8), b"000001V ")