Repository: secdev/scapy

Issue:
fix __str__, __bytes__ for ASN1_BIT_STRING to return BER encoded values
ASN1_BIT_STRING.\_\_bytes__() and ASN1_BIT_STRING.\_\_str__() should return BER encoded TLV, but returns self.val_readable instead.

fixes #1761



Discussion:
This apparently breaks Certificate reading :/
Well, the current implementation of \_\_str__ and \_\_bytes__ for ASN1_BIT_STRING is wrong. I discovered this while I was implementing the SV and GOOSE protocols (61850) for Scapy. It's possible that something in Certificate reading depends on this bug. I can take a look to see if  can fix it.

Can you point me to which test I should be looking at for that? Doesn't show up when I run the tests (i.e. no difference pre and post change).
BTW, maybe you can tell me a little about what the idea behind val_readable is here. Seems like you may know something about it.
@guedou @p-l- Could one of you take care of this please ? I’m not very familiar with the ASN1 implementation..
Neither am I :/. @guedou?
Any chance we are going to fix this? I have offered to look at the issue you have with [Certificate reading](https://github.com/secdev/scapy/pull/1765#issuecomment-450507896) - if you give me some details.

And, if you like, we can talk about why I am saying that this here is a bug.

I'd rather not maintain a separate fork just for this one line fix.
Could you take a look at the linked bug #1761? Please take a look at how the type ASN1_BOOLEAN or ASN1_INTEGER (or for that matter any of the ASN1 types) behave. Their \_\_str__ and \_\_bytes__ methods return the ASN1 encoded values. Also, look at the implementation of these methods in ASN1_Object to see what the original intent behind these methods is. The encoded values have the TLV format (which is not what you are showing above).

The packet "composition" mechanism relies on the correct implementation of these methods to put together the final packet. This bug causes a bad packet to be composed (for types where a ASN1_BIT_STRING is used).

This bug got introduced when someone did this implementation around val_readable. I see 2 developers in git annotate who may know something about the motivation behind that.

Resolving patch:
diff --git a/scapy/asn1/asn1.py b/scapy/asn1/asn1.py
--- a/scapy/asn1/asn1.py
+++ b/scapy/asn1/asn1.py
@@ -17,7 +17,7 @@
 from scapy.error import Scapy_Exception, warning
 from scapy.volatile import RandField, RandIP, GeneralizedTime
 from scapy.utils import Enum_metaclass, EnumElement, binrepr
-from scapy.compat import plain_str, chb, orb, bytes_encode
+from scapy.compat import plain_str, chb, orb
 import scapy.modules.six as six
 from scapy.modules.six.moves import range
 
@@ -318,7 +318,7 @@ class ASN1_BIT_STRING(ASN1_Object):
     """
      ASN1_BIT_STRING values are bit strings like "011101".
      A zero-bit padded readable string is provided nonetheless,
-     which is also output when __str__ is called.
+     which is stored in val_readable
     """
     tag = ASN1_Class_UNIVERSAL.BIT_STRING
 
@@ -329,59 +329,56 @@ def __init__(self, val, readable=False):
             self.val_readable = val
 
     def __setattr__(self, name, value):
-        if isinstance(value, str):
-            value = bytes_encode(value)
         if name == "val_readable":
-            if isinstance(value, bytes):
-                val = b"".join(binrepr(orb(x)).zfill(8).encode("utf8") for x in value)  # noqa: E501
+            if isinstance(value, (str, bytes)):
+                val = "".join(binrepr(orb(x)).zfill(8) for x in value)
             else:
+                warning("Invalid val: should be bytes")
                 val = "<invalid val_readable>"
             super(ASN1_Object, self).__setattr__("val", val)
             super(ASN1_Object, self).__setattr__(name, value)
             super(ASN1_Object, self).__setattr__("unused_bits", 0)
         elif name == "val":
-            if isinstance(value, bytes):
-                if any(True for x in range(len(value))
-                       if value[x:x + 1] not in [b"0", b"1"]):
-                    print("Invalid operation: 'val' is not a valid bit string.")  # noqa: E501
+            value = plain_str(value)
+            if isinstance(value, str):
+                if any(c for c in value if c not in ["0", "1"]):
+                    warning("Invalid operation: 'val' is not a valid bit string.")  # noqa: E501
                     return
                 else:
                     if len(value) % 8 == 0:
                         unused_bits = 0
                     else:
                         unused_bits = 8 - (len(value) % 8)
-                    padded_value = value + (b"0" * unused_bits)
-                    bytes_arr = zip(*[(padded_value[i:i + 1]
-                                       for i in range(len(padded_value)))] * 8)
-                    val_readable = b"".join(chb(int(b"".join(x), 2))
-                                            for x in bytes_arr)
+                    padded_value = value + ("0" * unused_bits)
+                    bytes_arr = zip(*[iter(padded_value)] * 8)
+                    val_readable = b"".join(chb(int("".join(x), 2)) for x in bytes_arr)  # noqa: E501
             else:
-                val_readable = "<invalid val>"
+                warning("Invalid val: should be str")
+                val_readable = b"<invalid val>"
                 unused_bits = 0
             super(ASN1_Object, self).__setattr__("val_readable", val_readable)
             super(ASN1_Object, self).__setattr__(name, value)
             super(ASN1_Object, self).__setattr__("unused_bits", unused_bits)
         elif name == "unused_bits":
-            print("Invalid operation: unused_bits rewriting is not supported.")
+            warning("Invalid operation: unused_bits rewriting "
+                    "is not supported.")
         else:
             super(ASN1_Object, self).__setattr__(name, value)
 
     def __repr__(self):
-        if len(self.val) <= 16:
-            v = plain_str(self.val)
-            return "<%s[%s] (%d unused bit%s)>" % (self.__dict__.get("name", self.__class__.__name__), v, self.unused_bits, "s" if self.unused_bits > 1 else "")  # noqa: E501
-        else:
-            s = self.val_readable
-            if len(s) > 20:
-                s = s[:10] + b"..." + s[-10:]
-            v = plain_str(self.val)
-            return "<%s[%s] (%d unused bit%s)>" % (self.__dict__.get("name", self.__class__.__name__), v, self.unused_bits, "s" if self.unused_bits > 1 else "")  # noqa: E501
-
-    def __str__(self):
-        return self.val_readable
-
-    def __bytes__(self):
-        return self.val_readable
+        s = self.val_readable
+        if len(s) > 16:
+            s = s[:10] + b"..." + s[-10:]
+        v = self.val
+        if len(v) > 20:
+            v = v[:10] + "..." + v[-10:]
+        return "<%s[%s]=%s (%d unused bit%s)>" % (
+            self.__dict__.get("name", self.__class__.__name__),
+            v,
+            s,
+            self.unused_bits,
+            "s" if self.unused_bits > 1 else ""
+        )
 
 
 class ASN1_STRING(ASN1_Object):
diff --git a/scapy/layers/tls/cert.py b/scapy/layers/tls/cert.py
--- a/scapy/layers/tls/cert.py
+++ b/scapy/layers/tls/cert.py
@@ -407,6 +407,13 @@ def __call__(cls, key_path=None):
         return obj
 
 
+class _Raw_ASN1_BIT_STRING(ASN1_BIT_STRING):
+    """A ASN1_BIT_STRING that ignores BER encoding"""
+    def __bytes__(self):
+        return self.val_readable
+    __str__ = __bytes__
+
+
 class PrivKey(six.with_metaclass(_PrivKeyFactory, object)):
     """
     Parent class for both PrivKeyRSA and PrivKeyECDSA.
@@ -433,7 +440,7 @@ def signTBSCert(self, tbsCert, h="sha256"):
         c = X509_Cert()
         c.tbsCertificate = tbsCert
         c.signatureAlgorithm = sigAlg
-        c.signatureValue = ASN1_BIT_STRING(sigVal, readable=True)
+        c.signatureValue = _Raw_ASN1_BIT_STRING(sigVal, readable=True)
         return c
 
     def resignCert(self, cert):


Test patch:
diff --git a/test/cert.uts b/test/cert.uts
--- a/test/cert.uts
+++ b/test/cert.uts
@@ -351,7 +351,7 @@ assert(pubkey.curve.name == 'secp384r1')
 pubkey.public_numbers().x == 3987178688175281746349180015490646948656137448666005327832107126183726641822596270780616285891030558662603987311874
 
 = Cert class : Checking ECDSA signature
-y.signatureValue == b'0d\x020%\xa4\x81E\x02k\x12KutO\xc8#\xe3p\xf2ur\xde|\x89\xf0\xcf\x91ra\x9e^\x10\x92YV\xb9\x83\xc7\x10\xe78\xe9X&6}\xd5\xe44\x869\x020|6S\xf00\xe5bc:\x99\xe2\xb6\xa3;\x9b4\xfa\x1e\xda\x10\x92q^\x91\x13\xa7\xdd\xa4n\x92\xcc2\xd6\xf5!f\xc7/\xea\x96cjeE\x92\x95\x01\xb4'
+raw(y.signatureValue) == b'0d\x020%\xa4\x81E\x02k\x12KutO\xc8#\xe3p\xf2ur\xde|\x89\xf0\xcf\x91ra\x9e^\x10\x92YV\xb9\x83\xc7\x10\xe78\xe9X&6}\xd5\xe44\x869\x020|6S\xf00\xe5bc:\x99\xe2\xb6\xa3;\x9b4\xfa\x1e\xda\x10\x92q^\x91\x13\xa7\xdd\xa4n\x92\xcc2\xd6\xf5!f\xc7/\xea\x96cjeE\x92\x95\x01\xb4'
 
 = Cert class : Test show
 awaited = """
diff --git a/test/regression.uts b/test/regression.uts
--- a/test/regression.uts
+++ b/test/regression.uts
@@ -1133,13 +1133,15 @@ random.seed(0x2807)
 assert raw(a) in [b'A\x02\x07q', b'C\x02\xfe\x92', b'\x1e\x023V']
 
 = ASN1 - ASN1_BIT_STRING
-a = ASN1_BIT_STRING("test", "value")
+a = ASN1_BIT_STRING("test", readable=True)
 a
-assert raw(a) == b'test'
+assert a.val == '01110100011001010111001101110100'
+assert raw(a) == b'\x03\x05\x00test'
 
-a = ASN1_BIT_STRING(b"\xff"*16, "value")
+a = ASN1_BIT_STRING(b"\xff"*16, readable=True)
 a
-assert raw(a) == b'\xff'*16
+assert a.val == "1" * 128
+assert raw(a) == b'\x03\x11\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff'
 
 = ASN1 - ASN1_SEQUENCE
 a = ASN1_SEQUENCE([ASN1_Object(1), ASN1_Object(0)])
@@ -1149,7 +1151,8 @@ assert a.strshow() == '# ASN1_SEQUENCE:\n  <ASN1_Object[1]>\n  <ASN1_Object[0]>\
 a = ASN1_DECODING_ERROR("error", exc=OSError(1))
 assert repr(a) == "<ASN1_DECODING_ERROR['error']{{1}}>"
 b = ASN1_DECODING_ERROR("error", exc=OSError(ASN1_BIT_STRING("0")))
-assert repr(b) == "<ASN1_DECODING_ERROR['error']{{<ASN1_BIT_STRING[0] (7 unused bits)>}}>"
+assert repr(b) in ["<ASN1_DECODING_ERROR['error']{{<ASN1_BIT_STRING[0]=b'\\x00' (7 unused bits)>}}>",
+                   "<ASN1_DECODING_ERROR['error']{{<ASN1_BIT_STRING[0]=\x00 (7 unused bits)>}}>"]
 
 = ASN1 - ASN1_INTEGER
 a = ASN1_INTEGER(int("1"*23))