Repository: ahawker/ulid

Issue:
Properly handle invalid base32 characters
As of today, it is possible to input non-base32 characters, `uU` for example, into any of the  `api` calls.

Doing this will cause the library to fail silently and perform an incorrect `base32` decode on the string.

The API should provide a feedback mechanism that informs the caller of the bad input. The implementation of that feedback is still TBD (separate API call vs. exception vs. ??).

**Considerations:**

* Performance of this computation for every decode call?
* Double-penality for callers that have already made this guarantee?
* Separate API call to validate? Is there use-cases for this outside of normal hot path?


Discussion:

Patch:
diff --git a/ulid/base32.py b/ulid/base32.py
index f7377b6..b63ba7a 100644
--- a/ulid/base32.py
+++ b/ulid/base32.py
@@ -248,14 +248,7 @@ def decode_ulid(value: str) -> bytes:
     :raises ValueError: when value is not 26 characters
     :raises ValueError: when value cannot be encoded in ASCII
     """
-    length = len(value)
-    if length != 26:
-        raise ValueError('Expects 26 characters for timestamp + randomness; got {}'.format(length))
-
-    try:
-        encoded = value.encode('ascii')
-    except UnicodeEncodeError as ex:
-        raise ValueError('Expects value that can be encoded in ASCII charset: {}'.format(ex))
+    encoded = str_to_bytes(value, 26)
 
     decoding = DECODING
 
@@ -296,14 +289,7 @@ def decode_timestamp(timestamp: str) -> bytes:
     :raises ValueError: when value is not 10 characters
     :raises ValueError: when value cannot be encoded in ASCII
     """
-    length = len(timestamp)
-    if length != 10:
-        raise ValueError('Expects 10 characters for timestamp; got {}'.format(length))
-
-    try:
-        encoded = timestamp.encode('ascii')
-    except UnicodeEncodeError as ex:
-        raise ValueError('Expects timestamp that can be encoded in ASCII charset: {}'.format(ex))
+    encoded = str_to_bytes(timestamp, 10)
 
     decoding = DECODING
 
@@ -334,14 +320,7 @@ def decode_randomness(randomness: str) -> bytes:
     :raises ValueError: when value is not 16 characters
     :raises ValueError: when value cannot be encoded in ASCII
     """
-    length = len(randomness)
-    if length != 16:
-        raise ValueError('Expects 16 characters for randomness; got {}'.format(length))
-
-    try:
-        encoded = randomness.encode('ascii')
-    except UnicodeEncodeError as ex:
-        raise ValueError('Expects randomness that can be encoded in ASCII charset: {}'.format(ex))
+    encoded = str_to_bytes(randomness, 16)
 
     decoding = DECODING
 
@@ -357,3 +336,34 @@ def decode_randomness(randomness: str) -> bytes:
         ((decoding[encoded[12]] << 7) | (decoding[encoded[13]] << 2) | (decoding[encoded[14]] >> 3)) & 0xFF,
         ((decoding[encoded[14]] << 5) | (decoding[encoded[15]])) & 0xFF
     ))
+
+
+def str_to_bytes(value: str, expected_length: int) -> bytes:
+    """
+    Convert the given string to bytes and validate it is within the Base32 character set.
+
+    :param value: String to convert to bytes
+    :type value: :class:`~str`
+    :param expected_length: Expected length of the input string
+    :type expected_length: :class:`~int`
+    :return: Value converted to bytes.
+    :rtype: :class:`~bytes`
+    """
+    length = len(value)
+    if length != expected_length:
+        raise ValueError('Expects {} characters for decoding; got {}'.format(expected_length, length))
+
+    try:
+        encoded = value.encode('ascii')
+    except UnicodeEncodeError as ex:
+        raise ValueError('Expects value that can be encoded in ASCII charset: {}'.format(ex))
+
+    decoding = DECODING
+
+    # Confirm all bytes are valid Base32 decode characters.
+    # Note: ASCII encoding handles the out of range checking for us.
+    for byte in encoded:
+        if decoding[byte] > 31:
+            raise ValueError('Non-base32 character found: "{}"'.format(chr(byte)))
+
+    return encoded


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 5e78e67..48e4ec1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,7 +13,9 @@ import random
 from ulid import base32
 
 
+ASCII_ALPHABET = ''.join(chr(d) for d in range(0, 128))
 EXTENDED_ASCII_ALPHABET = ''.join(chr(d) for d in range(128, 256))
+ASCII_NON_BASE_32_ALPHABET = ''.join(set(ASCII_ALPHABET).difference(set(base32.ENCODING)))
 
 
 @pytest.fixture(scope='session')
@@ -126,6 +128,14 @@ def invalid_bytes_48_80_128(request):
     return random_bytes(request.param, not_in=[6, 10, 16])
 
 
+@pytest.fixture(scope='function', params=[10, 16, 26])
+def valid_str_valid_length(request):
+    """
+    Fixture that yields :class:`~str` instances that are 10, 16, and 26 characters.
+    """
+    return random_str(request.param)
+
+
 @pytest.fixture(scope='function')
 def valid_str_26():
     """
@@ -182,6 +192,42 @@ def invalid_str_10_16_26(request):
     return random_str(request.param, not_in=[10, 16, 26])
 
 
+@pytest.fixture(scope='function', params=[10, 16, 26])
+def ascii_non_base32_str_valid_length(request):
+    """
+    Fixture that yields a :class:`~str` instance that is valid length for a ULID part but contains
+    any standard ASCII characters that are not in the Base 32 alphabet.
+    """
+    return random_str(request.param, alphabet=ASCII_NON_BASE_32_ALPHABET)
+
+
+@pytest.fixture(scope='function')
+def ascii_non_base32_str_26():
+    """
+    Fixture that yields a :class:`~str` instance that is 26 characters, the length of an entire ULID but
+    contains extended ASCII characters.
+    """
+    return random_str(26, alphabet=ASCII_NON_BASE_32_ALPHABET)
+
+
+@pytest.fixture(scope='function')
+def ascii_non_base32_str_10():
+    """
+    Fixture that yields a :class:`~str` instance that is 10 characters, the length of a ULID timestamp value but
+    contains extended ASCII characters.
+    """
+    return random_str(10, alphabet=ASCII_NON_BASE_32_ALPHABET)
+
+
+@pytest.fixture(scope='function')
+def ascii_non_base32_str_16():
+    """
+    Fixture that yields a :class:`~str` instance that is 16 characters, the length of a ULID randomness value but
+    contains extended ASCII characters.
+    """
+    return random_str(16, alphabet=ASCII_NON_BASE_32_ALPHABET)
+
+
 @pytest.fixture(scope='function', params=[10, 16, 26])
 def extended_ascii_str_valid_length(request):
     """
diff --git a/tests/test_base32.py b/tests/test_base32.py
index a922802..42cdf1f 100644
--- a/tests/test_base32.py
+++ b/tests/test_base32.py
@@ -9,6 +9,9 @@ import pytest
 from ulid import base32
 
 
+NON_BASE_32_EXP = r'^Non-base32 character found'
+
+
 @pytest.fixture(scope='session')
 def decoding_alphabet():
     """
@@ -115,7 +118,7 @@ def test_encode_randomness_raises_on_bytes_length_mismatch(invalid_bytes_80):
 
 def test_decode_handles_ulid_and_returns_16_bytes(valid_str_26):
     """
-    Assert that :func:`~ulid.base32.decode` decodes a valid 26 character string  into a :class:`~bytes`
+    Assert that :func:`~ulid.base32.decode` decodes a valid 26 character string into a :class:`~bytes`
     instance that is 128 bit.
     """
     decoded = base32.decode(valid_str_26)
@@ -125,7 +128,7 @@ def test_decode_handles_ulid_and_returns_16_bytes(valid_str_26):
 
 def test_decode_handles_timestamp_and_returns_6_bytes(valid_str_10):
     """
-    Assert that :func:`~ulid.base32.decode` decodes a valid 10 character string  into a :class:`~bytes`
+    Assert that :func:`~ulid.base32.decode` decodes a valid 10 character string into a :class:`~bytes`
     instance that is 48 bit.
     """
     decoded = base32.decode(valid_str_10)
@@ -135,7 +138,7 @@ def test_decode_handles_timestamp_and_returns_6_bytes(valid_str_10):
 
 def test_decode_handles_randomness_and_returns_10_bytes(valid_str_16):
     """
-    Assert that :func:`~ulid.base32.decode` decodes a valid 16 character string  into a :class:`~bytes`
+    Assert that :func:`~ulid.base32.decode` decodes a valid 16 character string into a :class:`~bytes`
     instance that is 80 bit.
     """
     decoded = base32.decode(valid_str_16)
@@ -161,9 +164,19 @@ def test_decode_raises_on_extended_ascii_str(extended_ascii_str_valid_length):
         base32.decode(extended_ascii_str_valid_length)
 
 
+def test_decode_raises_on_non_base32_decode_char(ascii_non_base32_str_valid_length):
+    """
+    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that includes ASCII characters not part of the Base 32 decoding character set.
+    """
+    with pytest.raises(ValueError) as ex:
+        base32.decode(ascii_non_base32_str_valid_length)
+    ex.match(NON_BASE_32_EXP)
+
+
 def test_decode_ulid_returns_16_bytes(valid_str_26):
     """
-    Assert that :func:`~ulid.base32.decode_ulid` decodes a valid 26 character string  into a :class:`~bytes`
+    Assert that :func:`~ulid.base32.decode_ulid` decodes a valid 26 character string into a :class:`~bytes`
     instance that is 128 bit.
     """
     decoded = base32.decode_ulid(valid_str_26)
@@ -189,9 +202,19 @@ def test_decode_ulid_raises_on_non_ascii_str(extended_ascii_str_26):
         base32.decode_ulid(extended_ascii_str_26)
 
 
+def test_decode_ulid_raises_on_non_base32_decode_char(ascii_non_base32_str_26):
+    """
+    Assert that :func:`~ulid.base32.decode_ulid` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that includes ASCII characters not part of the Base 32 decoding character set.
+    """
+    with pytest.raises(ValueError) as ex:
+        base32.decode_ulid(ascii_non_base32_str_26)
+    ex.match(NON_BASE_32_EXP)
+
+
 def test_decode_timestamp_returns_6_bytes(valid_str_10):
     """
-    Assert that :func:`~ulid.base32.decode_timestamp` decodes a valid 10 character string  into a :class:`~bytes`
+    Assert that :func:`~ulid.base32.decode_timestamp` decodes a valid 10 character string into a :class:`~bytes`
     instance that is 48 bit.
     """
     decoded = base32.decode_timestamp(valid_str_10)
@@ -217,9 +240,19 @@ def test_decode_timestamp_raises_on_non_ascii_str(extended_ascii_str_10):
         base32.decode_timestamp(extended_ascii_str_10)
 
 
+def test_decode_timestamp_raises_on_non_base32_decode_char(ascii_non_base32_str_10):
+    """
+    Assert that :func:`~ulid.base32.decode_timestamp` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that includes ASCII characters not part of the Base 32 decoding character set.
+    """
+    with pytest.raises(ValueError) as ex:
+        base32.decode_timestamp(ascii_non_base32_str_10)
+    ex.match(NON_BASE_32_EXP)
+
+
 def test_decode_randomness_returns_10_bytes(valid_str_16):
     """
-    Assert that :func:`~ulid.base32.decode_randomness` decodes a valid 16 character string  into a :class:`~bytes`
+    Assert that :func:`~ulid.base32.decode_randomness` decodes a valid 16 character string into a :class:`~bytes`
     instance that is 80 bit.
     """
     decoded = base32.decode_randomness(valid_str_16)
@@ -245,6 +278,16 @@ def test_decode_randomness_raises_on_non_ascii_str(extended_ascii_str_16):
         base32.decode_randomness(extended_ascii_str_16)
 
 
+def test_decode_randomness_raises_on_non_base32_decode_char(ascii_non_base32_str_16):
+    """
+    Assert that :func:`~ulid.base32.decode_randomness` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that includes ASCII characters not part of the Base 32 decoding character set.
+    """
+    with pytest.raises(ValueError) as ex:
+        base32.decode_randomness(ascii_non_base32_str_16)
+    ex.match(NON_BASE_32_EXP)
+
+
 def test_decode_table_has_value_for_entire_decoding_alphabet(decoding_alphabet):
     """
     Assert that :attr:`~ulid.base32.DECODING` stores a valid value mapping for all characters that
@@ -252,3 +295,41 @@ def test_decode_table_has_value_for_entire_decoding_alphabet(decoding_alphabet):
     """
     for char in decoding_alphabet:
         assert base32.DECODING[ord(char)] != 0xFF, 'Character "{}" decoded improperly'.format(char)
+
+
+def test_str_to_bytes_returns_expected_bytes(valid_str_valid_length):
+    """
+    Assert that :func:`~ulid.base32.str_to_bytes` decodes a valid string that is 10, 16, or 26 characters long
+    into a :class:`~bytes` instance.
+    """
+    decoded = base32.str_to_bytes(valid_str_valid_length, len(valid_str_valid_length))
+    assert isinstance(decoded, bytes)
+    assert len(decoded) == len(valid_str_valid_length)
+
+
+def test_str_to_bytes_raises_on_unexpected_length(invalid_str_26):
+    """
+    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that does not match the expected length.
+    """
+    with pytest.raises(ValueError):
+        base32.str_to_bytes(invalid_str_26, 26)
+
+
+def test_str_to_bytes_raises_on_extended_ascii_str(extended_ascii_str_valid_length):
+    """
+    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that includes extended ascii characters.
+    """
+    with pytest.raises(ValueError):
+        base32.str_to_bytes(extended_ascii_str_valid_length, len(extended_ascii_str_valid_length))
+
+
+def test_str_to_bytes_raises_on_non_base32_decode_char(ascii_non_base32_str_valid_length):
+    """
+    Assert that :func:`~ulid.base32.str_to_bytes` raises a :class:`~ValueError` when given a :class:`~str`
+    instance that includes ASCII characters not part of the Base 32 decoding character set.
+    """
+    with pytest.raises(ValueError) as ex:
+        base32.str_to_bytes(ascii_non_base32_str_valid_length, len(ascii_non_base32_str_valid_length))
+    ex.match(NON_BASE_32_EXP)