bloop/util.py
def ordered(obj):
    """
    Return sorted version of nested dicts/lists for comparing.

    Modified from:
    http://stackoverflow.com/a/25851972
    """
    if isinstance(obj, collections.abc.Mapping):
        return sorted((k, ordered(v)) for k, v in obj.items())
    # Special case str since it's a collections.abc.Iterable
    elif isinstance(obj, str):
        return obj
    elif isinstance(obj, collections.abc.Iterable):
        return sorted(ordered(x) for x in obj)
    else:
        return obj
bloop/exceptions.py
class RecordsExpired(BloopException):
    """The requested stream records are beyond the trim horizon."""
bloop/exceptions.py
class ShardIteratorExpired(BloopException):
    """The shard iterator is past its expiration date."""
bloop/types.py
class Timestamp(Integer):
    """Stores the unix (epoch) time in seconds.  Milliseconds are truncated to 0 on load and save.

    Naive datetimes (``tzinfo is None``) are not supported, and trying to use one will raise ``ValueError``.

    .. code-block:: python

        from datetime import datetime, timedelta, timezone

        class Model(Base):
            id = Column(Integer, hash_key=True)
            date = Column(Timestamp)
        engine.bind()

        obj = Model(id=1, date=datetime.now(timezone.utc))
        engine.save(obj)

        one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)

        query = engine.query(
            Model,
            key=Model.id==1,
            filter=Model.date >= one_day_ago)

        query.first().date

    .. note::

        To use common datetime libraries such as `arrow`_, `delorean`_, or `pendulum`_,
        see :ref:`DateTime and Timestamp Extensions <user-extensions-datetime>` in the user guide.  These
        are drop-in replacements and support non-utc timezones:

        .. code-block:: python

            from bloop import Timestamp  # becomes:
            from bloop.ext.pendulum import Timestamp

    .. _arrow: http://crsmithdev.com/arrow
    .. _delorean: https://delorean.readthedocs.io/en/latest/
    .. _pendulum: https://pendulum.eustace.io
    """
    python_type = datetime.datetime

    def dynamo_load(self, value, *, context, **kwargs):
        if value is None:
            return None
        value = super().dynamo_load(value, context=context, **kwargs)
        # we assume all stored values are utc, so we simply force timezone to utc
        # without changing the day/time values
        return datetime.datetime.fromtimestamp(value, tz=datetime.timezone.utc)

    def dynamo_dump(self, value, *, context, **kwargs):
        if value is None:
            return None
        if value.tzinfo is None:
            raise ValueError(
                "naive datetime instances are not supported.  You can set a timezone with either "
                "your_dt.replace(tzinfo=) or your_dt.astimezone(tz=).  WARNING: calling astimezone on a naive "
                "datetime will assume the naive datetime is in the system's timezone, even though "
                "datetime.utcnow() creates a naive object!  You almost certainly don't want to do that."
            )
        value = value.timestamp()
        return super().dynamo_dump(value, context=context, **kwargs)
bloop/types.py
class String(Type):
    python_type = str
    backing_type = STRING

    def dynamo_load(self, value, *, context, **kwargs):
        if not value:
            return ""
        return value

    def dynamo_dump(self, value, *, context, **kwargs):
        if not value:
            return None
        return value
tests/helpers/models.py
class SimpleModel(BaseModel):
    class Meta:
        table_name = "Simple"
    id = Column(String, hash_key=True)
bloop/exceptions.py
class InvalidStream(BloopException, ValueError):
    """This is not a valid stream definition."""
bloop/exceptions.py
class TransactionCanceled(BloopException):
    """The transaction was canceled.

    A WriteTransaction is canceled when:
        * A condition in one of the condition expressions is not met.
        * A table in the TransactWriteItems request is in a different account or region.
        * More than one action in the TransactWriteItems operation targets the same item.
        * There is insufficient provisioned capacity for the transaction to be completed.
        * An item size becomes too large (larger than 400 KB), or a local secondary index (LSI)
          becomes too large, or a similar validation error occurs because of changes made by the transaction.

    A ReadTransaction is canceled when:
        * There is an ongoing TransactGetItems operation that conflicts with a concurrent PutItem,
          UpdateItem, DeleteItem or TransactWriteItems request.
        * A table in the TransactGetItems request is in a different account or region.
        * There is insufficient provisioned capacity for the transaction to be completed.
        * There is a user error, such as an invalid data format.

    .. seealso::

        The API reference for `TransactionCanceledException`_

        .. _TransactionCanceledException: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html#API_TransactGetItems_Errors
    """  # noqa: E501
bloop/session.py
BATCH_GET_ITEM_CHUNK_SIZE = 100
SHARD_ITERATOR_TYPES = {
    "at_sequence": "AT_SEQUENCE_NUMBER",
    "after_sequence": "AFTER_SEQUENCE_NUMBER",
    "trim_horizon": "TRIM_HORIZON",
    "latest": "LATEST"
}
class SessionWrapper:
    def __init__(self, dynamodb=None, dynamodbstreams=None):
    def clear_cache(self):
    def save_item(self, item):
    def delete_item(self, item):
    def load_items(self, items):
    def query_items(self, request):
    def scan_items(self, request):
    def search_items(self, mode, request):
    def create_table(self, table_name, model):
    def describe_table(self, table_name):
    def validate_table(self, table_name, model):
    def enable_ttl(self, table_name, model):
    def enable_backups(self, table_name, model):
    def describe_stream(self, stream_arn, first_shard=None):
    def get_shard_iterator(self, *, stream_arn, shard_id, iterator_type, sequence_number=None):
    def get_stream_records(self, iterator_id):
    def transaction_read(self, items):
    def transaction_write(self, items, client_request_token):
def validate_search_mode(mode):
def validate_stream_iterator_type(iterator_type):
def handle_constraint_violation(error):
def handle_table_exists(error, model):
def standardize_query_response(response):
def create_batch_get_chunks(items):
def compare_tables(model, actual):
def attribute_definitions(model):
    def add_column(column):
def index_projection(index):
def is_valid_superset(actual_projection, index):
def key_schema(*, index=None, model=None):
def create_table_request(table_name, model):
def sanitize_table_description(description):
def simple_table_status(description):
def _read_field(root, default, *path):
bloop/models.py
class Column(ComparisonMixin):
    model: BaseModel
    """Represents a single attribute in DynamoDB.

    :param typedef: The type of this attribute.  Can be either a :class:`~bloop.types.Type` or
        an instance thereof.  If a type class is provided, the column will call the constructor without arguments
        to create an instance.  For example, ``Column(Integer)`` and ``Column(Integer())`` are equivalent.
    :param bool hash_key: *(Optional)* True if this is the model's hash key.
        A model must have exactly one Column with ``hash_key=True``.  Default is False.
    :param bool range_key:  *(Optional)* True if this is the model's range key.
        A model can have at most one Column with
        ``range_key=True``.  Default is False.
    :param str dynamo_name: *(Optional)* The column's name in in DynamoDB. Defaults to the index’s name in the model.
    """
    def __init__(self, typedef, hash_key=False, range_key=False, dynamo_name=None, default=missing):
        self.hash_key: bool = hash_key
        self.range_key: bool = range_key
        self._name: str = None
        self._dynamo_name: str = dynamo_name

        if not callable(default):
            self.default = lambda: default
        else:
            self.default = default

        if subclassof(typedef, Type):
            typedef = typedef()
        if instanceof(typedef, Type):
            self.typedef = typedef

        else:
            raise TypeError(f"Expected {typedef} to be instance or subclass of Type")
        super().__init__()

    def __copy__(self):
        """
        Create a shallow copy of this Column.  Primarily used when initializing models that subclass other abstract
        models or mixins (baseless classes that contain Columns and Indexes).  You can override this method to
        change how derived models are created:

        .. code-block:: python

            import copy


            class MyColumn(Column):
                def __copy__(self):
                    new = super().__copy__()
                    new.derived = True
                    return new


            column = MyColumn(Integer)
            same = copy.copy(column)
            assert same.derived  # True

        :return: A shallow copy of this Column, with the ``model`` and ``_name`` attributes unset.
        """
        cls = self.__class__
        obj = cls.__new__(cls)
        obj.__dict__.update(self.__dict__)
        obj.model = None
        obj._name = None
        return obj

    def __set_name__(self, owner, name):
        self._name = name

    __hash__ = object.__hash__

    def __set__(self, obj, value):
        if self._name is None:
            raise AttributeError("Can't set field without binding to model")
        obj.__dict__[self._name] = value
        # Notify the tracking engine that this value was intentionally mutated
        object_modified.send(self, obj=obj, column=self, value=value)

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        if self._name is None:
            raise AttributeError("Can't get field without binding to model")
        try:
            return obj.__dict__[self._name]
        except KeyError:
            raise AttributeError(f"'{obj.__class__}' has no attribute '{self._name}'")

    def __delete__(self, obj):
        try:
            if self._name is None:
                raise AttributeError("Can't delete field without binding to model")
            try:
                del obj.__dict__[self._name]
            except KeyError:
                raise AttributeError(f"'{obj.__class__}' has no attribute '{self._name}'")
        finally:
            # Unlike set, we always want to mark on delete.  If we didn't, and the column wasn't loaded
            # (say from a query) then the intention "ensure this doesn't have a value" wouldn't be captured.
            object_modified.send(self, obj=obj, column=self, value=None)

    def __repr__(self):
        if self.hash_key:
            extra = "=hash"
        elif self.range_key:
            extra = "=range"
        else:
            extra = ""

        # <Column[Pin.url]>
        # <Column[User.id=hash]>
        # <Column[File.fragment=range]>
        return f"<{self.__class__.__name__}[{self.model.__name__}.{self.name}{extra}]>"

    @property
    def name(self):
        """Name of the model's attr that references self"""
        return self._name

    @property
    def dynamo_name(self):
        if self._dynamo_name is None:
            return self.name
        return self._dynamo_name
bloop/models.py
class LocalSecondaryIndex(Index):
    """See `LocalSecondaryIndex`_ in the DynamoDB Developer Guide for details.

    Unlike :class:`~bloop.models.GlobalSecondaryIndex` each LSI shares its throughput with the table
    and their hash key is always the table hash key.

    :param projection: Either "keys", "all", or a list of column name or objects.
        Included columns will be projected into the index.  Key columns are always included.
    :param range_key: The column that the index can be sorted against.
    :param str dynamo_name: *(Optional)* The index's name in in DynamoDB. Defaults to the index’s name in the model.
    :param bool strict: *(Optional)* Restricts queries and scans on the LSI to columns in the projection.
        When False, DynamoDB may silently incur additional reads to load results.  You should not disable this
        unless you have an explicit need.  Default is True.

    .. _LocalSecondaryIndex: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LSI.html
    """
    def __init__(self, *, projection, range_key, dynamo_name=None, strict=True, **kwargs):
        # Hash key MUST be the table hash; do not specify
        if "hash_key" in kwargs:
            raise InvalidModel("An LSI shares its hash key with the Model.")
        if ("write_units" in kwargs) or ("read_units" in kwargs):
            raise InvalidModel("An LSI shares its provisioned throughput with the Model.")
        super().__init__(range_key=range_key, dynamo_name=dynamo_name, projection=projection, **kwargs)
        self.projection["strict"] = strict

    @property
    def hash_key(self):
        return self.model.Meta.hash_key

    @property
    def read_units(self):
        return self.model.Meta.read_units

    @read_units.setter
    def read_units(self, value):
        self.model.Meta.read_units = value

    @property
    def write_units(self):
        return self.model.Meta.write_units

    @write_units.setter
    def write_units(self, value):
        self.model.Meta.write_units = value
bloop/util.py
class Sentinel:
    """Simple string-based placeholders for missing or special values.

    Names are unique, and instances are re-used for the same name:

    .. code-block:: pycon

        >>> from bloop.util import Sentinel
        >>> empty = Sentinel("empty")
        >>> empty
        <Sentinel[empty]>
        >>> same_token = Sentinel("empty")
        >>> empty is same_token
        True

    This removes the need to import the same signal or placeholder value everywhere; two modules can create
    ``Sentinel("some-value")`` and refer to the same object.  This is especially helpful where ``None`` is a possible
    value, and so can't be used to indicate omission of an optional parameter.

    Implements ``__repr__`` to render nicely in function signatures.  Standard object-based sentinels:

    .. code-block:: pycon

        >>> missing = object()
        >>> def some_func(optional=missing):
        ...     pass
        ...
        >>> help(some_func)
        Help on function some_func in module __main__:

        some_func(optional=<object object at 0x7f0f3f29e5d0>)

    With the Sentinel class:

    .. code-block:: pycon

        >>> from bloop.util import Sentinel
        >>> missing = Sentinel("Missing")
        >>> def some_func(optional=missing):
        ...     pass
        ...
        >>> help(some_func)
        Help on function some_func in module __main__:

        some_func(optional=<Sentinel[Missing]>)

    :param str name: The name for this sentinel.
    """
    def __new__(cls, name, *args, **kwargs):
        name = name.lower()
        sentinel = _symbols.get(name, None)
        if sentinel is None:
            sentinel = _symbols[name] = super().__new__(cls)
        return sentinel

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return "<Sentinel[{}]>".format(self.name)
tests/helpers/models.py
class User(BaseModel):
    id = Column(String, hash_key=True)
    age = Column(Integer)
    name = Column(String)
    email = Column(String)
    joined = Column(DateTime, dynamo_name="j")
    by_email = GlobalSecondaryIndex(hash_key="email", projection="all")
tests/helpers/models.py
class ComplexModel(BaseModel):
    class Meta:
        write_units = 2
        read_units = 3
        table_name = "CustomTableName"

    name = Column(UUID, hash_key=True)
    date = Column(String, range_key=True)
    email = Column(String)
    joined = Column(String)
    not_projected = Column(Integer)
    by_email = GlobalSecondaryIndex(hash_key="email", read_units=4, projection="all", write_units=5)
    by_joined = LocalSecondaryIndex(range_key="joined", projection=["email"])
bloop/models.py
class BaseModel:
    """Abstract base that all models derive from.

    Provides a basic ``__init__`` method that takes ``**kwargs`` whose
    keys are columns names:

    .. code-block:: python

        class URL(BaseModel):
            id = Column(UUID, hash_key=True)
            ip = Column(IPv6)
            name = Column(String)

        url = URL(id=uuid.uuid4(), name="google")

    By default, the ``__init__`` method is not called when new instances are
    required, for example when iterating results from Query, Scan or a Stream.

    """
    class Meta(IMeta):
        abstract = True

    def __init__(self, **attrs):
        # Only set values from **attrs if there's a
        # corresponding `name` for a column in the model
        for column in self.Meta.columns:
            value = attrs.get(column.name, missing)
            if value is missing:
                value = column.default()
            if value is not missing:
                setattr(self, column.name, value)

    def __init_subclass__(cls: type, **kwargs):
        ensure_hash(cls)
        meta = initialize_meta(cls)

        # before we start binding, we should ensure that no combination of parent classes
        # will cause conflicts.  For example:
        #   class C(A, B) where
        #       A has a column named "foo" and dynamo_name "ddb"
        #       B has a column named "bar" and dynamo_name "ddb"
        # both A and B are valid mixins, but C must fail because there isn't a 1:1 binding to the "ddb" column.
        #
        # TODO | for now, we'll assume that the class being defined is special, and can replace columns with the
        # TODO | same dynamo_name.  In the example above, that would mean C has a column named "baz" and dynamo_name
        # TODO | "ddb" which would prevent the parent columns "foo" and "bar" from binding to the child class.

        modeled_attrs = set((attr for (_, attr) in inspect.getmembers(cls, lambda x: isinstance(x, (Column, Index)))))
        local_attrs = {
            name: value
            for name, value in cls.__dict__.items()
            if isinstance(value, (Column, Index))
        }
        derived_attrs = modeled_attrs - set(local_attrs.values())

        # 0.0 Pre-validation for collisions in derived columns/indexes
        dynamo_names = [x.dynamo_name for x in derived_attrs]
        collisions = [name for name, count in collections.Counter(dynamo_names).items() if count > 1]
        if collisions:
            collisions.sort()
            raise InvalidModel(
                f"The model {cls.__name__} subclasses one or more models with conflicting "
                f"column or index definitions for the following values of dynamo_name: {collisions}")
        derived_hash_keys = set((x.name for x in derived_attrs if isinstance(x, Column) and x.hash_key))
        if len(derived_hash_keys) > 1:
            derived_hash_keys = sorted(derived_hash_keys)
            raise InvalidModel(
                f"The model {cls.__name__} subclasses one or more models that declare multiple "
                f"columns as the hash key: {derived_hash_keys}")
        derived_range_keys = set((x.name for x in derived_attrs if isinstance(x, Column) and x.range_key))
        if len(derived_range_keys) > 1:
            derived_range_keys = sorted(derived_range_keys)
            raise InvalidModel(
                f"The model {cls.__name__} subclasses one or more models that declare multiple "
                f"columns as the range key: {derived_range_keys}")

        # 0.1 Pre-validation for collisions in local columns/indexes
        dynamo_names = [x.dynamo_name for x in local_attrs.values()]
        collisions = [name for name, count in collections.Counter(dynamo_names).items() if count > 1]
        if collisions:
            collisions.sort()
            raise InvalidModel(
                f"The model {cls.__name__} contains conflicting column or index definitions for the "
                f"following values of dynamo_name: {collisions}")
        local_hash_keys = [x.name for x in local_attrs.values() if isinstance(x, Column) and x.hash_key]
        if len(local_hash_keys) > 1:
            local_hash_keys = sorted(local_hash_keys)
            raise InvalidModel(
                f"The model {cls.__name__} defines multiple columns as hash columns: {local_hash_keys}")
        local_range_keys = [x.name for x in local_attrs.values() if isinstance(x, Column) and x.range_key]
        if len(local_range_keys) > 1:
            local_range_keys = sorted(local_range_keys)
            raise InvalidModel(
                f"The model {cls.__name__} defines multiple columns as range columns: {local_range_keys}")

        # 1.0 Bind derived columns so they can be referenced by derived indexes
        for attr in derived_attrs:
            if isinstance(attr, Column):
                bind_column(cls, attr.name, attr, copy=True)

        # 1.1 Bind derived indexes
        for attr in derived_attrs:
            if isinstance(attr, Index):
                bind_index(cls, attr.name, attr, copy=True)

        # 1.2 Bind local columns, allowing them to overwrite existing columns
        for name, attr in local_attrs.items():
            if isinstance(attr, Column):
                bind_column(cls, name, attr, force=True)

        # 1.3 Bind local indexes, allowing them to overwrite existing indexes
        for name, attr in local_attrs.items():
            if isinstance(attr, Index):
                bind_index(cls, name, attr, force=True)

        # 2.0 Ensure concrete models are valid
        # Currently, this just checks that a hash key is defined
        if not meta.abstract and not meta.hash_key:
            raise InvalidModel(f"{meta.model.__name__!r} has no hash key.")

        validate_stream(meta)
        validate_ttl(meta)
        validate_encryption(meta)
        validate_backups(meta)
        validate_billing(meta)

        # 3.0 Fire model_created for customizing the class after creation
        model_created.send(None, model=cls)

    def __repr__(self):
        attrs = ", ".join("{}={!r}".format(*item) for item in loaded_columns(self))
        return f"{self.__class__.__name__}({attrs})"
bloop/exceptions.py
class TableMismatch(BloopException):
    """The expected and actual tables for this Model do not match."""
bloop/exceptions.py
class InvalidShardIterator(BloopException, ValueError):
    """This is not a valid shard iterator."""
bloop/exceptions.py
class ConstraintViolation(BloopException):
    """A required condition was not met."""
bloop/models.py
class GlobalSecondaryIndex(Index):
    """See `GlobalSecondaryIndex`_ in the DynamoDB Developer Guide for details.

    :param projection: Either "keys", "all", or a list of column name or objects.
        Included columns will be projected into the index.  Key columns are always included.
    :param hash_key: The column that the index can be queried against.
    :param range_key: *(Optional)* The column that the index can be sorted on.  Default is None.
    :param int read_units: *(Optional)* Provisioned read units for the index.  Default is None.
        When no value is provided and the index does not exist, it will be created with 1 read unit.  If the index
        already exists, it will use the actual index's read units.
    :param int write_units:  *(Optional)* Provisioned write units for the index.  Default is None.
        When no value is provided and the index does not exist, it will be created with 1 write unit.  If the index
        already exists, it will use the actual index's write units.
    :param str dynamo_name: *(Optional)* The index's name in in DynamoDB. Defaults to the index’s name in the model.

    .. _GlobalSecondaryIndex: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.html
    """
    def __init__(
            self, *, projection,
            hash_key, range_key=None,
            read_units=None, write_units=None,
            dynamo_name=None, **kwargs):
        super().__init__(
            hash_key=hash_key, range_key=range_key,
            dynamo_name=dynamo_name, projection=projection, **kwargs)
        self.write_units = write_units
        self.read_units = read_units
bloop/exceptions.py
class InvalidSearch(BloopException, ValueError):
    """The search was malformed"""
bloop/exceptions.py
class BloopException(Exception):
    """An unexpected exception occurred."""


Question: import logging
import botocore.exceptions
import pytest
from unittest.mock import Mock
from bloop.exceptions import (
    BloopException,
    ConstraintViolation,
    InvalidSearch,
    InvalidShardIterator,
    InvalidStream,
    RecordsExpired,
    ShardIteratorExpired,
    TableMismatch,
    TransactionCanceled,
)
from bloop.models import (
    BaseModel,
    Column,
    GlobalSecondaryIndex,
    LocalSecondaryIndex,
)
from bloop.session import (
    BATCH_GET_ITEM_CHUNK_SIZE,
    SessionWrapper,
    compare_tables,
    create_table_request,
    ready,
    sanitize_table_description,
    simple_table_status,
)
from bloop.types import String, Timestamp
from bloop.util import Sentinel, ordered
from ..helpers.models import ComplexModel, SimpleModel, User





missing = Sentinel("missing")


@pytest.fixture
def dynamodb():
    # No spec since clients are generated dynamically.
    # We could use botocore.client.BaseClient but it's so generic
    # that we don't gain any useful inspections
    return Mock()


@pytest.fixture
def dynamodbstreams():
    # No spec since clients are generated dynamically.
    return Mock()


@pytest.fixture
def session(dynamodb, dynamodbstreams):
    return SessionWrapper(dynamodb=dynamodb, dynamodbstreams=dynamodbstreams)


@pytest.fixture
def model():
    """Return a clean model so each test can mutate the model's Meta"""
    class MyModel(BaseModel):
        class Meta:
            backups = {"enabled": True}
            billing = {"mode": "provisioned"}
            encryption = {"enabled": True}
            stream = {"include": {"old", "new"}}
            ttl = {"column": "expiry"}
            read_units = 3
            write_units = 7

        id = Column(String, hash_key=True)
        range = Column(String, range_key=True)

Answer:         expiry = Column(Timestamp)