bloop/models.py
def unpack_from_dynamodb(*, attrs, expected, model=None, obj=None, engine=None, context=None, **kwargs):
    """Push values by dynamo_name into an object"""
    context = util.default_context(engine, context)
    engine = context["engine"]
    if not engine:
        raise ValueError("You must provide engine or a context with an engine.")
    if model is None and obj is None:
        raise ValueError("You must provide a model or obj to unpack.")
    if model is not None and obj is not None:
        raise ValueError("Only specify model or obj.")
    if model:
        obj = model.Meta.init()

    for column in expected:
        value = attrs.get(column.dynamo_name, None)
        # noinspection PyProtectedMember
        value = column.typedef._load(value, context=context, **kwargs)
        setattr(obj, column.name, value)
    return obj
bloop/transactions.py
class ReadTransaction(Transaction):
    """
    Loads all items in the same transaction.  Items can be from different models and tables.
    """
    mode = "r"

    def load(self, *objs) -> "ReadTransaction":
        """
        Add one or more objects to be loaded in this transaction.

        At most 10 items can be loaded in the same transaction.  All objects will be loaded each time you
        call commit().


        :param objs: Objects to add to the set that are loaded in this transaction.
        :return: this transaction for chaining
        :raises bloop.exceptions.MissingObjects: if one or more objects aren't loaded.
        """
        self._extend([TxItem.new("get", obj) for obj in objs])
        return self
bloop/transactions.py
class WriteTransaction(Transaction):
    """
    Applies all updates in the same transaction.  Items can be from different models and tables.

    As with an engine, you can apply conditions to each object that you save or delete, or a condition for the entire
    transaction that won't modify the specified object:

    .. code-block:: python

        # condition on some_obj
        >>> tx.save(some_obj, condition=SomeModel.name.begins_with("foo"))
        # condition on the tx, based on the values of some_other_obj
        >>> tx.check(some_other_obj, condition=ThatModel.capacity >= 100)

    """
    mode = "w"

    def check(self, obj, condition) -> "WriteTransaction":
        """
        Add a condition which must be met for the transaction to commit.

        While the condition is checked against the provided object, that object will not be modified.  It is only
        used to provide the hash and range key to apply the condition to.

        At most 10 items can be checked, saved, or deleted in the same transaction.  The same idempotency token will
        be used for a single prepared transaction, which allows you to safely call commit on the PreparedCommit object
        multiple times.


        :param obj: The object to use for the transaction condition.  This object will not be modified.
        :param condition: A condition on an object which must hold for the transaction to commit.
        :return: this transaction for chaining
        """
        self._extend([TxItem.new("check", obj, condition)])
        return self

    def save(self, *objs, condition=None) -> "WriteTransaction":
        """
        Add one or more objects to be saved in this transaction.

        At most 10 items can be checked, saved, or deleted in the same transaction.  The same idempotency token will
        be used for a single prepared transaction, which allows you to safely call commit on the PreparedCommit object
        multiple times.

        :param objs: Objects to add to the set that are updated in this transaction.
        :param condition: A condition for these objects which must hold for the transaction to commit.
        :return: this transaction for chaining
        """
        self._extend([TxItem.new("save", obj, condition) for obj in objs])
        return self

    def delete(self, *objs, condition=None) -> "WriteTransaction":
        """
        Add one or more objects to be deleted in this transaction.

        At most 10 items can be checked, saved, or deleted in the same transaction.  The same idempotency token will
        be used for a single prepared transaction, which allows you to safely call commit on the PreparedCommit object
        multiple times.

        :param objs: Objects to add to the set that are deleted in this transaction.
        :param condition: A condition for these objects which must hold for the transaction to commit.
        :return: this transaction for chaining
        """
        self._extend([TxItem.new("delete", obj, condition) for obj in objs])
        return self
bloop/exceptions.py
class MissingObjects(BloopException):
    """Some objects were not found."""

    #: The objects that failed to load
    objects: list

    def __init__(self, *args, objects=None):
        super().__init__(*args)
        self.objects = list(objects) if objects else []
bloop/search.py
class Search:
    """A user-created search object.

    Used to prepare a :class:`~bloop.search.PreparedSearch` which build search iterators.

    :param str mode: Search type, either "query" or "scan".
    :param engine: :class:`~bloop.engine.Engine` to unpack models with.
    :param model: :class:`~bloop.models.BaseModel` being searched.
    :param index: :class:`~bloop.models.Index` to search, or None.
    :param key: *(Query only)* Key condition.  This must include an equality against the hash key,
        and optionally one of a restricted set of conditions on the range key.
    :param filter: Filter condition.  Only matching objects will be included in the results.
    :param projection: "all", "count", a set of column names, or a list of :class:`~bloop.models.Column`.
        When projection is "count", you must advance the iterator to retrieve the count.
    :param bool consistent: Use `strongly consistent reads`__ if True.  Not applicable to GSIs.  Default is False.
    :param bool forward: *(Query only)* Use ascending or descending order.  Default is True (ascending).
    :param tuple parallel: *(Scan only)* A tuple of (Segment, TotalSegments) for this portion of a `parallel scan`__.
            Default is None.

    __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadConsistency.html
    __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/QueryAndScan.html#QueryAndScanParallelScan
    """

    def __init__(
            self, mode=None, engine=None, model=None, index=None, key=None, filter=None,
            projection=None, consistent=False, forward=True, parallel=None):
        self.mode = mode
        self.engine = engine
        self.model = model
        self.index = index
        self.key = key
        self.filter = filter
        self.projection = projection
        self.consistent = consistent
        self.forward = forward
        self.parallel = parallel

    def __repr__(self):
        return search_repr(self.__class__, self.model, self.index)

    def prepare(self):
        """Constructs a :class:`~bloop.search.PreparedSearch`."""
        p = PreparedSearch()
        p.prepare(
            engine=self.engine,
            mode=self.mode,
            model=self.model,
            index=self.index,
            key=self.key,
            filter=self.filter,
            projection=self.projection,
            consistent=self.consistent,
            forward=self.forward,
            parallel=self.parallel
        )
        return p
bloop/session.py
class SessionWrapper:
    """Provides a consistent interface to DynamoDb and DynamoDbStreams clients.

    If either client is None, that client is built using :func:`boto3.client`.

    :param dynamodb: A boto3 client for DynamoDB.  Defaults to ``boto3.client("dynamodb")``.
    :param dynamodbstreams: A boto3 client for DynamoDbStreams.  Defaults to ``boto3.client("dynamodbstreams")``.
    """
    def __init__(self, dynamodb=None, dynamodbstreams=None):
        dynamodb = dynamodb or boto3.client("dynamodb")
        dynamodbstreams = dynamodbstreams or boto3.client("dynamodbstreams")

        self._tables = {}
        self.dynamodb_client = dynamodb
        self.stream_client = dynamodbstreams

    def clear_cache(self):
        """Clear all cached table descriptions."""
        self._tables.clear()

    def save_item(self, item):
        """Save an object to DynamoDB.

        Returns Optional[dict] of read attributes depending on the "ReturnValues" kwarg.
        Return value is None when no attributes were requested.

        :param item: Unpacked into kwargs for :func:`boto3.DynamoDB.Client.update_item`.
        :raises bloop.exceptions.ConstraintViolation: if the condition (or atomic) is not met.
        """
        try:
            resp = self.dynamodb_client.update_item(**item)
            return resp.get("Attributes", None)
        except botocore.exceptions.ClientError as error:
            handle_constraint_violation(error)

    def delete_item(self, item):
        """Delete an object in DynamoDB.

        Returns Optional[dict] of read attributes depending on the "ReturnValues" kwarg.
        Return value is None when no attributes were requested.

        :param item: Unpacked into kwargs for :func:`boto3.DynamoDB.Client.delete_item`.
        :raises bloop.exceptions.ConstraintViolation: if the condition (or atomic) is not met.
        """
        try:
            resp = self.dynamodb_client.delete_item(**item)
            return resp.get("Attributes", None)
        except botocore.exceptions.ClientError as error:
            handle_constraint_violation(error)

    def load_items(self, items):
        """Loads any number of items in chunks, handling continuation tokens.

        :param items: Unpacked in chunks into "RequestItems" for :func:`boto3.DynamoDB.Client.batch_get_item`.
        """
        loaded_items = {}
        requests = collections.deque(create_batch_get_chunks(items))
        while requests:
            request = requests.pop()
            try:
                response = self.dynamodb_client.batch_get_item(RequestItems=request)
            except botocore.exceptions.ClientError as error:
                raise BloopException("Unexpected error while loading items.") from error

            # Accumulate results
            for table_name, table_items in response.get("Responses", {}).items():
                loaded_items.setdefault(table_name, []).extend(table_items)

            # Push additional request onto the deque.
            # "UnprocessedKeys" is {} if this request is done
            if response["UnprocessedKeys"]:
                requests.append(response["UnprocessedKeys"])
        return loaded_items

    def query_items(self, request):
        """Wraps :func:`boto3.DynamoDB.Client.query`.

        Response always includes "Count" and "ScannedCount"

        :param request: Unpacked into :func:`boto3.DynamoDB.Client.query`
        """
        return self.search_items("query", request)

    def scan_items(self, request):
        """Wraps :func:`boto3.DynamoDB.Client.scan`.

        Response always includes "Count" and "ScannedCount"

        :param request: Unpacked into :func:`boto3.DynamoDB.Client.scan`
        """
        return self.search_items("scan", request)

    def search_items(self, mode, request):
        """Invoke query/scan by name.

        Response always includes "Count" and "ScannedCount"

        :param str mode: "query" or "scan"
        :param request: Unpacked into :func:`boto3.DynamoDB.Client.query` or :func:`boto3.DynamoDB.Client.scan`
        """
        validate_search_mode(mode)
        method = getattr(self.dynamodb_client, mode)
        try:
            response = method(**request)
        except botocore.exceptions.ClientError as error:
            raise BloopException("Unexpected error during {}.".format(mode)) from error
        standardize_query_response(response)
        return response

    def create_table(self, table_name, model):
        """Create the model's table.  Returns True if the table is being created, False otherwise.

        Does not wait for the table to create, and does not validate an existing table.
        Will not raise "ResourceInUseException" if the table exists or is being created.

        :param str table_name: The name of the table to create for the model.
        :param model: The :class:`~bloop.models.BaseModel` to create the table for.
        :return: True if the table is being created, False if the table exists
        :rtype: bool
        """
        table = create_table_request(table_name, model)
        try:
            self.dynamodb_client.create_table(**table)
            is_creating = True
        except botocore.exceptions.ClientError as error:
            handle_table_exists(error, model)
            is_creating = False
        return is_creating

    def describe_table(self, table_name):
        """
        Polls until the table is ready, then returns the first result when the table was ready.

        The returned dict is standardized to ensure all fields are present, even when empty or across different
        DynamoDB API versions.
        TTL information is also inserted.

        :param table_name: The name of the table to describe
        :return: The (sanitized) result of DescribeTable["Table"]
        :rtype: dict
        """
        if table_name in self._tables:
            return self._tables[table_name]
        status, description = None, {}
        calls = 0
        while status is not ready:
            calls += 1
            try:
                description = self.dynamodb_client.describe_table(TableName=table_name)["Table"]
            except botocore.exceptions.ClientError as error:
                raise BloopException("Unexpected error while describing table.") from error
            status = simple_table_status(description)
        logger.debug("describe_table: table \"{}\" was in ACTIVE state after {} calls".format(table_name, calls))
        try:
            ttl = self.dynamodb_client.describe_time_to_live(TableName=table_name)
        except botocore.exceptions.ClientError as error:
            raise BloopException("Unexpected error while describing ttl.") from error
        try:
            backups = self.dynamodb_client.describe_continuous_backups(TableName=table_name)
        except botocore.exceptions.ClientError as error:
            raise BloopException("Unexpected error while describing continuous backups.") from error

        description["TimeToLiveDescription"] = {
            "AttributeName": _read_field(ttl, None, "TimeToLiveDescription", "AttributeName"),
            "TimeToLiveStatus": _read_field(ttl, None, "TimeToLiveDescription", "TimeToLiveStatus"),
        }
        description["ContinuousBackupsDescription"] = {
            "ContinuousBackupsStatus": _read_field(
                backups, None, "ContinuousBackupsDescription", "ContinuousBackupsStatus"),
        }

        table = self._tables[table_name] = sanitize_table_description(description)
        return table

    def validate_table(self, table_name, model):
        """Polls until a creating table is ready, then verifies the description against the model's requirements.

        The model may have a subset of all GSIs and LSIs on the table, but the key structure must be exactly
        the same.  The table must have a stream if the model expects one, but not the other way around.  When read or
        write units are not specified for the model or any GSI, the existing values will always pass validation.

        :param str table_name: The name of the table to validate the model against.
        :param model: The :class:`~bloop.models.BaseModel` to validate the table of.
        :raises bloop.exceptions.TableMismatch: When the table does not meet the constraints of the model.
        """
        actual = self.describe_table(table_name)
        if not compare_tables(model, actual):
            raise TableMismatch("The expected and actual tables for {!r} do not match.".format(model.__name__))

        # Fill in values that Meta doesn't know ahead of time (such as arns).
        # These won't be populated unless Meta explicitly cares about the value
        if model.Meta.stream:
            stream_arn = model.Meta.stream["arn"] = actual["LatestStreamArn"]
            logger.debug(f"Set {model.__name__}.Meta.stream['arn'] to '{stream_arn}' from DescribeTable response")
        if model.Meta.ttl:
            ttl_enabled = actual["TimeToLiveDescription"]["TimeToLiveStatus"].lower() == "enabled"
            model.Meta.ttl["enabled"] = ttl_enabled
            logger.debug(f"Set {model.__name__}.Meta.ttl['enabled'] to '{ttl_enabled}' from DescribeTable response")

        # Fill in meta values that the table didn't care about (eg. billing=None)
        if model.Meta.encryption is None:
            sse_enabled = actual["SSEDescription"]["Status"].lower() == "enabled"
            model.Meta.encryption = {"enabled": sse_enabled}
            logger.debug(
                f"Set {model.__name__}.Meta.encryption['enabled'] to '{sse_enabled}' from DescribeTable response")
        if model.Meta.backups is None:
            backups = actual["ContinuousBackupsDescription"]["ContinuousBackupsStatus"] == "ENABLED"
            model.Meta.backups = {"enabled": backups}
            logger.debug(f"Set {model.__name__}.Meta.backups['enabled'] to '{backups}' from DescribeTable response")
        if model.Meta.billing is None:
            billing_mode = {
                "PAY_PER_REQUEST": "on_demand",
                "PROVISIONED": "provisioned"
            }[actual["BillingModeSummary"]["BillingMode"]]
            model.Meta.billing = {"mode": billing_mode}
            logger.debug(f"Set {model.__name__}.Meta.billing['mode'] to '{billing_mode}' from DescribeTable response")
        if model.Meta.read_units is None:
            read_units = model.Meta.read_units = actual["ProvisionedThroughput"]["ReadCapacityUnits"]
            logger.debug(
                f"Set {model.__name__}.Meta.read_units to {read_units} from DescribeTable response")
        if model.Meta.write_units is None:
            write_units = model.Meta.write_units = actual["ProvisionedThroughput"]["WriteCapacityUnits"]
            logger.debug(
                f"Set {model.__name__}.Meta.write_units to {write_units} from DescribeTable response")

        # Replace any ``None`` values for read_units, write_units in GSIs with their actual values
        gsis = {index["IndexName"]: index for index in actual["GlobalSecondaryIndexes"]}
        for index in model.Meta.gsis:
            read_units = gsis[index.dynamo_name]["ProvisionedThroughput"]["ReadCapacityUnits"]
            write_units = gsis[index.dynamo_name]["ProvisionedThroughput"]["WriteCapacityUnits"]
            if index.read_units is None:
                index.read_units = read_units
                logger.debug(
                    f"Set {model.__name__}.{index.name}.read_units to {read_units} from DescribeTable response")
            if index.write_units is None:
                index.write_units = write_units
                logger.debug(
                    f"Set {model.__name__}.{index.name}.write_units to {write_units} from DescribeTable response")

    def enable_ttl(self, table_name, model):
        """Calls UpdateTimeToLive on the table according to model.Meta["ttl"]

        :param table_name: The name of the table to enable the TTL setting on
        :param model: The model to get TTL settings from
        """
        self._tables.pop(table_name, None)
        ttl_name = model.Meta.ttl["column"].dynamo_name
        request = {
            "TableName": table_name,
            "TimeToLiveSpecification": {"AttributeName": ttl_name, "Enabled": True}
        }
        try:
            self.dynamodb_client.update_time_to_live(**request)
        except botocore.exceptions.ClientError as error:
            raise BloopException("Unexpected error while setting TTL.") from error

    def enable_backups(self, table_name, model):
        """Calls UpdateContinuousBackups on the table according to model.Meta["continuous_backups"]

        :param table_name: The name of the table to enable Continuous Backups on
        :param model: The model to get Continuous Backups settings from
        """
        self._tables.pop(table_name, None)
        request = {
            "TableName": table_name,
            "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": True}
        }
        try:
            self.dynamodb_client.update_continuous_backups(**request)
        except botocore.exceptions.ClientError as error:
            raise BloopException("Unexpected error while setting Continuous Backups.") from error

    def describe_stream(self, stream_arn, first_shard=None):
        """Wraps :func:`boto3.DynamoDBStreams.Client.describe_stream`, handling continuation tokens.

        :param str stream_arn: Stream arn, usually from the model's ``Meta.stream["arn"]``.
        :param str first_shard: *(Optional)* If provided, only shards after this shard id will be returned.
        :return: All shards in the stream, or a subset if ``first_shard`` is provided.
        :rtype: dict
        """
        description = {"Shards": []}

        request = {"StreamArn": stream_arn, "ExclusiveStartShardId": first_shard}
        # boto3 isn't down with literal Nones.
        if first_shard is None:
            request.pop("ExclusiveStartShardId")

        while request.get("ExclusiveStartShardId") is not missing:
            try:
                response = self.stream_client.describe_stream(**request)["StreamDescription"]
            except botocore.exceptions.ClientError as error:
                if error.response["Error"]["Code"] == "ResourceNotFoundException":
                    raise InvalidStream(f"The stream arn {stream_arn!r} does not exist.") from error
                raise BloopException("Unexpected error while describing stream.") from error
            # Docs aren't clear if the terminal value is null, or won't exist.
            # Since we don't terminate the loop on None, the "or missing" here
            # will ensure we stop on a falsey value.
            request["ExclusiveStartShardId"] = response.pop("LastEvaluatedShardId", None) or missing
            description["Shards"].extend(response.pop("Shards", []))
            description.update(response)
        return description

    def get_shard_iterator(self, *, stream_arn, shard_id, iterator_type, sequence_number=None):
        """Wraps :func:`boto3.DynamoDBStreams.Client.get_shard_iterator`.

        :param str stream_arn: Stream arn.  Usually :data:`Shard.stream_arn <bloop.stream.shard.Shard.stream_arn>`.
        :param str shard_id: Shard identifier.  Usually :data:`Shard.shard_id <bloop.stream.shard.Shard.shard_id>`.
        :param str iterator_type: "sequence_at", "sequence_after", "trim_horizon", or "latest"
        :param sequence_number:
        :return: Iterator id, valid for 15 minutes.
        :rtype: str
        :raises bloop.exceptions.RecordsExpired: Tried to get an iterator beyond the Trim Horizon.
        """
        real_iterator_type = validate_stream_iterator_type(iterator_type)
        request = {
            "StreamArn": stream_arn,
            "ShardId": shard_id,
            "ShardIteratorType": real_iterator_type,
            "SequenceNumber": sequence_number
        }
        # boto3 isn't down with literal Nones.
        if sequence_number is None:
            request.pop("SequenceNumber")
        try:
            return self.stream_client.get_shard_iterator(**request)["ShardIterator"]
        except botocore.exceptions.ClientError as error:
            if error.response["Error"]["Code"] == "TrimmedDataAccessException":
                raise RecordsExpired from error
            raise BloopException("Unexpected error while creating shard iterator") from error

    def get_stream_records(self, iterator_id):
        """Wraps :func:`boto3.DynamoDBStreams.Client.get_records`.

        :param iterator_id: Iterator id.  Usually :data:`Shard.iterator_id <bloop.stream.shard.Shard.iterator_id>`.
        :return: Dict with "Records" list (may be empty) and "NextShardIterator" str (may not exist).
        :rtype: dict
        :raises bloop.exceptions.RecordsExpired: The iterator moved beyond the Trim Horizon since it was created.
        :raises bloop.exceptions.ShardIteratorExpired: The iterator was created more than 15 minutes ago.
        """
        try:
            return self.stream_client.get_records(ShardIterator=iterator_id)
        except botocore.exceptions.ClientError as error:
            if error.response["Error"]["Code"] == "TrimmedDataAccessException":
                raise RecordsExpired from error
            elif error.response["Error"]["Code"] == "ExpiredIteratorException":
                raise ShardIteratorExpired from error
            raise BloopException("Unexpected error while getting records.") from error

    def transaction_read(self, items):
        """
        Wraps :func:`boto3.DynamoDB.Client.db.transact_get_items`.

        :param items: Unpacked into "TransactionItems" for :func:`boto3.DynamoDB.Client.transact_get_items`
        :raises bloop.exceptions.TransactionCanceled: if the transaction was canceled.
        :return: Dict with "Records" list
        """
        try:
            return self.dynamodb_client.transact_get_items(TransactItems=items)
        except botocore.exceptions.ClientError as error:
            if error.response["Error"]["Code"] == "TransactionCanceledException":
                raise TransactionCanceled from error
            raise BloopException("Unexpected error during transaction read.") from error

    def transaction_write(self, items, client_request_token):
        """
        Wraps :func:`boto3.DynamoDB.Client.db.transact_write_items`.

        :param items: Unpacked into "TransactionItems" for :func:`boto3.DynamoDB.Client.transact_write_items`
        :param client_request_token: Idempotency token valid for 10 minutes from first use.
            Unpacked into "ClientRequestToken"
        :raises bloop.exceptions.TransactionCanceled: if the transaction was canceled.
        """
        try:
            self.dynamodb_client.transact_write_items(
                TransactItems=items,
                ClientRequestToken=client_request_token
            )
        except botocore.exceptions.ClientError as error:
            if error.response["Error"]["Code"] == "TransactionCanceledException":
                raise TransactionCanceled from error
            raise BloopException("Unexpected error during transaction write.") from error
bloop/util.py
def extract_key(key_shape, item):
    """
    construct a key according to key_shape for building an index

    usage::

        key_shape = "foo", "bar"
        item = {"baz": 1, "bar": 2, "foo": 3}
        extract_key(key_shape, item) -> {"foo": 3, "bar": 2}
    """
    return {field: item[field] for field in key_shape}
bloop/models.py
def subclassof(obj, classinfo):
    """Wrap issubclass to only return True/False"""
    try:
        return issubclass(obj, classinfo)
    except TypeError:
        return False
bloop/models.py
class BaseModel:
    """Abstract base that all models derive from.

    Provides a basic ``__init__`` method that takes ``**kwargs`` whose
    keys are columns names:

    .. code-block:: python

        class URL(BaseModel):
            id = Column(UUID, hash_key=True)
            ip = Column(IPv6)
            name = Column(String)

        url = URL(id=uuid.uuid4(), name="google")

    By default, the ``__init__`` method is not called when new instances are
    required, for example when iterating results from Query, Scan or a Stream.

    """
    class Meta(IMeta):
        abstract = True

    def __init__(self, **attrs):
        # Only set values from **attrs if there's a
        # corresponding `name` for a column in the model
        for column in self.Meta.columns:
            value = attrs.get(column.name, missing)
            if value is missing:
                value = column.default()
            if value is not missing:
                setattr(self, column.name, value)

    def __init_subclass__(cls: type, **kwargs):
        ensure_hash(cls)
        meta = initialize_meta(cls)

        # before we start binding, we should ensure that no combination of parent classes
        # will cause conflicts.  For example:
        #   class C(A, B) where
        #       A has a column named "foo" and dynamo_name "ddb"
        #       B has a column named "bar" and dynamo_name "ddb"
        # both A and B are valid mixins, but C must fail because there isn't a 1:1 binding to the "ddb" column.
        #
        # TODO | for now, we'll assume that the class being defined is special, and can replace columns with the
        # TODO | same dynamo_name.  In the example above, that would mean C has a column named "baz" and dynamo_name
        # TODO | "ddb" which would prevent the parent columns "foo" and "bar" from binding to the child class.

        modeled_attrs = set((attr for (_, attr) in inspect.getmembers(cls, lambda x: isinstance(x, (Column, Index)))))
        local_attrs = {
            name: value
            for name, value in cls.__dict__.items()
            if isinstance(value, (Column, Index))
        }
        derived_attrs = modeled_attrs - set(local_attrs.values())

        # 0.0 Pre-validation for collisions in derived columns/indexes
        dynamo_names = [x.dynamo_name for x in derived_attrs]
        collisions = [name for name, count in collections.Counter(dynamo_names).items() if count > 1]
        if collisions:
            collisions.sort()
            raise InvalidModel(
                f"The model {cls.__name__} subclasses one or more models with conflicting "
                f"column or index definitions for the following values of dynamo_name: {collisions}")
        derived_hash_keys = set((x.name for x in derived_attrs if isinstance(x, Column) and x.hash_key))
        if len(derived_hash_keys) > 1:
            derived_hash_keys = sorted(derived_hash_keys)
            raise InvalidModel(
                f"The model {cls.__name__} subclasses one or more models that declare multiple "
                f"columns as the hash key: {derived_hash_keys}")
        derived_range_keys = set((x.name for x in derived_attrs if isinstance(x, Column) and x.range_key))
        if len(derived_range_keys) > 1:
            derived_range_keys = sorted(derived_range_keys)
            raise InvalidModel(
                f"The model {cls.__name__} subclasses one or more models that declare multiple "
                f"columns as the range key: {derived_range_keys}")

        # 0.1 Pre-validation for collisions in local columns/indexes
        dynamo_names = [x.dynamo_name for x in local_attrs.values()]
        collisions = [name for name, count in collections.Counter(dynamo_names).items() if count > 1]
        if collisions:
            collisions.sort()
            raise InvalidModel(
                f"The model {cls.__name__} contains conflicting column or index definitions for the "
                f"following values of dynamo_name: {collisions}")
        local_hash_keys = [x.name for x in local_attrs.values() if isinstance(x, Column) and x.hash_key]
        if len(local_hash_keys) > 1:
            local_hash_keys = sorted(local_hash_keys)
            raise InvalidModel(
                f"The model {cls.__name__} defines multiple columns as hash columns: {local_hash_keys}")
        local_range_keys = [x.name for x in local_attrs.values() if isinstance(x, Column) and x.range_key]
        if len(local_range_keys) > 1:
            local_range_keys = sorted(local_range_keys)
            raise InvalidModel(
                f"The model {cls.__name__} defines multiple columns as range columns: {local_range_keys}")

        # 1.0 Bind derived columns so they can be referenced by derived indexes
        for attr in derived_attrs:
            if isinstance(attr, Column):
                bind_column(cls, attr.name, attr, copy=True)

        # 1.1 Bind derived indexes
        for attr in derived_attrs:
            if isinstance(attr, Index):
                bind_index(cls, attr.name, attr, copy=True)

        # 1.2 Bind local columns, allowing them to overwrite existing columns
        for name, attr in local_attrs.items():
            if isinstance(attr, Column):
                bind_column(cls, name, attr, force=True)

        # 1.3 Bind local indexes, allowing them to overwrite existing indexes
        for name, attr in local_attrs.items():
            if isinstance(attr, Index):
                bind_index(cls, name, attr, force=True)

        # 2.0 Ensure concrete models are valid
        # Currently, this just checks that a hash key is defined
        if not meta.abstract and not meta.hash_key:
            raise InvalidModel(f"{meta.model.__name__!r} has no hash key.")

        validate_stream(meta)
        validate_ttl(meta)
        validate_encryption(meta)
        validate_backups(meta)
        validate_billing(meta)

        # 3.0 Fire model_created for customizing the class after creation
        model_created.send(None, model=cls)

    def __repr__(self):
        attrs = ", ".join("{}={!r}".format(*item) for item in loaded_columns(self))
        return f"{self.__class__.__name__}({attrs})"
bloop/exceptions.py
class InvalidTemplate(BloopException, ValueError):
    """This is not a valid template string."""
bloop/exceptions.py
class InvalidStream(BloopException, ValueError):
    """This is not a valid stream definition."""
bloop/exceptions.py
class InvalidModel(BloopException, ValueError):
    """This is not a valid Model."""
bloop/util.py
def dump_key(engine, obj):
    """dump the hash (and range, if there is one) key(s) of an object into
    a dynamo-friendly format.

    returns {dynamo_name: {type: value} for dynamo_name in hash/range keys}
    """
    key = {}
    context = default_context(engine)
    for key_column in obj.Meta.keys:
        key_value = getattr(obj, key_column.name, missing)
        if key_value is missing:
            raise MissingKey("{!r} is missing {}: {!r}".format(
                obj, "hash_key" if key_column.hash_key else "range_key",
                key_column.name
            ))
        # noinspection PyProtectedMember
        key_action = key_column.typedef._dump(key_value, context=context)
        if key_action.type is not ActionType.Set:
            raise ValueError(
                f"key value {key_value} for column {key_column} must be a SET action but was {key_action}")
        key[key_column.dynamo_name] = key_action.value
    return key
bloop/models.py
class Index:
    """Abstract base class for GSIs and LSIs.

    An index must be bound to a model by calling :func:`bind_index(meta, model) <bloop.models.bind_index>`,
    which lets the index compute projected columns, validate hash and range keys, etc.

    .. seealso::

        :class:`~bloop.models.GlobalSecondaryIndex` and :class:`~bloop.models.LocalSecondaryIndex`

    :param projection: Either "keys", "all", or a list of column name or objects.
        Included columns will be projected into the index.  Key columns are always included.
    :param hash_key: The column that the index can be queried against.  Always the table hash_key for LSIs.
    :param range_key: The column that the index can be sorted on.  Always required for an LSI.  Default is None.
    :param str dynamo_name: *(Optional)* The index's name in in DynamoDB. Defaults to the index’s name in the model.
    """
    def __init__(self, *, projection, hash_key=None, range_key=None, dynamo_name=None, **kwargs):
        self.model = None
        if not isinstance(hash_key, (str, Column, type(None))):
            raise InvalidModel(f"Index hash_key must be a str or Column, but was {type(hash_key)!r}")
        if not isinstance(range_key, (str, Column, type(None))):
            raise InvalidModel(f"Index range_key must be a str or Column, but was {type(range_key)!r}")
        self._hash_key = hash_key
        self._range_key = range_key
        self._name = None
        self._dynamo_name = dynamo_name

        self.projection = validate_projection(projection)

    def __copy__(self):
        """
        Create a shallow copy of this Index.  Primarily used when initializing models that subclass other abstract
        models or mixins (baseless classes that contain Columns and Indexes).  You can override this method to
        change how derived models are created:

        .. code-block:: python

            import copy


            class MyIndex(Index):
                def __copy__(self):
                    new = super().__copy__()
                    new.derived = True
                    return new


            index = MyIndex(projection="keys", hash_key="some_column")
            same = copy.copy(index)
            assert same.derived  # True

        :return: A shallow copy of this Index, with the ``model`` and ``_name`` attributes unset, and the
                 computed projection invalidated.
        """
        cls = self.__class__
        obj = cls.__new__(cls)
        obj.__dict__.update(self.__dict__)
        obj.model = None
        obj._name = None
        obj.projection = {
            "mode": self.projection["mode"],
            "included": None,
            "available": None,
            "strict": self.projection["strict"]
        }

        # special case for projection mode "include".
        # note that for "all" and "keys" modes the included and available columns are model-specific, and so
        #   must be recalculated during refresh_index.  Both "all" and "keys" are lazy values that describe the
        #   index's relation to the model.
        # however, "include" is tightly coupled to the index itself.  The value of "include" for an unbound index
        #   can be thought of as "names that will be looked up on the model during a bind_index call" and so
        #   it doesn't really matter if they're Column objects or strings.  But that information must travel with the
        #   index, since it is not a property of the model.
        # to handle this, __copy__ on "include" will pass along the value of projection["included"].
        if self.projection["mode"] == "include":
            obj.projection["included"] = set(self.projection["included"])

        return obj

    def __set_name__(self, owner, name):
        self._name = name

    def __repr__(self):
        if isinstance(self, LocalSecondaryIndex):
            cls_name = "LSI"
        elif isinstance(self, GlobalSecondaryIndex):
            cls_name = "GSI"
        else:
            cls_name = self.__class__.__name__

        # <GSI[User.by_email=all]>
        # <GSI[User.by_email=keys]>
        # <LSI[User.by_email=include]>
        return f"<{cls_name}[{self.model.__name__}.{self.name}={self.projection['mode']}]>"

    @property
    def name(self):
        """Name of the model's attr that references self"""
        return self._name

    @property
    def dynamo_name(self):
        if self._dynamo_name is None:
            return self.name
        return self._dynamo_name

    @property
    def hash_key(self):
        if isinstance(self._hash_key, Column):
            # replacement is late-binding to handle direct references in models
            # before BaseModel.__init_subclass__ can name each column
            self._hash_key = self._hash_key.name
        return self.model.Meta.columns_by_name[self._hash_key]

    @property
    def range_key(self):
        if self._range_key is None:
            return None
        if isinstance(self._range_key, Column):
            # replacement is late-binding to handle direct references in models
            # before BaseModel.__init_subclass__ can name each column
            self._range_key = self._range_key.name
        return self.model.Meta.columns_by_name[self._range_key]

    @property
    def keys(self):
        keys = {self.hash_key}
        if self.range_key:
            keys.add(self.range_key)
        return keys

    def __set__(self, obj, value):
        raise AttributeError(f"{self.model.__name__}.{self.name} is a {self.__class__.__name__}")

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        raise AttributeError(f"{self.model.__name__}.{self.name} is a {self.__class__.__name__}")

    def __delete__(self, obj):
        raise AttributeError(f"{self.model.__name__}.{self.name} is a {self.__class__.__name__}")
bloop/conditions.py
def render(engine, obj=None, filter=None, projection=None, key=None, condition=None, update=None):
    renderer = ConditionRenderer(engine)
    renderer.render(
        obj=obj, condition=condition,
        update=update,
        filter=filter, projection=projection, key=key,
    )
    return renderer.output
bloop/signals.py

bloop/util.py
def walk_subclasses(root):
    """Does not yield the input class"""
    classes = [root]
    visited = set()
    while classes:
        cls = classes.pop()
        if cls is type or cls in visited:
            continue
        classes.extend(cls.__subclasses__())
        visited.add(cls)
        if cls is not root:
            yield cls
bloop/util.py
def index_for(key):
    """stable hashable tuple of object keys for indexing an item in constant time.

    usage::

        index_for({'id': {'S': 'foo'}, 'range': {'S': 'bar'}}) -> ('bar', 'foo')
    """
    return tuple(sorted(value_of(k) for k in key.values()))
bloop/stream/stream.py
class Stream:
    """Iterator over all records in a stream.

    :param model: The model to stream records from.
    :param engine: The engine to load model objects through.
    :type engine: :class:`~bloop.engine.Engine`
    """
    def __init__(self, *, model, engine):

        self.model = model
        self.engine = engine
        self.coordinator = Coordinator(
            session=engine.session,
            stream_arn=model.Meta.stream["arn"])

    def __repr__(self):
        # <Stream[User]>
        return "<{}[{}]>".format(self.__class__.__name__, self.model.__name__)

    def __iter__(self):
        return self

    def __next__(self):
        record = next(self.coordinator)
        if record:
            meta = self.model.Meta
            for key, expected in [("new", meta.columns), ("old", meta.columns), ("key", meta.keys)]:
                if key not in meta.stream["include"]:
                    record[key] = None
                else:
                    self._unpack(record, key, expected)
        return record

    def heartbeat(self):
        """Refresh iterators without sequence numbers so they don't expire.

        Call this at least every 14 minutes.
        """
        self.coordinator.heartbeat()

    def move_to(self, position):
        """Move the Stream to a specific endpoint or time, or load state from a token.

        Moving to an endpoint with "trim_horizon" or "latest" and loading from a previous token are both
        very efficient.

        In contrast, seeking to a specific time requires iterating **all records in the stream up to that time**.
        This can be **very expensive**.  Once you have moved a stream to a time, you should save the
        :attr:`Stream.token <bloop.stream.stream.Stream.token>` so reloading will be extremely fast.

        :param position: "trim_horizon", "latest", :class:`~datetime.datetime`, or a
            :attr:`Stream.token <bloop.stream.stream.Stream.token>`
        """
        self.coordinator.move_to(position)

    @property
    def token(self):
        """JSON-serializable representation of the current Stream state.

        Use :func:`Engine.stream(YourModel, token) <bloop.engine.Engine.stream>` to create an identical stream,
        or :func:`stream.move_to(token) <bloop.stream.Stream.move_to>` to move an existing stream to this position.

        :returns: Stream state as a json-friendly dict
        :rtype: dict
        """
        return self.coordinator.token

    def _unpack(self, record, key, expected):
        """Replaces the attr dict at the given key with an instance of a Model"""
        attrs = record.get(key)
        if attrs is None:
            return
        obj = unpack_from_dynamodb(
            attrs=attrs,
            expected=expected,
            model=self.model,
            engine=self.engine
        )
        object_loaded.send(self.engine, engine=self.engine, obj=obj)
        record[key] = obj


Question: import logging
from typing import Any, Callable, Union
from .conditions import render
from .exceptions import (
    InvalidModel,
    InvalidStream,
    InvalidTemplate,
    MissingObjects,
)
from .models import BaseModel, Index, subclassof, unpack_from_dynamodb
from .search import Search
from .session import SessionWrapper
from .signals import (
    before_create_table,
    model_bound,
    model_validated,
    object_deleted,
    object_loaded,
    object_saved,
)
from .stream import Stream
from .transactions import ReadTransaction, WriteTransaction
from .util import dump_key, extract_key, index_for, walk_subclasses
        logger.info("successfully loaded {} objects".format(len(objs)))

    def query(self, model_or_index, key, filter=None, projection="all", consistent=False, forward=True):
        """Create a reusable :class:`~bloop.search.QueryIterator`.

        :param model_or_index: A model or index to query.  For example, ``User`` or ``User.by_email``.
        :param key:
            Key condition.  This must include an equality against the hash key, and optionally one
            of a restricted set of conditions on the range key.
        :param filter: Filter condition.  Only matching objects will be included in the results.
        :param projection:
            "all", "count", a set of column names, or a set of :class:`~bloop.models.Column`.  When projection is
            "count", you must advance the iterator to retrieve the count.
        :param bool consistent: Use `strongly consistent reads`__ if True.  Default is False.
        :param bool forward:  Query in ascending or descending order.  Default is True (ascending).

        :return: A reusable query iterator with helper methods.
        :rtype: :class:`~bloop.search.QueryIterator`

        __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadConsistency.html
        """
        if isinstance(model_or_index, Index):
            model, index = model_or_index.model, model_or_index
        else:
            model, index = model_or_index, None
        validate_not_abstract(model)
        q = Search(
            mode="query", engine=self, model=model, index=index, key=key, filter=filter,
            projection=projection, consistent=consistent, forward=forward)
        return iter(q.prepare())

    def save(self, *objs, condition=None, sync=None):
        """Save one or more objects.

        :param objs: objects to save.
        :param condition: only perform each save if this condition holds.
        :param sync:
            update objects after saving.  "new" loads attributes after the save;
            "old" loads attributes before the save; None does not mutate the object locally.  Default is None.
        :raises bloop.exceptions.ConstraintViolation: if the condition (or atomic) is not met.
        """
        objs = set(objs)
        validate_not_abstract(*objs)
        for obj in objs:
            attrs = self.session.save_item({
                "TableName": self._compute_table_name(obj.__class__),
                "Key": dump_key(self, obj),
                "ReturnValues": validate_sync("save", sync),
                **render(self, obj=obj, condition=condition, update=True)
            })
            if attrs is not None:
                unpack_from_dynamodb(attrs=attrs, expected=obj.Meta.columns, engine=self, obj=obj)
            object_saved.send(self, engine=self, obj=obj)
        logger.info("successfully saved {} objects".format(len(objs)))

    def scan(self, model_or_index, filter=None, projection="all", consistent=False, parallel=None):
        """Create a reusable :class:`~bloop.search.ScanIterator`.

        :param model_or_index: A model or index to scan.  For example, ``User`` or ``User.by_email``.
        :param filter: Filter condition.  Only matching objects will be included in the results.
        :param projection:
            "all", "count", a list of column names, or a list of :class:`~bloop.models.Column`.  When projection is
            "count", you must exhaust the iterator to retrieve the count.
        :param bool consistent: Use `strongly consistent reads`__ if True.  Default is False.
        :param tuple parallel: Perform a `parallel scan`__.  A tuple of (Segment, TotalSegments)
            for this portion the scan. Default is None.
        :return: A reusable scan iterator with helper methods.
        :rtype: :class:`~bloop.search.ScanIterator`

        __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadConsistency.html
        __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/QueryAndScan.html#QueryAndScanParallelScan
        """
        if isinstance(model_or_index, Index):
            model, index = model_or_index.model, model_or_index
        else:
            model, index = model_or_index, None
        validate_not_abstract(model)
        s = Search(
            mode="scan", engine=self, model=model, index=index, filter=filter,
            projection=projection, consistent=consistent, parallel=parallel)
        return iter(s.prepare())

    def stream(self, model, position):
        # noinspection PyUnresolvedReferences
        """Create a :class:`~bloop.stream.Stream` that provides approximate chronological ordering.

        .. code-block:: pycon

            # Create a user so we have a record
            >>> engine = Engine()
            >>> user = User(id=3, email="user@domain.com")
            >>> engine.save(user)
            >>> user.email = "admin@domain.com"
            >>> engine.save(user)

            # First record lacks an "old" value since it's an insert
            >>> stream = engine.stream(User, "trim_horizon")
            >>> next(stream)
            {'key': None,
             'old': None,
             'new': User(email='user@domain.com', id=3, verified=None),
             'meta': {
                 'created_at': datetime.datetime(2016, 10, 23, ...),
                 'event': {
                     'id': '3fe6d339b7cb19a1474b3d853972c12a',
                     'type': 'insert',
                     'version': '1.1'},
                 'sequence_number': '700000000007366876916'}
            }


        :param model: The model to stream records from.
        :param position: "trim_horizon", "latest", a stream token, or a :class:`datetime.datetime`.
        :return: An iterator for records in all shards.
        :rtype: :class:`~bloop.stream.Stream`
        :raises bloop.exceptions.InvalidStream: if the model does not have a stream.
        """
        validate_not_abstract(model)
        if not model.Meta.stream or not model.Meta.stream.get("arn"):
            raise InvalidStream("{!r} does not have a stream arn".format(model))

Answer:         stream = Stream(model=model, engine=self)