Repository: pynamodb/PynamoDB

Issue:
Another crash caused by zope.interface
Similar to #71

This one appear when using an index, the crash is in Model._get_indexes() 

``` python
Exception: AttributeError: __provides__

Exception class: AttributeError

Exception message: __provides__

Backtrace:
filename: "/xxx/local/lib/python2.7/site-packages/paste/deploy/config.py" line: 291 function: "__call__" text/code: "return self.app(environ, start_response)" 
filename: "/xxx/local/lib/python2.7/site-packages/wsgicors.py" line: 197 function: "__call__" text/code: "return self.application(environ, custom_start_response)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/router.py" line: 242 function: "__call__" text/code: "response = self.invoke_subrequest(request, use_tweens=True)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/router.py" line: 217 function: "invoke_subrequest" text/code: "response = handle_request(request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid_metrics/tween.py" line: 24 function: "performance_tween" text/code: "response = handler(request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/tweens.py" line: 21 function: "excview_tween" text/code: "response = handler(request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid_exclog/__init__.py" line: 115 function: "exclog_tween" text/code: "_handle_error(request, getLogger, get_message)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid_exclog/__init__.py" line: 111 function: "exclog_tween" text/code: "return handler(request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/router.py" line: 163 function: "handle_request" text/code: "response = view_callable(context, request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/config/views.py" line: 596 function: "__call__" text/code: "return view(context, request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/config/views.py" line: 329 function: "attr_view" text/code: "return view(context, request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/config/views.py" line: 305 function: "predicate_wrapper" text/code: "return view(context, request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/config/views.py" line: 245 function: "_secured_view" text/code: "return view(context, request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/config/views.py" line: 355 function: "rendered_view" text/code: "result = view(context, request)" 
filename: "/xxx/local/lib/python2.7/site-packages/pyramid/config/views.py" line: 491 function: "_class_view" text/code: "response = getattr(inst, attr)()" 
filename: "/xxx/local/lib/python2.7/site-packages/royal/views.py" line: 44 function: "index" text/code: "result = func(self.context.index_schema(query_params))" 
filename: "/xxx/local/lib/python2.7/site-packages/xxxx/resources.py" line: 286 function: "index" text/code: "devices = Device.get_devices_by_account(self.project_id, account_id)" 
filename: "/xxx/local/lib/python2.7/site-packages/xxxx/device.py" line: 67 function: "get_devices_by_account" text/code: "cls._get_devices_by_account(project_id, account_id)]" 
filename: "/xxx/local/lib/python2.7/site-packages/pynamodb/models.py" line: 521 function: "query" text/code: "cls._get_indexes()" 
filename: "/xxx/local/lib/python2.7/site-packages/pynamodb/models.py" line: 922 function: "_get_indexes" text/code: "item_cls = getattr(getattr(cls, item), "__class__", None)" 
```


Discussion:
jpinner-lyft: @garrettheel @jmphilli any idea why we do:
```
cls = getattr(item, '__class__', None)
issubclass(cls, classinfo)
```
instead of
```
issubclass(item, classinfo)
```
jpinner-lyft: Some comments while looking at the history from #280:
- previous implementations used `issubclass` directly and `type(cls)`
- inspect `getmembers` has an identical implementation for 2.7+

Patch:
diff --git a/pynamodb/attributes.py b/pynamodb/attributes.py
index 4f0e286..99cdb33 100644
--- a/pynamodb/attributes.py
+++ b/pynamodb/attributes.py
@@ -14,6 +14,7 @@ from pynamodb.constants import (
     STRING, STRING_SHORT, NUMBER, BINARY, UTC, DATETIME_FORMAT, BINARY_SET, STRING_SET, NUMBER_SET,
     MAP, MAP_SHORT, LIST, LIST_SHORT, DEFAULT_ENCODING, BOOLEAN, ATTR_TYPE_MAP, NUMBER_SHORT, NULL, SHORT_ATTR_TYPES
 )
+from pynamodb.compat import getmembers_issubclass
 from pynamodb.expressions.operand import Path
 import collections
 
@@ -183,34 +184,24 @@ class AttributeContainerMeta(type):
         cls._attributes = {}
         cls._dynamo_to_python_attrs = {}
 
-        for item_name in dir(cls):
-            try:
-                item_cls = getattr(getattr(cls, item_name), "__class__", None)
-            except AttributeError:
-                continue
+        for name, attribute in getmembers_issubclass(cls, Attribute):
+            initialized = False
+            if isinstance(attribute, MapAttribute):
+                # MapAttribute instances that are class attributes of an AttributeContainer class
+                # should behave like an Attribute instance and not an AttributeContainer instance.
+                initialized = attribute._make_attribute()
 
-            if item_cls is None:
-                continue
+            cls._attributes[name] = attribute
+            if attribute.attr_name is not None:
+                cls._dynamo_to_python_attrs[attribute.attr_name] = name
+            else:
+                attribute.attr_name = name
 
-            if issubclass(item_cls, Attribute):
-                instance = getattr(cls, item_name)
-                initialized = False
-                if isinstance(instance, MapAttribute):
-                    # MapAttribute instances that are class attributes of an AttributeContainer class
-                    # should behave like an Attribute instance and not an AttributeContainer instance.
-                    initialized = instance._make_attribute()
-
-                cls._attributes[item_name] = instance
-                if instance.attr_name is not None:
-                    cls._dynamo_to_python_attrs[instance.attr_name] = item_name
-                else:
-                    instance.attr_name = item_name
-
-                if initialized and isinstance(instance, MapAttribute):
-                    # To support creating expressions from nested attributes, MapAttribute instances
-                    # store local copies of the attributes in cls._attributes with `attr_path` set.
-                    # Prepend the `attr_path` lists with the dynamo attribute name.
-                    instance._update_attribute_paths(instance.attr_name)
+            if initialized and isinstance(attribute, MapAttribute):
+                # To support creating expressions from nested attributes, MapAttribute instances
+                # store local copies of the attributes in cls._attributes with `attr_path` set.
+                # Prepend the `attr_path` lists with the dynamo attribute name.
+                attribute._update_attribute_paths(attribute.attr_name)
 
 
 @add_metaclass(AttributeContainerMeta)
diff --git a/pynamodb/compat.py b/pynamodb/compat.py
index 2c246b1..5163399 100644
--- a/pynamodb/compat.py
+++ b/pynamodb/compat.py
@@ -59,3 +59,21 @@ class CompatTestCase(unittest.TestCase):
 class NullHandler(logging.Handler):
     def emit(self, record):
         pass
+
+
+# Replace this function with inspect.getmembers() once we drop Python 2.6 support
+# see https://bugs.python.org/issue1785
+# see https://bugs.launchpad.net/zope.interface/+bug/181371
+def getmembers_issubclass(object, classinfo):
+    results = []
+    for key in dir(object):
+        try:
+            value = getattr(object, key)
+        except AttributeError:
+            continue
+
+        value_cls = getattr(value, '__class__', None)
+        if value_cls and issubclass(value_cls, classinfo):
+            results.append((key, value))
+    results.sort()
+    return results
diff --git a/pynamodb/indexes.py b/pynamodb/indexes.py
index 1c8d960..09b9687 100644
--- a/pynamodb/indexes.py
+++ b/pynamodb/indexes.py
@@ -7,6 +7,7 @@ from pynamodb.constants import (
 )
 from pynamodb.attributes import Attribute
 from pynamodb.types import HASH, RANGE
+from pynamodb.compat import getmembers_issubclass
 from pynamodb.connection.util import pythonic
 from six import with_metaclass
 
@@ -131,10 +132,8 @@ class Index(with_metaclass(IndexMeta)):
         """
         if cls.Meta.attributes is None:
             cls.Meta.attributes = {}
-            for item in dir(cls):
-                item_cls = getattr(getattr(cls, item), "__class__", None)
-                if item_cls and issubclass(item_cls, (Attribute, )):
-                    cls.Meta.attributes[item] = getattr(cls, item)
+            for name, attribute in getmembers_issubclass(cls, Attribute):
+                cls.Meta.attributes[name] = attribute
         return cls.Meta.attributes
 
 
diff --git a/pynamodb/models.py b/pynamodb/models.py
index cacb817..01c71f5 100644
--- a/pynamodb/models.py
+++ b/pynamodb/models.py
@@ -15,7 +15,7 @@ from pynamodb.connection.base import MetaTable
 from pynamodb.connection.table import TableConnection
 from pynamodb.connection.util import pythonic
 from pynamodb.types import HASH, RANGE
-from pynamodb.compat import NullHandler
+from pynamodb.compat import NullHandler, getmembers_issubclass
 from pynamodb.indexes import Index, GlobalSecondaryIndex
 from pynamodb.pagination import ResultIterator
 from pynamodb.settings import get_settings_value
@@ -896,13 +896,18 @@ class Model(AttributeContainer):
         hash_key, attrs = data
         range_key = attrs.pop('range_key', None)
         attributes = attrs.pop(pythonic(ATTRIBUTES))
+        hash_keyname = cls._get_meta_data().hash_keyname
+        hash_keytype = cls._get_meta_data().get_attribute_type(hash_keyname)
+        attributes[hash_keyname] = {
+            hash_keytype: hash_key
+        }
         if range_key is not None:
             range_keyname = cls._get_meta_data().range_keyname
             range_keytype = cls._get_meta_data().get_attribute_type(range_keyname)
             attributes[range_keyname] = {
                 range_keytype: range_key
             }
-        item = cls(hash_key)
+        item = cls()
         item._deserialize(attributes)
         return item
 
@@ -1103,34 +1108,29 @@ class Model(AttributeContainer):
                 pythonic(ATTR_DEFINITIONS): []
             }
             cls._index_classes = {}
-            for item in dir(cls):
-                item_cls = getattr(getattr(cls, item), "__class__", None)
-                if item_cls is None:
-                    continue
-                if issubclass(item_cls, (Index, )):
-                    item_cls = getattr(cls, item)
-                    cls._index_classes[item_cls.Meta.index_name] = item_cls
-                    schema = item_cls._get_schema()
-                    idx = {
-                        pythonic(INDEX_NAME): item_cls.Meta.index_name,
-                        pythonic(KEY_SCHEMA): schema.get(pythonic(KEY_SCHEMA)),
-                        pythonic(PROJECTION): {
-                            PROJECTION_TYPE: item_cls.Meta.projection.projection_type,
-                        },
+            for name, index in getmembers_issubclass(cls, Index):
+                cls._index_classes[index.Meta.index_name] = index
+                schema = index._get_schema()
+                idx = {
+                    pythonic(INDEX_NAME): index.Meta.index_name,
+                    pythonic(KEY_SCHEMA): schema.get(pythonic(KEY_SCHEMA)),
+                    pythonic(PROJECTION): {
+                        PROJECTION_TYPE: index.Meta.projection.projection_type,
+                    },
 
+                }
+                if issubclass(index.__class__, GlobalSecondaryIndex):
+                    idx[pythonic(PROVISIONED_THROUGHPUT)] = {
+                        READ_CAPACITY_UNITS: index.Meta.read_capacity_units,
+                        WRITE_CAPACITY_UNITS: index.Meta.write_capacity_units
                     }
-                    if issubclass(item_cls.__class__, GlobalSecondaryIndex):
-                        idx[pythonic(PROVISIONED_THROUGHPUT)] = {
-                            READ_CAPACITY_UNITS: item_cls.Meta.read_capacity_units,
-                            WRITE_CAPACITY_UNITS: item_cls.Meta.write_capacity_units
-                        }
-                    cls._indexes[pythonic(ATTR_DEFINITIONS)].extend(schema.get(pythonic(ATTR_DEFINITIONS)))
-                    if item_cls.Meta.projection.non_key_attributes:
-                        idx[pythonic(PROJECTION)][NON_KEY_ATTRIBUTES] = item_cls.Meta.projection.non_key_attributes
-                    if issubclass(item_cls.__class__, GlobalSecondaryIndex):
-                        cls._indexes[pythonic(GLOBAL_SECONDARY_INDEXES)].append(idx)
-                    else:
-                        cls._indexes[pythonic(LOCAL_SECONDARY_INDEXES)].append(idx)
+                cls._indexes[pythonic(ATTR_DEFINITIONS)].extend(schema.get(pythonic(ATTR_DEFINITIONS)))
+                if index.Meta.projection.non_key_attributes:
+                    idx[pythonic(PROJECTION)][NON_KEY_ATTRIBUTES] = index.Meta.projection.non_key_attributes
+                if issubclass(index.__class__, GlobalSecondaryIndex):
+                    cls._indexes[pythonic(GLOBAL_SECONDARY_INDEXES)].append(idx)
+                else:
+                    cls._indexes[pythonic(LOCAL_SECONDARY_INDEXES)].append(idx)
         return cls._indexes
 
     def _get_json(self):
@@ -1257,13 +1257,13 @@ class Model(AttributeContainer):
 
         :param attrs: A dictionary of attributes to update this item with.
         """
-        for name, attr in attrs.items():
-            attr_instance = self._get_attributes().get(name, None)
-            if attr_instance:
-                attr_type = ATTR_TYPE_MAP[attr_instance.attr_type]
-                value = attr.get(attr_type, None)
+        for name, attr in self._get_attributes().items():
+            value = attrs.get(attr.attr_name, None)
+            if value is not None:
+                value = value.get(ATTR_TYPE_MAP[attr.attr_type], None)
                 if value is not None:
-                    setattr(self, name, attr_instance.deserialize(value))
+                    value = attr.deserialize(value)
+            setattr(self, name, value)
 
     def _serialize(self, attr_map=False, null_check=True):
         """


Test patch:
diff --git a/pynamodb/tests/data.py b/pynamodb/tests/data.py
index 3a76e22..973ac23 100644
--- a/pynamodb/tests/data.py
+++ b/pynamodb/tests/data.py
@@ -466,7 +466,7 @@ COMPLEX_MODEL_ITEM_DATA = {
                     'N': '31'
                 },
                 'is_dude': {
-                    'N': '1'
+                    'BOOL': True
                 }
             }
         }
@@ -826,6 +826,33 @@ SERIALIZED_TABLE_DATA = [
     ]
 ]
 
+COMPLEX_MODEL_SERIALIZED_TABLE_DATA = [
+    [
+        "123",
+        {
+            "attributes": {
+                'weird_person': {
+                    'M': {
+                        'firstName': {
+                            'S': 'Justin'
+                        },
+                        'lname': {
+                            'S': 'Phillips'
+                        },
+                        'age': {
+                            'N': '31'
+                        },
+                        'is_dude': {
+                            'N': '1'
+                        }
+                    }
+
+                }
+            }
+        }
+    ]
+]
+
 OFFICE_EMPLOYEE_MODEL_TABLE_DATA = {
     "Table": {
         "AttributeDefinitions": [
diff --git a/pynamodb/tests/test_model.py b/pynamodb/tests/test_model.py
index 9870dbd..0e90d20 100644
--- a/pynamodb/tests/test_model.py
+++ b/pynamodb/tests/test_model.py
@@ -36,7 +36,7 @@ from pynamodb.tests.data import (
     BATCH_GET_ITEMS, SIMPLE_BATCH_GET_ITEMS, COMPLEX_TABLE_DATA,
     COMPLEX_ITEM_DATA, INDEX_TABLE_DATA, LOCAL_INDEX_TABLE_DATA, DOG_TABLE_DATA,
     CUSTOM_ATTR_NAME_INDEX_TABLE_DATA, CUSTOM_ATTR_NAME_ITEM_DATA,
-    BINARY_ATTR_DATA, SERIALIZED_TABLE_DATA, OFFICE_EMPLOYEE_MODEL_TABLE_DATA,
+    BINARY_ATTR_DATA, SERIALIZED_TABLE_DATA, OFFICE_EMPLOYEE_MODEL_TABLE_DATA, COMPLEX_MODEL_SERIALIZED_TABLE_DATA,
     GET_OFFICE_EMPLOYEE_ITEM_DATA, GET_OFFICE_EMPLOYEE_ITEM_DATA_WITH_NULL,
     GROCERY_LIST_MODEL_TABLE_DATA, GET_GROCERY_LIST_ITEM_DATA,
     GET_OFFICE_ITEM_DATA, OFFICE_MODEL_TABLE_DATA, COMPLEX_MODEL_TABLE_DATA, COMPLEX_MODEL_ITEM_DATA,
@@ -702,10 +702,12 @@ class ModelTestCase(TestCase):
 
         with patch(PATCH_METHOD) as req:
             req.return_value = GET_MODEL_ITEM_DATA
+            item.picture = b'to-be-removed'
             item.refresh()
             self.assertEqual(
                 item.custom_user_name,
                 GET_MODEL_ITEM_DATA.get(ITEM).get('user_name').get(STRING_SHORT))
+            self.assertIsNone(item.picture)
 
     def test_complex_key(self):
         """
@@ -3732,6 +3734,20 @@ class ModelTestCase(TestCase):
         }
         self.assert_dict_lists_equal(req.call_args[0][1]['RequestItems']['UserModel'], args['UserModel'])
 
+    def test_loads_complex_model(self):
+        with patch(PATCH_METHOD) as req:
+            req.return_value = {}
+            ComplexModel.loads(json.dumps(COMPLEX_MODEL_SERIALIZED_TABLE_DATA))
+
+        args = {
+            'ComplexModel': [
+                {
+                    'PutRequest': COMPLEX_MODEL_ITEM_DATA
+                }
+            ]
+        }
+        self.assert_dict_lists_equal(req.call_args[0][1]['RequestItems']['ComplexModel'], args['ComplexModel'])
+
     def _get_office_employee(self):
         justin = Person(
             fname='Justin',
@@ -4193,8 +4209,8 @@ class ModelTestCase(TestCase):
                 'mapy': {'M': {'baz': {'S': 'bongo'}}}
             }
         }
-        instance = ExplicitRawMapModel(map_attr=map_native)
-        instance._deserialize(map_serialized)
+        instance = ExplicitRawMapModel()
+        instance._deserialize({'map_attr': map_serialized})
         actual = instance.map_attr
         for k, v in six.iteritems(map_native):
             self.assertEqual(v, actual[k])