Repository: pynamodb/PynamoDB

Issue:
Last Evaluated Key and Limited Queries/Scans
The `last_evaluated_key` value returns whatever was last evaluated for the current result page. When limiting the query but not setting the page size to the limit, the value does not represent the last item returned by the iterator.

Discussion:

Patch:
diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py
index ac92ae0..bf425ef 100644
--- a/pynamodb/connection/base.py
+++ b/pynamodb/connection/base.py
@@ -95,6 +95,22 @@ class MetaTable(object):
                     break
         return self._hash_keyname
 
+    def get_key_names(self, index_name=None):
+        """
+        Returns the names of the primary key attributes and index key attributes (if index_name is specified)
+        """
+        key_names = [self.hash_keyname]
+        if self.range_keyname:
+            key_names.append(self.range_keyname)
+        if index_name is not None:
+            index_hash_keyname = self.get_index_hash_keyname(index_name)
+            if index_hash_keyname not in key_names:
+                key_names.append(index_hash_keyname)
+            index_range_keyname = self.get_index_range_keyname(index_name)
+            if index_range_keyname is not None and index_range_keyname not in key_names:
+                key_names.append(index_range_keyname)
+        return key_names
+
     def get_index_hash_keyname(self, index_name):
         """
         Returns the name of the hash key for a given index
diff --git a/pynamodb/connection/table.py b/pynamodb/connection/table.py
index 1e7a439..35f46bd 100644
--- a/pynamodb/connection/table.py
+++ b/pynamodb/connection/table.py
@@ -28,6 +28,12 @@ class TableConnection(object):
                                      max_retry_attempts=max_retry_attempts,
                                      base_backoff_ms=base_backoff_ms)
 
+    def get_meta_table(self, refresh=False):
+        """
+        Returns a MetaTable
+        """
+        return self.connection.get_meta_table(self.table_name, refresh=refresh)
+
     def delete_item(self, hash_key,
                     range_key=None,
                     condition=None,
diff --git a/pynamodb/pagination.py b/pynamodb/pagination.py
index 8f008e2..6f948b6 100644
--- a/pynamodb/pagination.py
+++ b/pynamodb/pagination.py
@@ -35,6 +35,16 @@ class PageIterator(object):
     def next(self):
         return self.__next__()
 
+    @property
+    def key_names(self):
+        # If the current page has a last_evaluated_key, use it to determine key attributes
+        if self._last_evaluated_key:
+            return self._last_evaluated_key.keys()
+
+        # Use the table meta data to determine the key attributes
+        table_meta = self._operation.im_self.get_meta_table()
+        return table_meta.get_key_names(self._kwargs.get('index_name'))
+
     @property
     def page_size(self):
         return self._kwargs.get('limit')
@@ -100,7 +110,20 @@ class ResultIterator(object):
 
     @property
     def last_evaluated_key(self):
-        return self.page_iter.last_evaluated_key
+        if self._first_iteration:
+            # Not started iterating yet: there cannot be a last_evaluated_key
+            return None
+
+        if self._index == self._count:
+            # Entire page has been consumed: last_evaluated_key is whatever DynamoDB returned
+            # It may correspond to the current item, or it may correspond to an item evaluated but not returned.
+            return self.page_iter.last_evaluated_key
+
+        # In the middle of a page of results: reconstruct a last_evaluated_key from the current item
+        # The operation should be resumed starting at the last item returned, not the last item evaluated.
+        # This can occur if the 'limit' is reached in the middle of a page.
+        item = self._items[self._index - 1]
+        return dict((key, item[key]) for key in self.page_iter.key_names)
 
     @property
     def total_count(self):


Test patch:
diff --git a/pynamodb/tests/test_base_connection.py b/pynamodb/tests/test_base_connection.py
index 4095d6c..b9b33fa 100644
--- a/pynamodb/tests/test_base_connection.py
+++ b/pynamodb/tests/test_base_connection.py
@@ -6,10 +6,12 @@ import json
 import six
 from pynamodb.compat import CompatTestCase as TestCase
 from pynamodb.connection import Connection
+from pynamodb.connection.base import MetaTable
 from botocore.vendored import requests
 from pynamodb.exceptions import (VerboseClientError,
     TableError, DeleteError, UpdateError, PutError, GetError, ScanError, QueryError, TableDoesNotExist)
-from pynamodb.constants import DEFAULT_REGION, UNPROCESSED_ITEMS, STRING_SHORT, BINARY_SHORT, DEFAULT_ENCODING
+from pynamodb.constants import (
+    DEFAULT_REGION, UNPROCESSED_ITEMS, STRING_SHORT, BINARY_SHORT, DEFAULT_ENCODING, TABLE_KEY)
 from pynamodb.expressions.operand import Path
 from pynamodb.tests.data import DESCRIBE_TABLE_DATA, GET_ITEM_DATA, LIST_TABLE_DATA
 from pynamodb.tests.deep_eq import deep_eq
@@ -25,6 +27,23 @@ else:
 PATCH_METHOD = 'pynamodb.connection.Connection._make_api_call'
 
 
+class MetaTableTestCase(TestCase):
+    """
+    Tests for the meta table class
+    """
+
+    def setUp(self):
+        self.meta_table = MetaTable(DESCRIBE_TABLE_DATA.get(TABLE_KEY))
+
+    def test_get_key_names(self):
+        key_names = self.meta_table.get_key_names()
+        self.assertEqual(key_names, ["ForumName", "Subject"])
+
+    def test_get_key_names_index(self):
+        key_names = self.meta_table.get_key_names("LastPostIndex")
+        self.assertEqual(key_names, ["ForumName", "Subject", "LastPostDateTime"])
+
+
 class ConnectionTestCase(TestCase):
     """
     Tests for the base connection class
diff --git a/pynamodb/tests/test_model.py b/pynamodb/tests/test_model.py
index f0ac212..ab93c07 100644
--- a/pynamodb/tests/test_model.py
+++ b/pynamodb/tests/test_model.py
@@ -2125,9 +2125,9 @@ class ModelTestCase(TestCase):
                 items.append(item)
 
             req.side_effect = [
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': 'x'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': 'y'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30], 'LastEvaluatedKey': 'z'},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': {'user_id': 'x'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': {'user_id': 'y'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30], 'LastEvaluatedKey': {'user_id': 'z'}},
             ]
             results_iter = UserModel.query('foo', limit=25)
             results = list(results_iter)
@@ -2136,7 +2136,7 @@ class ModelTestCase(TestCase):
             self.assertEquals(req.mock_calls[0][1][1]['Limit'], 25)
             self.assertEquals(req.mock_calls[1][1][1]['Limit'], 25)
             self.assertEquals(req.mock_calls[2][1][1]['Limit'], 25)
-            self.assertEquals(results_iter.last_evaluated_key, 'z')
+            self.assertEquals(results_iter.last_evaluated_key, {'user_id': items[24]['user_id']})
             self.assertEquals(results_iter.total_count, 30)
             self.assertEquals(results_iter.page_iter.total_scanned_count, 60)
 
@@ -2153,9 +2153,9 @@ class ModelTestCase(TestCase):
                 items.append(item)
 
             req.side_effect = [
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': 'x'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': 'y'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30], 'LastEvaluatedKey': 'z'},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': {'user_id': 'x'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': {'user_id': 'y'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30], 'LastEvaluatedKey': {'user_id': 'x'}},
             ]
             results_iter = UserModel.query('foo', limit=25, page_size=10)
             results = list(results_iter)
@@ -2164,7 +2164,7 @@ class ModelTestCase(TestCase):
             self.assertEquals(req.mock_calls[0][1][1]['Limit'], 10)
             self.assertEquals(req.mock_calls[1][1][1]['Limit'], 10)
             self.assertEquals(req.mock_calls[2][1][1]['Limit'], 10)
-            self.assertEquals(results_iter.last_evaluated_key, 'z')
+            self.assertEquals(results_iter.last_evaluated_key, {'user_id': items[24]['user_id']})
             self.assertEquals(results_iter.total_count, 30)
             self.assertEquals(results_iter.page_iter.total_scanned_count, 60)
 
@@ -2181,8 +2181,8 @@ class ModelTestCase(TestCase):
                 items.append(item)
 
             req.side_effect = [
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': 'x'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': 'y'},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': {'user_id': 'x'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': {'user_id': 'y'}},
                 {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30]},
             ]
             results_iter = UserModel.query('foo', limit=50)
@@ -2209,8 +2209,8 @@ class ModelTestCase(TestCase):
                 items.append(item)
 
             req.side_effect = [
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': 'x'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': 'y'},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': {'user_id': 'x'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': {'user_id': 'y'}},
                 {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30]},
             ]
             results_iter = UserModel.query('foo', limit=50, page_size=10)
@@ -2545,9 +2545,9 @@ class ModelTestCase(TestCase):
                 items.append(item)
 
             req.side_effect = [
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': 'x'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': 'y'},
-                {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30], 'LastEvaluatedKey': 'z'},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[:10], 'LastEvaluatedKey': {'user_id': 'x'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[10:20], 'LastEvaluatedKey': {'user_id': 'y'}},
+                {'Count': 10, 'ScannedCount': 20, 'Items': items[20:30], 'LastEvaluatedKey': {'user_id': 'z'}},
             ]
             results_iter = UserModel.scan(limit=25, page_size=10)
             results = list(results_iter)
@@ -2556,7 +2556,7 @@ class ModelTestCase(TestCase):
             self.assertEquals(req.mock_calls[0][1][1]['Limit'], 10)
             self.assertEquals(req.mock_calls[1][1][1]['Limit'], 10)
             self.assertEquals(req.mock_calls[2][1][1]['Limit'], 10)
-            self.assertEquals(results_iter.last_evaluated_key, 'z')
+            self.assertEquals(results_iter.last_evaluated_key, {'user_id': items[24]['user_id']})
             self.assertEquals(results_iter.total_count, 30)
             self.assertEquals(results_iter.page_iter.total_scanned_count, 60)