Repository: pyslackers/slack-sansio

Issue:
Rate limit
Slack has a rate limit of 1 query / second. When using the iteration mode for paginated request it's easy to get rate limited.

We should add a way to ensure x seconds have passed between multiple api call using the iteration mode.

Discussion:
coveralls: 
[![Coverage Status](https://coveralls.io/builds/14970855/badge)](https://coveralls.io/builds/14970855)

Coverage increased (+0.1%) to 90.765% when pulling **1087375ff50f2a8db607c880a465723b8f58d458 on rate-limit-wait** into **28b1c6928d086f8a4d2322a207ac99faf19a6839 on master**.


Patch:
diff --git a/README.rst b/README.rst
index 7578d2f..fa28ba4 100644
--- a/README.rst
+++ b/README.rst
@@ -73,6 +73,8 @@ Changelog
 dev
 ```
 
+* Add ``minimum_time`` argument to ``SlackAPI.iter`` in order to force a minimum elapsed time between two call to the API
+
 0.3.2
 `````
 
diff --git a/slack/io/abc.py b/slack/io/abc.py
index 4a5ea84..7332345 100644
--- a/slack/io/abc.py
+++ b/slack/io/abc.py
@@ -80,7 +80,7 @@ class SlackAPI(abc.ABC):
                                                     global_headers=self._headers, token=self._token)
         return await self._make_query(url, body, headers)
 
-    async def iter(self, url, data=None, headers=None, *, limit=200, iterkey=None, itermode=None):
+    async def iter(self, url, data=None, headers=None, *, limit=200, iterkey=None, itermode=None, minimum_time=None):
         """
         Iterate over a slack API method supporting pagination
 
@@ -91,6 +91,8 @@ class SlackAPI(abc.ABC):
             limit: Maximum number of results to return per call.
             iterkey: Key in response data to iterate over (required for url string).
             itermode: Iteration mode (required for url string) (one of `cursor`, `page` or `timeline`)
+            minimum_time: Minimum elapsed time (in seconds) between two calls to the Slack API (default to 0).
+             If not reached the client will sleep for the remaining time.
 
         Returns:
             Async iterator over `response_data[key]`
@@ -101,9 +103,15 @@ class SlackAPI(abc.ABC):
         if not data:
             data = {}
 
+        last_request_time = None
         while True:
+            current_time = time.time()
+            if minimum_time and last_request_time and last_request_time + minimum_time > current_time:
+                await self.sleep(last_request_time + minimum_time - current_time)
+
             data, iterkey, itermode = sansio.prepare_iter_request(url, data, iterkey=iterkey, itermode=itermode,
                                                                   limit=limit, itervalue=itervalue)
+            last_request_time = time.time()
             response_data = await self.query(url, data, headers)
             itervalue = sansio.decode_iter_request(response_data)
             for item in response_data[iterkey]:
diff --git a/slack/io/requests.py b/slack/io/requests.py
index 5e0a87a..c384053 100644
--- a/slack/io/requests.py
+++ b/slack/io/requests.py
@@ -76,7 +76,7 @@ class SlackAPI(abc.SlackAPI):
                                                     global_headers=self._headers, token=self._token)
         return self._make_query(url, body, headers)
 
-    def iter(self, url, data=None, headers=None, *, limit=200, iterkey=None, itermode=None):
+    def iter(self, url, data=None, headers=None, *, limit=200, iterkey=None, itermode=None, minimum_time=None):
         """
         Iterate over a slack API method supporting pagination
 
@@ -87,6 +87,8 @@ class SlackAPI(abc.SlackAPI):
             limit: Maximum number of results to return per call.
             iterkey: Key in response data to iterate over (required for url string).
             itermode: Iteration mode (required for url string) (one of `cursor`, `page` or `timeline`)
+            minimum_time: Minimum elapsed time (in seconds) between two calls to the Slack API (default to 0).
+             If not reached the client will sleep for the remaining time.
 
         Returns:
             Async iterator over `response_data[key]`
@@ -97,9 +99,15 @@ class SlackAPI(abc.SlackAPI):
         if not data:
             data = {}
 
+        last_request_time = None
         while True:
+            current_time = time.time()
+            if minimum_time and last_request_time and last_request_time + minimum_time > current_time:
+                self.sleep(last_request_time + minimum_time - current_time)
+
             data, iterkey, itermode = sansio.prepare_iter_request(url, data, iterkey=iterkey, itermode=itermode,
                                                                   limit=limit, itervalue=itervalue)
+            last_request_time = time.time()
             response_data = self.query(url, data, headers)
             itervalue = sansio.decode_iter_request(response_data)
             for item in response_data[iterkey]:


Test patch:
diff --git a/tests/test_io.py b/tests/test_io.py
index 6ff909d..fc01c8c 100644
--- a/tests/test_io.py
+++ b/tests/test_io.py
@@ -1,9 +1,11 @@
 import json
 import asks
 import trio
+import time
 import slack
 import curio
 import pytest
+import asyncio
 import aiohttp
 import requests
 import datetime
@@ -166,6 +168,40 @@ class TestABC:
             async for _ in client.iter('https://slack.com/api/channels.list'):  # noQa: F841
                 pass
 
+    @pytest.mark.parametrize('client', ({'retry_when_rate_limit': False,
+                                         '_request': [
+                                             {'body': 'channels_iter'},
+                                             {'body': 'channels'}
+                                         ]}, ), indirect=True)
+    async def test_iter_wait(self, client):
+        client.sleep = asynctest.CoroutineMock()
+
+        channels = 0
+        async for _ in client.iter(methods.CHANNELS_LIST, minimum_time=2):  # noQa: F841
+            channels += 1
+
+        assert channels == 4
+        assert client._request.call_count == 2
+        assert client.sleep.call_count == 1
+        assert 2 > client.sleep.call_args[0][0] > 1.9
+
+    @pytest.mark.parametrize('client', ({'retry_when_rate_limit': False,
+                                         '_request': [
+                                             {'body': 'channels_iter'},
+                                             {'body': 'channels'}
+                                         ]}, ), indirect=True)
+    async def test_iter_no_wait(self, client):
+        client.sleep = asynctest.CoroutineMock()
+
+        channels = 0
+        async for _ in client.iter(methods.CHANNELS_LIST, minimum_time=1):  # noQa: F841
+            channels += 1
+            await asyncio.sleep(0.5)
+
+        assert channels == 4
+        assert client._request.call_count == 2
+        assert client.sleep.call_count == 0
+
     @pytest.mark.parametrize('client', ({'_request': [{'body': 'auth.test'}, {'body': 'users.info'}]}, ), indirect=True)
     async def test_find_bot_id(self, client):
         bot_id = await client._find_bot_id()
@@ -314,6 +350,40 @@ class TestNoAsync:
             'POST', 'https://slack.com/api/channels.list', {}, {'limit': 200, 'token': token, 'cursor': itercursor}
         )
 
+    @pytest.mark.parametrize('client', ({'retry_when_rate_limit': False,
+                                         '_request': [
+                                             {'body': 'channels_iter'},
+                                             {'body': 'channels'}
+                                         ]}, ), indirect=True)
+    def test_iter_wait(self, client):
+        client.sleep = asynctest.CoroutineMock()
+
+        channels = 0
+        for _ in client.iter(methods.CHANNELS_LIST, minimum_time=2):  # noQa: F841
+            channels += 1
+
+        assert channels == 4
+        assert client._request.call_count == 2
+        assert client.sleep.call_count == 1
+        assert 2 > client.sleep.call_args[0][0] > 1.9
+
+    @pytest.mark.parametrize('client', ({'retry_when_rate_limit': False,
+                                         '_request': [
+                                             {'body': 'channels_iter'},
+                                             {'body': 'channels'}
+                                         ]}, ), indirect=True)
+    def test_iter_no_wait(self, client):
+        client.sleep = asynctest.CoroutineMock()
+
+        channels = 0
+        for _ in client.iter(methods.CHANNELS_LIST, minimum_time=1):  # noQa: F841
+            channels += 1
+            time.sleep(0.5)
+
+        assert channels == 4
+        assert client._request.call_count == 2
+        assert client.sleep.call_count == 0
+
     @pytest.mark.parametrize('client', ({'_request': [{'body': 'auth.test'}, {'body': 'users.info'}]}, ), indirect=True)
     def test_find_bot_id(self, client):
         bot_id = client._find_bot_id()