Repository: Kinto/kinto

Issue:
Usage profiles for load tests
Currently the simultation load test runs random actions on the API, with arbitrary percentages.

Could be nice to have like a _presets_ argument that would adjust the actions proportions:
- `kinto.js` : polling for changes and batch create/updates/delete
- Firefox security settings: lots of polling for changes with very rare create/update/delete
- exhaustive: make sure every action is run at least once ([as suggested by @Natim](https://github.com/mozilla-services/cliquet/pull/533#discussion_r43618102))

Related : #201 

Usage profiles for load tests
Currently the simultation load test runs random actions on the API, with arbitrary percentages.

Could be nice to have like a _presets_ argument that would adjust the actions proportions:
- `kinto.js` : polling for changes and batch create/updates/delete
- Firefox security settings: lots of polling for changes with very rare create/update/delete
- exhaustive: make sure every action is run at least once ([as suggested by @Natim](https://github.com/mozilla-services/cliquet/pull/533#discussion_r43618102))

Related : #201 



Discussion:

Resolving patch:
diff --git a/loadtests/loadtest/simulation.py b/loadtests/loadtest/simulation.py
--- a/loadtests/loadtest/simulation.py
+++ b/loadtests/loadtest/simulation.py
@@ -40,9 +40,9 @@ class SimulationLoadTest(BaseLoadTest):
     def __init__(self, *args, **kwargs):
         super(SimulationLoadTest, self).__init__(*args, **kwargs)
         self.collection = 'articles'
-        self.init_article()
+        self.init_user()
 
-    def init_article(self, *args, **kwargs):
+    def init_user(self, *args, **kwargs):
         """Initialization that happens once per user.
 
         :note:
@@ -54,13 +54,12 @@ def init_article(self, *args, **kwargs):
         self.nb_initial_records = random.randint(3, max_initial_records)
         self.batch_requests_size = self.conf.get('batch_requests_size', 25)
 
-    def setUp(self):
-        """Choose some random records in the whole collection.
-
-        :note:
+        self.bucket = os.getenv('BUCKET', self.conf.get('bucket', 'default'))
+        if self.bucket != 'default':
+            # Create bucket + collection:
+            self.collection_url()
 
-            This method is called as many times as number of hits.
-        """
+    def _pickRecords(self):
         resp = self.session.get(self.collection_url())
         self.records = resp.json()['data']
 
@@ -82,25 +81,65 @@ def setUp(self):
         self.random_id_2 = self.random_record_2['id']
         self.random_url_2 = self.record_url(self.random_id_2)
 
-    def test_simulation(self):
-        """Choose a random action among available, if not frequent enough,
-        try again recursively.
+    def setUp(self):
+        """Choose some random records in the whole collection.
 
         :note:
 
             This method is called as many times as number of hits.
         """
-        action, percentage = random.choice(ACTIONS_FREQUENCIES)
+        pass
 
-        forced_action = os.getenv('LOAD_ACTION')
-        if forced_action:
-            action, percentage = forced_action, 101
+    def test_simulation(self):
+        """
+        :note:
 
-        if random.randint(0, 100) < percentage:
+            This method is called as many times as number of hits.
+        """
+        preset = os.getenv('LOAD_PRESET', self.conf.get('preset', 'random'))
+
+        rand = random.randint(0, 100)
+        if preset == 'random':
+            # Choose a random action among available, if not frequent enough,
+            # try again recursively.
+            action, percentage = random.choice(ACTIONS_FREQUENCIES)
+            if rand < percentage:
+                self.incr_counter(action)
+                return getattr(self, action)()
+            else:
+                self.test_simulation()
+
+        elif preset == "exhaustive":
+            # Make sure we exhaustive every action.
+            actions = [a for (a, _) in ACTIONS_FREQUENCIES]
+            for action in actions:
+                getattr(self, action)()
+
+        elif preset == "read":
+            if rand < 2:
+                action = 'batch_create_put'
+            elif rand < 80:
+                action = 'poll_changes'
+            elif rand < 90:
+                action = 'list_deleted'
+            else:
+                action = 'filter_sort'
+            self.incr_counter(action)
+            getattr(self, action)()
+
+        elif preset == "write":
+            if rand < 20:
+                action = 'batch_create'
+            elif rand < 40:
+                action = 'batch_create_put'
+            elif rand < 60:
+                action = 'batch_replace'
+            elif rand < 90:
+                action = 'batch_delete'
+            else:
+                action = 'poll_changes'
             self.incr_counter(action)
-            return getattr(self, action)()
-        else:
-            self.test_simulation()
+            getattr(self, action)()
 
     def create(self):
         article = build_article()
@@ -176,6 +215,7 @@ def _run_batch(self, data):
             self.incr_counter('status-%s' % subresponse['status'])
 
     def update(self):
+        self._pickRecords()
         data = {
             "title": "Some title {}".format(random.randint(0, 1)),
             "archived": bool(random.randint(0, 1)),
@@ -185,6 +225,7 @@ def update(self):
         self._patch(self.random_url, {"data": data})
 
     def batch_replace(self):
+        self._pickRecords()
         data = {
             "defaults": {
                 "method": "PUT"
@@ -200,6 +241,7 @@ def batch_replace(self):
         self._run_batch(data)
 
     def batch_update(self):
+        self._pickRecords()
         data = {
             "defaults": {
                 "method": "PATCH"
@@ -216,39 +258,40 @@ def batch_update(self):
         self._run_batch(data)
 
     def delete(self):
+        self._pickRecords()
         resp = self.session.delete(self.random_url)
         self.incr_counter('status-%s' % resp.status_code)
         self.assertEqual(resp.status_code, 200)
 
     def batch_delete(self):
+        self._pickRecords()
         # Get some random articles on which the batch will be applied
-        url = self.collection_url() + '?_limit=5&_sort=title'
-        resp = self.session.get(url)
-        articles = resp.json()['data']
-        urls = [self.record_url(a['id'], prefix=False)
-                for a in articles]
-
+        nb_deleted = min(self.nb_initial_records, len(self.records))
+        records = random.sample(self.records, nb_deleted)
         data = {
             "defaults": {
                 "method": "DELETE"
             }
         }
-        for i in range(self.batch_requests_size):
-            request = {"path": urls[i % len(urls)]}
+        for record in records[:self.batch_requests_size]:
+            request = {"path": self.record_url(record['id'], prefix=False)}
             data.setdefault("requests", []).append(request)
 
         self._run_batch(data)
 
     def poll_changes(self):
+        self._pickRecords()
         last_modified = self.random_record['last_modified']
         filters = '?_since=%s' % last_modified
         modified_url = self.collection_url() + filters
         resp = self.session.get(modified_url)
+        self.incr_counter('status-%s' % resp.status_code)
         self.assertEqual(resp.status_code, 200)
 
     def list_archived(self):
         archived_url = self.collection_url() + '?archived=true'
         resp = self.session.get(archived_url)
+        self.incr_counter('status-%s' % resp.status_code)
         self.assertEqual(resp.status_code, 200)
 
     def batch_count(self):
@@ -268,26 +311,20 @@ def batch_count(self):
         self._run_batch(data)
 
     def list_deleted(self):
+        self._pickRecords()
         modif = self.random_record['last_modified']
         filters = '?_since=%s&deleted=true' % modif
         deleted_url = self.collection_url() + filters
         resp = self.session.get(deleted_url)
+        self.incr_counter('status-%s' % resp.status_code)
         self.assertEqual(resp.status_code, 200)
 
     def list_continuated_pagination(self):
         paginated_url = self.collection_url() + '?_limit=20'
-
-        urls = []
-
         while paginated_url:
             resp = self.session.get(paginated_url)
+            self.incr_counter('status-%s' % resp.status_code)
             self.assertEqual(resp.status_code, 200)
             next_page = resp.headers.get("Next-Page")
-            if next_page is not None and next_page not in urls:
-                self.assertNotEqual(paginated_url, next_page)
-                paginated_url = next_page
-                urls.append(next_page)
-            else:
-                # XXX: we shouldn't have to keep the full list.
-                # See mozilla-services/cliquet#366
-                break
+            self.assertNotEqual(paginated_url, next_page)
+            paginated_url = next_page
diff --git a/loadtests/loadtest/tutorial.py b/loadtests/loadtest/tutorial.py
--- a/loadtests/loadtest/tutorial.py
+++ b/loadtests/loadtest/tutorial.py
@@ -138,13 +138,10 @@ def play_user_shared_bucket_tutorial(self):
         collection_url = self.collection_url(bucket_id, collection_id)
 
         # Create a new bucket and check for permissions
-        resp = self.session.put(
-            self.bucket_url(bucket_id))
+        resp = self.session.put(self.bucket_url(bucket_id))
         self.incr_counter("status-%s" % resp.status_code)
-        # We should always have a 201 here. See mozilla-services/cliquet#367
-        # if resp.status_code == 200:
-        #     import pdb; pdb.set_trace()
-        # self.assertEqual(resp.status_code, 201)
+        # In case of concurrent execution, it may have been created already.
+        self.assertIn(resp.status_code, (200, 201))
         record = resp.json()
         self.assertIn('write', record['permissions'])
 
@@ -154,10 +151,8 @@ def play_user_shared_bucket_tutorial(self):
             re.sub('/records$', '', collection_url),
             data=json.dumps({'permissions': permissions}))
         self.incr_counter("status-%s" % resp.status_code)
-        # We should always have a 201 here. See mozilla-services/cliquet#367
-        # if resp.status_code == 200:
-        #     import pdb; pdb.set_trace()
-        # self.assertEqual(resp.status_code, 201)
+        # In case of concurrent execution, it may have been created already.
+        self.assertIn(resp.status_code, (200, 201))
         record = resp.json()
         self.assertIn('record:create', record['permissions'])
         self.assertIn('system.Authenticated',
diff --git a/loadtests/simulation.py b/loadtests/simulation.py
deleted file mode 100644
--- a/loadtests/simulation.py
+++ /dev/null
@@ -1,318 +0,0 @@
-import json
-import os
-import random
-import uuid
-
-from konfig import Config
-from loads.case import TestCase
-from requests.auth import HTTPBasicAuth
-
-
-ACTIONS_FREQUENCIES = [
-    ('create', 20),
-    ('create_put', 20),
-    ('batch_create', 50),
-    ('batch_create_put', 50),
-    ('batch_replace', 50),
-    ('batch_update', 50),
-    ('update', 50),
-    ('filter_sort', 60),
-    ('delete', 10),
-    ('batch_delete', 10),
-    ('poll_changes', 90),
-    ('list_deleted', 40),
-    ('batch_count', 50),
-    ('list_continuated_pagination', 80),
-]
-
-
-def build_record():
-    suffix = uuid.uuid4().hex
-    data = {
-        "name": "Mushroom {0}".format(suffix),
-        "editable": (random.randint(0, 1) == 0),
-        "size": random.randint(0, 10),
-    }
-    return data
-
-
-class SimulationLoadTest(TestCase):
-
-    collection = 'psilos'
-
-    def __init__(self, *args, **kwargs):
-        """Initialization that happens once per user.
-
-        :note:
-
-            This method is called as many times as number of users.
-        """
-        super(SimulationLoadTest, self).__init__(*args, **kwargs)
-
-        self.conf = self._get_configuration()
-
-        self.random_user = uuid.uuid4().hex
-        self.auth = HTTPBasicAuth(self.random_user, 'secret')
-        self.session.auth = self.auth
-        self.session.headers.update({'Content-Type': 'application/json'})
-
-        # Create at least some records for this user
-        max_initial_records = self.conf.get('max_initial_records', 100)
-        self.nb_initial_records = random.randint(3, max_initial_records)
-        self.batch_requests_size = self.conf.get('batch_requests_size', 25)
-
-    def _get_configuration(self):
-        # Loads is removing the extra information contained in the ini files,
-        # so we need to parse it again.
-        config_file = self.config['config']
-        # When copying the configuration files, we lose the config/ prefix so,
-        # try to read from this folder in case the file doesn't exist.
-        if not os.path.isfile(config_file):
-            config_file = os.path.basename(config_file)
-            if not os.path.isfile(config_file):
-                msg = 'Unable to locate the configuration file, aborting.'
-                raise LookupError(msg)
-        return Config(config_file).get_map('loads')
-
-    def api_url(self, path):
-        url = "{0}/v0/{1}".format(self.server_url.rstrip('/'), path)
-        return url
-
-    def collection_url(self, prefix=True):
-        if prefix:
-            return self.api_url(self.collection)
-        return '/' + self.collection
-
-    def record_url(self, record_id, prefix=True):
-        return self.collection_url(prefix) + '/%s' % record_id
-
-    def setUp(self):
-        """Choose some random records in the whole collection.
-
-        :note:
-
-            This method is called as many times as number of hits.
-        """
-        resp = self.session.get(self.collection_url())
-        self.incr_counter(resp.status_code)
-        self.records = resp.json()['data']
-
-        # Create some records, if not any in collection.
-        missing = self.nb_initial_records - len(self.records)
-        if missing > 0:
-            self.batch_create(size=missing)
-            resp = self.session.get(self.collection_url())
-            self.records = resp.json()['data']
-
-        # Pick a random record
-        self.random_record = random.choice(self.records)
-        self.random_id = self.random_record['id']
-        self.random_url = self.record_url(self.random_id)
-
-        # Pick another random, different
-        self.records.remove(self.random_record)
-        self.random_record_2 = random.choice(self.records)
-        self.random_id_2 = self.random_record_2['id']
-        self.random_url_2 = self.record_url(self.random_id_2)
-
-    def test_simulation(self):
-        """Choose a random action among available, if not frequent enough,
-        try again recursively.
-
-        :note:
-
-            This method is called as many times as number of hits.
-        """
-        action, percentage = random.choice(ACTIONS_FREQUENCIES)
-
-        forced_action = self.conf.get('load_action',
-                                      os.getenv('LOAD_ACTION'))
-        if forced_action:
-            action, percentage = forced_action, 101
-
-        if random.randint(0, 100) < percentage:
-            self.incr_counter(action)
-            return getattr(self, action)()
-        else:
-            self.test_simulation()
-
-    def create(self):
-        record = build_record()
-        resp = self.session.post(
-            self.collection_url(),
-            data=json.dumps({'data': record}),
-            headers={'Content-Type': 'application/json'})
-        self.incr_counter(resp.status_code)
-        self.assertEqual(resp.status_code, 201)
-
-    def create_put(self):
-        record = build_record()
-        resp = self.session.put(
-            self.record_url(uuid.uuid4()),
-            data=json.dumps({'data': record}),
-            headers={'Content-Type': 'application/json'})
-        self.incr_counter(resp.status_code)
-        self.assertEqual(resp.status_code, 201)
-
-    def batch_create(self, size=None):
-        data = {
-            "defaults": {
-                "method": "POST",
-                "path": self.collection_url(prefix=False)
-            }
-        }
-        for i in range(size or self.batch_requests_size):
-            request = {"body": {"data": build_record()}}
-            data.setdefault("requests", []).append(request)
-
-        self._run_batch(data)
-
-    def batch_create_put(self):
-        data = {
-            "defaults": {
-                "method": "PUT",
-            }
-        }
-        for i in range(self.batch_requests_size):
-            path = self.record_url(uuid.uuid4(), prefix=False)
-            request = {"path": path, "body": {"data": build_record()}}
-            data.setdefault("requests", []).append(request)
-
-        self._run_batch(data)
-
-    def filter_sort(self):
-        queries = [
-            [('edible', 'false')],
-            [('edible', 'true'), ('min_size', '3')],
-            [('_sort', '-last_modified'), ('max_size', '5')],
-            [('_sort', 'name')],
-            [('_sort', '-size,-name'), ('edible', 'false')],
-        ]
-        queryparams = random.choice(queries)
-        query_url = '&'.join(['='.join(param) for param in queryparams])
-        url = self.collection_url() + '?' + query_url
-        resp = self.session.get(url)
-        self.incr_counter(resp.status_code)
-        self.assertEqual(resp.status_code, 200)
-
-    def _patch(self, url, data, status=200):
-        data = json.dumps(data)
-        resp = self.session.patch(url, data)
-        self.incr_counter(resp.status_code)
-        self.assertEqual(resp.status_code, status)
-
-    def _run_batch(self, data):
-        resp = self.session.post(self.api_url('batch'),
-                                 data=json.dumps(data))
-        self.incr_counter(resp.status_code)
-        self.assertEqual(resp.status_code, 200)
-        for subresponse in resp.json()['responses']:
-            self.incr_counter(subresponse['status'])
-
-    def update(self):
-        data = {
-            "name": "Some title {}".format(random.randint(0, 1)),
-            "edible": bool(random.randint(0, 1)),
-            "size": random.randint(10, 20),
-        }
-        self._patch(self.random_url, {"data": data})
-
-    def batch_replace(self):
-        data = {
-            "defaults": {
-                "method": "PUT"
-            }
-        }
-        nb_batched = min(self.nb_initial_records, len(self.records))
-        records = random.sample(self.records, nb_batched)
-        for record in records:
-            record_url = self.record_url(record['id'], prefix=False)
-            request = {"path": record_url, "body": {"data": build_record()}}
-            data.setdefault("requests", []).append(request)
-
-        self._run_batch(data)
-
-    def batch_update(self):
-        data = {
-            "defaults": {
-                "method": "PATCH"
-            }
-        }
-        nb_batched = min(self.nb_initial_records, len(self.records))
-        records = random.sample(self.records, nb_batched)
-        for record in records:
-            record_url = self.record_url(record['id'], prefix=False)
-            change = {"name": "Name %s" % random.randint(0, 100)}
-            request = {"path": record_url, "body": {"data": change}}
-            data.setdefault("requests", []).append(request)
-
-        self._run_batch(data)
-
-    def delete(self):
-        resp = self.session.delete(self.random_url)
-        self.incr_counter(resp.status_code)
-        self.assertEqual(resp.status_code, 200)
-
-    def batch_delete(self):
-        # Get some random records on which the batch will be applied
-        url = self.collection_url() + '?_limit=5&_sort=name'
-        resp = self.session.get(url)
-        records = resp.json()['data']
-        urls = [self.record_url(a['id'], prefix=False)
-                for a in records]
-
-        data = {
-            "defaults": {
-                "method": "DELETE"
-            }
-        }
-        for i in range(self.batch_requests_size):
-            request = {"path": urls[i % len(urls)]}
-            data.setdefault("requests", []).append(request)
-
-        self._run_batch(data)
-
-    def poll_changes(self):
-        last_modified = self.random_record['last_modified']
-        filters = '?_since=%s' % last_modified
-        modified_url = self.collection_url() + filters
-        resp = self.session.get(modified_url)
-        self.assertEqual(resp.status_code, 200)
-
-    def batch_count(self):
-        base_url = self.collection_url(prefix=False)
-        data = {
-            "defaults": {
-                "method": "HEAD",
-            },
-            "requests": [
-                {"path": base_url + "?edible=true"},
-                {"path": base_url + "?min_size=5"}
-            ]
-        }
-        self._run_batch(data)
-
-    def list_deleted(self):
-        modif = self.random_record['last_modified']
-        filters = '?_since=%s&deleted=true' % modif
-        deleted_url = self.collection_url() + filters
-        resp = self.session.get(deleted_url)
-        self.assertEqual(resp.status_code, 200)
-
-    def list_continuated_pagination(self):
-        paginated_url = self.collection_url() + '?_limit=20'
-
-        urls = []
-
-        while paginated_url:
-            resp = self.session.get(paginated_url)
-            self.assertEqual(resp.status_code, 200)
-            next_page = resp.headers.get("Next-Page")
-            if next_page is not None and next_page not in urls:
-                self.assertNotEqual(paginated_url, next_page)
-                paginated_url = next_page
-                urls.append(next_page)
-            else:
-                # XXX: we shouldn't have to keep the full list.
-                # See mozilla-services/cliquet#366
-                break


Test patch:
diff --git a/loadtests/config/test.ini b/loadtests/config/test.ini
--- a/loadtests/config/test.ini
+++ b/loadtests/config/test.ini
@@ -4,3 +4,6 @@ users = 1
 max_initial_records = 25
 no-dns-resolve = true
 include_file = loadtest
+
+bucket = blog
+preset = read
diff --git a/loadtests/test.ini b/loadtests/test.ini
deleted file mode 100644
--- a/loadtests/test.ini
+++ /dev/null
@@ -1,12 +0,0 @@
-[loads]
-# Number of parallel users
-users = 3
-
-# Limit tests by number of hits
-hits = 30
-# Or by duration (seconds)
-# duration = 600
-
-max_initial_records = 25
-no-dns-resolve = true
-include_file = loadtest