Repository: gratipay/gratipay.com

Issue:
fake_data should result in sane balances
A database filled with fake_data fails our self check for sane balances, because there aren't transfers and exchanges equal to each balance. This results in a traceback the first time a new contributor launches Gittip, which is a drag.

```
pid-16504 thread-140735191843600 (MainThread)   show_tracebacks        True                           environment variable ASPEN_SHOW_TRACEBACKS=yes
pid-16504 thread-140735191843600 (MainThread)   www_root               None                           default                 
pid-16504 thread-140735191843600 (MainThread)   www_root               www/                           environment variable ASPEN_WWW_ROOT=www/
pid-16504 thread-140735191843600 (MainThread) project_root is relative to CWD: '.'.
pid-16504 thread-140735191843600 (MainThread) project_root set to /Users/whit537/personal/gittip/www.gittip.com.
pid-16504 thread-140735191843600 (MainThread) Found plugin for renderer 'jinja2'
pid-16504 thread-140735191843600 (MainThread) Won't log to Sentry (SENTRY_DSN is empty).
pid-16504 thread-140735191843600 (MainThread) Renderers (*ed are unavailable, CAPS is default):
pid-16504 thread-140735191843600 (MainThread)   stdlib_percent   
pid-16504 thread-140735191843600 (MainThread)   json_dump        
pid-16504 thread-140735191843600 (MainThread)   stdlib_format    
pid-16504 thread-140735191843600 (MainThread)   JINJA2           
pid-16504 thread-140735191843600 (MainThread)   stdlib_template  
pid-16504 thread-4359208960 (Thread-2) Traceback (most recent call last):
pid-16504 thread-4359208960 (Thread-2)   File "/Users/whit537/personal/gittip/www.gittip.com/configure-aspen.py", line 79, in f
pid-16504 thread-4359208960 (Thread-2)     func()
pid-16504 thread-4359208960 (Thread-2)   File "/Users/whit537/personal/gittip/www.gittip.com/gittip/models/__init__.py", line 17, in self_check
pid-16504 thread-4359208960 (Thread-2)     check_db(cursor)
pid-16504 thread-4359208960 (Thread-2)   File "/Users/whit537/personal/gittip/www.gittip.com/gittip/models/__init__.py", line 23, in check_db
pid-16504 thread-4359208960 (Thread-2)     _check_balances(cursor)
pid-16504 thread-4359208960 (Thread-2)   File "/Users/whit537/personal/gittip/www.gittip.com/gittip/models/__init__.py", line 96, in _check_balances
pid-16504 thread-4359208960 (Thread-2)     assert b == 0, "conflicting balances: {}".format(b)
pid-16504 thread-4359208960 (Thread-2) AssertionError: conflicting balances: 101
```

<bountysource-plugin>

---

Want to back this issue? **[Place a bounty on it!](https://www.bountysource.com/issues/3403835-fake_data-should-result-in-sane-balances?utm_campaign=plugin&utm_content=tracker%2F85909&utm_medium=issues&utm_source=github)** We accept bounties via [Bountysource](https://www.bountysource.com/?utm_campaign=plugin&utm_content=tracker%2F85909&utm_medium=issues&utm_source=github).
</bountysource-plugin>



Discussion:
This is also a blocker for running payday against fake_data. [IRC](https://botbot.me/freenode/gittip/2014-07-30/?msg=18865568&page=1)


Resolving patch:
diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py
--- a/gratipay/utils/fake_data.py
+++ b/gratipay/utils/fake_data.py
@@ -1,18 +1,20 @@
+import datetime
+from decimal import Decimal as D
+import random
+import string
+import sys
+
 from faker import Factory
+from psycopg2 import IntegrityError
+
 from gratipay import wireup, MAX_TIP_SINGULAR, MIN_TIP
 from gratipay.elsewhere import PLATFORMS
 from gratipay.models.participant import Participant
 from gratipay.models import community
-
-import datetime
-import decimal
-import random
-import string
-
+from gratipay.models import check_db
 
 faker = Factory.create()
 
-
 def _fake_thing(db, tablename, **kw):
     column_names = []
     column_value_placeholders = []
@@ -39,18 +41,6 @@ def fake_text_id(size=6, chars=string.ascii_lowercase + string.digits):
     return ''.join(random.choice(chars) for x in range(size))
 
 
-def fake_balance(max_amount=100):
-    """Return a random amount between 0 and max_amount.
-    """
-    return random.random() * max_amount
-
-
-def fake_int_id(nmax=2 ** 31 -1):
-    """Create a random int id.
-    """
-    return random.randint(0, nmax)
-
-
 def fake_sentence(start=1, stop=100):
     """Create a sentence of random length.
     """
@@ -61,24 +51,26 @@ def fake_participant(db, number="singular", is_admin=False):
     """Create a fake User.
     """
     username = faker.first_name() + fake_text_id(3)
-    _fake_thing( db
-               , "participants"
-               , id=fake_int_id()
-               , username=username
-               , username_lower=username.lower()
-               , ctime=faker.date_time_this_year()
-               , is_admin=is_admin
-               , balance=fake_balance()
-               , anonymous_giving=(random.randrange(5) == 0)
-               , anonymous_receiving=(number != 'plural' and random.randrange(5) == 0)
-               , goal=fake_balance()
-               , balanced_customer_href=faker.uri()
-               , last_ach_result=''
-               , is_suspicious=False
-               , last_bill_result=''  # Needed to not be suspicious
-               , claimed_time=faker.date_time_this_year()
-               , number=number
-                )
+    try:
+        _fake_thing( db
+                   , "participants"
+                   , username=username
+                   , username_lower=username.lower()
+                   , ctime=faker.date_time_this_year()
+                   , is_admin=is_admin
+                   , balance=0
+                   , anonymous_giving=(random.randrange(5) == 0)
+                   , anonymous_receiving=(number != 'plural' and random.randrange(5) == 0)
+                   , balanced_customer_href=faker.uri()
+                   , last_ach_result=''
+                   , is_suspicious=False
+                   , last_bill_result=''  # Needed to not be suspicious
+                   , claimed_time=faker.date_time_this_year()
+                   , number=number
+                    )
+    except IntegrityError:
+      return fake_participant(db, number, is_admin)
+
     #Call participant constructor to perform other DB initialization
     return Participant.from_username(username)
 
@@ -98,13 +90,13 @@ def fake_community(db, creator):
 
 
 def fake_tip_amount():
-    amount = ((decimal.Decimal(random.random()) * (MAX_TIP_SINGULAR - MIN_TIP))
+    amount = ((D(random.random()) * (MAX_TIP_SINGULAR - MIN_TIP))
             + MIN_TIP)
 
-    decimal_amount = decimal.Decimal(amount).quantize(decimal.Decimal('.01'))
-    while decimal_amount == decimal.Decimal('0.00'):
+    decimal_amount = D(amount).quantize(D('.01'))
+    while decimal_amount == D('0.00'):
         # https://github.com/gratipay/gratipay.com/issues/2950
-        decimal_amount = fake_tip_amount() 
+        decimal_amount = fake_tip_amount()
     return decimal_amount
 
 
@@ -112,14 +104,13 @@ def fake_tip(db, tipper, tippee):
     """Create a fake tip.
     """
     return _fake_thing( db
-               , "tips"
-               , id=fake_int_id()
-               , ctime=faker.date_time_this_year()
-               , mtime=faker.date_time_this_month()
-               , tipper=tipper.username
-               , tippee=tippee.username
-               , amount=fake_tip_amount()
-                )
+                      , "tips"
+                      , ctime=faker.date_time_this_year()
+                      , mtime=faker.date_time_this_month()
+                      , tipper=tipper.username
+                      , tippee=tippee.username
+                      , amount=fake_tip_amount()
+                       )
 
 
 def fake_elsewhere(db, participant, platform):
@@ -127,7 +118,6 @@ def fake_elsewhere(db, participant, platform):
     """
     _fake_thing( db
                , "elsewhere"
-               , id=fake_int_id()
                , platform=platform
                , user_id=fake_text_id()
                , user_name=participant.username
@@ -137,41 +127,134 @@ def fake_elsewhere(db, participant, platform):
 
 
 def fake_transfer(db, tipper, tippee):
-        return _fake_thing( db
-               , "transfers"
-               , id=fake_int_id()
-               , timestamp=faker.date_time_this_year()
-               , tipper=tipper.username
-               , tippee=tippee.username
-               , amount=fake_tip_amount()
-               , context='tip'
-                )
+    return _fake_thing( db
+           , "transfers"
+           , timestamp=faker.date_time_this_year()
+           , tipper=tipper.username
+           , tippee=tippee.username
+           , amount=fake_tip_amount()
+           , context='tip'
+            )
+
+def fake_exchange(db, participant, amount, fee, timestamp):
+    return _fake_thing( db
+                      , "exchanges"
+                      , timestamp=timestamp
+                      , participant=participant.username
+                      , amount=amount
+                      , fee=fee
+                      , status='succeeded'
+                       )
+
+def prep_db(db):
+    db.run("""
+        CREATE OR REPLACE FUNCTION process_transfer() RETURNS trigger AS $$
+            BEGIN
+                UPDATE participants
+                   SET balance = balance + NEW.amount
+                 WHERE username = NEW.tippee;
+
+                UPDATE participants
+                   SET balance = balance - NEW.amount
+                 WHERE username = NEW.tipper;
+
+                RETURN NULL;
+            END;
+        $$ language plpgsql;
+
+        CREATE TRIGGER process_transfer AFTER INSERT ON transfers
+            FOR EACH ROW EXECUTE PROCEDURE process_transfer();
+
+        CREATE OR REPLACE FUNCTION process_exchange() RETURNS trigger AS $$
+            BEGIN
+                IF NEW.amount > 0 THEN
+                    UPDATE participants
+                       SET balance = balance + NEW.amount
+                     WHERE username = NEW.participant;
+                ELSE
+                    UPDATE participants
+                       SET balance = balance + NEW.amount - NEW.fee
+                     WHERE username = NEW.participant;
+                END IF;
+
+                RETURN NULL;
+            END;
+        $$ language plpgsql;
+
+        CREATE TRIGGER process_exchange AFTER INSERT ON exchanges
+            FOR EACH ROW EXECUTE PROCEDURE process_exchange();
+
+        CREATE OR REPLACE FUNCTION process_payday() RETURNS trigger AS $$
+            BEGIN
+                SELECT COALESCE(SUM(amount+fee), 0)
+                  FROM exchanges
+                 WHERE timestamp > NEW.ts_start
+                   AND timestamp < NEW.ts_end
+                   AND amount > 0
+                  INTO NEW.charge_volume;
+
+                SELECT COALESCE(SUM(fee), 0)
+                  FROM exchanges
+                 WHERE timestamp > NEW.ts_start
+                   AND timestamp < NEW.ts_end
+                   AND amount > 0
+                  INTO NEW.charge_fees_volume;
+
+                SELECT COALESCE(SUM(amount), 0)
+                  FROM exchanges
+                 WHERE timestamp > NEW.ts_start
+                   AND timestamp < NEW.ts_end
+                   AND amount < 0
+                  INTO NEW.ach_volume;
+
+                SELECT COALESCE(SUM(fee), 0)
+                  FROM exchanges
+                 WHERE timestamp > NEW.ts_start
+                   AND timestamp < NEW.ts_end
+                   AND amount < 0
+                  INTO NEW.ach_fees_volume;
+
+                RETURN NEW;
+            END;
+        $$ language plpgsql;
+
+        CREATE TRIGGER process_payday BEFORE INSERT ON paydays
+            FOR EACH ROW EXECUTE PROCEDURE process_payday();
+    """)
+
+def clean_db(db):
+    db.run("""
+        DROP FUNCTION process_transfer() CASCADE;
+        DROP FUNCTION process_exchange() CASCADE;
+        DROP FUNCTION process_payday() CASCADE;
+    """)
 
 
 def populate_db(db, num_participants=100, num_tips=200, num_teams=5, num_transfers=5000, num_communities=20):
     """Populate DB with fake data.
     """
-    #Make the participants
+    print("Making Participants")
     participants = []
     for i in xrange(num_participants):
         participants.append(fake_participant(db))
 
-    #Make the "Elsewhere's"
+    print("Making Teams")
+    for i in xrange(num_teams):
+        team = fake_participant(db, number="plural")
+        #Add 1 to 3 members to the team
+        members = random.sample(participants, random.randint(1, 3))
+        for p in members:
+            team.add_member(p)
+        participants.append(team)
+
+    print("Making Elsewheres")
     for p in participants:
         #All participants get between 1 and 3 elsewheres
         num_elsewheres = random.randint(1, 3)
         for platform_name in random.sample(PLATFORMS, num_elsewheres):
             fake_elsewhere(db, p, platform_name)
 
-    #Make teams
-    for i in xrange(num_teams):
-        t = fake_participant(db, number="plural")
-        #Add 1 to 3 members to the team
-        members = random.sample(participants, random.randint(1, 3))
-        for p in members:
-            t.add_member(p)
-
-    #Make communities
+    print("Making Communities")
     for i in xrange(num_communities):
         creator = random.sample(participants, 1)
         community = fake_community(db, creator[0])
@@ -180,33 +263,55 @@ def populate_db(db, num_participants=100, num_tips=200, num_teams=5, num_transfe
         for p in members:
             p.insert_into_communities(True, community.name, community.slug)
 
-    #Make the tips
+    print("Making Tips")
     tips = []
     for i in xrange(num_tips):
         tipper, tippee = random.sample(participants, 2)
         tips.append(fake_tip(db, tipper, tippee))
 
-
-    #Make the transfers
+    # Transfers
     transfers = []
     for i in xrange(num_transfers):
+        sys.stdout.write("\rMaking Transfers (%i/%i)" % (i+1, num_transfers))
+        sys.stdout.flush()
         tipper, tippee = random.sample(participants, 2)
         transfers.append(fake_transfer(db, tipper, tippee))
+    print("")
 
-
-    #Make some paydays
-    #First determine the boundaries - min and max date
+    # Paydays
+    # First determine the boundaries - min and max date
     min_date = min(min(x['ctime'] for x in tips), \
                    min(x['timestamp'] for x in transfers))
     max_date = max(max(x['ctime'] for x in tips), \
                    max(x['timestamp'] for x in transfers))
-    #iterate through min_date, max_date one week at a time
+    # iterate through min_date, max_date one week at a time
+    payday_counter = 1
     date = min_date
+    paydays_total = (max_date - min_date).days/7 + 1
     while date < max_date:
+        sys.stdout.write("\rMaking Paydays (%i/%i)" % (payday_counter, paydays_total))
+        sys.stdout.flush()
+        payday_counter += 1
         end_date = date + datetime.timedelta(days=7)
-        week_tips = filter(lambda x: date <= x['ctime'] <= end_date, tips)
-        week_transfers = filter(lambda x: date <= x['timestamp'] <= end_date, transfers)
-        week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) <= end_date, participants)
+        week_tips = filter(lambda x: date < x['ctime'] < end_date, tips)
+        week_transfers = filter(lambda x: date < x['timestamp'] < end_date, transfers)
+        week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) < end_date, participants)
+        for p in week_participants:
+            transfers_in = filter(lambda x: x['tippee'] == p.username, week_transfers)
+            transfers_out = filter(lambda x: x['tipper'] == p.username, week_transfers)
+            amount_in = sum([t['amount'] for t in transfers_in])
+            amount_out = sum([t['amount'] for t in transfers_out])
+            amount = amount_out - amount_in
+            if amount != 0:
+                fee = amount * D('0.02')
+                fee = abs(fee.quantize(D('.01')))
+                fake_exchange(
+                    db=db,
+                    participant=p,
+                    amount=amount,
+                    fee=fee,
+                    timestamp=(end_date - datetime.timedelta(seconds=1))
+                )
         actives=set()
         tippers=set()
         for xfers in week_tips, week_transfers:
@@ -214,7 +319,6 @@ def populate_db(db, num_participants=100, num_tips=200, num_teams=5, num_transfe
             actives.update(x['tippee'] for x in xfers)
             tippers.update(x['tipper'] for x in xfers)
         payday = {
-            'id': fake_int_id(),
             'ts_start': date,
             'ts_end': end_date,
             'ntips': len(week_tips),
@@ -224,18 +328,17 @@ def populate_db(db, num_participants=100, num_tips=200, num_teams=5, num_transfe
             'nactive': len(actives),
             'transfer_volume': sum(x['amount'] for x in week_transfers)
         }
-        #Make ach_volume and charge_volume between 0 and 10% of transfer volume
-        def rand_part():
-            return decimal.Decimal(random.random()* 0.1)
-        payday['ach_volume']   = -1 * payday['transfer_volume'] * rand_part()
-        payday['charge_volume'] = payday['transfer_volume'] * rand_part()
         _fake_thing(db, "paydays", **payday)
         date = end_date
-
-
+    print("")
 
 def main():
-    populate_db(wireup.db(wireup.env()))
+    db = wireup.db(wireup.env())
+    prep_db(db)
+    populate_db(db)
+    clean_db(db)
+    check_db(db)
+
 
 if __name__ == '__main__':
     main()


Test patch: