Repository: gratipay/gratipay.com

Issue:
make fake: record "new" has no field "charge_volume" CONTEXT:  PL/pgSQL function process_payday() line 3 at SQL statement
Hello. I would like to help with Gratipay development, but I got stuck at `make fake`:

```
$ make fake
env/bin/honcho run -e defaults.env,local.env env/bin/fake_data fake_data
Making Participants
Making Teams
Making Payment Instructions
Making Elsewheres
Making Communities
Making Tips
Making Transfers (5000/5000)
Making Paydays (1/53)Traceback (most recent call last):
  File "env/bin/fake_data", line 9, in <module>
    load_entry_point('gratipay==1913', 'console_scripts', 'fake_data')()
  File "/home/fenryxo/dev/projects/gratipay/gratipay.com/gratipay/utils/fake_data.py", line 387, in main
    populate_db(db)
  File "/home/fenryxo/dev/projects/gratipay/gratipay.com/gratipay/utils/fake_data.py", line 380, in populate_db
    _fake_thing(db, "paydays", **payday)
  File "/home/fenryxo/dev/projects/gratipay/gratipay.com/gratipay/utils/fake_data.py", line 34, in _fake_thing
    , column_values
  File "/home/fenryxo/dev/projects/gratipay/gratipay.com/env/lib/python2.7/site-packages/postgres/__init__.py", line 357, in run
    cursor.run(sql, parameters)
  File "/home/fenryxo/dev/projects/gratipay/gratipay.com/env/lib/python2.7/site-packages/postgres/cursors.py", line 92, in run
    self.execute(sql, parameters)
  File "/home/fenryxo/dev/projects/gratipay/gratipay.com/env/lib/python2.7/site-packages/psycopg2/extras.py", line 288, in execute
    return super(NamedTupleCursor, self).execute(query, vars)
psycopg2.ProgrammingError: record "new" has no field "charge_volume"
CONTEXT:  PL/pgSQL function process_payday() line 3 at SQL statement

Makefile:45: recipe for target 'fake' failed
make: *** [fake] Error 1
```

I'm running Debian Jessie with Python 2.7.9 and Postgresql 9.4.2, if that matters.



Discussion:
cc: @rorepo 

Confirmed. I get the same error here. Sorry, @fenryxo. :-/

The problem is in `prep_db`, which looks like it's copied and pasted from `sql/payday.sql`. What's up with that?

I really thought that I got rid of the `ts_start` interpolation in `payday.sql` at some point, but now it looks like it's still there.

Diffing the sql in `prep_db` with `payday.sql` isn't helpful:

``` diff
--- prep_db.sql 2015-09-13 15:14:20.000000000 -0400
+++ sql/payday.sql  2015-09-09 12:09:23.000000000 -0400
@@ -1,73 +1,243 @@
-        CREATE OR REPLACE FUNCTION process_transfer() RETURNS trigger AS $$
+-- Recreate the necessary tables and indexes
+
+DROP TABLE IF EXISTS payday_participants;
+CREATE TABLE payday_participants AS
+    SELECT id
+         , username
+         , claimed_time
+         , balance AS old_balance
+         , balance AS new_balance
+         , is_suspicious
+         , false AS card_hold_ok
+         , ( SELECT count(*)
+               FROM current_exchange_routes r
+              WHERE r.participant = p.id
+                AND network = 'braintree-cc'
+                AND error = ''
+           ) > 0 AS has_credit_card
+          , braintree_customer_id
+      FROM participants p
+     WHERE is_suspicious IS NOT true
+       AND claimed_time < %(ts_start)s
+  ORDER BY claimed_time;
+
+CREATE UNIQUE INDEX ON payday_participants (id);
+CREATE UNIQUE INDEX ON payday_participants (username);
+
+DROP TABLE IF EXISTS payday_teams;
+CREATE TABLE payday_teams AS
+    SELECT t.id
+         , slug
+         , owner
+         , 0::numeric(35, 2) AS balance
+         , false AS is_drained
+      FROM teams t
+      JOIN participants p
+        ON t.owner = p.username
+     WHERE t.is_approved IS true
+       AND t.is_closed IS NOT true
+       AND p.claimed_time IS NOT null
+       AND p.is_closed IS NOT true
+       AND p.is_suspicious IS NOT true
+       AND (SELECT count(*)
+              FROM current_exchange_routes er
+             WHERE er.participant = p.id
+               AND network = 'paypal'
+               AND error = ''
+            ) > 0
+    ;
+
+DROP TABLE IF EXISTS payday_payments_done;
+CREATE TABLE payday_payments_done AS
+    SELECT *
+      FROM payments p
+     WHERE p.timestamp > %(ts_start)s;
+
+DROP TABLE IF EXISTS payday_payment_instructions;
+CREATE TABLE payday_payment_instructions AS
+    SELECT s.id, participant, team, amount, due
+      FROM ( SELECT DISTINCT ON (participant, team) *
+               FROM payment_instructions
+              WHERE mtime < %(ts_start)s
+           ORDER BY participant, team, mtime DESC
+           ) s
+      JOIN payday_participants p ON p.username = s.participant
+      JOIN payday_teams t ON t.slug = s.team
+     WHERE s.amount > 0
+       AND ( SELECT id
+               FROM payday_payments_done done
+              WHERE s.participant = done.participant
+                AND s.team = done.team
+                AND direction = 'to-team'
+           ) IS NULL
+  ORDER BY p.claimed_time ASC, s.ctime ASC;
+
+CREATE INDEX ON payday_payment_instructions (participant);
+CREATE INDEX ON payday_payment_instructions (team);
+ALTER TABLE payday_payment_instructions ADD COLUMN is_funded boolean;
+
+ALTER TABLE payday_participants ADD COLUMN giving_today numeric(35,2);
+UPDATE payday_participants pp
+   SET giving_today = COALESCE((
+           SELECT sum(amount + due)
+             FROM payday_payment_instructions
+            WHERE participant = pp.username
+       ), 0);
+
+DROP TABLE IF EXISTS payday_takes;
+CREATE TABLE payday_takes
+( team text
+, member text
+, amount numeric(35,2)
+ );
+
+DROP TABLE IF EXISTS payday_payments;
+CREATE TABLE payday_payments
+( timestamp timestamptz         DEFAULT now()
+, participant text              NOT NULL
+, team text                     NOT NULL
+, amount numeric(35,2)          NOT NULL
+, direction payment_direction   NOT NULL
+ );
+
+
+-- Prepare a statement that makes and records a payment
+
+CREATE OR REPLACE FUNCTION pay(text, text, numeric, payment_direction)
+RETURNS void AS $$
+    DECLARE
+        participant_delta numeric;
+        team_delta numeric;
+        payload json;
             BEGIN
-                UPDATE participants
-                   SET balance = balance + NEW.amount
-                 WHERE username = NEW.tippee;
-
-                UPDATE participants
-                   SET balance = balance - NEW.amount
-                 WHERE username = NEW.tipper;
+        IF ($3 = 0) THEN RETURN; END IF;

-                RETURN NULL;
+        IF ($4 = 'to-team') THEN
+            participant_delta := -$3;
+            team_delta := $3;
+        ELSE
+            participant_delta := $3;
+            team_delta := -$3;
+        END IF;
+
+        UPDATE payday_participants
+           SET new_balance = (new_balance + participant_delta)
+         WHERE username = $1;
+        UPDATE payday_teams
+           SET balance = (balance + team_delta)
+         WHERE slug = $2;
+        UPDATE current_payment_instructions
+           SET due = 0
+         WHERE participant = $1
+           AND team = $2
+           AND due > 0;
+        IF ($4 = 'to-team') THEN
+            payload = '{"action":"pay","participant":"' || $1 || '", "team":"'
+                || $2 || '", "amount":' || $3 || '}';
+            INSERT INTO events(type, payload)
+                VALUES ('payday',payload);
+        END IF;
+        INSERT INTO payday_payments
+                    (participant, team, amount, direction)
+             VALUES ( ( SELECT p.username
+                          FROM participants p
+                          JOIN payday_participants p2 ON p.id = p2.id
+                         WHERE p2.username = $1 )
+                    , ( SELECT t.slug
+                          FROM teams t
+                          JOIN payday_teams t2 ON t.id = t2.id
+                         WHERE t2.slug = $2 )
+                    , $3
+                    , $4
+                     );
             END;
-        $$ language plpgsql;
+$$ LANGUAGE plpgsql;
+
+-- Add payments that were not met on to due
+
+CREATE OR REPLACE FUNCTION park(text, text, numeric)
+RETURNS void AS $$
+    DECLARE payload json;
+    BEGIN
+        IF ($3 = 0) THEN RETURN; END IF;
+
+        UPDATE current_payment_instructions
+           SET due = $3
+         WHERE participant = $1
+           AND team = $2;
+
+        payload = '{"action":"due","participant":"' || $1 || '", "team":"'
+            || $2 || '", "due":' || $3 || '}';
+        INSERT INTO events(type, payload)
+            VALUES ('payday',payload);
+
+    END;
+$$ LANGUAGE plpgsql;
+

-        CREATE TRIGGER process_transfer AFTER INSERT ON transfers
-            FOR EACH ROW EXECUTE PROCEDURE process_transfer();
+-- Create a trigger to process payment_instructions

-        CREATE OR REPLACE FUNCTION process_exchange() RETURNS trigger AS $$
+CREATE OR REPLACE FUNCTION process_payment_instruction() RETURNS trigger AS $$
+    DECLARE
+        participant payday_participants;
             BEGIN
-                IF NEW.amount > 0 THEN
-                    UPDATE participants
-                       SET balance = balance + NEW.amount
-                     WHERE username = NEW.participant;
+        participant := (
+            SELECT p.*::payday_participants
+              FROM payday_participants p
+             WHERE username = NEW.participant
+        );
+        IF (NEW.amount + NEW.due <= participant.new_balance OR participant.card_hold_ok) THEN
+            EXECUTE pay(NEW.participant, NEW.team, NEW.amount + NEW.due, 'to-team');
+            RETURN NEW;
                 ELSE
-                    UPDATE participants
-                       SET balance = balance + NEW.amount - NEW.fee
-                     WHERE username = NEW.participant;
+            EXECUTE park(NEW.participant, NEW.team, NEW.amount + NEW.due);
+            RETURN NULL;
                 END IF;
+        RETURN NULL;
+    END;
+$$ LANGUAGE plpgsql;

+CREATE TRIGGER process_payment_instruction BEFORE UPDATE OF is_funded ON payday_payment_instructions
+    FOR EACH ROW
+    WHEN (NEW.is_funded IS true AND OLD.is_funded IS NOT true)
+    EXECUTE PROCEDURE process_payment_instruction();
+
+-- Create a trigger to process takes
+
+CREATE OR REPLACE FUNCTION process_take() RETURNS trigger AS $$
+    DECLARE
+        actual_amount numeric(35,2);
+        team_balance numeric(35,2);
+    BEGIN
+        team_balance := (
+            SELECT new_balance
+              FROM payday_participants
+             WHERE username = NEW.team
+        );
+        IF (team_balance <= 0) THEN RETURN NULL; END IF;
+        actual_amount := NEW.amount;
+        IF (team_balance < NEW.amount) THEN
+            actual_amount := team_balance;
+        END IF;
+        EXECUTE transfer(NEW.team, NEW.member, actual_amount, 'take');
                 RETURN NULL;
             END;
-        $$ language plpgsql;
+$$ LANGUAGE plpgsql;

-        CREATE TRIGGER process_exchange AFTER INSERT ON exchanges
-            FOR EACH ROW EXECUTE PROCEDURE process_exchange();
+CREATE TRIGGER process_take AFTER INSERT ON payday_takes
+    FOR EACH ROW EXECUTE PROCEDURE process_take();

-        CREATE OR REPLACE FUNCTION process_payday() RETURNS trigger AS $$
-            BEGIN
-                SELECT COALESCE(SUM(amount+fee), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount > 0
-                  INTO NEW.charge_volume;
-
-                SELECT COALESCE(SUM(fee), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount > 0
-                  INTO NEW.charge_fees_volume;
-
-                SELECT COALESCE(SUM(amount), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount < 0
-                  INTO NEW.ach_volume;
-
-                SELECT COALESCE(SUM(fee), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount < 0
-                  INTO NEW.ach_fees_volume;

-                RETURN NEW;
-            END;
-        $$ language plpgsql;
+-- Create a trigger to process draws

-        CREATE TRIGGER process_payday BEFORE INSERT ON paydays
-            FOR EACH ROW EXECUTE PROCEDURE process_payday();
+CREATE OR REPLACE FUNCTION process_draw() RETURNS trigger AS $$
+    BEGIN
+        EXECUTE pay(NEW.owner, NEW.slug, NEW.balance, 'to-participant');
+        RETURN NULL;
+    END;
+$$ LANGUAGE plpgsql;

+CREATE TRIGGER process_draw BEFORE UPDATE OF is_drained ON payday_teams
+    FOR EACH ROW
+    WHEN (NEW.is_drained IS true AND OLD.is_drained IS NOT true)
+    EXECUTE PROCEDURE process_draw();
```


Resolving patch:
diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py
--- a/gratipay/utils/fake_data.py
+++ b/gratipay/utils/fake_data.py
@@ -225,50 +225,12 @@ def prep_db(db):
 
         CREATE TRIGGER process_exchange AFTER INSERT ON exchanges
             FOR EACH ROW EXECUTE PROCEDURE process_exchange();
-
-        CREATE OR REPLACE FUNCTION process_payday() RETURNS trigger AS $$
-            BEGIN
-                SELECT COALESCE(SUM(amount+fee), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount > 0
-                  INTO NEW.charge_volume;
-
-                SELECT COALESCE(SUM(fee), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount > 0
-                  INTO NEW.charge_fees_volume;
-
-                SELECT COALESCE(SUM(amount), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount < 0
-                  INTO NEW.ach_volume;
-
-                SELECT COALESCE(SUM(fee), 0)
-                  FROM exchanges
-                 WHERE timestamp > NEW.ts_start
-                   AND timestamp < NEW.ts_end
-                   AND amount < 0
-                  INTO NEW.ach_fees_volume;
-
-                RETURN NEW;
-            END;
-        $$ language plpgsql;
-
-        CREATE TRIGGER process_payday BEFORE INSERT ON paydays
-            FOR EACH ROW EXECUTE PROCEDURE process_payday();
     """)
 
 def clean_db(db):
     db.run("""
-        DROP FUNCTION process_transfer() CASCADE;
-        DROP FUNCTION process_exchange() CASCADE;
-        DROP FUNCTION process_payday() CASCADE;
+        DROP FUNCTION IF EXISTS process_transfer() CASCADE;
+        DROP FUNCTION IF EXISTS process_exchange() CASCADE;
     """)
 
 
@@ -381,10 +343,12 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=
         date = end_date
     print("")
 
-def main():
-    db = wireup.db(wireup.env())
+
+def main(db=None, *a, **kw):
+    db = db or wireup.db(wireup.env())
+    clean_db(db)
     prep_db(db)
-    populate_db(db)
+    populate_db(db, *a, **kw)
     clean_db(db)
     check_db(db)
 


Test patch:
diff --git a/tests/py/test_fake_data.py b/tests/py/test_fake_data.py
--- a/tests/py/test_fake_data.py
+++ b/tests/py/test_fake_data.py
@@ -14,7 +14,7 @@ def test_fake_data(self):
         num_tips = 5
         num_teams = 1
         num_transfers = 5
-        fake_data.populate_db(self.db, num_participants, num_tips, num_teams, num_transfers)
+        fake_data.main(self.db, num_participants, num_tips, num_teams, num_transfers)
         tips = self.db.all("SELECT * FROM tips")
         participants = self.db.all("SELECT * FROM participants")
         transfers = self.db.all("SELECT * FROM transfers")