Repository: algorand/indexer
Pull request #284 (2021-01-21 21:13:43)

Json for postgres
## Summary

Fix handling of some transactions whose JSON encoding would fail to be received by PostgreSQL. Emit stricter JSON, keys are strings, strings are utf8 or base64

## Test Plan

Tested against test data from go-algorand e2e test scripts (including new data added there to exercise this condition).

brianolson: Indexer mostly doesn't use the json encoding of the txn. A few specific fields are used in filters but they aren't affected by this change. The value side of fields was already base64, just the key strings needed fixup.
tsachiherman: > Indexer mostly doesn't use the json encoding of the txn. A few specific fields are used in filters but they aren't affected by this change. The value side of fields was already base64, just the key strings needed fixup.

I meant that when a new block is being received and applied, you need to decode the old keys since these might need to get overridden by the new ones. ( i.e. the old one is the b64('key') whereas the new one just just the `key` )

Patch:
diff --git a/.travis.yml b/.travis.yml
index ac536b9..9c8a85b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ script:
 - make
 - make integration
 - make test
-- ./test/perf.sh "$TEST_CONNECTION_STRING_1" && ./test/perf.sh "$TEST_CONNECTION_STRING_2"
+#- ./test/perf.sh "$TEST_CONNECTION_STRING_1" && ./test/perf.sh "$TEST_CONNECTION_STRING_2"
 - make fakepackage
 - docker build -t e2e_tests -f docker/Dockerfile.mule .
 - docker run --rm -t e2e_tests mule/e2e.sh
diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go
index a4d6d2a..4f5842a 100644
--- a/idb/postgres/postgres.go
+++ b/idb/postgres/postgres.go
@@ -142,14 +142,34 @@ func (db *IndexerDb) StartBlock() (err error) {
 	return nil
 }
 
+// For App apply data, convert "string" keys which are secretly []byte blobs to their base64 representation so that JSON systems that require strings to be utf8 don't panic.
+func stxnToJSON(txn types.SignedTxnWithAD) []byte {
+	jt := txn
+	if len(jt.EvalDelta.GlobalDelta) > 0 {
+		gd := make(map[string]types.ValueDelta, len(jt.EvalDelta.GlobalDelta))
+		for k, v := range jt.EvalDelta.GlobalDelta {
+			gd[b64([]byte(k))] = v
+		}
+		jt.EvalDelta.GlobalDelta = gd
+	}
+	if len(jt.EvalDelta.LocalDeltas) > 0 {
+		ldout := make(map[uint64]types.StateDelta, len(jt.EvalDelta.LocalDeltas))
+		for i, ld := range jt.EvalDelta.LocalDeltas {
+			nld := make(map[string]types.ValueDelta, len(ld))
+			for k, v := range ld {
+				nld[b64([]byte(k))] = v
+			}
+			ldout[i] = nld
+		}
+		jt.EvalDelta.LocalDeltas = ldout
+	}
+	return idb.JSONOneLine(jt)
+}
+
 // AddTransaction is part of idb.IndexerDB
 func (db *IndexerDb) AddTransaction(round uint64, intra int, txtypeenum int, assetid uint64, txn types.SignedTxnWithAD, participation [][]byte) error {
 	txnbytes := msgpack.Encode(txn)
-	var jsonbytes []byte
-	jsonbytes, err := idb.MsgpackToJSON(txnbytes)
-	if err != nil {
-		return err
-	}
+	jsonbytes := stxnToJSON(txn)
 	txid := crypto.TransactionIDString(txn.Txn)
 	tx := []interface{}{round, intra, txtypeenum, assetid, txid[:], txnbytes, jsonbytes}
 	db.txrows = append(db.txrows, tx)
@@ -183,6 +203,7 @@ func (db *IndexerDb) CommitBlock(round uint64, timestamp int64, rewardslevel uin
 	}
 	_, err = addtx.Exec()
 	if err != nil {
+		db.log.Errorf("CommitBlock failed: %v (%#v)", err, err)
 		for _, txr := range db.txrows {
 			ntxr := make([]interface{}, len(txr))
 			for i, v := range txr {
@@ -197,7 +218,7 @@ func (db *IndexerDb) CommitBlock(round uint64, timestamp int64, rewardslevel uin
 					ntxr[i] = v
 				}
 			}
-			db.log.Printf("txr %#v", ntxr)
+			db.log.Errorf("txr %#v", ntxr)
 		}
 		return fmt.Errorf("COPY txn Exec() %v", err)
 	}
diff --git a/idb/postgres/postgres_migrations.go b/idb/postgres/postgres_migrations.go
index d651df7..d7d66b6 100644
--- a/idb/postgres/postgres_migrations.go
+++ b/idb/postgres/postgres_migrations.go
@@ -34,6 +34,7 @@ func init() {
 		{m3acfgFix, false, "Recompute asset configurations with corrected merge function."},
 		{m4RewardsAndDatesPart1, true, "Update DB Schema for cumulative account reward support and creation dates."},
 		{m5RewardsAndDatesPart2, false, "Compute cumulative account rewards for all accounts."},
+		{m6MarkTxnJSONSplit, true, "record round at which txn json recording changes, for future migration to fixup prior records"},
 	}
 
 	// Verify ensure the constant is pointing to the right index
@@ -547,7 +548,7 @@ func initM5AccountData() *m5AccountData {
 	}
 }
 
-func processAccountTransactionsWithRetry(db *IndexerDb, addressStr string, address types.Address, nextRound uint64, retries int) (results *m5AccountData, err error){
+func processAccountTransactionsWithRetry(db *IndexerDb, addressStr string, address types.Address, nextRound uint64, retries int) (results *m5AccountData, err error) {
 	for i := 0; i < retries; i++ {
 		// Query transactions for the account
 		txnrows := db.Transactions(context.Background(), idb.TransactionFilter{
@@ -559,7 +560,7 @@ func processAccountTransactionsWithRetry(db *IndexerDb, addressStr string, addre
 		results, err = processAccountTransactions(txnrows, addressStr, address)
 		if err != nil {
 			db.log.Errorf("%s: (attempt %d) failed to update %s: %v", rewardsCreateCloseUpdateErr, i+1, addressStr, err)
-			time.Sleep(10*time.Second)
+			time.Sleep(10 * time.Second)
 		} else {
 			return
 		}
@@ -580,7 +581,6 @@ func processAccountTransactions(txnrows <-chan idb.TxnRow, addressStr string, ad
 		}
 		if len(txn.TxnBytes) == 0 {
 			return nil, fmt.Errorf("%s: processing account %s: found empty TxnBytes (rnd %d, intra %d):  %v", rewardsCreateCloseUpdateErr, addressStr, txn.Round, txn.Intra, err)
-			continue
 		}
 		numTxn++
 
@@ -692,7 +692,7 @@ func m5RewardsAndDatesPart2UpdateAccounts(db *IndexerDb, state *MigrationState,
 		if err != nil {
 			return fmt.Errorf("%s: failed to update %s: %v", rewardsCreateCloseUpdateErr, addressStr, err)
 		}
-		if dur > 5 * time.Minute {
+		if dur > 5*time.Minute {
 			db.log.Warnf("%s: slowness detected, spent %s migrating %s", rewardsCreateCloseUpdateMessage, dur, addressStr)
 		}
 
@@ -1090,3 +1090,12 @@ func (mtxid *txidFiuxpMigrationContext) readHeaders(minRound, maxRound uint64) (
 	}
 	return headers, nil
 }
+
+// Record round at which behavior changed for encoding txn.txn JSON.
+// A future migration should go back and apply new encoding to prior txn rows then delete this row in metastate.
+func m6MarkTxnJSONSplit(db *IndexerDb, state *MigrationState) error {
+	sqlLines := []string{
+		`INSERT INTO metastate (k,v) SELECT 'm6MarkTxnJSONSplit', m.v FROM metastate m WHERE m.k = 'state'`,
+	}
+	return sqlMigration(db, state, sqlLines)
+}
diff --git a/idb/transcode.go b/idb/transcode.go
index c190ebe..0a5c599 100644
--- a/idb/transcode.go
+++ b/idb/transcode.go
@@ -66,4 +66,5 @@ func init() {
 	oneLineJSONCodecHandle.RecursiveEmptyCheck = true
 	oneLineJSONCodecHandle.HTMLCharsAsIs = true
 	oneLineJSONCodecHandle.Indent = 0
+	oneLineJSONCodecHandle.MapKeyAsString = true
 }


Test patch:
diff --git a/idb/postgres/postgres_test.go b/idb/postgres/postgres_test.go
index b93edd3..80aaecc 100644
--- a/idb/postgres/postgres_test.go
+++ b/idb/postgres/postgres_test.go
@@ -5,7 +5,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/algorand/go-algorand-sdk/encoding/msgpack"
+
 	"github.com/algorand/indexer/idb"
+	"github.com/algorand/indexer/types"
 	"github.com/stretchr/testify/require"
 )
 
@@ -111,3 +114,50 @@ func TestTealKeyValue(t *testing.T) {
 	_, ok = tkv.get(k2)
 	a.False(ok)
 }
+
+type txnMsgpackJSON struct {
+	msgpack []byte
+	json    string
+}
+
+var testTxns = []txnMsgpackJSON{
+	{
+		[]uint8{0x83, 0xa2, 0x64, 0x74, 0x81, 0xa2, 0x67, 0x64, 0x81, 0xa9, 0xfe, 0xfe, 0xff, 0xef, 0x0, 0x0, 0x11, 0x22, 0x33, 0x82, 0xa2, 0x61, 0x74, 0x1, 0xa2, 0x62, 0x73, 0xc4, 0x3, 0x78, 0x78, 0x78, 0xa3, 0x73, 0x69, 0x67, 0xc4, 0x40, 0x51, 0xca, 0x9f, 0x32, 0xca, 0x9d, 0x66, 0x4b, 0xde, 0xa0, 0x98, 0xd9, 0x1b, 0xd, 0xe, 0x4d, 0x39, 0xca, 0x2, 0x4c, 0x4e, 0xc4, 0xba, 0x88, 0x1a, 0xb6, 0xa, 0x63, 0xff, 0xb0, 0x95, 0xc6, 0xb6, 0x7d, 0x0, 0xb4, 0xdc, 0xef, 0x41, 0xe6, 0x3b, 0xc3, 0x43, 0x3e, 0xb5, 0xa2, 0xa0, 0x27, 0xad, 0x9c, 0xc0, 0x57, 0x93, 0x5c, 0x4e, 0xcd, 0x18, 0xea, 0xb0, 0x6b, 0xe3, 0x97, 0x17, 0x3, 0xa3, 0x74, 0x78, 0x6e, 0x8b, 0xa4, 0x61, 0x70, 0x61, 0x61, 0x91, 0xc4, 0x3, 0x78, 0x78, 0x78, 0xa4, 0x61, 0x70, 0x61, 0x70, 0xc4, 0x17, 0x2, 0x20, 0x1, 0x1, 0x26, 0x1, 0x9, 0xfe, 0xfe, 0xff, 0xef, 0x0, 0x0, 0x11, 0x22, 0x33, 0x28, 0x36, 0x1a, 0x0, 0x67, 0x22, 0x43, 0xa4, 0x61, 0x70, 0x67, 0x73, 0x81, 0xa3, 0x6e, 0x62, 0x73, 0x1, 0xa4, 0x61, 0x70, 0x73, 0x75, 0xc4, 0x5, 0x2, 0x20, 0x1, 0x1, 0x22, 0xa3, 0x66, 0x65, 0x65, 0xcd, 0x3, 0xe8, 0xa2, 0x66, 0x76, 0x4, 0xa2, 0x67, 0x68, 0xc4, 0x20, 0x8a, 0xae, 0xf2, 0xee, 0x8f, 0x3, 0x93, 0xb9, 0xa5, 0x47, 0x41, 0x35, 0x3b, 0x97, 0x96, 0xf3, 0xd, 0xcc, 0x52, 0x10, 0x9d, 0x21, 0x15, 0x9a, 0x64, 0xe8, 0x47, 0x52, 0xb2, 0xcc, 0x90, 0x6a, 0xa2, 0x6c, 0x76, 0xcd, 0x3, 0xec, 0xa4, 0x6e, 0x6f, 0x74, 0x65, 0xc4, 0x8, 0x13, 0xfa, 0x3c, 0x55, 0xe8, 0x7b, 0x23, 0xea, 0xa3, 0x73, 0x6e, 0x64, 0xc4, 0x20, 0x4a, 0x82, 0x63, 0xeb, 0xc0, 0xd2, 0xee, 0xed, 0xac, 0x73, 0xdb, 0xb9, 0xd0, 0x27, 0xa1, 0xb2, 0x32, 0x99, 0x7a, 0xed, 0xc5, 0xde, 0xa2, 0x25, 0x7f, 0x7f, 0x2c, 0x8b, 0xcd, 0x42, 0x5f, 0x1a, 0xa4, 0x74, 0x79, 0x70, 0x65, 0xa4, 0x61, 0x70, 0x70, 0x6c},
+		"{\"dt\":{\"gd\":{\"/v7/7wAAESIz\":{\"at\":1,\"bs\":\"eHh4\"}}},\"sig\":\"UcqfMsqdZkveoJjZGw0OTTnKAkxOxLqIGrYKY/+wlca2fQC03O9B5jvDQz61oqAnrZzAV5NcTs0Y6rBr45cXAw==\",\"txn\":{\"apaa\":[\"eHh4\"],\"apap\":\"AiABASYBCf7+/+8AABEiMyg2GgBnIkM=\",\"apgs\":{\"nbs\":1},\"apsu\":\"AiABASI=\",\"fee\":1000,\"fv\":4,\"gh\":\"iq7y7o8Dk7mlR0E1O5eW8w3MUhCdIRWaZOhHUrLMkGo=\",\"lv\":1004,\"note\":\"E/o8Veh7I+o=\",\"snd\":\"SoJj68DS7u2sc9u50CehsjKZeu3F3qIlf38si81CXxo=\",\"type\":\"appl\"}}",
+	},
+	{
+		[]byte{0x83, 0xa2, 0x64, 0x74, 0x82, 0xa2, 0x67, 0x64, 0x82, 0xa3, 0x67, 0x6b, 0x62, 0x82, 0xa2, 0x61, 0x74, 0x1, 0xa2, 0x62, 0x73, 0xc4, 0x4, 0x74, 0x65, 0x73, 0x74, 0xa3, 0x67, 0x6b, 0x69, 0x82, 0xa2, 0x61, 0x74, 0x2, 0xa2, 0x75, 0x69, 0x64, 0xa2, 0x6c, 0x64, 0x81, 0x0, 0x82, 0xa3, 0x6c, 0x6b, 0x62, 0x82, 0xa2, 0x61, 0x74, 0x1, 0xa2, 0x62, 0x73, 0xc4, 0xb, 0x61, 0x6e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x74, 0x65, 0x73, 0x74, 0xa3, 0x6c, 0x6b, 0x69, 0x82, 0xa2, 0x61, 0x74, 0x2, 0xa2, 0x75, 0x69, 0xcc, 0xc8, 0xa3, 0x73, 0x69, 0x67, 0xc4, 0x40, 0xc9, 0x25, 0xb2, 0xa, 0x42, 0xda, 0x15, 0xbe, 0x74, 0x16, 0x1d, 0x45, 0xc9, 0x3b, 0xf, 0xa4, 0xcc, 0xdd, 0x86, 0xbd, 0xa, 0x53, 0x1e, 0x43, 0xb3, 0x7e, 0xf9, 0xcc, 0xaf, 0x44, 0x38, 0xce, 0x35, 0xa5, 0xaa, 0xb, 0x96, 0x28, 0x79, 0x6, 0xf8, 0xe1, 0xfb, 0x96, 0xe3, 0x79, 0x9b, 0x27, 0xfa, 0xa4, 0x51, 0x10, 0xc7, 0xb1, 0x84, 0x79, 0x46, 0xf8, 0xd8, 0x6a, 0x6c, 0x96, 0x93, 0x6, 0xa3, 0x74, 0x78, 0x6e, 0x8a, 0xa4, 0x61, 0x70, 0x61, 0x61, 0x91, 0xc4, 0x5, 0x66, 0x69, 0x72, 0x73, 0x74, 0xa4, 0x61, 0x70, 0x61, 0x6e, 0x1, 0xa4, 0x61, 0x70, 0x69, 0x64, 0x23, 0xa3, 0x66, 0x65, 0x65, 0xcd, 0x3, 0xe8, 0xa2, 0x66, 0x76, 0x7, 0xa2, 0x67, 0x68, 0xc4, 0x20, 0x8a, 0xae, 0xf2, 0xee, 0x8f, 0x3, 0x93, 0xb9, 0xa5, 0x47, 0x41, 0x35, 0x3b, 0x97, 0x96, 0xf3, 0xd, 0xcc, 0x52, 0x10, 0x9d, 0x21, 0x15, 0x9a, 0x64, 0xe8, 0x47, 0x52, 0xb2, 0xcc, 0x90, 0x6a, 0xa2, 0x6c, 0x76, 0xcd, 0x3, 0xef, 0xa4, 0x6e, 0x6f, 0x74, 0x65, 0xc4, 0x8, 0xc9, 0x83, 0x5, 0x5f, 0x20, 0x45, 0x8f, 0x98, 0xa3, 0x73, 0x6e, 0x64, 0xc4, 0x20, 0x32, 0xf8, 0xa1, 0x14, 0x66, 0x60, 0x7, 0xb7, 0xfe, 0x8, 0xd2, 0x48, 0x83, 0xdf, 0x28, 0x86, 0x16, 0x74, 0xa3, 0xb2, 0x5, 0x48, 0x1d, 0x4a, 0x45, 0x8e, 0x50, 0xb4, 0xba, 0x2f, 0x34, 0xde, 0xa4, 0x74, 0x79, 0x70, 0x65, 0xa4, 0x61, 0x70, 0x70, 0x6c},
+		"{\"dt\":{\"gd\":{\"/v7/7wAAESIz\":{\"at\":1,\"bs\":\"eHh4\"},\"Z2ti\":{\"at\":1,\"bs\":\"dGVzdA==\"},\"Z2tp\":{\"at\":2,\"ui\":100}},\"ld\":{\"0\":{\"bGti\":{\"at\":1,\"bs\":\"YW5vdGhlcnRlc3Q=\"},\"bGtp\":{\"at\":2,\"ui\":200}}}},\"sig\":\"ySWyCkLaFb50Fh1FyTsPpMzdhr0KUx5Ds375zK9EOM41paoLlih5Bvjh+5bjeZsn+qRREMexhHlG+NhqbJaTBg==\",\"txn\":{\"apaa\":[\"Zmlyc3Q=\"],\"apan\":1,\"apap\":\"AiABASYBCf7+/+8AABEiMyg2GgBnIkM=\",\"apgs\":{\"nbs\":1},\"apid\":35,\"apsu\":\"AiABASI=\",\"fee\":1000,\"fv\":7,\"gh\":\"iq7y7o8Dk7mlR0E1O5eW8w3MUhCdIRWaZOhHUrLMkGo=\",\"lv\":1007,\"note\":\"yYMFXyBFj5g=\",\"snd\":\"MvihFGZgB7f+CNJIg98ohhZ0o7IFSB1KRY5QtLovNN4=\",\"type\":\"appl\"}}",
+	},
+}
+
+func TestStxnJSON(t *testing.T) {
+	var stxn types.SignedTxnWithAD
+	for i, mt := range testTxns {
+		t.Run(fmt.Sprintf("i=%d", i), func(t *testing.T) {
+			msgpack.Decode(mt.msgpack, &stxn)
+			js := stxnToJSON(stxn)
+			require.Equal(t, mt.json, string(js))
+		})
+	}
+}
+
+func TestStxnJSONSynthetic(t *testing.T) {
+	nonutf8b := []byte{254, 254, 255, 239, 0, 0, 17, 34, 51}
+	nonutf8 := string(nonutf8b)
+	var stxn types.SignedTxnWithAD
+	stxn.EvalDelta.GlobalDelta = make(map[string]types.ValueDelta)
+	stxn.EvalDelta.GlobalDelta[nonutf8] = types.ValueDelta{
+		Action: types.SetBytesAction,
+		Bytes:  nonutf8b,
+	}
+	stxn.EvalDelta.LocalDeltas = make(map[uint64]types.StateDelta, 1)
+	ld := make(map[string]types.ValueDelta)
+	ld[nonutf8] = types.ValueDelta{
+		Action: types.SetBytesAction,
+		Bytes:  nonutf8b,
+	}
+	stxn.EvalDelta.LocalDeltas[1] = ld
+	js := stxnToJSON(stxn)
+	require.Equal(t, "{\"dt\":{\"gd\":{\"/v7/7wAAESIz\":{\"at\":1,\"bs\":\"/v7/7wAAESIz\"}},\"ld\":{\"1\":{\"/v7/7wAAESIz\":{\"at\":1,\"bs\":\"/v7/7wAAESIz\"}}}}}", string(js))
+}
diff --git a/test/common.sh b/test/common.sh
index fc54d89..0caaeff 100755
--- a/test/common.sh
+++ b/test/common.sh
@@ -7,6 +7,8 @@ NET=localhost:8981
 CURL_TEMPFILE=curl_out.txt
 PIDFILE=testindexerpidfile
 CONNECTION_STRING="host=localhost user=algorand password=algorand dbname=DB_NAME_HERE port=5434 sslmode=disable"
+MAX_TIME=20
+
 
 ###################
 ## Print Helpers ##
@@ -78,7 +80,7 @@ function sql_test {
   done
 
   local ELAPSED=$(($SECONDS - $START))
-  if [ $ELAPSED -gt $MAX_TIME ]; then
+  if [[ $ELAPSED -gt $MAX_TIME ]]; then
     fail_and_exit "$DESCRIPTION" "$QUERY" "query duration too long, $ELAPSED > $MAX_TIME"
   fi
 
@@ -134,7 +136,7 @@ function rest_test {
   fi
 
   local ELAPSED=$(($SECONDS - $START))
-  if [ $ELAPSED -gt $MAX_TIME ]; then
+  if [[ $ELAPSED -gt $MAX_TIME ]]; then
     fail_and_exit "$DESCRIPTION" "$QUERY" "query duration too long, $ELAPSED > $MAX_TIME"
   fi
 
@@ -200,7 +202,7 @@ function start_indexer_with_blocks() {
   tar -xf "$2" -C $TEMPDIR
 
   if [ ! -z $3 ]; then
-    echo "Start args 'import -P \"${CONNECTION_STRING/DB_NAME_HERE/$1}\" --genesis \"$TEMPDIR/algod/genesis.json\" $TEMPDIR/blocktars/*"
+    echo "Start args 'import -P \"${CONNECTION_STRING/DB_NAME_HERE/$1}\" --genesis \"$TEMPDIR/algod/genesis.json\" $TEMPDIR/blocktars/*'"
     sleep infinity
   fi
   ALGORAND_DATA= ../cmd/algorand-indexer/algorand-indexer import \