Repository: trustwallet/blockatlas
Pull request #1286 (2020-11-26 10:37:05)

Improve logging / remove metrics

codecov[bot]: # [Codecov](https://codecov.io/gh/trustwallet/blockatlas/pull/1286?src=pr&el=h1) Report
> Merging [#1286](https://codecov.io/gh/trustwallet/blockatlas/pull/1286?src=pr&el=desc) (7874aba) into [master](https://codecov.io/gh/trustwallet/blockatlas/commit/94f7e1795c9e918255c369c2e96bbf0200a08837?el=desc) (94f7e17) will **not change** coverage.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/trustwallet/blockatlas/pull/1286/graphs/tree.svg?width=650&height=150&src=pr&token=jnrjoEQAqv)](https://codecov.io/gh/trustwallet/blockatlas/pull/1286?src=pr&el=tree)

```diff
@@          Coverage Diff           @@
##           master   #1286   +/-   ##
======================================
  Coverage      42%     42%           
======================================
  Files         152     152           
  Lines        4640    4640           
======================================
  Hits         1994    1994           
  Misses       2468    2468           
  Partials      178     178           
```



------

[Continue to review full report at Codecov](https://codecov.io/gh/trustwallet/blockatlas/pull/1286?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/trustwallet/blockatlas/pull/1286?src=pr&el=footer). Last update [94f7e17...135e075](https://codecov.io/gh/trustwallet/blockatlas/pull/1286?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/api/registry.go b/api/registry.go
index aae152a..153fd04 100644
--- a/api/registry.go
+++ b/api/registry.go
@@ -3,9 +3,7 @@ package api
 import (
 	"time"
 
-	"github.com/chenjiandongx/ginprom"
 	"github.com/gin-gonic/gin"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/trustwallet/blockatlas/api/endpoint"
 	"github.com/trustwallet/blockatlas/api/middleware"
 	"github.com/trustwallet/blockatlas/pkg/blockatlas"
@@ -102,7 +100,6 @@ func RegisterBatchAPI(router gin.IRouter) {
 
 func RegisterBasicAPI(router gin.IRouter) {
 	router.GET("/", endpoint.GetStatus)
-	router.GET("/metrics", ginprom.PromHandler(promhttp.Handler()))
 }
 
 func RegisterTokensSearcherAPI(router gin.IRouter, instance tokensearcher.Instance) {
diff --git a/app.json b/app.json
index 386e7d8..a06ef2b 100644
--- a/app.json
+++ b/app.json
@@ -14,11 +14,11 @@
       "addons": [
         {
           "plan":"heroku-postgresql:hobby-dev",
-          "as":"ATLAS_POSTGRES"
+          "as":"POSTGRES"
         },
         {
           "plan":"cloudamqp:lemur",
-          "as":"ATLAS_OBSERVER_RABBITMQ"
+          "as":"OBSERVER_RABBITMQ"
         }
       ]
     }
diff --git a/config.yml b/config.yml
index 9a67366..28ef7bb 100644
--- a/config.yml
+++ b/config.yml
@@ -220,7 +220,7 @@ near:
   api: https://staging-rpc.nearprotocol.com
 
 elrond:
-  api: https://api.elrond.com
+  api: https://gateway.elrond.com
 
 filecoin:
   api: https://api.filscan.io:8700/rpc/v1
diff --git a/platform/elrond/client.go b/platform/elrond/client.go
index 63a21a4..b5e3e0b 100644
--- a/platform/elrond/client.go
+++ b/platform/elrond/client.go
@@ -28,14 +28,14 @@ func (c *Client) CurrentBlockNumber() (num int64, err error) {
 func (c *Client) GetBlockByNumber(height int64) (*blockatlas.Block, error) {
 	var blockRes BlockResponse
 
-	path := fmt.Sprintf("block/%s/%d", metachainID, uint64(height))
+	path := fmt.Sprintf("hyperblock/by-nonce/%d", uint64(height))
 	err := c.getResponse(&blockRes, path, nil)
 	if err != nil {
 		return nil, err
 	}
 
 	block := blockRes.Block
-	txs := NormalizeTxs(block.Transactions, "")
+	txs := NormalizeTxs(block.Transactions, "", blockRes.Block)
 
 	return &blockatlas.Block{
 		Number: int64(block.Nonce),
@@ -54,7 +54,7 @@ func (c *Client) GetTxsOfAddress(address string) (blockatlas.TxPage, error) {
 		return nil, err
 	}
 
-	txs := NormalizeTxs(txPage.Transactions, address)
+	txs := NormalizeTxs(txPage.Transactions, address, Block{})
 
 	return txs, nil
 }
diff --git a/platform/elrond/model.go b/platform/elrond/model.go
index c4576ed..3e75cd7 100644
--- a/platform/elrond/model.go
+++ b/platform/elrond/model.go
@@ -2,11 +2,15 @@ package elrond
 
 import (
 	"encoding/json"
+	"math/big"
 	"time"
 
 	"github.com/trustwallet/blockatlas/pkg/blockatlas"
 )
 
+const roundDurationInSeconds = 6
+const mainnetStartTime = 1596117600
+
 type GenericResponse struct {
 	Data  json.RawMessage `json:"data"`
 	Code  string          `json:"code"`
@@ -24,11 +28,12 @@ type StatusDetails struct {
 }
 
 type BlockResponse struct {
-	Block Block `json:"block"`
+	Block Block `json:"hyperblock"`
 }
 
 type Block struct {
 	Nonce        uint64        `json:"nonce"`
+	Round        uint64        `json:"round"`
 	Hash         string        `json:"hash"`
 	Transactions []Transaction `json:"transactions"`
 }
@@ -47,6 +52,23 @@ type Transaction struct {
 	Timestamp time.Duration `json:"timestamp"`
 	Status    string        `json:"status"`
 	Fee       string        `json:"fee"`
+	GasPrice  uint64        `json:"gasPrice,omitempty"`
+	GasLimit  uint64        `json:"gasLimit,omitempty"`
+}
+
+func (tx *Transaction) TxFee() blockatlas.Amount {
+	if tx.Fee != "0" && tx.Fee != "" {
+		return blockatlas.Amount(tx.Fee)
+	}
+
+	// Hyperblocks API V1 does not provide the transaction fees (nor "gasUsed"). Hyperblocks API V2 will provide this information, as well.
+	// Until then, we can compute the fees deterministically (best-effort) in BlockAtlas, based on gasLimit and gasPrice. This logic will be soon removed from BlockAtlas, in a future PR, when the Hyperblocks API v2 becomes available.
+	// Note: For Smart Contract transactions, the refunds will be incompletely provided by the API (until Hyperblocks V2 becomes available): e.g. intra-shard refunds are not visible etc.)
+
+	txFee := big.NewInt(0).SetUint64(tx.GasPrice)
+	txFee = txFee.Mul(txFee, big.NewInt(0).SetUint64(tx.GasLimit))
+
+	return blockatlas.Amount(txFee.String())
 }
 
 func (tx *Transaction) TxStatus() blockatlas.Status {
@@ -70,3 +92,20 @@ func (tx *Transaction) Direction(address string) blockatlas.Direction {
 		return blockatlas.DirectionIncoming
 	}
 }
+
+func (tx *Transaction) TxTimestamp(blockRound uint64) time.Duration {
+	if int64(tx.Timestamp) != 0 {
+		return tx.Timestamp
+	}
+
+	// Minor issue (slight inconsistency) about the "timestamp" field:
+	// The transactions fetched by querying the endpoint "address/erd1.../transactions" come from our central Elastic Search database, and their timestamp refers to the moment of *including* those transactions in shard blocks (detail: at destination).
+	// However, the transactions fetched by querying the endpoint "hyperblock/by-nonce/..." come from another source - from our Observer Nodes - and their timestamp refers to the moment of *final acknowledgement*,
+	// according to the Protocol (detail: the moment when the Metachain notarizes the destination shard block containing the transactions).
+	// Note: the differences are small - e.g. a few seconds.
+	// This inconsistency will be fixed in Hyperblocks API V2.
+
+	txTimestamp := mainnetStartTime + blockRound*roundDurationInSeconds
+
+	return time.Duration(txTimestamp)
+}
diff --git a/platform/elrond/transaction.go b/platform/elrond/transaction.go
index 156bf0e..2ac4219 100644
--- a/platform/elrond/transaction.go
+++ b/platform/elrond/transaction.go
@@ -12,9 +12,9 @@ func (p *Platform) GetTxsByAddress(address string) (blockatlas.TxPage, error) {
 }
 
 // NormalizeTx converts an slice of Elrond transaction info a slice of generic model transaction
-func NormalizeTxs(srcTxs []Transaction, address string) (txs []blockatlas.Tx) {
+func NormalizeTxs(srcTxs []Transaction, address string, block Block) (txs []blockatlas.Tx) {
 	for _, srcTx := range srcTxs {
-		tx, ok := NormalizeTx(srcTx, address)
+		tx, ok := NormalizeTx(srcTx, address, block)
 		if !ok {
 			continue
 		}
@@ -24,14 +24,15 @@ func NormalizeTxs(srcTxs []Transaction, address string) (txs []blockatlas.Tx) {
 }
 
 // NormalizeTx converts an Elrond transaction into the generic model
-func NormalizeTx(srcTx Transaction, address string) (tx blockatlas.Tx, ok bool) {
+func NormalizeTx(srcTx Transaction, address string, block Block) (tx blockatlas.Tx, ok bool) {
 	tx = blockatlas.Tx{
 		ID:       srcTx.Hash,
 		Coin:     coin.Elrond().ID,
-		Date:     int64(srcTx.Timestamp),
+		Date:     int64(srcTx.TxTimestamp(block.Round)),
+		Block:    block.Nonce,
 		From:     srcTx.Sender,
 		To:       srcTx.Receiver,
-		Fee:      blockatlas.Amount(srcTx.Fee),
+		Fee:      srcTx.TxFee(),
 		Status:   srcTx.TxStatus(),
 		Sequence: srcTx.Nonce,
 		Memo:     srcTx.Data,
diff --git a/services/parser/parser.go b/services/parser/parser.go
index 6fcaef3..0a9e209 100644
--- a/services/parser/parser.go
+++ b/services/parser/parser.go
@@ -75,7 +75,7 @@ func parse(params Params) {
 
 	lastParsedBlock, currentBlock, err := GetBlocksIntervalToFetch(params, ctx)
 	if err != nil || lastParsedBlock > currentBlock {
-		log.WithFields(log.Fields{"operation": "fetch GetBlocksIntervalToFetch", "coin": params.Api.Coin().Handle}).Error(err)
+		log.WithFields(log.Fields{"operation": "fetch GetBlocksIntervalToFetch", "lastParsedBlock": lastParsedBlock, "currentBlock": currentBlock, "coin": params.Api.Coin().Handle}).Error(err)
 		time.Sleep(params.ParsingBlocksInterval)
 		return
 	}
diff --git a/services/subscriber/tokens.go b/services/subscriber/tokens.go
index 2a69c2e..1fbe15e 100644
--- a/services/subscriber/tokens.go
+++ b/services/subscriber/tokens.go
@@ -8,7 +8,6 @@ import (
 	"github.com/trustwallet/blockatlas/db"
 	"github.com/trustwallet/blockatlas/db/models"
 	"go.elastic.co/apm"
-	"strconv"
 )
 
 const Tokens Subscriber = "tokens"
@@ -30,7 +29,7 @@ func RunTokensSubscriber(database *db.Instance, delivery amqp.Delivery) {
 			log.Error("Failed to AddAssociationsForAddress: " + err.Error())
 		}
 	}
-	log.WithFields(log.Fields{"service": Tokens}).Info("Subscribed " + strconv.Itoa(len(event)))
+	log.WithFields(log.Fields{"service": Tokens, "count": len(event)}).Info("Subscribed")
 	if err := delivery.Ack(false); err != nil {
 		log.Fatal(err, err)
 	}


Test patch:
diff --git a/api/middleware/prometheus_test.go b/api/middleware/prometheus_test.go
index a3c952f..44e8ad6 100644
--- a/api/middleware/prometheus_test.go
+++ b/api/middleware/prometheus_test.go
@@ -1,24 +1,19 @@
 package middleware
 
 import (
-	"github.com/chenjiandongx/ginprom"
-	"github.com/gin-gonic/gin"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/stretchr/testify/assert"
-	"net/http"
 	"testing"
 )
 
-func TestPrometheus(t *testing.T) {
-	router := gin.New()
-	router.Use(Prometheus())
-	router.GET("/metrics", ginprom.PromHandler(promhttp.Handler()))
-
-	w1 := performRequest("GET", "/metrics", router)
-
-	assert.Equal(t, http.StatusOK, w1.Code)
-	assert.NotNil(t, w1.Body.String())
-}
+//func TestPrometheus(t *testing.T) {
+//	router := gin.New()
+//	router.Use(Prometheus())
+//	router.GET("/metrics", ginprom.PromHandler(promhttp.Handler()))
+//
+//	w1 := performRequest("GET", "/metrics", router)
+//
+//	assert.Equal(t, http.StatusOK, w1.Code)
+//	assert.NotNil(t, w1.Body.String())
+//}
 
 func Test_removeAddress(t *testing.T) {
 	tests := []struct {
diff --git a/platform/elrond/transaction_test.go b/platform/elrond/transaction_test.go
index 6d20d0c..16a15c3 100644
--- a/platform/elrond/transaction_test.go
+++ b/platform/elrond/transaction_test.go
@@ -52,8 +52,10 @@ const txTransferSrc3 = `
 	"data":"test",
 	"signature":"",
 	"timestamp":1588757256,
-	"status":"Not executed",
-	"fee": "5000"
+	"status":"Fail",
+	"fee": "0",
+	"gasPrice": 5,
+    "gasLimit": 1000
 }`
 
 const txTransferSrc4 = `
@@ -86,6 +88,21 @@ const txTransferSrc5 = `
 	"fee": "5000"
 }`
 
+const txTransferSrc6 = `
+{
+	"hash":"30d404cc7a42b0158b95f6adfbf9a517627d60f6c7e497c1442dfdb6460285df",
+	"nonce":25,
+	"value":"2",
+	"receiver":"erd1v0ce6rapup6rwma5sltyv05xhp33u543nex75a7j39vsz9m6squq6mxm7y",
+	"sender":"erd10yagg2vme2jns9zqf9xn8kl86fkc6dr063vnuj0mz2kk2jw0qwuqmfmaw0",
+	"data":"test",
+	"signature":"",
+	"status":"success",
+	"fee": "0",
+	"gasPrice": 5,
+    "gasLimit": 1000
+}`
+
 var txTransfer1Normalized = blockatlas.Tx{
 	ID:       "30d404cc7a42b0158b95f6adfbf9a517627d60f6c7e497c1442dfdb6460285df",
 	Coin:     coin.ERD,
@@ -176,6 +193,25 @@ var txTransfer5Normalized = blockatlas.Tx{
 	Direction: blockatlas.DirectionOutgoing,
 }
 
+var txTransfer6Normalized = blockatlas.Tx{
+	ID:       "30d404cc7a42b0158b95f6adfbf9a517627d60f6c7e497c1442dfdb6460285df",
+	Coin:     coin.ERD,
+	From:     "erd10yagg2vme2jns9zqf9xn8kl86fkc6dr063vnuj0mz2kk2jw0qwuqmfmaw0",
+	To:       "erd1v0ce6rapup6rwma5sltyv05xhp33u543nex75a7j39vsz9m6squq6mxm7y",
+	Fee:      "5000",
+	Status:   blockatlas.StatusCompleted,
+	Memo:     "test",
+	Sequence: 25,
+	Block:    620,
+	Date:     1596121554,
+	Meta: blockatlas.Transfer{
+		Value:    "2",
+		Symbol:   coin.Elrond().Symbol,
+		Decimals: coin.Elrond().Decimals,
+	},
+	Direction: blockatlas.DirectionOutgoing,
+}
+
 type test struct {
 	name        string
 	apiResponse string
@@ -222,7 +258,7 @@ func TestNormalizeTxs(t *testing.T) {
 	_ = json.Unmarshal([]byte(txTransferSrc1), &tx3)
 
 	txs := []Transaction{tx1, tx2, tx3}
-	normalizedTxs := NormalizeTxs(txs, userAddress)
+	normalizedTxs := NormalizeTxs(txs, userAddress, Block{})
 	require.Equal(t, len(txs), len(normalizedTxs))
 }
 
@@ -234,7 +270,7 @@ func testNormalize(t *testing.T, _test *test) {
 		return
 	}
 
-	normalizedTx, ok := NormalizeTx(tx, tx.Sender)
+	normalizedTx, ok := NormalizeTx(tx, tx.Sender, Block{})
 	require.True(t, ok, _test.name+": cannot normalize tx")
 
 	resJSON, err := json.Marshal(&normalizedTx)
@@ -245,3 +281,18 @@ func testNormalize(t *testing.T, _test *test) {
 
 	require.Equal(t, string(dstJSON), string(resJSON))
 }
+
+func TestNormalizeTxsFromHyperblock(t *testing.T) {
+	var tx Transaction
+
+	_ = json.Unmarshal([]byte(txTransferSrc6), &tx)
+	txs := []Transaction{tx}
+
+	normalizedTxs := NormalizeTxs(txs, userAddress, Block{
+		Nonce: 620,
+		Round: 659,
+	})
+	require.Equal(t, len(txs), len(normalizedTxs))
+
+	require.Equal(t, []blockatlas.Tx{txTransfer6Normalized}, normalizedTxs)
+}