Repository: algorand/indexer
Pull request #484 (2021-05-03 15:02:03)

Converting GT/LT uint struct fields to uint pointers 
Addresses https://github.com/algorand/indexer/issues/144

I wrote no new tests for this, though it probably should have some.

I chose to convert any struct fields that contained GT/LT or GreaterThan/LessThan that were uint64 to *uint64.  A 0 value for AlgosLT will never (?) happen in reality but it offended me to leave it out.

I redefined uint64Ptr in one of the test files because exporting it from another one was weird. 






tolikzinovyev: Thanks, looks good! I will let @winder also take a look.

Patch:
diff --git a/Makefile b/Makefile
index f72617f..bdf5555 100644
--- a/Makefile
+++ b/Makefile
@@ -68,4 +68,4 @@ sign:
 test-package:
 	mule/e2e.sh
 
-.PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer
+.PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go
diff --git a/accounting/rewind.go b/accounting/rewind.go
index 181a1da..8e8bfa0 100644
--- a/accounting/rewind.go
+++ b/accounting/rewind.go
@@ -205,7 +205,7 @@ func AccountAtRound(account models.Account, round uint64, db idb.IndexerDb) (acc
 				return
 			}
 			var proto types.ConsensusParams
-			proto, err = db.GetProto(string(blockheader.CurrentProtocol))
+			proto, err = types.Protocol(string(blockheader.CurrentProtocol))
 			if err != nil {
 				return
 			}
diff --git a/api/converter_utils.go b/api/converter_utils.go
index a85d65b..14dcc7b 100644
--- a/api/converter_utils.go
+++ b/api/converter_utils.go
@@ -528,11 +528,11 @@ func transactionParamsToTransactionFilter(params generated.SearchForTransactions
 
 	// filter Algos or Asset but not both.
 	if filter.AssetID != 0 {
-		filter.AssetAmountLT = uintOrDefault(params.CurrencyLessThan)
-		filter.AssetAmountGT = uintOrDefault(params.CurrencyGreaterThan)
+		filter.AssetAmountLT = params.CurrencyLessThan
+		filter.AssetAmountGT = params.CurrencyGreaterThan
 	} else {
-		filter.AlgosLT = uintOrDefault(params.CurrencyLessThan)
-		filter.AlgosGT = uintOrDefault(params.CurrencyGreaterThan)
+		filter.AlgosLT = params.CurrencyLessThan
+		filter.AlgosGT = params.CurrencyGreaterThan
 	}
 	filter.Round = params.Round
 
diff --git a/api/handlers.go b/api/handlers.go
index 6a207ad..1ef1bbc 100644
--- a/api/handlers.go
+++ b/api/handlers.go
@@ -155,11 +155,11 @@ func (si *ServerImplementation) SearchForAccounts(ctx echo.Context, params gener
 
 	// Set GT/LT on Algos or Asset depending on whether or not an assetID was specified
 	if options.HasAssetID == 0 {
-		options.AlgosGreaterThan = uintOrDefault(params.CurrencyGreaterThan)
-		options.AlgosLessThan = uintOrDefault(params.CurrencyLessThan)
+		options.AlgosGreaterThan = params.CurrencyGreaterThan
+		options.AlgosLessThan = params.CurrencyLessThan
 	} else {
-		options.AssetGT = uintOrDefault(params.CurrencyGreaterThan)
-		options.AssetLT = uintOrDefault(params.CurrencyLessThan)
+		options.AssetGT = params.CurrencyGreaterThan
+		options.AssetLT = params.CurrencyLessThan
 	}
 
 	if params.Next != nil {
@@ -308,8 +308,8 @@ func (si *ServerImplementation) LookupAssetByID(ctx echo.Context, assetID uint64
 func (si *ServerImplementation) LookupAssetBalances(ctx echo.Context, assetID uint64, params generated.LookupAssetBalancesParams) error {
 	query := idb.AssetBalanceQuery{
 		AssetID:        assetID,
-		AmountGT:       uintOrDefault(params.CurrencyGreaterThan),
-		AmountLT:       uintOrDefault(params.CurrencyLessThan),
+		AmountGT:       params.CurrencyGreaterThan,
+		AmountLT:       params.CurrencyLessThan,
 		IncludeDeleted: boolOrDefault(params.IncludeAll),
 		Limit:          min(uintOrDefaultValue(params.Limit, defaultBalancesLimit), maxBalancesLimit),
 	}
diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go
index 5e6e3a8..f5bf0dc 100644
--- a/cmd/algorand-indexer/daemon.go
+++ b/cmd/algorand-indexer/daemon.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -57,9 +56,6 @@ var daemonCmd = &cobra.Command{
 			bot, err = fetcher.ForNetAndToken(algodAddr, algodToken, logger)
 			maybeFail(err, "fetcher setup, %v", err)
 		} else if algodDataDir != "" {
-			if genesisJSONPath == "" {
-				genesisJSONPath = filepath.Join(algodDataDir, "genesis.json")
-			}
 			bot, err = fetcher.ForDataDir(algodDataDir, logger)
 			maybeFail(err, "fetcher setup, %v", err)
 		} else {
@@ -71,13 +67,6 @@ var daemonCmd = &cobra.Command{
 			opts.ReadOnly = true
 		}
 		db := indexerDbFromFlags(opts)
-		if !noAlgod {
-			// Only do this if we're going to be writing
-			// to the db, to allow for read-only query
-			// servers that hit the db backend.
-			err := importer.ImportProto(db)
-			maybeFail(err, "import proto, %v", err)
-		}
 		if bot != nil {
 			logger.Info("Initializing block import handler.")
 			maxRound, err := db.GetMaxRoundLoaded()
@@ -100,7 +89,7 @@ var daemonCmd = &cobra.Command{
 				waitForDBAvailable(db)
 
 				// Initial import if needed.
-				importer.InitialImport(db, genesisJSONPath, logger)
+				importer.InitialImport(db, genesisJSONPath, bot.Algod(), logger)
 
 				logger.Info("Starting block importer.")
 				bot.Run()
diff --git a/cmd/algorand-indexer/main.go b/cmd/algorand-indexer/main.go
index 6ed3c37..50f6ded 100644
--- a/cmd/algorand-indexer/main.go
+++ b/cmd/algorand-indexer/main.go
@@ -29,7 +29,7 @@ func maybeFail(err error, errfmt string, params ...interface{}) {
 var rootCmd = &cobra.Command{
 	Use:   "indexer",
 	Short: "Algorand Indexer",
-	Long:  `indexer imports blocks from an algod node or from local files into an SQL database for querying. indexer is a daemon that can serve queries from that database.`,
+	Long:  `Indexer imports blocks from an algod node into an SQL database for querying. It is a daemon that can serve queries from that database.`,
 	Args:  cobra.NoArgs,
 	Run: func(cmd *cobra.Command, args []string) {
 		//If no arguments passed, we should fallback to help
@@ -106,6 +106,7 @@ func init() {
 	logger.SetLevel(log.InfoLevel)
 
 	rootCmd.AddCommand(importCmd)
+	importCmd.Hidden = true
 	rootCmd.AddCommand(daemonCmd)
 	rootCmd.AddCommand(resetCmd)
 	resetCmd.Hidden = true
diff --git a/go.mod b/go.mod
index 121c0c5..27b58b1 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/algorand/indexer
 go 1.13
 
 require (
-	github.com/algorand/go-algorand-sdk v1.5.1
+	github.com/algorand/go-algorand-sdk v1.7.0
 	github.com/algorand/go-codec/codec v1.1.7
 	github.com/algorand/oapi-codegen v1.3.5-algorand5
 	github.com/getkin/kin-openapi v0.19.0
@@ -17,6 +17,5 @@ require (
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1
 	github.com/valyala/fasttemplate v1.2.0 // indirect
-	github.com/vektra/mockery v1.1.2 // indirect
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect
 )
diff --git a/go.sum b/go.sum
index 5b2f703..6ba21ea 100644
--- a/go.sum
+++ b/go.sum
@@ -55,8 +55,10 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/algorand/go-algorand-sdk v1.5.1 h1:5THZvMOlfAc+HHRhbV/V89LJC4k0EzhK1mn3oizSt+c=
-github.com/algorand/go-algorand-sdk v1.5.1/go.mod h1:U12d8fTN/CyKPR1HObrt51ITxb6OgXxpGCH743Ds2GQ=
+github.com/algorand/go-algorand-sdk v1.6.0 h1:0xl694OAz611CNuZ88l510DPvzTPH4D1dJX+KNNB428=
+github.com/algorand/go-algorand-sdk v1.6.0/go.mod h1:U12d8fTN/CyKPR1HObrt51ITxb6OgXxpGCH743Ds2GQ=
+github.com/algorand/go-algorand-sdk v1.7.0 h1:eRg9UQy7FndRE5nTgniDpbyjTvP2FUKX4/0e8NdXn/E=
+github.com/algorand/go-algorand-sdk v1.7.0/go.mod h1:U12d8fTN/CyKPR1HObrt51ITxb6OgXxpGCH743Ds2GQ=
 github.com/algorand/go-codec v1.1.7 h1:6nvCh2nfgnfkaoVHKQyk2wxyl2GQBAlI7IkbqbB/e4s=
 github.com/algorand/go-codec v1.1.7/go.mod h1:pVLQYhIVCsx9D3iy4W4Qqi0SKhx6IVhMwOvj/agFL4g=
 github.com/algorand/go-codec/codec v1.1.7 h1:EFOyWf5duxbh2ru+AW1YDgmZ+MRVgqklELSqTArgp3M=
@@ -304,7 +306,6 @@ github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz
 github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI=
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
-github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.5.1 h1:Jn6HYxiYrtQ92CopqJLvfPCJUrrruw1+1cn0jM9dKrI=
 github.com/lib/pq v1.5.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -449,8 +450,6 @@ github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy
 github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.0 h1:y3yXRCoDvC2HTtIHvL2cc7Zd+bqA+zqDO6oQzsJO07E=
 github.com/valyala/fasttemplate v1.2.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
-github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4=
-github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
@@ -683,7 +682,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200423205358-59e73619c742 h1:9OGWpORUXvk8AsaBJlpzzDx7Srv/rSK6rvjcsJq4rJo=
 golang.org/x/tools v0.0.0-20200423205358-59e73619c742/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
diff --git a/idb/dummy.go b/idb/dummy.go
index 2bfe32e..a65b76c 100644
--- a/idb/dummy.go
+++ b/idb/dummy.go
@@ -46,32 +46,11 @@ func (db *dummyIndexerDb) CommitBlock(round uint64, timestamp int64, rewardsleve
 	return nil
 }
 
-// AlreadyImported is part of idb.IndexerDB
-func (db *dummyIndexerDb) AlreadyImported(path string) (imported bool, err error) {
-	return false, nil
-}
-
-// MarkImported is part of idb.IndexerDB
-func (db *dummyIndexerDb) MarkImported(path string) (err error) {
-	return nil
-}
-
 // LoadGenesis is part of idb.IndexerDB
 func (db *dummyIndexerDb) LoadGenesis(genesis types.Genesis) (err error) {
 	return nil
 }
 
-// SetProto is part of idb.IndexerDB
-func (db *dummyIndexerDb) SetProto(version string, proto types.ConsensusParams) (err error) {
-	return nil
-}
-
-// GetProto is part of idb.IndexerDB
-func (db *dummyIndexerDb) GetProto(version string) (proto types.ConsensusParams, err error) {
-	err = nil
-	return
-}
-
 // GetImportState is part of idb.IndexerDB
 func (db *dummyIndexerDb) GetImportState() (is ImportState, err error) {
 	return ImportState{}, nil
@@ -221,12 +200,7 @@ type IndexerDb interface {
 	AddTransaction(round uint64, intra int, txtypeenum int, assetid uint64, txn types.SignedTxnWithAD, participation [][]byte) error
 	CommitBlock(round uint64, timestamp int64, rewardslevel uint64, headerbytes []byte) error
 
-	AlreadyImported(path string) (imported bool, err error)
-	MarkImported(path string) (err error)
-
 	LoadGenesis(genesis types.Genesis) (err error)
-	SetProto(version string, proto types.ConsensusParams) (err error)
-	GetProto(version string) (proto types.ConsensusParams, err error)
 
 	// GetImportState returns ErrorNotInitialized if there is no import state.
 	GetImportState() (is ImportState, err error)
@@ -306,18 +280,18 @@ type TransactionFilter struct {
 	OffsetGT   *uint64 // nil for no filter
 	SigType    string  // ["", "sig", "msig", "lsig"]
 	NotePrefix []byte
-	AlgosGT    uint64 // implictly filters on "pay" txns for Algos > this. This will be a slightly faster query than EffectiveAmountGt.
-	AlgosLT    uint64
+	AlgosGT    *uint64 // implictly filters on "pay" txns for Algos > this. This will be a slightly faster query than EffectiveAmountGT.
+	AlgosLT    *uint64
 	RekeyTo    *bool // nil for no filter
 
 	AssetID       uint64 // filter transactions relevant to an asset
-	AssetAmountGT uint64
-	AssetAmountLT uint64
+	AssetAmountGT *uint64
+	AssetAmountLT *uint64
 
 	ApplicationID uint64 // filter transactions relevant to an application
 
-	EffectiveAmountGt uint64 // Algo: Amount + CloseAmount > x
-	EffectiveAmountLt uint64 // Algo: Amount + CloseAmount < x
+	EffectiveAmountGT *uint64 // Algo: Amount + CloseAmount > x
+	EffectiveAmountLT *uint64 // Algo: Amount + CloseAmount < x
 
 	// pointer to last returned object of previous query
 	NextToken string
@@ -334,16 +308,16 @@ type AccountQueryOptions struct {
 	EqualToAuthAddr []byte
 
 	// Filter on accounts with current balance greater than x
-	AlgosGreaterThan uint64
+	AlgosGreaterThan *uint64
 	// Filter on accounts with current balance less than x.
-	AlgosLessThan uint64
+	AlgosLessThan *uint64
 
 	// HasAssetID, AssetGT, and AssetLT are implemented in Go code
 	// after data has returned from Postgres and thus are slightly
 	// less efficient. They will turn on IncludeAssetHoldings.
 	HasAssetID uint64
-	AssetGT    uint64
-	AssetLT    uint64
+	AssetGT    *uint64
+	AssetLT    *uint64
 
 	HasAppID uint64
 
@@ -397,8 +371,8 @@ type AssetRow struct {
 // AssetBalanceQuery is a parameter object with all of the asset balance filter options.
 type AssetBalanceQuery struct {
 	AssetID  uint64
-	AmountGT uint64 // only rows > this
-	AmountLT uint64 // only rows < this
+	AmountGT *uint64 // only rows > this
+	AmountLT *uint64 // only rows < this
 
 	// IncludeDeleted indicated whether to include deleted AssetHoldingss in the results.
 	IncludeDeleted bool
diff --git a/idb/mocks/IndexerDb.go b/idb/mocks/IndexerDb.go
index 2346849..d2dbe26 100644
--- a/idb/mocks/IndexerDb.go
+++ b/idb/mocks/IndexerDb.go
@@ -32,27 +32,6 @@ func (_m *IndexerDb) AddTransaction(round uint64, intra int, txtypeenum int, ass
 	return r0
 }
 
-// AlreadyImported provides a mock function with given fields: path
-func (_m *IndexerDb) AlreadyImported(path string) (bool, error) {
-	ret := _m.Called(path)
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(string) bool); ok {
-		r0 = rf(path)
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(string) error); ok {
-		r1 = rf(path)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
 // Applications provides a mock function with given fields: ctx, filter
 func (_m *IndexerDb) Applications(ctx context.Context, filter *generated.SearchForApplicationsParams) (<-chan idb.ApplicationRow, uint64) {
 	ret := _m.Called(ctx, filter)
@@ -289,27 +268,6 @@ func (_m *IndexerDb) GetMaxRoundLoaded() (uint64, error) {
 	return r0, r1
 }
 
-// GetProto provides a mock function with given fields: version
-func (_m *IndexerDb) GetProto(version string) (types.ConsensusParams, error) {
-	ret := _m.Called(version)
-
-	var r0 types.ConsensusParams
-	if rf, ok := ret.Get(0).(func(string) types.ConsensusParams); ok {
-		r0 = rf(version)
-	} else {
-		r0 = ret.Get(0).(types.ConsensusParams)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(string) error); ok {
-		r1 = rf(version)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
 // GetSpecialAccounts provides a mock function with given fields:
 func (_m *IndexerDb) GetSpecialAccounts() (idb.SpecialAccounts, error) {
 	ret := _m.Called()
@@ -366,20 +324,6 @@ func (_m *IndexerDb) LoadGenesis(genesis types.Genesis) error {
 	return r0
 }
 
-// MarkImported provides a mock function with given fields: path
-func (_m *IndexerDb) MarkImported(path string) error {
-	ret := _m.Called(path)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(string) error); ok {
-		r0 = rf(path)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
 // Reset provides a mock function with given fields:
 func (_m *IndexerDb) Reset() error {
 	ret := _m.Called()
@@ -408,20 +352,6 @@ func (_m *IndexerDb) SetImportState(_a0 idb.ImportState) error {
 	return r0
 }
 
-// SetProto provides a mock function with given fields: version, proto
-func (_m *IndexerDb) SetProto(version string, proto types.ConsensusParams) error {
-	ret := _m.Called(version, proto)
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func(string, types.ConsensusParams) error); ok {
-		r0 = rf(version, proto)
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
 // StartBlock provides a mock function with given fields:
 func (_m *IndexerDb) StartBlock() error {
 	ret := _m.Called()
diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go
index b5bd3b8..f859a8a 100644
--- a/idb/postgres/postgres.go
+++ b/idb/postgres/postgres.go
@@ -49,7 +49,7 @@ func OpenPostgres(connection string, opts idb.IndexerDbOptions, log *log.Logger)
 	db, err := sql.Open("postgres", connection)
 
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("connecting to postgres: %v", err)
 	}
 
 	if strings.Contains(connection, "readonly") {
@@ -77,6 +77,9 @@ func openPostgres(db *sql.DB, opts idb.IndexerDbOptions, logger *log.Logger) (pd
 	// e.g. a user named "readonly" is in the connection string
 	if !opts.ReadOnly {
 		err = pdb.init(opts)
+		if err != nil {
+			return nil, fmt.Errorf("initializing postgres: %v", err)
+		}
 	}
 	return
 }
@@ -148,11 +151,15 @@ func (db *IndexerDb) init(opts idb.IndexerDbOptions) (err error) {
 	// new database, run setup
 	_, err = db.db.Exec(setup_postgres_sql)
 	if err != nil {
-		return
+		return fmt.Errorf("unable to setup postgres: %v", err)
 	}
 
 	err = db.markMigrationsAsDone()
-	return
+	if err != nil {
+		return fmt.Errorf("unable to confirm migration: %v", err)
+	}
+
+	return nil
 }
 
 // Reset is part of idb.IndexerDB
@@ -166,20 +173,6 @@ func (db *IndexerDb) Reset() (err error) {
 	return
 }
 
-// AlreadyImported is part of idb.IndexerDB
-func (db *IndexerDb) AlreadyImported(path string) (imported bool, err error) {
-	row := db.db.QueryRow(`SELECT COUNT(path) FROM imported WHERE path = $1`, path)
-	numpath := 0
-	err = row.Scan(&numpath)
-	return numpath == 1, err
-}
-
-// MarkImported is part of idb.IndexerDB
-func (db *IndexerDb) MarkImported(path string) (err error) {
-	_, err = db.db.Exec(`INSERT INTO imported (path) VALUES ($1)`, path)
-	return err
-}
-
 // StartBlock is part of idb.IndexerDB
 func (db *IndexerDb) StartBlock() (err error) {
 	db.txrows = make([][]interface{}, 0, 6000)
@@ -389,35 +382,6 @@ func (db *IndexerDb) LoadGenesis(genesis types.Genesis) (err error) {
 
 }
 
-// SetProto is part of idb.IndexerDB
-func (db *IndexerDb) SetProto(version string, proto types.ConsensusParams) (err error) {
-	pj := idb.JSONOneLine(proto)
-	if err != nil {
-		return err
-	}
-	_, err = db.db.Exec(`INSERT INTO protocol (version, proto) VALUES ($1, $2) ON CONFLICT (version) DO UPDATE SET proto = EXCLUDED.proto`, version, pj)
-	return err
-}
-
-// GetProto is part of idb.IndexerDB
-func (db *IndexerDb) GetProto(version string) (proto types.ConsensusParams, err error) {
-	proto, hit := db.protoCache[version]
-	if hit {
-		return
-	}
-	row := db.db.QueryRow(`SELECT proto FROM protocol WHERE version = $1`, version)
-	var protostr string
-	err = row.Scan(&protostr)
-	if err != nil {
-		return
-	}
-	err = json.Decode([]byte(protostr), &proto)
-	if err == nil {
-		db.protoCache[version] = proto
-	}
-	return
-}
-
 func (db *IndexerDb) getMetastate(key string) (jsonStrValue string, err error) {
 	row := db.db.QueryRow(`SELECT v FROM metastate WHERE k = $1`, key)
 	err = row.Scan(&jsonStrValue)
@@ -1535,14 +1499,14 @@ func buildTransactionQuery(tf idb.TransactionFilter) (query string, whereArgs []
 		whereArgs = append(whereArgs, creatableID)
 		partNumber++
 	}
-	if tf.AssetAmountGT != 0 {
+	if tf.AssetAmountGT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("(t.txn -> 'txn' -> 'aamt')::bigint > $%d", partNumber))
-		whereArgs = append(whereArgs, tf.AssetAmountGT)
+		whereArgs = append(whereArgs, *tf.AssetAmountGT)
 		partNumber++
 	}
-	if tf.AssetAmountLT != 0 {
+	if tf.AssetAmountLT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("(t.txn -> 'txn' -> 'aamt')::bigint < $%d", partNumber))
-		whereArgs = append(whereArgs, tf.AssetAmountLT)
+		whereArgs = append(whereArgs, *tf.AssetAmountLT)
 		partNumber++
 	}
 	if tf.TypeEnum != 0 {
@@ -1585,24 +1549,24 @@ func buildTransactionQuery(tf idb.TransactionFilter) (query string, whereArgs []
 		whereArgs = append(whereArgs, tf.NotePrefix)
 		partNumber++
 	}
-	if tf.AlgosGT != 0 {
+	if tf.AlgosGT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("(t.txn -> 'txn' -> 'amt')::bigint > $%d", partNumber))
-		whereArgs = append(whereArgs, tf.AlgosGT)
+		whereArgs = append(whereArgs, *tf.AlgosGT)
 		partNumber++
 	}
-	if tf.AlgosLT != 0 {
+	if tf.AlgosLT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("(t.txn -> 'txn' -> 'amt')::bigint < $%d", partNumber))
-		whereArgs = append(whereArgs, tf.AlgosLT)
+		whereArgs = append(whereArgs, *tf.AlgosLT)
 		partNumber++
 	}
-	if tf.EffectiveAmountGt != 0 {
+	if tf.EffectiveAmountGT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("((t.txn -> 'ca')::bigint + (t.txn -> 'txn' -> 'amt')::bigint) > $%d", partNumber))
-		whereArgs = append(whereArgs, tf.EffectiveAmountGt)
+		whereArgs = append(whereArgs, *tf.EffectiveAmountGT)
 		partNumber++
 	}
-	if tf.EffectiveAmountLt != 0 {
+	if tf.EffectiveAmountLT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("((t.txn -> 'ca')::bigint + (t.txn -> 'txn' -> 'amt')::bigint) < $%d", partNumber))
-		whereArgs = append(whereArgs, tf.EffectiveAmountLt)
+		whereArgs = append(whereArgs, *tf.EffectiveAmountLT)
 		partNumber++
 	}
 	if tf.RekeyTo != nil && (*tf.RekeyTo) {
@@ -1995,7 +1959,7 @@ func (db *IndexerDb) yieldAccountsThread(req *getAccountsRequest) {
 			account.PendingRewards = 0
 		} else {
 			// TODO: pending rewards calculation doesn't belong in database layer (this is just the most covenient place which has all the data)
-			proto, err := db.GetProto(string(req.blockheader.CurrentProtocol))
+			proto, err := types.Protocol(string(req.blockheader.CurrentProtocol))
 			if err != nil {
 				err = fmt.Errorf("get protocol err (%s) %v", req.blockheader.CurrentProtocol, err)
 				req.out <- idb.AccountRow{Error: err}
@@ -2345,6 +2309,13 @@ func nullableBoolPtr(x sql.NullBool) *bool {
 	return &x.Bool
 }
 
+func uintOrDefault(x *uint64) uint64 {
+	if x != nil {
+		return *x
+	}
+	return 0
+}
+
 func uint64Ptr(x uint64) *uint64 {
 	out := new(uint64)
 	*out = x
@@ -2433,8 +2404,8 @@ func (db *IndexerDb) GetAccounts(ctx context.Context, opts idb.AccountQueryOptio
 
 	if opts.HasAssetID != 0 {
 		opts.IncludeAssetHoldings = true
-	} else if (opts.AssetGT != 0) || (opts.AssetLT != 0) {
-		err := fmt.Errorf("AssetGT=%d, AssetLT=%d, but HasAssetID=%d", opts.AssetGT, opts.AssetLT, opts.HasAssetID)
+	} else if (opts.AssetGT != nil) || (opts.AssetLT != nil) {
+		err := fmt.Errorf("AssetGT=%d, AssetLT=%d, but HasAssetID=%d", uintOrDefault(opts.AssetGT), uintOrDefault(opts.AssetLT), opts.HasAssetID)
 		out <- idb.AccountRow{Error: err}
 		close(out)
 		return out, 0
@@ -2517,14 +2488,14 @@ func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions) (query stri
 		aq := fmt.Sprintf("SELECT addr FROM account_asset WHERE assetid = $%d", partNumber)
 		whereArgs = append(whereArgs, opts.HasAssetID)
 		partNumber++
-		if opts.AssetGT != 0 {
+		if opts.AssetGT != nil {
 			aq += fmt.Sprintf(" AND amount > $%d", partNumber)
-			whereArgs = append(whereArgs, opts.AssetGT)
+			whereArgs = append(whereArgs, *opts.AssetGT)
 			partNumber++
 		}
-		if opts.AssetLT != 0 {
+		if opts.AssetLT != nil {
 			aq += fmt.Sprintf(" AND amount < $%d", partNumber)
-			whereArgs = append(whereArgs, opts.AssetLT)
+			whereArgs = append(whereArgs, *opts.AssetLT)
 			partNumber++
 		}
 		aq = "qasf AS (" + aq + ")"
@@ -2546,14 +2517,14 @@ func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions) (query stri
 		whereArgs = append(whereArgs, opts.EqualToAddress)
 		partNumber++
 	}
-	if opts.AlgosGreaterThan != 0 {
+	if opts.AlgosGreaterThan != nil {
 		whereParts = append(whereParts, fmt.Sprintf("a.microalgos > $%d", partNumber))
-		whereArgs = append(whereArgs, opts.AlgosGreaterThan)
+		whereArgs = append(whereArgs, *opts.AlgosGreaterThan)
 		partNumber++
 	}
-	if opts.AlgosLessThan != 0 {
+	if opts.AlgosLessThan != nil {
 		whereParts = append(whereParts, fmt.Sprintf("a.microalgos < $%d", partNumber))
-		whereArgs = append(whereArgs, opts.AlgosLessThan)
+		whereArgs = append(whereArgs, *opts.AlgosLessThan)
 		partNumber++
 	}
 	if !opts.IncludeDeleted {
@@ -2764,14 +2735,14 @@ func (db *IndexerDb) AssetBalances(ctx context.Context, abq idb.AssetBalanceQuer
 		whereArgs = append(whereArgs, abq.AssetID)
 		partNumber++
 	}
-	if abq.AmountGT != 0 {
+	if abq.AmountGT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("aa.amount > $%d", partNumber))
-		whereArgs = append(whereArgs, abq.AmountGT)
+		whereArgs = append(whereArgs, *abq.AmountGT)
 		partNumber++
 	}
-	if abq.AmountLT != 0 {
+	if abq.AmountLT != nil {
 		whereParts = append(whereParts, fmt.Sprintf("aa.amount < $%d", partNumber))
-		whereArgs = append(whereArgs, abq.AmountLT)
+		whereArgs = append(whereArgs, *abq.AmountLT)
 		partNumber++
 	}
 	if len(abq.PrevAddress) != 0 {
diff --git a/idb/postgres/setup_postgres.sql b/idb/postgres/setup_postgres.sql
index 5947071..ab8e12d 100644
--- a/idb/postgres/setup_postgres.sql
+++ b/idb/postgres/setup_postgres.sql
@@ -2,11 +2,6 @@
 --
 -- TODO? replace all 'addr bytea' with 'addr_id bigint' and a mapping table? makes addrs an 8 byte int that fits in a register instead of a 32 byte string
 
-CREATE TABLE IF NOT EXISTS protocol (
-  version text PRIMARY KEY,
-  proto jsonb
-);
-
 CREATE TABLE IF NOT EXISTS block_header (
 round bigint PRIMARY KEY,
 realtime timestamp without time zone NOT NULL,
@@ -45,9 +40,6 @@ intra smallint NOT NULL
 -- For query account transactions
 CREATE UNIQUE INDEX IF NOT EXISTS txn_participation_i ON txn_participation ( addr, round DESC, intra DESC );
 
--- bookeeping for local file import
-CREATE TABLE IF NOT EXISTS imported (path text);
-
 -- expand data.basics.AccountData
 CREATE TABLE IF NOT EXISTS account (
   addr bytea primary key,
diff --git a/idb/postgres/setup_postgres_sql.go b/idb/postgres/setup_postgres_sql.go
index 8f86e28..defb905 100644
--- a/idb/postgres/setup_postgres_sql.go
+++ b/idb/postgres/setup_postgres_sql.go
@@ -6,11 +6,6 @@ const setup_postgres_sql = `-- This file is setup_postgres.sql which gets compil
 --
 -- TODO? replace all 'addr bytea' with 'addr_id bigint' and a mapping table? makes addrs an 8 byte int that fits in a register instead of a 32 byte string
 
-CREATE TABLE IF NOT EXISTS protocol (
-  version text PRIMARY KEY,
-  proto jsonb
-);
-
 CREATE TABLE IF NOT EXISTS block_header (
 round bigint PRIMARY KEY,
 realtime timestamp without time zone NOT NULL,
@@ -49,9 +44,6 @@ intra smallint NOT NULL
 -- For query account transactions
 CREATE UNIQUE INDEX IF NOT EXISTS txn_participation_i ON txn_participation ( addr, round DESC, intra DESC );
 
--- bookeeping for local file import
-CREATE TABLE IF NOT EXISTS imported (path text);
-
 -- expand data.basics.AccountData
 CREATE TABLE IF NOT EXISTS account (
   addr bytea primary key,
diff --git a/importer/helper.go b/importer/helper.go
index 6b76150..0c4b85e 100644
--- a/importer/helper.go
+++ b/importer/helper.go
@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/algorand/go-algorand-sdk/client/v2/algod"
 	"github.com/algorand/go-algorand-sdk/encoding/json"
 	log "github.com/sirupsen/logrus"
 
@@ -57,9 +58,6 @@ type ImportState struct {
 
 // Import is the main ImportHelper function that glues together a directory full of block files and an Importer objects.
 func (h *ImportHelper) Import(db idb.IndexerDb, args []string) {
-	err := ImportProto(db)
-	maybeFail(err, h.Log, "import proto, %v", err)
-
 	imp := NewDBImporter(db)
 	blocks := 0
 	txCount := 0
@@ -94,7 +92,7 @@ func (h *ImportHelper) Import(db idb.IndexerDb, args []string) {
 	var startRound uint64 = 0
 	accountingRounds := 0
 	if err == idb.ErrorNotInitialized {
-		if InitialImport(db, h.GenesisJSONPath, h.Log) {
+		if InitialImport(db, h.GenesisJSONPath, nil, h.Log) {
 			accountingRounds++
 		}
 	} else {
@@ -185,11 +183,6 @@ func importFile(db idb.IndexerDb, imp Importer, fname string, l *log.Logger) (bl
 	blocks = 0
 	txCount = 0
 	var btxns int
-	imported, err := db.AlreadyImported(fname)
-	maybeFail(err, l, "%s: %v", fname, err)
-	if imported {
-		return
-	}
 	l.Infof("importing %s ...", fname)
 	if strings.HasSuffix(fname, ".tar") {
 		fin, err := os.Open(fname)
@@ -217,8 +210,6 @@ func importFile(db idb.IndexerDb, imp Importer, fname string, l *log.Logger) (bl
 		blocks++
 		txCount += btxns
 	}
-	err = db.MarkImported(fname)
-	maybeFail(err, l, "%s: %v", fname, err)
 	return
 }
 
@@ -237,27 +228,37 @@ func loadGenesis(db idb.IndexerDb, in io.Reader) (err error) {
 }
 
 // InitialImport imports the genesis block if needed. Returns true if the initial import occurred.
-func InitialImport(db idb.IndexerDb, genesisJSONPath string, l *log.Logger) bool {
-	state, err := db.GetImportState()
+func InitialImport(db idb.IndexerDb, genesisJSONPath string, client *algod.Client, l *log.Logger) bool {
+	log.Infof("Initial import. Client is null:  %t", client == nil)
+	_, err := db.GetImportState()
 
-	// Only import when the state is not found.
-	if err == idb.ErrorNotInitialized {
-		state.AccountRound = -1
-		if genesisJSONPath != "" {
-			l.Infof("loading genesis %s", genesisJSONPath)
-			// if we're given no previous state and we're given a genesis file, import it as initial account state
-			gf, err := os.Open(genesisJSONPath)
-			maybeFail(err, l, "%s: %v", genesisJSONPath, err)
-			err = loadGenesis(db, gf)
-			maybeFail(err, l, "%s: could not load genesis json, %v", genesisJSONPath, err)
-			return true
-		}
-		l.Errorf("no import state recorded; need --genesis genesis.json file to get started")
-		os.Exit(1)
+	// Exit immediately or crash if we don't see ErrorNotInitialized.
+	if err != idb.ErrorNotInitialized {
+		maybeFail(err, l, "getting import state, %v", err)
 		return false
 	}
-	maybeFail(err, l, "getting import state, %v", err)
-	return false
+
+	// Import genesis file from file or algod.
+	var genesisReader io.Reader
+
+	if genesisJSONPath != "" {
+		// Read file if specified.
+		l.Infof("loading genesis file %s", genesisJSONPath)
+		genesisReader, err = os.Open(genesisJSONPath)
+		maybeFail(err, l, "unable to read genesis file %s", genesisJSONPath)
+	} else if client != nil {
+		// Fallback to asking algod for genesis if file is not specified.
+		l.Infof("fetching genesis from algod")
+		genesisString, err := client.GetGenesis().Do(context.Background())
+		maybeFail(err, l, "unable to fetch genesis from algod")
+		genesisReader = strings.NewReader(genesisString)
+	} else {
+		l.Fatal("Neither genesis file path or algod client provided for initial import.")
+	}
+
+	err = loadGenesis(db, genesisReader)
+	maybeFail(err, l, "%s: could not load genesis json, %v", genesisJSONPath, err)
+	return true
 }
 
 // UpdateAccounting triggers an accounting update.
diff --git a/importer/importer.go b/importer/importer.go
index c8d3cdb..70244fb 100644
--- a/importer/importer.go
+++ b/importer/importer.go
@@ -134,15 +134,3 @@ func (imp *dbImporter) ImportDecodedBlock(blockContainer *types.EncodedBlockCert
 func NewDBImporter(db idb.IndexerDb) Importer {
 	return &dbImporter{db: db}
 }
-
-// ImportProto writes compiled-in protocol information to the database
-func ImportProto(db idb.IndexerDb) (err error) {
-	types.ForeachProtocol(func(version string, proto types.ConsensusParams) (err error) {
-		err = db.SetProto(version, proto)
-		if err != nil {
-			return fmt.Errorf("db set proto %s, %v", version, err)
-		}
-		return nil
-	})
-	return nil
-}
diff --git a/types/consensus.go b/types/consensus.go
index 597aef8..6e96de9 100644
--- a/types/consensus.go
+++ b/types/consensus.go
@@ -49,18 +49,3 @@ func Protocol(version string) (proto ConsensusParams, err error) {
 	}
 	return
 }
-
-// ForeachProtocol executes a function against each protocol version.
-func ForeachProtocol(f func(version string, proto ConsensusParams) error) (err error) {
-	err = ensureProtos()
-	if err != nil {
-		return
-	}
-	for version, proto := range protocols {
-		err = f(version, proto)
-		if err != nil {
-			return
-		}
-	}
-	return nil
-}


Test patch:
diff --git a/api/handlers_test.go b/api/handlers_test.go
index c03d6c5..47475a3 100644
--- a/api/handlers_test.go
+++ b/api/handlers_test.go
@@ -115,12 +115,12 @@ func TestTransactionParamToTransactionFilter(t *testing.T) {
 				AssetID:           4,
 				BeforeTime:        time.Date(2021, 1, 1, 1, 0, 0, 0, time.FixedZone("UTC", 0)),
 				AfterTime:         time.Date(2022, 2, 2, 2, 0, 0, 0, time.FixedZone("UTC", 0)),
-				AlgosGT:           0,
-				AlgosLT:           0,
-				AssetAmountGT:     5,
-				AssetAmountLT:     6,
-				EffectiveAmountGt: 0,
-				EffectiveAmountLt: 0,
+				AlgosGT:           nil,
+				AlgosLT:           nil,
+				AssetAmountGT:     uint64Ptr(5),
+				AssetAmountLT:     uint64Ptr(6),
+				EffectiveAmountGT: nil,
+				EffectiveAmountLT: nil,
 				Address:           []byte{197, 204, 27, 84, 42, 255, 184, 163, 97, 247, 241, 26, 128, 13, 87, 125, 150, 22, 131, 63, 100, 238, 60, 209, 121, 177, 156, 23, 106, 222, 64, 35},
 				AddressRole:       9,
 				Offset:            nil,
@@ -197,7 +197,7 @@ func TestTransactionParamToTransactionFilter(t *testing.T) {
 		{
 			name:          "Currency to Algos when no asset-id",
 			params:        generated.SearchForTransactionsParams{CurrencyGreaterThan: uint64Ptr(10), CurrencyLessThan: uint64Ptr(20)},
-			filter:        idb.TransactionFilter{AlgosGT: 10, AlgosLT: 20, Limit: defaultTransactionsLimit},
+			filter:        idb.TransactionFilter{AlgosGT: uint64Ptr(10), AlgosLT: uint64Ptr(20), Limit: defaultTransactionsLimit},
 			errorContains: nil,
 		},
 		{
diff --git a/cmd/idbtest/idbtest.go b/cmd/idbtest/idbtest.go
index 9da6f06..d1c19ad 100644
--- a/cmd/idbtest/idbtest.go
+++ b/cmd/idbtest/idbtest.go
@@ -143,7 +143,7 @@ func main() {
 	maybeFail(err, "open postgres, %v", err)
 
 	if accounttest {
-		printAccountQuery(db, idb.AccountQueryOptions{IncludeAssetHoldings: true, IncludeAssetParams: true, AlgosGreaterThan: 10000000000, Limit: 20})
+		printAccountQuery(db, idb.AccountQueryOptions{IncludeAssetHoldings: true, IncludeAssetParams: true, AlgosGreaterThan: uint64Ptr(10000000000), Limit: 20})
 		printAccountQuery(db, idb.AccountQueryOptions{HasAssetID: 312769, Limit: 19})
 	}
 	if assettest {
@@ -180,14 +180,14 @@ func main() {
 		printTxnQuery(db, idb.TransactionFilter{Offset: &offset, Limit: 2})
 		printTxnQuery(db, idb.TransactionFilter{SigType: "lsig", Limit: 2})
 		printTxnQuery(db, idb.TransactionFilter{NotePrefix: []byte("a"), Limit: 2})
-		printTxnQuery(db, idb.TransactionFilter{AlgosGT: 10000000, Limit: 2})
-		printTxnQuery(db, idb.TransactionFilter{EffectiveAmountGt: 10000000, Limit: 2})
-		printTxnQuery(db, idb.TransactionFilter{EffectiveAmountLt: 1000000, Limit: 2})
+		printTxnQuery(db, idb.TransactionFilter{AlgosGT: uint64Ptr(10000000), Limit: 2})
+		printTxnQuery(db, idb.TransactionFilter{EffectiveAmountGT: uint64Ptr(10000000), Limit: 2})
+		printTxnQuery(db, idb.TransactionFilter{EffectiveAmountLT: uint64Ptr(1000000), Limit: 2})
 		printTxnQuery(db, idb.TransactionFilter{Address: xa[:], Limit: 6})
 		printTxnQuery(db, idb.TransactionFilter{Address: xa[:], AddressRole: idb.AddressRoleSender, Limit: 2})
 		printTxnQuery(db, idb.TransactionFilter{Address: xa[:], AddressRole: idb.AddressRoleReceiver, Limit: 2})
-		printTxnQuery(db, idb.TransactionFilter{AssetAmountGT: 99, Limit: 2})
-		printTxnQuery(db, idb.TransactionFilter{AssetAmountLT: 100, Limit: 2})
+		printTxnQuery(db, idb.TransactionFilter{AssetAmountGT: uint64Ptr(99), Limit: 2})
+		printTxnQuery(db, idb.TransactionFilter{AssetAmountLT: uint64Ptr(100), Limit: 2})
 	}
 
 	//printTxnQuery(db, idb.TransactionFilter{AssetID: 312769, Limit: 30})
@@ -198,7 +198,7 @@ func main() {
 		xa, _ := sdk_types.DecodeAddress("QRP4AJLQXHJ42VJ5PSGAH53IVVACYCI6ZDRJMF4JPRFY5VKSYKFWKKMFVU")
 		testTxnPaging(db, idb.TransactionFilter{Address: xa[:]})
 		testTxnPaging(db, idb.TransactionFilter{TypeEnum: 2})
-		testTxnPaging(db, idb.TransactionFilter{AlgosGT: 1})
+		testTxnPaging(db, idb.TransactionFilter{AlgosGT: uint64Ptr(1)})
 	}
 
 	dt := time.Now().Sub(start)
@@ -210,3 +210,7 @@ func main() {
 	}
 	os.Exit(exitValue)
 }
+
+func uint64Ptr(x uint64) *uint64 {
+	return &x
+}