Repository: algorand/indexer
Pull request #265 (2020-12-15 16:46:23)

Degrade ReadOnly Node Service When Migration is Required
## Summary

Update Health endpoint to report migration information from read-only nodes, update migration middleware. 



Patch:
diff --git a/api/middlewares/migration_middleware.go b/api/middlewares/migration_middleware.go
index b107942..e93b601 100644
--- a/api/middlewares/migration_middleware.go
+++ b/api/middlewares/migration_middleware.go
@@ -8,8 +8,8 @@ import (
 	"github.com/algorand/indexer/idb"
 )
 
-// InProgressError is the error returned when a migration is in progress.
-var InProgressError = "Indexer migration in progress, please wait."
+// DBUnavailableError is the error returned when a migration is in progress or required.
+var DBUnavailableError = "Indexer DB is not available, try again later."
 
 // MigrationMiddleware makes sure a 500 error is returned when the IndexerDb has a migration in progress.
 type MigrationMiddleware struct {
@@ -33,8 +33,8 @@ func (mm *MigrationMiddleware) handler(next echo.HandlerFunc) echo.HandlerFunc {
 			return echo.NewHTTPError(http.StatusInternalServerError, "Indexer health error: %s", err.Error())
 		}
 
-		if h.IsMigrating && !h.DBAvailable {
-			return echo.NewHTTPError(http.StatusInternalServerError, InProgressError)
+		if !h.DBAvailable {
+			return echo.NewHTTPError(http.StatusInternalServerError, DBUnavailableError)
 		}
 
 		return next(ctx)
diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go
index 724cf2f..bf22b46 100644
--- a/idb/postgres/postgres.go
+++ b/idb/postgres/postgres.go
@@ -2436,16 +2436,16 @@ func (db *IndexerDb) yieldApplicationsThread(ctx context.Context, rows *sql.Rows
 }
 
 // Health is part of idb.IndexerDB
-func (db *IndexerDb) Health() (health idb.Health, err error) {
-	var ptr *map[string]interface{}
+func (db *IndexerDb) Health() (idb.Health, error) {
+	migrationRequired := false
 	migrating := false
 	blocking := false
+	var data = make(map[string]interface{})
 
 	// If we are not in read-only mode, there will be a migration object.
 	if db.migration != nil {
 		state := db.migration.GetStatus()
 
-		var data = make(map[string]interface{})
 		if state.Err != nil {
 			data["migration-error"] = state.Err.Error()
 		}
@@ -2453,17 +2453,23 @@ func (db *IndexerDb) Health() (health idb.Health, err error) {
 			data["migration-status"] = state.Status
 		}
 
+		migrationRequired = state.Running
 		migrating = state.Running
 		blocking = state.Blocking
-
-		if len(data) > 0 {
-			ptr = &data
+	} else {
+		data["read-only-node"] = true
+		state, err := db.getMigrationState()
+		if err == nil {
+			blocking = migrationStateBlocked(*state)
+			migrationRequired = needsMigration(*state)
 		}
 	}
 
+	data["migration-required"] = migrationRequired
+
 	round, err := db.GetMaxRound()
 	return idb.Health{
-		Data:        ptr,
+		Data:        &data,
 		Round:       round,
 		IsMigrating: migrating,
 		DBAvailable: !blocking,
diff --git a/idb/postgres/postgres_migrations.go b/idb/postgres/postgres_migrations.go
index 46cb4c5..d651df7 100644
--- a/idb/postgres/postgres_migrations.go
+++ b/idb/postgres/postgres_migrations.go
@@ -84,6 +84,16 @@ func wrapPostgresHandler(handler postgresMigrationFunc, db *IndexerDb, state *Mi
 	}
 }
 
+// migrationStateBlocked returns true if a migration is required for running in read only mode.
+func migrationStateBlocked(state MigrationState) bool {
+	return state.NextMigration < rewardsMigrationIndex
+}
+
+// needsMigration returns true if there is an incomplete migration.
+func needsMigration(state MigrationState) bool {
+	return state.NextMigration < len(migrations)
+}
+
 func (db *IndexerDb) runAvailableMigrations(migrationStateJSON string) (err error) {
 	var state MigrationState
 	if len(migrationStateJSON) > 0 {
@@ -136,7 +146,7 @@ func (db *IndexerDb) markMigrationsAsDone() (err error) {
 	return db.SetMetastate(migrationMetastateKey, migrationStateJSON)
 }
 
-func getMigrationState(db *IndexerDb) (*MigrationState, error) {
+func (db *IndexerDb) getMigrationState() (*MigrationState, error) {
 	migrationStateJSON, err := db.GetMetastate(migrationMetastateKey)
 	if err == sql.ErrNoRows {
 		// no previous state, ok
@@ -174,7 +184,7 @@ func (db *IndexerDb) hasTotalRewardsSupport() bool {
 		// Set this unconditionally, if there's a failure lets not spam the DB.
 		lastCheckTs = time.Now()
 
-		state, err := getMigrationState(db)
+		state, err := db.getMigrationState()
 		if err != nil || state == nil {
 			hasRewardsSupport = false
 			return hasRewardsSupport
@@ -976,7 +986,7 @@ func (mtxid *txidFiuxpMigrationContext) putTxidFixupBatch(batch []idb.TxnRow) er
 	}
 	defer tx.Rollback() // ignored if .Commit() first
 	// Check that migration state in db is still what we think it is
-	txstate, err := getMigrationState(db)
+	txstate, err := db.getMigrationState()
 	if err != nil {
 		db.log.WithError(err).Errorf("%s, get m state err", txidMigrationErrMsg)
 		return err


Test patch:
diff --git a/api/middlewares/migration_middleware_test.go b/api/middlewares/migration_middleware_test.go
index 983d6f7..8fc017a 100644
--- a/api/middlewares/migration_middleware_test.go
+++ b/api/middlewares/migration_middleware_test.go
@@ -32,7 +32,7 @@ func TestMigrationMiddlewareWaiting(t *testing.T) {
 	handler := MakeMigrationMiddleware(mockIndexer)(success)
 	err := handler(e.NewContext(nil, nil))
 
-	require.Error(t, err, InProgressError, `'IsMigrating' is true, so we should see an InProgressError`)
+	require.Error(t, err, DBUnavailableError, `'IsMigrating' is true, so we should see an DBUnavailableError`)
 }
 
 func TestMigrationMiddlewareDone(t *testing.T) {