Bug: Master Failed to Start after upgrading from 2.5.10 to 3.0.0-beta-1	
Status: Resolved | Priority: Major | Resolution: Duplicate
Created: 01/Dec/24 13:38

Master Failed to Start after upgrading from 2.5.10 to 3.0.0-beta-1
{code:java}
2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster] master.HMaster: Failed to become active master
java.lang.IllegalStateException: Expected the service ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED
        at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384) ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
        at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324) ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
        at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1]
        at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException: Failed 18 actions: RetriesExhaustedException: 18 times, servers with issues:
        at org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.makeError(BufferedMutatorOverAsyncBufferedMutator.java:107) ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.internalFlush(BufferedMutatorOverAsyncBufferedMutator.java:122) ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.client.BufferedMutatorOverAsyncBufferedMutator.close(BufferedMutatorOverAsyncBufferedMutator.java:166) ~[hbase-client-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.TableNamespaceManager.migrateNamespaceTable(TableNamespaceManager.java:92) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:122) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:61) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:252) ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
        at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1532) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        ... 5 more
2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster] master.HMaster: Master server abort: loaded coprocessors are: []
2024-12-01T19:00:47,390 ERROR [master/master:16000:becomeActiveMaster] master.HMaster: ***** ABORTING master master,16000,1733059831148: Unhandled exception. Starting shutdown. *****
java.lang.IllegalStateException: Expected the service ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED
        at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:384) ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
        at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:324) ~[hbase-shaded-miscellaneous-4.1.5.jar:4.1.5]
        at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1535) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1204) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2494) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-1.jar:3.0.0-beta-1]
        at java.lang.Thread.run(Thread.java:840) ~[?:?]{code}
Region Server has below warning log

 
{code:java}
2024-12-01T19:00:47,354 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16020] regionserver.HRegion: No such column family in batch mutation in region org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation@74bf756d
org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException: Column family ns does not exist in region hbase:meta,,1.1588230740 in table 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
        at org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5344) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion.checkFamily(HRegion.java:5331) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion.checkFamilies(HRegion.java:5325) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion.access$100(HRegion.java:239) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPreparePut(HRegion.java:3849) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3486) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.checkAndPrepareMutation(HRegion.java:3498) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation$1.visit(HRegion.java:3878) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion$BatchOperation.visitBatchOperations(HRegion.java:3377) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion$MutationBatchOperation.checkAndPrepare(HRegion.java:3857) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4753) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4673) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:984) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:902) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:867) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2830) ~[hbase-server-3.0.0-beta-1.jar:3.0.0-beta-1]
        at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45018) ~[hbase-protocol-shaded-3.0.0-beta-1.jar:3.0.0-beta-1]{code}
complete log files are attached