Bug: NotServingRegionException while running IntegrationTestIngest with ChaosMonkey
Status: Open | Priority: Major | Resolution: 
Created: 06/Jun/23 02:04

While Running the command:
{noformat}
hbase org.apache.hadoop.hbase.IntegrationTestIngest --monkey slowDeterministic{noformat}
Getting below errors:
{noformat}
2023-06-05 08:44:07,362 INFO org.apache.hadoop.hbase.procedure2.ProcedureExecutor: Finished pid=771, ppid=763, state=SUCCESS; CloseRegionProcedure a5882e880b2ea5eaa33bc972ef98ae9a, server=ozn-hbnew19-2.ozn-hbnew19.root.hwx.site,22101,1685953635506 in 12.8940 sec
2023-06-05 08:44:07,431 INFO org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure: Starting pid=763, ppid=731, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=IntegrationTestIngest, region=a5882e880b2ea5eaa33bc972ef98ae9a, REOPEN/MOVE; state=CLOSED, location=ozn-hbnew19-2.ozn-hbnew19.root.hwx.site,22101,1685953635506; forceNewPlan=false, retain=true
2023-06-05 08:44:07,543 ERROR org.apache.hadoop.hbase.ipc.RpcServer: Unexpected throwable object
org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable: org.apache.hadoop.hbase.NotServingRegionException: IntegrationTestIngest,182be0c5cdcd5072bb1864cdee4d3d6e-33,1685954003397.e02c6a03f3499091552f84577b27b942. is closing
        at org.apache.hadoop.hbase.procedure.flush.RegionServerFlushTableProcedureManager$FlushTableSubprocedurePool.waitForOutstandingTasks(RegionServerFlushTableProcedureManager.java:274)
        at org.apache.hadoop.hbase.procedure.flush.FlushTableSubprocedure.flushRegions(FlushTableSubprocedure.java:115)
        at org.apache.hadoop.hbase.procedure.flush.FlushTableSubprocedure.acquireBarrier(FlushTableSubprocedure.java:126)
        at org.apache.hadoop.hbase.procedure.Subprocedure.call(Subprocedure.java:160)
        at org.apache.hadoop.hbase.procedure.Subprocedure.call(Subprocedure.java:46)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
2023-06-05 08:44:07,550 ERROR org.apache.hadoop.hbase.procedure.Procedure: Procedure 'IntegrationTestIngest' execution failed!
org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable via ozn-hbnew19-1.ozn-hbnew19.root.hwx.site,22101,1685953632178:org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable: org.apache.hadoop.hbase.NotServingRegionException: IntegrationTestIngest,182be0c5cdcd5072bb1864cdee4d3d6e-33,1685954003397.e02c6a03f3499091552f84577b27b942. is closing
        at org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher.rethrowException(ForeignExceptionDispatcher.java:81)
        at org.apache.hadoop.hbase.procedure.Procedure.waitForLatch(Procedure.java:382)
        at org.apache.hadoop.hbase.procedure.Procedure.call(Procedure.java:204)
        at org.apache.hadoop.hbase.procedure.Procedure.call(Procedure.java:67)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable: org.apache.hadoop.hbase.NotServingRegionException: IntegrationTestIngest,182be0c5cdcd5072bb1864cdee4d3d6e-33,1685954003397.e02c6a03f3499091552f84577b27b942. is closing
 at org.apache.hadoop.hbase.procedure.flush.RegionServerFlushTableProcedureManager$FlushTableSubprocedurePool.waitForOutstandingTasks(RegionServerFlushTableProcedureManager.java:274)
        at org.apache.hadoop.hbase.procedure.flush.FlushTableSubprocedure.flushRegions(FlushTableSubprocedure.java:115)
        at org.apache.hadoop.hbase.procedure.flush.FlushTableSubprocedure.acquireBarrier(FlushTableSubprocedure.java:126)
        at org.apache.hadoop.hbase.procedure.Subprocedure.call(Subprocedure.java:160)
        at org.apache.hadoop.hbase.procedure.Subprocedure.call(Subprocedure.java:46)
        ... 4 more
2023-06-05 08:44:07,552 INFO org.apache.hadoop.hbase.procedure.ZKProcedureUtil: Clearing all znodes for procedure IntegrationTestIngestincluding nodes /hbase2/flush-table-proc/acquired /hbase2/flush-table-proc/reached /hbase2/flush-table-proc/abort
2023-06-05 08:44:07,582 INFO org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer: Reassigned 1 regions. 1 retained the pre-restart assignment.
2023-06-05 08:44:07,583 INFO org.apache.hadoop.hbase.master.assignment.RegionStateStore: pid=741 updating hbase:meta row=76a8eaee60cc2aec2af1bbdfba6259a4, regionState=CLOSED
2023-06-05 08:44:07,650 INFO org.apache.hadoop.hbase.master.MasterRpcServices: Client=hbase//10.64.62.16 procedure request for: flush-table-proc
2023-06-05 08:44:07,659 INFO org.apache.hadoop.hbase.procedure.Procedure: Starting procedure 'IntegrationTestIngest'
2023-06-05 08:44:08,160 ERROR org.apache.hadoop.hbase.ipc.RpcServer: Unexpected throwable object
org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable: org.apache.hadoop.hbase.NotServingRegionException: IntegrationTestIngest,07507507,1685953771325.6c59d6c5a3b35231b11d5d4c854729f3. is closing
        at org.apache.hadoop.hbase.procedure.flush.RegionServerFlushTableProcedureManager$FlushTableSubprocedurePool.waitForOutstandingTasks(RegionServerFlushTableProcedureManager.java:274)
        at org.apache.hadoop.hbase.procedure.flush.FlushTableSubprocedure.flushRegions(FlushTableSubprocedure.java:115)
        at org.apache.hadoop.hbase.procedure.flush.FlushTableSubprocedure.acquireBarrier(FlushTableSubprocedure.java:126)
        at org.apache.hadoop.hbase.procedure.Subprocedure.call(Subprocedure.java:160)
        at org.apache.hadoop.hbase.procedure.Subprocedure.call(Subprocedure.java:46)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
2023-06-05 08:44:08,167 ERROR org.apache.hadoop.hbase.procedure.Procedure: Procedure 'IntegrationTestIngest' execution failed!
org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable via ozn-hbnew19-3.ozn-hbnew19.root.hwx.site,22101,1685953632330:org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable: org.apache.hadoop.hbase.NotServingRegionException: IntegrationTestIngest,07507507,1685953771325.6c59d6c5a3b35231b11d5d4c854729f3. is closing
        at org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher.rethrowException(ForeignExceptionDispatcher.java:81)
        at org.apache.hadoop.hbase.procedure.Procedure.waitForLatch(Procedure.java:382)
        at org.apache.hadoop.hbase.procedure.Procedure.call(Procedure.java:204)
        at org.apache.hadoop.hbase.procedure.Procedure.call(Procedure.java:67)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hbase.errorhandling.ForeignException$ProxyThrowable: org.apache.hadoop.hbase.NotServingRegionException: IntegrationTestIngest,07507507,1685953771325.6c59d6c5a3b35231b11d5d4c854729f3. is closing{noformat}