Bug: Region Server crash in our production environment.
Status: Resolved | Priority: Major | Resolution: Duplicate
Created: 11/Mar/24 22:33

Recently we are seeing lot of RS crash in our production environment creating core dump file and hs_err_pid.log file.
HBase:  hbase-2.5
Java: openjdk 1.8

Copying contents from hs_err_pid.log below:
{noformat}
#
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00007f9fb1415ba2, pid=50172, tid=0x00007f92a97ec700
#
# JRE version: OpenJDK Runtime Environment (Zulu 8.76.0.18-SA-linux64) (8.0_402-b06) (build 1.8.0_402-b06)
# Java VM: OpenJDK 64-Bit Server VM (25.402-b06 mixed mode linux-amd64 )
# Problematic frame:
# J 19801 C2 org.apache.hadoop.hbase.util.ByteBufferUtils.copyBufferToStream(Ljava/io/OutputStream;Ljava/nio/ByteBuffer;II)V (75 bytes) @ 0x00007f9fb1415ba2 [0x00007f9fb14159a0+0x202]
#
# Core dump written. Default location: /home/sfdc/core or core.50172
#
# If you would like to submit a bug report, please visit:
#   http://www.azul.com/support/
#

---------------  T H R E A D  ---------------

Current thread (0x00007f9fa2d13000):  JavaThread "RS-EventLoopGroup-1-92" daemon [_thread_in_Java, id=54547, stack(0x00007f92a96ec000,0x00007f92a97ed000)]

siginfo: si_signo: 11 (SIGSEGV), si_code: 1 (SEGV_MAPERR), si_addr: 0x0000559869daf000

Registers:
RAX=0x00007f9dbd8b6460, RBX=0x0000000000080000, RCX=0x000000000005c86b, RDX=0x00007f9dbd8b6460
RSP=0x00007f92a97eaf20, RBP=0x0000000000000002, RSI=0x00007f92d225e970, RDI=0x0000000000000069
R8 =0x000055986975f028, R9 =0x000000000064ffd8, R10=0x000000000000005f, R11=0x00007f94a778b290
R12=0x00007f9e62855ae8, R13=0x0000000000000000, R14=0x00007f9e5a14b1e0, R15=0x00007f9fa2d13000
RIP=0x00007f9fb1415ba2, EFLAGS=0x0000000000010216, CSGSFS=0x0000000000000033, ERR=0x0000000000000004
  TRAPNO=0x000000000000000e

Top of Stack: (sp=0x00007f92a97eaf20)
0x00007f92a97eaf20:   000000690064ff79 00007f9dbd8b6460
0x00007f92a97eaf30:   00007f9dbd8b6460 0000005700000003
0x00007f92a97eaf40:   00007f94a778b290 0004000100040000
0x00007f92a97eaf50:   00040000d090c130 00007f9db5522220
0x00007f92a97eaf60:   0008000000040001 00007f92a97eaf90
0x00007f92a97eaf70:   00007f92d0908648 0000000000000001
0x00007f92a97eaf80:   0000000000000001 000000000000005c
0x00007f92a97eaf90:   00007f94ee8078d0 0000000000000206
0x00007f92a97eafa0:   00007f9db5545a00 00007f9fafb63670
0x00007f92a97eafb0:   00007f9e5a13ed70 0000006900000001
0x00007f92a97eafc0:   00007f93ab8965b8 00007f93b9959210
0x00007f92a97eafd0:   00007f9db5545a00 00007f9fb04b3e30
0x00007f92a97eafe0:   00007f9e5a13ed70 00007f9300000001
0x00007f92a97eaff0:   00007f93ab8965b8 00007f93a8ae3920
0x00007f92a97eb000:   00007f93b9959210 00007f94a778b290
0x00007f92a97eb010:   00007f9b60707c20 00007f93a8938c28
0x00007f92a97eb020:   00007f94ee8078d0 00007f9b60708608
0x00007f92a97eb030:   00007f9b60707bc0 00007f9b60707c20
0x00007f92a97eb040:   0000000000000069 00007f93ab8965b8
0x00007f92a97eb050:   00007f94a778b290 00007f94a778b290
0x00007f92a97eb060:   0005c80d0005c80c 00000000a828a590
0x00007f92a97eb070:   00007f9e5a13ed70 000000010000270e
0x00007f92a97eb080:   00007f9db5545790 0144002200000000
0x00007f92a97eb090:   00007f95ddc800c0 00007f93ab89a6c8
0x00007f92a97eb0a0:   00007f93ae65c270 00007f9fb24af990
0x00007f92a97eb0b0:   00007f93ae65c290 00007f93ae65c270
0x00007f92a97eb0c0:   00007f9e5a13ed70 00007f92ca328528
0x00007f92a97eb0d0:   00007f9e5a13ed98 00007f9e5e1e88b0
0x00007f92a97eb0e0:   00007f92ca32d870 00007f9e5a13ed98
0x00007f92a97eb0f0:   00007f9e5e1e88b0 00007f93b9956288
0x00007f92a97eb100:   00007f9e5a13ed70 00007f9fb23c3aac
0x00007f92a97eb110:   00007f9317c9c8d0 00007f9b60708608 

Instructions: (pc=0x00007f9fb1415ba2)
0x00007f9fb1415b82:   44 3b d7 0f 8d 6d fe ff ff 4c 8b 40 10 45 8b ca
0x00007f9fb1415b92:   44 03 0c 24 c4 c1 f9 7e c3 4d 8b 5b 18 4d 63 c9
0x00007f9fb1415ba2:   47 0f be 04 08 4d 85 db 0f 84 49 03 00 00 4d 8b
0x00007f9fb1415bb2:   4b 08 48 b9 10 1c be 10 93 7f 00 00 4c 3b c9 0f 

Register to memory mapping:

RAX=0x00007f9dbd8b6460 is an oop
java.nio.DirectByteBuffer 
 - klass: 'java/nio/DirectByteBuffer'
RBX=0x0000000000080000 is an unknown value
RCX=0x000000000005c86b is an unknown value
RDX=0x00007f9dbd8b6460 is an oop
java.nio.DirectByteBuffer 
 - klass: 'java/nio/DirectByteBuffer'
RSP=0x00007f92a97eaf20 is pointing into the stack for thread: 0x00007f9fa2d13000
RBP=0x0000000000000002 is an unknown value
RSI=0x00007f92d225e970 is pointing into metadata
RDI=0x0000000000000069 is an unknown value
R8 =0x000055986975f028 is an unknown value
R9 =0x000000000064ffd8 is an unknown value
R10=0x000000000000005f is an unknown value
R11=0x00007f94a778b290 is an oop
org.apache.hbase.thirdparty.io.netty.buffer.PooledUnsafeDirectByteBuf 
 - klass: 'org/apache/hbase/thirdparty/io/netty/buffer/PooledUnsafeDirectByteBuf'
R12=0x00007f9e62855ae8 is an oop
java.lang.StringBuilder 
 - klass: 'java/lang/StringBuilder'
R13=0x0000000000000000 is an unknown value
R14=0x00007f9e5a14b1e0 is an oop
java.util.ArrayList$Itr 
 - klass: 'java/util/ArrayList$Itr'
R15=0x00007f9fa2d13000 is a thread


Stack: [0x00007f92a96ec000,0x00007f92a97ed000],  sp=0x00007f92a97eaf20,  free space=1019k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
J 19801 C2 org.apache.hadoop.hbase.util.ByteBufferUtils.copyBufferToStream(Ljava/io/OutputStream;Ljava/nio/ByteBuffer;II)V (75 bytes) @ 0x00007f9fb1415ba2 [0x00007f9fb14159a0+0x202]
J 38915 C2 org.apache.hadoop.hbase.ByteBufferKeyValue.write(Ljava/io/OutputStream;Z)I (21 bytes) @ 0x00007f9fafb63670 [0x00007f9fafb63620+0x50]
J 37217 C2 org.apache.hadoop.hbase.KeyValueUtil.oswrite(Lorg/apache/hadoop/hbase/Cell;Ljava/io/OutputStream;Z)I (256 bytes) @ 0x00007f9fb04b3e30 [0x00007f9fb04b3de0+0x50]
J 37223 C2 org.apache.hadoop.hbase.ipc.CellBlockBuilder.encodeCellsTo(Ljava/io/OutputStream;Lorg/apache/hadoop/hbase/CellScanner;Lorg/apache/hadoop/hbase/codec/Codec;Lorg/apache/hadoop/io/compress/CompressionCodec;)V (138 bytes) @ 0x00007f9fb24af990 [0x00007f9fb24af140+0x850]
J 28122 C2 org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.writeRequest(Lorg/apache/hbase/thirdparty/io/netty/channel/ChannelHandlerContext;Lorg/apache/hadoop/hbase/ipc/Call;Lorg/apache/hbase/thirdparty/io/netty/channel/ChannelPromise;)V (376 bytes) @ 0x00007f9fb2a4bac4 [0x00007f9fb2a4b600+0x4c4]
J 28121 C2 org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.write(Lorg/apache/hbase/thirdparty/io/netty/channel/ChannelHandlerContext;Ljava/lang/Object;Lorg/apache/hbase/thirdparty/io/netty/channel/ChannelPromise;)V (144 bytes) @ 0x00007f9fb15531d4 [0x00007f9fb1553140+0x94]
J 41543 C2 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.write(Ljava/lang/Object;ZLorg/apache/hbase/thirdparty/io/netty/channel/ChannelPromise;)V (149 bytes) @ 0x00007f9fb57da15c [0x00007f9fb57d96c0+0xa9c]
J 39549 C2 org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(Lorg/apache/hbase/thirdparty/com/google/protobuf/RpcCallback;Lorg/apache/hadoop/hbase/ipc/HBaseRpcController$CancellationCallback;)V (36 bytes) @ 0x00007f9fb42e2c44 [0x00007f9fb42e2520+0x724]
J 39675 C2 org.apache.hadoop.hbase.ipc.NettyRpcConnection$$Lambda$135.run()V (16 bytes) @ 0x00007f9faeb56e3c [0x00007f9faeb56ac0+0x37c]
J 24927 C2 org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(J)Z (109 bytes) @ 0x00007f9fb260a408 [0x00007f9fb2609fc0+0x448]
J 25959% C2 org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run()V (603 bytes) @ 0x00007f9faf772248 [0x00007f9faf772000+0x248]
j  org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run()V+44
j  org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run()V+11
j  org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run()V+4
J 8122 C1 java.lang.Thread.run()V (17 bytes) @ 0x00007f9faf7920c4 [0x00007f9faf791f80+0x144]
v  ~StubRoutines::call_stub
V  [libjvm.so+0x6aa9d6]  JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0xea6
V  [libjvm.so+0x6a7f73]  JavaCalls::call_virtual(JavaValue*, KlassHandle, Symbol*, Symbol*, JavaCallArguments*, Thread*)+0x263
V  [libjvm.so+0x6a8577]  JavaCalls::call_virtual(JavaValue*, Handle, KlassHandle, Symbol*, Symbol*, Thread*)+0x57
V  [libjvm.so+0x749f6c]  thread_entry(JavaThread*, Thread*)+0x6c
V  [libjvm.so+0xae8613]  JavaThread::thread_main_inner()+0x1f3
V  [libjvm.so+0xae89ad]  JavaThread::run()+0x36d

V  [libjvm.so+0x971532]  java_start(Thread*)+0x102
C  [libpthread.so.0+0x7ea5]  start_thread+0xc5

{noformat}