Bug: RS crash on ipc request to big
Status: Open | Priority: Major | Resolution: 
Created: 24/Feb/23 09:46

 2023-02-17 16:44:09,601 WARN  [RS-EventLoopGroup-1-46] ipc.NettyRpcServer: RPC data length of 825701220 received from client_ip1 is greater than max allowed 268435456. Set "hba      se.ipc.max.request.size" on server to override this limit (not recommended)
 9423 2023-02-17 16:44:12,668 ERROR [RpcServer.default.RWQ.Fifo.write.handler=62,queue=62,port=16020] ipc.RpcServer: Unexpected throwable object
 9424 java.lang.RuntimeException: Unknown code 98
 9425     at org.apache.hadoop.hbase.KeyValue$Type.codeToType(KeyValue.java:276)
 9426     at org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(CellUtil.java:1340)
 9427     at org.apache.hadoop.hbase.CellUtil.getCellKeyAsString(CellUtil.java:1318)
 9428     at org.apache.hadoop.hbase.CellUtil.toString(CellUtil.java:1512)
 9429     at org.apache.hadoop.hbase.ByteBufferKeyValue.toString(ByteBufferKeyValue.java:301)
 9430     at org.apache.hadoop.hbase.client.Mutation.add(Mutation.java:786)
 9431     at org.apache.hadoop.hbase.client.Put.add(Put.java:282)
 9432     at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toPut(ProtobufUtil.java:656)
 9433     at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1033)
 9434     at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:987)
 9435     at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:950)
 9436     at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2949)
 9437     at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265)
 9438     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:394)
 9439     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
 9440     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
 9441     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
 9442 2023-02-17 16:44:19,430 WARN  [RS-EventLoopGroup-1-60] ipc.RpcServer: Invalid request header: , should have param set in it
 9443 2023-02-17 16:44:19,431 WARN  [RS-EventLoopGroup-1-60] ipc.RpcServer: /hostname_rs1:16020 is unable to read call parameter from client client_ip1
 9444 org.apache.hadoop.hbase.DoNotRetryIOException: Invalid request header: , should have param set in it
 9445     at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:654)
 9446     at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:448)
 9447     at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89)
 9448     at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:63)
 9449     at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead(NettyRpcServerRequestDecoder.java:62)
 9450     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
 9451     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
 9452     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
 9453     at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327)
 9454     at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:314)
 9455     at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:435)
 9456     at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279)
 9457     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
 9458     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
 9459     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
 9460     at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
 9461     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
 9462     at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
 9463     at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
 9464     at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795)
 9465     at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480)
 9466     at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378)
 9467     at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)
 9468     at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
 9469     at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
 9470     at java.base/java.lang.Thread.run(Thread.java:832)
 9471 2023-02-17 16:44:19,431 WARN  [RS-EventLoopGroup-1-60] ipc.NettyRpcServer: RPC data length of 4294967044 received from client_ip1 is greater than max allowed 268435456. Set "hb      ase.ipc.max.request.size" on server to override this limit (not recommended)
 9472 2023-02-17 16:44:24,215 INFO  [qtp1820051628-4977731] requests.regionserver: 127.0.0.1 - - [17/2月/2023:08:44:24 +0000] "GET //127.0.0.1:16030/jmx HTTP/1.1" 200 1397322
 9473 2023-02-17 16:44:24,703 ERROR [RpcServer.default.RWQ.Fifo.write.handler=532,queue=132,port=16020] ipc.RpcServer: Unexpected throwable object
 9474 java.lang.NegativeArraySizeException: -620684796
 9475     at org.apache.hadoop.hbase.CellUtil.cloneValue(CellUtil.java:126)
 9476     at org.apache.hadoop.hbase.ByteBufferKeyValue.getValueArray(ByteBufferKeyValue.java:168)
 9477     at org.apache.hadoop.hbase.CellUtil.toString(CellUtil.java:1522)
 9478     at org.apache.hadoop.hbase.ByteBufferKeyValue.toString(ByteBufferKeyValue.java:301)
 9479     at org.apache.hadoop.hbase.client.Mutation.add(Mutation.java:786)
 9480     at org.apache.hadoop.hbase.client.Put.add(Put.java:282)
 9481     at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toPut(ProtobufUtil.java:656)
 9482     at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:1033)
 9483     at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicBatchOp(RSRpcServices.java:987)
 9484     at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:950)
 9485     at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2949)
 9486     at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45265)
 9487     at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:394)
 9488     at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
 9489     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)
 9490     at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)
 9491 2023-02-17 16:44:25,356 WARN  [RS-EventLoopGroup-1-13] ipc.NettyRpcServer: RPC data length of 929117235 received from client_ip1 is greater than max allowed 268435456. Set "hbase.ipc.max.request.size" on server to override this limit (not recommended)