Bug: Suspected memory leak caused by major compaction
Status: Open | Priority: Major | Resolution: 
Created: 18/Sep/22 04:47
Affects Version/s: 2.2.6

There are 49G Hfiles in a certain region, and major compact has been unsuccessful, and the Old area in the heap memory continues to fill up, causing frequent Full GCs and causing the RegionServer process to OOM. After analyzing the dump file of heap memory using MAT, we found that:

!image-2022-09-18-12-46-08-478.png!

The full stack involved is as follows:

regionserver/node71:60020-longCompactions-3
  at org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSums(IILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;IILjava/lang/String;JZ)V (Native Method)
  at org.apache.hadoop.util.NativeCrc32.verifyChunkedSums(IILjava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V (NativeCrc32.java:63)
  at org.apache.hadoop.util.DataChecksum.verifyChunkedSums(Ljava/nio/ByteBuffer;Ljava/nio/ByteBuffer;Ljava/lang/String;J)V (DataChecksum.java:380)
  at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.verifyChunkedSums(Lorg/apache/hadoop/util/DataChecksum;Lorg/apache/hadoop/hbase/nio/ByteBuff;Lorg/apache/hadoop/hbase/nio/ByteBuff;Ljava/lang/String;)Z (ChecksumUtil.java:104)
  at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateChecksum(Lorg/apache/hadoop/hbase/nio/ByteBuff;Ljava/lang/String;JI)Z (ChecksumUtil.java:191)
  at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateChecksum(JLorg/apache/hadoop/hbase/nio/ByteBuff;I)Z (HFileBlock.java:1842)
  at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(Lorg/apache/hadoop/fs/FSDataInputStream;JJZZZZ)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileBlock.java:1777)
  at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(JJZZZ)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileBlock.java:1570)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.readBlock(JJZZZZLorg/apache/hadoop/hbase/io/hfile/BlockType;Lorg/apache/hadoop/hbase/io/encoding/DataBlockEncoding;)Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileReaderImpl.java:1519)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.readNextDataBlock()Lorg/apache/hadoop/hbase/io/hfile/HFileBlock; (HFileReaderImpl.java:928)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.isNextBlock()Z (HFileReaderImpl.java:1061)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.positionForNextBlock()Z (HFileReaderImpl.java:1055)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl._next()Z (HFileReaderImpl.java:1073)
  at org.apache.hadoop.hbase.io.hfile.HFileReaderImpl$HFileScannerImpl.next()Z (HFileReaderImpl.java:1094)
  at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.next()Z (HalfStoreFileReader.java:169)
  at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(Lorg/apache/hadoop/hbase/io/hfile/HFileScanner;Lorg/apache/hadoop/hbase/Cell;)Z (StoreFileScanner.java:351)
  at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(Lorg/apache/hadoop/hbase/Cell;)Z (StoreFileScanner.java:244)
  at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(Lorg/apache/hadoop/hbase/regionserver/KeyValueScanner;Lorg/apache/hadoop/hbase/Cell;Z)Z (NonLazyKeyValueScanner.java:55)
  at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(ZLorg/apache/hadoop/hbase/Cell;ZZ)Z (KeyValueHeap.java:324)
  at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(Lorg/apache/hadoop/hbase/Cell;)Z (KeyValueHeap.java:267)
  at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(Lorg/apache/hadoop/hbase/Cell;)Z (StoreScanner.java:1099)
  at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(Lorg/apache/hadoop/hbase/Cell;)Z (StoreScanner.java:1088)
  at org.apache.hadoop.hbase.regionserver.StoreScanner.seekOrSkipToNextColumn(Lorg/apache/hadoop/hbase/Cell;)V (StoreScanner.java:823)
  at org.apache.hadoop.hbase.regionserver.StoreScanner.next(Ljava/util/List;Lorg/apache/hadoop/hbase/regionserver/ScannerContext;)Z (StoreScanner.java:730)
  at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$FileDetails;Lorg/apache/hadoop/hbase/regionserver/InternalScanner;Lorg/apache/hadoop/hbase/regionserver/CellSink;JZLorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;ZI)Z (Compactor.java:387)
  at org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$InternalScannerFactory;Lorg/apache/hadoop/hbase/regionserver/compactions/Compactor$CellSinkFactory;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (Compactor.java:326)
  at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionRequestImpl;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (DefaultCompactor.java:65)
  at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (DefaultStoreEngine.java:126)
  at org.apache.hadoop.hbase.regionserver.HStore.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Ljava/util/List; (HStore.java:1469)
  at org.apache.hadoop.hbase.regionserver.HRegion.compact(Lorg/apache/hadoop/hbase/regionserver/compactions/CompactionContext;Lorg/apache/hadoop/hbase/regionserver/HStore;Lorg/apache/hadoop/hbase/regionserver/throttle/ThroughputController;Lorg/apache/hadoop/hbase/security/User;)Z (HRegion.java:2264)
  at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(Lorg/apache/hadoop/hbase/security/User;)V (CompactSplit.java:624)
  at org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run()V (CompactSplit.java:666)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V (ThreadPoolExecutor.java:1142)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run()V (ThreadPoolExecutor.java:617)
  at java.lang.Thread.run()V (Thread.java:748)