Source:
<code>private void handleUpsert(ImmutableSegment immutableSegment) { Preconditions.checkArgument(!_primaryKeyColumns.isEmpty(), "the primary key columns cannot be empty"); Map<String, PinotSegmentColumnReader> columnToReaderMap = new HashMap<>(); for (String primaryKeyColumn : _primaryKeyColumns) { columnToReaderMap.put(primaryKeyColumn, new PinotSegmentColumnReader(immutableSegment, primaryKeyColumn)); } columnToReaderMap.put(_timeColumnName, new PinotSegmentColumnReader(immutableSegment, _timeColumnName)); int numTotalDocs = immutableSegment.getSegmentMetadata().getTotalDocs(); <START> Map<PrimaryKey, RecordLocation> primaryKeyIndex = new HashMap<>(); <END> ThreadSafeMutableRoaringBitmap validDocIndex = new ThreadSafeMutableRoaringBitmap(); String segmentName = immutableSegment.getSegmentName(); int partitionId = new LLCSegmentName(immutableSegment.getSegmentName()).getPartitionId(); for (int docId = 0; docId < numTotalDocs; docId++) { Object[] fields = new Object[_primaryKeyColumns.size()]; for (int i = 0; i < _primaryKeyColumns.size(); i++) { fields[i] = columnToReaderMap.get(_primaryKeyColumns.get(i)).getValue(docId); } PrimaryKey primaryKey = new PrimaryKey(fields); Object timeValue = columnToReaderMap.get(_timeColumnName).getValue(docId); Preconditions.checkArgument(timeValue instanceof Comparable, "time column shall be comparable"); long timestamp = IngestionUtils.extractTimeValue((Comparable) timeValue); UpsertProcessorUtil .handleUpsert(primaryKey, timestamp, segmentName, docId, partitionId, primaryKeyIndex, validDocIndex, _upsertMetadataTableManager); } _upsertMetadataTableManager.putUpsertMetadataOfPartition(partitionId, segmentName, primaryKeyIndex, validDocIndex); }</code><technical_language>estimated memory usage for extra data structure? if primary key+RecordLocation is 1K, this means 10G heap for this data structure for 1M records. want careful part of memory usage</technical_language>

Target:
private void handleUpsert(ImmutableSegment immutableSegment) { Map<String, PinotSegmentColumnReader> columnToReaderMap = new HashMap<>(); for (String primaryKeyColumn : _primaryKeyColumns) { columnToReaderMap.put(primaryKeyColumn, new PinotSegmentColumnReader(immutableSegment, primaryKeyColumn)); } columnToReaderMap.put(_timeColumnName, new PinotSegmentColumnReader(immutableSegment, _timeColumnName)); int numTotalDocs = immutableSegment.getSegmentMetadata().getTotalDocs(); String segmentName = immutableSegment.getSegmentName(); int partitionId = new LLCSegmentName(immutableSegment.getSegmentName()).getPartitionId(); PartitionUpsertMetadataManager partitionUpsertMetadataManager = _tableUpsertMetadataManager.getOrCreatePartitionManager(partitionId); ThreadSafeMutableRoaringBitmap validDocIds = partitionUpsertMetadataManager.getValidDocIds(segmentName); int numPrimaryKeyColumns = _primaryKeyColumns.size(); for (int docId = 0; docId < numTotalDocs; docId++) { Object[] values = new Object[numPrimaryKeyColumns]; for (int i = 0; i < numPrimaryKeyColumns; i++) { Object value = columnToReaderMap.get(_primaryKeyColumns.get(i)).getValue(docId); if (value instanceof byte[]) { value = new ByteArray((byte[]) value); } values[i] = value; } PrimaryKey primaryKey = new PrimaryKey(values); Object timeValue = columnToReaderMap.get(_timeColumnName).getValue(docId); Preconditions.checkArgument(timeValue instanceof Comparable, "time column shall be comparable"); long timestamp = IngestionUtils.extractTimeValue((Comparable) timeValue); RecordLocation recordLocation = new RecordLocation(segmentName, docId, timestamp, false); partitionUpsertMetadataManager.updateRecordLocation(primaryKey, recordLocation, validDocIds); } }