auphong2707/dl4se-code-review-dataset huggingface dataset:
Source:
private static MessageInfo getMessageInfo(StoreKey id, List<MessageInfo> messageInfos, boolean deleteMsg) { MessageInfo toRet = null; for (MessageInfo messageInfo : messageInfos) { if (messageInfo.getStoreKey().equals(id)) { START if ((deleteMsg && messageInfo.isDeleted()) || (!deleteMsg && !messageInfo.isDeleted())) { END toRet = messageInfo; break; } } } return toRet; } `if(deleteMsg == messageInfo.isDeleted())`?

Target:
private static MessageInfo getMessageInfo(StoreKey id, List<MessageInfo> messageInfos, boolean deleteMsg) { MessageInfo toRet = null; for (MessageInfo messageInfo : messageInfos) { if (messageInfo.getStoreKey().equals(id)) { if (deleteMsg == messageInfo.isDeleted()) { toRet = messageInfo; break; } } } return toRet; }
Source:
public boolean addAccountUpdateConsumer(Consumer<Collection<Account>> accountUpdateConsumer) { START checkOpen(); END Objects.requireNonNull(accountUpdateConsumer, "accountUpdateConsumer to subscribe cannot be null"); return accountUpdateConsumers.add(accountUpdateConsumer); } why not make these no-ops? Adding and removing doesn't seem to make sense here

Target:
public boolean addAccountUpdateConsumer(Consumer<Collection<Account>> accountUpdateConsumer) { checkOpen(); Objects.requireNonNull(accountUpdateConsumer, "accountUpdateConsumer to subscribe cannot be null"); return true; }
Source:
private int numberOfEntries(ByteBuffer mmap) { START return (mmap.capacity() - indexSizeExcludingEntries) / (persistedEntrySize); END } minor: parentheses on denominator not required

Target:
private int numberOfEntries(ByteBuffer mmap) { return (mmap.capacity() - indexSizeExcludingEntries) / persistedEntrySize; }
Source:
private StoreKey getKeyAt(ByteBuffer mmap, int index) throws IOException { START mmap.position(firstKeyRelativeOffset + (index * (persistedEntrySize))); END return factory.getStoreKey(new DataInputStream(new ByteBufferInputStream(mmap))); } minor: parentheses around `persistedEntrySize` not required.

Target:
private StoreKey getKeyAt(ByteBuffer mmap, int index) throws IOException { mmap.position(firstKeyRelativeOffset + index * persistedEntrySize); return factory.getStoreKey(new DataInputStream(new ByteBufferInputStream(mmap))); }
Source:
public Object setArg(String key, Object value) { Object oldVal = allArgs.put(key, value); START allArgsReadOnly = Collections.unmodifiableMap(allArgs); END return oldVal; } why is this step needed? changing the underlying map should have changed this too.

Target:
public Object setArg(String key, Object value) { return allArgs.put(key, value); }
Source:
public Object setArg(String key, Object value) { START Object oldVal = allArgs.put(key, value); END return oldVal; } `return allArgs.put(key,value)`

Target:
public Object setArg(String key, Object value) { return allArgs.put(key, value); }
Source:
public AccountService getAccountService() { logger.info("Starting an InMemoryUnknownAccountService."); long startTime = System.currentTimeMillis(); START AccountService res = new InMemoryUnknownAccountService(); END logger.info("InMemoryUnknownAccountService started, took ms", System.currentTimeMillis() - startTime); return res; } call this variable accountService

Target:
public AccountService getAccountService() { logger.info("Starting an InMemoryUnknownAccountService."); return new InMemoryUnknownAccountService(); }
Source:
public Collection<Account> getAllAccounts() { START return Collections.unmodifiableCollection(accounts); END } missed this in the first iteration, sorry - but can you make _this_ the static class variable (the unmodifiable collection).

Target:
public Collection<Account> getAllAccounts() { return accounts; }
Source:
public void onLiveInstanceChange(List<LiveInstance> liveInstances, NotificationContext changeContext) { logger.trace("Live instance change triggered from with: {}", dcName, liveInstances); updateInstanceLiveness(liveInstances); START if (!liveStateInitialized) { END logger.info("Received initial notification for live instance change from {}", dcName); liveStateInitialized = true; } helixClusterManagerMetrics.liveInstanceChangeTriggerCount.inc(); } minor: since this one is not under a lock, could you use `AtomicBoolean` and `compareAndSet()`?

Target:
public void onLiveInstanceChange(List<LiveInstance> liveInstances, NotificationContext changeContext) { logger.trace("Live instance change triggered from with: {}", dcName, liveInstances); updateInstanceLiveness(liveInstances); if (!liveStateInitialized.get()) { logger.info("Received initial notification for live instance change from {}", dcName); liveStateInitialized.set(true); } helixClusterManagerMetrics.liveInstanceChangeTriggerCount.inc(); }
Source:
public void setup() throws Exception { storeKeyToMockStoreMap.clear(); helixNotifier = new HelixNotifier(getMockHelixStore(storeConfig)); helixNotifier2 = new HelixNotifier(getMockHelixStore(storeConfig)); resetReferenceTopicsAndMessages(); resetListeners(); START deleteStoreIfExists(); END } could you move this to the top?

Target:
public void setup() throws Exception { deleteStoreIfExists(); storeKeyToMockStoreMap.clear(); helixNotifier = new HelixNotifier(getMockHelixStore(storeConfig)); resetReferenceTopicsAndMessages(); resetListeners(); }
Source:
public void close() { try { isOpen = false; cache.close(); helixStore.stop(); } catch (Exception e) { logger.error("Exception occurred when closing HelixAccountService.", e); } START } END Does nuage require metrics on the update path?

Target:
public void close() { if (isOpen) { try { isOpen = false; helixStore.stop(); } catch (Exception e) { logger.error("Exception occurred when closing HelixAccountService.", e); } } }
Source:
public void close() { if (isOpen) { try { isOpen = false; START accountInfoMapRef.set(null); END helixStore.stop(); } catch (Exception e) { logger.error("Exception occurred when closing HelixAccountService.", e); } } } why is this required?

Target:
public void close() { if (isOpen) { try { isOpen = false; helixStore.stop(); } catch (Exception e) { logger.error("Exception occurred when closing HelixAccountService.", e); } } }
Source:
START public static void serializeBlobPropertiesV2Record(ByteBuffer outputBuffer, BlobProperties properties) { END int startOffset = outputBuffer.position(); outputBuffer.putShort(BlobProperties_Version_V1); BlobPropertiesSerDe.putBlobPropertiesToBufferV2(outputBuffer, properties); Crc32 crc = new Crc32(); crc.update(outputBuffer.array(), startOffset, getBlobPropertiesV2RecordSize(properties) - Crc_Size); outputBuffer.putLong(crc.getValue()); } private for both functions?

Target:
private void serializeBlobPropertiesV2Record(ByteBuffer outputBuffer, BlobProperties properties) { int startOffset = outputBuffer.position(); outputBuffer.putShort(BlobProperties_Version_V1); putBlobPropertiesToBufferV2(outputBuffer, properties); Crc32 crc = new Crc32(); crc.update(outputBuffer.array(), startOffset, getBlobPropertiesV2RecordSize(properties) - Crc_Size); outputBuffer.putLong(crc.getValue()); }
Source:
public DiskSpaceRequirements(long segmentSizeInBytes, long segmentsNeeded, long swapSegmentsInUse) { START if (segmentSizeInBytes < 0 || segmentsNeeded < 0 || swapSegmentsInUse < 0) { END throw new IllegalArgumentException( "Arguments cannot be negative. segmentSizeInBytes: " + segmentSizeInBytes + ", segmentsNeeded: " + segmentsNeeded + ", swapSegmentsInUse: " + swapSegmentsInUse); } this.segmentSizeInBytes = segmentSizeInBytes; this.segmentsNeeded = segmentsNeeded; this.swapSegmentsInUse = swapSegmentsInUse; } `segmentSizeInBytes <= 0`?

Target:
DiskSpaceRequirements(long segmentSizeInBytes, long segmentsNeeded, long swapSegmentsInUse) { if (segmentSizeInBytes <= 0 || segmentsNeeded < 0 || swapSegmentsInUse < 0) { throw new IllegalArgumentException( "Arguments cannot be negative. segmentSizeInBytes: " + segmentSizeInBytes + ", segmentsNeeded: " + segmentsNeeded + ", swapSegmentsInUse: " + swapSegmentsInUse); } this.segmentSizeInBytes = segmentSizeInBytes; this.segmentsNeeded = segmentsNeeded; this.swapSegmentsInUse = swapSegmentsInUse; }
Source:
Set<Long> getFileSizeSet() { rwLock.readLock().lock(); try { return internalMap.keySet(); } finally { rwLock.readLock().unlock(); START } END } would we need the lock if we used a `ConcurrentHashMap` and `LinkedinBlockingQueue`?

Target:
Set<Long> getFileSizeSet() { return internalMap.keySet(); }
Source:
private File allocate(String filename, long size) throws IOException { File segmentFile = new File(dataDir, filename); if (!segmentFile.exists()) { if (diskSpaceAllocator != null) { diskSpaceAllocator.allocate(segmentFile, size); } else { START Utils.preAllocateFileIfNeeded(segmentFile, size); END } } return segmentFile; } as discussed, let's do this inside the allocator

Target:
private File allocate(String filename, long size) throws IOException { File segmentFile = new File(dataDir, filename); if (!segmentFile.exists()) { diskSpaceAllocator.allocate(segmentFile, size); } return segmentFile; }
Source:
private static Long getFileSizeForDirName(String fileSizeDirName) { Long sizeInBytes = null; if (fileSizeDirName.startsWith(FILE_SIZE_DIR_PREFIX)) { String sizeString = fileSizeDirName.substring(FILE_SIZE_DIR_PREFIX.length()); try { sizeInBytes = Long.parseLong(sizeString); START } catch (NumberFormatException ignored) { END } } return sizeInBytes; } why do we ignore this?

Target:
private static Long getFileSizeForDirName(String fileSizeDirName) { Long sizeInBytes = null; if (fileSizeDirName.startsWith(FILE_SIZE_DIR_PREFIX)) { String sizeString = fileSizeDirName.substring(FILE_SIZE_DIR_PREFIX.length()); sizeInBytes = Long.parseLong(sizeString); } return sizeInBytes; }
Source:
START short getCurrentVersion() { END return CURRENT_VERSION; } protected

Target:
protected short getCurrentVersion() { return CURRENT_VERSION; }
Source:
public StoreKey getStoreKey(DataInputStream value) throws IOException { START return BlobId.fromDataInputStream(value, clusterMap); END } same here - why is this changed from a constructor to this?

Target:
public StoreKey getStoreKey(DataInputStream value) throws IOException { return new BlobId(value, clusterMap); }
Source:
public short sizeInBytes() { short sizeForBlobIdV1 = (short) (VERSION_FIELD_LENGTH_IN_BYTES + partitionId.getBytes().length + UUID_SIZE_FIELD_LENGTH_IN_BYTES START + uuid.length()); END switch (version) { case BLOB_ID_V1: return sizeForBlobIdV1; case BLOB_ID_V2: return (short) (FLAG_FIELD_LENGTH_IN_BYTES + DATACENTER_ID_FIELD_LENGTH_IN_BYTES + ACCOUNT_ID_FIELD_LENGTH_IN_BYTES + CONTAINER_ID_FIELD_LENGTH_IN_BYTES + sizeForBlobIdV1); default: throw new IllegalArgumentException("blobId version=" + version + " not supported"); } } shouldn't this be `uuid.getBytes().length`?

Target:
public short sizeInBytes() { short sizeForBlobIdV1 = (short) (VERSION_FIELD_LENGTH_IN_BYTES + partitionId.getBytes().length + UUID_SIZE_FIELD_LENGTH_IN_BYTES + uuid.getBytes().length); switch (version) { case BLOB_ID_V1: return sizeForBlobIdV1; case BLOB_ID_V2: return (short) (FLAG_FIELD_LENGTH_IN_BYTES + DATACENTER_ID_FIELD_LENGTH_IN_BYTES + ACCOUNT_ID_FIELD_LENGTH_IN_BYTES + CONTAINER_ID_FIELD_LENGTH_IN_BYTES + sizeForBlobIdV1); default: throw new IllegalArgumentException("blobId version=" + version + " not supported"); } }
Source:
START public short getDatacenterId() { END return datacenterId; } why isn't the return value a `byte`?

Target:
public byte getDatacenterId() { return datacenterId; }
Source:
public boolean remove(String path, int options) { if (path.equals("/")) { notifyListeners("/", HelixPropertyStoreUtils.StoreOperationType.DELETE); pathToRecords.clear(); return true; } else { START throw new IllegalStateException("Not implemented"); END } } do we have test cases that remove a particular path that is not "/"?

Target:
public boolean remove(String path, int options) { if (path.equals("/")) { pathToRecords.clear(); notifyListeners("/", HelixPropertyStoreUtils.StoreOperationType.DELETE); return true; } else { throw new IllegalStateException("Not implemented"); } }
Source:
START public boolean equals(Object o) { END if (this == o) { return true; } if (!(o instanceof Account)) { return false; } Account account = (Account) o; if (id != account.id) { return false; } if (!name.equals(account.name)) { return false; } return status.equals(account.status); } was this auto generated ?

Target:
public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof Account)) { return false; } Account account = (Account) o; if (!id.equals(account.id)) { return false; } if (!name.equals(account.name)) { return false; } return status == account.status; }
Source:
public AccountBuilder(Account origin) { if (origin == null) { throw new IllegalArgumentException("origin cannot be null."); } this.id = origin.getId(); this.name = origin.getName(); this.status = origin.getStatus(); for (Container container : origin.getAllContainers()) { idToContainerMetadataMap.put(container.getId(), container); } START this.version = ACCOUNT_METADATA_VERSION_1; END } something to think about. Incase we introduce a new version for Account sometime in future, how will this code look like. Some fields in V2 may not be available in V1. Do we have dummy values for it ?

Target:
public AccountBuilder(Account origin) { if (origin == null) { throw new IllegalArgumentException("origin cannot be null."); } id = origin.getId(); name = origin.getName(); status = origin.getStatus(); for (Container container : origin.getAllContainers()) { idToContainerMetadataMap.put(container.getId(), container); } }
Source:
public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Partition partition = (Partition) o; START if (id == null ^ partition.id == null){ END // one of them is null (XOR, so not both) return false; } return id.equals(partition.id); } I think this check is unnecessary and can be omitted. `id` is supposed to be non-null, the construction of `Partition` will fail otherwise.

Target:
public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Partition partition = (Partition) o; return id.equals(partition.id); }
Source:
CompactAllPolicy(StoreConfig storeConfig, Time time) { this.storeConfig = storeConfig; this.time = time; START this.messageRetentionTimeInMs = (long) storeConfig.storeDeletedMessageRetentionDays * Time.SecsPerDay * Time.MsPerSec; END } This is being fixed in a different way (using the preferred `TimeUnit` utilities) as part of [this](https://github.com/linkedin/ambry/pull/617/files#diff-fad74bb47f2e54088777e34754ff84d6L52) patch.

Target:
CompactAllPolicy(StoreConfig storeConfig, Time time) { this.storeConfig = storeConfig; this.time = time; this.messageRetentionTimeInMs = storeConfig.storeDeletedMessageRetentionDays * Time.SecsPerDay * Time.MsPerSec; }
Source:
void setBelongsToRecentIndexSegment(boolean belongsToRecentIndexSegment) { START this.belongsToRecentIndexSegment.compareAndSet(false, belongsToRecentIndexSegment); END } could be just ``` this. belongsToRecentIndexSegment = this. belongsToRecentIndexSegment || belongsToRecentIndexSegment ``` don't see why `compareAndSet()` (which is used in multithreading scenarios) is needed.

Target:
void setBelongsToRecentIndexSegment(boolean belongsToRecentIndexSegment) { this.belongsToRecentIndexSegment = this.belongsToRecentIndexSegment || belongsToRecentIndexSegment; }
Source:
START public void setValue(long value) { END this.value = value; } are these setters used anywhere?

Target:
void setValue(long value) { this.value = value; }
Source:
void enable() { if (compactionExecutor != null) { logger.info("Compaction thread started for {}", mountPath); compactionThread = Utils.newThread(THREAD_NAME_PREFIX + mountPath, compactionExecutor, true); compactionThread.start(); try { START if (!compactionExecutor.startLatch.await(1, TimeUnit.SECONDS)) { END throw new IllegalStateException("Compaction thread did not start in 1 second"); } } catch (InterruptedException e) { throw new IllegalStateException(e); } } } why do you care about this? This is essentially checking if the JVM started the thread we asked it to start, isn't it? That seems like an arbitrary check to do. If this is being done for tests, perhaps you can use `TestUtils.getThreadByThisName()`?

Target:
void enable() { if (compactionExecutor != null) { logger.info("Compaction thread started for {}", mountPath); compactionThread = Utils.newThread(THREAD_NAME_PREFIX + mountPath, compactionExecutor, true); compactionThread.start(); } }
Source:
public void cleanup() throws InterruptedException, IOException, StoreException { state.destroy(); assertTrue(tempDir.getAbsolutePath() + " could not be deleted", StoreTestUtils.cleanDirectory(tempDir, true)); START indexScannerScheduler.shutdown(); END if (!indexScannerScheduler.awaitTermination(1000, TimeUnit.MILLISECONDS)) { fail("Could not terminate index scanner task after BlobStoreStats shutdown"); } queueProcessorScheduler.shutdown(); if (!queueProcessorScheduler.awaitTermination(1000, TimeUnit.MILLISECONDS)) { fail("Could not terminate queue processor task after BlobStoreStats shutdown"); } } shouldn't you shutdown the schedulers and then cleanup the directory

Target:
public void cleanup() throws InterruptedException, IOException, StoreException { indexScannerScheduler.shutdown(); if (!indexScannerScheduler.awaitTermination(1000, TimeUnit.MILLISECONDS)) { fail("Could not terminate index scanner task after BlobStoreStats shutdown"); } queueProcessorScheduler.shutdown(); if (!queueProcessorScheduler.awaitTermination(1000, TimeUnit.MILLISECONDS)) { fail("Could not terminate queue processor task after BlobStoreStats shutdown"); } state.destroy(); assertTrue(tempDir.getAbsolutePath() + " could not be deleted", StoreTestUtils.cleanDirectory(tempDir, true)); }
Source:
public CompactAllPolicyTest() throws InterruptedException { Pair<MockBlobStore, StoreConfig> initState = CompactionPolicyTest.initializeBlobStore(properties, time, -1, -1, -1); START this.config = initState.getSecond(); END this.messageRetentionTimeInMs = config.storeDeletedMessageRetentionDays * Time.HoursPerDay * Time.SecsPerHour * Time.MsPerSec; this.blobStore = initState.getFirst(); this.compactionPolicy = new CompactAllPolicy(config, time); } minor: `this` qualifiers not required.

Target:
public CompactAllPolicyTest() throws InterruptedException { Pair<MockBlobStore, StoreConfig> initState = CompactionPolicyTest.initializeBlobStore(properties, time, -1, -1, -1); config = initState.getSecond(); messageRetentionTimeInMs = config.storeDeletedMessageRetentionDays * Time.HoursPerDay * Time.SecsPerHour * Time.MsPerSec; blobStore = initState.getFirst(); compactionPolicy = new CompactAllPolicy(config, time); }
Source:
START CompactionDetails getCompactionDetails(BlobStore blobStore) throws StoreException { END return blobStore.getCompactionDetails(compactionPolicy); } can we make this a `private` function now? It will be tested along with the `CompactionExecutor` anyway. The actual policy tests are in the respective tests.

Target:
private CompactionDetails getCompactionDetails(BlobStore blobStore) throws StoreException { return blobStore.getCompactionDetails(compactionPolicy); }
Source:
public void shutdown() throws InterruptedException { if (statsAggregator != null) { START statsAggregator.cancel(true); END } scheduler.shutdown(); if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) { logger.error("Could not terminate aggregator tasks after StatsManager shutdown"); } } wouldn't it be better to safely shutdown the actual stats job rather than cancelling it and terminating it in the middle? If this was being pushed to a central location, it would cut off the data abruptly.

Target:
public void shutdown() throws InterruptedException { if (statsAggregator != null) { statsAggregator.cancel(); } if (scheduler != null) { scheduler.shutdown(); if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) { logger.error("Could not terminate aggregator tasks after StatsManager shutdown"); } } }
Source:
public void requestHandleWithGoodInputTest() throws Exception { doRequestHandleTest(HttpMethod.POST, "POST", false, false); doRequestHandleTest(HttpMethod.GET, "GET", false, false); doRequestHandleTest(HttpMethod.DELETE, "DELETE", false, false); // SSL enabled START doRequestHandleTest(HttpMethod.GET, "GET", false, true); END } why just for GET and not for all request types?

Target:
public void requestHandleWithGoodInputTest() throws Exception { doRequestHandleTest(HttpMethod.POST, "POST", false, false); doRequestHandleTest(HttpMethod.GET, "GET", false, false); doRequestHandleTest(HttpMethod.DELETE, "DELETE", false, false); // SSL enabled doRequestHandleTest(HttpMethod.POST, "POST", false, true); doRequestHandleTest(HttpMethod.GET, "GET", false, true); doRequestHandleTest(HttpMethod.DELETE, "DELETE", false, true); }
Source:
public void getLogSegmentsNotInJournalTest() throws InterruptedException, IOException, StoreException { if (isLogSegmented) { testGetLogSegmentsNotInJournal(2, 2 * MAX_IN_MEM_ELEMENTS, false); START testGetLogSegmentsNotInJournal(3, 2 * MAX_IN_MEM_ELEMENTS, false); END } else { assertEquals("LogSegments mismatch for non segmented log ", null, state.index.getLogSegmentsNotInJournal()); } } what is the intent of this function call? `testGetLogSegmentsNotInJournal()` seems to reduce this to the case tested in the previous function call.

Target:
public void getLogSegmentsNotInJournalTest() throws InterruptedException, IOException, StoreException { if (isLogSegmented) { testGetLogSegmentsNotInJournal(2, 2 * MAX_IN_MEM_ELEMENTS); testGetLogSegmentsNotInJournal(3, 2 * MAX_IN_MEM_ELEMENTS); } else { assertEquals("LogSegments mismatch for non segmented log ", null, state.index.getLogSegmentsNotInJournal()); } }
Source:
public ClusterParticipant getClusterParticipant() throws IOException { if (clusterMapConfig == null) { throw new IllegalArgumentException( "Cluster participant creation requires the factory to be initialized with a non-null ClusterMapConfig"); } START if (clusterParticipant == null) { END clusterParticipant = new ClusterParticipant() { @Override public void initialize(String hostname, int port) throws Exception { } @Override public void terminate() { } }; } return clusterParticipant; } any reason to not do this in the constructor? Is it `null` in the normal flow?

Target:
public ClusterParticipant getClusterParticipant() throws IOException { if (clusterParticipant == null) { clusterParticipant = new ClusterParticipant() { @Override public void initialize(String hostname, int port) { } @Override public void close() { } }; } return clusterParticipant; }
Source:
START public HelixClusterAgentsFactory(ClusterMapConfig clusterMapConfig, MetricRegistry metricRegistry) { END this.clusterMapConfig = clusterMapConfig; this.instanceName = ClusterMapUtils.getInstanceName(clusterMapConfig.clusterMapHostName, clusterMapConfig.clusterMapPort); helixFactory = new HelixFactory(); this.metricRegistry = metricRegistry; } why are there two public constructors?

Target:
HelixClusterAgentsFactory(ClusterMapConfig clusterMapConfig, MetricRegistry metricRegistry) { this.clusterMapConfig = clusterMapConfig; this.instanceName = ClusterMapUtils.getInstanceName(clusterMapConfig.clusterMapHostName, clusterMapConfig.clusterMapPort); helixFactory = new HelixFactory(); this.metricRegistry = metricRegistry; }
Source:
private boolean alreadyExistsInTgt(StoreKey key, IndexValue srcValue) throws StoreException { IndexValue tgtValue = tgtIndex.findKey(key); return tgtValue != null && srcValue.isFlagSet(IndexValue.Flags.Delete_Index) == tgtValue.isFlagSet( IndexValue.Flags.Delete_Index); START } END minor: exists in the targetIndex

Target:
private boolean alreadyExistsInTgt(StoreKey key, IndexValue srcValue) throws StoreException { IndexValue tgtValue = tgtIndex.findKey(key); return tgtValue != null && (tgtValue.isFlagSet(IndexValue.Flags.Delete_Index) || !srcValue.isFlagSet( IndexValue.Flags.Delete_Index)); }
Source:
public void basicTest() throws Exception { refreshState(false, true); List<String> segmentsUnderCompaction = getLogSegments(0, 2); START reduceValidDataSizeInLogSegments(segmentsUnderCompaction, state.log.getSegmentCapacity() - LogSegment.HEADER_SIZE); END long deleteReferenceTimeMs = state.time.milliseconds(); compactAndVerify(segmentsUnderCompaction, deleteReferenceTimeMs, true); } minor: I do know that we don't increment the time here. But just to mimic the actual flow, can you make reduceValidDataSizeInLogSegments return the deleteRefTime and use the same in next line.

Target:
public void basicTest() throws Exception { refreshState(false, true); List<String> segmentsUnderCompaction = getLogSegments(0, 2); long deleteReferenceTimeMs = reduceValidDataSizeInLogSegments(segmentsUnderCompaction, state.log.getSegmentCapacity() - LogSegment.HEADER_SIZE); compactAndVerify(segmentsUnderCompaction, deleteReferenceTimeMs, true); }
Source:
void notifyForFirstChunkCreation() { START if (firstChunkIdAndProperties != null) { END String chunkId = firstChunkIdAndProperties.getFirst().getID(); BlobProperties chunkProperties = firstChunkIdAndProperties.getSecond(); notificationSystem.onBlobCreated(chunkId, chunkProperties, userMetadata, NotificationBlobType.DataChunk); } } Maybe omit this check since this method will now only be called if it is known for sure that there is a first chunk that needs notification (which is also why it is no longer prefixed with "maybe").

Target:
void notifyForFirstChunkCreation() { String chunkId = firstChunkIdAndProperties.getFirst().getID(); BlobProperties chunkProperties = firstChunkIdAndProperties.getSecond(); notificationSystem.onBlobCreated(chunkId, chunkProperties, userMetadata, NotificationBlobType.DataChunk); }
Source:
START SortedMap<String, Long> getUsedCapacityBySegment() { END SortedMap<String, Long> usedCapacityBySegments = new TreeMap<>(); LogSegment logSegment = log.getFirstSegment(); while (logSegment != null) { usedCapacityBySegments.put(logSegment.getName(), logSegment.getEndOffset()); logSegment = log.getNextSegment(logSegment); } return usedCapacityBySegments; } Why the map needs to be sorted?

Target:
public SortedMap<String, Long> getUsedCapacityBySegment() { SortedMap<String, Long> usedCapacityBySegments = new TreeMap<>(); LogSegment logSegment = log.getFirstSegment(); while (logSegment != null) { usedCapacityBySegments.put(logSegment.getName(), logSegment.getEndOffset()); logSegment = log.getNextSegment(logSegment); } return usedCapacityBySegments; }
Source:
Pair<StoreKey, PersistentIndex.IndexEntryType> getResetKey() { START if (version == (short) 1 && resetKey != null) { END return new Pair<>(resetKey.getFirst(), PersistentIndex.IndexEntryType.values()[resetKey.getSecond()]); } else { return null; } } Should for ease of test be the reason to change the main code?

Target:
Pair<StoreKey, PersistentIndex.IndexEntryType> getResetKey() { return resetKey; }
Source:
short getVersion() { START return this.version; END } minor: this not required.

Target:
short getVersion() { return version; }
Source:
private void initializeAdminsAndAddCluster() { for (Map.Entry<String, String> entry : dataCenterToZkAddress.entrySet()) { ZKHelixAdmin admin = new ZKHelixAdmin(entry.getValue()); adminForDc.put(entry.getKey(), admin); // Add a cluster entry in every DC if (!admin.getClusters().contains(clusterName)) { admin.addCluster(clusterName); START admin.addStateModelDef(clusterName, "LeaderStandby", LeaderStandbySMD.build()); END } } } replace the string "LeaderStandby" with LeaderStandbySMD.name

Target:
private void initializeAdminsAndAddCluster() { for (Map.Entry<String, String> entry : dataCenterToZkAddress.entrySet()) { ZKHelixAdmin admin = new ZKHelixAdmin(entry.getValue()); adminForDc.put(entry.getKey(), admin); // Add a cluster entry in every DC if (!admin.getClusters().contains(clusterName)) { admin.addCluster(clusterName); admin.addStateModelDef(clusterName, LeaderStandbySMD.name, LeaderStandbySMD.build()); } } }
Source:
START StoreFindToken() { END this(Type.Uninitialized, null, null, null, null); } Can/Will this constructor be gotten rid of once Version_0 and Version_1 are phased out?

Target:
StoreFindToken() { this(Type.Uninitialized, null, null, null, null, true); }
Source:
boolean getInclusive() { START return inclusive == 1 ? true : false; END } 1. In general, avoid `expr ? true : false` pattern and simply substitute that with `expr`. 2. Ideally, this check should be `... == (byte) 1` or `(... != 0)`.

Target:
boolean getInclusive() { return inclusive == (byte) 1; }
Source:
private void deleteBlob(final String blobId, FutureResult<Void> futureResult, final Callback<Void> callback) { deleteManager.submitDeleteBlobOperation(blobId, futureResult, new Callback<Void>() { @Override public void onCompletion(Void result, Exception exception) { if (exception == null) { backgroundDeleter.initiateChunkDeletesIfAny(blobId); } START callback.onCompletion(result, exception); END } }); readyForPollCallback.onPollReady(); } `callback` can be `null`.

Target:
private void deleteBlob(final String blobId, FutureResult<Void> futureResult, final Callback<Void> callback) { deleteManager.submitDeleteBlobOperation(blobId, futureResult, new Callback<Void>() { @Override public void onCompletion(Void result, Exception exception) { if (exception == null) { backgroundDeleter.initiateChunkDeletesIfAny(blobId); } if (callback != null) { callback.onCompletion(result, exception); } } }); readyForPollCallback.onPollReady(); }
Source:
BackgroundDeleter() throws IOException { super("backgroundDeleter"); START super.putManager.close(); END } minor: do you need the `super` qualifier here?

Target:
BackgroundDeleter() throws IOException { super("backgroundDeleter"); putManager.close(); }
Source:
static long getGeneration(String name) { if (name.isEmpty()) { throw new IllegalArgumentException("Name provided cannot be empty"); } START return Long.parseLong(name.substring(name.indexOf(BlobStore.SEPARATOR) + 1, name.length())); END } we don't need the 2nd param(name.length()) in subString.

Target:
static long getGeneration(String name) { if (name.isEmpty()) { throw new IllegalArgumentException("Name provided cannot be empty"); } return Long.parseLong(name.substring(name.indexOf(BlobStore.SEPARATOR) + 1)); }
Source:
DiskIOScheduler(Map<String, Throttler> throttlers) { if (throttlers != null) { START this.throttlers.putAll(throttlers); END } } minor: currently, i don't think the concurrent map instance in this class is required (might be in the future if we allow injecting of throttlers). For now, won't `this.throttlers = throttlers` do?

Target:
DiskIOScheduler(Map<String, Throttler> throttlers) { this.throttlers = throttlers != null ? throttlers : new HashMap<String, Throttler>(); }
Source:
Store getStore(PartitionId id) { BlobStore store = stores.get(id); START return store.isStarted() ? store : null; END } if store is null, NullPoinerException ?

Target:
Store getStore(PartitionId id) { BlobStore store = stores.get(id); return (store != null && store.isStarted()) ? store : null; }
Source:
START public IndexRecord(String msg, boolean isDeleted, boolean isExpired) { END this.message = msg; this.isDeleted = isDeleted; this.isExpired = isExpired; } Remove `public`

Target:
IndexRecord(String msg, boolean isDeleted, boolean isExpired) { this.message = msg; this.isDeleted = isDeleted; this.isExpired = isExpired; }
Source:
private void reportBytesRead(int count) { START if (bytesAvailable - count >= 0) { END bytesAvailable -= count; } } `if (bytesAvailable >= count)` looks simpler?

Target:
private void reportBytesRead(int count) { if (bytesAvailable >= count) { bytesAvailable -= count; } }
Source:
START private boolean waitForThreadState(Thread thread, Thread.State state) { END long MAX_WAIT_MS = 2000; long checkStartTimeMs = SystemTime.getInstance().milliseconds(); while (thread.getState() != state) { if (SystemTime.getInstance().milliseconds() - checkStartTimeMs > MAX_WAIT_MS) { return false; } else { Thread.yield(); } } return true; } docs

Target:
private boolean waitForThreadState(Thread thread, Thread.State state) { long checkStartTimeMs = SystemTime.getInstance().milliseconds(); while (thread.getState() != state) { if (SystemTime.getInstance().milliseconds() - checkStartTimeMs > MAX_WAIT_MS) { return false; } else { Thread.yield(); } } return true; }
Source:
private void onDequeue() { START router.metrics.totalOperationsInExecution.incrementAndGet(); END if (operationQueueStartTime != null) { long queueTime = System.currentTimeMillis() - operationQueueStartTime; router.metrics.operationQueuingTimeInMs.update(queueTime); logger.trace("Operation spent ms in execution queue", queueTime); router.metrics.operationDequeuingRate.mark(); } else { router.metrics.queueStartTimeNotRecordedError.inc(); } } perhaps this can go in where `onDequeue()` is called, so that the increment and decrement are within the same method?

Target:
private void onDequeue() { if (operationQueueStartTime != null) { long queueTime = System.currentTimeMillis() - operationQueueStartTime; router.metrics.operationQueuingTimeInMs.update(queueTime); logger.trace("Operation spent ms in execution queue", queueTime); router.metrics.operationDequeuingRate.mark(); } else { router.metrics.queueStartTimeNotRecordedError.inc(); } }
Source:
void onFillComplete() { buf.flip(); prepareForSending(); chunkReadyAtMs = time.milliseconds(); START routerMetrics.chunkFillTimeMs.update(chunkReadyAtMs - chunkFreeAtMs); END } Are metadata chunk and data chunk treated in the same way, so it is a single metric that reports `chunkFillTime` for both chunk types?

Target:
void onFillComplete() { buf.flip(); prepareForSending(); chunkReadyAtMs = time.milliseconds(); }
Source:
private PutChunk getChunkToFill() { if (chunkToFill != null && chunkToFill.isBuilding()) { return chunkToFill; } chunkToFill = null; for (PutChunk chunk : putChunks) { if (chunk.isFree()) { chunkCounter++; chunk.prepareForBuilding(chunkCounter, getSizeOfChunkAt(chunkCounter)); chunkToFill = chunk; START routerMetrics.chunkFillerWaitingTrunkTimeMs.update(time.milliseconds() - timeFilledPreviousChunkMs); END timeStartFillingCurrentChunkMs = time.milliseconds(); break; } } return chunkToFill; } I don't think this is correct. You are updating `timeFilledPreviousChunkMs` when the fill is complete and use that as the threshold for how much we wait. But once the fill is complete, we can't reuse the PutChunk until the chunk it holds is sent out, and we seem to end up calculating that in the "wait time"

Target:
private PutChunk getChunkToFill() { if (chunkToFill != null && chunkToFill.isBuilding()) { return chunkToFill; } chunkToFill = null; for (PutChunk chunk : putChunks) { if (chunk.isFree()) { chunkCounter++; chunk.prepareForBuilding(chunkCounter, getSizeOfChunkAt(chunkCounter)); chunkToFill = chunk; break; } } return chunkToFill; }
Source:
void poll(RequestRegistrationCallback<DeleteOperation> requestRegistrationCallback) { try { cleanupExpiredInflightRequests(); checkAndMaybeComplete(); if (!isOperationComplete()) { fetchRequests(requestRegistrationCallback); } } catch (Exception e) { START logger.error("Unexpected error in DeleteBlobOperation poll; operation cancelled.", e); END routerMetrics.unexpectedErrorCount.inc(); setOperationException(new RouterException("DeleteOperation poll encountered unexpected error", e, RouterErrorCode.UnexpectedInternalError)); } } a general rule of thumb to follow i think would be to not log unless the exception is being swallowed or it is the very last destination of the exception. In this case, the exception is being set and will be passed to someone else who will log it eventually. If you log when the exception is not being swallowed, there might be double logging and pollution of the log.

Target:
void poll(RequestRegistrationCallback<DeleteOperation> requestRegistrationCallback) { cleanupExpiredInflightRequests(); checkAndMaybeComplete(); if (!isOperationComplete()) { fetchRequests(requestRegistrationCallback); } }
Source:
void poll(RequestRegistrationCallback<DeleteOperation> requestRegistrationCallback) { try { cleanupExpiredInflightRequests(); checkAndMaybeComplete(); if (!isOperationComplete()) { fetchRequests(requestRegistrationCallback); } } catch (Exception e) { START routerMetrics.unexpectedErrorCount.inc(); END setOperationException(new RouterException("DeleteOperation poll encountered unexpected error", e, RouterErrorCode.UnexpectedInternalError)); } } did you consider pros and cons of doing this at the operation level v/s doing this try-catch at a higher level i.e. at the `RequestResponseHandler` thread itself?

Target:
void poll(RequestRegistrationCallback<DeleteOperation> requestRegistrationCallback) { cleanupExpiredInflightRequests(); checkAndMaybeComplete(); if (!isOperationComplete()) { fetchRequests(requestRegistrationCallback); } }
Source:
NodeLevelMetrics getDataNodeBasedMetrics(DataNodeId dataNodeId) { START return dataNodeToMetrics.containsKey(dataNodeId) ? dataNodeToMetrics.get(dataNodeId) : dataNodeToMetrics.get(null); END } why would we have a default case? what is the point in updating metrics for an "unknown" datanode?

Target:
NodeLevelMetrics getDataNodeBasedMetrics(DataNodeId dataNodeId) { return dataNodeToMetrics.get(dataNodeId); }
Source:
START public void setDigestAlgorithm(String digestAlgorithm) END throws NoSuchAlgorithmException { throw new IllegalStateException("Not implemented"); } how did you decide between throwing vs. silently ignoring (which is what you do in `ReadableStreamChannelInputStreamTest`)?

Target:
public void setDigestAlgorithm(String digestAlgorithm) { throw new IllegalStateException("Not implemented"); }
Source:
START public synchronized ResponseStatus getStatus() { END ResponseStatus status = null; try { if (responseMetadata.has(RESPONSE_STATUS_KEY)) { status = ResponseStatus.valueOf(responseMetadata.getString(RESPONSE_STATUS_KEY)); } } catch (Exception e) { throw new IllegalStateException(e); } return status; } For curiosity, will this method be called by multiple threads?

Target:
public ResponseStatus getStatus() { ResponseStatus status = null; try { if (responseMetadata.has(RESPONSE_STATUS_KEY)) { status = ResponseStatus.valueOf(responseMetadata.getString(RESPONSE_STATUS_KEY)); } } catch (Exception e) { throw new IllegalStateException(e); } return status; }
Source:
public void testSSLConnect() throws IOException { String connectionId = selector.connect(new InetSocketAddress("localhost", server.port), BUFFER_SIZE, BUFFER_SIZE, PortType.SSL); START Assert.assertFalse("Channel should not be ready by now (until handshake completes)", END selector.isChannelReady(connectionId)); while (!selector.connected().contains(connectionId)) { selector.poll(10000L); } Assert.assertTrue("Channel should have been ready by now ", selector.isChannelReady(connectionId)); } Is it possible that a handshake completes very fast and becomes ready before the connectionId is added to connected?

Target:
public void testSSLConnect() throws IOException { String connectionId = selector.connect(new InetSocketAddress("localhost", server.port), BUFFER_SIZE, BUFFER_SIZE, PortType.SSL); while (!selector.connected().contains(connectionId)) { selector.poll(10000L); } Assert.assertTrue("Channel should have been ready by now ", selector.isChannelReady(connectionId)); }
Source:
START public void initializeSelectorMetrics(final AtomicLong activeConnections) { END numActiveConnections = new Gauge<Long>() { @Override public Long getValue() { return activeConnections.get(); } }; } does this need to be `public`?

Target:
void initializeSelectorMetrics(final AtomicLong activeConnections) { numActiveConnections = new Gauge<Long>() { @Override public Long getValue() { return activeConnections.get(); } }; }
Source:
private void maybeWriteToChannel() { // if there are chunks available to be written out, do now. if (firstChunk.isComplete() && readCalled) { while (operationException.get() == null && chunkIndexToBuffer.containsKey(indexOfNextChunkToWriteOut)) { ByteBuffer chunkBuf = chunkIndexToBuffer.remove(indexOfNextChunkToWriteOut); START asyncWritableChannel.write(chunkBuf, chunkAsyncWriteCallback); END indexOfNextChunkToWriteOut++; } if (operationException.get() != null || numChunksWrittenOut.get() == numChunksTotal) { completeRead(); } } } Might be an orthogonal question, but if this (or any other statement really) were to throw a `RuntimeException`, who would catch and what is the behavior? And is that the right behavior?

Target:
private void maybeWriteToChannel() { // if there are chunks available to be written out, do now. if (firstChunk.isComplete() && readCalled) { while (operationException.get() == null && chunkIndexToBuffer.containsKey(indexOfNextChunkToWriteOut)) { ByteBuffer chunkBuf = chunkIndexToBuffer.remove(indexOfNextChunkToWriteOut); asyncWritableChannel.write(chunkBuf, chunkAsyncWriteCallback); indexOfNextChunkToWriteOut++; } if (operationException.get() != null || numChunksWrittenOut == numChunksTotal) { completeRead(); } } }
Source:
void handleResponse(ResponseInfo responseInfo) { GetRequest getRequest = (GetRequest) responseInfo.getRequest(); GetOperation getOperation = correlationIdToGetOperation.remove(getRequest.getCorrelationId()); if (getOperations.contains(getOperation)) { getOperation.handleResponse(responseInfo); if (getOperation.isOperationComplete()) { getOperations.remove(getOperation); START routerMetrics.operationDequeuingRate.mark(); END } } } is it possible you will double count if `close()` has removed the operation already?

Target:
void handleResponse(ResponseInfo responseInfo) { GetRequest getRequest = (GetRequest) responseInfo.getRequest(); GetOperation getOperation = correlationIdToGetOperation.remove(getRequest.getCorrelationId()); if (getOperations.contains(getOperation)) { getOperation.handleResponse(responseInfo); if (getOperation.isOperationComplete()) { remove(getOperation); } } }
Source:
public void testCompositeBlobNotChunkSizeMultipleGetSuccess() throws Exception { START END maxChunkSize = random.nextInt(1024 * 1024) + 1; for (int i = 0; i < 10; i++) { blobSize = maxChunkSize * i + random.nextInt(maxChunkSize - 1) + 1; doPut(); getAndAssertSuccess(); } } why extra line

Target:
public void testCompositeBlobNotChunkSizeMultipleGetSuccess() throws Exception { for (int i = 0; i < 10; i++) { blobSize = maxChunkSize * i + random.nextInt(maxChunkSize - 1) + 1; doPut(); getAndAssertSuccess(); } }
Source:
public void testSimpleBlobGetSuccess() throws Exception { maxChunkSize = random.nextInt(1024 * 1024) + 1; for (int i = 0; i < 10; i++) { // blobSize in the range [1, maxChunkSize] blobSize = random.nextInt(maxChunkSize) + 1; START doPut(); END getAndAssertSuccess(); } } once again, personal pref - but i would find the tests _much_ more readable if `doPut()` just took the arguments it needed instead of us setting the arguments via class variables. One of the reasons I say that is because if I want to reuse the function and want to know what its knobs are, I _have_ to read the whole function. If it took arguments, I would simply read the description of the arguments and know what I need. Any new arguments will also need to be added as class variables and that might make it more confusing.

Target:
public void testSimpleBlobGetSuccess() throws Exception { for (int i = 0; i < 10; i++) { // blobSize in the range [1, maxChunkSize] blobSize = random.nextInt(maxChunkSize) + 1; doPut(); getAndAssertSuccess(); } }
Source:
START boolean remove(GetOperation op) { END if (getOperations.remove(op)) { routerMetrics.operationDequeuingRate.mark(); return true; } else { return false; } } why is this not `private`?

Target:
private boolean remove(GetOperation op) { if (getOperations.remove(op)) { routerMetrics.operationDequeuingRate.mark(); return true; } else { return false; } }
Source:
START public void initializeTests() END throws Exception { coordinatorProps = new Properties(); notificationSystem = new MockNotificationSystem(9); plaintextCluster = new MockCluster(notificationSystem, SystemTime.getInstance()); plaintextCluster.startServers(); } these could even go into the constructor. Test classes get instantiated for each individual test.

Target:
public void initializeTests() throws Exception { coordinatorProps = new Properties(); notificationSystem = new MockNotificationSystem(9); plaintextCluster = new MockCluster(notificationSystem, false, SystemTime.getInstance()); plaintextCluster.startServers(); }
Source:
protected GetRequest createGetRequest(BlobId blobId, MessageFormatFlags flag) { List<BlobId> blobIds = Collections.singletonList(blobId); START List<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>(); END PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobId.getPartition(), blobIds); partitionRequestInfoList.add(partitionRequestInfo); return new GetRequest(NonBlockingRouter.correlationIdGenerator.incrementAndGet(), routerConfig.routerHostname, flag, partitionRequestInfoList, GetOptions.None); } A singletonList() can be used here as well.

Target:
protected GetRequest createGetRequest(BlobId blobId, MessageFormatFlags flag) { List<BlobId> blobIds = Collections.singletonList(blobId); List<PartitionRequestInfo> partitionRequestInfoList = Collections.singletonList(new PartitionRequestInfo(blobId .getPartition(), blobIds)); return new GetRequest(NonBlockingRouter.correlationIdGenerator.incrementAndGet(), routerConfig.routerHostname, flag, partitionRequestInfoList, GetOptions.None); }
Source:
private void markOperationStart() { logger.trace("{} started for {}", operationType, blobId); START operationStartTime = System.currentTimeMillis(); END } Why don't we have rate metrics

Target:
void markOperationStart() { logger.trace("{} started for {}", operationType, blobId); operationStartTime = System.currentTimeMillis(); }
Source:
START protected void markStartTime() { END callbackTracker.markOperationStart(); } We can make these methods as private I guess.

Target:
void markStartTime() { callbackTracker.markOperationStart(); }
Source:
void submitDeleteBlobOperation(String blobIdString, FutureResult<Void> futureResult, Callback<Void> callback) { try { START BlobId blobId = RouterUtils.getBlobIdFromString(blobIdString, clusterMap); END DeleteOperation deleteOperation = new DeleteOperation(routerConfig, blobId, clusterMap, responseHandler, futureResult, callback, new DeleteRequestRegistrationCallbackImpl(), time); deleteOperations.add(deleteOperation); } catch (RouterException e) { NonBlockingRouter.completeOperation(futureResult, callback, null, e); } } I hope you plan to work on logging and metrics in a different patch. We need lot of trace logging in general.

Target:
void submitDeleteBlobOperation(String blobIdString, FutureResult<Void> futureResult, Callback<Void> callback) { try { BlobId blobId = RouterUtils.getBlobIdFromString(blobIdString, clusterMap); DeleteOperation deleteOperation = new DeleteOperation(routerConfig, responseHandler, blobId, futureResult, callback, time); deleteOperations.add(deleteOperation); } catch (RouterException e) { NonBlockingRouter.completeOperation(futureResult, callback, null, e); } }
Source:
private void prepareForSending() { try { // if this is part of a retry, make sure no previously attempted partitions are retried. if (partitionId != null) { attemptedPartitionIds.add(partitionId); } partitionId = getPartitionForPut(attemptedPartitionIds); chunkBlobId = new BlobId(partitionId); operationTracker = new SimpleOperationTracker(routerConfig.routerDatacenterName, partitionId, false, START routerConfig.routerPutSuccessTarget, routerConfig.routerPutRequestParallelism, false); END correlationIdToChunkPutRequestInfo.clear(); state = ChunkState.Ready; } catch (RouterException e) { setOperationException(e); } } why you dont want to shuffle replicas for put

Target:
private void prepareForSending() { try { // if this is part of a retry, make sure no previously attempted partitions are retried. if (partitionId != null) { attemptedPartitionIds.add(partitionId); } partitionId = getPartitionForPut(attemptedPartitionIds); chunkBlobId = new BlobId(partitionId); operationTracker = new SimpleOperationTracker(routerConfig.routerDatacenterName, partitionId, false, routerConfig.routerPutSuccessTarget, routerConfig.routerPutRequestParallelism); correlationIdToChunkPutRequestInfo.clear(); state = ChunkState.Ready; } catch (RouterException e) { setOperationException(e); } }