Repository: microsoft/garnet
File: libs/storage/Tsavorite/cs/src/core/Index/Recovery/Recovery.cs (C#)

Code before:
                    int pageIndex = hlog.GetPageIndexForPage(p);
                    recoveryStatus.WaitRead(pageIndex);

                    // We make an extra pass to clear locks when reading every page back into memory
                    ClearLocksOnPage(p, options);

                    ProcessReadPageAndFlush(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex);
                }
            }

            WaitUntilAllPagesHaveBeenFlushed(startPage, endPage, recoveryStatus);
        }

        private async ValueTask RecoverHybridLogAsync(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, CheckpointType checkpointType, RecoveryOptions options, CancellationToken cancellationToken)
        {
            if (untilAddress <= scanFromAddress)
                return;
            var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadFirst);

            for (long page = startPage; page < endPage; page += numPagesToReadFirst)
            {
                var end = Math.Min(page + numPagesToReadFirst, endPage);
                var numPagesToReadThisIteration = (int)(end - page);

                // Ensure that page slots that will be read into, have been flushed from previous reads. 
                await WaitUntilAllPagesHaveBeenFlushedAsync(page, end, recoveryStatus, cancellationToken).ConfigureAwait(false);

                ReadPagesWithMemoryConstraint(untilAddress, capacity, recoveryStatus, page, end, numPagesToReadThisIteration);

                for (var p = page; p < end; p++)
                {
                    // Ensure page has been read into memory
                    int pageIndex = hlog.GetPageIndexForPage(p);
                    await recoveryStatus.WaitReadAsync(pageIndex, cancellationToken).ConfigureAwait(false);

                    // We make an extra pass to clear locks when reading every page back into memory
                    ClearLocksOnPage(p, options);

                    ProcessReadPageAndFlush(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex);
                }
            }

            await WaitUntilAllPagesHaveBeenFlushedAsync(startPage, endPage, recoveryStatus, cancellationToken).ConfigureAwait(false);
        }

        private RecoveryStatus GetPageRangesToRead(long scanFromAddress, long untilAddress, CheckpointType checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadFirst)
        {
            startPage = hlog.GetPage(scanFromAddress);
            endPage = hlog.GetPage(untilAddress);
            if (untilAddress > hlog.GetStartLogicalAddress(endPage) && untilAddress > scanFromAddress)
            {

Review comment (performance):
add comment:
// Due to the use of a single read semaphore, this must be done in batches of "all flushes' followed by "all reads" to ensure proper sequencing of reads when usableCapacity != capacity (and thus the page-read index is not equal to the page-flush index). #Resolved

Code after:
            var readEndPage = Math.Min(page + numPagesToReadPerIteration, endPage);
            if (page < readEndPage)
            {
                var numPagesToRead = (int)(readEndPage - page);

                // Ensure that page slots that will be read into, have been flushed from previous reads. Due to the use of a single read semaphore,
                // this must be done in batches of "all flushes' followed by "all reads" to ensure proper sequencing of reads when
                // usableCapacity != capacity (and thus the page-read index is not equal to the page-flush index).
                await WaitUntilAllPagesHaveBeenFlushedAsync(page, readEndPage, recoveryStatus, cancellationToken).ConfigureAwait(false);
                ReadPagesWithMemoryConstraint(untilAddress, capacity, recoveryStatus, page, readEndPage, numPagesToRead);
            }

            return readEndPage;
        }

        private void RecoverHybridLog(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, CheckpointType checkpointType, RecoveryOptions options)
        {
            if (untilAddress <= scanFromAddress)
                return;
            var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadPerIteration);

            for (long page = startPage; page < endPage; page += numPagesToReadPerIteration)
            {
                var end = ReadPagesForRecovery(untilAddress, recoveryStatus, endPage, capacity, numPagesToReadPerIteration, page);

                for (var p = page; p < end; p++)
                {
                    // Ensure page has been read into memory
                    int pageIndex = hlog.GetPageIndexForPage(p);
                    recoveryStatus.WaitRead(pageIndex);

                    // We make an extra pass to clear locks when reading every page back into memory
                    ClearLocksOnPage(p, options);

                    ProcessReadPageAndFlush(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex);
                }
            }

            WaitUntilAllPagesHaveBeenFlushed(startPage, endPage, recoveryStatus);
        }

        private async ValueTask RecoverHybridLogAsync(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, CheckpointType checkpointType, RecoveryOptions options, CancellationToken cancellationToken)
        {
            if (untilAddress <= scanFromAddress)
                return;
            var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadPerIteration);

            for (long page = startPage; page < endPage; page += numPagesToReadPerIteration)
            {
                var end = await ReadPagesForRecoveryAsync(untilAddress, recoveryStatus, endPage, capacity, numPagesToReadPerIteration, page, cancellationToken).ConfigureAwait(false);