Repository: MystenLabs/sui
File: crates/sui-core/src/consensus_handler.rs (Rust)

Code before:
            let sequenced_transaction = SequencedConsensusTransaction {
                certificate_author_index: cert_origin,
                certificate_author,
                consensus_index: current_tx_index,
                transaction,
            };

            let Some(verified_transaction) = self
                .epoch_store
                .verify_consensus_transaction(sequenced_transaction)
            else {
                continue;
            };

            let key = verified_transaction.0.key();

            if let Some(tx_digest) = key.user_transaction_digest() {
                self.epoch_store
                    .cache_recently_finalized_transaction(tx_digest);
            }

            // Increment count and check if this is a duplicate within this commit.
            // This replaces the separate processed_set HashSet.
            let count = occurrence_counts.entry(key.clone()).or_insert(0);
            *count += 1;
            let in_commit = *count > 1;

            let in_cache = self.processed_cache.put(key.clone(), ()).is_some();
            if in_commit || in_cache {
                self.metrics.skipped_consensus_txns_cache_hit.inc();
                continue;
            }
            if self
                .epoch_store
                .is_consensus_message_processed(&key)
                .expect("db error")
            {
                self.metrics.skipped_consensus_txns.inc();
                continue;
            }

            first_commit_keys.insert(key.clone());

            state.output.record_consensus_message_processed(key);

            all_transactions.push(verified_transaction);
        }

        for key in first_commit_keys {
            if let Some(&count) = occurrence_counts.get(&key)
                && count > 1

Review comment (performance):
Out of curiosity - a deduped transaction will still appear in the checkpoint? I am mostly thinking the requirements around the determinism against the check on the LRU cache `processed_cache`. I am thinking that all honest validators will perform the exact same evictions - if needed - so in the end they'll end up with same dedupe results? 

Code after:
                consensus_index: current_tx_index,
                transaction,
            };

            let Some(verified_transaction) = self
                .epoch_store
                .verify_consensus_transaction(sequenced_transaction)
            else {
                continue;
            };

            let key = verified_transaction.0.key();

            if let Some(tx_digest) = key.user_transaction_digest() {
                self.epoch_store
                    .cache_recently_finalized_transaction(tx_digest);
            }

            // Increment count and check if this is a duplicate within this commit.
            // This replaces the separate processed_set HashSet.
            let count = occurrence_counts.entry(key.clone()).or_insert(0);
            *count += 1;
            let in_commit = *count > 1;

            let in_cache = self.processed_cache.put(key.clone(), ()).is_some();
            if in_commit || in_cache {
                self.metrics.skipped_consensus_txns_cache_hit.inc();
                continue;
            }
            if self
                .epoch_store
                .is_consensus_message_processed(&key)
                .expect("db error")
            {
                self.metrics.skipped_consensus_txns.inc();
                continue;
            }

            first_commit_keys.insert(key.clone());

            state.output.record_consensus_message_processed(key);

            all_transactions.push(verified_transaction);
        }

        for key in first_commit_keys {
            if let Some(&count) = occurrence_counts.get(&key)
                && count > 1
            {
                self.metrics
                    .consensus_handler_duplicate_tx_count