Repository: aptos-labs/aptos-core
File: testsuite/single_node_performance.py (Python)

Code before:

    RunGroupConfig(key=RunGroupKey("liquidity-pool-swap-stable"), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("liquidity-pool-swap-stable", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS),

    RunGroupConfig(key=RunGroupKey("deserialize-u256"), included_in=Flow.CONTINUOUS),

    # fee payer sequentializes transactions today. in these tests module publisher is the fee payer, so larger number of modules tests throughput with multiple fee payers
    RunGroupConfig(key=RunGroupKey("no-op-fee-payer"), included_in=LAND_BLOCKING_AND_C),
    RunGroupConfig(key=RunGroupKey("no-op-fee-payer", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C),

    RunGroupConfig(key=RunGroupKey("vector-trim-append-len3000-size1"), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("vector-remove-insert-len3000-size1"), included_in=Flow.CONTINUOUS),

    # waived because of missing monotonic counter native.
    RunGroupConfig(key=RunGroupKey("order-book-no-matches1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches25-pct1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches80-pct1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-size-skewed80-pct1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-no-matches50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches25-pct50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches80-pct50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-size-skewed80-pct50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("monotonic-counter-single"), included_in=Flow.CONTINUOUS, waived=True),

    RunGroupConfig(key=RunGroupKey("fibonacci-recursive-10"), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("fibonacci-tail-recursive-20"), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("fibonacci-iterative-30"), included_in=Flow.CONTINUOUS),

    RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True),
    RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True),

    # setting separately for previewnet, as we run on a different number of cores.
    RunGroupConfig(expected_tps=20000, key=RunGroupKey("apt-fa-transfer"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=15000, key=RunGroupKey("account-generation"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=60, key=RunGroupKey("publish-package"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=6800, key=RunGroupKey("token-v2-ambassador-mint"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=17000 if NUM_ACCOUNTS < 5000000 else 28000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True),
    RunGroupConfig(expected_tps=27000 if NUM_ACCOUNTS < 5000000 else 23000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True),
] + [
    # no-commit throughput of different executor, used on continuous flow
    RunGroupConfig(
        key=RunGroupKey(
            "no_commit_{}{}".format(
                transaction_type:="apt-fa-transfer" if FA_MIGRATION_COMPLETE else "coin-transfer", 
                "_sharding" if executor_sharding else "",
            ),
            executor_type=executor_type
        ), 
        key_extra=RunGroupKeyExtra(
            transaction_type_override=transaction_type,

Review comment (suggestion):
If you have not calibrated numbers, probably add `, waived=True` for now so CI ignores numbers for these

Code after:

    RunGroupConfig(key=RunGroupKey("liquidity-pool-swap-stable"), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("liquidity-pool-swap-stable", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS),

    RunGroupConfig(key=RunGroupKey("deserialize-u256"), included_in=Flow.CONTINUOUS),

    # fee payer sequentializes transactions today. in these tests module publisher is the fee payer, so larger number of modules tests throughput with multiple fee payers
    RunGroupConfig(key=RunGroupKey("no-op-fee-payer"), included_in=LAND_BLOCKING_AND_C),
    RunGroupConfig(key=RunGroupKey("no-op-fee-payer", module_working_set_size=DEFAULT_MODULE_WORKING_SET_SIZE), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("simple-script"), included_in=LAND_BLOCKING_AND_C),

    RunGroupConfig(key=RunGroupKey("vector-trim-append-len3000-size1"), included_in=Flow.CONTINUOUS),
    RunGroupConfig(key=RunGroupKey("vector-remove-insert-len3000-size1"), included_in=Flow.CONTINUOUS),

    # waived because of missing monotonic counter native.
    RunGroupConfig(key=RunGroupKey("order-book-no-matches1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches25-pct1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches80-pct1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-size-skewed80-pct1-market"), included_in=Flow.ORDER_BOOK, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-no-matches50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches25-pct50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-matches80-pct50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("order-book-balanced-size-skewed80-pct50-markets"), included_in=Flow.ORDER_BOOK | Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("monotonic-counter-single"), included_in=Flow.CONTINUOUS, waived=True),

    RunGroupConfig(key=RunGroupKey("fibonacci-recursive20"), included_in=Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("fibonacci-tail-recursive20"), included_in=Flow.CONTINUOUS, waived=True),
    RunGroupConfig(key=RunGroupKey("fibonacci-iterative20"), included_in=Flow.CONTINUOUS, waived=True),

    RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True),
    RunGroupConfig(expected_tps=50000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.REPRESENTATIVE, waived=True),

    # setting separately for previewnet, as we run on a different number of cores.
    RunGroupConfig(expected_tps=20000, key=RunGroupKey("apt-fa-transfer"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=15000, key=RunGroupKey("account-generation"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=60, key=RunGroupKey("publish-package"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=6800, key=RunGroupKey("token-v2-ambassador-mint"), included_in=Flow.MAINNET_LARGE_DB),
    RunGroupConfig(expected_tps=17000 if NUM_ACCOUNTS < 5000000 else 28000, key=RunGroupKey("coin_transfer_connected_components", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--connected-tx-grps 5000", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True),
    RunGroupConfig(expected_tps=27000 if NUM_ACCOUNTS < 5000000 else 23000, key=RunGroupKey("coin_transfer_hotspot", executor_type="sharded"), key_extra=RunGroupKeyExtra(sharding_traffic_flags="--hotspot-probability 0.8", transaction_type_override=""), included_in=Flow.MAINNET | Flow.MAINNET_LARGE_DB, waived=True),
] + [
    # no-commit throughput of different executor, used on continuous flow
    RunGroupConfig(
        key=RunGroupKey(
            "no_commit_{}{}".format(
                transaction_type:="apt-fa-transfer" if FA_MIGRATION_COMPLETE else "coin-transfer", 
                "_sharding" if executor_sharding else "",
            ),
            executor_type=executor_type
        ), 
        key_extra=RunGroupKeyExtra(
            transaction_type_override=transaction_type,