Repository: MinaProtocol/mina
Pull request #16767: [CI] Test ledger apply in benchmarks
Author: login: dkijania
Created: 2025-03-24T19:49:10Z | Merged: 2025-03-30T18:05:30Z

Incorporated test ledger apply run into our benchmark infra. Unfortunately it was not 100% following the flow which usually execute bench from python app. In this case there is a semi complicated script which runs benchmarks and put data in stdout. 

I made some changes to test ledger apply command and added new option makes json file with benchmark data to be dumped on file system. After that benchmark manager is only parsing mentioned json file and puts it to influx db. 

On a side node, looks like we need to invest more effort into moving influx output builder more into particular benchmark app (ocaml code, apart for jane street framework which suffers from inability to bump its version to one which supports json output) then just parse and upload benchmark. This will make entire benchmark solution more elastic 


Diff:
diff --git a/buildkite/scripts/bench/run.sh b/buildkite/scripts/bench/run.sh
index e0355d3dd579..327b2075113d 100755
--- a/buildkite/scripts/bench/run.sh
+++ b/buildkite/scripts/bench/run.sh
@@ -6,11 +6,11 @@ set -eox pipefail
 
 YELLOW_THRESHOLD="0.1"
 RED_THRESHOLD="0.3"
-EXTRA_ARGS=""
+EXTRA_ARGS="${EXTRA_ARGS:-}"
 
 source buildkite/scripts/bench/install.sh
 
-MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/enhance_benchmarks"
+MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/bench_for_ledger_test"
 while [[ "$#" -gt 0 ]]; do case $1 in
   heap-usage) BENCHMARK="heap-usage"; ;;
   mina-base) BENCHMARK="mina-base"; ;;
@@ -18,6 +18,9 @@ while [[ "$#" -gt 0 ]]; do case $1 in
     BENCHMARK="ledger-export"
     EXTRA_ARGS="--genesis-ledger-path ./genesis_ledgers/devnet.json"
   ;;
+  ledger-apply) 
+    BENCHMARK="ledger-apply"
+  ;;
   snark) 
     BENCHMARK="snark";
     K=1
diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall
index 93d44617718d..21f36ce2cd20 100644
--- a/buildkite/src/Command/Bench/Base.dhall
+++ b/buildkite/src/Command/Bench/Base.dhall
@@ -16,6 +16,8 @@ let Profiles = ../../Constants/Profiles.dhall
 
 let Command = ../../Command/Base.dhall
 
+let Cmd = ../../Lib/Cmds.dhall
+
 let Docker = ../../Command/Docker/Type.dhall
 
 let Size = ../Size.dhall
@@ -39,6 +41,7 @@ let Spec =
           , additionalDirtyWhen : List SelectFiles.Type
           , yellowThreshold : Double
           , redThreshold : Double
+          , preCommands : List Cmd.Type
           }
       , default =
           { mode = PipelineMode.Type.PullRequest
@@ -50,6 +53,7 @@ let Spec =
           , additionalDirtyWhen = [] : List SelectFiles.Type
           , yellowThreshold = 0.1
           , redThreshold = 0.2
+          , preCommands = [] : List Cmd.Type
           }
       }
 
@@ -59,11 +63,12 @@ let command
       ->  Command.build
             Command.Config::{
             , commands =
-                RunInToolchain.runInToolchain
-                  (Benchmarks.toEnvList Benchmarks.Type::{=})
-                  "./buildkite/scripts/bench/run.sh  ${spec.bench} --red-threshold ${Double/show
-                                                                                       spec.redThreshold} --yellow-threshold ${Double/show
-                                                                                                                                 spec.yellowThreshold}"
+                  spec.preCommands
+                # RunInToolchain.runInToolchain
+                    (Benchmarks.toEnvList Benchmarks.Type::{=})
+                    "./buildkite/scripts/bench/run.sh  ${spec.bench} --red-threshold ${Double/show
+                                                                                         spec.redThreshold} --yellow-threshold ${Double/show
+                                                                                                                                   spec.yellowThreshold}"
             , label =
                 "Perf: ${spec.label} ${PipelineMode.capitalName spec.mode}"
             , key = spec.key
diff --git a/buildkite/src/Command/Bench/LedgerApply.dhall b/buildkite/src/Command/Bench/LedgerApply.dhall
new file mode 100644
index 000000000000..ffa63a3e11d0
--- /dev/null
+++ b/buildkite/src/Command/Bench/LedgerApply.dhall
@@ -0,0 +1,61 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let Profiles = ../../Constants/Profiles.dhall
+
+let BuildFlags = ../../Constants/BuildFlags.dhall
+
+let SelectFiles = ../../Lib/SelectFiles.dhall
+
+let Spec =
+      { Type =
+          { key : Text, name : Text, label : Text, mode : PipelineMode.Type }
+      , default.mode = PipelineMode.Type.PullRequest
+      }
+
+let dependsOn =
+      DebianVersions.dependsOnStep
+        (None Text)
+        DebianVersions.DebVersion.Bullseye
+        Profiles.Type.Standard
+        BuildFlags.Type.Instrumented
+        "build"
+
+let pipeline
+    : Spec.Type -> Pipeline.Config.Type
+    =     \(spec : Spec.Type)
+      ->  BenchBase.pipeline
+            BenchBase.Spec::{
+            , path = "Bench"
+            , name = spec.name
+            , label = spec.label
+            , key = spec.key
+            , bench = "ledger-apply"
+            , mode = spec.mode
+            , dependsOn = dependsOn
+            , additionalDirtyWhen =
+              [ SelectFiles.exactly
+                  "buildkite/src/Command/Bench/LedgerApply"
+                  "dhall"
+              , SelectFiles.exactly
+                  "buildkite/scripts/tests/ledger_test_apply"
+                  "sh"
+              , SelectFiles.exactly "scripts/tests/ledger_test_apply" "sh"
+              ]
+            , preCommands =
+                RunInToolchain.runInToolchain
+                  [ "DUNE_INSTRUMENT_WITH=bisect_ppx"
+                  , "COVERALLS_TOKEN"
+                  , "BENCHMARK_FILE=input.json"
+                  ]
+                  "buildkite/scripts/tests/ledger_test_apply.sh && buildkite/scripts/upload-partial-coverage-data.sh ${spec.key}"
+            }
+
+in  { pipeline = pipeline, Spec = Spec }
diff --git a/buildkite/src/Jobs/Bench/LedgerApplyStable.dhall b/buildkite/src/Jobs/Bench/LedgerApplyStable.dhall
new file mode 100755
index 000000000000..a25572788099
--- /dev/null
+++ b/buildkite/src/Jobs/Bench/LedgerApplyStable.dhall
@@ -0,0 +1,15 @@
+let LedgerApply = ../../Command/Bench/LedgerApply.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in  Pipeline.build
+      ( LedgerApply.pipeline
+          LedgerApply.Spec::{
+          , name = "LedgerApplyStable"
+          , label = "Ledger Apply"
+          , key = "ledger-apply"
+          , mode = PipelineMode.Type.Stable
+          }
+      )
diff --git a/buildkite/src/Jobs/Bench/LedgerApplyUnstable.dhall b/buildkite/src/Jobs/Bench/LedgerApplyUnstable.dhall
new file mode 100755
index 000000000000..62546c0fa3ac
--- /dev/null
+++ b/buildkite/src/Jobs/Bench/LedgerApplyUnstable.dhall
@@ -0,0 +1,12 @@
+let LedgerApply = ../../Command/Bench/LedgerApply.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in  Pipeline.build
+      ( LedgerApply.pipeline
+          LedgerApply.Spec::{
+          , name = "LedgerApplyUnstable"
+          , label = "Ledger Apply"
+          , key = "ledger-apply"
+          }
+      )
diff --git a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall b/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall
deleted file mode 100644
index c0a4b2c26cc0..000000000000
--- a/buildkite/src/Jobs/Test/LedgerTestApplyTx.dhall
+++ /dev/null
@@ -1,80 +0,0 @@
-let PipelineTag = ../../Pipeline/Tag.dhall
-
-let PipelineMode = ../../Pipeline/Mode.dhall
-
-let S = ../../Lib/SelectFiles.dhall
-
-let Pipeline = ../../Pipeline/Dsl.dhall
-
-let JobSpec = ../../Pipeline/JobSpec.dhall
-
-let Command = ../../Command/Base.dhall
-
-let RunInToolchain = ../../Command/RunInToolchain.dhall
-
-let DebianVersions = ../../Constants/DebianVersions.dhall
-
-let Profiles = ../../Constants/Profiles.dhall
-
-let BuildFlags = ../../Constants/BuildFlags.dhall
-
-let Docker = ../../Command/Docker/Type.dhall
-
-let Size = ../../Command/Size.dhall
-
-let dependsOn =
-        DebianVersions.dependsOnStep
-          (None Text)
-          DebianVersions.DebVersion.Bullseye
-          Profiles.Type.Standard
-          BuildFlags.Type.Instrumented
-          "build"
-      # DebianVersions.dependsOnStep
-          (None Text)
-          DebianVersions.DebVersion.Bullseye
-          Profiles.Type.Standard
-          BuildFlags.Type.None
-          "build"
-
-let buildTestCmd
-    : Size -> Command.Type
-    =     \(cmd_target : Size)
-      ->  let key = "ledger-test-apply"
-
-          in  Command.build
-                Command.Config::{
-                , commands =
-                    RunInToolchain.runInToolchain
-                      [ "DUNE_INSTRUMENT_WITH=bisect_ppx", "COVERALLS_TOKEN" ]
-                      "buildkite/scripts/tests/ledger_test_apply.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}"
-                , label = "Test: Ledger apply txs"
-                , key = key
-                , target = cmd_target
-                , docker = None Docker.Type
-                , depends_on = dependsOn
-                }
-
-in  Pipeline.build
-      Pipeline.Config::{
-      , spec =
-          let unitDirtyWhen =
-                [ S.strictlyStart (S.contains "src")
-                , S.strictly (S.contains "Makefile")
-                , S.exactly "buildkite/src/Jobs/Test/LedgerTestApplyTx" "dhall"
-                , S.exactly "buildkite/scripts/tests/ledger_test_apply" "sh"
-                , S.exactly "scripts/tests/ledger_test_apply" "sh"
-                ]
-
-          in  JobSpec::{
-              , dirtyWhen = unitDirtyWhen
-              , path = "Test"
-              , name = "LedgerTestApplyTx"
-              , tags =
-                [ PipelineTag.Type.Long
-                , PipelineTag.Type.Test
-                , PipelineTag.Type.Stable
-                ]
-              , mode = PipelineMode.Type.PullRequest
-              }
-      , steps = [ buildTestCmd Size.XLarge ]
-      }
diff --git a/scripts/benchmarks/__main__.py b/scripts/benchmarks/__main__.py
index 2f4a56298e24..a8639a556b61 100644
--- a/scripts/benchmarks/__main__.py
+++ b/scripts/benchmarks/__main__.py
@@ -24,6 +24,7 @@
 run_bench.add_argument("--path", help="override path to benchmark")
 run_bench.add_argument("--branch", default="test", help="Required only if --format=csv. Add branch name to csv file")
 run_bench.add_argument("--genesis-ledger-path", default="./genesis_ledgers/devnet.json",  help="Applicable only for ledger-export benchmark. Location of genesis config file")
+run_bench.add_argument("--privkeys-path", default="./privkey",  help="Applicable only for test apply ledger. Location of privkeys")
 run_bench.add_argument("--k", default=1)
 run_bench.add_argument("--max-num-updates", default=4 , type=int)
 run_bench.add_argument("--min-num-updates", default=2, type=int)
@@ -53,7 +54,8 @@
 
 test_bench = subparsers.add_parser('test', help="Performs entire cycle of operations from run till upload")
 test_bench.add_argument("--benchmark", type=BenchmarkType, help="benchmark to test")
-test_bench.add_argument("--tmpfile", help="temporary location of result file")
+test_bench.add_argument("--tmpfile", help="temporary location of result file", default="/tmp/bench.tmp")
+test_bench.add_argument("--input-file", help="input file with raw benchmark", default="input.json")
 test_bench.add_argument("--path")
 test_bench.add_argument("--yellow-threshold",
                         help="defines how many percent current measurement can exceed average so app will trigger warning",
@@ -91,6 +93,8 @@ def select_benchmark(kind):
         return HeapUsageBenchmark()
     elif kind == BenchmarkType.snark:
         return SnarkBenchmark(args.k, args.max_num_updates, args.min_num_updates)
+    elif kind == BenchmarkType.ledger_apply:
+        return LedgerApplyBenchmark(args.input_file)
     elif kind == BenchmarkType.ledger_export:
         if args.genesis_ledger_path is None:
             print(
diff --git a/scripts/benchmarks/lib/bench.py b/scripts/benchmarks/lib/bench.py
index c2208989406c..a7bc4649e061 100644
--- a/scripts/benchmarks/lib/bench.py
+++ b/scripts/benchmarks/lib/bench.py
@@ -4,6 +4,7 @@
 import parse
 from pathlib import Path
 import io
+import json
 import os
 from enum import Enum
 import logging
@@ -154,6 +155,7 @@ class BenchmarkType(Enum):
     heap_usage = 'heap-usage'
     zkapp = 'zkapp'
     ledger_export = 'ledger-export'
+    ledger_apply = 'ledger-apply'
 
     def __str__(self):
         return self.value
@@ -374,6 +376,70 @@ def default_path(self):
         return "mina-ledger-export-benchmark"
 
 
+class LedgerApplyBenchmark(Benchmark):
+    """
+     Concrete implementation of Benchmark for ledger test apply benchmark.
+     It requires input json file with benchmark data in format
+     {
+        "final_time":"0.4000" # In ms
+        , "preparation_steps_mean": "0.432"
+     }
+    """
+
+    name = MeasurementColumn("Name", 0)
+    time = FieldColumn("time",  1, "ms")
+    preps_mean = FieldColumn("preps mean",  2, "")
+    category = TagColumn("category", 3)
+    branch = TagColumn("gitbranch", 4)
+
+    def __init__(self, benchmark_input_json):
+        Benchmark.__init__(self, BenchmarkType.ledger_apply)
+        self.benchmark_input_json = benchmark_input_json
+
+    def run(self, path=None):
+        pass
+
+    def fields(self):
+        return [
+            self.time, self.preps_mean
+        ]
+
+    def name_header(self):
+        return self.name
+
+    def branch_header(self):
+        return self.branch
+
+    def default_path(self):
+        return "mina"
+
+    def headers(self):
+        return [
+            LedgerApplyBenchmark.name, LedgerApplyBenchmark.time,
+            LedgerApplyBenchmark.preps_mean, LedgerApplyBenchmark.category,
+            LedgerApplyBenchmark.branch
+        ]
+
+    def parse(self, content, output_filename, influxdb, branch):
+
+        final_time_header = "final_time"
+        preparation_steps_mean_header = "preparation_steps_mean"
+
+        with open(self.benchmark_input_json) as f:
+            data = json.load(f)
+            final_time = data[final_time_header]
+            preparation_steps_mean = data[preparation_steps_mean_header]
+
+            with open(output_filename, 'w') as csvfile:
+                if influxdb:
+                    csvfile.write(
+                        self.headers_to_influx(self.headers()) + "\n")
+                csvwriter = csv.writer(csvfile)
+                csvwriter.writerow((self.name.name, self.time.name,self.preps_mean.name, self.category.name, self.branch.name))
+                csvwriter.writerow(("ledger-apply",final_time, preparation_steps_mean,"ledger-apply", branch))
+
+        return [output_filename]
+
 class ZkappLimitsBenchmark(Benchmark):
     """
      ZkappLimit benchmark has it's own output which we need to handle separately. It is similar to:
diff --git a/scripts/tests/ledger_test_apply.sh b/scripts/tests/ledger_test_apply.sh
index 8c04578aff98..5a6e88d1a552 100755
--- a/scripts/tests/ledger_test_apply.sh
+++ b/scripts/tests/ledger_test_apply.sh
@@ -14,6 +14,7 @@ export MINA_PRIVKEY_PASS='naughty blue worm'
 MINA_APP=_build/default/src/app/cli/src/mina.exe
 RUNTIME_LEDGER_APP=_build/default/src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe
 
+BENCHMARK_FILE=${BENCHMARK_FILE:-ledger_test_apply_bench.json}
 TEMP_FOLDER=$(mktemp -d)
 ACCOUNTS_FILE=$TEMP_FOLDER/accounts.json
 TEMP_ACCOUNTS_FILE=$TEMP_FOLDER/accounts_tmp.json
@@ -54,5 +55,5 @@ mkdir $TEMP_FOLDER/genesis/ledger
 tar -zxf  $TEMP_FOLDER/genesis/genesis_ledger_*.tar.gz -C $TEMP_FOLDER/genesis/ledger
 
 echo "running test:"
-time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 200
+time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 200 --dump-benchmark $BENCHMARK_FILE
 
diff --git a/src/app/cli/src/init/client.ml b/src/app/cli/src/init/client.ml
index 363eb9398367..338e72b18009 100644
--- a/src/app/cli/src/init/client.ml
+++ b/src/app/cli/src/init/client.ml
@@ -2309,7 +2309,11 @@ let test_ledger_application =
        flag "--has-second-partition"
          ~doc:"Assume there is a second partition (scan state)" no_arg
      and tracing = flag "--tracing" ~doc:"Wrap test into tracing" no_arg
-     and no_masks = flag "--no-masks" ~doc:"Do not create masks" no_arg in
+     and no_masks = flag "--no-masks" ~doc:"Do not create masks" no_arg
+     and benchmark =
+       flag "--dump-benchmark" ~doc:"Dump json file with benchmark data"
+         (optional string)
+     in
      Cli_lib.Exceptions.handle_nicely
      @@ fun () ->
      let first_partition_slots =
@@ -2325,7 +2329,7 @@ let test_ledger_application =
      Test_ledger_application.test ~privkey_path ~ledger_path ?prev_block_path
        ~first_partition_slots ~no_new_stack ~has_second_partition
        ~num_txs_per_round ~rounds ~no_masks ~max_depth ~tracing num_txs
-       ~constraint_constants ~genesis_constants )
+       ~constraint_constants ~genesis_constants ~benchmark )
 
 let itn_create_accounts =
   let compile_config = Mina_compile_config.Compiled.t in
diff --git a/src/app/cli/src/init/test_ledger_application.ml b/src/app/cli/src/init/test_ledger_application.ml
index 7adddbef204c..bdb1d6ed8c41 100644
--- a/src/app/cli/src/init/test_ledger_application.ml
+++ b/src/app/cli/src/init/test_ledger_application.ml
@@ -158,11 +158,13 @@ let apply_txs ~action_elements ~event_elements ~constraint_constants
   with
   | Ok (b, _, _, _, _) ->
       let root = Ledger.merkle_root ledger in
+      let elapsed = Time.diff (Time.now ()) start in
       printf
         !"Result of application %d: %B (took %s): new root %s\n%!"
         i b
-        Time.(Span.to_string @@ diff (now ()) start)
-        (Ledger_hash.to_base58_check root)
+        Time.(Span.to_string elapsed)
+        (Ledger_hash.to_base58_check root) ;
+      elapsed
   | Error e ->
       eprintf
         !"Error applying staged ledger: %s\n%!"
@@ -171,7 +173,8 @@ let apply_txs ~action_elements ~event_elements ~constraint_constants
 
 let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
     ~no_new_stack ~has_second_partition ~num_txs_per_round ~rounds ~no_masks
-    ~max_depth ~tracing num_txs_final ~(genesis_constants : Genesis_constants.t)
+    ~max_depth ~tracing num_txs_final ~benchmark
+    ~(genesis_constants : Genesis_constants.t)
     ~(constraint_constants : Genesis_constants.Constraint_constants.t) =
   O1trace.thread "mina"
   @@ fun () ->
@@ -218,7 +221,35 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := time :: !results
+  in
+  let save_and_dump_benchmarks final_time =
+    let calculate_mean preparation_steps =
+      let prep_steps_len = Float.of_int (List.length preparation_steps) in
+      let prep_steps_total_time =
+        List.fold preparation_steps ~init:Float.zero ~f:(fun acc time ->
+            acc +. Time.Span.to_ms time )
+      in
+      prep_steps_total_time /. prep_steps_len
+    in
+    match benchmark with
+    | Some benchmark ->
+        let preparation_steps_mean = calculate_mean !results in
+        let json =
+          `Assoc
+            [ ( "final_time"
+              , `String (Printf.sprintf "%.2f" (Time.Span.to_ms final_time)) )
+            ; ( "preparation_steps_mean"
+              , `String (Printf.sprintf "%.2f" preparation_steps_mean) )
+            ]
+        in
+        Yojson.Safe.to_file benchmark json
+    | None ->
+        ()
+  in
   printf !"Init root %s\n%!" (Ledger_hash.to_base58_check init_root) ;
   Deferred.List.fold (List.init rounds ~f:ident) ~init:(init_ledger, [])
     ~f:(fun (ledger, ledgers) i ->
@@ -229,10 +260,10 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
       |> Option.iter ~f:drop_old_ledger ;
       apply ~action_elements:0 ~event_elements:0 ~num_txs:num_txs_per_round ~i
         ledger
-      >>| mask_handler ledger
+      >>| save_preparation_times >>| mask_handler ledger
       >>| Fn.flip Tuple2.create (ledger :: ledgers) )
   >>| fst
   >>= apply ~num_txs:num_txs_final
         ~action_elements:genesis_constants.max_action_elements
         ~event_elements:genesis_constants.max_event_elements ~i:rounds
-  >>| stop_tracing
+  >>| stop_tracing >>| save_and_dump_benchmarks


Reviews:
user: login: dkijania
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-25T19:28:58Z
user: login: georgeee
body: Left some minor comments
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-03-26T11:35:57Z
user: login: dkijania
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T07:06:00Z
user: login: dkijania
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T07:09:16Z
user: login: georgeee
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-03-27T09:43:30Z
user: login: dkijania
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T13:31:16Z
user: login: dkijania
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T13:32:18Z
user: login: georgeee
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-27T15:01:43Z
user: login: SanabriaRusso
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-27T20:31:20Z

Review comments:
diff_hunk: @@ -6,18 +6,21 @@ set -eox pipefail
 
 YELLOW_THRESHOLD="0.1"
 RED_THRESHOLD="0.3"
-EXTRA_ARGS=""
+EXTRA_ARGS="${EXTRA_ARGS:-}"
 
 source buildkite/scripts/bench/install.sh
 
-MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/enhance_benchmarks"
+MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/bench_for_ledger_test"
path: buildkite/scripts/bench/run.sh
user: login: dkijania
body: This is a way to test upload of benchmark
created_at: 2025-03-25T19:28:58Z
author_association: MEMBER
diff_hunk: @@ -6,18 +6,21 @@ set -eox pipefail
 
 YELLOW_THRESHOLD="0.1"
 RED_THRESHOLD="0.3"
-EXTRA_ARGS=""
+EXTRA_ARGS="${EXTRA_ARGS:-}"
 
 source buildkite/scripts/bench/install.sh
 
-MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/enhance_benchmarks"
+MAINLINE_BRANCHES="-m develop -m compatible -m master -m dkijania/bench_for_ledger_test"
path: buildkite/scripts/bench/run.sh
user: login: georgeee
body: Will we remove it in a follow-up PR?
created_at: 2025-03-26T11:23:02Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,15 @@
+let LedgerApply = ../../Command/Bench/LedgerApply.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in  Pipeline.build
+      ( LedgerApply.pipeline
+          LedgerApply.Spec::{
+          , name = "LedgerApplyUnstable"
+          , label = "Ledger Apply"
+          , key = "ledger-apply"
+          , mode = PipelineMode.Type.Stable
path: buildkite/src/Jobs/Bench/LedgerApplyUnstable.dhall
user: login: georgeee
body: Is it a mistake? A bit counter-intuitive that the **LedgerApplyUnstable.dhall** file uses `Stable` pipeline mode.
created_at: 2025-03-26T11:26:01Z
author_association: MEMBER
diff_hunk: @@ -54,5 +55,5 @@ mkdir $TEMP_FOLDER/genesis/ledger
 tar -zxf  $TEMP_FOLDER/genesis/genesis_ledger_*.tar.gz -C $TEMP_FOLDER/genesis/ledger
 
 echo "running test:"
-time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 200
+time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 20 --dump-benchmark $BENCHMARK_FILE
path: scripts/tests/ledger_test_apply.sh
user: login: georgeee
body: ```suggestion
time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 200 --dump-benchmark $BENCHMARK_FILE
```

I don't think we should reduce number of txs to 20. If time to run it is a concern, let's make it `125` at least.
created_at: 2025-03-26T11:29:09Z
author_association: MEMBER
diff_hunk: @@ -158,11 +158,13 @@ let apply_txs ~action_elements ~event_elements ~constraint_constants
   with
   | Ok (b, _, _, _, _) ->
       let root = Ledger.merkle_root ledger in
+      let time = Time.diff (Time.now ()) start in
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: ```suggestion
      let elapsed = Time.diff (Time.now ()) start in
```
created_at: 2025-03-26T11:29:53Z
author_association: MEMBER
diff_hunk: @@ -158,11 +158,13 @@ let apply_txs ~action_elements ~event_elements ~constraint_constants
   with
   | Ok (b, _, _, _, _) ->
       let root = Ledger.merkle_root ledger in
+      let time = Time.diff (Time.now ()) start in
       printf
         !"Result of application %d: %B (took %s): new root %s\n%!"
         i b
-        Time.(Span.to_string @@ diff (now ()) start)
-        (Ledger_hash.to_base58_check root)
+        Time.(Span.to_string time)
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: ```suggestion
        Time.(Span.to_string elapsed)
```
created_at: 2025-03-26T11:30:12Z
author_association: MEMBER
diff_hunk: @@ -229,10 +259,10 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
       |> Option.iter ~f:drop_old_ledger ;
       apply ~action_elements:0 ~event_elements:0 ~num_txs:num_txs_per_round ~i
         ledger
-      >>| mask_handler ledger
+      >>| save_preparation_times >>| mask_handler ledger
       >>| Fn.flip Tuple2.create (ledger :: ledgers) )
   >>| fst
   >>= apply ~num_txs:num_txs_final
         ~action_elements:genesis_constants.max_action_elements
         ~event_elements:genesis_constants.max_event_elements ~i:rounds
-  >>| stop_tracing
+  >>| save_and_dump_benchmarks >>| stop_tracing
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: ```suggestion
  >>| stop_tracing >>| save_and_dump_benchmarks
```
created_at: 2025-03-26T11:31:10Z
author_association: MEMBER
diff_hunk: @@ -218,7 +221,34 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := !results @ [ time ]
+  in
+  let save_and_dump_benchmarks final_time =
+    let calculate_mean results =
+      let preparation_steps = List.drop_last_exn results in
+      let sum = Float.of_int (List.length preparation_steps) in
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: This is not a sum, this is number of preparation steps
created_at: 2025-03-26T11:34:06Z
author_association: MEMBER
diff_hunk: @@ -218,7 +221,34 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := !results @ [ time ]
+  in
+  let save_and_dump_benchmarks final_time =
+    let calculate_mean results =
+      let preparation_steps = List.drop_last_exn results in
+      let sum = Float.of_int (List.length preparation_steps) in
+      printf !"sum: %f\n" sum ;
+      List.fold preparation_steps ~init:Float.zero ~f:(fun acc time ->
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: I suggest that we just add times up in the `List.fold` and divide by number of preparation steps at the very end. This is slightly more intuitive for taking an average
created_at: 2025-03-26T11:35:27Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,15 @@
+let LedgerApply = ../../Command/Bench/LedgerApply.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in  Pipeline.build
+      ( LedgerApply.pipeline
+          LedgerApply.Spec::{
+          , name = "LedgerApplyUnstable"
+          , label = "Ledger Apply"
+          , key = "ledger-apply"
+          , mode = PipelineMode.Type.Stable
path: buildkite/src/Jobs/Bench/LedgerApplyUnstable.dhall
user: login: dkijania
body: Ups. Yes, this is a mistake
created_at: 2025-03-27T07:06:00Z
author_association: MEMBER
diff_hunk: @@ -54,5 +55,5 @@ mkdir $TEMP_FOLDER/genesis/ledger
 tar -zxf  $TEMP_FOLDER/genesis/genesis_ledger_*.tar.gz -C $TEMP_FOLDER/genesis/ledger
 
 echo "running test:"
-time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 200
+time $MINA_APP ledger test apply --ledger-path $TEMP_FOLDER/genesis/ledger  --privkey-path $SENDER --num-txs 20 --dump-benchmark $BENCHMARK_FILE
path: scripts/tests/ledger_test_apply.sh
user: login: dkijania
body: Yes, another good catch! I decreased for testing purposes
created_at: 2025-03-27T07:09:16Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,67 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let Profiles = ../../Constants/Profiles.dhall
+
+let BuildFlags = ../../Constants/BuildFlags.dhall
+
+let SelectFiles = ../../Lib/SelectFiles.dhall
+
+let Spec =
+      { Type =
+          { key : Text, name : Text, label : Text, mode : PipelineMode.Type }
+      , default.mode = PipelineMode.Type.PullRequest
+      }
+
+let dependsOn =
+        DebianVersions.dependsOnStep
+          (None Text)
+          DebianVersions.DebVersion.Bullseye
+          Profiles.Type.Standard
+          BuildFlags.Type.Instrumented
+          "build"
+      # DebianVersions.dependsOnStep
+          (None Text)
+          DebianVersions.DebVersion.Bullseye
+          Profiles.Type.Standard
+          BuildFlags.Type.None
path: buildkite/src/Command/Bench/LedgerApply.dhall
user: login: georgeee
body: Why it depends on both instrumented and regular debian?
created_at: 2025-03-27T09:38:10Z
author_association: MEMBER
diff_hunk: @@ -218,7 +221,36 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := !results @ [ time ]
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: ```suggestion
    if Option.is_some benchmark then results := time :: !results
```
created_at: 2025-03-27T09:39:25Z
author_association: MEMBER
diff_hunk: @@ -218,7 +221,36 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := !results @ [ time ]
+  in
+  let save_and_dump_benchmarks final_time =
+    let calculate_mean results =
+      let preparation_steps = List.drop_last_exn results in
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: I do not think we should be dropping anything here.

How I understand the code, final call will not go through `save_preparation_times`, hence no need to drop anything.
created_at: 2025-03-27T09:42:36Z
author_association: MEMBER
diff_hunk: @@ -218,7 +221,36 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := !results @ [ time ]
path: src/app/cli/src/init/test_ledger_application.ml
user: login: georgeee
body: We don't care about order of operations, `::` is slightly "cleaner"
created_at: 2025-03-27T09:42:57Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,67 @@
+let BenchBase = ../../Command/Bench/Base.dhall
+
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let Profiles = ../../Constants/Profiles.dhall
+
+let BuildFlags = ../../Constants/BuildFlags.dhall
+
+let SelectFiles = ../../Lib/SelectFiles.dhall
+
+let Spec =
+      { Type =
+          { key : Text, name : Text, label : Text, mode : PipelineMode.Type }
+      , default.mode = PipelineMode.Type.PullRequest
+      }
+
+let dependsOn =
+        DebianVersions.dependsOnStep
+          (None Text)
+          DebianVersions.DebVersion.Bullseye
+          Profiles.Type.Standard
+          BuildFlags.Type.Instrumented
+          "build"
+      # DebianVersions.dependsOnStep
+          (None Text)
+          DebianVersions.DebVersion.Bullseye
+          Profiles.Type.Standard
+          BuildFlags.Type.None
path: buildkite/src/Command/Bench/LedgerApply.dhall
user: login: dkijania
body: Oh right! I forgot that runtime_genesis_ledger is packed in instrumented debian too
created_at: 2025-03-27T13:31:16Z
author_association: MEMBER
diff_hunk: @@ -218,7 +221,36 @@ let test ~privkey_path ~ledger_path ?prev_block_path ~first_partition_slots
   let stop_tracing =
     if tracing then (fun x -> Mina_tracing.stop () ; x) else ident
   in
+  let results = ref [] in
   let init_root = Ledger.merkle_root init_ledger in
+  let save_preparation_times time =
+    if Option.is_some benchmark then results := !results @ [ time ]
+  in
+  let save_and_dump_benchmarks final_time =
+    let calculate_mean results =
+      let preparation_steps = List.drop_last_exn results in
path: src/app/cli/src/init/test_ledger_application.ml
user: login: dkijania
body: Absolutely!
created_at: 2025-03-27T13:32:18Z
author_association: MEMBER

PR comments:
user: login: dkijania
created_at: 2025-03-24T19:49:19Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-24T21:17:30Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-25T08:10:39Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-25T12:39:21Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-25T14:26:57Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-25T19:28:19Z
body: !ci-nightly-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-27T07:45:09Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-27T07:59:44Z
body: !ci-build-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-27T08:36:37Z
body: !ci-nightly-me
author_association: MEMBER
user: login: dkijania
created_at: 2025-03-27T13:36:07Z
body: !ci-build-me
author_association: MEMBER
user: login: glyh
created_at: 2025-03-31T07:54:28Z
body: Thanks Dariusz, I'll look into the code here and try to implement benchmark for de-serialization later :) 
author_association: MEMBER