Repository: PumasAI/QuartoNotebookRunner.jl (julia)

Issue:
Seeded RNGs only work reproducibly within a cell
## Description

Calling `Random.seed!` at the top of a notebook doesn't ensure reproducibility of random number generation in other cells; it only works within the same cell that `seed!` is called.

I'm not sure if this is intentional, but it's slightly counterintuitive behaviour so if it's intentional it might be good to document it somewhere.

A related issue (and perhaps an example of how this behaviour is confusing!) is that the [Pumas tutorial on reproducibility](https://tutorials.pumas.ai/html/simulation/simulating_reproducible.html#local-reproducibility-setting-the-seed-inside-a-function-call) says that the second batch of samples in `Random.seed!(123); rand(3); rand(3)` should always result in "the same behavior [...] every time we run the code". This is true if done in a REPL, but the Quarto source has the second `rand(3)` in a different cell and consequently the second `rand(3)` will not always return the same numbers.

## MWE

The following notebook works to demonstrate the behaviour. The first call to randn() should always give the same result no matter how many times the notebook is rendered. The second call however will not.

````markdown
---
engine: julia
---

```{julia}
versioninfo()
```

```{julia}
using Random
Random.seed!(42)
randn()
```

```{julia}
randn()
```
````

## My `versioninfo` from the notebook above

```
Julia Version 1.10.5
Commit 6f3fdf7b362 (2024-08-27 14:19 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: macOS (arm64-apple-darwin22.4.0)
  CPU: 10 × Apple M1 Pro
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, apple-m1)
Threads: 1 default, 0 interactive, 1 GC (on 8 virtual cores)
Environment:
  JULIA_LOAD_PATH = @:@stdlib
```

Pull request:
Fixes #192

Patch:
diff --git a/src/server.jl b/src/server.jl
index f881d75f..f1abab6c 100644
--- a/src/server.jl
+++ b/src/server.jl
@@ -92,6 +92,14 @@ end
 
 # Implementation.
 
+function remote_eval_fetch_channeled(worker, expr)
+    code = quote
+        put!(stable_execution_task_channel_in, $(QuoteNode(expr)))
+        take!(stable_execution_task_channel_out)
+    end
+    return Malt.remote_eval_fetch(worker, code)
+end
+
 function init!(file::File, options::Dict)
     worker = file.worker
     Malt.remote_eval_fetch(worker, worker_init(file, options))
@@ -106,7 +114,7 @@ function refresh!(file::File, options::Dict)
         init!(file, options)
     end
     expr = :(refresh!($(options)))
-    Malt.remote_eval_fetch(file.worker, expr)
+    remote_eval_fetch_channeled(file.worker, expr)
 end
 
 """
@@ -648,7 +656,7 @@ function evaluate_raw_cells!(
                     $(chunk.cell_options),
                 ))
 
-                worker_results, expand_cell = Malt.remote_eval_fetch(f.worker, expr)
+                worker_results, expand_cell = remote_eval_fetch_channeled(f.worker, expr)
 
                 # When the result of the cell evaluation is a cell expansion
                 # then we insert the original cell contents before the expanded
@@ -827,7 +835,7 @@ function evaluate_raw_cells!(
                             # inline evaluation since you can't pass cell
                             # options and so `expand` will always be `false`.
                             worker_results, expand_cell =
-                                Malt.remote_eval_fetch(f.worker, expr)
+                                remote_eval_fetch_channeled(f.worker, expr)
                             expand_cell && error("inline code cells cannot be expanded")
                             remote = only(worker_results)
                             if !isnothing(remote.error)
@@ -888,7 +896,7 @@ function evaluate_params!(f, params::Dict{String})
         :(@eval getfield(Main, :Notebook) const $(Symbol(key::String)) = $value)
     end
     expr = Expr(:block, exprs...)
-    Malt.remote_eval_fetch(f.worker, expr)
+    remote_eval_fetch_channeled(f.worker, expr)
     return
 end
 
diff --git a/src/worker.jl b/src/worker.jl
index 986b2c08..68dc44de 100644
--- a/src/worker.jl
+++ b/src/worker.jl
@@ -2,6 +2,19 @@ function worker_init(f::File, options::Dict)
     project = WorkerSetup.LOADER_ENV[]
     return lock(WorkerSetup.WORKER_SETUP_LOCK) do
         return quote
+            # issue #192
+            # Malt itself uses a new task for each `remote_eval` and because of this, random number streams
+            # are not consistent across runs even if seeded, as each task introduces a new state for its
+            # task-local RNG. As a workaround, we feed all `remote_eval` requests through these channels, such
+            # that the task executing code is always the same.
+            const stable_execution_task_channel_out = Channel()
+            const stable_execution_task_channel_in = Channel() do chan
+                for expr in chan
+                    result = Core.eval(Main, expr)
+                    put!(stable_execution_task_channel_out, result)
+                end
+            end
+
             push!(LOAD_PATH, $(project))
 
             let QNW = task_local_storage(:QUARTO_NOTEBOOK_WORKER_OPTIONS, $(options)) do


Test patch:
diff --git a/test/examples/random_seed/Project.toml b/test/examples/random_seed/Project.toml
new file mode 100644
index 00000000..576ba3cb
--- /dev/null
+++ b/test/examples/random_seed/Project.toml
@@ -0,0 +1,2 @@
+[deps]
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
diff --git a/test/examples/random_seed/random_seed.qmd b/test/examples/random_seed/random_seed.qmd
new file mode 100644
index 00000000..e97b76ce
--- /dev/null
+++ b/test/examples/random_seed/random_seed.qmd
@@ -0,0 +1,17 @@
+---
+title: Random seed
+---
+
+```{julia}
+using Random
+Random.seed!(123)
+rand()
+```
+
+```{julia}
+rand()
+```
+
+```{julia}
+rand()
+```
diff --git a/test/testsets/random_seed.jl b/test/testsets/random_seed.jl
new file mode 100644
index 00000000..01853a46
--- /dev/null
+++ b/test/testsets/random_seed.jl
@@ -0,0 +1,25 @@
+include("../utilities/prelude.jl")
+
+@testset "seeded random numbers are consistent across runs" begin
+    notebook = joinpath(@__DIR__, "../examples/random_seed/random_seed.qmd")
+
+    server = QuartoNotebookRunner.Server()
+
+    jsons = map(1:2) do _
+        QuartoNotebookRunner.run!(server, notebook; showprogress = false)
+    end
+
+    _output(cell) = only(cell.outputs).data["text/plain"]
+
+    @test tryparse(Float64, _output(jsons[1].cells[2])) !== nothing
+    @test tryparse(Float64, _output(jsons[1].cells[4])) !== nothing
+    @test tryparse(Float64, _output(jsons[1].cells[6])) !== nothing
+
+    @test length(unique([_output(jsons[1].cells[i]) for i in [2, 4, 6]])) == 3
+
+    @test _output(jsons[1].cells[2]) == _output(jsons[2].cells[2])
+    @test _output(jsons[1].cells[4]) == _output(jsons[2].cells[4])
+    @test _output(jsons[1].cells[6]) == _output(jsons[2].cells[6])
+
+    close!(server)
+end