Repository: nim-works/nimskull
Pull request #1504: fix memory corruption with indirect `.tailcall` calls  
Author: login: zerbina
Labels: name: bug
description: Something isn't working
name: compiler/backend
description: Related to backend system of the compiler
Created: 2025-03-31T18:43:36Z | Merged: 2025-04-03T00:29:39Z

## Summary

Fix a C/JS code generator bug leading to some `.tailcall` routines
receiving garbage values as their parameters, or failing to compile,
when called indirectly.

## Details

Both `cgen` and `jsgen` use the original `PType` to decide how
parameter passing works. For `.tailcall` proc types representing
procedure pointers, the proc type doesn't match the actual signature
(which is always `proc(env: pointer): Continuation[T]`), leading to the
environment pointer being passed incorrectly (e.g., by address).

This is worked around by having both `cgen` and `jsgen` manually emit
the argument setup for indirect `.tailcall` calls.


Diff:
diff --git a/compiler/backend/ccgcalls.nim b/compiler/backend/ccgcalls.nim
index 5d9076e7f4d..7acad2725ba 100644
--- a/compiler/backend/ccgcalls.nim
+++ b/compiler/backend/ccgcalls.nim
@@ -238,6 +238,16 @@ proc genClosureCall(p: BProc, le, ri: CgNode, d: var TLoc) =
 proc genAsgnCall(p: BProc, le, ri: CgNode, d: var TLoc) =
   if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
+  elif ri[0].typ.skipTypes(abstractInst).callConv == ccTailcall and
+       ri[0].kind != cnkProc:
+    # an indirect tailcall call. The real signature doesn't match the `tyProc`,
+    # so we manually emit the call
+    # XXX: this is fundamentally a hack see https://github.com/nim-works/nimskull/pull/1504
+    assert numArgs(ri) == 1
+    var callee, arg: TLoc
+    initLocExpr(p, ri[0], callee)
+    initLocExpr(p, ri[1], arg)
+    fixupCall(p, le, ri, d, rdLoc(callee), rdLoc(arg))
   else:
     genPrefixCall(p, le, ri, d)
 
diff --git a/compiler/backend/jsgen.nim b/compiler/backend/jsgen.nim
index 6a4b86fe36c..9091fd3c05c 100644
--- a/compiler/backend/jsgen.nim
+++ b/compiler/backend/jsgen.nim
@@ -1195,10 +1195,10 @@ proc genArgNoParam(p: PProc, n: CgNode, r: var TCompRes) =
   else:
     r.res.add(a.res)
 
-proc genArg(p: PProc, n: CgNode, param: PSym, r: var TCompRes; emitted: ptr int = nil) =
+proc genArg(p: PProc, n: CgNode, param: PType, r: var TCompRes; emitted: ptr int = nil) =
   var a: TCompRes
   gen(p, n, a)
-  if skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs} and
+  if skipTypes(param, abstractVar).kind in {tyOpenArray, tyVarargs} and
       a.typ == etyBaseIndex:
     r.res.add("$1[$2]" % [a.address, a.res])
   elif a.typ == etyBaseIndex:
@@ -1230,7 +1230,7 @@ proc genArgs(p: PProc, n: CgNode, r: var TCompRes; start=1) =
     if paramType.isNil:
       genArgNoParam(p, it, r)
     else:
-      genArg(p, it, paramType.sym, r, addr emitted)
+      genArg(p, it, paramType.sym.typ, r, addr emitted)
     inc emitted
     hasArgs = true
   r.res.add(")")
@@ -1253,7 +1253,7 @@ proc genOtherArg(p: PProc; n: CgNode; i: int; typ: PType;
   if paramType.isNil:
     genArgNoParam(p, it, r)
   else:
-    genArg(p, it, paramType.sym, r)
+    genArg(p, it, paramType.sym.typ, r)
   inc generated
 
 proc genPatternCall(p: PProc; n: CgNode; pat: string; typ: PType;
@@ -1317,7 +1317,16 @@ proc genInfixCall(p: PProc, n: CgNode, r: var TCompRes) =
 
 proc genCall(p: PProc, n: CgNode, r: var TCompRes) =
   gen(p, n[0], r)
-  genArgs(p, n, r)
+  if n[0].kind != cnkProc and
+     n[0].typ.skipTypes(abstractInst).callConv == ccTailcall:
+    # an indirect tailcall call. The signature doesn't match reality, so we
+    # manually handle the argument
+    # XXX: this is fundamentally a hack see https://github.com/nim-works/nimskull/pull/1504
+    r.res.add "("
+    genArg(p, n[1], p.module.graph.getSysType(n[1].info, tyPointer), r, nil)
+    r.res.add ")"
+  else:
+    genArgs(p, n, r)
   if n.typ != nil:
     let t = mapType(n.typ)
     if t == etyBaseIndex:
diff --git a/tests/lang_callable/proc/tindirect_tailcall_bug.nim b/tests/lang_callable/proc/tindirect_tailcall_bug.nim
new file mode 100644
index 00000000000..3749b370824
--- /dev/null
+++ b/tests/lang_callable/proc/tindirect_tailcall_bug.nim
@@ -0,0 +1,28 @@
+discard """
+  targets: "c js vm"
+"""
+
+block:
+  # parameters had garbage values when the first parameter of the indirectly-
+  # called tailcall routine is a pass-by-reference parameter. This only
+  # affected the C backend
+  type Large = object
+    ## something that's passed by reference
+    pad: array[256, int]
+
+  proc test(x: Large, y: int) {.tailcall.} =
+    doAssert y == 128
+
+  var p = test
+  p(Large(), 128)
+
+block:
+  # parameters had garbage values when the first parameter of the indirectly-
+  # called tailcall routine is a an openArray. This affected both the C and JS
+  # backend
+  proc test(x: openArray[int], y: int) {.tailcall.} =
+    doAssert x[1] == 128
+    doAssert y == 256
+
+  var p = test
+  p([1, 128, 2], 256)


Reviews:
user: login: saem
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T07:16:19Z
user: login: saem
body: Two minor suggestions so a future person knows where to look for fix ideas
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-02T07:18:07Z

Review comments:
diff_hunk: @@ -1317,7 +1317,16 @@ proc genInfixCall(p: PProc, n: CgNode, r: var TCompRes) =
 
 proc genCall(p: PProc, n: CgNode, r: var TCompRes) =
   gen(p, n[0], r)
-  genArgs(p, n, r)
+  if n[0].kind != cnkProc and
+     n[0].typ.skipTypes(abstractInst).callConv == ccTailcall:
+    # an indirect tailcall call. The signature doesn't match reality, so we
+    # manually handle the argument
+    # XXX: this is fundamentally a hack
path: compiler/backend/jsgen.nim
user: login: saem
body: ```suggestion
    # XXX: this is fundamentally a hack see https://github.com/nim-works/nimskull/pull/1504
```
created_at: 2025-04-02T07:16:19Z
author_association: COLLABORATOR
diff_hunk: @@ -238,6 +238,15 @@ proc genClosureCall(p: BProc, le, ri: CgNode, d: var TLoc) =
 proc genAsgnCall(p: BProc, le, ri: CgNode, d: var TLoc) =
   if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
+  elif ri[0].typ.skipTypes(abstractInst).callConv == ccTailcall and
+       ri[0].kind != cnkProc:
+    # an indirect tailcall call. The real signature doesn't match the `tyProc`,
+    # so we manually emit the call
path: compiler/backend/ccgcalls.nim
user: login: saem
body: ```suggestion
    # so we manually emit the call
    # XXX: this is fundamentally a hack see https://github.com/nim-works/nimskull/pull/1504
```
created_at: 2025-04-02T07:16:42Z
author_association: COLLABORATOR

PR comments:
user: login: zerbina
created_at: 2025-03-31T18:48:18Z
body: The proper fix would be to:
* distinguish between signature and procedure pointer types at the MIR level
* only lower `.tailcall` procedure pointer types (not signature types)
* use MIR types instead of `PType` for the argument handling in the code generators

After all, being able to selectively lower or modify types (without the changes being visible to sem) was what the MIR type IR was intended for, among other things.
author_association: COLLABORATOR
user: login: zerbina
created_at: 2025-04-02T23:17:29Z
body: /merge
author_association: COLLABORATOR
user: login: github-actions[bot]
created_at: 2025-04-02T23:18:48Z
body: <!-- /merge info comment -->
<!-- Autogenerated by merge-command. DO NOT EDIT -->
Merge requested by: @zerbina

Contents after the first section break of the PR description has been removed and preserved below:

---

<blockquote>

</blockquote>
author_association: NONE