Repository: cocotb/cocotb

Issue:
libgpilog.so: undefined symbol: PyGILState_Release
Greetings.
Current master (currently at 5a53f6b) is broken for me (Arch Linux w/ Python 3.8.5).

E.g., running `test_vhdl_access` returns:
```
[...]

/usr/bin/ghdl -r  --workdir=sim_build --work=dec_viterbi dec_viterbi_ent --vpi=/home/nicola/.local/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so  
loading VPI module '/home/nicola/.local/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so'
/home/nicola/.local/lib/python3.8/site-packages/cocotb/libs/libgpilog.so: undefined symbol: PyGILState_Release
/usr/bin/ghdl:error: cannot load VPI module

[...]
```
Same for iverilog. git bisect tells me that the responsible commit is 606f2485b082c917195f20b7e5b23ad8dcacd82a by @tpambor (indeed, reverting 606f2485b082c917195f20b7e5b23ad8dcacd82a and ef181522e26c4da6d52ed9443fcbf775155098d5 fixes the issue).


Discussion:
Could you provide the output of `ldd /home/nicola/.local/lib/python3.8/site-packages/cocotb/libs/libgpilog.so`?
```
linux-vdso.so.1 (0x00007fff51ddd000)
libc.so.6 => /usr/lib/libc.so.6 (0x00007fcb9d1c3000)
/usr/lib64/ld-linux-x86-64.so.2 (0x00007fcb9d3d6000)
```
Damn, I should have caught this. Yes, Python 3.8 no longer links Extensions against libpython. libgpilog depends upon libpython. libgpilog is loaded before libcocotb which runtime loads libpython.

You can see my output for libgpilog on my Python 3.8 installation:
```
> ldd libgpilog.so 
	linux-vdso.so.1 (0x00007ffc5777d000)
	libstdc++.so.6 => /home/kaleb/miniconda3/lib/libstdc++.so.6 (0x00007fa370081000)
	libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fa36ff29000)
	libgcc_s.so.1 => /home/kaleb/miniconda3/lib/libgcc_s.so.1 (0x00007fa36ff15000)
	libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007fa36fef3000)
	libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fa36fd2e000)
	/lib64/ld-linux-x86-64.so.2 (0x00007fa3701fc000)
```

No libpython listed....

The alternative fix is #1993.

EDIT: nvm this, the Python logger isn't registered until well after libcocotb runtime loads libpython. This shouldn't be an issue. And it works on my system (Debian Unstable) using either the system Python 3.8 installation or a conda 3.8 environment.
Somehow it works for more on Debian with Python 3.8.5 and also for the CI
Library loading looks for me like this:
```
MODULE= TESTCASE= TOPLEVEL=dec_viterbi_ent TOPLEVEL_LANG=vhdl strace /usr/bin/ghdl -r  --workdir=sim_build --work=dec_viterbi dec_viterbi_ent --vpi=/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so 2>&1 | grep -E '^open(at)?(.*)\.so' | grep -v ENOENT
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgnarl-9.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgnat-9.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libgpi.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libgpilog.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotbutils.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotb.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpython3.8.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libexpat.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libutil.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/python3.8/lib-dynload/_ctypes.cpython-38-x86_64-linux-gnu.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libffi.so.7", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/simulator.cpython-38-x86_64-linux-gnu.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/python3.8/lib-dynload/_opcode.cpython-38-x86_64-linux-gnu.so", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/python3.8/lib-dynload/mmap.cpython-38-x86_64-linux-gnu.so", O_RDONLY|O_CLOEXEC) = 3
```

```
$ ldd libgpilog.so
	linux-vdso.so.1 (0x00007ffd4c50c000)
	libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f0ba0c5b000)
	/lib64/ld-linux-x86-64.so.2 (0x00007f0ba0e3f000)
```
If I don't have `libpython.so` laying around I get:
```
MODULE= TESTCASE= TOPLEVEL=dec_viterbi_ent TOPLEVEL_LANG=vhdl /usr/bin/ghdl -r  --workdir=sim_build --work=dec_viterbi dec_viterbi_ent --vpi=/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so
loading VPI module '/home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so'
     -.--ns ERROR    cocotb.gpi                         ..s/cocotb_utils.cpp:69   in utils_dyn_open                  Unable to open lib libpython3.8.so: libpython3.8.so: cannot open shared object file: No such file or directory

     -.--ns ERROR    cocotb.gpi                         ..mbed/gpi_embed.cpp:127  in embed_init_python               Failed to find Python shared library
/usr/bin/ghdl-mcode: symbol lookup error: /home/tim/projects/cocotb/venv/lib/python3.8/site-packages/cocotb/libs/libcocotb.so: undefined symbol: Py_DecodeLocale
```
So I'm wondering if it gets loaded but wrong version or something.

@corna Could you run `MODULE= TESTCASE= TOPLEVEL=dec_viterbi_ent TOPLEVEL_LANG=vhdl strace /usr/bin/ghdl -r  --workdir=sim_build --work=dec_viterbi dec_viterbi_ent --vpi=/home/nicola/.local/lib/python3.8/site-packages/cocotb/libs/libcocotbvpi_ghdl.so | grep -E '^open(at)?(.*)\.so'`
and `nm -gD /usr/lib/libpython3.8.so | grep PyGILState_Release`?

Do you use system python or a local version?

@tpambor That what I expected if it couldn't load load libython, it still prints a few logs, including an error message, before the Python logger is registered. In Corna's case it seems like it's failing on the first logging call. Any ideas on why that would happen?
We dlopen only `libpython.so`. I think it's some kind of clash between a system-wide and a local python installation. Maybe we should load the absolute path. I will prepare a patch.
> Maybe we should load the absolute path. I will prepare a patch.

Yes, this is a good idea.
System python.

```
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
```
```
00000000001ecdf0 T PyGILState_Release
```
Archlinux:
```
readelf -r venv/lib/python3.8/site-packages/cocotb/libs/libgpilog.so

Relocation section '.rela.dyn' at offset 0x788 contains 30 entries:
  Offset          Info           Type           Sym. Value    Sym. Name + Addend
000000003d78  000000000008 R_X86_64_RELATIVE                    10e0
000000003d80  000000000008 R_X86_64_RELATIVE                    1090
000000004000  000000000008 R_X86_64_RELATIVE                    4000
000000004028  000000000008 R_X86_64_RELATIVE                    2053
000000004038  000000000008 R_X86_64_RELATIVE                    2059
000000004048  000000000008 R_X86_64_RELATIVE                    205e
000000004058  000000000008 R_X86_64_RELATIVE                    2066
000000004068  000000000008 R_X86_64_RELATIVE                    206c
000000003f50  000100000006 R_X86_64_GLOB_DAT 0000000000000000 _ITM_deregisterTMClone + 0
000000003f58  000200000006 R_X86_64_GLOB_DAT 0000000000000000 stdout@GLIBC_2.2.5 + 0
000000003f60  000300000006 R_X86_64_GLOB_DAT 0000000000000000 PyGILState_Release + 0
000000003f68  000400000006 R_X86_64_GLOB_DAT 0000000000000000 vsnprintf@GLIBC_2.2.5 + 0
000000003f70  000500000006 R_X86_64_GLOB_DAT 0000000000000000 _Py_Dealloc + 0
000000003f78  000600000006 R_X86_64_GLOB_DAT 0000000000000000 strlen@GLIBC_2.2.5 + 0
000000003f80  000700000006 R_X86_64_GLOB_DAT 0000000000000000 __stack_chk_fail@GLIBC_2.4 + 0
000000003f88  000800000006 R_X86_64_GLOB_DAT 0000000000000000 PyGILState_Ensure + 0
000000003f90  000900000006 R_X86_64_GLOB_DAT 0000000000000000 fputs@GLIBC_2.2.5 + 0
000000003f98  000a00000006 R_X86_64_GLOB_DAT 0000000000000000 PyErr_Print + 0
000000003fa0  000b00000006 R_X86_64_GLOB_DAT 0000000000000000 fputc@GLIBC_2.2.5 + 0
000000003fa8  000c00000006 R_X86_64_GLOB_DAT 0000000000000000 PyLong_FromLong + 0
000000003fb0  000d00000006 R_X86_64_GLOB_DAT 0000000000000000 fprintf@GLIBC_2.2.5 + 0
000000003fb8  000e00000006 R_X86_64_GLOB_DAT 0000000000000000 __gmon_start__ + 0
000000003fc0  000f00000006 R_X86_64_GLOB_DAT 0000000000000000 PyObject_CallFunctionO + 0
000000003fc8  001000000006 R_X86_64_GLOB_DAT 0000000000000000 fflush@GLIBC_2.2.5 + 0
000000003fd0  001100000006 R_X86_64_GLOB_DAT 0000000000000000 PyObject_IsTrue + 0
000000003fd8  001200000006 R_X86_64_GLOB_DAT 0000000000000000 PyUnicode_FromString + 0
000000003fe0  001300000006 R_X86_64_GLOB_DAT 0000000000000000 fwrite@GLIBC_2.2.5 + 0
000000003fe8  001400000006 R_X86_64_GLOB_DAT 0000000000000000 _ITM_registerTMCloneTa + 0
000000003ff0  001500000006 R_X86_64_GLOB_DAT 0000000000000000 __cxa_finalize@GLIBC_2.2.5 + 0
000000003ff8  001600000006 R_X86_64_GLOB_DAT 0000000000000000 stderr@GLIBC_2.2.5 + 0
```

Debian:
```
eadelf -r venv/lib/python3.8/site-packages/cocotb/libs/libgpilog.so 

Relocation section '.rela.dyn' at offset 0x750 contains 14 entries:
  Offset          Info           Type           Sym. Value    Sym. Name + Addend
000000003df0  000000000008 R_X86_64_RELATIVE                    11c0
000000003df8  000000000008 R_X86_64_RELATIVE                    1180
000000004080  000000000008 R_X86_64_RELATIVE                    4080
0000000040a8  000000000008 R_X86_64_RELATIVE                    2058
0000000040b8  000000000008 R_X86_64_RELATIVE                    205e
0000000040c8  000000000008 R_X86_64_RELATIVE                    2063
0000000040d8  000000000008 R_X86_64_RELATIVE                    206b
0000000040e8  000000000008 R_X86_64_RELATIVE                    2071
000000003fd0  000100000006 R_X86_64_GLOB_DAT 0000000000000000 _ITM_deregisterTM[...] + 0
000000003fd8  000200000006 R_X86_64_GLOB_DAT 0000000000000000 stdout@GLIBC_2.2.5 + 0
000000003fe0  000a00000006 R_X86_64_GLOB_DAT 0000000000000000 __gmon_start__ + 0
000000003fe8  001100000006 R_X86_64_GLOB_DAT 0000000000000000 _ITM_registerTMCl[...] + 0
000000003ff0  001200000006 R_X86_64_GLOB_DAT 0000000000000000 __cxa_finalize@GLIBC_2.2.5 + 0
000000003ff8  001300000006 R_X86_64_GLOB_DAT 0000000000000000 stderr@GLIBC_2.2.5 + 0

Relocation section '.rela.plt' at offset 0x8a0 contains 13 entries:
  Offset          Info           Type           Sym. Value    Sym. Name + Addend
000000004018  000300000007 R_X86_64_JUMP_SLO 0000000000000000 PyGILState_Release + 0
000000004020  000400000007 R_X86_64_JUMP_SLO 0000000000000000 _Py_Dealloc + 0
000000004028  000500000007 R_X86_64_JUMP_SLO 0000000000000000 strlen@GLIBC_2.2.5 + 0
000000004030  000600000007 R_X86_64_JUMP_SLO 0000000000000000 __stack_chk_fail@GLIBC_2.4 + 0
000000004038  000700000007 R_X86_64_JUMP_SLO 0000000000000000 PyGILState_Ensure + 0
000000004040  000800000007 R_X86_64_JUMP_SLO 0000000000000000 PyErr_Print + 0
000000004048  000900000007 R_X86_64_JUMP_SLO 0000000000000000 PyLong_FromLong + 0
000000004050  000b00000007 R_X86_64_JUMP_SLO 0000000000000000 PyObject_CallFunc[...] + 0
000000004058  000c00000007 R_X86_64_JUMP_SLO 0000000000000000 fflush@GLIBC_2.2.5 + 0
000000004060  000d00000007 R_X86_64_JUMP_SLO 0000000000000000 PyObject_IsTrue + 0
000000004068  000e00000007 R_X86_64_JUMP_SLO 0000000000000000 __vsnprintf_chk@GLIBC_2.3.4 + 0
000000004070  000f00000007 R_X86_64_JUMP_SLO 0000000000000000 PyUnicode_FromString + 0
000000004078  001000000007 R_X86_64_JUMP_SLO 0000000000000000 __fprintf_chk@GLIBC_2.3.4 + 0
```

On Archlinux non-PLT relocations (`R_X86_64_GLOB_DAT`) are used for python symbols which cannot be lazy bound whereas on Debian PLT relocations (`R_X86_64_JUMP_SLO`) are used.

Archlinux passes: `-Wl,-z,now`:
> Marks the object to override the runtime linker's default mode and require non-lazy runtime binding. This is similar to adding the object to the process by using dlopen(3DL) with the RTLD_NOW mode, or setting the LD_BIND_NOW environment variable in effect. 
This is an issue with the GCC on Arch Linux always passing that flag, or a customized distutils in their system Python installation?

Resolving patch:
diff --git a/cocotb_build_libs.py b/cocotb_build_libs.py
--- a/cocotb_build_libs.py
+++ b/cocotb_build_libs.py
@@ -270,17 +270,9 @@ def _get_python_lib():
     """ Get the library for embedded the python interpreter """
 
     if os.name == "nt":
-        python_lib = _get_python_lib_link() + ".dll"
+        python_lib = _get_python_lib_link() + "." + _get_lib_ext_name()
     else:
-        python_lib = "lib" + _get_python_lib_link() + "."
-        if sys.platform == "darwin":
-            python_lib = os.path.join(sysconfig.get_config_var("LIBDIR"), python_lib)
-            if os.path.exists(python_lib + "dylib"):
-                python_lib += "dylib"
-            else:
-                python_lib += "so"
-        else:
-            python_lib += "so"
+        python_lib = "lib" + _get_python_lib_link() + "." + _get_lib_ext_name()
 
     return python_lib
 
@@ -325,6 +317,10 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
     #
     #  libgpilog
     #
+    python_lib_dirs = []
+    if sys.platform == "darwin":
+        python_lib_dirs = [sysconfig.get_config_var("LIBDIR")]
+
     libgpilog_sources = [
         os.path.join(share_lib_dir, "gpi_log", "gpi_logging.cpp")
     ]
@@ -334,6 +330,7 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         os.path.join("cocotb", "libs", "libgpilog"),
         define_macros=[("GPILOG_EXPORTS", "")]  + _extra_defines,
         include_dirs=[include_dir],
+        library_dirs=python_lib_dirs,
         sources=libgpilog_sources,
         extra_link_args=_extra_link_args(lib_name="libgpilog", rpaths=["$ORIGIN"]),
         extra_compile_args=_extra_cxx_compile_args,
@@ -351,9 +348,10 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         os.path.join("cocotb", "libs", "libcocotb"),
         define_macros=[("COCOTB_EMBED_EXPORTS", ""), ("PYTHON_SO_LIB", _get_python_lib())] + _extra_defines,
         include_dirs=[include_dir],
-        libraries=["gpilog", "cocotbutils"],
+        libraries=[_get_python_lib_link(), "gpilog", "cocotbutils"],
+        library_dirs=python_lib_dirs,
         sources=libcocotb_sources,
-        extra_link_args=_extra_link_args(lib_name="libcocotb", rpaths=["$ORIGIN"]),
+        extra_link_args=_extra_link_args(lib_name="libcocotb", rpaths=["$ORIGIN"] + python_lib_dirs),
         extra_compile_args=_extra_cxx_compile_args,
     )
 
@@ -389,6 +387,7 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         define_macros=_extra_defines,
         include_dirs=[include_dir],
         libraries=["cocotbutils", "gpilog", "gpi"],
+        library_dirs=python_lib_dirs,
         sources=simulator_sources,
         extra_compile_args=_extra_cxx_compile_args,
         extra_link_args=_extra_link_args(rpaths=["$ORIGIN/libs"]),


Test patch: