Repository: cocotb/cocotb

Issue:
Add support for MSVC
#1306 adds support for using setuptools, which knows how to invoke MSVC. However, initial experimentation revealed some follow-up work:

* Compiler flags need fixing (they are currently `-Werror` instead of `/W4` for instance)
* `.lib` files seem missing (https://travis-ci.org/themperek/cocotb/builds/646876123)
  * Can't use `dlltool`, need `lib /def:modelsim.def /out:modelsim.lib /machine:x86` etc

Opening this issue to track future work.


Discussion:
MSVC does not support C99 which is called out as our minimum supported C version. I don't know how trivial it would be to knock C support back to the C89 standard. Honestly, there does not need to be any C files in this project. Then C standards are not an issue at all...
Numpy is able to build under MSVC and uses (most of) C99 - so I don't expect an issue there.

Resolving patch:
diff --git a/cocotb_build_libs.py b/cocotb_build_libs.py
--- a/cocotb_build_libs.py
+++ b/cocotb_build_libs.py
@@ -14,12 +14,24 @@
 from distutils.spawn import find_executable
 from setuptools.command.build_ext import build_ext as _build_ext
 from distutils.file_util import copy_file
+from distutils.ccompiler import get_default_compiler
 from typing import List
 
 
 logger = logging.getLogger(__name__)
 cocotb_share_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "cocotb", "share"))
 
+_base_warns = ["-Wall", "-Wextra", "-Wcast-qual", "-Wwrite-strings", "-Wconversion"]
+_ccx_warns = _base_warns + ["-Wnon-virtual-dtor", "-Woverloaded-virtual"]
+_extra_cxx_compile_args = ["-std=c++11", "-fvisibility=hidden", "-fvisibility-inlines-hidden"] + _ccx_warns
+if os.name != "nt":
+    _extra_cxx_compile_args += ["-flto"]
+
+_extra_cxx_compile_args_msvc = ["/permissive-"]
+
+# Make PRI* format macros available with C++11 compiler but older libc, e.g. on RHEL6.
+_extra_defines = [("__STDC_FORMAT_MACROS", "")]
+
 
 def create_sxs_assembly_manifest(name: str, filename: str, libraries: List[str], dependency_only=False) -> str:
     """
@@ -94,7 +106,7 @@ def create_sxs_appconfig(filename):
         f.write(config_body)
 
 
-def create_rc_file(name, filename, libraries):
+def create_rc_file(rc_filename, name, filename, libraries, runtime_libraries):
     """
     Creates windows resource definition script to embed the side-by-side assembly manifest into the libraries.
 
@@ -119,9 +131,8 @@ def create_rc_file(name, filename, libraries):
         END
         ''') % manifest
 
-    # Add the runtime dependency on libcocotb to libembed dependencies
-    if name == "libembed":
-        manifest = create_sxs_assembly_manifest(name, filename, ["libcocotb"], dependency_only=True)
+    if runtime_libraries is not None:
+        manifest = create_sxs_assembly_manifest(name, filename, runtime_libraries, dependency_only=True)
 
         # Escape double quotes and put every line between double quotes for embedding into rc file
         manifest = manifest.replace('"', '""')
@@ -134,7 +145,7 @@ def create_rc_file(name, filename, libraries):
             END
             ''') % manifest
 
-    with open(name + ".rc", "w", encoding='utf-8') as f:
+    with open(rc_filename, "w", encoding='utf-8') as f:
         f.write(rc_body)
 
 
@@ -150,13 +161,33 @@ def _get_lib_ext_name():
 
 
 class build_ext(_build_ext):
+    def _uses_msvc(self):
+        if self.compiler == "msvc":
+            return True
+        if self.compiler is None:
+            return get_default_compiler() == "msvc"
+        else:
+            return "msvc" == getattr(self.compiler, "compiler_type", None)
+
     def run(self):
+        if os.name == "nt":
+            create_sxs_appconfig(self.get_ext_fullpath(os.path.join("cocotb", "simulator")))
 
-        def_dir = os.path.join(cocotb_share_dir, "def")
-        self._gen_import_libs(def_dir)
+        super().run()
 
+    def build_extensions(self):
         if os.name == "nt":
-            create_sxs_appconfig(self.get_ext_fullpath(os.path.join("cocotb", "simulator")))
+            if self._uses_msvc():
+                # Initialize the compiler now so that compiler/linker flags are populated
+                if not self.compiler.initialized:
+                    self.compiler.initialize()
+
+                # Setuptools defaults to activate automatic manifest generation for msvc,
+                # disable it here as we manually generate it to also support mingw on windows
+                for (k, ldflags) in self.compiler._ldflags.items():
+                    self.compiler._ldflags[k] = [x for x in ldflags if not x.startswith("/MANIFEST")] + ["/MANIFEST:NO"]
+
+                self.compiler.compile_options = [x for x in self.compiler.compile_options if not x.startswith("/W")] + ["/W4"]
 
             ext_names = {os.path.split(ext.name)[-1] for ext in self.extensions}
             for ext in self.extensions:
@@ -165,15 +196,73 @@ def run(self):
                 name = os.path.split(fullname)[-1]
                 filename = os.path.split(filename)[-1]
                 libraries = {"lib" + lib for lib in ext.libraries}.intersection(ext_names)
-                create_rc_file(name, filename, libraries)
+                rc_filename = name + ".rc"
+                runtime_libraries = None
 
-        super().run()
+                # Add the runtime dependency on libcocotb to libembed
+                if name == "libembed":
+                    runtime_libraries = ["libcocotb"]
+
+                # Strip lib prefix for msvc
+                if self._uses_msvc():
+                    name = name[3:] if name.startswith("lib") else name
+                    libraries = {(lib[3:] if lib.startswith("lib") else lib) for lib in libraries}
+                    if runtime_libraries is not None:
+                        runtime_libraries = {(lib[3:] if lib.startswith("lib") else lib) for lib in runtime_libraries}
+                create_rc_file(rc_filename, name, filename, libraries, runtime_libraries)
+
+            def_dir = os.path.join(cocotb_share_dir, "def")
+            self._gen_import_libs(def_dir)
+
+            for e in self.extensions:
+                e.library_dirs += [def_dir]
+
+        super().build_extensions()
 
     def build_extension(self, ext):
         """Build each extension in its own temp directory to make gcov happy.
 
         A normal PEP 517 install still works as the temp directories are discarded anyway.
         """
+        lib_name = os.path.split(ext.name)[-1]
+
+        if self._uses_msvc():
+            ext.extra_compile_args += _extra_cxx_compile_args_msvc
+        else:
+            ext.extra_compile_args += _extra_cxx_compile_args
+
+            if os.name == "nt":
+                # Align behavior of gcc with msvc and export only symbols marked with __declspec(dllexport)
+                ext.extra_link_args += ["-Wl,--exclude-all-symbols"]
+            else:
+                ext.extra_link_args += ["-flto"]
+
+                rpaths = []
+                if lib_name == "simulator":
+                    rpaths += ["$ORIGIN/libs"]
+                    install_name = None
+                else:
+                    rpaths += ["$ORIGIN"]
+                    install_name = lib_name
+
+                if sys.platform == "darwin":
+                    rpaths = [rpath.replace("$ORIGIN", "@loader_path") for rpath in rpaths]
+                    if install_name is not None:
+                        ext.extra_link_args += ["-Wl,-install_name,@rpath/%s.so" % install_name]
+
+                ext.extra_link_args += ["-Wl,-rpath,%s" % rpath for rpath in rpaths]
+
+        # vpi_user.h and vhpi_user.h require that WIN32 is defined
+        if os.name == "nt":
+            ext.define_macros += [("WIN32", "")]
+
+        if lib_name == "libembed":
+            if self._uses_msvc():
+                embed_lib_name = "cocotb"
+            else:
+                embed_lib_name = "libcocotb"
+            ext.define_macros += [("EMBED_IMPL_LIB", embed_lib_name + "." + _get_lib_ext_name())]
+
         old_build_temp = self.build_temp
         self.build_temp = os.path.join(self.build_temp, ext.name)
         super().build_extension(ext)
@@ -203,11 +292,16 @@ def get_ext_filename(self, ext_name):
         # mingw on msys2 uses `-` as seperator
         tail_split = tail_split[0].split("-")
 
+        # strip lib prefix if msvc is used
+        if self._uses_msvc() and tail_split[0].startswith("lib"):
+            tail_split[0] = tail_split[0][3:]
+
         filename_short = os.path.join(head, tail_split[0] + "." + _get_lib_ext_name())
 
         # icarus requires vpl extension
         filename_short = filename_short.replace("libcocotbvpi_icarus.so", "libcocotbvpi_icarus.vpl")
         filename_short = filename_short.replace("libcocotbvpi_icarus.dll", "libcocotbvpi_icarus.vpl")
+        filename_short = filename_short.replace("cocotbvpi_icarus.dll", "cocotbvpi_icarus.vpl")
 
         return filename_short
 
@@ -247,8 +341,17 @@ def _gen_import_libs(self, def_dir):
         load the DLL (.a) based on module definition files (.def)
         """
 
-        if os.name == "nt":
-            for sim in ["icarus", "modelsim", "aldec", "ghdl"]:
+        for sim in ["icarus", "modelsim", "aldec", "ghdl"]:
+            if self._uses_msvc():
+                subprocess.run(
+                    [
+                        self.compiler.lib,
+                        "/def:" + os.path.join(def_dir, sim + ".def"),
+                        "/out:" + os.path.join(def_dir, sim + ".lib"),
+                        "/machine:" + ("X64" if sys.maxsize > 2**32 else "X86")
+                    ]
+                )
+            else:
                 subprocess.run(
                     [
                         "dlltool",
@@ -260,28 +363,6 @@ def _gen_import_libs(self, def_dir):
                 )
 
 
-def _extra_link_args(lib_name=None, rpaths=[]):
-    """
-    Add linker argument to load dependencies from the directory where vpi/vhpi/fli library is located
-    On osx use `install_name`.
-    Use `rpath` on all platforms
-    """
-
-    args = []
-    if sys.platform == "darwin" and lib_name is not None:
-        args += ["-Wl,-install_name,@rpath/%s.so" % lib_name]
-    for rpath in rpaths:
-        if sys.platform == "darwin":
-            rpath = rpath.replace("$ORIGIN", "@loader_path")
-        args += ["-Wl,-rpath,%s" % rpath]
-    if os.name == "nt":
-        # Align behavior of gcc with msvc and export only symbols marked with __declspec(dllexport)
-        args += ["-Wl,--exclude-all-symbols"]
-    else:
-        args += ["-flto"]
-    return args
-
-
 def _get_python_lib_link():
     """ Get name of python library used for linking """
 
@@ -316,17 +397,6 @@ def _get_python_lib():
     return python_lib
 
 
-# TODO [gh-1372]: make this work for MSVC which has a different flag syntax
-_base_warns = ["-Wall", "-Wextra", "-Wcast-qual", "-Wwrite-strings", "-Wconversion"]
-_ccx_warns = _base_warns + ["-Wnon-virtual-dtor", "-Woverloaded-virtual"]
-_extra_cxx_compile_args = ["-std=c++11", "-fvisibility=hidden", "-fvisibility-inlines-hidden"] + _ccx_warns
-if os.name != "nt":
-    _extra_cxx_compile_args += ["-flto"]
-
-# Make PRI* format macros available with C++11 compiler but older libc, e.g. on RHEL6.
-_extra_defines = [("__STDC_FORMAT_MACROS", "")]
-
-
 def _get_common_lib_ext(include_dir, share_lib_dir):
     """
     Defines common libraries.
@@ -352,8 +422,6 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         include_dirs=[include_dir],
         libraries=libcocotbutils_libraries,
         sources=libcocotbutils_sources,
-        extra_link_args=_extra_link_args(lib_name="libcocotbutils", rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     #
@@ -373,8 +441,6 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         define_macros=[("GPILOG_EXPORTS", "")] + _extra_defines,
         include_dirs=[include_dir],
         sources=libgpilog_sources,
-        extra_link_args=_extra_link_args(lib_name="libgpilog", rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     #
@@ -391,8 +457,6 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         include_dirs=[include_dir],
         libraries=["gpilog"],
         sources=libpygpilog_sources,
-        extra_link_args=_extra_link_args(lib_name="libpygpilog", rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     #
@@ -407,13 +471,10 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         os.path.join("cocotb", "libs", "libembed"),
         define_macros=[
             ("COCOTB_EMBED_EXPORTS", ""),
-            ("EMBED_IMPL_LIB", "libcocotb." + _get_lib_ext_name()),
             ("PYTHON_LIB", _get_python_lib())] + _extra_defines,
         include_dirs=[include_dir],
         libraries=["gpilog", "cocotbutils"],
         sources=libembed_sources,
-        extra_link_args=_extra_link_args(lib_name="libembed", rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     #
@@ -430,8 +491,6 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         include_dirs=[include_dir],
         libraries=["gpilog", "cocotbutils", "pygpilog"],
         sources=libcocotb_sources,
-        extra_link_args=_extra_link_args(lib_name="libcocotb", rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     #
@@ -449,8 +508,6 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         include_dirs=[include_dir],
         libraries=["cocotbutils", "gpilog", "embed"],
         sources=libgpi_sources,
-        extra_link_args=_extra_link_args(lib_name="libgpi", rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     #
@@ -468,8 +525,6 @@ def _get_common_lib_ext(include_dir, share_lib_dir):
         libraries=["cocotbutils", "gpilog", "gpi", "pygpilog"],
         library_dirs=python_lib_dirs,
         sources=simulator_sources,
-        extra_compile_args=_extra_cxx_compile_args,
-        extra_link_args=_extra_link_args(rpaths=["$ORIGIN/libs"]),
     )
 
     # The libraries in this list are compiled in order of their appearance.
@@ -495,8 +550,6 @@ def _get_vpi_lib_ext(
         libraries=["gpi", "gpilog"] + extra_lib,
         library_dirs=extra_lib_dir,
         sources=libcocotbvpi_sources,
-        extra_link_args=_extra_link_args(lib_name=lib_name, rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     return libcocotbvpi
@@ -519,8 +572,6 @@ def _get_vhpi_lib_ext(
         libraries=["gpi", "gpilog"] + extra_lib,
         library_dirs=extra_lib_dir,
         sources=libcocotbvhpi_sources,
-        extra_link_args=_extra_link_args(lib_name=lib_name, rpaths=["$ORIGIN"]),
-        extra_compile_args=_extra_cxx_compile_args,
     )
 
     return libcocotbvhpi
@@ -535,7 +586,6 @@ def get_ext():
 
     share_lib_dir = os.path.relpath(os.path.join(cocotb_share_dir, "lib"))
     include_dir = os.path.relpath(os.path.join(cocotb_share_dir, "include"))
-    share_def_dir = os.path.relpath(os.path.join(cocotb_share_dir, "def"))
 
     ext = []
 
@@ -547,18 +597,15 @@ def get_ext():
     #  Icarus Verilog
     #
     icarus_extra_lib = []
-    icarus_extra_lib_path = []
     logger.info("Compiling libraries for Icarus Verilog")
     if os.name == "nt":
         icarus_extra_lib = ["icarus"]
-        icarus_extra_lib_path = [share_def_dir]
 
     icarus_vpi_ext = _get_vpi_lib_ext(
         include_dir=include_dir,
         share_lib_dir=share_lib_dir,
         sim_define="ICARUS",
         extra_lib=icarus_extra_lib,
-        extra_lib_dir=icarus_extra_lib_path,
     )
     ext.append(icarus_vpi_ext)
 
@@ -566,18 +613,15 @@ def get_ext():
     #  Modelsim/Questa
     #
     modelsim_extra_lib = []
-    modelsim_extra_lib_path = []
     logger.info("Compiling libraries for Modelsim/Questa")
     if os.name == "nt":
         modelsim_extra_lib = ["modelsim"]
-        modelsim_extra_lib_path = [share_def_dir]
 
     modelsim_vpi_ext = _get_vpi_lib_ext(
         include_dir=include_dir,
         share_lib_dir=share_lib_dir,
         sim_define="MODELSIM",
         extra_lib=modelsim_extra_lib,
-        extra_lib_dir=modelsim_extra_lib_path,
     )
     ext.append(modelsim_vpi_ext)
 
@@ -604,10 +648,7 @@ def get_ext():
                 define_macros=[("COCOTBFLI_EXPORTS", "")] + _extra_defines,
                 include_dirs=[include_dir, modelsim_include_dir],
                 libraries=["gpi", "gpilog"] + modelsim_extra_lib,
-                library_dirs=modelsim_extra_lib_path,
                 sources=fli_sources,
-                extra_link_args=_extra_link_args(lib_name=lib_name, rpaths=["$ORIGIN"]),
-                extra_compile_args=_extra_cxx_compile_args,
             )
 
             ext.append(fli_ext)
@@ -624,18 +665,15 @@ def get_ext():
     # GHDL
     #
     ghdl_extra_lib = []
-    ghdl_extra_lib_path = []
     logger.info("Compiling libraries for GHDL")
     if os.name == "nt":
         ghdl_extra_lib = ["ghdl"]
-        ghdl_extra_lib_path = [share_def_dir]
 
     ghdl_vpi_ext = _get_vpi_lib_ext(
         include_dir=include_dir,
         share_lib_dir=share_lib_dir,
         sim_define="GHDL",
         extra_lib=ghdl_extra_lib,
-        extra_lib_dir=ghdl_extra_lib_path,
     )
     ext.append(ghdl_vpi_ext)
 
@@ -668,18 +706,15 @@ def get_ext():
     # Aldec Riviera Pro
     #
     aldec_extra_lib = []
-    aldec_extra_lib_path = []
     logger.info("Compiling libraries for Riviera")
     if os.name == "nt":
         aldec_extra_lib = ["aldec"]
-        aldec_extra_lib_path = [share_def_dir]
 
     aldec_vpi_ext = _get_vpi_lib_ext(
         include_dir=include_dir,
         share_lib_dir=share_lib_dir,
         sim_define="ALDEC",
         extra_lib=aldec_extra_lib,
-        extra_lib_dir=aldec_extra_lib_path,
     )
     ext.append(aldec_vpi_ext)
 
@@ -688,7 +723,6 @@ def get_ext():
         share_lib_dir=share_lib_dir,
         sim_define="ALDEC",
         extra_lib=aldec_extra_lib,
-        extra_lib_dir=aldec_extra_lib_path,
     )
     ext.append(aldec_vhpi_ext)
 


Test patch:
diff --git a/tests/sxs.ps1 b/tests/sxs.ps1
new file mode 100644
--- /dev/null
+++ b/tests/sxs.ps1
@@ -0,0 +1,8 @@
+$j = Start-Job -ScriptBlock { SxsTrace Trace -logfile:SxsTrace.etl }
+Start-Sleep -s 5
+python -c "import cocotb.simulator"
+Start-Sleep -s 5
+$j | Stop-Job
+SxsTrace Stoptrace
+SxsTrace Parse -logfile:SxsTrace.etl -outfile:SxsTrace.txt
+Get-Content SxsTrace.txt