Repository: intel/code-base-investigator

Issue:
symlink files are not handled correctly
### Expected behavior

Source files that are symlinks to other source files should not be counted as duplicates, as per #72, #79 and #116. 

### Actual behavior

Code Base Investigator correctly "sees through" _directories_ that are symlinks to other directories, but double-counts source files that are directly symlinked to other source files.

### Steps to reproduce the problem

```python
    def test_symlink_files(self):
        """Check that symlink files do not count towards divergence."""
        tmp = tempfile.TemporaryDirectory()
        p = Path(tmp.name)
        with open(p / "base.cpp", mode="w") as f:
            f.write("void foo();")
        os.symlink(p / "base.cpp", p / "symlink.cpp")

        codebase = CodeBase(p)
        configuration = {
            "test": [
                {
                    "file": str(p / "base.cpp"),
                    "defines": [],
                    "include_paths": [],
                    "include_files": [],
                },
                {
                    "file": str(p / "symlink.cpp"),
                    "defines": [],
                    "include_paths": [],
                    "include_files": [],
                },
            ],
        }

        expected_setmap = {frozenset(["test"]): 1}

        state = finder.find(self.rootdir, codebase, configuration)
        setmap = state.get_setmap(codebase)
        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")

        tmp.cleanup()
```

This test fails:
```
======================================================================
FAIL: test_symlink_files (tests.duplicates.test_duplicates.TestDuplicates.test_symlink_files)
Check that symlink files do not count towards divergence.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "tests/duplicates/test_duplicates.py", line 118, in test_symlink_files
    self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
AssertionError: defaultdict(<class 'int'>, {frozenset({'test'}): 2}) != {frozenset({'test'}): 1}
- defaultdict(<class 'int'>, {frozenset({'test'}): 2})
+ {frozenset({'test'}): 1} : Mismatch in setmap

----------------------------------------------------------------------
```

### Specifications

This affects 1.2.0 and the tip of main.

Hints from issue comments:

Solution patch:
diff --git a/codebasin/finder.py b/codebasin/finder.py
index cde8474..5c376bc 100644
--- a/codebasin/finder.py
+++ b/codebasin/finder.py
@@ -98,6 +98,12 @@ def get_setmap(self, codebase: CodeBase) -> dict[frozenset, int]:
         """
         setmap = collections.defaultdict(int)
         for fn in codebase:
+            # Don't count symlinks if their target is in the code base.
+            # The target will be counted separately.
+            path = Path(fn)
+            if path.is_symlink() and path.resolve() in codebase:
+                continue
+
             tree = self.get_tree(fn)
             association = self.get_map(fn)
             for node in [n for n in tree.walk() if isinstance(n, CodeNode)]:
@@ -117,7 +123,7 @@ def associator(node: Node) -> Visit:
             association[node].add(platform.name)
             active = node.evaluate_for_platform(
                 platform=platform,
-                filename=filename,
+                filename=self._get_realpath(filename),
                 state=self,
             )
 
diff --git a/codebasin/report.py b/codebasin/report.py
index 34a480a..aa24fec 100644
--- a/codebasin/report.py
+++ b/codebasin/report.py
@@ -77,7 +77,7 @@ def divergence(setmap):
         npairs += 1
 
     if npairs == 0:
-        return 0
+        return float("nan")
     return d / float(npairs)
 
 
diff --git a/docs/source/specialization.rst b/docs/source/specialization.rst
index d124f03..a1d5102 100644
--- a/docs/source/specialization.rst
+++ b/docs/source/specialization.rst
@@ -57,7 +57,7 @@ pairwise distances:
 
 .. math::
     CD(a, p, H) = \binom{|H|}{2}^{-1}
-                  \sum_{\{i, j\} \in H \times H} {d_{i, j}(a, p)}
+                  \sum_{\{i, j\} \in \binom{H}{2}} {d_{i, j}(a, p)}
 
 where :math:`d_{i, j}(a, p)` represents the distance between the source
 code required by platforms :math:`i` and :math:`j` for application


Test patch:
diff --git a/tests/duplicates/test_duplicates.py b/tests/duplicates/test_duplicates.py
index 222efca..c224a46 100644
--- a/tests/duplicates/test_duplicates.py
+++ b/tests/duplicates/test_duplicates.py
@@ -2,6 +2,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import logging
+import os
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -50,8 +52,8 @@ def test_duplicates(self):
         setmap = state.get_setmap(codebase)
         self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
 
-    def test_symlinks(self):
-        """Check that symlinks do not count towards divergence."""
+    def test_symlink_directories(self):
+        """Check that symlink directories do not count towards divergence."""
 
         cpufile = str(self.rootdir / "cpu/foo.cpp")
         cpu2file = str(self.rootdir / "cpu2/foo.cpp")
@@ -83,6 +85,40 @@ def test_symlinks(self):
         setmap = state.get_setmap(codebase)
         self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
 
+    def test_symlink_files(self):
+        """Check that symlink files do not count towards divergence."""
+        tmp = tempfile.TemporaryDirectory()
+        p = Path(tmp.name)
+        with open(p / "base.cpp", mode="w") as f:
+            f.write("void foo();")
+        os.symlink(p / "base.cpp", p / "symlink.cpp")
+
+        codebase = CodeBase(p)
+        configuration = {
+            "test": [
+                {
+                    "file": str(p / "base.cpp"),
+                    "defines": [],
+                    "include_paths": [],
+                    "include_files": [],
+                },
+                {
+                    "file": str(p / "symlink.cpp"),
+                    "defines": [],
+                    "include_paths": [],
+                    "include_files": [],
+                },
+            ],
+        }
+
+        expected_setmap = {frozenset(["test"]): 1}
+
+        state = finder.find(self.rootdir, codebase, configuration)
+        setmap = state.get_setmap(codebase)
+        self.assertDictEqual(setmap, expected_setmap, "Mismatch in setmap")
+
+        tmp.cleanup()
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/include/test_include.py b/tests/include/test_include.py
index 27e298e..05f7606 100644
--- a/tests/include/test_include.py
+++ b/tests/include/test_include.py
@@ -2,6 +2,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import logging
+import os
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -43,6 +45,30 @@ def test_include(self):
             "Mismatch in setmap",
         )
 
+    def test_include_from_symlink(self):
+        """Check included file correctly identifies its parent"""
+        tmp = tempfile.TemporaryDirectory()
+        p = Path(tmp.name)
+        with open(p / "test.cpp", mode="w") as f:
+            f.write('#include "test.h"')
+        open(p / "test.h", mode="w").close()
+        os.symlink(p / "test.cpp", p / "symlink.cpp")
+
+        codebase = CodeBase(p)
+        configuration = {
+            "test": [
+                {
+                    "file": str(p / "symlink.cpp"),
+                    "defines": [],
+                    "include_paths": [],
+                    "include_files": [],
+                },
+            ],
+        }
+        _ = finder.find(self.rootdir, codebase, configuration)
+
+        tmp.cleanup()
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/metrics/test_divergence.py b/tests/metrics/test_divergence.py
new file mode 100644
index 0000000..64eebb9
--- /dev/null
+++ b/tests/metrics/test_divergence.py
@@ -0,0 +1,49 @@
+# Copyright (C) 2019-2024 Intel Corporation
+# SPDX-License-Identifier: BSD-3-Clause
+
+import logging
+import math
+import unittest
+import warnings
+
+from codebasin.report import divergence
+
+
+class TestDivergence(unittest.TestCase):
+    """
+    Test computation of code divergence.
+    """
+
+    def setUp(self):
+        logging.disable()
+        warnings.simplefilter("ignore", ResourceWarning)
+
+    def test_divergence(self):
+        """Check divergence computation for simple setmap."""
+        setmap = {
+            frozenset(["A"]): 1,
+            frozenset(["B"]): 2,
+            frozenset(["A", "B"]): 3,
+            frozenset([]): 4,
+        }
+        intersection = 3
+        union = 1 + 2 + 3
+
+        expected_divergence = intersection / union
+        self.assertEqual(divergence(setmap), expected_divergence)
+
+    def test_null_divergence(self):
+        """Check divergence computation for null cases."""
+        setmap = {
+            frozenset(""): 0,
+        }
+        self.assertTrue(math.isnan(divergence(setmap)))
+
+        setmap = {
+            frozenset("A"): 1,
+        }
+        self.assertTrue(math.isnan(divergence(setmap)))
+
+
+if __name__ == "__main__":
+    unittest.main()