Repository: PolyJIT/benchbuild (language: python)

Issue:
Project with multiple sources do not correctly get their source version mapped
In a project with multiple sources, the execution instantly terminates with the message "Could not find any unprocessed variants."  although there are multiple unprocessed variants.

According to @vulder, the issue seems to be related to the ordering of the values for the sources.

For example, for the following sources

```
SOURCE = [
    PaperConfigSpecificGit(
        project_name="clasp",
        remote="https://github.com/potassco/clasp.git",
        local="clasp",
        refspec="origin/HEAD",
        limit=None,
        shallow=False
    ),
    FeatureSource(),
    HTTP(
        local="SAT_H_instances_childsnack_p08.hddl_2.cnf",
        remote={
            "1.0":
                "https://github.com/bnico99/picoSAT-mirror/releases/"
                "download/picoSAT-965/SAT_H_instances_childsnack_p08.hddl_2.cnf"
        }
    )
]
```
the following list of projects is generated for the slurm script:

```
# List of available projects
projects=(
'clasp-cpp_projects@4c708a73fd,1.0,0'
'clasp-cpp_projects@4c708a73fd,1.0,1'
'clasp-cpp_projects@6382096f5a,1.0,0'
'clasp-cpp_projects@6382096f5a,1.0,1'
)
```




Discussion / hints:
The unprocessed variants does not have anything to do with BB itself that's on our side. However, the underlying problem is related to the wrong mapping of values to the corresponding sources for a project.

Fix patch:
diff --git a/benchbuild/source/base.py b/benchbuild/source/base.py
index 928dbde7d..20805bdd2 100644
--- a/benchbuild/source/base.py
+++ b/benchbuild/source/base.py
@@ -149,10 +149,24 @@ def source_by_name(self, name: str) -> 'FetchableSource':
 
     @property
     def primary(self) -> Variant:
-        return self.variants[0]
+        variants = self.sorted()
+        return variants[0]
+
+    def sorted(self) -> tp.Sequence[Variant]:
+        """
+        Return an ordered list of Variants from this revision.
+
+        The order is defined by the order in the SOURCE attribute of the
+        associated project class.
+        """
+        sources = self.project_cls.SOURCE
+        sources = [src for src in sources if self.has_variant(src.key)]
+
+        return [self.variant_by_name(src.key) for src in sources]
 
     def __str__(self) -> str:
-        return to_str(*self.variants)
+        variants = self.sorted()
+        return to_str(*variants)
 
     def __repr__(self) -> str:
         return str(self)


Test patch:
diff --git a/tests/source/conftest.py b/tests/source/conftest.py
index 84b91e071..83c1547a8 100644
--- a/tests/source/conftest.py
+++ b/tests/source/conftest.py
@@ -84,8 +84,7 @@ def versions_with_context(self, ctx: Revision) -> tp.Sequence[Variant]:
         if ctx.primary.version == "0":
             ret = [Variant(self, "v0.1"), Variant(self, "v0.2")]
             return ret
-        else:
-            return []
+        return []
 
 
 class Config1(CAWSource):
@@ -100,8 +99,7 @@ def versions_with_context(self, ctx: Revision) -> tp.Sequence[Variant]:
         if ctx.primary.version == "1":
             ret = [Variant(self, "v1.1"), Variant(self, "v1.2")]
             return ret
-        else:
-            return []
+        return []
 
 
 @pytest.fixture
diff --git a/tests/source/test_caw_source.py b/tests/source/test_caw_source.py
index 4a51da473..f804ac75f 100644
--- a/tests/source/test_caw_source.py
+++ b/tests/source/test_caw_source.py
@@ -1,11 +1,13 @@
 import typing as tp
+from unittest.mock import patch
 
-from benchbuild.project import Project
+from benchbuild.project import Project, ProjectRegistry, populate
 from benchbuild.source import (
     enumerate_revisions,
     Variant,
     SingleVersionFilter,
     FetchableSource,
+    Revision,
 )
 
 
@@ -68,3 +70,31 @@ def test_caw_filter(make_source, caw_src_0):
 
     test = [rev.variants for rev in revs]
     assert test == expected_variants
+
+
+def test_source_mapping(make_source, caw_src_0):
+    """
+    Test if we can map a list of revisions to the correct sources.
+    """
+    src_primary = make_source(["0", "1"])
+    src_secondary = make_source(["s1", "s2"])
+    TestProject.SOURCE = [src_primary, caw_src_0, src_secondary]
+
+    with patch.dict(
+        ProjectRegistry.projects, {"test/test": TestProject}, clear=True
+    ):
+        res: tp.Mapping[str, tp.Type[Project]] = populate(["test/test"])
+        assert res["test/test"] == TestProject
+
+        expected_revisions = [
+            str(
+                Revision(
+                    TestProject, Variant(src_primary, "0"),
+                    Variant(caw_src_0, "v0.1"), Variant(src_secondary, "s1")
+                )
+            )
+        ]
+
+        res = populate(["test/test@0,v0.1,s1"])
+        revisions = enumerate_revisions(res["test/test"])
+        assert [str(rev) for rev in revisions] == expected_revisions