Repository: CycloneDX/cdxgen (js)

Issue:
Python requirements file without pinned version crashes cdxgen
Created an empty project `abcd`
```
ubuntu@MACHINE:~$ tree abcd
abcd
└── requirements.txt
```

Requirements contain multiple unpinned dependencies
```
ubuntu@ip-172-24-11-180:~$ cat abcd/requirements.txt
ipython
apache-airflow-providers-pagerduty
matplotlib
seaborn
retrying
dataframe-image
lxml
openpyxl
xlrd
apache-airflow-providers-apache-livy
apache-airflow-providers-http
markupsafe
gspread
oauth2client
duckdb
facebook-business
facebookads
apache-airflow-providers-slack
apache-airflow-providers-ssh
pyzipper
apache-airflow-providers-mysql
jira
cron-descriptor
```

Then run `cdxgen`
```
ubuntu@MACHINE:~$ ~/path/to/dir/node_modules/.bin/cdxgen abcd
 Traceback (most recent call last):
  File "/tmp/cdxgen-piptree-y9GpJ5/piptree.py", line 87, in <module>
    main(sys.argv)
  File "/tmp/cdxgen-piptree-y9GpJ5/piptree.py", line 72, in main
    "dependencies": find_deps(idx, p.requires()),
  File "/tmp/cdxgen-piptree-y9GpJ5/piptree.py", line 48, in find_deps
    "dependencies": find_deps(idx, d.requires()),
  File "/tmp/cdxgen-piptree-y9GpJ5/piptree.py", line 48, in find_deps
    "dependencies": find_deps(idx, d.requires()),
  File "/tmp/cdxgen-piptree-y9GpJ5/piptree.py", line 48, in find_deps
    "dependencies": find_deps(idx, d.requires()),
  [Previous line repeated 981 more times]
  File "/tmp/cdxgen-piptree-y9GpJ5/piptree.py", line 46, in find_deps
    "version": importlib_metadata.version(r.key),
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 984, in version
    return distribution(distribution_name).version
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 957, in distribution
    return Distribution.from_name(distribution_name)
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 543, in from_name
    dists = resolver(DistributionFinder.Context(name=name))
  File "<frozen importlib._bootstrap_external>", line 1482, in find_distributions
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 896, in find_distributions
    found = cls._search_paths(context.name, context.path)
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 902, in _search_paths
    prepared = Prepared(name)
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 863, in __init__
    self.normalized = self.normalize(name)
  File "/usr/lib/python3.10/importlib/metadata/__init__.py", line 871, in normalize
    return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_')
  File "/usr/lib/python3.10/re.py", line 209, in sub
    return _compile(pattern, flags).sub(repl, string, count)
  File "/usr/lib/python3.10/re.py", line 290, in _compile
    if isinstance(flags, RegexFlag):
RecursionError: maximum recursion depth exceeded while calling a Python object

file:///path/to/dir/node_modules/@cyclonedx/cdxgen/utils.js:5559
    for (const t of tree) {
                    ^

TypeError: tree is not iterable
    at getPipFrozenTree (file:///path/to/dir/node_modules/@cyclonedx/cdxgen/utils.js:5559:21)
    at createPythonBom (file:///path/to/dir/node_modules/@cyclonedx/cdxgen/index.js:2125:28)
    at createMultiXBom (file:///path/to/dir/node_modules/@cyclonedx/cdxgen/index.js:3840:21)
    at async createBom (file:///path/to/dir/node_modules/@cyclonedx/cdxgen/index.js:4695:16)
    at async file:///path/to/dir/node_modules/@cyclonedx/cdxgen/bin/cdxgen.js:229:22

Node.js v18.16.1
```

It's a wild guess but it appears unpinned dependencies are causing a problem here.

Pull request description:
Makes pnpm work without breaking docker and repotests. 1 Breaking change
- [ ] BREAKING: @types packages are no longer ignored for pnpm projects. With deep validation, it is not possible to rebuild all dependency trees by ignoring specific packages easily
- [ ] Due to some legacy baggage, `purl` cannot be set in `parsePkgJson`
- [ ] Fixes #411 


@ajmalab @cerrussell  could you kindly test and review this PR?
@Hritik14 Could you test #411 with this?

Patch:
diff --git a/contrib/piptree.py b/contrib/piptree.py
index 96865b06..e98682ed 100644
--- a/contrib/piptree.py
+++ b/contrib/piptree.py
@@ -32,19 +32,23 @@ def get_installed_distributions():
     return [d._dist for d in dists]
 
 
-def find_deps(idx, reqs):
+def find_deps(idx, visited, reqs):
     freqs = []
     for r in reqs:
         d = idx.get(r.key)
         r.project_name = d.project_name if d is not None else r.project_name
+        if visited.get(r.project_name):
+            return freqs
         specs = sorted(r.specs, reverse=True)
         specs_str = ",".join(["".join(sp) for sp in specs]) if specs else ""
+        visited[r.project_name] = True
+        dreqs = d.requires()
         freqs.append(
             {
                 "name": r.project_name,
                 "version": importlib_metadata.version(r.key),
                 "versionSpecifiers": specs_str,
-                "dependencies": find_deps(idx, d.requires()),
+                "dependencies": find_deps(idx, visited, dreqs) if dreqs else [],
             }
         )
     return freqs
@@ -55,6 +59,7 @@ def main(argv):
     tree = []
     pkgs = get_installed_distributions()
     idx = {p.key: p for p in pkgs}
+    visited = {}
     for p in pkgs:
         fr = frozen_req_from_dist(p)
         tmpA = fr.split("==")
@@ -68,7 +73,7 @@ def main(argv):
             {
                 "name": name,
                 "version": version,
-                "dependencies": find_deps(idx, p.requires()),
+                "dependencies": find_deps(idx, visited, p.requires()),
             }
         )
     all_deps = {}
diff --git a/index.js b/index.js
index b4e4f7f8..c820cb0b 100644
--- a/index.js
+++ b/index.js
@@ -1775,6 +1775,16 @@ export const createNodejsBom = async (path, options) => {
         if (pcs.length) {
           parentComponent = pcs[0];
           parentComponent.type = "application";
+          ppurl = new PackageURL(
+            "npm",
+            parentComponent.group,
+            parentComponent.name,
+            parentComponent.version,
+            null,
+            null
+          ).toString();
+          parentComponent["bom-ref"] = decodeURIComponent(ppurl);
+          parentComponent["purl"] = ppurl;
         }
       } else {
         let dirName = dirname(f);
@@ -1906,6 +1916,16 @@ export const createNodejsBom = async (path, options) => {
         if (pcs.length) {
           const tmpParentComponent = pcs[0];
           tmpParentComponent.type = "application";
+          ppurl = new PackageURL(
+            "npm",
+            tmpParentComponent.group,
+            tmpParentComponent.name,
+            tmpParentComponent.version,
+            null,
+            null
+          ).toString();
+          tmpParentComponent["bom-ref"] = decodeURIComponent(ppurl);
+          tmpParentComponent["purl"] = ppurl;
           if (!Object.keys(parentComponent).length) {
             parentComponent = tmpParentComponent;
           } else {
diff --git a/piptree.js b/piptree.js
index b48923eb..0936bfff 100644
--- a/piptree.js
+++ b/piptree.js
@@ -50,19 +50,23 @@ def get_installed_distributions():
     return [d._dist for d in dists]
 
 
-def find_deps(idx, reqs):
+def find_deps(idx, visited, reqs):
     freqs = []
     for r in reqs:
         d = idx.get(r.key)
         r.project_name = d.project_name if d is not None else r.project_name
+        if visited.get(r.project_name):
+            return freqs
         specs = sorted(r.specs, reverse=True)
         specs_str = ",".join(["".join(sp) for sp in specs]) if specs else ""
+        visited[r.project_name] = True
+        dreqs = d.requires()
         freqs.append(
             {
                 "name": r.project_name,
                 "version": importlib_metadata.version(r.key),
                 "versionSpecifiers": specs_str,
-                "dependencies": find_deps(idx, d.requires()),
+                "dependencies": find_deps(idx, visited, dreqs) if dreqs else [],
             }
         )
     return freqs
@@ -73,6 +77,7 @@ def main(argv):
     tree = []
     pkgs = get_installed_distributions()
     idx = {p.key: p for p in pkgs}
+    visited = {}
     for p in pkgs:
         fr = frozen_req_from_dist(p)
         tmpA = fr.split("==")
@@ -86,7 +91,7 @@ def main(argv):
             {
                 "name": name,
                 "version": version,
-                "dependencies": find_deps(idx, p.requires()),
+                "dependencies": find_deps(idx, visited, p.requires()),
             }
         )
     all_deps = {}
@@ -108,7 +113,7 @@ if __name__ == "__main__":
  * Execute the piptree plugin and return the generated tree as json object
  */
 export const getTreeWithPlugin = (env, python_cmd, basePath) => {
-  let tree = undefined;
+  let tree = [];
   const tempDir = mkdtempSync(join(tmpdir(), "cdxgen-piptree-"));
   const pipPlugin = join(tempDir, "piptree.py");
   const pipTreeJson = join(tempDir, "piptree.json");
diff --git a/utils.js b/utils.js
index dc29419b..757346a9 100644
--- a/utils.js
+++ b/utils.js
@@ -448,7 +448,7 @@ export const parsePkgJson = async (pkgJsonFile) => {
       const purl = new PackageURL(
         "npm",
         group,
-        encodeForPurl(name),
+        name,
         pkgData.version,
         null,
         null
@@ -457,7 +457,6 @@ export const parsePkgJson = async (pkgJsonFile) => {
         name,
         group,
         version: pkgData.version,
-        purl,
         "bom-ref": decodeURIComponent(purl),
         properties: [
           {
@@ -865,33 +864,31 @@ export const parseNodeShrinkwrap = async function (swFile) {
           }
           version = parts[2];
         }
-        if (group !== "@types") {
-          pkgList.push({
-            group: group,
-            name: name,
-            version: version,
-            _integrity: integrity,
-            properties: [
-              {
-                name: "SrcFile",
-                value: swFile
-              }
-            ],
-            evidence: {
-              identity: {
-                field: "purl",
-                confidence: 1,
-                methods: [
-                  {
-                    technique: "manifest-analysis",
-                    confidence: 1,
-                    value: swFile
-                  }
-                ]
-              }
+        pkgList.push({
+          group: group,
+          name: name,
+          version: version,
+          _integrity: integrity,
+          properties: [
+            {
+              name: "SrcFile",
+              value: swFile
             }
-          });
-        }
+          ],
+          evidence: {
+            identity: {
+              field: "purl",
+              confidence: 1,
+              methods: [
+                {
+                  technique: "manifest-analysis",
+                  confidence: 1,
+                  value: swFile
+                }
+              ]
+            }
+          }
+        });
       }
     }
   }
@@ -1007,7 +1004,7 @@ export const parsePnpmLock = async function (pnpmLock, parentComponent = null) {
           );
           continue;
         }
-        if (group !== "@types" && name.indexOf("file:") !== 0) {
+        if (name.indexOf("file:") !== 0) {
           const purlString = new PackageURL(
             "npm",
             group,


Test patch:
diff --git a/utils.test.js b/utils.test.js
index 436c81a8..df374943 100644
--- a/utils.test.js
+++ b/utils.test.js
@@ -1356,8 +1356,8 @@ test("parseSetupPyFile", async () => {
 
 test("parsePnpmLock", async () => {
   let parsedList = await parsePnpmLock("./test/pnpm-lock.yaml");
-  expect(parsedList.pkgList.length).toEqual(1610);
-  expect(parsedList.dependenciesList.length).toEqual(1610);
+  expect(parsedList.pkgList.length).toEqual(1706);
+  expect(parsedList.dependenciesList.length).toEqual(1706);
   expect(parsedList.pkgList[0]).toEqual({
     _integrity:
       "sha512-IGhtTmpjGbYzcEDOw7DcQtbQSXcG9ftmAXtWTu9V936vDye4xjjekktFAtgZsWpzTj/X01jocB46mTywm/4SZw==",
@@ -1386,8 +1386,8 @@ test("parsePnpmLock", async () => {
     }
   });
   parsedList = await parsePnpmLock("./test/data/pnpm-lock.yaml");
-  expect(parsedList.pkgList.length).toEqual(308);
-  expect(parsedList.dependenciesList.length).toEqual(308);
+  expect(parsedList.pkgList.length).toEqual(318);
+  expect(parsedList.dependenciesList.length).toEqual(318);
   expect(parsedList.pkgList[0]).toEqual({
     _integrity:
       "sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg==",
@@ -1450,8 +1450,8 @@ test("parsePnpmLock", async () => {
     ]
   });
   parsedList = await parsePnpmLock("./test/data/pnpm-lock3.yaml");
-  expect(parsedList.pkgList.length).toEqual(448);
-  expect(parsedList.dependenciesList.length).toEqual(448);
+  expect(parsedList.pkgList.length).toEqual(449);
+  expect(parsedList.dependenciesList.length).toEqual(449);
   expect(parsedList.pkgList[0]).toEqual({
     group: "@nodelib",
     name: "fs.scandir",
@@ -1483,8 +1483,8 @@ test("parsePnpmLock", async () => {
   expect(parsedList.pkgList.length).toEqual(1);
 
   parsedList = await parsePnpmLock("./test/data/pnpm-lock6.yaml");
-  expect(parsedList.pkgList.length).toEqual(195);
-  expect(parsedList.dependenciesList.length).toEqual(195);
+  expect(parsedList.pkgList.length).toEqual(200);
+  expect(parsedList.dependenciesList.length).toEqual(200);
   expect(parsedList.pkgList[0]).toEqual({
     group: "@babel",
     name: "code-frame",
@@ -1530,8 +1530,8 @@ test("parsePnpmLock", async () => {
     }
   });
   parsedList = await parsePnpmLock("./test/data/pnpm-lock6a.yaml");
-  expect(parsedList.pkgList.length).toEqual(229);
-  expect(parsedList.dependenciesList.length).toEqual(229);
+  expect(parsedList.pkgList.length).toEqual(234);
+  expect(parsedList.dependenciesList.length).toEqual(234);
   expect(parsedList.pkgList[0]).toEqual({
     group: "@babel",
     name: "code-frame",