Repository: FHIR/sushi (ts)

Issue:
SUSHI should allow wildcard directory paths in the path-resource parameter
It appears that the IG publisher allows using a directory wildcard `/*` in the `path-resource` parameter. However, SUSHI does not.

In SUSHI, you can list out the path to each subfolder to include, but trying to include multiple directories with a `/*` at the end does not work. For example, you need to do something like this:

```yaml
parameters:
  path-resource:
    - input/resources/group1
    - input/resources/group2
```

rather than something like

```yaml
parameters:
  path-resource:
    - input/resources/*
```

A small project that replicates the issue: [NestedFolders.zip](https://github.com/FHIR/sushi/files/13993751/NestedFolders.zip)

Zulip: https://chat.fhir.org/#narrow/stream/215610-shorthand/topic/sub-folder.20support

Pull request:
Allow path-resource paths with a wildcard
Fixes #1415 

This PR allows a user to specify a `*` wildcard in the `path-resource` parameter file path they specify. When a single `<path>/*` is used, SUSHI will allow any level of nested folders and load any file within those folders. I tested this with the IG Publisher, and specifying `<path>/*` allows you to add any resources at any nested level within that path. The IG Publisher does not support specifying `<path>` or `<path>/` or `<path>/**`, so this PR doesn't support those cases.

To test this, you can use the various options in this sample project: [NestedFoldersExample.zip](https://github.com/FHIR/sushi/files/14285686/NestedFoldersExample.zip)

I did change up my implementation a little bit because I was originally using something that was added in Node 18, but I figured since I found a way to accomplish the same thing pretty easily with Node 16 features, that was better than knowingly breaking SUSHI support for Node 16 (even though we don't explicitly support it).

Patch:
diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml
index a7dfac7..0ffeae8 100644
--- a/.github/workflows/ci-workflow.yml
+++ b/.github/workflows/ci-workflow.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        node-version: [16, 18]
+        node-version: [18, 20]
 
     steps:
     - uses: actions/checkout@v1
diff --git a/src/ig/IGExporter.ts b/src/ig/IGExporter.ts
index c25bf41..a2fe001 100644
--- a/src/ig/IGExporter.ts
+++ b/src/ig/IGExporter.ts
@@ -942,8 +942,8 @@ export class IGExporter {
    * capabilities, extensions, models, operations, profiles, resources, vocabulary, examples
    * Based on: https://build.fhir.org/ig/FHIR/ig-guidance/using-templates.html#root.input
    *
-   * NOTE: This does not include files nested in subfolders in supported paths since the
-   * IG Exporter does not handle those well.
+   * NOTE: This only includes files nested in subfolders when specified in the path-resource
+   * parameter, which is based on how the IG Publisher works.
    *
    * This function has similar operation to addResources, and both should be
    * analyzed when making changes to either.
@@ -963,14 +963,23 @@ export class IGExporter {
     const predefinedResourcePaths = pathEnds.map(pathEnd =>
       path.join(this.inputPath, 'input', pathEnd)
     );
-    let pathResourceDirectories: string[];
+    const pathResourceDirectories: string[] = [];
     const pathResources = this.config.parameters
       ?.filter(parameter => parameter.value && parameter.code === 'path-resource')
       .map(parameter => parameter.value);
     if (pathResources) {
-      pathResourceDirectories = pathResources
-        .map(directoryPath => path.join(this.inputPath, directoryPath))
-        .filter(directoryPath => existsSync(directoryPath));
+      pathResources.forEach(directoryPath => {
+        const fullPath = path.join(this.inputPath, ...directoryPath.split('/'));
+        if (existsSync(fullPath)) {
+          pathResourceDirectories.push(fullPath);
+        } else if (directoryPath.endsWith('/*') && existsSync(fullPath.slice(0, -2))) {
+          pathResourceDirectories.push(
+            ...readdirSync(fullPath.slice(0, -2), { withFileTypes: true, recursive: true })
+              .filter(file => file.isDirectory())
+              .map(dir => path.join(dir.path, dir.name))
+          );
+        }
+      });
       if (pathResourceDirectories) predefinedResourcePaths.push(...pathResourceDirectories);
     }
     const deeplyNestedFiles: string[] = [];
@@ -987,7 +996,9 @@ export class IGExporter {
             path.dirname(file) !== dirPath &&
             !pathResourceDirectories?.includes(path.dirname(file))
           ) {
-            deeplyNestedFiles.push(file);
+            if (!deeplyNestedFiles.includes(file)) {
+              deeplyNestedFiles.push(file);
+            }
             continue;
           }
           const resourceJSON: InstanceDefinition = this.fhirDefs.getPredefinedResource(file);
diff --git a/src/import/importConfiguration.ts b/src/import/importConfiguration.ts
index d7a0843..937e892 100644
--- a/src/import/importConfiguration.ts
+++ b/src/import/importConfiguration.ts
@@ -748,7 +748,7 @@ function parseParameters(
   }
   if (yamlConfig.parameters) {
     for (const [code, values] of Object.entries(yamlConfig.parameters)) {
-      normalizeToArray(values).forEach(value => parameters.push({ code, value: `${value}` }));
+      normalizeToArray(values)?.forEach(value => parameters.push({ code, value: `${value}` }));
     }
   } else if (parameters.length === 0) {
     return; // return undefined rather than an empty []


Test patch:
diff --git a/test/ig/IGExporter.IG.test.ts b/test/ig/IGExporter.IG.test.ts
index 7a7572e..9524774 100644
--- a/test/ig/IGExporter.IG.test.ts
+++ b/test/ig/IGExporter.IG.test.ts
@@ -2921,7 +2921,7 @@ describe('IGExporter', () => {
       config.parameters = [];
       config.parameters.push({
         code: 'path-resource',
-        value: path.join('input', 'resources', 'path-resource-nest')
+        value: 'input/resources/path-resource-nest'
       });
       const pkg = new Package(config);
       exporter = new IGExporter(pkg, defs, path.resolve(fixtures));
@@ -2973,6 +2973,12 @@ describe('IGExporter', () => {
       );
       expect(warning).toInclude(path.join('nested1', 'StructureDefinition-MyTitlePatient.json'));
       expect(warning).toInclude(path.join('nested2', 'ValueSet-MyVS.json'));
+      expect(warning).toInclude(
+        path.join('path-resource-double-nest', 'john', 'Patient-John.json')
+      );
+      expect(warning).toInclude(
+        path.join('path-resource-double-nest', 'jack', 'examples', 'Patient-Jack.json')
+      );
       expect(warning).not.toInclude('Patient-BarPatient.json');
       expect(warning).not.toInclude('StructureDefinition-MyPatient.json');
     });
@@ -2989,10 +2995,63 @@ describe('IGExporter', () => {
       );
       expect(warning).toInclude(path.join('nested1', 'StructureDefinition-MyTitlePatient.json'));
       expect(warning).toInclude(path.join('nested2', 'ValueSet-MyVS.json'));
+      // path-resource-double-nest is not included in config
+      expect(warning).toInclude(
+        path.join('path-resource-double-nest', 'john', 'Patient-John.json')
+      );
+      expect(warning).toInclude(
+        path.join('path-resource-double-nest', 'jack', 'examples', 'Patient-Jack.json')
+      );
+      // path-resource-nest is included in config
       expect(warning).not.toInclude(
         path.join('path-resource-nest', 'StructureDefinition-MyCorrectlyNestedPatient.json')
       );
     });
+
+    it('should not warn on deeply nested resources that are included in the path-resource parameter with a directory and wildcard', () => {
+      const config = cloneDeep(minimalConfig);
+      config.parameters = [];
+      config.parameters.push({
+        code: 'path-resource',
+        value: 'input/resources/path-resource-double-nest/*'
+      });
+      const pkg = new Package(config);
+      exporter = new IGExporter(pkg, defs, path.resolve(fixtures));
+      exporter.export(tempOut);
+      expect(loggerSpy.getAllMessages('warn')).toHaveLength(1);
+      const warning = loggerSpy.getFirstMessage('warn');
+      expect(warning).not.toInclude(
+        path.join('path-resource-double-nest', 'john', 'Patient-John.json')
+      );
+      expect(warning).not.toInclude(
+        path.join('path-resource-double-nest', 'jack', 'examples', 'Patient-Jack.json')
+      );
+    });
+
+    it('should warn on deeply nested resources that are included in the path-resource parameter with a directory but NO wildcard', () => {
+      const config = cloneDeep(minimalConfig);
+      config.parameters = [];
+      config.parameters.push({
+        code: 'path-resource',
+        // NOTE: file path does not include the "*" portion (it just lists a directory), which is not sufficient
+        value: 'input/resources/path-resource-double-nest'
+      });
+      const pkg = new Package(config);
+      exporter = new IGExporter(pkg, defs, path.resolve(fixtures));
+      exporter.export(tempOut);
+      expect(loggerSpy.getAllMessages('warn')).toHaveLength(1);
+      const warning = loggerSpy.getFirstMessage('warn');
+      const warningLines = warning.split('\n');
+      const johnLine = warningLines.filter(w =>
+        w.includes(path.join('path-resource-double-nest', 'john', 'Patient-John.json'))
+      );
+      const jackLine = warningLines.filter(w =>
+        w.includes(path.join('path-resource-double-nest', 'john', 'Patient-John.json'))
+      );
+      // Check that both nested files are logged in the warning, but check that they're only there once
+      expect(johnLine).toHaveLength(1);
+      expect(jackLine).toHaveLength(1);
+    });
   });
 
   describe('#customized-ig-with-logical-model-example', () => {
diff --git a/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/path-resource-double-nest/jack/examples/Patient-Jack.json b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/path-resource-double-nest/jack/examples/Patient-Jack.json
new file mode 100644
index 0000000..366338c
--- /dev/null
+++ b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/path-resource-double-nest/jack/examples/Patient-Jack.json
@@ -0,0 +1,13 @@
+{
+  "resourceType": "Patient",
+  "id": "Jack",
+  "name": [
+    {
+      "family": "Anyperson",
+      "given": [
+        "Jack",
+        "C."
+      ]
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/path-resource-double-nest/john/Patient-John.json b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/path-resource-double-nest/john/Patient-John.json
new file mode 100644
index 0000000..5b6ceb2
--- /dev/null
+++ b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/path-resource-double-nest/john/Patient-John.json
@@ -0,0 +1,13 @@
+{
+  "resourceType": "Patient",
+  "id": "John",
+  "name": [
+    {
+      "family": "Anyperson",
+      "given": [
+        "John",
+        "B."
+      ]
+    }
+  ]
+}
\ No newline at end of file