Repository: FHIR/sushi (ts)

Issue:
SUSHI does not recursively read input files
If the user has a folder structure like:
```
input
- fsh
    - fsh-file1.fsh
- resources
    - structuredefinitions
        - file1.json
    - codesystems
    - valuesets
```
Then SUSHI does not recursively read in `file1.json` as a predefined resource. This should probably be supported.

Pull request description:
Load predefined resources in nested folders
The IG templates define a set of supported input folders (e.g., input/profiles, input/resources, etc.). If an author creates further subfolders (e.g., input/resources/nested), SUSHI will now load resources from them as well.  Since the IG publisher doesn't support those paths, however, it will emit a warning and exclude them from the IG JSON.

You can manually test w/ the attached project: [RecursiveInput.zip](https://github.com/FHIR/sushi/files/6785975/RecursiveInput.zip).

Fixes #786 

Patch:
diff --git a/src/fhirdefs/load.ts b/src/fhirdefs/load.ts
index 1b2b988..cd6328d 100644
--- a/src/fhirdefs/load.ts
+++ b/src/fhirdefs/load.ts
@@ -7,7 +7,7 @@ import tar from 'tar';
 import axios from 'axios';
 import junk from 'junk';
 import temp from 'temp';
-import { logger } from '../utils';
+import { logger, getFilesRecursive } from '../utils';
 import { Fhir as FHIRConverter } from 'fhir/fhir';
 
 /**
@@ -210,7 +210,7 @@ export function loadCustomResources(resourceDir: string, defs: FHIRDefinitions):
     let foundSpreadsheets = false;
     const dirPath = path.join(resourceDir, pathEnd);
     if (fs.existsSync(dirPath)) {
-      const files = fs.readdirSync(dirPath);
+      const files = getFilesRecursive(dirPath);
       for (const file of files) {
         let resourceJSON: any;
         try {
@@ -218,12 +218,12 @@ export function loadCustomResources(resourceDir: string, defs: FHIRDefinitions):
             // Ignore "junk" files created by the OS, like .DS_Store on macOS and Thumbs.db on Windows
             continue;
           } else if (file.endsWith('.json')) {
-            resourceJSON = fs.readJSONSync(path.join(dirPath, file));
+            resourceJSON = fs.readJSONSync(file);
           } else if (file.endsWith('-spreadsheet.xml')) {
             foundSpreadsheets = true;
             continue;
           } else if (file.endsWith('xml')) {
-            const xml = fs.readFileSync(path.join(dirPath, file)).toString();
+            const xml = fs.readFileSync(file).toString();
             if (/<\?mso-application progid="Excel\.Sheet"\?>/m.test(xml)) {
               foundSpreadsheets = true;
               continue;
diff --git a/src/ig/IGExporter.ts b/src/ig/IGExporter.ts
index 97db1da..b505042 100644
--- a/src/ig/IGExporter.ts
+++ b/src/ig/IGExporter.ts
@@ -25,7 +25,7 @@ import {
   ImplementationGuideDependsOn
 } from '../fhirtypes';
 import { ConfigurationMenuItem } from '../fshtypes';
-import { logger, Type } from '../utils';
+import { logger, Type, getFilesRecursive } from '../utils';
 import { FHIRDefinitions } from '../fhirdefs';
 import { Configuration } from '../fshtypes';
 
@@ -801,6 +801,9 @@ export class IGExporter {
    * capabilities, extensions, models, operations, profiles, resources, vocabulary, examples
    * Based on: https://build.fhir.org/ig/FHIR/ig-guidance/using-templates.html#root.input
    *
+   * NOTE: This does not include files nested in subfolders in supported paths since the
+   * IG Exporter does not handle those well.
+   *
    * This function has similar operation to addResources, and both should be
    * analyzed when making changes to either.
    */
@@ -816,17 +819,20 @@ export class IGExporter {
       'vocabulary',
       'examples'
     ];
+    const deeplyNestedFiles: string[] = [];
     for (const pathEnd of pathEnds) {
-      const dirPath = path.join(this.inputPath, 'input', pathEnd);
+      const dirPath = path.resolve(this.inputPath, 'input', pathEnd);
       if (existsSync(dirPath)) {
-        const files = readdirSync(dirPath);
+        const files = getFilesRecursive(dirPath);
         for (const file of files) {
+          if (path.dirname(file) !== dirPath) {
+            deeplyNestedFiles.push(file);
+            continue;
+          }
           const resourceJSON: InstanceDefinition = this.fhirDefs.getPredefinedResource(file);
           if (resourceJSON) {
             if (resourceJSON.resourceType == null || resourceJSON.id == null) {
-              logger.error(
-                `Resource at ${path.join(dirPath, file)} must define resourceType and id.`
-              );
+              logger.error(`Resource at ${file} must define resourceType and id.`);
               continue;
             }
 
@@ -920,6 +926,15 @@ export class IGExporter {
         }
       }
     }
+    if (deeplyNestedFiles.length) {
+      logger.warn(
+        'The following files were not added to the ImplementationGuide JSON because they are nested too deep. While ' +
+          'SUSHI supports these paths, the IG Publisher does not. To fix this, move these files so they are directly ' +
+          `under a supported input folder (e.g., input/resources, input/profiles, etc.):\n  - ${deeplyNestedFiles.join(
+            '\n  - '
+          )}`
+      );
+    }
   }
 
   /**
diff --git a/src/utils/Processing.ts b/src/utils/Processing.ts
index 42fbae2..9495f7d 100644
--- a/src/utils/Processing.ts
+++ b/src/utils/Processing.ts
@@ -505,15 +505,17 @@ export async function init(): Promise<void> {
   );
 }
 
-function getFilesRecursive(dir: string): string[] {
+export function getFilesRecursive(dir: string): string[] {
+  // always return absolute paths
+  const absPath = path.resolve(dir);
   try {
-    if (fs.statSync(dir).isDirectory()) {
+    if (fs.statSync(absPath).isDirectory()) {
       const descendants = fs
-        .readdirSync(dir, 'utf8')
-        .map(f => getFilesRecursive(path.join(dir, f)));
+        .readdirSync(absPath, 'utf8')
+        .map(f => getFilesRecursive(path.join(absPath, f)));
       return [].concat(...descendants);
     } else {
-      return [dir];
+      return [absPath];
     }
   } catch {
     return [];


Test patch:
diff --git a/test/fhirdefs/fixtures/customized-ig-with-resources/ig-data/input/resources/nested/StructureDefinition-MyNestedPatient.json b/test/fhirdefs/fixtures/customized-ig-with-resources/ig-data/input/resources/nested/StructureDefinition-MyNestedPatient.json
new file mode 100644
index 0000000..567176d
--- /dev/null
+++ b/test/fhirdefs/fixtures/customized-ig-with-resources/ig-data/input/resources/nested/StructureDefinition-MyNestedPatient.json
@@ -0,0 +1,6 @@
+{
+  "resourceType": "StructureDefinition",
+  "id": "MyNestedPatient",
+  "kind": "resource",
+  "derivation": "constraint"
+}
diff --git a/test/fhirdefs/load.test.ts b/test/fhirdefs/load.test.ts
index 15049f3..7065856 100644
--- a/test/fhirdefs/load.test.ts
+++ b/test/fhirdefs/load.test.ts
@@ -392,17 +392,18 @@ describe('#loadDependency()', () => {
 
 describe('#loadCustomResources', () => {
   let defs: FHIRDefinitions;
+  let pathToInput: string;
   beforeAll(() => {
     loggerSpy.reset();
     defs = new FHIRDefinitions();
-    const fixtures = path.join(
+    pathToInput = path.join(
       __dirname,
       'fixtures',
       'customized-ig-with-resources',
       'ig-data',
       'input'
     );
-    loadCustomResources(fixtures, defs);
+    loadCustomResources(pathToInput, defs);
   });
 
   it('should load custom JSON and XML resources', () => {
@@ -410,7 +411,7 @@ describe('#loadCustomResources', () => {
     const profiles = defs.allProfiles();
     const valueSets = defs.allValueSets();
     const extensions = defs.allExtensions();
-    expect(profiles).toHaveLength(1);
+    expect(profiles).toHaveLength(2);
     expect(profiles[0].id).toBe('MyPatient');
     expect(valueSets).toHaveLength(1);
     expect(valueSets[0].id).toBe('MyVS');
@@ -430,19 +431,51 @@ describe('#loadCustomResources', () => {
   });
 
   it('should add all predefined resources to the FHIRDefs with file information', () => {
-    expect(defs.getPredefinedResource('CapabilityStatement-MyCS.json').id).toBe('MyCS');
-    expect(defs.getPredefinedResource('Patient-MyPatient.json').id).toBe('MyPatient');
-    expect(defs.getPredefinedResource('StructureDefinition-patient-birthPlace.json').id).toBe(
-      'patient-birthPlace'
-    );
-    expect(defs.getPredefinedResource('StructureDefinition-patient-birthPlaceXML.xml').id).toBe(
-      'patient-birthPlaceXML'
-    );
-    expect(defs.getPredefinedResource('StructureDefinition-MyLM.json').id).toBe('MyLM');
-    expect(defs.getPredefinedResource('OperationDefinition-MyOD.json').id).toBe('MyOD');
-    expect(defs.getPredefinedResource('StructureDefinition-MyPatient.json').id).toBe('MyPatient');
-    expect(defs.getPredefinedResource('Patient-BazPatient.json').id).toBe('BazPatient');
-    expect(defs.getPredefinedResource('ValueSet-MyVS.json').id).toBe('MyVS');
+    expect(
+      defs.getPredefinedResource(
+        path.join(pathToInput, 'capabilities', 'CapabilityStatement-MyCS.json')
+      ).id
+    ).toBe('MyCS');
+    expect(
+      defs.getPredefinedResource(path.join(pathToInput, 'examples', 'Patient-MyPatient.json')).id
+    ).toBe('MyPatient');
+    expect(
+      defs.getPredefinedResource(
+        path.join(pathToInput, 'extensions', 'StructureDefinition-patient-birthPlace.json')
+      ).id
+    ).toBe('patient-birthPlace');
+    expect(
+      defs.getPredefinedResource(
+        path.join(pathToInput, 'extensions', 'StructureDefinition-patient-birthPlaceXML.xml')
+      ).id
+    ).toBe('patient-birthPlaceXML');
+    expect(
+      defs.getPredefinedResource(path.join(pathToInput, 'models', 'StructureDefinition-MyLM.json'))
+        .id
+    ).toBe('MyLM');
+    expect(
+      defs.getPredefinedResource(
+        path.join(pathToInput, 'operations', 'OperationDefinition-MyOD.json')
+      ).id
+    ).toBe('MyOD');
+    expect(
+      defs.getPredefinedResource(
+        path.join(pathToInput, 'profiles', 'StructureDefinition-MyPatient.json')
+      ).id
+    ).toBe('MyPatient');
+    expect(
+      defs.getPredefinedResource(path.join(pathToInput, 'resources', 'Patient-BazPatient.json')).id
+    ).toBe('BazPatient');
+    // NOTE: It loads nested predefined resources even thought the IG Publisher doesn't handle them well
+    expect(
+      defs.getPredefinedResource(
+        path.join(pathToInput, 'resources', 'nested', 'StructureDefinition-MyNestedPatient.json')
+      ).id
+    ).toBe('MyNestedPatient');
+
+    expect(
+      defs.getPredefinedResource(path.join(pathToInput, 'vocabulary', 'ValueSet-MyVS.json')).id
+    ).toBe('MyVS');
   });
 
   it('should log an info message for non JSON or XML input files', () => {
@@ -453,7 +486,7 @@ describe('#loadCustomResources', () => {
 
   it('should log an error for invalid XML files', () => {
     expect(loggerSpy.getLastMessage('error')).toMatch(
-      /Loading InvalidFile.xml failed with the following error:/
+      /Loading .*InvalidFile.xml failed with the following error:/
     );
   });
 
@@ -477,7 +510,7 @@ describe('#loadCustomResources', () => {
 
   it('should log an error for invalid JSON files', () => {
     expect(loggerSpy.getMessageAtIndex(-2, 'error')).toMatch(
-      /Loading InvalidFile.json failed with the following error:/
+      /Loading .*InvalidFile.json failed with the following error:/
     );
   });
 });
diff --git a/test/ig/IGExporter.IG.test.ts b/test/ig/IGExporter.IG.test.ts
index 1ca7d14..0379f5e 100644
--- a/test/ig/IGExporter.IG.test.ts
+++ b/test/ig/IGExporter.IG.test.ts
@@ -1517,6 +1517,75 @@ describe('IGExporter', () => {
     });
   });
 
+  describe('#customized-ig-with-nested-resources', () => {
+    let exporter: IGExporter;
+    let tempOut: string;
+    let fixtures: string;
+    let defs: FHIRDefinitions;
+
+    beforeAll(() => {
+      defs = new FHIRDefinitions();
+      loadFromPath(
+        path.join(__dirname, '..', 'testhelpers', 'testdefs', 'package'),
+        'testPackage',
+        defs
+      );
+      fixtures = path.join(__dirname, 'fixtures', 'customized-ig-with-nested-resources');
+      loadCustomResources(path.join(fixtures, 'input'), defs);
+    });
+
+    beforeEach(() => {
+      loggerSpy.reset();
+      tempOut = temp.mkdirSync('sushi-test');
+      const config = cloneDeep(minimalConfig);
+      const pkg = new Package(config);
+      exporter = new IGExporter(pkg, defs, path.resolve(fixtures));
+    });
+
+    afterAll(() => {
+      temp.cleanupSync();
+    });
+
+    it('should add only non-nested resource references to the ImplementationGuide resource', () => {
+      exporter.export(tempOut);
+      const igPath = path.join(
+        tempOut,
+        'fsh-generated',
+        'resources',
+        'ImplementationGuide-fhir.us.minimal.json'
+      );
+      expect(fs.existsSync(igPath)).toBeTruthy();
+      const igContent: ImplementationGuide = fs.readJSONSync(igPath);
+      expect(igContent.definition.resource).toHaveLength(2);
+      expect(igContent.definition.resource).toContainEqual({
+        reference: {
+          reference: 'Patient/BarPatient'
+        },
+        name: 'BarPatient',
+        exampleCanonical: 'http://hl7.org/fhir/sushi-test/StructureDefinition/MyPatient'
+      });
+      expect(igContent.definition.resource).toContainEqual({
+        reference: {
+          reference: 'StructureDefinition/MyPatient'
+        },
+        name: 'MyPatient',
+        exampleBoolean: false
+      });
+    });
+
+    it('should warn on deeply nested resources', () => {
+      exporter.export(tempOut);
+      const warning = loggerSpy.getFirstMessage('warn');
+      expect(warning).toInclude(
+        'The following files were not added to the ImplementationGuide JSON'
+      );
+      expect(warning).toInclude(path.join('nested1', 'StructureDefinition-MyTitlePatient.json'));
+      expect(warning).toInclude(path.join('nested2', 'ValueSet-MyVS.json'));
+      expect(warning).not.toInclude('Patient-BarPatient.json');
+      expect(warning).not.toInclude('StructureDefinition-MyPatient.json');
+    });
+  });
+
   describe('#pages-folder-ig', () => {
     let pkg: Package;
     let exporter: IGExporter;
diff --git a/test/ig/fixtures/customized-ig-with-nested-resources/input/examples/Patient-BarPatient.json b/test/ig/fixtures/customized-ig-with-nested-resources/input/examples/Patient-BarPatient.json
new file mode 100644
index 0000000..05174a8
--- /dev/null
+++ b/test/ig/fixtures/customized-ig-with-nested-resources/input/examples/Patient-BarPatient.json
@@ -0,0 +1,8 @@
+{
+  "resourceType": "Patient",
+  "id": "BarPatient",
+  "meta": {
+    "profile": ["http://hl7.org/fhir/sushi-test/StructureDefinition/MyPatient"]
+  },
+  "name": [{ "family": "Bar" }]
+}
diff --git a/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/StructureDefinition-MyPatient.json b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/StructureDefinition-MyPatient.json
new file mode 100644
index 0000000..4f65cc4
--- /dev/null
+++ b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/StructureDefinition-MyPatient.json
@@ -0,0 +1,25 @@
+{
+  "resourceType": "StructureDefinition",
+  "id": "MyPatient",
+  "url": "http://hl7.org/fhir/sushi-test/StructureDefinition/MyPatient",
+  "name": "MyPatient",
+  "status": "active",
+  "fhirVersion": "4.0.1",
+  "kind": "resource",
+  "type": "Patient",
+  "baseDefinition": "http://hl7.org/fhir/StructureDefinition/Patient",
+  "derivation": "constraint",
+  "differential": {
+    "element": [
+      {
+        "id": "Patient.deceased[x]",
+        "path": "Patient.deceased[x]",
+        "type": [
+          {
+            "code": "boolean"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/nested1/StructureDefinition-MyTitlePatient.json b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/nested1/StructureDefinition-MyTitlePatient.json
new file mode 100644
index 0000000..a1dec3d
--- /dev/null
+++ b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/nested1/StructureDefinition-MyTitlePatient.json
@@ -0,0 +1,26 @@
+{
+  "resourceType": "StructureDefinition",
+  "id": "MyTitlePatient",
+  "url": "http://hl7.org/fhir/us/mcode/StructureDefinition/MyTitlePatient",
+  "title": "This patient has a title",
+  "name": "MyTitlePatient",
+  "status": "active",
+  "fhirVersion": "4.0.1",
+  "kind": "resource",
+  "type": "Patient",
+  "baseDefinition": "http://hl7.org/fhir/StructureDefinition/Patient",
+  "derivation": "constraint",
+  "differential": {
+    "element": [
+      {
+        "id": "Patient.deceased[x]",
+        "path": "Patient.deceased[x]",
+        "type": [
+          {
+            "code": "boolean"
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/nested2/ValueSet-MyVS.json b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/nested2/ValueSet-MyVS.json
new file mode 100644
index 0000000..0498df0
--- /dev/null
+++ b/test/ig/fixtures/customized-ig-with-nested-resources/input/resources/nested2/ValueSet-MyVS.json
@@ -0,0 +1,59 @@
+{
+  "resourceType": "ValueSet",
+  "id": "MyVS",
+  "meta": {
+    "extension": [
+      {
+        "url": "http://hl7.org/fhir/StructureDefinition/instance-description",
+        "valueMarkdown": "Ignored Description"
+      },
+      {
+        "url": "http://hl7.org/fhir/StructureDefinition/instance-name",
+        "valueString": "Ignored Name"
+      }
+    ]
+  },
+  "url": "http://hl7.org/fhir/ValueSet/MyVS",
+  "version": "4.0.1",
+  "name": "Yes/No/Don't Know",
+  "status": "draft",
+  "compose": {
+    "include": [
+      {
+        "valueSet": [
+          "http://terminology.hl7.org/ValueSet/v2-0136"
+        ]
+      },
+      {
+        "system": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
+        "concept": [
+          {
+            "code": "asked-unknown",
+            "display": "Don't know"
+          }
+        ]
+      }
+    ]
+  },
+  "expansion": {
+    "identifier": "urn:uuid:bf99fe50-2c2b-41ad-bd63-bee6919810b4",
+    "timestamp": "2015-07-14T10:00:00Z",
+    "contains": [
+      {
+        "system": "http://terminology.hl7.org/CodeSystem/v2-0136",
+        "code": "Y",
+        "display": "Yes"
+      },
+      {
+        "system": "http://terminology.hl7.org/CodeSystem/v2-0136",
+        "code": "N",
+        "display": "No"
+      },
+      {
+        "system": "http://terminology.hl7.org/CodeSystem/data-absent-reason",
+        "code": "asked-unknown",
+        "display": "Don't know"
+      }
+    ]
+  }
+}
\ No newline at end of file