Repository: FHIR/sushi (ts)

Issue:
Support New Syntax to Declare Multiple Versions of the Same Dependency
_**NOTE: Proposed syntax has changed since this issue was reported. See latest comment below.**_

Some IGs may need to support multiple versions of the same dependency package. To support this, the following syntax will be used:
```
 "hl7.fhir.us.core-old" : "npm:hl7.fhir.us.core@3.1.1",
 "hl7.fhir.us.core" : "6.1.0",
```

SUSHI has been identified as a core tool that will need to support this.

For more information, see: https://chat.fhir.org/#narrow/stream/179239-tooling/topic/Advance.20Notice/near/439946498

Also logged as a FHIR Package Loader issue here: https://github.com/standardhealth/fhir-package-loader/issues/23

Pull request description:
Support for NPM aliases in dependencies
**Description:** Allow users to use NPM aliases in sushi-config.yaml so that they can load multiple versions of the same package. E.g.:
```
dependencies:
  hl7.fhir.us.core: 7.0.0
  v610@npm:hl7.fhir.us.core: 6.1.0
  v311@npm:hl7.fhir.us.core: 3.1.1
```
The only point of these aliases is to allow loading multiple versions of the same package -- so the aliases are not used or referenced in any actual FSH content.

Packages w/ multiple versions are loaded such that the latest version is loaded last. This way versionless resolution always goes to the latest version.

See: https://chat.fhir.org/#narrow/channel/179239-tooling/topic/NPM.20Aliases/near/517985527

_NOTE: I originally thought we might need to update FHIR Package Loader to support NPM aliases, but I opted to keep its interface clean (since it already supports loading multiple versions of same package) and have SUSHI handle aliases before requesting packages from FHIR Package Loader._

**Testing Instructions:** Create a project that uses NPM aliases to declare dependencies on multiple versions of the same project. Ensure that SUSHI loads the dependencies without error and that you are able to reference definitions from any of those versions (note: there is a known issue w/ canonical versions declared as `Parent:` (#1553) or `InstanceOf:` (#1559), so it's probably best to test referencing different versions by constraining references using `Reference(a.b.c|1.2.3)`).

To run the code from this branch:
1. Check out the branch in git
2. Run `npm install` to install the dependencies
3. Run the following command: `ts-node src/app.ts build /path/to/fshproject` (may require `npm install -g ts-node`)

**Related Issue:** Fixes #1463 


Patch:
diff --git a/src/ig/IGExporter.ts b/src/ig/IGExporter.ts
index 7b841c2..19fbca0 100644
--- a/src/ig/IGExporter.ts
+++ b/src/ig/IGExporter.ts
@@ -261,6 +261,15 @@ export class IGExporter {
     // Clone it so we don't mutate the original
     const dependsOn = cloneDeep(dependency);
 
+    // Get the alias and real package id (in cases where it uses an NPM alias)
+    // See: https://chat.fhir.org/#narrow/channel/179239-tooling/topic/NPM.20Aliases/near/517985527
+    let alias: string = null;
+    let realPackageId = dependsOn.packageId ?? '';
+    const npmAliasMatcher = realPackageId.match(/^([^@]+)@npm:(.+)$/);
+    if (npmAliasMatcher) {
+      [alias, realPackageId] = npmAliasMatcher.slice(1);
+    }
+
     // By default, dependsOn.reason should not be supported because it is an R5 element
     delete dependsOn.reason;
 
@@ -277,12 +286,12 @@ export class IGExporter {
     if (dependsOn.version === 'latest') {
       // TODO: This assumes only a single version of a package is in scope
       // NOTE: It replaces latest with the real version since IG Publisher doesn't support "latest"
-      const dependencyIG = igs.find(ig => ig.packageId === dependsOn.packageId);
+      const dependencyIG = igs.find(ig => ig.packageId === realPackageId);
       if (dependencyIG?.version != null) {
         resolvedVersion = dependsOn.version = dependencyIG.version;
       } else {
         const packageInfos = this.fhirDefs
-          .findPackageInfos(dependsOn.packageId)
+          .findPackageInfos(realPackageId)
           .filter(info => info.version != null);
         if (packageInfos.length) {
           resolvedVersion = dependsOn.version = packageInfos[0].version;
@@ -292,7 +301,7 @@ export class IGExporter {
 
     if (dependsOn.version?.endsWith('.x')) {
       const matchingVersions = igs
-        .filter(ig => ig.packageId === dependsOn.packageId && ig.version != null)
+        .filter(ig => ig.packageId === realPackageId && ig.version != null)
         .map(ig => ig.version);
       const max = maxSatisfying(matchingVersions, dependsOn.version);
       if (max) {
@@ -305,7 +314,7 @@ export class IGExporter {
       // Need to find the URI from the IG in the FHIR cache
       const dependencyIG = igs.find(
         ig =>
-          ig.packageId === dependsOn.packageId &&
+          ig.packageId === realPackageId &&
           (ig.version === resolvedVersion ||
             'current' === resolvedVersion ||
             'dev' === resolvedVersion)
@@ -313,27 +322,25 @@ export class IGExporter {
       dependsOn.uri = dependencyIG?.url;
       if (dependsOn.uri == null) {
         // there may be a package.json that can help us here
-        const dependencyPackageJson = this.fhirDefs.findPackageJSON(
-          dependsOn.packageId,
-          resolvedVersion
-        );
+        const dependencyPackageJson = this.fhirDefs.findPackageJSON(realPackageId, resolvedVersion);
         dependsOn.uri = dependencyPackageJson?.canonical;
       }
 
       if (dependsOn.uri == null) {
         // setting uri to required format as indicated in zulip discussion:
         // https://chat.fhir.org/#narrow/stream/179252-IG-creation/topic/fhir.2Edicom.20has.20no.20ImplementationGuide.20resource/near/265772431
-        dependsOn.uri = `http://fhir.org/packages/${dependsOn.packageId}/ImplementationGuide/${dependsOn.packageId}`;
+        dependsOn.uri = `http://fhir.org/packages/${realPackageId}/ImplementationGuide/${realPackageId}`;
       }
     }
 
     if (dependsOn.id == null) {
+      const dependsOnId = alias ? `${realPackageId}_${alias}` : realPackageId;
       // packageId should be "a..z, A..Z, 0..9, and _ and it must start with a..z | A..Z" per
       // https://chat.fhir.org/#narrow/stream/215610-shorthand/topic/SUSHI.200.2E12.2E7/near/199193333
       // depId should be [A-Za-z0-9\-\.]{1,64}, so we replace . and - with _ and prepend "id_" if it does not start w/ a-z|A-Z
-      dependsOn.id = /[A-Za-z]/.test(dependsOn.packageId[0])
-        ? dependsOn.packageId.replace(/\.|-/g, '_')
-        : 'id_' + dependsOn.packageId.replace(/\.|-/g, '_');
+      dependsOn.id = /[A-Za-z]/.test(dependsOnId[0])
+        ? dependsOnId.replace(/\.|-/g, '_')
+        : 'id_' + dependsOnId.replace(/\.|-/g, '_');
     }
 
     // Keep dependsOn.extension as configured, so no special handling is needed
diff --git a/src/utils/Processing.ts b/src/utils/Processing.ts
index 1124521..92bf05c 100644
--- a/src/utils/Processing.ts
+++ b/src/utils/Processing.ts
@@ -3,6 +3,7 @@ import process from 'process';
 import fs from 'fs-extra';
 import readlineSync from 'readline-sync';
 import YAML from 'yaml';
+import semver from 'semver';
 import { execSync } from 'child_process';
 import { YAMLMap, Collection } from 'yaml/types';
 import { isPlainObject, padEnd, startCase, sortBy, upperFirst } from 'lodash';
@@ -257,8 +258,11 @@ export async function updateExternalDependencies(
     return true;
   }
   const promises = config.dependencies.map(async dep => {
+    if (/@npm/.test(dep.packageId)) {
+      logger.info(`Skipping dependency version check for NPM aliased package: ${dep.packageId}`);
+    }
     // current and dev have special meanings, so don't try to update those dependencies
-    if (dep.version != 'current' && dep.version != 'dev') {
+    else if (dep.version != 'current' && dep.version != 'dev') {
       try {
         const latestVersion = await registryClient.resolveVersion(dep.packageId, 'latest');
         if (dep.version !== latestVersion) {
@@ -321,8 +325,37 @@ export async function loadExternalDependencies(
   defs: FHIRDefinitions,
   config: Configuration
 ): Promise<void> {
+  // Check for special npm alias syntax and fix the packageIds as necessary, sorting dependencies with
+  // the same package id so the latest version is last (due to FPL's last in first out resolution strategy)
+  // See: https://chat.fhir.org/#narrow/channel/179239-tooling/topic/NPM.20Aliases/near/517985527
+
+  // First collect all the packages in a map, grouping multiple versions of the same package together
+  const packageIdMap = new Map<string, ImplementationGuideDependsOn[]>();
+  (config.dependencies ?? []).forEach(dep => {
+    const npmAliasMatcher = dep.packageId?.match(/^([^@]+)@npm:(.+)$/);
+    if (npmAliasMatcher) {
+      const [alias, packageId] = npmAliasMatcher.slice(1);
+      if (!/^[A-Za-z0-9\-\.]+$/.test(alias)) {
+        logger.warn(
+          `NPM aliases should contain only the following characters: upper- or lower-case ASCII letters ('A'..'Z', and 'a'..'z'), numerals ('0'..'9'), '-' and '.'. Found '${alias}'.`
+        );
+      }
+      dep = Object.assign({}, dep, { packageId });
+    }
+    if (packageIdMap.has(dep.packageId)) {
+      packageIdMap.get(dep.packageId).push(dep);
+    } else {
+      packageIdMap.set(dep.packageId, [dep]);
+    }
+  });
+
+  // Then iterate the map in insertion order, preserving original order except packages with multiple versions
+  const dependencies: ImplementationGuideDependsOn[] = [];
+  packageIdMap.forEach(dependsOns => {
+    dependencies.push(...dependsOns.sort((a, b) => semver.compareLoose(a.version, b.version)));
+  });
+
   // Add FHIR to the dependencies so it is loaded
-  const dependencies = (config.dependencies ?? []).slice(); // slice so we don't modify actual config;
   const fhirVersionInfo = config.fhirVersion
     .map(v => getFHIRVersionInfo(v))
     .find(v => v.isSupported);


Test patch:
diff --git a/test/ig/IGExporter.IG.test.ts b/test/ig/IGExporter.IG.test.ts
index e61a7ce..941e4bf 100644
--- a/test/ig/IGExporter.IG.test.ts
+++ b/test/ig/IGExporter.IG.test.ts
@@ -23,7 +23,7 @@ import {
 import { cloneDeep } from 'lodash';
 import { minimalConfig } from '../utils/minimalConfig';
 import { minimalConfigWithMenu } from '../utils/minimalConfigWithMenu';
-import { DiskBasedVirtualPackage } from 'fhir-package-loader';
+import { DiskBasedVirtualPackage, InMemoryVirtualPackage } from 'fhir-package-loader';
 
 describe('IGExporter', () => {
   temp.track();
@@ -5446,4 +5446,76 @@ describe('IGExporter', () => {
       });
     });
   });
+
+  describe('#npm-aliases', () => {
+    let ig: any;
+
+    beforeAll(async () => {
+      loggerSpy.reset();
+      const tempOut = temp.mkdirSync('sushi-test');
+      const fixtures = path.join(__dirname, 'fixtures', 'simple-ig');
+      // r4-definitions contains ImplementationGuide-hl7.fhir.us.core.json used for resolution
+      const defs = await getTestFHIRDefinitions(false, testDefsPath('r4-definitions'));
+
+      // Add the additional US Core package that will be aliased (along with minimal ImplementationGuide)
+      const additionalUSCoreMap = new Map<string, any>();
+      additionalUSCoreMap.set('ImplementationGuide-hl7.fhir.us.core', {
+        resourceType: 'ImplementationGuide',
+        id: 'hl7.fhir.us.core',
+        url: 'http://hl7.org/fhir/us/core/ImplementationGuide/hl7.fhir.us.core',
+        version: '7.0.0',
+        name: 'USCore',
+        title: 'US Core',
+        status: 'active',
+        packageId: 'hl7.fhir.us.core',
+        fhirVersion: ['4.0.1']
+      });
+      const additionalUSCorePackage = new InMemoryVirtualPackage(
+        { name: 'hl7.fhir.us.core', version: '7.0.0' },
+        additionalUSCoreMap
+      );
+      await defs.loadVirtualPackage(additionalUSCorePackage);
+
+      const config = cloneDeep(minimalConfig);
+      config.dependencies = [
+        { packageId: 'hl7.fhir.us.core', version: '7.0.0' },
+        { packageId: 'v310@npm:hl7.fhir.us.core', version: '3.1.0' }
+      ];
+      const pkg = new Package(config);
+      const exporter = new IGExporter(pkg, defs, fixtures);
+      // No need to regenerate the IG on every test -- generate it once and inspect what you
+      // need to in the tests
+      exporter.export(tempOut);
+      const igPath = path.join(
+        tempOut,
+        'fsh-generated',
+        'resources',
+        'ImplementationGuide-fhir.us.minimal.json'
+      );
+      expect(fs.existsSync(igPath)).toBeTruthy();
+      ig = fs.readJSONSync(igPath);
+    });
+
+    afterAll(() => {
+      temp.cleanupSync();
+    });
+
+    it('should pass along the aliased package as-is, but detect correct IG URL and generate id w/ alias', () => {
+      const dependencies: ImplementationGuideDependsOn[] = ig.dependsOn;
+      expect(loggerSpy.getAllLogs('error')).toHaveLength(0);
+      // Ensure US Core is exported with the resolved version
+      expect(dependencies).toContainEqual({
+        id: 'hl7_fhir_us_core',
+        uri: 'http://hl7.org/fhir/us/core/ImplementationGuide/hl7.fhir.us.core',
+        packageId: 'hl7.fhir.us.core',
+        version: '7.0.0'
+      });
+      expect(dependencies).toContainEqual({
+        id: 'hl7_fhir_us_core_v310',
+        uri: 'http://hl7.org/fhir/us/core/ImplementationGuide/hl7.fhir.us.core',
+        packageId: 'v310@npm:hl7.fhir.us.core',
+        version: '3.1.0'
+      });
+    });
+  });
 });
diff --git a/test/utils/Processing.test.ts b/test/utils/Processing.test.ts
index 0d1d024..2d1c4ed 100644
--- a/test/utils/Processing.test.ts
+++ b/test/utils/Processing.test.ts
@@ -658,6 +658,32 @@ describe('Processing', () => {
       expect(result).toBe(true);
       expect(keyInSpy).toHaveBeenCalledTimes(0);
     });
+
+    it('should log a message when skipping dependencies with NPM aliases', async () => {
+      config.dependencies = [
+        {
+          packageId: 'hl7.fhir.us.core',
+          version: '3.1.0'
+        },
+        {
+          packageId: 'v200@npm:hl7.fhir.us.core',
+          version: '2.0.0'
+        },
+        {
+          packageId: 'hl7.fhir.us.davinci-pas',
+          version: 'dev'
+        }
+      ];
+      const result = await updateExternalDependencies(config, registryClientMock);
+      expect(result).toBe(true);
+      expect(keyInSpy).toHaveBeenCalledTimes(0);
+      expect(loggerSpy.getAllMessages('info')).toContain(
+        'Skipping dependency version check for NPM aliased package: v200@npm:hl7.fhir.us.core'
+      );
+      expect(loggerSpy.getAllMessages('info')).not.toContain(
+        'Skipping dependency version check for NPM aliased package: hl7.fhir.us.core'
+      );
+    });
   });
 
   describe('#loadExternalDependencies()', () => {
@@ -679,6 +705,36 @@ describe('Processing', () => {
       });
     });
 
+    it('should load specified dependencies when NPM aliases are used', async () => {
+      // See: https://chat.fhir.org/#narrow/channel/179239-tooling/topic/NPM.20Aliases/near/517985527
+      const npmAliasDependencyConfig = cloneDeep(minimalConfig);
+      npmAliasDependencyConfig.dependencies = [
+        { packageId: 'hl7.fhir.us.mcode', version: '4.0.0' },
+        { packageId: 'hl7.fhir.us.core', version: '7.0.0' },
+        { packageId: 'v610@npm:hl7.fhir.us.core', version: '6.1.0' },
+        { packageId: 'hl7.fhir.us.qicore', version: '6.0.0' },
+        { packageId: 'v311@npm:hl7.fhir.us.core', version: '3.1.1' }
+      ];
+      const defs = await getTestFHIRDefinitions();
+      return loadExternalDependencies(defs, npmAliasDependencyConfig).then(async () => {
+        const loadedPackages = defs.findPackageInfos('*').map(pkg => `${pkg.name}#${pkg.version}`);
+        expect(loadedPackages).toHaveLength(6 + NUM_R4_AUTO_DEPENDENCIES);
+        // Check that specified packages are loaded in correct order after autodependencies.
+        // Correct order includes sorting multi-version packages so most recent is last.
+        expect(loadedPackages.slice(NUM_R4_AUTO_DEPENDENCIES)).toEqual([
+          'hl7.fhir.us.mcode#4.0.0',
+          'hl7.fhir.us.core#3.1.1',
+          'hl7.fhir.us.core#6.1.0',
+          'hl7.fhir.us.core#7.0.0',
+          'hl7.fhir.us.qicore#6.0.0',
+          'hl7.fhir.r4.core#4.0.1'
+        ]);
+        assertAutomaticR4Dependencies(loadedPackages);
+        expect(loggerSpy.getAllLogs('warn')).toHaveLength(0);
+        expect(loggerSpy.getAllLogs('error')).toHaveLength(0);
+      });
+    });
+
     it('should load automatic dependencies first so they have lowest priority', async () => {
       const usCoreDependencyConfig = cloneDeep(minimalConfig);
       usCoreDependencyConfig.dependencies = [{ packageId: 'hl7.fhir.us.core', version: '3.1.0' }];
@@ -918,6 +974,29 @@ describe('Processing', () => {
       });
     });
 
+    it('should log a warning when an NPM alias uses a character not valid for FHIR ids', async () => {
+      // See: https://chat.fhir.org/#narrow/channel/179239-tooling/topic/NPM.20Aliases/near/518051795
+      const npmAliasDependencyConfig = cloneDeep(minimalConfig);
+      npmAliasDependencyConfig.dependencies = [
+        { packageId: 'hl7.fhir.us.mcode', version: '4.0.0' },
+        { packageId: 'hl7.fhir.us.core', version: '7.0.0' },
+        { packageId: 'v610!@npm:hl7.fhir.us.core', version: '6.1.0' },
+        { packageId: 'hl7.fhir.us.qicore', version: '6.0.0' },
+        { packageId: 'v311@npm:hl7.fhir.us.core', version: '3.1.1' }
+      ];
+      const defs = await getTestFHIRDefinitions();
+      return loadExternalDependencies(defs, npmAliasDependencyConfig).then(async () => {
+        const loadedPackages = defs.findPackageInfos('*').map(pkg => `${pkg.name}#${pkg.version}`);
+        expect(loadedPackages).toHaveLength(6 + NUM_R4_AUTO_DEPENDENCIES);
+        // Check that the package w/ the invalid alias was still loaded anyway
+        expect(loadedPackages).toContain('hl7.fhir.us.core#6.1.0');
+        expect(loggerSpy.getLastMessage('warn')).toMatch(
+          /NPM aliases should .+\. Found 'v610!'\./s
+        );
+        expect(loggerSpy.getAllLogs('error')).toHaveLength(0);
+      });
+    });
+
     it('should log an error when it fails to load a dependency', async () => {
       const badDependencyConfig = cloneDeep(minimalConfig);
       badDependencyConfig.dependencies = [{ packageId: 'hl7.does.not.exist', version: 'current' }];