Repository: FHIR/sushi
Pull request #975 (2021-12-10 13:35:08)

Don't use HTTP HEAD to determine package availability
The packages.fhir.org server no longer allows HTTP HEAD requests.  Since SUSHI used HTTP HEAD to determine if a package was available on packages.fhir.org, this is obviously a problem.

Now we will just attempt the download on packages.fhir.org first and if there is an error, redo it on packages2.fhir.org.  In the future, however, we should use the version-listing capability that is now part of the API ([see zulip](https://chat.fhir.org/#narrow/stream/215610-shorthand/topic/.5BGoFSH.5D.20error.20on.20loading.20IGs/near/264431169)).  But.. this will suffice for now and causes less churn in the code while @jafeltra continues to refactor this code into a new module.

To test, create a SUSHI project with this dependency:
```yaml
dependencies:
  de.gematik.isik-basismodul: 1.0.2
```

On master, it will fail to download it.  On this branch, it will succeed.


Patch:
diff --git a/src/fhirdefs/load.ts b/src/fhirdefs/load.ts
index b061146..d17a443 100644
--- a/src/fhirdefs/load.ts
+++ b/src/fhirdefs/load.ts
@@ -112,49 +112,58 @@ export async function loadDependency(
     }
   } else if (!loadedPackage) {
     packageUrl = `https://packages.fhir.org/${packageName}/${version}`;
-
-    // If the package is not available in packages, then we may need to get it from packages2
-    try {
-      await axios.head(packageUrl);
-    } catch {
-      // It didn't exist in the normal registry.  Fallback to packages2 registry.
-      // See: https://chat.fhir.org/#narrow/stream/179252-IG-creation/topic/Registry.20for.20FHIR.20Core.20packages.20.3E.204.2E0.2E1
-      // See: https://chat.fhir.org/#narrow/stream/179252-IG-creation/topic/fhir.2Edicom/near/262334652
-      packageUrl = `https://packages2.fhir.org/packages/${packageName}/${version}`;
-    }
   }
 
   // If the packageUrl is set, we must download the package from that url, and extract it to our local cache
   if (packageUrl) {
-    logger.info(`Downloading ${fullPackageName}...`);
-    const res = await axios.get(packageUrl, {
-      responseType: 'arraybuffer'
-    });
-    if (res?.data) {
-      logger.info(`Downloaded ${fullPackageName}`);
-      // Create a temporary file and write the package to there
-      temp.track();
-      const tempFile = temp.openSync();
-      fs.writeFileSync(tempFile.path, res.data);
-      // Extract the package to a temporary directory
-      const tempDirectory = temp.mkdirSync();
-      tar.x({
-        cwd: tempDirectory,
-        file: tempFile.path,
-        sync: true,
-        strict: true
+    const doDownload = async (url: string) => {
+      logger.info(`Downloading ${fullPackageName}...`);
+      const res = await axios.get(url, {
+        responseType: 'arraybuffer'
       });
-      cleanCachedPackage(tempDirectory);
-      // Add or replace the package in the FHIR cache
-      const targetDirectory = path.join(cachePath, fullPackageName);
-      if (fs.existsSync(targetDirectory)) {
-        fs.removeSync(targetDirectory);
+      if (res?.data) {
+        logger.info(`Downloaded ${fullPackageName}`);
+        // Create a temporary file and write the package to there
+        temp.track();
+        const tempFile = temp.openSync();
+        fs.writeFileSync(tempFile.path, res.data);
+        // Extract the package to a temporary directory
+        const tempDirectory = temp.mkdirSync();
+        tar.x({
+          cwd: tempDirectory,
+          file: tempFile.path,
+          sync: true,
+          strict: true
+        });
+        cleanCachedPackage(tempDirectory);
+        // Add or replace the package in the FHIR cache
+        const targetDirectory = path.join(cachePath, fullPackageName);
+        if (fs.existsSync(targetDirectory)) {
+          fs.removeSync(targetDirectory);
+        }
+        fs.moveSync(tempDirectory, targetDirectory);
+        // Now try to load again from the path
+        loadedPackage = loadFromPath(cachePath, fullPackageName, FHIRDefs);
+      } else {
+        logger.info(`Unable to download most current version of ${fullPackageName}`);
+      }
+    };
+    try {
+      await doDownload(packageUrl);
+    } catch (e) {
+      if (packageUrl === `https://packages.fhir.org/${packageName}/${version}`) {
+        // It didn't exist in the normal registry.  Fallback to packages2 registry.
+        // See: https://chat.fhir.org/#narrow/stream/179252-IG-creation/topic/Registry.20for.20FHIR.20Core.20packages.20.3E.204.2E0.2E1
+        // See: https://chat.fhir.org/#narrow/stream/179252-IG-creation/topic/fhir.2Edicom/near/262334652
+        packageUrl = `https://packages2.fhir.org/packages/${packageName}/${version}`;
+        try {
+          await doDownload(packageUrl);
+        } catch (e) {
+          throw new PackageLoadError(fullPackageName);
+        }
+      } else {
+        throw new PackageLoadError(fullPackageName);
       }
-      fs.moveSync(tempDirectory, targetDirectory);
-      // Now try to load again from the path
-      loadedPackage = loadFromPath(cachePath, fullPackageName, FHIRDefs);
-    } else {
-      logger.info(`Unable to download most current version of ${fullPackageName}`);
     }
   }
 


Test patch:
diff --git a/test/fhirdefs/load.test.ts b/test/fhirdefs/load.test.ts
index c8e4061..0a56fac 100644
--- a/test/fhirdefs/load.test.ts
+++ b/test/fhirdefs/load.test.ts
@@ -77,15 +77,22 @@ describe('#loadDependency()', () => {
   // Many tests check that the right package was downloaded to the right place.
   // This function encapsulates that testing logic. It's coupled more tightly to
   // the actual implementation than I'd prefer, but... at least it's in one place.
-  const expectDownloadSequence = (source: string, destination: string, isCurrent = false): void => {
+  const expectDownloadSequence = (
+    sources: string | string[],
+    destination: string,
+    isCurrent = false
+  ): void => {
+    if (!Array.isArray(sources)) {
+      sources = [sources];
+    }
     if (isCurrent) {
       expect(axiosSpy.mock.calls).toEqual([
         ['https://build.fhir.org/ig/qas.json'],
-        [source.replace(/package\.tgz$/, 'package.manifest.json')],
-        [source, { responseType: 'arraybuffer' }]
+        [sources[0].replace(/package\.tgz$/, 'package.manifest.json')],
+        [sources[0], { responseType: 'arraybuffer' }]
       ]);
     } else {
-      expect(axiosSpy.mock.calls).toEqual([[source, { responseType: 'arraybuffer' }]]);
+      expect(axiosSpy.mock.calls).toEqual(sources.map(s => [s, { responseType: 'arraybuffer' }]));
     }
     if (destination != null) {
       const tempTarFile = writeSpy.mock.calls[0][0];
@@ -179,12 +186,7 @@ describe('#loadDependency()', () => {
             some: 'zipfile'
           }
         };
-      } else {
-        return {};
-      }
-    });
-    axiosHeadSpy = jest.spyOn(axios, 'head').mockImplementation((uri: string): any => {
-      if (
+      } else if (
         uri === 'https://packages.fhir.org/hl7.fhir.r4b.core/4.1.0' ||
         uri === 'https://packages.fhir.org/hl7.fhir.r5.core/4.5.0' ||
         uri === 'https://packages.fhir.org/fhir.dicom/2021.4.20210910'
@@ -194,6 +196,9 @@ describe('#loadDependency()', () => {
         return {};
       }
     });
+    axiosHeadSpy = jest.spyOn(axios, 'head').mockImplementation((): any => {
+      throw 'Method Not Allowed';
+    });
     tarSpy = jest.spyOn(tar, 'x').mockImplementation(() => {});
     writeSpy = jest.spyOn(fs, 'writeFileSync').mockImplementation(() => {});
     removeSpy = jest.spyOn(fs, 'removeSync').mockImplementation(() => {});
@@ -256,7 +261,10 @@ describe('#loadDependency()', () => {
       'The package hl7.fhir.r4b.core#4.1.0 could not be loaded locally or from the FHIR package registry'
     ); // the package is never actually added to the cache, since tar is mocked
     expectDownloadSequence(
-      'https://packages2.fhir.org/packages/hl7.fhir.r4b.core/4.1.0',
+      [
+        'https://packages.fhir.org/hl7.fhir.r4b.core/4.1.0',
+        'https://packages2.fhir.org/packages/hl7.fhir.r4b.core/4.1.0'
+      ],
       path.join('foo', 'hl7.fhir.r4b.core#4.1.0')
     );
   });
@@ -276,7 +284,10 @@ describe('#loadDependency()', () => {
       'The package hl7.fhir.r5.core#4.5.0 could not be loaded locally or from the FHIR package registry'
     ); // the package is never actually added to the cache, since tar is mocked
     expectDownloadSequence(
-      'https://packages2.fhir.org/packages/hl7.fhir.r5.core/4.5.0',
+      [
+        'https://packages.fhir.org/hl7.fhir.r5.core/4.5.0',
+        'https://packages2.fhir.org/packages/hl7.fhir.r5.core/4.5.0'
+      ],
       path.join('foo', 'hl7.fhir.r5.core#4.5.0')
     );
   });
@@ -286,7 +297,10 @@ describe('#loadDependency()', () => {
       'The package fhir.dicom#2021.4.20210910 could not be loaded locally or from the FHIR package registry'
     ); // the package is never actually added to the cache, since tar is mocked
     expectDownloadSequence(
-      'https://packages2.fhir.org/packages/fhir.dicom/2021.4.20210910',
+      [
+        'https://packages.fhir.org/fhir.dicom/2021.4.20210910',
+        'https://packages2.fhir.org/packages/fhir.dicom/2021.4.20210910'
+      ],
       path.join('foo', 'fhir.dicom#2021.4.20210910')
     );
   });