Repository: FHIR/sushi
Pull request #1059 (2022-04-06 22:16:39)

1045 sushi does not install dependencies proxy
- Created axiosUtils.ts and created axiosGet function to address posible proxy
- Changed all axios.get calls to axiosGet within /src
- Wrote tests for axiosGet in axiosUtils.test.ts

To test:
- Make sure you have docker installed
- Set up a proxy server through docker: `docker run --name squid -d -p 3128:3128 datadog/squid`
- Set up environment variables `HTTP_PROXY` and `HTTPS_PROXY` as it was done in the issue.
- Use `env|grep HTTP_PROXY` and `env|grep HTTPS_PROXY` to verify that you're using a proxy
- `curl google.com` just to make sure you're connected
- Delete local .fhir cache
- Run sushi on a local sushi project
- If packages download correctly, it is working correctly. If it displays an error regarding the package download failing, then it is breaking.
- `unset HTTP_PROXY` and `unset HTTPS_PROXY` will let you stop using the proxy. Again you can use `env|grep` to verify the two proxies that are not running anymore
- Turn off the docker container: `docker stop squid`


guhanthuran: Check latest changes, it should handle the new 'options' logic. Please test it with proxy again (with the directions above), since I am having socket issues on my end. 
guhanthuran: Resolved Chris' changes. 

Patch:
diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index 02c57bd..428bab6 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -2288,7 +2288,6 @@
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
       "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
-      "dev": true,
       "requires": {
         "debug": "4"
       }
@@ -2909,7 +2908,6 @@
       "version": "4.3.1",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
       "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
-      "dev": true,
       "requires": {
         "ms": "2.1.2"
       },
@@ -2917,8 +2915,7 @@
         "ms": {
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-          "dev": true
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
         }
       }
     },
@@ -4170,7 +4167,6 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
       "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
-      "dev": true,
       "requires": {
         "agent-base": "6",
         "debug": "4"
diff --git a/package.json b/package.json
index ae24971..5f5a0d7 100644
--- a/package.json
+++ b/package.json
@@ -85,6 +85,7 @@
     "fhir": "^4.9.0",
     "fs-extra": "^8.1.0",
     "html-minifier": "^4.0.0",
+    "https-proxy-agent": "^5.0.0",
     "ini": "^1.3.8",
     "junk": "^3.1.0",
     "lodash": "^4.17.21",
diff --git a/regression/find-repos.ts b/regression/find-repos.ts
index fba881d..df3d0c4 100644
--- a/regression/find-repos.ts
+++ b/regression/find-repos.ts
@@ -2,6 +2,7 @@ import path from 'path';
 import axios from 'axios';
 import fs from 'fs-extra';
 import { remove, uniqBy } from 'lodash';
+import { axiosGet } from '../src/utils/axiosUtils';
 
 const BUILD_URL_RE = /^([^/]+)\/([^/]+)\/branches\/([^/]+)\/qa\.json$/;
 const FSHY_PATHS = ['sushi-config.yaml', 'input/fsh', 'fsh'];
@@ -55,7 +56,7 @@ async function getReposFromGitHub(): Promise<GHRepo[]> {
       if (process.env.GITHUB_API_KEY) {
         options.headers = { Authorization: `token ${process.env.GITHUB_API_KEY}` };
       }
-      const res = await axios.get(
+      const res = await axiosGet(
         `https://api.github.com/orgs/HL7/repos?sort=full_name&per_page=100&page=${page}`,
         options
       );
@@ -89,7 +90,7 @@ async function getNonHL7ReposFromBuild(): Promise<GHRepo[]> {
   console.log('Getting non-HL7 repos from the auto-builder report...');
   const repoToBranches: Map<string, string[]> = new Map();
   // Build up the map
-  const res = await axios.get('https://build.fhir.org/ig/qas.json');
+  const res = await axiosGet('https://build.fhir.org/ig/qas.json');
   if (Array.isArray(res?.data)) {
     res.data.forEach(build => {
       const matches = build.repo?.match(BUILD_URL_RE);
diff --git a/src/fhirdefs/load.ts b/src/fhirdefs/load.ts
index d163c12..1217e28 100644
--- a/src/fhirdefs/load.ts
+++ b/src/fhirdefs/load.ts
@@ -4,12 +4,12 @@ import fs from 'fs-extra';
 import path from 'path';
 import os from 'os';
 import tar from 'tar';
-import axios from 'axios';
 import junk from 'junk';
 import temp from 'temp';
 import { logger, getFilesRecursive } from '../utils';
 import { Fhir as FHIRConverter } from 'fhir/fhir';
 import { ImplementationGuideDefinitionParameter } from '../fhirtypes';
+import { axiosGet } from '../utils/axiosUtils';
 
 /**
  * Loads a dependency from user FHIR cache or from online
@@ -64,7 +64,7 @@ export async function loadDependency(
     // Even if a local current package is loaded, we must still check that the local package date matches
     // the date on the most recent version on build.fhir.org. If the date does not match, we re-download to the cache
     const baseUrl = 'https://build.fhir.org/ig';
-    const res = await axios.get(`${baseUrl}/qas.json`);
+    const res = await axiosGet(`${baseUrl}/qas.json`);
     const qaData: { 'package-id': string; date: string; repo: string }[] = res?.data;
     // Find matching packages and sort by date to get the most recent
     let newestPackage;
@@ -78,7 +78,7 @@ export async function loadDependency(
       const [org, repo] = newestPackage.repo.split('/');
       const igUrl = `${baseUrl}/${org}/${repo}`;
       // get the package.manifest.json for the newest version of the package on build.fhir.org
-      const manifest = await axios.get(`${igUrl}/package.manifest.json`);
+      const manifest = await axiosGet(`${igUrl}/package.manifest.json`);
       let cachedPackageJSON;
       if (fs.existsSync(path.join(loadPath, 'package.json'))) {
         cachedPackageJSON = fs.readJSONSync(path.join(loadPath, 'package.json'));
@@ -119,9 +119,7 @@ export async function loadDependency(
   if (packageUrl) {
     const doDownload = async (url: string) => {
       logger.info(`Downloading ${fullPackageName}...`);
-      const res = await axios.get(url, {
-        responseType: 'arraybuffer'
-      });
+      const res = await axiosGet(url, { responseType: 'arraybuffer' });
       if (res?.data) {
         logger.info(`Downloaded ${fullPackageName}`);
         // Create a temporary file and write the package to there
diff --git a/src/utils/Processing.ts b/src/utils/Processing.ts
index 7948d04..8b8afad 100644
--- a/src/utils/Processing.ts
+++ b/src/utils/Processing.ts
@@ -1,4 +1,3 @@
-import axios from 'axios';
 import path from 'path';
 import fs from 'fs-extra';
 import readlineSync from 'readline-sync';
@@ -17,6 +16,7 @@ import {
 } from '../import';
 import { Package } from '../export';
 import { Configuration } from '../fshtypes';
+import { axiosGet } from './axiosUtils';
 
 const EXT_PKG_TO_FHIR_PKG_MAP: { [key: string]: string } = {
   'hl7.fhir.extensions.r2': 'hl7.fhir.r2.core#1.0.2',
@@ -506,7 +506,7 @@ export async function init(): Promise<void> {
   ]) {
     const url = `http://raw.githubusercontent.com/HL7/ig-publisher-scripts/main/${script}`;
     try {
-      const res = await axios.get(url);
+      const res = await axiosGet(url);
       fs.writeFileSync(path.join(outputDir, script), res.data);
       if (script.endsWith('.sh')) {
         fs.chmodSync(path.join(outputDir, script), 0o755);
@@ -560,7 +560,7 @@ export function getLocalSushiVersion(): string {
 
 export async function getLatestSushiVersion(): Promise<string> {
   try {
-    const res = await axios.get('https://registry.npmjs.org/fsh-sushi');
+    const res = await axiosGet('https://registry.npmjs.org/fsh-sushi');
     const latestVer = res.data['dist-tags'].latest;
     if (latestVer == null) {
       logger.error('Unable to determine the latest version of sushi.');
diff --git a/src/utils/axiosUtils.ts b/src/utils/axiosUtils.ts
new file mode 100644
index 0000000..b5069bc
--- /dev/null
+++ b/src/utils/axiosUtils.ts
@@ -0,0 +1,28 @@
+import axios, { AxiosRequestConfig, AxiosResponse } from 'axios';
+import HttpsProxyAgent from 'https-proxy-agent';
+
+/**
+ * This function is called to better handle axios.get calls with logic that allows
+ * the use of proxies. Not needed in tests unless specifically testing with proxies.
+ * Check https://github.com/axios/axios/issues/3459#issuecomment-766171276 for more info.
+ * @param url {string} - string representation of url to get
+ * @param responseType {any} - optional parameter to change the data type needed from get
+ * (ex. arraybuffer). In default it returns JSON
+ */
+export async function axiosGet(
+  url: string,
+  options?: AxiosRequestConfig
+): Promise<AxiosResponse<any>> {
+  const httpsProxy = process.env.HTTPS_PROXY;
+  const axiosOptions = options ?? {};
+  if (httpsProxy) {
+    // https://github.com/axios/axios/issues/3459
+    axiosOptions.httpsAgent = new (HttpsProxyAgent as any)(httpsProxy);
+    axiosOptions.proxy = false;
+  }
+  if (Object.keys(axiosOptions).length > 0) {
+    return await axios.get(url, axiosOptions);
+  } else {
+    return await axios.get(url);
+  }
+}


Test patch:
diff --git a/test/fhirdefs/load.test.ts b/test/fhirdefs/load.test.ts
index f45d989..7f7633c 100644
--- a/test/fhirdefs/load.test.ts
+++ b/test/fhirdefs/load.test.ts
@@ -205,6 +205,7 @@ describe('#loadDependency()', () => {
     removeSpy = jest.spyOn(fs, 'removeSync').mockImplementation(() => {});
     moveSpy = jest.spyOn(fs, 'moveSync').mockImplementation(() => {});
     cachePath = path.join(__dirname, 'fixtures');
+    delete process.env.HTTPS_PROXY;
   });
 
   beforeEach(() => {
diff --git a/test/utils/axiosUtils.test.ts b/test/utils/axiosUtils.test.ts
new file mode 100644
index 0000000..3f884f8
--- /dev/null
+++ b/test/utils/axiosUtils.test.ts
@@ -0,0 +1,37 @@
+import axios from 'axios';
+import HttpsProxyAgent from 'https-proxy-agent';
+import { axiosGet } from '../../src/utils/axiosUtils';
+
+describe('axiosUtils', () => {
+  beforeAll(() => {
+    delete process.env.HTTPS_PROXY;
+  });
+  afterEach(() => {
+    delete process.env.HTTPS_PROXY;
+  });
+  describe('#axiosGet', () => {
+    it('returns get with responseType assigned when it is used as argument', async () => {
+      const getSpy = jest.spyOn(axios, 'get').mockImplementationOnce(() => Promise.resolve({}));
+      await axiosGet('packages.fhir.org/de.basisprofil.r4/1.1.0', { responseType: 'arraybuffer' });
+      expect(getSpy).toHaveBeenCalledWith('packages.fhir.org/de.basisprofil.r4/1.1.0', {
+        responseType: 'arraybuffer'
+      });
+    });
+
+    it('returns get without responseType assigned by default', async () => {
+      const getSpy = jest.spyOn(axios, 'get').mockImplementationOnce(() => Promise.resolve({}));
+      await axiosGet('packages.fhir.org/de.basisprofil.r4/1.1.0');
+      expect(getSpy).toHaveBeenCalledWith('packages.fhir.org/de.basisprofil.r4/1.1.0');
+    });
+
+    it('calls get with proxy agent when https env variable is set', async () => {
+      const getSpy = jest.spyOn(axios, 'get').mockImplementationOnce(() => Promise.resolve({}));
+      process.env.HTTPS_PROXY = 'https://127.0.0.1:2626';
+      await axiosGet('packages.fhir.org/de.basisprofil.r4/1.1.0');
+      expect(getSpy).toHaveBeenCalledWith('packages.fhir.org/de.basisprofil.r4/1.1.0', {
+        httpsAgent: new (HttpsProxyAgent as any)('https://127.0.0.1:2626'),
+        proxy: false
+      });
+    });
+  });
+});