Repository: wix/Detox
Pull request #2471 (2020-11-18 14:46:49)

Genycloud teardown
## Description
Another (mandatory) step towards support for Genymotion's SaaS solution (see #2429): supporting tear-down.

One of the most important aspects of using cloud emulators is the proper management of their deletion (in particular, because pricing is usage-time based). This change introduces instances deletion that in a Jest-based environment, is based on the global-teardown hook, with the API invocation looking pretty much like this:
```js
// global-teardown.js
async function globalTeardown() {
  const detox = require('detox');
  await detox.globalCleanup();
}
module.exports = globalTeardown;
```
_Namely, such that the internal details of the shutdown are obscure from the Detox user's perspective, and are available through an API that feels natural._

> In Mocha - or any other test runner that does not support parallel-execution, this can be integrated by dispatching `detox.globalCleanup()` immediately after `detox.cleanup()`. In due time, we will add this to our Documentation and scaffolding scripts.

Underneath, what happens in a nutshell is this:
1. A new, custom device-registry is initialized (on top the existing ones), which never clears its content throughout the execution.
2. Each driver spinning up a new cloud instance in it's init, adds the associated `uuid` to the device-registry.
3. Eventually, through global-teardown (at global context), the Genymotion driver statically reads the registry, issues and executes deletion operations in a `Promise.all()` fashion.
4. The end.

> Important: This technique is quite robust and may seem elegant, but it's important to keep in mind that it is nevertheless **not ideal**. Being charged by the minute, ideally, we would like to delete each instance once its worker finishes the last bits of its work. I've experimented ways of implementing this by registering a `process.on('beforeExit', () => {...})`. While it simplified the implementation a great deal, it didn't work, for two reasons:
> 1. Jest seems to hold workers up and running right until the very bitter end (global teardown), so the callback isn't called earlier than with the current approach.
> 2. Jest doesn't approve of asynchronous work inside each worker as it waits for it to be killed: it warns the user with the famous _open handles_ log, and sometimes force-kills before the `beforeExit` callback completes.
As a bottom line, then, in order to introduce the _ideal_ optimization, we would have to push this into Jest itself (see [here](https://github.com/facebook/jest/issues/8708#issuecomment-708492340)).

Last but not least, this also adds support for `driver.shutdown()`, such that genymotion-cloud testing would respect Detox' `--cleanup` argument.

d4vidi: @noomorph please review post-factum, along with #2466. Although I'm still not done with all of the comments from the first PR (#2446), I will get to everything, eventually.

Patch:
diff --git a/detox/src/DetoxExportWrapper.js b/detox/src/DetoxExportWrapper.js
index 066c58d74..0d3e1eac6 100644
--- a/detox/src/DetoxExportWrapper.js
+++ b/detox/src/DetoxExportWrapper.js
@@ -1,4 +1,3 @@
-const _ = require('lodash');
 const funpermaproxy = require('funpermaproxy');
 const Detox = require('./Detox');
 const DetoxConstants = require('./DetoxConstants');
@@ -105,4 +104,14 @@ class DetoxExportWrapper {
   }
 }
 
+DetoxExportWrapper.prototype.globalCleanup = async function() {
+  try {
+    // TODO For the next consumer, need to come up with some kind of infra-code to allow for dynamic registration of cleanup-callbacks.
+    const GenyCloudDriver = require('./devices/drivers/android/genycloud/GenyCloudDriver');
+    await GenyCloudDriver.globalCleanup();
+  } catch (error) {
+    log.warn({ event: 'GLOBAL_CLEANUP' }, 'An error occurred trying to shut down Genymotion-cloud emulator instances!', error);
+  }
+}
+
 module.exports = DetoxExportWrapper;
diff --git a/detox/src/devices/DeviceRegistry.js b/detox/src/devices/DeviceRegistry.js
index 8b34b904a..1aeec9f63 100644
--- a/detox/src/devices/DeviceRegistry.js
+++ b/detox/src/devices/DeviceRegistry.js
@@ -52,6 +52,14 @@ class DeviceRegistry {
     return this._lockfile.read();
   }
 
+  async readRegisteredDevices() {
+    let result;
+    await this._lockfile.exclusively(() => {
+      result = this.getRegisteredDevices();
+    })
+    return result;
+  }
+
   /***
    * @private
    */
diff --git a/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.js b/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.js
index 8195ceed5..4bd081982 100644
--- a/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.js
+++ b/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.js
@@ -3,15 +3,19 @@ const retry = require('../../../../utils/retry');
 const logger = require('../../../../utils/logger').child({ __filename });
 
 class GenyCloudDeviceAllocator extends AndroidDeviceAllocator {
-  constructor(deviceRegistry, instanceLookupService, instanceLifecycleService) {
+  constructor(deviceRegistry, deviceCleanupRegistry, instanceLookupService, instanceLifecycleService) {
     super(deviceRegistry, logger);
 
+    this.deviceCleanupRegistry = deviceCleanupRegistry;
     this.instanceLookupService = instanceLookupService;
     this.instanceLifecycleService = instanceLifecycleService;
   }
 
   async _doAllocateDevice(recipe) {
     let { instance, isNew } = await this._doSynchronizedAllocation(recipe);
+    if (isNew) {
+      await this.deviceCleanupRegistry.allocateDevice(instance.uuid);
+    }
 
     instance = await this._waitForInstanceBoot(instance);
     instance = await this._adbConnectIfNeeded(instance);
@@ -47,7 +51,7 @@ class GenyCloudDeviceAllocator extends AndroidDeviceAllocator {
     }
 
     const options = {
-      backoff: 'none',
+      backoff: 'none', // TODO apply reverse-linear polling
       retries: 18,
       interval: 10000,
     };
diff --git a/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.js b/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.js
index 6de9e956c..1564f84e1 100644
--- a/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.js
+++ b/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.js
@@ -1,6 +1,6 @@
 const AndroidDriver = require('../AndroidDriver');
-const DeviceRegistry = require('../../../DeviceRegistry');
 const GenyCloudDeviceAllocator = require('./GenyCloudDeviceAllocator');
+const GenyDeviceRegistryFactory = require('./GenyDeviceRegistryFactory');
 const GenyCloudExec = require('./exec/GenyCloudExec');
 const RecipesService = require('./services/GenyRecipesService');
 const InstanceLookupService = require('./services/GenyInstanceLookupService');
@@ -17,13 +17,14 @@ class GenyCloudDriver extends AndroidDriver {
 
     const exec = new GenyCloudExec();
     const instanceNaming = new InstanceNaming(); // TODO should consider a permissive impl for debug/dev mode. Maybe even a custom arg in package.json (Detox > ... > genycloud > sharedAccount: false)
-    this._deviceRegistry = DeviceRegistry.forAndroid();
+    this._deviceRegistry = GenyDeviceRegistryFactory.forRuntime();
+    this._deviceCleanupRegistry = GenyDeviceRegistryFactory.forGlobalShutdown();
 
     const recipeService = new RecipesService(exec, logger);
     const instanceLookupService = new InstanceLookupService(exec, instanceNaming, this._deviceRegistry);
-    const instanceLifecycleService = new InstanceLifecycleService(exec, instanceNaming);
+    this._instanceLifecycleService = new InstanceLifecycleService(exec, instanceNaming);
     this._deviceQueryHelper = new DeviceQueryHelper(recipeService);
-    this._deviceAllocator = new GenyCloudDeviceAllocator(this._deviceRegistry, instanceLookupService, instanceLifecycleService);
+    this._deviceAllocator = new GenyCloudDeviceAllocator(this._deviceRegistry, this._deviceCleanupRegistry, instanceLookupService, this._instanceLifecycleService);
   }
 
   get name() {
@@ -58,6 +59,13 @@ class GenyCloudDriver extends AndroidDriver {
     await super.cleanup(instance, bundleId);
   }
 
+  async shutdown(instance) {
+    await this.emitter.emit('beforeShutdownDevice', { deviceId: instance.adbName });
+    await this._instanceLifecycleService.deleteInstance(instance.uuid);
+    await this._deviceCleanupRegistry.disposeDevice(instance.uuid);
+    await this.emitter.emit('shutdownDevice', { deviceId: instance.adbName });
+  }
+
   _assertRecipe(deviceQuery, recipe) {
     if (!recipe) {
       throw new DetoxRuntimeError({
@@ -66,6 +74,47 @@ class GenyCloudDriver extends AndroidDriver {
       });
     }
   }
+
+  static async globalCleanup(instanceLifecycleService) {
+    if (!instanceLifecycleService) {
+      instanceLifecycleService = new InstanceLifecycleService(new GenyCloudExec(), null);
+    }
+
+    const deviceCleanupRegistry = GenyDeviceRegistryFactory.forGlobalShutdown();
+    const deviceUUIDs = await deviceCleanupRegistry.readRegisteredDevices();
+    if (deviceUUIDs.length) {
+      await doCleanup(instanceLifecycleService, deviceUUIDs)
+    }
+  }
+}
+
+const cleanupLogData = {
+  event: 'GENYCLOUD_TEARDOWN',
+};
+
+async function doCleanup(instanceLifecycleService, deviceUUIDs) {
+  logger.info(cleanupLogData, 'Initiating Genymotion cloud instances teardown...');
+
+  const deletionLeaks = [];
+  const killPromises = deviceUUIDs.map((uuid) =>
+    instanceLifecycleService.deleteInstance(uuid)
+      .catch((error) => deletionLeaks.push({ uuid, error })));
+
+  await Promise.all(killPromises);
+  reportGlobalCleanupSummary(deletionLeaks);
+}
+
+function reportGlobalCleanupSummary(deletionLeaks) {
+  if (deletionLeaks.length) {
+    logger.warn(cleanupLogData, 'WARNING! Detected a Genymotion cloud instance leakage, for the following instances:');
+
+    deletionLeaks.forEach(({ uuid, error }) =>
+      logger.warn(cleanupLogData, `Instance UUID ${uuid}: ${error}`));
+
+    logger.info(cleanupLogData, 'Instances teardown completed with warnings');
+  } else {
+    logger.info(cleanupLogData, 'Instances teardown completed successfully');
+  }
 }
 
 module.exports = GenyCloudDriver;
diff --git a/detox/src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory.js b/detox/src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory.js
new file mode 100644
index 000000000..ef1ca2b56
--- /dev/null
+++ b/detox/src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory.js
@@ -0,0 +1,16 @@
+const DeviceRegistry = require('../../../DeviceRegistry');
+const environment = require('../../../../utils/environment');
+
+class GenyDeviceRegistryFactory {
+  forRuntime() {
+    return DeviceRegistry.forAndroid();
+  }
+
+  forGlobalShutdown() {
+    return new DeviceRegistry({
+      lockfilePath: environment.getGenyCloudGlobalCleanupFilePath(),
+    });
+  }
+}
+
+module.exports = new GenyDeviceRegistryFactory();
diff --git a/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.js b/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.js
index 1755df387..691392388 100644
--- a/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.js
+++ b/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.js
@@ -18,20 +18,24 @@ class GenyCloudExec {
   }
 
   stopInstance(instanceUUID) {
-    return this._exec(`instances stop ${instanceUUID}`);
+    const options = {
+      retries: 3,
+    };
+    return this._exec(`instances stop ${instanceUUID}`, options);
   }
 
-  async _exec(args) {
+  async _exec(args, options) {
     try {
-      const rawResult = await this.__exec(args);
+      const rawResult = await this.__exec(args, options);
       return JSON.parse(rawResult);
     } catch (error) {
-      throw JSON.parse(error.stderr);
+      throw new Error(error.stderr);
     }
   }
 
-  async __exec(args) {
+  async __exec(args, _options) {
     const options = {
+      ..._options,
       statusLogs: {
         retrying: true,
       },
diff --git a/detox/src/devices/drivers/android/genycloud/services/GenyInstanceLookupService.js b/detox/src/devices/drivers/android/genycloud/services/GenyInstanceLookupService.js
index 35f6f75c4..4d2816035 100644
--- a/detox/src/devices/drivers/android/genycloud/services/GenyInstanceLookupService.js
+++ b/detox/src/devices/drivers/android/genycloud/services/GenyInstanceLookupService.js
@@ -2,10 +2,10 @@ const _ = require('lodash');
 const Instance = require('./dto/GenyInstance');
 
 class GenyInstanceLookupService {
-  constructor(genyCloudExec, instanceNaming, genycloudDeviceRegistry) {
+  constructor(genyCloudExec, instanceNaming, genyCloudDeviceRegistry) {
     this.genyCloudExec = genyCloudExec;
     this.instanceNaming = instanceNaming;
-    this.deviceRegistry = genycloudDeviceRegistry;
+    this.deviceRegistry = genyCloudDeviceRegistry;
   }
 
   async findFreeInstance() {
diff --git a/detox/src/utils/environment.js b/detox/src/utils/environment.js
index 3122a7e1f..c3ae9b765 100644
--- a/detox/src/utils/environment.js
+++ b/detox/src/utils/environment.js
@@ -17,6 +17,7 @@ const MISSING_SDK_ERROR = `$ANDROID_SDK_ROOT is not defined, set the path to the
 Go to https://developer.android.com/studio/command-line/variables.html for more details`;
 const DEVICE_LOCK_FILE_PATH_IOS = path.join(DETOX_LIBRARY_ROOT_PATH, 'device.registry.state.lock');
 const DEVICE_LOCK_FILE_PATH_ANDROID = path.join(DETOX_LIBRARY_ROOT_PATH, 'android-device.registry.state.lock');
+const GENYCLOUD_GLOBAL_CLEANUP_FILE_PATH = path.join(DETOX_LIBRARY_ROOT_PATH, 'genycloud-cleanup.lock');
 const LAST_FAILED_TESTS_PATH = path.join(DETOX_LIBRARY_ROOT_PATH, 'last-failed.txt');
 
 function getAndroidSDKPath() {
@@ -162,10 +163,15 @@ function getDeviceLockFilePathIOS() {
   return DEVICE_LOCK_FILE_PATH_IOS;
 }
 
+// TODO This can probably be merged with IOS' by now
 function getDeviceLockFilePathAndroid() {
   return DEVICE_LOCK_FILE_PATH_ANDROID;
 }
 
+function getGenyCloudGlobalCleanupFilePath() {
+  return GENYCLOUD_GLOBAL_CLEANUP_FILE_PATH;
+}
+
 function getLastFailedTestsPath() {
   return LAST_FAILED_TESTS_PATH;
 }
@@ -188,6 +194,7 @@ module.exports = {
   getDetoxLibraryRootPath,
   getDeviceLockFilePathIOS,
   getDeviceLockFilePathAndroid,
+  getGenyCloudGlobalCleanupFilePath,
   getLastFailedTestsPath,
   getHomeDir,
 };


Test patch:
diff --git a/detox/local-cli/test.js b/detox/local-cli/test.js
index 4383e6541..d95e5a799 100644
--- a/detox/local-cli/test.js
+++ b/detox/local-cli/test.js
@@ -6,6 +6,7 @@ const { parse, quote } = require('./utils/shellQuote');
 const splitArgv = require('./utils/splitArgv');
 const DetoxRuntimeError = require('../src/errors/DetoxRuntimeError');
 const DeviceRegistry = require('../src/devices/DeviceRegistry');
+const GenyDeviceRegistryFactory = require('../src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory');
 const { loadLastFailedTests, resetLastFailedTests } = require('../src/utils/lastFailedTests');
 const { composeDetoxConfig } = require('../src/configuration');
 const log = require('../src/utils/logger').child({ __filename });
@@ -196,6 +197,7 @@ async function resetLockFile({ platform }) {
 
   if (platform === 'android') {
     await DeviceRegistry.forAndroid().reset();
+    await GenyDeviceRegistryFactory.forGlobalShutdown().reset();
   }
 }
 
diff --git a/detox/local-cli/test.test.js b/detox/local-cli/test.test.js
index 729f8ce84..4b1405b59 100644
--- a/detox/local-cli/test.test.js
+++ b/detox/local-cli/test.test.js
@@ -1,6 +1,7 @@
 jest.mock('child_process');
 jest.mock('../src/utils/logger');
 jest.mock('../src/devices/DeviceRegistry');
+jest.mock('../src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory');
 jest.mock('../src/utils/lastFailedTests');
 
 const _ = require('lodash');
@@ -14,6 +15,7 @@ describe('CLI', () => {
   let detoxConfig;
   let detoxConfigPath;
   let DeviceRegistry;
+  let GenyDeviceRegistryFactory;
   let _env;
 
   beforeEach(() => {
@@ -32,8 +34,11 @@ describe('CLI', () => {
     cp = require('child_process');
     logger = require('../src/utils/logger');
     temporaryFiles = [];
-    DeviceRegistry = require('../src/devices/DeviceRegistry')
-    DeviceRegistry.forAndroid = DeviceRegistry.forIOS = () => new DeviceRegistry();
+    DeviceRegistry = require('../src/devices/DeviceRegistry');
+    DeviceRegistry.forAndroid.mockImplementation(() => new DeviceRegistry());
+    DeviceRegistry.forIOS.mockImplementation(() => new DeviceRegistry());
+    GenyDeviceRegistryFactory = require('../src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory');
+    GenyDeviceRegistryFactory.forGlobalShutdown.mockImplementation(() => new DeviceRegistry());
   });
 
   afterEach(async () => {
diff --git a/detox/src/devices/DeviceRegistry.test.js b/detox/src/devices/DeviceRegistry.test.js
index 399446b57..e2857ed90 100644
--- a/detox/src/devices/DeviceRegistry.test.js
+++ b/detox/src/devices/DeviceRegistry.test.js
@@ -37,6 +37,10 @@ describe('DeviceRegistry', () => {
       });
     }
 
+    async function disposeDevice(deviceId) {
+      return registry.disposeDevice(() => deviceId);
+    }
+
     async function checkDeviceNotRegistered(deviceId) {
       return registry.allocateDevice(async () => {
         expect(registry.includes(deviceId)).toBe(false);
@@ -44,6 +48,10 @@ describe('DeviceRegistry', () => {
       }).catch((e) => { if (e.message !== 'ignored') throw e });
     }
 
+    async function checkRegisteredDevicesEqual(...deviceIds) {
+      expect(await registry.readRegisteredDevices()).toEqual([ ...deviceIds ]);
+    }
+
     const assertForbiddenOutOfContextRegistryQuery = () =>
       expect(() => registry.includes('whatever')).toThrowError();
 
@@ -87,6 +95,17 @@ describe('DeviceRegistry', () => {
       assertForbiddenOutOfContextDeviceListQuery();
     });
 
+    it('should be able to read a valid list of registered devices', async () => {
+      const deviceId = 'emulator-5554';
+      const anotherDeviceId = 'emulator-5556';
+
+      await allocateDevice(deviceId);
+      await allocateDevice(anotherDeviceId);
+      await checkRegisteredDevicesEqual(deviceId, anotherDeviceId);
+      await disposeDevice(deviceId);
+      await checkRegisteredDevicesEqual(anotherDeviceId);
+    });
+
     describe('.reset() method', () => {
       it('should create a lock file with an empty array if it does not exist', async () => {
         expect(await fs.exists(lockfilePath)).toBe(false);
diff --git a/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.test.js b/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.test.js
index 5bb7276a9..5ba399705 100644
--- a/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.test.js
+++ b/detox/src/devices/drivers/android/genycloud/GenyCloudDeviceAllocator.test.js
@@ -4,6 +4,7 @@ describe('Genymotion-Cloud device allocator', () => {
   let logger;
   let retry;
   let deviceRegistry;
+  let deviceCleanupRegistry;
   let instanceLookupService;
   let instanceLifecycleService;
   let GenyInstance;
@@ -19,6 +20,7 @@ describe('Genymotion-Cloud device allocator', () => {
     const DeviceRegistry = jest.genMockFromModule('../../../../devices/DeviceRegistry');
     deviceRegistry = new DeviceRegistry();
     deviceRegistry.allocateDevice.mockImplementation((func) => func());
+    deviceCleanupRegistry = new DeviceRegistry();
 
     const InstanceLookupService = jest.genMockFromModule('./services/GenyInstanceLookupService');
     instanceLookupService = new InstanceLookupService();
@@ -29,7 +31,7 @@ describe('Genymotion-Cloud device allocator', () => {
     GenyInstance = jest.genMockFromModule('./services/dto/GenyInstance');
 
     const DeviceAllocator = require('./GenyCloudDeviceAllocator');
-    uut = new DeviceAllocator(deviceRegistry, instanceLookupService, instanceLifecycleService);
+    uut = new DeviceAllocator(deviceRegistry, deviceCleanupRegistry, instanceLookupService, instanceLifecycleService);
   });
 
   const aRecipe = () => ({
diff --git a/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.test.js b/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.test.js
index 65b1cb9d3..5a7241de7 100644
--- a/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.test.js
+++ b/detox/src/devices/drivers/android/genycloud/GenyCloudDriver.test.js
@@ -11,12 +11,12 @@ const mockBaseClassesDependencies = () => {
   jest.mock('../tools/MonitoredInstrumentation');
   jest.mock('../../../../artifacts/utils/AndroidDevicePathBuilder');
   jest.mock('../../../../android/espressoapi/UiDeviceProxy');
-  jest.mock('../../../DeviceRegistry');
   jest.mock('../../../../utils/logger');
 };
 
 const mockDirectDependencies = () => {
   jest.mock('./exec/GenyCloudExec');
+  jest.mock('./GenyDeviceRegistryFactory');
   jest.mock('./services/GenyRecipesService');
   jest.mock('./services/GenyInstanceLookupService');
   jest.mock('./services/GenyInstanceLifecycleService');
@@ -43,14 +43,19 @@ describe('Genymotion-cloud driver', () => {
   beforeEach(mockBaseClassesDependencies);
   beforeEach(mockDirectDependencies);
 
+  let logger;
   let emitter;
   let adb;
   let deviceQueryHelper;
   let deviceRegistry;
+  let deviceCleanupRegistry;
   let deviceAllocator;
+  let InstanceLifecycleService;
   let GenyCloudDriver;
   let uut;
   beforeEach(() => {
+    logger = require('../../../../utils/logger');
+
     const Emitter = jest.genMockFromModule('../../../../utils/AsyncEmitter');
     emitter = new Emitter();
 
@@ -60,10 +65,14 @@ describe('Genymotion-cloud driver', () => {
     const ADB = require('../exec/ADB');
     adb = () => latestInstanceOf(ADB);
 
-    const DeviceRegistry = require('../../../DeviceRegistry');
+    const GenyDeviceRegistryFactory = require('./GenyDeviceRegistryFactory');
+    const DeviceRegistry = jest.genMockFromModule('../../../DeviceRegistry');
     deviceRegistry = new DeviceRegistry();
     deviceRegistry.allocateDevice.mockImplementation((doAllocateFn) => doAllocateFn());
-    DeviceRegistry.forAndroid.mockReturnValue(deviceRegistry);
+    GenyDeviceRegistryFactory.forRuntime.mockReturnValue(deviceRegistry);
+
+    deviceCleanupRegistry = new DeviceRegistry();
+    GenyDeviceRegistryFactory.forGlobalShutdown.mockReturnValue(deviceCleanupRegistry);
 
     jest.mock('./helpers/GenyDeviceQueryHelper');
     const DeviceQueryHelper = require('./helpers/GenyDeviceQueryHelper');
@@ -73,6 +82,8 @@ describe('Genymotion-cloud driver', () => {
     const DeviceAllocator = require('./GenyCloudDeviceAllocator');
     deviceAllocator = () => latestInstanceOf(DeviceAllocator);
 
+    InstanceLifecycleService = require('./services/GenyInstanceLifecycleService');
+
     GenyCloudDriver = require('./GenyCloudDriver');
     uut = new GenyCloudDriver({
       invocationManager,
@@ -211,7 +222,7 @@ describe('Genymotion-cloud driver', () => {
     });
   });
 
-  describe('Cleanup', () => {
+  describe('clean-up', () => {
     const instrumentationObj = () => latestInstanceOf(Instrumentation);
 
     let Instrumentation;
@@ -230,4 +241,118 @@ describe('Genymotion-cloud driver', () => {
       expect(instrumentationObj().terminate).toHaveBeenCalled();
     });
   });
+
+  describe('shutdown', () => {
+    const instanceLifecycleServiceObj = () => latestInstanceOf(InstanceLifecycleService);
+
+    const givenAnInstanceDeletionError = () => instanceLifecycleServiceObj().deleteInstance.mockRejectedValue(new Error());
+
+    it('should delete the instance', async () => {
+      const instance = anInstance();
+      await uut.shutdown(instance);
+      expect(instanceLifecycleServiceObj().deleteInstance).toHaveBeenCalledWith(instance.uuid);
+    });
+
+    it('should throw if deletion fails', async () => {
+      givenAnInstanceDeletionError();
+
+      try {
+        await uut.shutdown(anInstance());
+        fail('Expected an error');
+      } catch (e) {}
+    });
+
+    it('should emit associated events', async () => {
+      const instance = anInstance();
+      await uut.shutdown(instance);
+      expect(emitter.emit).toHaveBeenCalledWith('beforeShutdownDevice', { deviceId: instance.adbName });
+      expect(emitter.emit).toHaveBeenCalledWith('shutdownDevice', { deviceId: instance.adbName });
+    });
+
+    it('should not emit shutdown even in case of failure', async () => {
+      givenAnInstanceDeletionError();
+
+      try {
+        await uut.shutdown(anInstance());
+        fail('Expected an error');
+      } catch (e) {
+        expect(emitter.emit).toHaveBeenCalledWith('beforeShutdownDevice', expect.anything());
+        expect(emitter.emit).not.toHaveBeenCalledWith('shutdownDevice', expect.anything());
+      }
+    });
+
+    it('should remove instance from device-registry', async () => {
+      const instance = anInstance();
+      await uut.shutdown(instance);
+      expect(deviceCleanupRegistry.disposeDevice).toHaveBeenCalledWith(instance.uuid);
+    });
+  });
+
+  describe('global (static) clean-up', () => {
+    const anInstanceLifecycleService = () => {
+      const InstanceLifecycleService = require('./services/GenyInstanceLifecycleService');
+      return new InstanceLifecycleService();
+    }
+
+    const assertablePendingPromise = () => {
+      let promiseAck = jest.fn();
+      const promise = new Promise(resolve => setTimeout(resolve, 1)).then(promiseAck);
+      promise.assertResolved = () => expect(promiseAck).toHaveBeenCalled();
+      return promise;
+    };
+
+    it('should kill all devices preregistered for clean-up', async () => {
+      const killPromise1 = assertablePendingPromise();
+      const killPromise2 = assertablePendingPromise();
+
+      const instanceLifecycleService = anInstanceLifecycleService();
+      instanceLifecycleService.deleteInstance
+        .mockReturnValueOnce(killPromise1)
+        .mockReturnValueOnce(killPromise2);
+
+      const deviceUUIDs = ['device1-uuid', 'device2-uuid'];
+      deviceCleanupRegistry.readRegisteredDevices.mockResolvedValue(deviceUUIDs);
+
+      await GenyCloudDriver.globalCleanup(instanceLifecycleService);
+
+      killPromise1.assertResolved();
+      killPromise2.assertResolved();
+      expect(instanceLifecycleService.deleteInstance).toHaveBeenCalledWith(deviceUUIDs[0]);
+      expect(instanceLifecycleService.deleteInstance).toHaveBeenCalledWith(deviceUUIDs[1]);
+    });
+
+    it('should fallback to a default lifecycle-service', async () => {
+      deviceCleanupRegistry.readRegisteredDevices.mockResolvedValue([]);
+      await GenyCloudDriver.globalCleanup();
+    });
+
+    it('should warn of instances cleanup rejects', async () => {
+      const instanceLifecycleService = anInstanceLifecycleService();
+      instanceLifecycleService.deleteInstance
+        .mockRejectedValueOnce(new Error('mock-error1'))
+        .mockResolvedValueOnce(anInstance())
+        .mockRejectedValueOnce(new Error('mock-error2'));
+
+      const deviceUUIDs = ['failing-uuid1', 'nonfailing-uuid', 'failing-uuid2'];
+      deviceCleanupRegistry.readRegisteredDevices.mockResolvedValue(deviceUUIDs);
+
+      await GenyCloudDriver.globalCleanup(instanceLifecycleService);
+
+      expect(logger.warn).toHaveBeenCalledWith({ event: 'GENYCLOUD_TEARDOWN' }, 'WARNING! Detected a Genymotion cloud instance leakage, for the following instances:');
+      expect(logger.warn).toHaveBeenCalledWith({ event: 'GENYCLOUD_TEARDOWN' }, expect.stringMatching(/failing-uuid1:.*mock-error1/));
+      expect(logger.warn).toHaveBeenCalledWith({ event: 'GENYCLOUD_TEARDOWN' }, expect.stringMatching(/failing-uuid2:.*mock-error2/));
+    });
+
+    it('should not warn of cleanup rejects if all went well', async () => {
+      const instanceLifecycleService = anInstanceLifecycleService();
+      instanceLifecycleService.deleteInstance.mockResolvedValue(anInstance);
+
+      const deviceUUIDs = ['device-uuid1'];
+      deviceCleanupRegistry.readRegisteredDevices.mockResolvedValue(deviceUUIDs);
+
+      await GenyCloudDriver.globalCleanup(instanceLifecycleService);
+
+      expect(logger.warn).not.toHaveBeenCalled();
+    });
+  });
 });
diff --git a/detox/src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory.test.js b/detox/src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory.test.js
new file mode 100644
index 000000000..3c5d10850
--- /dev/null
+++ b/detox/src/devices/drivers/android/genycloud/GenyDeviceRegistryFactory.test.js
@@ -0,0 +1,33 @@
+const _ = require('lodash');
+const environment = require('../../../../utils/environment');
+
+const latestInstanceOf = (clazz) => _.last(clazz.mock.instances);
+
+describe('Genymotion-cloud device-registries factory', () => {
+  let DeviceRegistry;
+  let GenyDeviceRegistryFactory;
+  beforeEach(() => {
+    jest.mock('../../../DeviceRegistry');
+    DeviceRegistry = require('../../../DeviceRegistry');
+
+    GenyDeviceRegistryFactory = require('./GenyDeviceRegistryFactory');
+  });
+
+  it('should expose method for runtime registry creation', () => {
+    const androidDeviceRegistry = new DeviceRegistry();
+    DeviceRegistry.forAndroid.mockReturnValue(androidDeviceRegistry);
+
+    const result = GenyDeviceRegistryFactory.forRuntime();
+    expect(result).toEqual(androidDeviceRegistry);
+  });
+
+  it('should expose method for genymotion clean-up registry creation', () => {
+    const result = GenyDeviceRegistryFactory.forGlobalShutdown();
+    expect(DeviceRegistry).toHaveBeenCalledWith({
+      lockfilePath: environment.getGenyCloudGlobalCleanupFilePath(),
+    });
+    expect(result).toBeInstanceOf(DeviceRegistry);
+    expect(result).toEqual(latestInstanceOf(DeviceRegistry));
+  });
+
+});
diff --git a/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.test.js b/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.test.js
index 980944bb1..2de56e35f 100644
--- a/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.test.js
+++ b/detox/src/devices/drivers/android/genycloud/exec/GenyCloudExec.test.js
@@ -62,6 +62,7 @@ describe('Genymotion-cloud executable', () => {
       commandName: 'Stop Instance',
       commandExecFn: () => uut.stopInstance(instanceUUID),
       expectedExec: `"gmsaas" --format compactjson instances stop ${instanceUUID}`,
+      expectedExecOptions: { retries: 3 },
     },
   ].forEach((testCase) => {
     describe(`${testCase.commandName} command`, () => {
@@ -69,6 +70,7 @@ describe('Genymotion-cloud executable', () => {
         givenSuccessResult();
 
         const expectedOptions = {
+          ...testCase.expectedExecOptions,
           statusLogs: {
             retrying: true,
           }
@@ -95,7 +97,7 @@ describe('Genymotion-cloud executable', () => {
           await testCase.commandExecFn();
           fail('Expected an error');
         } catch (e) {
-          expect(e).toEqual(failResponse);
+          expect(e.message).toEqual(JSON.stringify(failResponse));
         }
       });
     });
diff --git a/detox/src/index.test.js b/detox/src/index.test.js
index 6c5d11342..d21101498 100644
--- a/detox/src/index.test.js
+++ b/detox/src/index.test.js
@@ -159,6 +159,32 @@ describe('index (regular)', () => {
         expect(Detox.none.initContext).not.toHaveBeenCalled();
       });
     });
+
+    describe('global API', () => {
+
+      let GenyCloudDriver;
+      beforeEach(() => {
+        jest.mock('./devices/drivers/android/genycloud/GenyCloudDriver');
+        GenyCloudDriver = require('./devices/drivers/android/genycloud/GenyCloudDriver');
+      });
+
+      it('should invoke genymotion-cloud\'s global cleanup API', async () => {
+        await detox.globalCleanup();
+        expect(GenyCloudDriver.globalCleanup).toHaveBeenCalled();
+      });
+
+      it('should catch and warn errors from genymotion-cloud driver', async () => {
+        const error = new Error('mocked-error');
+        GenyCloudDriver.globalCleanup.mockRejectedValue(error);
+
+        await detox.globalCleanup();
+        expect(logger.warn).toHaveBeenCalledWith(
+          { event: 'GLOBAL_CLEANUP' },
+          'An error occurred trying to shut down Genymotion-cloud emulator instances!',
+          error,
+        );
+      });
+    });
   });
 
   describe('detox.cleanup()', () => {
diff --git a/detox/test/e2e/config.js b/detox/test/e2e/config.js
index f6c9b861d..c8ab9d2ff 100644
--- a/detox/test/e2e/config.js
+++ b/detox/test/e2e/config.js
@@ -4,6 +4,7 @@ module.exports = {
   "testRunner": "./test/node_modules/jest-circus/runner",
   "setupFilesAfterEnv": [],
   "globalSetup": "./test/e2e/global-setup.js",
+  "globalTeardown": "./test/e2e/global-teardown.js",
   "testTimeout": 120000,
   "reporters": process.env.DISABLE_JUNIT_REPORTER === '1'
     ? ["<rootDir>/runners/jest/streamlineReporter"]
diff --git a/detox/test/e2e/global-teardown.js b/detox/test/e2e/global-teardown.js
new file mode 100644
index 000000000..c2625cf2f
--- /dev/null
+++ b/detox/test/e2e/global-teardown.js
@@ -0,0 +1,7 @@
+const detox = require('detox');
+
+async function globalTeardown() {
+  await detox.globalCleanup();
+}
+
+module.exports = globalTeardown;