Repository: openvstorage/framework

Issue:
Delete snapshots - aborts completely on error
The scheduled delete snapshots aborts the complete job when a single error is encountered.

Seeing it happen frequently on larger environments when scrubbings takes a while (the error thrown by the delete snapshot when the scrubber is running, aborts the whole job)



Discussion:

Resolving patch:
diff --git a/ovs/constants/vdisk.py b/ovs/constants/vdisk.py
--- a/ovs/constants/vdisk.py
+++ b/ovs/constants/vdisk.py
@@ -23,3 +23,4 @@
 
 # Scrub related
 SCRUB_VDISK_LOCK = '{0}_{{0}}'.format(LOCK_NAMESPACE)  # Second format is the vdisk guid
+SCRUB_VDISK_EXCEPTION_MESSAGE = 'VDisk is being scrubbed. Unable to remove snapshots at this time'
diff --git a/ovs/lib/generic.py b/ovs/lib/generic.py
--- a/ovs/lib/generic.py
+++ b/ovs/lib/generic.py
@@ -23,6 +23,7 @@
 from datetime import datetime, timedelta
 from threading import Thread
 from time import mktime
+from ovs.constants.vdisk import SCRUB_VDISK_EXCEPTION_MESSAGE
 from ovs.dal.hybrids.servicetype import ServiceType
 from ovs.dal.lists.servicelist import ServiceList
 from ovs.dal.lists.storagerouterlist import StorageRouterList
@@ -180,12 +181,27 @@ def make_timestamp(offset):
                     bucket['snapshots'] = [s for s in bucket['snapshots'] if
                                            s['timestamp'] != oldest['timestamp']]
 
+        exceptions = []
         # Delete obsolete snapshots
         for bucket_chain in bucket_chains:
-            for bucket in bucket_chain:
-                for snapshot in bucket['snapshots']:
-                    VDiskController.delete_snapshot(vdisk_guid=snapshot['vdisk_guid'],
-                                                    snapshot_id=snapshot['snapshot_id'])
+            # Each bucket chain represents one vdisk's snapshots
+            try:
+                for bucket in bucket_chain:
+                    for snapshot in bucket['snapshots']:
+                        VDiskController.delete_snapshot(vdisk_guid=snapshot['vdisk_guid'],
+                                                        snapshot_id=snapshot['snapshot_id'])
+            except RuntimeError as ex:
+                vdisk_guid = next((snapshot['vdisk_guid'] for bucket in bucket_chain for snapshot in bucket['snapshots']), '')
+                vdisk_id_log = ''
+                if vdisk_guid:
+                    vdisk_id_log = ' for VDisk with guid {}'.format(vdisk_guid)
+                if SCRUB_VDISK_EXCEPTION_MESSAGE in ex.message:
+                    GenericController._logger.warning('Being scrubbed exception occurred while deleting snapshots{}'.format(vdisk_id_log))
+                else:
+                    GenericController._logger.exception('Exception occurred while deleting snapshots{}'.format(vdisk_id_log))
+                    exceptions.append(ex)
+        if exceptions:
+            raise RuntimeError('Exceptions occurred while deleting snapshots: \n- {}'.format('\n- '.join((str(ex) for ex in exceptions))))
         GenericController._logger.info('Delete snapshots finished')
 
     @staticmethod
diff --git a/ovs/lib/vdisk.py b/ovs/lib/vdisk.py
--- a/ovs/lib/vdisk.py
+++ b/ovs/lib/vdisk.py
@@ -25,6 +25,7 @@
 import uuid
 import pickle
 import random
+from ovs.constants.vdisk import SCRUB_VDISK_EXCEPTION_MESSAGE
 from ovs.dal.exceptions import ObjectNotFoundException
 from ovs.dal.hybrids.domain import Domain
 from ovs.dal.hybrids.j_vdiskdomain import VDiskDomain
@@ -566,7 +567,7 @@ def delete_snapshots(snapshot_mapping):
                     results[vdisk_guid] = [False, ex.message]
                 continue
             if vdisk.being_scrubbed:
-                msg = 'VDisk is being scrubbed. Unable to remove snapshots at this time'
+                msg = SCRUB_VDISK_EXCEPTION_MESSAGE
                 results[vdisk_guid].update({'success': False,
                                             'error': msg})
                 if backwards_compat is True:


Test patch:
diff --git a/ovs/extensions/storageserver/tests/mockups.py b/ovs/extensions/storageserver/tests/mockups.py
--- a/ovs/extensions/storageserver/tests/mockups.py
+++ b/ovs/extensions/storageserver/tests/mockups.py
@@ -84,6 +84,7 @@ class StorageRouterClient(object):
     _dtl_config_cache = {}
     _metadata_backend_config = {}
     _snapshots = {}
+    delete_snapshot_callbacks = {}
     volumes = {}
     exceptions = {}
     vrouter_id = {}
@@ -201,6 +202,9 @@ def delete_snapshot(self, volume_id, snapshot_id, req_timeout_secs=None):
         Delete snapshot mockup
         """
         _ = req_timeout_secs
+        callback = self.delete_snapshot_callbacks.get(volume_id, {}).get(snapshot_id)
+        if callback:
+            callback()
         del StorageRouterClient._snapshots[self.vpool_guid][volume_id][snapshot_id]
 
     def empty_info(self, req_timeout_secs=None):
diff --git a/ovs/lib/tests/generic_tests/test_snapshot.py b/ovs/lib/tests/generic_tests/test_snapshot.py
--- a/ovs/lib/tests/generic_tests/test_snapshot.py
+++ b/ovs/lib/tests/generic_tests/test_snapshot.py
@@ -21,9 +21,11 @@
 import time
 import datetime
 import unittest
+from ovs.constants.vdisk import SCRUB_VDISK_EXCEPTION_MESSAGE
 from ovs.dal.tests.helpers import DalHelper
 from ovs.lib.generic import GenericController
 from ovs.lib.vdisk import VDiskController
+from ovs.extensions.storageserver.tests.mockups import StorageRouterClient
 
 
 class SnapshotTestCase(unittest.TestCase):
@@ -35,6 +37,7 @@ def setUp(self):
         """
         (Re)Sets the stores on every test
         """
+        StorageRouterClient.delete_snapshot_callbacks = {}
         self.volatile, self.persistent = DalHelper.setup()
 
     def tearDown(self):
@@ -449,6 +452,66 @@ def test_happypath(self):
                                                               'is_consistent': True,
                                                               'timestamp': str(ts)})
 
+    def test_exception_handling(self):
+        """
+        Test if the scheduled job can handle exceptions
+        """
+        def raise_an_exception(*args, **kwargs):
+            raise RuntimeError('Emulated snapshot delete error')
+
+        structure = DalHelper.build_dal_structure(
+            {'vpools': [1],
+             'vdisks': [(1, 1, 1, 1), (2, 1, 1, 1)],  # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
+             'mds_services': [(1, 1)],
+             'storagerouters': [1],
+             'storagedrivers': [(1, 1, 1)]}  # (<id>, <vpool_id>, <storagerouter_id>)
+        )
+
+        vdisk_1, vdisk_2 = structure['vdisks'].values()
+        vdisks = [vdisk_1, vdisk_2]
+
+        for vdisk in vdisks:
+            [dynamic for dynamic in vdisk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
+            for i in xrange(0, 2):
+                metadata = {'label': str(i),
+                            'is_consistent': False,
+                            'is_sticky': False,
+                            'timestamp': str((int(time.time() - datetime.timedelta(2).total_seconds() - i)))}
+                snapshot_id = VDiskController.create_snapshot(vdisk.guid, metadata)
+                if vdisk == vdisk_1:
+                    StorageRouterClient.delete_snapshot_callbacks[vdisk.volume_id] = {snapshot_id: raise_an_exception}
+        with self.assertRaises(RuntimeError):
+            GenericController.delete_snapshots()
+        self.assertEqual(1, len(vdisk_2.snapshot_ids), 'One snapshot should be removed for vdisk 2')
+        self.assertEqual(2, len(vdisk_1.snapshot_ids), 'No snapshots should be removed for vdisk 1')
+
+    def test_scrubbing_exception_handling(self):
+        """
+        Test if the scheduled job can handle scrub related exceptions
+        """
+        def raise_an_exception(*args, **kwargs):
+            raise RuntimeError(SCRUB_VDISK_EXCEPTION_MESSAGE)
+
+        structure = DalHelper.build_dal_structure(
+            {'vpools': [1],
+             'vdisks': [(1, 1, 1, 1)],  # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
+             'mds_services': [(1, 1)],
+             'storagerouters': [1],
+             'storagedrivers': [(1, 1, 1)]}  # (<id>, <vpool_id>, <storagerouter_id>)
+        )
+        vdisk_1 = structure['vdisks'][1]
+        [dynamic for dynamic in vdisk_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
+
+        for i in xrange(0, 2):
+            metadata = {'label': str(i),
+                        'is_consistent': False,
+                        'is_sticky': False,
+                        'timestamp': str((int(time.time() - datetime.timedelta(2).total_seconds() - i)))}
+            snapshot_id = VDiskController.create_snapshot(vdisk_1.guid, metadata)
+            StorageRouterClient.delete_snapshot_callbacks[vdisk_1.volume_id] = {snapshot_id: raise_an_exception}
+
+        GenericController.delete_snapshots()
+
     ##################
     # HELPER METHODS #
     ##################