CP-42675: send messages to Xapi if GC has insufficient space

drivers/cleanup.py

@@ -304,6 +304,10 @@ def __del__(self):
         if self.sessionPrivate:
             self.session.xenapi.session.logout()
 
+    @property
+    def srRef(self):
+        return self._srRef
+
     def isPluggedHere(self):
         pbds = self.getAttachedPBDs()
         for pbdRec in pbds:
@@ -496,6 +500,7 @@ class VDI:
     DB_GC = "gc"
     DB_COALESCE = "coalesce"
     DB_LEAFCLSC = "leaf-coalesce"  # config key
+    DB_GC_NO_SPACE = "gc_no_space"
     LEAFCLSC_DISABLED = "false"  # set by user; means do not leaf-coalesce
     LEAFCLSC_FORCE = "force"     # set by user; means skip snap-coalesce
     LEAFCLSC_OFFLINE = "offline"  # set here for informational purposes: means
@@ -518,6 +523,7 @@ class VDI:
             DB_LEAFCLSC: XAPI.CONFIG_OTHER,
             DB_ONBOOT: XAPI.CONFIG_ON_BOOT,
             DB_ALLOW_CACHING: XAPI.CONFIG_ALLOW_CACHING,
+            DB_GC_NO_SPACE: XAPI.CONFIG_SM
     }
 
     LIVE_LEAF_COALESCE_MAX_SIZE = 20 * 1024 * 1024  # bytes
@@ -1579,6 +1585,55 @@ def __init__(self, uuid, xapi, createLock, force):
         elif not self.xapi.isMaster():
             raise util.SMException("This host is NOT master, will not run")
 
+        self.no_space_candidates = {}
+
+    def msg_cleared(self, xapi_session, msg_ref):
+        try:
+            msg = xapi_session.xenapi.message.get_record(msg_ref)
+        except XenAPI.Failure:
+            return True
+
+        return msg is None
+
+    def check_no_space_candidates(self):
+        xapi_session = self.xapi.getSession()
+
+        msg_id = self.xapi.srRecord["sm_config"].get(VDI.DB_GC_NO_SPACE)
+        if self.no_space_candidates:
+            if msg_id is None or self.msg_cleared(xapi_session, msg_id):
+                util.SMlog("Could not coalesce due to a lack of space "
+                           f"in SR {self.uuid}")
+                msg_body = ("Unable to perform data coalesce due to a lack "
+                            f"of space in SR {self.uuid}")
+                msg_id = xapi_session.xenapi.message.create(
+                    'SM_GC_NO_SPACE',
+                    3,
+                    "SR",
+                    self.uuid,
+                    msg_body)
+                xapi_session.xenapi.SR.remove_from_sm_config(
+                    self.xapi.srRef, VDI.DB_GC_NO_SPACE)
+                xapi_session.xenapi.SR.add_to_sm_config(
+                    self.xapi.srRef, VDI.DB_GC_NO_SPACE, msg_id)
+
+            for candidate in self.no_space_candidates.values():
+                candidate.setConfig(VDI.DB_GC_NO_SPACE, msg_id)
+        elif msg_id is not None:
+            # Everything was coalescable, remove the message
+            xapi_session.xenapi.message.destroy(msg_id)
+
+    def clear_no_space_msg(self, vdi):
+        msg_id = None
+        try:
+            msg_id = vdi.getConfig(VDI.DB_GC_NO_SPACE)
+        except XenAPI.Failure:
+            pass
+
+        self.no_space_candidates.pop(vdi.uuid, None)
+        if msg_id is not None:
+            vdi.delConfig(VDI.DB_GC_NO_SPACE)
+
+
     def wait_for_plug(self):
         for _ in range(1, 10):
             time.sleep(2)
@@ -1664,8 +1719,10 @@ def findCoalesceable(self):
                 spaceNeeded = c._calcExtraSpaceForCoalescing()
                 if spaceNeeded <= freeSpace:
                     Util.log("Coalesce candidate: %s (tree height %d)" % (c, h))
+                    self.clear_no_space_msg(c)
                     return c
                 else:
+                    self.no_space_candidates[c.uuid] = c
                     Util.log("No space to coalesce %s (free space: %d)" % \
                             (c, freeSpace))
         return None
@@ -1716,14 +1773,16 @@ def findLeafCoalesceable(self):
 
             if spaceNeeded <= freeSpace:
                 Util.log("Leaf-coalesce candidate: %s" % candidate)
+                self.clear_no_space_msg(candidate)
                 return candidate
             else:
                 Util.log("No space to leaf-coalesce %s (free space: %d)" % \
                         (candidate, freeSpace))
                 if spaceNeededLive <= freeSpace:
                     Util.log("...but enough space if skip snap-coalesce")
                     candidate.setConfig(VDI.DB_LEAFCLSC,
-                            VDI.LEAFCLSC_OFFLINE)
+                                        VDI.LEAFCLSC_OFFLINE)
+                self.no_space_candidates[candidate.uuid] = candidate
 
         return None
 
@@ -3096,6 +3155,7 @@ def _gc(session, srUuid, dryRun=False, immediate=False):
     try:
         _gcLoop(sr, dryRun, immediate=immediate)
     finally:
+        sr.check_no_space_candidates()
         sr.cleanup()
         sr.logFilter.logState()
         del sr.xapi

tests/test_cleanup.py

@@ -2,6 +2,7 @@
 import signal
 import unittest
 import unittest.mock as mock
+import uuid
 
 from uuid import uuid4
 
@@ -13,6 +14,8 @@
 
 import ipc
 
+import XenAPI
+
 
 class FakeFile(object):
     pass
@@ -71,6 +74,8 @@ def setUp(self):
         self.xapi_mock.srRecord = {'name_label': 'dummy'}
         self.xapi_mock.isPluggedHere.return_value = True
         self.xapi_mock.isMaster.return_value = True
+        self.mock_xapi_session = mock.MagicMock(name="MockSession")
+        self.xapi_mock.getSession.return_value = self.mock_xapi_session
 
         self.addCleanup(mock.patch.stopall)
 
@@ -94,6 +99,82 @@ def mock_cleanup_locks(self):
         cleanup.lockGCRunning = TestRelease()
         cleanup.lockGCRunning.release = mock.Mock(return_value=None)
 
+    def test_check_no_space_candidates_none(self):
+        sr = create_cleanup_sr(self.xapi_mock)
+        sr.xapi.srRecord.update({
+            "sm_config": {}
+        })
+
+        sr.check_no_space_candidates()
+
+        self.mock_xapi_session.xenapi.message.create.assert_not_called()
+
+    def test_check_no_space_candidates_one_not_reported(self):
+        sr = create_cleanup_sr(self.xapi_mock)
+        vdi_uuid = str(uuid.uuid4())
+        mock_vdi = mock.MagicMock()
+        mock_vdi.uuid = vdi_uuid
+
+        sr.no_space_candidates = {
+            vdi_uuid: mock_vdi
+        }
+        sr.xapi.srRecord.update({
+            "sm_config": {}
+        })
+        self.mock_xapi_session.xenapi.VDI.get_other_config.return_value = {}
+        xapi_message = self.mock_xapi_session.xenapi.message
+        xapi_message.get_record.side_effect = XenAPI.Failure(
+            details='No such message')
+
+        sr.check_no_space_candidates()
+
+        self.mock_xapi_session.xenapi.message.create.assert_called_once_with(
+            'SM_GC_NO_SPACE', 3, 'SR', sr.uuid,
+            "Unable to perform data coalesce "
+            f"due to a lack of space in SR {sr.uuid}")
+
+    def test_check_no_space_candidates_one_already_reported(self):
+        sr = create_cleanup_sr(self.xapi_mock)
+        vdi_uuid = str(uuid.uuid4())
+        mock_vdi = mock.MagicMock()
+        mock_vdi.uuid = vdi_uuid
+
+        sr.no_space_candidates = {
+            vdi_uuid: mock_vdi
+        }
+        sr.xapi.srRecord.update({
+            "sm_config": {"gc_no_space": "dummy ref"}
+        })
+        self.mock_xapi_session.xenapi.VDI.get_other_config.return_value = {}
+        self.mock_xapi_session.xenapi.message.get_record.side_effect = {
+            'name': 'SM_GC_NO_SPACE'
+        }
+
+        sr.check_no_space_candidates()
+
+        self.mock_xapi_session.xenapi.message.create.assert_not_called()
+
+    def test_check_no_space_candidates_none_clear_message(self):
+        sr = create_cleanup_sr(self.xapi_mock)
+        vdi_uuid = str(uuid.uuid4())
+        mock_vdi = mock.MagicMock()
+        mock_vdi.uuid = vdi_uuid
+
+        sr.no_space_candidates = {}
+        sr.xapi.srRecord.update({
+            "sm_config": {"gc_no_space": "dummy ref"}
+        })
+        self.mock_xapi_session.xenapi.VDI.get_other_config.return_value = {}
+        self.mock_xapi_session.xenapi.message.get_record.side_effect = {
+            'name': 'SM_GC_NO_SPACE'
+        }
+
+        sr.check_no_space_candidates()
+
+        self.mock_xapi_session.xenapi.message.destroy.assert_called_once_with(
+            "dummy ref"
+        )
+
     def test_term_handler(self):
         self.assertFalse(cleanup.SIGTERM)
 

tests/test_fairlock.py

@@ -89,7 +89,7 @@ def test_double_lock_deadlock(self):
         with self.assertRaises(FairlockDeadlock) as err:
             with Fairlock("test") as l:
                 n = Fairlock("test")
-                self.assertEquals(l, n)
+                self.assertEqual(l, n)
                 # Real code would use another 'with Fairlock("test")' here but we cannot
                 # do that because it insists on having a code block as a body, which would
                 # then not be reached, causing a "Test code not fully covered" failure