From 94214801928d2a07fcc247cb8fa0bca07eb8b718 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 15:02:30 +0200 Subject: [PATCH 01/29] Prevent wrong mypy error regarding `_linstor` member not set Signed-off-by: Ronan Abhamon --- drivers/LinstorSR.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index 829c48f8..7640f93d 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -613,16 +613,16 @@ def create(self, uuid, size): logger=util.SMlog ) self._vhdutil = LinstorVhdUtil(self.session, self._linstor) - except Exception as e: - util.SMlog('Failed to create LINSTOR SR: {}'.format(e)) - raise xs_errors.XenError('LinstorSRCreate', opterr=str(e)) - try: util.SMlog( "Finishing SR creation, enable drbd-reactor on all hosts..." ) self._update_drbd_reactor_on_all_hosts(enabled=True) except Exception as e: + if not self._linstor: + util.SMlog('Failed to create LINSTOR SR: {}'.format(e)) + raise xs_errors.XenError('LinstorSRCreate', opterr=str(e)) + try: self._linstor.destroy() except Exception as e2: @@ -637,6 +637,7 @@ def delete(self, uuid): util.SMlog('LinstorSR.delete for {}'.format(self.uuid)) cleanup.gc_force(self.session, self.uuid) + assert self._linstor if self.vdis or self._linstor._volumes: raise xs_errors.XenError('SRNotEmpty') From 1e54484a7a0b1ed0f639fb282983d1bb4f080a50 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 11 Oct 2024 18:10:14 +0200 Subject: [PATCH 02/29] Fix many invalid escape sequences Signed-off-by: Ronan Abhamon --- drivers/cifutils.py | 2 +- tests/test_SMBSR.py | 2 +- tests/test_cifutils.py | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/cifutils.py b/drivers/cifutils.py index c183fe61..be82a37c 100755 --- a/drivers/cifutils.py +++ b/drivers/cifutils.py @@ -60,7 +60,7 @@ def splitDomainAndUsername(uname): username = dom_username[1] else: raise CIFSException("A maximum of 2 tokens are expected " - "(\). {} were given." + "(\\). {} were given." .format(len(dom_username))) return username, domain diff --git a/tests/test_SMBSR.py b/tests/test_SMBSR.py index d20bcc0c..0bb6cfb8 100644 --- a/tests/test_SMBSR.py +++ b/tests/test_SMBSR.py @@ -135,7 +135,7 @@ def test_attach_with_cifs_password( def test_attach_with_cifs_password_and_domain( self, symlink, mock_lock, makeMountPoint, mock_checkmount, mock_checklinks, mock_checkwritable): - smbsr = self.create_smbsr(username="citrix\jsmith", dconf_update={"password": "winter2019"}) + smbsr = self.create_smbsr(username="citrix\\jsmith", dconf_update={"password": "winter2019"}) mock_checkmount.return_value = False makeMountPoint.return_value = "/var/mount" smbsr.attach('asr_uuid') diff --git a/tests/test_cifutils.py b/tests/test_cifutils.py index 924aa3c6..0b8bf835 100644 --- a/tests/test_cifutils.py +++ b/tests/test_cifutils.py @@ -44,7 +44,7 @@ def test_password_and_username_smbsr(self): self.assertEqual(domain, None) def test_password_and_username_domain(self): - junk_dconf = {"cifspassword": "123", "username": "citrix\jsmith"} + junk_dconf = {"cifspassword": "123", "username": "citrix\\jsmith"} junk_session = 123 credentials, domain = cifutils.getCIFCredentials(junk_dconf, junk_session, @@ -54,7 +54,7 @@ def test_password_and_username_domain(self): self.assertEqual(domain, "citrix") def test_password_and_username_domain_smbsr(self): - junk_dconf = {"password": "123", "username": "citrix\jsmith"} + junk_dconf = {"password": "123", "username": "citrix\\jsmith"} junk_session = 123 credentials, domain = cifutils.getCIFCredentials(junk_dconf, junk_session) @@ -90,7 +90,7 @@ def test_password_secret_and_username_smbsr(self, get_secret): @mock.patch('util.get_secret', autospec=True) def test_password_secret_and_username_also_domain(self, get_secret): junk_dconf = {"cifspassword_secret": "123", - "username": "citrix\jsmith"} + "username": "citrix\\jsmith"} junk_session = 123 get_secret.return_value = 'winter2019' credentials, domain = cifutils.getCIFCredentials(junk_dconf, @@ -104,7 +104,7 @@ def test_password_secret_and_username_also_domain(self, get_secret): @mock.patch('util.get_secret', autospec=True) def test_password_secret_and_username_also_domain_smbsr(self, get_secret): junk_dconf = {"password_secret": "123", - "username": "citrix\jsmith"} + "username": "citrix\\jsmith"} junk_session = 123 get_secret.return_value = 'winter2019' credentials, domain = cifutils.getCIFCredentials(junk_dconf, @@ -116,23 +116,23 @@ def test_password_secret_and_username_also_domain_smbsr(self, get_secret): def test_username_bad_domain(self): junk_dconf = {"cifspassword_secret": "123", - "username": "citrix\gjk\jsmith"} + "username": "citrix\\gjk\\jsmith"} junk_session = 123 with self.assertRaises(cifutils.CIFSException) as cm: cifutils.getCIFCredentials(junk_dconf, junk_session, prefix="cifs") expected_message = ("A maximum of 2 tokens are expected " - "(\). 3 were given.") + "(\\). 3 were given.") the_exception = cm.exception self.assertEqual(the_exception.errstr, expected_message) def test_username_bad_domain_smbsr(self): junk_dconf = {"password_secret": "123", - "username": "citrix\gjk\jsmith"} + "username": "citrix\\gjk\\jsmith"} junk_session = 123 with self.assertRaises(cifutils.CIFSException) as cm: cifutils.getCIFCredentials(junk_dconf, junk_session) expected_message = ("A maximum of 2 tokens are expected " - "(\). 3 were given.") + "(\\). 3 were given.") the_exception = cm.exception self.assertEqual(the_exception.errstr, expected_message) From 294cde689110a36c2859a44c01cee8000fc8edff Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 11 Oct 2024 18:21:07 +0200 Subject: [PATCH 03/29] Fix many invalid escape sequences on regexes Signed-off-by: Ronan Abhamon --- drivers/util.py | 4 ++-- drivers/vhdutil.py | 2 +- tests/test_storage_init.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/util.py b/drivers/util.py index 5d7b320b..d9356d67 100755 --- a/drivers/util.py +++ b/drivers/util.py @@ -1080,7 +1080,7 @@ def diskFromPartition(partition): return m.group(2) numlen = 0 # number of digit characters - m = re.match("\D+(\d+)", partition) + m = re.match(r"\D+(\d+)", partition) if m is not None: numlen = len(m.group(1)) @@ -1091,7 +1091,7 @@ def diskFromPartition(partition): # is it a mapper path? if partition.startswith("mapper"): if re.search("p[0-9]*$", partition): - numlen = len(re.match("\d+", partition[::-1]).group(0)) + 1 + numlen = len(re.match(r"\d+", partition[::-1]).group(0)) + 1 SMlog("Found mapper part, len %d" % numlen) else: numlen = 0 diff --git a/drivers/vhdutil.py b/drivers/vhdutil.py index 6a71804c..8ac7b08d 100755 --- a/drivers/vhdutil.py +++ b/drivers/vhdutil.py @@ -206,7 +206,7 @@ def hasParent(path): cmd = [VHD_UTIL, "read", OPT_LOG_ERR, "-p", "-n", path] ret = ioretry(cmd) # pylint: disable=no-member - m = re.match(".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S) + m = re.match(r".*Disk type\s+: (\S+) hard disk.*", ret, flags=re.S) vhd_type = m.group(1) assert(vhd_type == "Differencing" or vhd_type == "Dynamic") return vhd_type == "Differencing" diff --git a/tests/test_storage_init.py b/tests/test_storage_init.py index 9b7138a4..d91d9089 100644 --- a/tests/test_storage_init.py +++ b/tests/test_storage_init.py @@ -355,7 +355,7 @@ def _xe_command(self, args): # pragma: no cover combined_args = " ".join(sorted(args[1:])) if subcmd == "sm-list": - m = re.match("--minimal params=uuid type=(\S+)$", combined_args) + m = re.match(r"--minimal params=uuid type=(\S+)$", combined_args) if m: sm_uuid = "uuid-for-sr-type-" + m.group(1) return CmdResult(stdout=f"{sm_uuid}\n") @@ -365,7 +365,7 @@ def _xe_command(self, args): # pragma: no cover if not self.created_srs: return CmdResult() - m = re.match("--minimal params=uuid type=(\S+)$", combined_args) + m = re.match(r"--minimal params=uuid type=(\S+)$", combined_args) if m: sr_type = m.group(1) num_srs = len(self.created_srs[sr_type]) From 2be912d548cf8104d9b170e52a10bca5cbfae41d Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Mon, 14 Oct 2024 17:33:49 +0200 Subject: [PATCH 04/29] Fix override of FileSR.attach The current attach method of FileSR doesn't correctly override the method of the SR class. It actually adds a "bind" parameter, which is seen as an error by analyzers like mypy. The "bind" parameter was added by this commit: "CA-371791: Fix world readable permissions on EXTSR" Signed-off-by: Ronan Abhamon --- drivers/EXTSR.py | 2 +- drivers/FileSR.py | 5 ++++- tests/test_FileSR.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/EXTSR.py b/drivers/EXTSR.py index 61e88dcd..16481620 100755 --- a/drivers/EXTSR.py +++ b/drivers/EXTSR.py @@ -123,7 +123,7 @@ def attach(self, sr_uuid): 'LVMMount', opterr='FSCK failed on %s. Errno is %d' % (self.remotepath, inst.code)) - super(EXTSR, self).attach(sr_uuid, bind=False) + self.attach_and_bind(sr_uuid, bind=False) self.attached = True diff --git a/drivers/FileSR.py b/drivers/FileSR.py index ba9b1863..8b832983 100755 --- a/drivers/FileSR.py +++ b/drivers/FileSR.py @@ -156,7 +156,10 @@ def delete(self, sr_uuid): raise xs_errors.XenError('FileSRDelete', \ opterr='error %d' % inst.code) - def attach(self, sr_uuid, bind=True): + def attach(self, sr_uuid): + self.attach_and_bind(sr_uuid) + + def attach_and_bind(self, sr_uuid, bind=True): if not self._checkmount(): try: util.ioretry(lambda: util.makedirs(self.path, mode=0o700)) diff --git a/tests/test_FileSR.py b/tests/test_FileSR.py index e755f2d8..a82d0e5b 100644 --- a/tests/test_FileSR.py +++ b/tests/test_FileSR.py @@ -682,7 +682,7 @@ def test_attach_can_do_non_bind_mount(self, mock_chmod, mock_util_makedirs): sr.path = mount_dst sr.remotepath = mount_src - sr.attach(None, bind=False) + sr.attach_and_bind(None, bind=False) self.assertTrue(sr.attached) From fc5b6d9b527aa15eca6e5c833aa348e5ecf89025 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Tue, 15 Oct 2024 14:35:40 +0200 Subject: [PATCH 05/29] Fix override of BaseISCSISR.detach The current detach method of BaseISCSISR doesn't correctly override the method of the SR class. It actually adds a "delete" parameter, which is seen as an error by analyzers like mypy. The "delete" parameter was added by this commit: "iscsi: Delete LUN on detach of RawISCSI" Signed-off-by: Ronan Abhamon --- drivers/BaseISCSI.py | 5 ++++- drivers/RawISCSISR.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/BaseISCSI.py b/drivers/BaseISCSI.py index 67388dc1..798491d0 100755 --- a/drivers/BaseISCSI.py +++ b/drivers/BaseISCSI.py @@ -429,7 +429,10 @@ def attach(self, sr_uuid): realdev = os.path.realpath(os.path.join(dev_path, dev)) util.set_scheduler(os.path.basename(realdev)) - def detach(self, sr_uuid, delete=False): + def detach(self, sr_uuid): + self.detach_and_delete(sr_uuid, delete=False) + + def detach_and_delete(self, sr_uuid, delete=True): keys = [] pbdref = None try: diff --git a/drivers/RawISCSISR.py b/drivers/RawISCSISR.py index 0b17cfa9..1df1c7a2 100644 --- a/drivers/RawISCSISR.py +++ b/drivers/RawISCSISR.py @@ -64,7 +64,7 @@ def load(self, vdi_uuid): self.managed = True def detach(self, sr_uuid): - super(RawISCSISR, self).detach(sr_uuid, True) + super(RawISCSISR, self).detach_and_delete(sr_uuid) def vdi(self, uuid): return ISCSIVDI(self, uuid) From 5e62373419a9cd468b3230e31854ececbd96415b Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Tue, 15 Oct 2024 17:12:57 +0200 Subject: [PATCH 06/29] Fix override of VDI.delete in many subclasses It triggers warns in analyzers like mypy. Signed-off-by: Ronan Abhamon --- drivers/DummySR.py | 2 +- drivers/ISOSR.py | 2 +- drivers/LUNperVDI.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/DummySR.py b/drivers/DummySR.py index 44f571e5..c733fdf7 100755 --- a/drivers/DummySR.py +++ b/drivers/DummySR.py @@ -159,7 +159,7 @@ def create(self, sr_uuid, vdi_uuid, size): self.run_corner_cases_tests() return self.get_params() - def delete(self, sr_uuid, vdi_uuid): + def delete(self, sr_uuid, vdi_uuid, data_only=False): self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) assert(len(self.sr.srcmd.params['args']) == 0) diff --git a/drivers/ISOSR.py b/drivers/ISOSR.py index 9ca44506..cbe0b658 100755 --- a/drivers/ISOSR.py +++ b/drivers/ISOSR.py @@ -757,7 +757,7 @@ def create(self, sr_uuid, vdi_uuid, size): raise xs_errors.XenError('VDICreate', \ opterr='could not create file: "%s"' % self.path) - def delete(self, sr_uuid, vdi_uuid): + def delete(self, sr_uuid, vdi_uuid, data_only=False): util.SMlog("Deleting...") self.uuid = vdi_uuid diff --git a/drivers/LUNperVDI.py b/drivers/LUNperVDI.py index 306d7041..497ba6b5 100755 --- a/drivers/LUNperVDI.py +++ b/drivers/LUNperVDI.py @@ -98,7 +98,7 @@ def create(self, sr_uuid, vdi_uuid, size): return super(RAWVDI, self.sr.vdis[v['uuid']]).get_params() raise xs_errors.XenError('SRNoSpace') - def delete(self, sr_uuid, vdi_uuid): + def delete(self, sr_uuid, vdi_uuid, data_only=False): try: vdi = util._getVDI(self.sr, vdi_uuid) if not vdi['managed']: From 04b406d1ff3a4f04d60f86803de01b5244a4aaf7 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 14:52:40 +0200 Subject: [PATCH 07/29] Fix override of `VDI._do_snapshot` `cloneOp` must be present. Parameters like `snapType` must must be written in the same way between the parent class and the child class. Otherwise a linter like mypy may return an error. Signed-off-by: Ronan Abhamon --- drivers/FileSR.py | 6 +++--- drivers/LinstorSR.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/FileSR.py b/drivers/FileSR.py index 8b832983..4d2e0a21 100755 --- a/drivers/FileSR.py +++ b/drivers/FileSR.py @@ -709,8 +709,8 @@ def reset_leaf(self, sr_uuid, vdi_uuid): vhdutil.killData(self.path) - def _do_snapshot(self, sr_uuid, vdi_uuid, snap_type, - secondary=None, cbtlog=None): + def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, + cloneOp=False, secondary=None, cbtlog=None): # If cbt enabled, save file consistency state if cbtlog is not None: if blktap2.VDI.tap_status(self.session, vdi_uuid): @@ -728,7 +728,7 @@ def _do_snapshot(self, sr_uuid, vdi_uuid, snap_type, if not blktap2.VDI.tap_pause(self.session, sr_uuid, vdi_uuid): raise util.SMException("failed to pause VDI %s" % vdi_uuid) try: - return self._snapshot(snap_type, cbtlog, consistency_state) + return self._snapshot(snapType, cbtlog, consistency_state) finally: blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid, secondary) diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index 7640f93d..81c4b7e0 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -2316,9 +2316,8 @@ def _rename(self, oldpath, newpath): volume_uuid = self._linstor.get_volume_uuid_from_device_path(oldpath) self._linstor.update_volume_name(volume_uuid, newpath) - def _do_snapshot( - self, sr_uuid, vdi_uuid, snap_type, secondary=None, cbtlog=None - ): + def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, + cloneOp=False, secondary=None, cbtlog=None): # If cbt enabled, save file consistency state. if cbtlog is not None: if blktap2.VDI.tap_status(self.session, vdi_uuid): @@ -2338,7 +2337,7 @@ def _do_snapshot( if not blktap2.VDI.tap_pause(self.session, sr_uuid, vdi_uuid): raise util.SMException('Failed to pause VDI {}'.format(vdi_uuid)) try: - return self._snapshot(snap_type, cbtlog, consistency_state) + return self._snapshot(snapType, cbtlog, consistency_state) finally: blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid, secondary) From be79a5e9f3443eadb7b3c6ba510e2f1702849e2d Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Tue, 22 Oct 2024 14:33:13 +0200 Subject: [PATCH 08/29] Fix override of VDI.load in LVHDVDI cleanup.py It triggers warns in analyzers like mypy. Signed-off-by: Ronan Abhamon --- drivers/cleanup.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/cleanup.py b/drivers/cleanup.py index 4d1c6681..712ab855 100755 --- a/drivers/cleanup.py +++ b/drivers/cleanup.py @@ -1205,20 +1205,22 @@ class LVHDVDI(VDI): JRN_ZERO = "zero" # journal entry type for zeroing out end of parent DRIVER_NAME_RAW = "aio" - def load(self, vdiInfo): + def load(self, info=None): + # `info` is always set. `None` default value is only here to match parent method. + assert info, "No info given to LVHDVDI.load" self.parent = None self.children = [] self._sizeVHD = -1 self._sizeAllocated = -1 - self.scanError = vdiInfo.scanError - self.sizeLV = vdiInfo.sizeLV - self.sizeVirt = vdiInfo.sizeVirt - self.fileName = vdiInfo.lvName - self.lvActive = vdiInfo.lvActive - self.lvOpen = vdiInfo.lvOpen - self.lvReadonly = vdiInfo.lvReadonly - self.hidden = vdiInfo.hidden - self.parentUuid = vdiInfo.parentUuid + self.scanError = info.scanError + self.sizeLV = info.sizeLV + self.sizeVirt = info.sizeVirt + self.fileName = info.lvName + self.lvActive = info.lvActive + self.lvOpen = info.lvOpen + self.lvReadonly = info.lvReadonly + self.hidden = info.hidden + self.parentUuid = info.parentUuid self.path = os.path.join(self.sr.path, self.fileName) @staticmethod From 5d53761a96cdaeacc41fcfebe05366b70a7d3a26 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 14:48:38 +0200 Subject: [PATCH 09/29] Use a specific var for NFS options in ISOSR.attach Prevent mypy errors when a variable type is changed dynamically from list to string. Signed-off-by: Ronan Abhamon --- drivers/ISOSR.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/ISOSR.py b/drivers/ISOSR.py index cbe0b658..64f99b48 100755 --- a/drivers/ISOSR.py +++ b/drivers/ISOSR.py @@ -335,14 +335,15 @@ def attach(self, sr_uuid): util.makedirs(self.mountpoint) mountcmd = [] - options = '' + options = [] + nfs_options = '' if 'options' in self.dconf: options = self.dconf['options'].split(' ') if protocol == 'cifs': options = [x for x in options if x != ""] else: - options = self.getNFSOptions(options) + nfs_options = self.getNFSOptions(options) # SMB options are passed differently for create via # XC/xe sr-create and create via xe-mount-iso-sr @@ -392,7 +393,7 @@ def attach(self, sr_uuid): io_timeout = nfs.get_nfs_timeout(self.other_config) io_retrans = nfs.get_nfs_retrans(self.other_config) nfs.soft_mount(self.mountpoint, server, path, - transport, useroptions=options, nfsversion=self.nfsversion, + transport, useroptions=nfs_options, nfsversion=self.nfsversion, timeout=io_timeout, retrans=io_retrans) else: if self.smbversion in SMB_VERSION_3: From c5ddfc7a86d9eff5d5622c6bb66b3cc10182d955 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Tue, 15 Oct 2024 15:15:02 +0200 Subject: [PATCH 10/29] Modernize Lock class using `staticmethod` decorator Signed-off-by: Ronan Abhamon --- drivers/lock.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/lock.py b/drivers/lock.py index ceb48fe1..56bf9fcc 100755 --- a/drivers/lock.py +++ b/drivers/lock.py @@ -64,6 +64,7 @@ def release(self): def held(self): raise NotImplementedError("Lock methods implemented in LockImplementation") + @staticmethod def _mknamespace(ns): if ns is None: @@ -72,7 +73,6 @@ def _mknamespace(ns): assert not ns.startswith(".") assert ns.find(os.path.sep) < 0 return ns - _mknamespace = staticmethod(_mknamespace) @staticmethod def clearAll(): @@ -82,6 +82,7 @@ def clearAll(): Lock.INSTANCES = {} Lock.BASE_INSTANCES = {} + @staticmethod def cleanup(name, ns=None): if ns: if ns in Lock.INSTANCES: @@ -97,8 +98,7 @@ def cleanup(name, ns=None): if os.path.exists(path): Lock._unlink(path) - cleanup = staticmethod(cleanup) - + @staticmethod def cleanupAll(ns=None): ns = Lock._mknamespace(ns) nspath = os.path.join(Lock.BASE_DIR, ns) @@ -112,11 +112,11 @@ def cleanupAll(ns=None): Lock._rmdir(nspath) - cleanupAll = staticmethod(cleanupAll) # # Lock and attribute file management # + @staticmethod def _mkdirs(path): """Concurrent makedirs() catching EEXIST.""" if os.path.exists(path): @@ -126,8 +126,8 @@ def _mkdirs(path): except OSError as e: if e.errno != errno.EEXIST: raise LockException("Failed to makedirs(%s)" % path) - _mkdirs = staticmethod(_mkdirs) + @staticmethod def _unlink(path): """Non-raising unlink().""" util.SMlog("lock: unlinking lock file %s" % path) @@ -135,8 +135,8 @@ def _unlink(path): os.unlink(path) except Exception as e: util.SMlog("Failed to unlink(%s): %s" % (path, e)) - _unlink = staticmethod(_unlink) + @staticmethod def _rmdir(path): """Non-raising rmdir().""" util.SMlog("lock: removing lock dir %s" % path) @@ -144,7 +144,6 @@ def _rmdir(path): os.rmdir(path) except Exception as e: util.SMlog("Failed to rmdir(%s): %s" % (path, e)) - _rmdir = staticmethod(_rmdir) class LockImplementation(object): From 59b7196a273dc17222f3b52cd50e62299071f3c1 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Tue, 15 Oct 2024 15:33:11 +0200 Subject: [PATCH 11/29] Modernize GC using `staticmethod` decorator Signed-off-by: Ronan Abhamon --- drivers/cleanup.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/cleanup.py b/drivers/cleanup.py index 712ab855..940e6802 100755 --- a/drivers/cleanup.py +++ b/drivers/cleanup.py @@ -112,10 +112,11 @@ class Util: PREFIX = {"G": 1024 * 1024 * 1024, "M": 1024 * 1024, "K": 1024} + @staticmethod def log(text): util.SMlog(text, ident="SMGC") - log = staticmethod(log) + @staticmethod def logException(tag): info = sys.exc_info() if info[0] == SystemExit: @@ -129,8 +130,8 @@ def logException(tag): Util.log("%s: EXCEPTION %s, %s" % (tag, info[0], info[1])) Util.log(tb) Util.log("*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*") - logException = staticmethod(logException) + @staticmethod def doexec(args, expectedRC, inputtext=None, ret=None, log=True): "Execute a subprocess, then return its return code, stdout, stderr" proc = subprocess.Popen(args, @@ -159,8 +160,8 @@ def doexec(args, expectedRC, inputtext=None, ret=None, log=True): if ret == Util.RET_STDERR: return stderr return stdout - doexec = staticmethod(doexec) + @staticmethod def runAbortable(func, ret, ns, abortTest, pollInterval, timeOut): """execute func in a separate thread and kill it if abortTest signals so""" @@ -211,23 +212,23 @@ def runAbortable(func, ret, ns, abortTest, pollInterval, timeOut): resultFlag.set("failure") Util.logException("This exception has occured") os._exit(0) - runAbortable = staticmethod(runAbortable) + @staticmethod def num2str(number): for prefix in ("G", "M", "K"): if number >= Util.PREFIX[prefix]: return "%.3f%s" % (float(number) / Util.PREFIX[prefix], prefix) return "%s" % number - num2str = staticmethod(num2str) + @staticmethod def numBits(val): count = 0 while val: count += val & 1 val = val >> 1 return count - numBits = staticmethod(numBits) + @staticmethod def countBits(bitmap1, bitmap2): """return bit count in the bitmap produced by ORing the two bitmaps""" len1 = len(bitmap1) @@ -249,14 +250,13 @@ def countBits(bitmap1, bitmap2): val = bitmapLong[i] count += Util.numBits(val) return count - countBits = staticmethod(countBits) + @staticmethod def getThisScript(): thisScript = util.get_real_path(__file__) if thisScript.endswith(".pyc"): thisScript = thisScript[:-1] return thisScript - getThisScript = staticmethod(getThisScript) ################################################################################ @@ -282,11 +282,11 @@ class XAPI: class LookupError(util.SMException): pass + @staticmethod def getSession(): session = XenAPI.xapi_local() session.xenapi.login_with_password(XAPI.USER, '', '', 'SM') return session - getSession = staticmethod(getSession) def __init__(self, session, srUuid): self.sessionPrivate = False @@ -836,6 +836,7 @@ def _runTapdiskDiff(self): Util.doexec(cmd, 0) return True + @staticmethod def _reportCoalesceError(vdi, ce): """Reports a coalesce error to XenCenter. @@ -888,12 +889,12 @@ def _reportCoalesceError(vdi, ce): str(now.strftime('%s'))) if xcmsg: xapi.message.create(msg_name, "3", "SR", vdi.sr.uuid, msg_body) - _reportCoalesceError = staticmethod(_reportCoalesceError) def coalesce(self): # size is returned in sectors return vhdutil.coalesce(self.path) * 512 + @staticmethod def _doCoalesceVHD(vdi): try: startTime = time.time() @@ -913,7 +914,6 @@ def _doCoalesceVHD(vdi): raise ce except: raise - _doCoalesceVHD = staticmethod(_doCoalesceVHD) def _vdi_is_raw(self, vdi_path): """ @@ -1811,6 +1811,7 @@ def _getTreeStr(self, vdi, indent=8): KEY_OFFLINE_COALESCE_NEEDED = "leaf_coalesce_need_offline" KEY_OFFLINE_COALESCE_OVERRIDE = "leaf_coalesce_offline_override" + @staticmethod def getInstance(uuid, xapiSession, createLock=True, force=False): xapi = XAPI(xapiSession, uuid) type = normalizeType(xapi.srRecord["type"]) @@ -1821,7 +1822,6 @@ def getInstance(uuid, xapiSession, createLock=True, force=False): elif type == SR.TYPE_LINSTOR: return LinstorSR(uuid, xapi, createLock, force) raise util.SMException("SR type %s not recognized" % type) - getInstance = staticmethod(getInstance) def __init__(self, uuid, xapi, createLock, force): self.logFilter = self.LogFilter(self) From 8e3bb38b69440b89bf967d5a35ccfaf5ad06b83e Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Tue, 15 Oct 2024 15:38:33 +0200 Subject: [PATCH 12/29] Modernize RefCounter using `staticmethod` decorator Signed-off-by: Ronan Abhamon --- drivers/refcounter.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/drivers/refcounter.py b/drivers/refcounter.py index 5418c858..20585da8 100644 --- a/drivers/refcounter.py +++ b/drivers/refcounter.py @@ -39,6 +39,7 @@ class RefCounter: BASE_DIR = "/var/run/sm/refcount" + @staticmethod def get(obj, binary, ns=None): """Get (inc ref count) 'obj' in namespace 'ns' (optional). Returns new ref count""" @@ -46,8 +47,8 @@ def get(obj, binary, ns=None): return RefCounter._adjust(ns, obj, 0, 1) else: return RefCounter._adjust(ns, obj, 1, 0) - get = staticmethod(get) + @staticmethod def put(obj, binary, ns=None): """Put (dec ref count) 'obj' in namespace 'ns' (optional). If ref count was zero already, this operation is a no-op. @@ -56,8 +57,8 @@ def put(obj, binary, ns=None): return RefCounter._adjust(ns, obj, 0, -1) else: return RefCounter._adjust(ns, obj, -1, 0) - put = staticmethod(put) + @staticmethod def set(obj, count, binaryCount, ns=None): """Set normal & binary counts explicitly to the specified values. Returns new ref count""" @@ -66,14 +67,14 @@ def set(obj, count, binaryCount, ns=None): if binaryCount > 1: raise RefCounterException("Binary count = %d > 1" % binaryCount) RefCounter._set(ns, obj, count, binaryCount) - set = staticmethod(set) + @staticmethod def check(obj, ns=None): """Get the ref count values for 'obj' in namespace 'ns' (optional)""" (obj, ns) = RefCounter._getSafeNames(obj, ns) return RefCounter._get(ns, obj) - check = staticmethod(check) + @staticmethod def checkLocked(obj, ns): """Lock-protected access""" lock = Lock(obj, ns) @@ -82,13 +83,13 @@ def checkLocked(obj, ns): return RefCounter.check(obj, ns) finally: lock.release() - checkLocked = staticmethod(checkLocked) + @staticmethod def reset(obj, ns=None): """Reset ref counts for 'obj' in namespace 'ns' (optional) to 0.""" RefCounter.resetAll(ns, obj) - reset = staticmethod(reset) + @staticmethod def resetAll(ns=None, obj=None): """Reset ref counts of 'obj' in namespace 'ns' to 0. If obj is not provided, reset all existing objects in 'ns' to 0. If neither obj nor @@ -106,8 +107,8 @@ def resetAll(ns=None, obj=None): raise RefCounterException("failed to get namespace list") for ns in nsList: RefCounter._reset(ns, obj) - resetAll = staticmethod(resetAll) + @staticmethod def _adjust(ns, obj, delta, binaryDelta): """Add 'delta' to the normal refcount and 'binaryDelta' to the binary refcount of 'obj' in namespace 'ns'. @@ -133,8 +134,8 @@ def _adjust(ns, obj, delta, binaryDelta): newCount, newBinaryCount)) RefCounter._set(ns, obj, newCount, newBinaryCount) return newCount + newBinaryCount - _adjust = staticmethod(_adjust) + @staticmethod def _get(ns, obj): """Get the ref count values for 'obj' in namespace 'ns'""" objFile = os.path.join(RefCounter.BASE_DIR, ns, obj) @@ -142,8 +143,8 @@ def _get(ns, obj): if util.pathexists(objFile): (count, binaryCount) = RefCounter._readCount(objFile) return (count, binaryCount) - _get = staticmethod(_get) + @staticmethod def _set(ns, obj, count, binaryCount): """Set the ref count values for 'obj' in namespace 'ns'""" util.SMlog("Refcount for %s:%s set => (%d, %db)" % \ @@ -156,8 +157,7 @@ def _set(ns, obj, count, binaryCount): while not RefCounter._writeCount(objFile, count, binaryCount): RefCounter._createNamespace(ns) - _set = staticmethod(_set) - + @staticmethod def _getSafeNames(obj, ns): """Get a name that can be used as a file name""" if not ns: @@ -167,8 +167,8 @@ def _getSafeNames(obj, ns): for char in ['/', '*', '?', '\\']: obj = obj.replace(char, "_") return (obj, ns) - _getSafeNames = staticmethod(_getSafeNames) + @staticmethod def _createNamespace(ns): nsDir = os.path.join(RefCounter.BASE_DIR, ns) try: @@ -177,8 +177,8 @@ def _createNamespace(ns): if e.errno != errno.EEXIST: raise RefCounterException("failed to makedirs '%s' (%s)" % \ (nsDir, e)) - _createNamespace = staticmethod(_createNamespace) + @staticmethod def _removeObject(ns, obj): nsDir = os.path.join(RefCounter.BASE_DIR, ns) objFile = os.path.join(nsDir, obj) @@ -199,8 +199,8 @@ def _removeObject(ns, obj): pass else: raise RefCounterException("failed to remove '%s'" % nsDir) - _removeObject = staticmethod(_removeObject) + @staticmethod def _reset(ns, obj=None): nsDir = os.path.join(RefCounter.BASE_DIR, ns) if not util.pathexists(nsDir): @@ -216,8 +216,8 @@ def _reset(ns, obj=None): raise RefCounterException("failed to list '%s'" % ns) for obj in objList: RefCounter._removeObject(ns, obj) - _reset = staticmethod(_reset) + @staticmethod def _readCount(fn): try: f = open(fn, 'r') @@ -229,8 +229,8 @@ def _readCount(fn): except IOError: raise RefCounterException("failed to read file '%s'" % fn) return (count, binaryCount) - _readCount = staticmethod(_readCount) + @staticmethod def _writeCount(fn, count, binaryCount): try: f = open(fn, 'w') @@ -243,8 +243,8 @@ def _writeCount(fn, count, binaryCount): return False raise RefCounterException("failed to write '(%d %d)' to '%s': %s" \ % (count, binaryCount, fn, e)) - _writeCount = staticmethod(_writeCount) + @staticmethod def _runTests(): "Unit tests" @@ -535,7 +535,6 @@ def _runTests(): RefCounter.resetAll() return 0 - _runTests = staticmethod(_runTests) if __name__ == '__main__': From 3cfda90017c573590dd15a775ba9dae0f0fcb316 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Mon, 21 Oct 2024 17:10:39 +0200 Subject: [PATCH 13/29] Simplify FakeSMBSR implementation (remove member vars in class) Signed-off-by: Ronan Abhamon --- tests/test_SMBSR.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_SMBSR.py b/tests/test_SMBSR.py index 0bb6cfb8..4cfd2733 100644 --- a/tests/test_SMBSR.py +++ b/tests/test_SMBSR.py @@ -12,17 +12,10 @@ class FakeSMBSR(SMBSR.SMBSR): - uuid = None - sr_ref = None - mountpoint = None - linkpath = None - path = None - session = None - remoteserver = None - def __init__(self, srcmd, none): self.dconf = srcmd.dconf self.srcmd = srcmd + self.session = None self.uuid = 'auuid' self.sr_ref = 'asr_ref' self.mountpoint = 'aMountpoint' From c38e7eb660e94ac19beb4e23c4856bd62fce5a82 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Mon, 21 Oct 2024 17:49:20 +0200 Subject: [PATCH 14/29] Use `for session` instead of `for e` Avoid mypy error: ``` error: Assignment to variable "e" outside except: block [misc] ``` Signed-off-by: Ronan Abhamon --- drivers/iscsilib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iscsilib.py b/drivers/iscsilib.py index e77d17d2..e54de1bc 100644 --- a/drivers/iscsilib.py +++ b/drivers/iscsilib.py @@ -535,8 +535,8 @@ def _checkAnyTGT(): except Exception as e: util.SMlog("%s failed with %s" % (cmd, e.args)) stdout = "" - for e in filter(match_session, stdout.split('\n')): - iqn = e.split()[-1] + for session in filter(match_session, stdout.split('\n')): + iqn = session.split()[-1] if not iqn in rootIQNs: return True return False From 3ab297de5c982b5ac788b884a6354bc5c2fb8f4a Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 14:41:58 +0200 Subject: [PATCH 15/29] Define and `details` attr on `Failure` mock Required by mypy. Attr is referenced in ISOSR.py. Signed-off-by: Ronan Abhamon --- mocks/XenAPI/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mocks/XenAPI/__init__.py b/mocks/XenAPI/__init__.py index 8e45a1b4..9dd4441f 100644 --- a/mocks/XenAPI/__init__.py +++ b/mocks/XenAPI/__init__.py @@ -1,6 +1,6 @@ class Failure(Exception): def __init__(self, details): - pass + self.details = details def xapi_local(): # Mock stub From 4f662048c143ff6fc95f1fda8ddcb21ca0df032b Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 14:44:19 +0200 Subject: [PATCH 16/29] Fix util.SRtoXML calls in many drivers Without this change, mypy triggers an error `var-annoted`: `Need type annotation for "sr_dict"` Signed-off-by: Ronan Abhamon --- drivers/CephFSSR.py | 3 +-- drivers/GlusterFSSR.py | 3 +-- drivers/MooseFSSR.py | 3 +-- drivers/SMBSR.py | 5 +---- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/CephFSSR.py b/drivers/CephFSSR.py index f7c26336..ca79a6a3 100644 --- a/drivers/CephFSSR.py +++ b/drivers/CephFSSR.py @@ -197,8 +197,7 @@ def probe(self): except (util.CommandException, xs_errors.XenError): raise # Create a dictionary from the SR uuids to feed SRtoXML() - sr_dict = {sr_uuid: {} for sr_uuid in sr_list} - return util.SRtoXML(sr_dict) + return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) def detach(self, sr_uuid): if not self.checkmount(): diff --git a/drivers/GlusterFSSR.py b/drivers/GlusterFSSR.py index 42e5ab52..8adfe700 100644 --- a/drivers/GlusterFSSR.py +++ b/drivers/GlusterFSSR.py @@ -177,8 +177,7 @@ def probe(self): except (util.CommandException, xs_errors.XenError): raise # Create a dictionary from the SR uuids to feed SRtoXML() - sr_dict = {sr_uuid: {} for sr_uuid in sr_list} - return util.SRtoXML(sr_dict) + return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) def detach(self, sr_uuid): if not self.checkmount(): diff --git a/drivers/MooseFSSR.py b/drivers/MooseFSSR.py index 3911b096..6f86054f 100755 --- a/drivers/MooseFSSR.py +++ b/drivers/MooseFSSR.py @@ -195,8 +195,7 @@ def probe(self): except (util.CommandException, xs_errors.XenError): raise # Create a dictionary from the SR uuids to feed SRtoXML() - sr_dict = {sr_uuid: {} for sr_uuid in sr_list} - return util.SRtoXML(sr_dict) + return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) def detach(self, sr_uuid): if not self.checkmount(): diff --git a/drivers/SMBSR.py b/drivers/SMBSR.py index aa9bda38..962060cd 100755 --- a/drivers/SMBSR.py +++ b/drivers/SMBSR.py @@ -220,11 +220,8 @@ def probe(self): raise xs_errors.XenError(err, opterr=inst.errstr) except (util.CommandException, xs_errors.XenError): raise - # Create a dictionary from the SR uuids to feed SRtoXML() - sr_dict = {sr_uuid: {} for sr_uuid in sr_list} - - return util.SRtoXML(sr_dict) + return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) def detach(self, sr_uuid): """Detach the SR: Unmounts and removes the mountpoint""" From 7568d2b9e4d5985fa21c6c22371c9f0cc06c2e1e Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 15:10:55 +0200 Subject: [PATCH 17/29] Replace `Dict` variable with `info` in `LVHDSR` Prevent reuse of the `Dict` symbol from the `typing` module. Signed-off-by: Ronan Abhamon --- drivers/LVHDSR.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/LVHDSR.py b/drivers/LVHDSR.py index 66a50fa2..361e88eb 100755 --- a/drivers/LVHDSR.py +++ b/drivers/LVHDSR.py @@ -691,28 +691,28 @@ def scan(self, uuid): for vdi in vdis: vdi_uuids.add(self.session.xenapi.VDI.get_uuid(vdi)) - Dict = LVMMetadataHandler(self.mdpath, False).getMetadata()[1] + info = LVMMetadataHandler(self.mdpath, False).getMetadata()[1] - for vdi in list(Dict.keys()): - vdi_uuid = Dict[vdi][UUID_TAG] - if bool(int(Dict[vdi][IS_A_SNAPSHOT_TAG])): - if Dict[vdi][SNAPSHOT_OF_TAG] in vdiToSnaps: - vdiToSnaps[Dict[vdi][SNAPSHOT_OF_TAG]].append(vdi_uuid) + for vdi in list(info.keys()): + vdi_uuid = info[vdi][UUID_TAG] + if bool(int(info[vdi][IS_A_SNAPSHOT_TAG])): + if info[vdi][SNAPSHOT_OF_TAG] in vdiToSnaps: + vdiToSnaps[info[vdi][SNAPSHOT_OF_TAG]].append(vdi_uuid) else: - vdiToSnaps[Dict[vdi][SNAPSHOT_OF_TAG]] = [vdi_uuid] + vdiToSnaps[info[vdi][SNAPSHOT_OF_TAG]] = [vdi_uuid] if vdi_uuid not in vdi_uuids: util.SMlog("Introduce VDI %s as it is present in " \ "metadata and not in XAPI." % vdi_uuid) sm_config = {} - sm_config['vdi_type'] = Dict[vdi][VDI_TYPE_TAG] + sm_config['vdi_type'] = info[vdi][VDI_TYPE_TAG] lvname = "%s%s" % \ (lvhdutil.LV_PREFIX[sm_config['vdi_type']], vdi_uuid) self.lvmCache.activateNoRefcount(lvname) activated = True lvPath = os.path.join(self.path, lvname) - if Dict[vdi][VDI_TYPE_TAG] == vhdutil.VDI_TYPE_RAW: + if info[vdi][VDI_TYPE_TAG] == vhdutil.VDI_TYPE_RAW: size = self.lvmCache.getSize( \ lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_RAW] + \ vdi_uuid) @@ -736,31 +736,31 @@ def scan(self, uuid): vdi_ref = self.session.xenapi.VDI.db_introduce( vdi_uuid, - Dict[vdi][NAME_LABEL_TAG], - Dict[vdi][NAME_DESCRIPTION_TAG], + info[vdi][NAME_LABEL_TAG], + info[vdi][NAME_DESCRIPTION_TAG], self.sr_ref, - Dict[vdi][TYPE_TAG], + info[vdi][TYPE_TAG], False, - bool(int(Dict[vdi][READ_ONLY_TAG])), + bool(int(info[vdi][READ_ONLY_TAG])), {}, vdi_uuid, {}, sm_config) self.session.xenapi.VDI.set_managed(vdi_ref, - bool(int(Dict[vdi][MANAGED_TAG]))) + bool(int(info[vdi][MANAGED_TAG]))) self.session.xenapi.VDI.set_virtual_size(vdi_ref, str(size)) self.session.xenapi.VDI.set_physical_utilisation( \ vdi_ref, str(utilisation)) self.session.xenapi.VDI.set_is_a_snapshot( \ - vdi_ref, bool(int(Dict[vdi][IS_A_SNAPSHOT_TAG]))) - if bool(int(Dict[vdi][IS_A_SNAPSHOT_TAG])): + vdi_ref, bool(int(info[vdi][IS_A_SNAPSHOT_TAG]))) + if bool(int(info[vdi][IS_A_SNAPSHOT_TAG])): self.session.xenapi.VDI.set_snapshot_time( \ - vdi_ref, DateTime(Dict[vdi][SNAPSHOT_TIME_TAG])) - if Dict[vdi][TYPE_TAG] == 'metadata': + vdi_ref, DateTime(info[vdi][SNAPSHOT_TIME_TAG])) + if info[vdi][TYPE_TAG] == 'metadata': self.session.xenapi.VDI.set_metadata_of_pool( \ - vdi_ref, Dict[vdi][METADATA_OF_POOL_TAG]) + vdi_ref, info[vdi][METADATA_OF_POOL_TAG]) # Update CBT status of disks either just added # or already in XAPI From c314b5ecfec796c2329f35d1ed3e913b360c21a1 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 14:58:18 +0200 Subject: [PATCH 18/29] Prevent mypy errors when a variable type is changed in `BaseISCSISR` Log without this change on `chappasword` and `incoming_chappassword`: ``` error: Incompatible types in assignment (expression has type "bytes", variable has type "str") [assignment] ``` Signed-off-by: Ronan Abhamon --- drivers/BaseISCSI.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/BaseISCSI.py b/drivers/BaseISCSI.py index 798491d0..a9d06743 100755 --- a/drivers/BaseISCSI.py +++ b/drivers/BaseISCSI.py @@ -175,11 +175,11 @@ def load(self, sr_uuid): and ('chappassword' in self.dconf or 'chappassword_secret' in self.dconf): self.chapuser = self.dconf['chapuser'].encode('utf-8') if 'chappassword_secret' in self.dconf: - self.chappassword = util.get_secret(self.session, self.dconf['chappassword_secret']) + chappassword = util.get_secret(self.session, self.dconf['chappassword_secret']) else: - self.chappassword = self.dconf['chappassword'] + chappassword = self.dconf['chappassword'] - self.chappassword = self.chappassword.encode('utf-8') + self.chappassword = chappassword.encode('utf-8') self.incoming_chapuser = "" self.incoming_chappassword = "" @@ -187,11 +187,11 @@ def load(self, sr_uuid): and ('incoming_chappassword' in self.dconf or 'incoming_chappassword_secret' in self.dconf): self.incoming_chapuser = self.dconf['incoming_chapuser'].encode('utf-8') if 'incoming_chappassword_secret' in self.dconf: - self.incoming_chappassword = util.get_secret(self.session, self.dconf['incoming_chappassword_secret']) + incoming_chappassword = util.get_secret(self.session, self.dconf['incoming_chappassword_secret']) else: - self.incoming_chappassword = self.dconf['incoming_chappassword'] + incoming_chappassword = self.dconf['incoming_chappassword'] - self.incoming_chappassword = self.incoming_chappassword.encode('utf-8') + self.incoming_chappassword = incoming_chappassword.encode('utf-8') self.port = DEFAULT_PORT if 'port' in self.dconf and self.dconf['port']: From 999ad04d864029992e45b11d26120a6ea2e07830 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 16 Oct 2024 15:13:52 +0200 Subject: [PATCH 19/29] Prevent bad mypy error in TestMultiLUNISCSISR using formatted-string Avoid: ``` error: Incompatible types in string interpolation (expression has type "object", placeholder has type "int | float | SupportsInt") [str-format] ``` Signed-off-by: Ronan Abhamon --- tests/test_ISCSISR.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ISCSISR.py b/tests/test_ISCSISR.py index 39529be1..e71ac268 100644 --- a/tests/test_ISCSISR.py +++ b/tests/test_ISCSISR.py @@ -129,7 +129,7 @@ def setUp(self): 'tpgt': 'TPGT' } self.node_records = [( - "%s:%d" % (self.node2['ip'], self.node2['port']), + f"{self.node2['ip']}:{self.node2['port']}", self.node2['tpgt'], self.node2['iqn'] )] From 40b2d2d9873cb0e38d0e6e92889558631db14dd4 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Mon, 21 Oct 2024 18:44:42 +0200 Subject: [PATCH 20/29] Count correctly IQN sessions during ISCSISR attach Before this change, IQNs were concatenated into a single string when `multiSession` was used. Signed-off-by: Ronan Abhamon --- drivers/BaseISCSI.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/BaseISCSI.py b/drivers/BaseISCSI.py index a9d06743..077fa91c 100755 --- a/drivers/BaseISCSI.py +++ b/drivers/BaseISCSI.py @@ -391,10 +391,9 @@ def attach(self, sr_uuid): util._incr_iscsiSR_refcount(self.targetIQN, sr_uuid) IQNs = [] if "multiSession" in self.dconf: - IQNs = "" for iqn in self.dconf['multiSession'].split("|"): if len(iqn): - IQNs += iqn.split(',')[2] + IQNs.append(iqn.split(',')[2]) else: IQNs.append(self.targetIQN) From 03850cac616f61ea33f4280309d5f140702c9571 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Thu, 10 Oct 2024 14:58:02 +0200 Subject: [PATCH 21/29] Use importlib instead of imp which is deprecated in python 3.4 Signed-off-by: Ronan Abhamon --- drivers/SR.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/SR.py b/drivers/SR.py index 9ca0045c..d1aacdb3 100755 --- a/drivers/SR.py +++ b/drivers/SR.py @@ -153,7 +153,7 @@ def __init__(self, srcmd, sr_uuid): @staticmethod def from_uuid(session, sr_uuid): - import imp + import importlib.util _SR = session.xenapi.SR sr_ref = _SR.get_by_uuid(sr_uuid) @@ -169,7 +169,10 @@ def from_uuid(session, sr_uuid): driver_real = os.path.realpath(driver_path) module_name = os.path.basename(driver_path) - module = imp.load_source(module_name, driver_real) + spec = importlib.util.spec_from_file_location(module_name, driver_real) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + target = driver(sm_type) # NB. get the host pbd's device_config From 03a6de9bddf9dd5db7c4640e49ff10a1f7d898ad Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 11 Oct 2024 15:48:29 +0200 Subject: [PATCH 22/29] Replace deprecated calls to distutils.spawn.find_executable Signed-off-by: Ronan Abhamon --- drivers/CephFSSR.py | 3 +-- drivers/GlusterFSSR.py | 3 +-- drivers/MooseFSSR.py | 3 +-- drivers/XFSSR.py | 3 +-- drivers/ZFSSR.py | 3 +-- drivers/util.py | 4 ++++ 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/CephFSSR.py b/drivers/CephFSSR.py index ca79a6a3..80884767 100644 --- a/drivers/CephFSSR.py +++ b/drivers/CephFSSR.py @@ -264,8 +264,7 @@ def vdi(self, uuid, loadLocked=False): @staticmethod def _is_ceph_available(): - import distutils.spawn - return distutils.spawn.find_executable('ceph') + return util.find_executable('ceph') class CephFSFileVDI(FileSR.FileVDI): def attach(self, sr_uuid, vdi_uuid): diff --git a/drivers/GlusterFSSR.py b/drivers/GlusterFSSR.py index 8adfe700..041a9142 100644 --- a/drivers/GlusterFSSR.py +++ b/drivers/GlusterFSSR.py @@ -244,8 +244,7 @@ def vdi(self, uuid, loadLocked=False): @staticmethod def _is_glusterfs_available(): - import distutils.spawn - return distutils.spawn.find_executable('glusterfs') + return util.find_executable('glusterfs') class GlusterFSFileVDI(FileSR.FileVDI): diff --git a/drivers/MooseFSSR.py b/drivers/MooseFSSR.py index 6f86054f..e4dc4ad6 100755 --- a/drivers/MooseFSSR.py +++ b/drivers/MooseFSSR.py @@ -276,8 +276,7 @@ def vdi(self, uuid, loadLocked=False): @staticmethod def _is_moosefs_available(): - import distutils.spawn - return distutils.spawn.find_executable('mfsmount') + return util.find_executable('mfsmount') class MooseFSFileVDI(FileSR.FileVDI): def attach(self, sr_uuid, vdi_uuid): diff --git a/drivers/XFSSR.py b/drivers/XFSSR.py index 1dfde095..ad4aca74 100755 --- a/drivers/XFSSR.py +++ b/drivers/XFSSR.py @@ -229,8 +229,7 @@ def vdi(self, uuid, loadLocked = False): @staticmethod def _is_xfs_available(): - import distutils.spawn - return distutils.spawn.find_executable('mkfs.xfs') + return util.find_executable('mkfs.xfs') class XFSFileVDI(FileSR.FileVDI): diff --git a/drivers/ZFSSR.py b/drivers/ZFSSR.py index 5301d5ec..cf5eb12d 100644 --- a/drivers/ZFSSR.py +++ b/drivers/ZFSSR.py @@ -58,8 +58,7 @@ def is_zfs_available(): - import distutils.spawn - return distutils.spawn.find_executable('zfs') and \ + return util.find_executable('zfs') and \ util.pathexists('/sys/module/zfs/initstate') diff --git a/drivers/util.py b/drivers/util.py index d9356d67..759648af 100755 --- a/drivers/util.py +++ b/drivers/util.py @@ -2091,3 +2091,7 @@ def gen_path(path): cProfile.runctx('function()', None, locals(), profile_path) finally: SMlog('* End profiling of {} ({}) *'.format(name, filename)) + + +def find_executable(name): + return shutil.which(name) From 23a9256bc3dedf0c8584c6dbef4dc71d5fa17d95 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 11 Oct 2024 15:52:18 +0200 Subject: [PATCH 23/29] Replace deprecated calls to distutils.util.strtobool Signed-off-by: Ronan Abhamon --- drivers/LinstorSR.py | 3 +-- drivers/MooseFSSR.py | 7 ++----- drivers/linstor-manager | 15 +++++++-------- drivers/linstorvhdutil.py | 5 ++--- drivers/linstorvolumemanager.py | 5 ++--- drivers/util.py | 17 +++++++++++++++++ 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index 81c4b7e0..1197723a 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -34,7 +34,6 @@ from lock import Lock, LOCK_TYPE_GC_RUNNING import blktap2 import cleanup -import distutils import errno import functools import lvutil @@ -337,7 +336,7 @@ def load(self, sr_uuid): monitor_db_quorum = self.dconf.get('monitor-db-quorum') self._monitor_db_quorum = (monitor_db_quorum is None) or \ - distutils.util.strtobool(monitor_db_quorum) + util.strtobool(monitor_db_quorum) # Note: We don't have access to the session field if the # 'vdi_attach_from_config' command is executed. diff --git a/drivers/MooseFSSR.py b/drivers/MooseFSSR.py index e4dc4ad6..8fc4a4ad 100755 --- a/drivers/MooseFSSR.py +++ b/drivers/MooseFSSR.py @@ -18,7 +18,6 @@ # # MooseFSSR: Based on CEPHFSSR and FileSR, mounts MooseFS share -import distutils.util import errno import os import syslog as _syslog @@ -113,9 +112,7 @@ def load(self, sr_uuid): self.sm_config = self.srcmd.params.get('sr_sm_config') or {} if self.srcmd.cmd != 'sr_create': - self.subdir = distutils.util.strtobool( - self.sm_config.get('subdir') or '0' - ) + self.subdir = util.strtobool(self.sm_config.get('subdir')) if self.subdir: self.remotepath = os.path.join(self.remotepath, sr_uuid) @@ -228,7 +225,7 @@ def create(self, sr_uuid, size): if self.subdir is None: self.subdir = True else: - self.subdir = distutils.util.strtobool(self.subdir) + self.subdir = util.strtobool(self.subdir) self.sm_config['subdir'] = str(self.subdir) self.session.xenapi.SR.set_sm_config(self.sr_ref, self.sm_config) diff --git a/drivers/linstor-manager b/drivers/linstor-manager index 8ee6f149..fb2b9fe6 100755 --- a/drivers/linstor-manager +++ b/drivers/linstor-manager @@ -21,7 +21,6 @@ import sys sys.path[0] = '/opt/xensource/sm/' import base64 -import distutils.util import os import socket import XenAPI @@ -313,7 +312,7 @@ def release_sr(session, args): def update_drbd_reactor(session, args): try: - enabled = distutils.util.strtobool(args['enabled']) + enabled = util.strtobool(args['enabled']) update_drbd_reactor_service(start=enabled) return str(True) except Exception as e: @@ -389,10 +388,10 @@ def destroy(session, args): def check(session, args): try: device_path = args['devicePath'] - ignore_missing_footer = distutils.util.strtobool( + ignore_missing_footer = util.strtobool( args['ignoreMissingFooter'] ) - fast = distutils.util.strtobool(args['fast']) + fast = util.strtobool(args['fast']) check_ex(device_path, ignore_missing_footer, fast) return str(True) except Exception as e: @@ -404,7 +403,7 @@ def get_vhd_info(session, args): try: device_path = args['devicePath'] group_name = args['groupName'] - include_parent = distutils.util.strtobool(args['includeParent']) + include_parent = util.strtobool(args['includeParent']) linstor = LinstorVolumeManager( get_controller_uri(), @@ -560,7 +559,7 @@ def deflate(session, args): device_path = args['devicePath'] new_size = int(args['newSize']) old_size = int(args['oldSize']) - zeroize = distutils.util.strtobool(args['zeroize']) + zeroize = util.strtobool(args['zeroize']) group_name = args['groupName'] linstor = LinstorVolumeManager( @@ -581,7 +580,7 @@ def lock_vdi(session, args): sr_uuid = args['srUuid'] vdi_uuid = args['vdiUuid'] group_name = args['groupName'] - locked = distutils.util.strtobool(args['locked']) + locked = util.strtobool(args['locked']) # We must lock to mark the VDI. lock = Lock(vhdutil.LOCK_TYPE_SR, sr_uuid) @@ -824,7 +823,7 @@ def create_sr(session, args): elif provisioning != 'thin' and provisioning != 'thick': raise Exception('unsupported provisioning') - force = distutils.util.strtobool(args.get('force') or '0') + force = util.strtobool(args.get('force')) return exec_create_sr( session, name, description, disks, volume_group, redundancy, provisioning, force diff --git a/drivers/linstorvhdutil.py b/drivers/linstorvhdutil.py index b3df004a..6ad4787d 100644 --- a/drivers/linstorvhdutil.py +++ b/drivers/linstorvhdutil.py @@ -17,7 +17,6 @@ from linstorjournaler import LinstorJournaler from linstorvolumemanager import LinstorVolumeManager import base64 -import distutils.util import errno import json import socket @@ -205,7 +204,7 @@ def check(self, vdi_uuid, ignore_missing_footer=False, fast=False): @linstorhostcall(check_ex, 'check') def _check(self, vdi_uuid, response): - return distutils.util.strtobool(response) + return util.strtobool(response) def get_vhd_info(self, vdi_uuid, include_parent=True): kwargs = { @@ -233,7 +232,7 @@ def _get_vhd_info(self, vdi_uuid, response): @linstorhostcall(vhdutil.hasParent, 'hasParent') def has_parent(self, vdi_uuid, response): - return distutils.util.strtobool(response) + return util.strtobool(response) def get_parent(self, vdi_uuid): return self._get_parent(vdi_uuid, self._extract_uuid) diff --git a/drivers/linstorvolumemanager.py b/drivers/linstorvolumemanager.py index a470dfec..55327254 100755 --- a/drivers/linstorvolumemanager.py +++ b/drivers/linstorvolumemanager.py @@ -16,7 +16,6 @@ # -import distutils.util import errno import json import linstor @@ -183,7 +182,7 @@ def _get_controller_uri(): for host_ref, host_record in session.xenapi.host.get_all_records().items(): node_name = host_record['hostname'] try: - if distutils.util.strtobool( + if util.strtobool( session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, {}) ): return 'linstor://' + host_record['address'] @@ -234,7 +233,7 @@ def get_controller_node_name(): )['live']: continue - if distutils.util.strtobool(session.xenapi.host.call_plugin( + if util.strtobool(session.xenapi.host.call_plugin( host_ref, PLUGIN, PLUGIN_CMD, {} )): return node_name diff --git a/drivers/util.py b/drivers/util.py index 759648af..544603a3 100755 --- a/drivers/util.py +++ b/drivers/util.py @@ -2093,5 +2093,22 @@ def gen_path(path): SMlog('* End profiling of {} ({}) *'.format(name, filename)) +def strtobool(str): + # Note: `distutils` package is deprecated and slated for removal in Python 3.12. + # There is not alternative for strtobool. + # See: https://peps.python.org/pep-0632/#migration-advice + # So this is a custom implementation with differences: + # - A boolean is returned instead of integer + # - Empty string and None are supported (False is returned in this case) + if not str: + return False + str = str.lower() + if str in ('y', 'yes', 't', 'true', 'on', '1'): + return True + if str in ('n', 'no', 'f', 'false', 'off', '0'): + return False + raise ValueError("invalid truth value '{}'".format(str)) + + def find_executable(name): return shutil.which(name) From 21107692fb711ec01a36f36132e32c37c951a992 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 11 Oct 2024 15:55:02 +0200 Subject: [PATCH 24/29] Fix _locked_load calls compatibility with python 3.10 Signed-off-by: Ronan Abhamon --- drivers/LinstorSR.py | 6 ++++++ drivers/util.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index 1197723a..22c1b37e 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -45,6 +45,7 @@ import SR import SRCommand import subprocess +import sys import time import traceback import util @@ -371,6 +372,9 @@ def load(self, sr_uuid): self._all_volume_info_cache = None self._all_volume_metadata_cache = None + # To remove in python 3.10. + # Use directly @staticmethod instead. + @util.conditional_decorator(staticmethod, sys.version_info >= (3, 10, 0)) def _locked_load(method): def wrapped_method(self, *args, **kwargs): self._init_status = self.INIT_STATUS_OK @@ -792,6 +796,8 @@ def is_master(self): def vdi(self, uuid): return LinstorVDI(self, uuid) + # To remove in python 3.10 + # See: https://stackoverflow.com/questions/12718187/python-version-3-9-calling-class-staticmethod-within-the-class-body _locked_load = staticmethod(_locked_load) # -------------------------------------------------------------------------- diff --git a/drivers/util.py b/drivers/util.py index 544603a3..c027f1c8 100755 --- a/drivers/util.py +++ b/drivers/util.py @@ -2112,3 +2112,11 @@ def strtobool(str): def find_executable(name): return shutil.which(name) + + +def conditional_decorator(decorator, condition): + def wrapper(func): + if not condition: + return func + return decorator(func) + return wrapper From 9cc1409b13bbc35aed4077eb2a3c8666811d808c Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 11 Oct 2024 17:51:25 +0200 Subject: [PATCH 25/29] Use static analysis tool (mypy) Signed-off-by: Ronan Abhamon --- .github/workflows/main.yml | 19 +++++++++++++++++++ Makefile | 1 + dev_requirements_static_analysis.txt | 3 +++ drivers/DummySR.py | 4 +++- drivers/FileSR.py | 4 +++- drivers/LVHDSR.py | 4 +++- drivers/blktap2.py | 19 +++++++++++-------- drivers/flock.py | 4 +++- drivers/lock.py | 6 ++++-- drivers/mpathcount.py | 6 ++++-- drivers/srmetadata.py | 5 ++++- misc/fairlock/fairlock.py | 6 ++++-- sm_typing/Makefile | 12 ++++++++++++ sm_typing/__init__.py | 2 ++ tests/test_cleanup.py | 4 +++- 15 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 dev_requirements_static_analysis.txt create mode 100644 sm_typing/Makefile create mode 100644 sm_typing/__init__.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6271a21b..d89b968c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,25 @@ name: Test SM on: [push, pull_request] jobs: + static-analysis: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3 + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r dev_requirements_static_analysis.txt + + - name: mypy + run: | + mypy . + build: runs-on: ubuntu-20.04 diff --git a/Makefile b/Makefile index 472413e5..84cbb5a7 100755 --- a/Makefile +++ b/Makefile @@ -231,6 +231,7 @@ install: precheck install -m 755 scripts/set-iscsi-initiator $(SM_STAGING)$(LIBEXEC) $(MAKE) -C dcopy install DESTDIR=$(SM_STAGING) $(MAKE) -C linstor install DESTDIR=$(SM_STAGING) + $(MAKE) -C sm_typing install DESTDIR=$(SM_STAGING) ln -sf $(SM_DEST)blktap2.py $(SM_STAGING)$(BIN_DEST)/blktap2 ln -sf $(SM_DEST)lcache.py $(SM_STAGING)$(BIN_DEST)tapdisk-cache-stats ln -sf /dev/null $(SM_STAGING)$(UDEV_RULES_DIR)/69-dm-lvm-metad.rules diff --git a/dev_requirements_static_analysis.txt b/dev_requirements_static_analysis.txt new file mode 100644 index 00000000..8a59c8f2 --- /dev/null +++ b/dev_requirements_static_analysis.txt @@ -0,0 +1,3 @@ +bitarray +mypy +python-linstor diff --git a/drivers/DummySR.py b/drivers/DummySR.py index c733fdf7..be0298b9 100755 --- a/drivers/DummySR.py +++ b/drivers/DummySR.py @@ -17,6 +17,8 @@ # # DummySR: an example dummy SR for the SDK +from sm_typing import List, Tuple + import SR import VDI import SRCommand @@ -28,7 +30,7 @@ "VDI_ACTIVATE", "VDI_DEACTIVATE", "VDI_CLONE", "VDI_SNAPSHOT", "VDI_RESIZE", "VDI_INTRODUCE", "VDI_MIRROR"] -CONFIGURATION = [] +CONFIGURATION: List[Tuple[str, str]] = [] DRIVER_INFO = { 'name': 'dummy', diff --git a/drivers/FileSR.py b/drivers/FileSR.py index 4d2e0a21..9f2ec28f 100755 --- a/drivers/FileSR.py +++ b/drivers/FileSR.py @@ -17,6 +17,8 @@ # # FileSR: local-file storage repository +from sm_typing import Dict, List + import SR import VDI import SRCommand @@ -36,7 +38,7 @@ import XenAPI # pylint: disable=import-error from constants import CBTLOG_TAG -geneology = {} +geneology: Dict[str, List[str]] = {} CAPABILITIES = ["SR_PROBE", "SR_UPDATE", \ "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH", \ "VDI_CLONE", "VDI_SNAPSHOT", "VDI_RESIZE", "VDI_MIRROR", diff --git a/drivers/LVHDSR.py b/drivers/LVHDSR.py index 361e88eb..0cb86bea 100755 --- a/drivers/LVHDSR.py +++ b/drivers/LVHDSR.py @@ -18,6 +18,8 @@ # LVHDSR: VHD on LVM storage repository # +from sm_typing import Dict, List + import SR from SR import deviceCheck import VDI @@ -54,7 +56,7 @@ from fairlock import Fairlock DEV_MAPPER_ROOT = os.path.join('/dev/mapper', lvhdutil.VG_PREFIX) -geneology = {} +geneology: Dict[str, List[str]] = {} CAPABILITIES = ["SR_PROBE", "SR_UPDATE", "SR_TRIM", "VDI_CREATE", "VDI_DELETE", "VDI_ATTACH", "VDI_DETACH", "VDI_MIRROR", "VDI_CLONE", "VDI_SNAPSHOT", "VDI_RESIZE", "ATOMIC_PAUSE", diff --git a/drivers/blktap2.py b/drivers/blktap2.py index 669b9f65..672f7448 100755 --- a/drivers/blktap2.py +++ b/drivers/blktap2.py @@ -17,6 +17,9 @@ # # blktap2: blktap/tapdisk management layer # + +from sm_typing import Any, Callable, ClassVar, Dict + import grp import os import re @@ -513,7 +516,7 @@ def mkdirs(path, mode=0o777): class KObject(object): - SYSFS_CLASSTYPE = None + SYSFS_CLASSTYPE: ClassVar[str] = "" def sysfs_devname(self): raise NotImplementedError("sysfs_devname is undefined") @@ -521,7 +524,7 @@ def sysfs_devname(self): class Attribute(object): - SYSFS_NODENAME = None + SYSFS_NODENAME: ClassVar[str] = "" def __init__(self, path): self.path = path @@ -1171,7 +1174,7 @@ class Link(object): # before VDI.activate. Therefore those link steps where we # relink existing devices under deterministic path names. - BASEDIR = None + BASEDIR: ClassVar[str] = "" def _mklink(self, target): raise NotImplementedError("_mklink is not defined") @@ -2130,7 +2133,7 @@ def __str__(self): return "Uevent '%s' not handled by %s" % \ (self.event, self.handler.__class__.__name__) - ACTIONS = {} + ACTIONS: Dict[str, Callable] = {} def run(self): @@ -2226,7 +2229,7 @@ def get_size(self): class BusDevice(KObject): - SYSFS_BUSTYPE = None + SYSFS_BUSTYPE: ClassVar[str] = "" @classmethod def sysfs_bus_path(cls): @@ -2244,7 +2247,7 @@ class XenbusDevice(BusDevice): XBT_NIL = "" - XENBUS_DEVTYPE = None + XENBUS_DEVTYPE: ClassVar[str] = "" def __init__(self, domid, devid): self.domid = int(domid) @@ -2393,7 +2396,7 @@ def __init__(self, domid, devid): self._q_events = None class XenstoreValueError(Exception): - KEY = None + KEY: ClassVar[str] = "" def __init__(self, vbd, _str): self.vbd = vbd @@ -2830,7 +2833,7 @@ def usage(stream): elif _class == 'tap': - attrs = {} + attrs: Dict[str, Any] = {} for item in sys.argv[2:]: try: key, val = item.split('=') diff --git a/drivers/flock.py b/drivers/flock.py index dceb0428..2d295ec4 100644 --- a/drivers/flock.py +++ b/drivers/flock.py @@ -23,6 +23,8 @@ got to grow our own. """ +from sm_typing import ClassVar + import os import fcntl import struct @@ -73,7 +75,7 @@ class FcntlLockBase: definition of LOCK_TYPE (fcntl.{F_RDLCK|F_WRLCK}) determines the type.""" - LOCK_TYPE = None + LOCK_TYPE: ClassVar[int] if __debug__: ERROR_ISLOCKED = "Attempt to acquire lock held." diff --git a/drivers/lock.py b/drivers/lock.py index 56bf9fcc..2e6e2c9c 100755 --- a/drivers/lock.py +++ b/drivers/lock.py @@ -16,6 +16,8 @@ """Serialization for concurrent operations""" +from sm_typing import Dict + import os import errno import flock @@ -37,8 +39,8 @@ class Lock(object): BASE_DIR = "/var/lock/sm" - INSTANCES = {} - BASE_INSTANCES = {} + INSTANCES: Dict[str, 'LockImplementation'] = {} + BASE_INSTANCES: Dict[str, 'LockImplementation'] = {} def __new__(cls, name, ns=None, *args, **kwargs): if ns: diff --git a/drivers/mpathcount.py b/drivers/mpathcount.py index a31ec1b7..73123a8c 100755 --- a/drivers/mpathcount.py +++ b/drivers/mpathcount.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from sm_typing import Dict + import util import os import sys @@ -238,7 +240,7 @@ def _add(key, val): mpc_exit(session, -1) try: - mpath_status = {} + mpath_status: Dict[str, str] = {} for pbd in pbds: def remove(key): session.xenapi.PBD.remove_from_other_config(pbd, key) @@ -258,7 +260,7 @@ def add(key, val): util.atomicFileWrite(MPATH_FILE_NAME, MPATHS_DIR, json.dumps(mpath_status)) os.chmod(MPATH_FILE_NAME, 0o0644) except: - util.SMlog("MPATH: Failure updating db. %s" % sys.exc_info()) + util.SMlog("MPATH: Failure updating db. %s" % str(sys.exc_info())) mpc_exit(session, -1) util.SMlog("MPATH: Update done") diff --git a/drivers/srmetadata.py b/drivers/srmetadata.py index f86711e2..c80fb6d7 100755 --- a/drivers/srmetadata.py +++ b/drivers/srmetadata.py @@ -15,6 +15,9 @@ # # Functions to read and write SR metadata # + +from sm_typing import ClassVar + from io import SEEK_SET import util @@ -254,7 +257,7 @@ def getMetadataLength(fd): # ----------------- # General helper functions - end # ----------------- class MetadataHandler: - VDI_INFO_SIZE_IN_SECTORS = None + VDI_INFO_SIZE_IN_SECTORS: ClassVar[int] # constructor def __init__(self, path=None, write=True): diff --git a/misc/fairlock/fairlock.py b/misc/fairlock/fairlock.py index 9217c697..391fbea3 100644 --- a/misc/fairlock/fairlock.py +++ b/misc/fairlock/fairlock.py @@ -1,3 +1,5 @@ +from sm_typing import Any, Callable, Dict, Optional + import os import socket import inspect @@ -7,8 +9,8 @@ START_SERVICE_TIMEOUT_SECS = 2 class SingletonWithArgs(type): - _instances = {} - _init = {} + _instances: Dict[Any, Any] = {} + _init: Dict[type, Optional[Callable[..., None]]] = {} def __init__(cls, name, bases, dct): cls._init[cls] = dct.get('__init__', None) diff --git a/sm_typing/Makefile b/sm_typing/Makefile new file mode 100644 index 00000000..1e2ea815 --- /dev/null +++ b/sm_typing/Makefile @@ -0,0 +1,12 @@ +PYTHONLIBDIR = $(shell python3 -c "import sys; print(sys.path.pop())") +DESTDIR ?= + +.PHONY: install +install: + install -D -m 644 __init__.py $(DESTDIR)$(PYTHONLIBDIR)/sm_typing.py + python3 -m compileall $(DESTDIR)$(PYTHONLIBDIR)/sm_typing.py + +.PHONY: uninstall +uninstall: + rm -rf $(DESTDIR)$(PYTHONLIBDIR)/sm_typing.py + rm -rf $(DESTDIR)$(PYTHONLIBDIR)/__pycache__/sm_typing.* diff --git a/sm_typing/__init__.py b/sm_typing/__init__.py new file mode 100644 index 00000000..c515056c --- /dev/null +++ b/sm_typing/__init__.py @@ -0,0 +1,2 @@ +import typing +from typing import * diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index cc4570d9..bd789515 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -1,3 +1,5 @@ +from sm_typing import List + import errno import unittest import unittest.mock as mock @@ -22,7 +24,7 @@ class FakeException(Exception): class FakeUtil: - record = [] + record: List[str] = [] def log(input): FakeUtil.record.append(input) From dbf1312383c2a8923ce6bbc21ad12d1e00dcf4cf Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Wed, 9 Oct 2024 15:44:05 +0200 Subject: [PATCH 26/29] Add mypy stubs Co-authored-by: Damien Thenot Co-authored-by: Ronan Abhamon Signed-off-by: Damien Thenot --- stubs/XenAPIPlugin.pyi | 5 +++++ stubs/xen/__init__.pyi | 0 stubs/xen/lowlevel/xs.pyi | 0 3 files changed, 5 insertions(+) create mode 100644 stubs/XenAPIPlugin.pyi create mode 100644 stubs/xen/__init__.pyi create mode 100644 stubs/xen/lowlevel/xs.pyi diff --git a/stubs/XenAPIPlugin.pyi b/stubs/XenAPIPlugin.pyi new file mode 100644 index 00000000..ca5f52ca --- /dev/null +++ b/stubs/XenAPIPlugin.pyi @@ -0,0 +1,5 @@ +class Failure(Exception): + def __init__(self, code, params) -> None: ... + def __str__(self) -> str: ... + +def dispatch(fn_table) -> None: ... diff --git a/stubs/xen/__init__.pyi b/stubs/xen/__init__.pyi new file mode 100644 index 00000000..e69de29b diff --git a/stubs/xen/lowlevel/xs.pyi b/stubs/xen/lowlevel/xs.pyi new file mode 100644 index 00000000..e69de29b From e4f2b5b8a7f8ad0c5642a78c6992e2685fb49e26 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Fri, 8 Nov 2024 13:44:57 +0100 Subject: [PATCH 27/29] Use `override` everywhere! Also use return type annotations on these methods. Signed-off-by: Ronan Abhamon --- .mypy.ini | 3 + drivers/BaseISCSI.py | 42 +++-- drivers/CephFSSR.py | 43 +++-- drivers/DummySR.py | 72 ++++--- drivers/EXTSR.py | 32 +++- drivers/FileSR.py | 102 ++++++---- drivers/GlusterFSSR.py | 43 +++-- drivers/HBASR.py | 36 ++-- drivers/ISOSR.py | 56 ++++-- drivers/LUNperVDI.py | 20 +- drivers/LVHDSR.py | 95 ++++++---- drivers/LVHDoFCoESR.py | 14 +- drivers/LVHDoHBASR.py | 38 ++-- drivers/LVHDoISCSISR.py | 41 ++-- drivers/LargeBlockSR.py | 23 ++- drivers/LinstorSR.py | 76 +++++--- drivers/MooseFSSR.py | 43 +++-- drivers/NFSSR.py | 40 ++-- drivers/RawISCSISR.py | 20 +- drivers/SHMSR.py | 42 +++-- drivers/SMBSR.py | 41 ++-- drivers/SR.py | 37 ++-- drivers/VDI.py | 54 +++--- drivers/XFSSR.py | 32 +++- drivers/ZFSSR.py | 32 +++- drivers/blktap2.py | 129 ++++++++----- drivers/cleanup.py | 320 ++++++++++++++++++++------------ drivers/flock.py | 5 +- drivers/lcache.py | 14 +- drivers/linstorvhdutil.py | 5 +- drivers/linstorvolumemanager.py | 4 +- drivers/lock.py | 2 +- drivers/mpath_cli.py | 5 +- drivers/nfs-on-slave | 6 +- drivers/srmetadata.py | 22 ++- drivers/udevSR.py | 53 ++++-- misc/fairlock/fairlock.py | 7 +- sm_typing/__init__.py | 12 ++ stubs/XenAPIPlugin.pyi | 2 +- tests/lvmlib.py | 8 +- tests/shared_iscsi_test_base.py | 11 +- tests/test_BaseISCSI.py | 5 +- tests/test_FileSR.py | 28 ++- tests/test_ISCSISR.py | 21 ++- tests/test_LVHDSR.py | 16 +- tests/test_LVHDoHBASR.py | 6 +- tests/test_LVHDoISCSISR.py | 10 +- tests/test_SMBSR.py | 8 +- tests/test_SR.py | 8 +- tests/test_blktap2.py | 12 +- tests/test_cbt.py | 43 +++-- tests/test_cleanup.py | 11 +- tests/test_fairlock.py | 5 +- tests/test_fjournaler.py | 8 +- tests/test_keymanagerutil.py | 9 +- tests/test_lock.py | 9 +- tests/test_lock_queue.py | 5 +- tests/test_lvutil.py | 15 +- tests/test_mpath_dmp.py | 6 +- tests/test_on_slave.py | 11 +- tests/test_sr_health_check.py | 6 +- tests/test_srmetadata.py | 11 +- tests/test_storage_init.py | 14 +- tests/test_util.py | 16 +- tests/testlib.py | 13 +- 65 files changed, 1296 insertions(+), 682 deletions(-) create mode 100644 .mypy.ini diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 00000000..651016c0 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,3 @@ +[mypy] +enable_error_code = explicit-override + diff --git a/drivers/BaseISCSI.py b/drivers/BaseISCSI.py index 077fa91c..34ce5f5e 100755 --- a/drivers/BaseISCSI.py +++ b/drivers/BaseISCSI.py @@ -18,7 +18,10 @@ # ISCSISR: ISCSI software initiator SR driver # +from sm_typing import override + import SR +import VDI import util import time import LUNperVDI @@ -100,11 +103,12 @@ def address(self): self._initPaths() return self._address - def handles(type): + @override + @staticmethod + def handles(type) -> bool: return False - handles = staticmethod(handles) - def _synchroniseAddrList(self, addrlist): + def _synchroniseAddrList(self, addrlist) -> None: if not self.multihomed: return change = False @@ -133,7 +137,8 @@ def _synchroniseAddrList(self, addrlist): except: pass - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if self.force_tapdisk: self.sr_vditype = 'aio' else: @@ -276,7 +281,7 @@ def _initPaths(self): self._address = self.tgtidx self._synchroniseAddrList(addrlist) - def _init_adapters(self): + def _init_adapters(self) -> None: # Generate a list of active adapters ids = scsiutil._genHostList(ISCSI_PROCNAME) util.SMlog(ids) @@ -293,7 +298,8 @@ def _init_adapters(self): pass self._devs = scsiutil.cacheSCSIidentifiers() - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: self._mpathHandle() multiTargets = False @@ -428,10 +434,11 @@ def attach(self, sr_uuid): realdev = os.path.realpath(os.path.join(dev_path, dev)) util.set_scheduler(os.path.basename(realdev)) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: self.detach_and_delete(sr_uuid, delete=False) - def detach_and_delete(self, sr_uuid, delete=True): + def detach_and_delete(self, sr_uuid, delete=True) -> None: keys = [] pbdref = None try: @@ -472,7 +479,8 @@ def detach_and_delete(self, sr_uuid, delete=True): self.attached = False - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: # Check whether an SR already exists SRs = self.session.xenapi.SR.get_all_records() for sr in SRs: @@ -501,11 +509,13 @@ def create(self, sr_uuid, size): self.session.xenapi.SR.set_sm_config(self.sr_ref, self.sm_config) return - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: self.detach(sr_uuid) return - def probe(self): + @override + def probe(self) -> str: SRs = self.session.xenapi.SR.get_all_records() Recs = {} for sr in SRs: @@ -515,8 +525,9 @@ def probe(self): sm_config['targetIQN'] == self.targetIQN: Recs[record["uuid"]] = sm_config return self.srlist_toxml(Recs) - - def scan(self, sr_uuid): + + @override + def scan(self, sr_uuid) -> None: if not self.passthrough: if not self.attached: raise xs_errors.XenError('SRUnavailable') @@ -528,9 +539,10 @@ def scan(self, sr_uuid): if vdi.managed: self.physical_utilisation += vdi.size self.virtual_allocation = self.physical_utilisation - return super(BaseISCSISR, self).scan(sr_uuid) + super(BaseISCSISR, self).scan(sr_uuid) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return LUNperVDI.RAWVDI(self, uuid) def _scan_IQNs(self): diff --git a/drivers/CephFSSR.py b/drivers/CephFSSR.py index 80884767..9e8e46e1 100644 --- a/drivers/CephFSSR.py +++ b/drivers/CephFSSR.py @@ -18,6 +18,8 @@ # # CEPHFSSR: Based on FileSR, mounts ceph fs share +from sm_typing import override + import errno import os import socket @@ -33,6 +35,7 @@ import SRCommand import FileSR # end of careful +import VDI import cleanup import util import vhdutil @@ -83,13 +86,14 @@ class CephFSSR(FileSR.FileSR): DRIVER_TYPE = 'cephfs' - def handles(sr_type): + @override + @staticmethod + def handles(sr_type) -> bool: # fudge, because the parent class (FileSR) checks for smb to alter its behavior return sr_type == CephFSSR.DRIVER_TYPE or sr_type == 'smb' - handles = staticmethod(handles) - - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not self._is_ceph_available(): raise xs_errors.XenError( 'SRUnavailable', @@ -180,7 +184,8 @@ def unmount(self, mountpoint, rmmountpoint): except OSError as inst: raise CephFSException("rmdir failed with error '%s'" % inst.strerror) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self.checkmount(): try: self.mount() @@ -189,7 +194,8 @@ def attach(self, sr_uuid): raise xs_errors.SROSError(12, exc.errstr) self.attached = True - def probe(self): + @override + def probe(self) -> str: try: self.mount(PROBE_MOUNTPOINT) sr_list = filter(util.match_uuid, util.listdir(PROBE_MOUNTPOINT)) @@ -199,7 +205,8 @@ def probe(self): # Create a dictionary from the SR uuids to feed SRtoXML() return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: if not self.checkmount(): return util.SMlog("Aborting GC/coalesce") @@ -210,7 +217,8 @@ def detach(self, sr_uuid): os.unlink(self.path) self.attached = False - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: if self.checkmount(): raise xs_errors.SROSError(113, 'CephFS mount point already attached') @@ -244,7 +252,8 @@ def create(self, sr_uuid, size): os.strerror(inst.code))) self.detach(sr_uuid) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: # try to remove/delete non VDI contents first super(CephFSSR, self).delete(sr_uuid) try: @@ -259,7 +268,8 @@ def delete(self, sr_uuid): if inst.code != errno.ENOENT: raise xs_errors.SROSError(114, "Failed to remove CephFS mount point") - def vdi(self, uuid, loadLocked=False): + @override + def vdi(self, uuid, loadLocked=False) -> VDI.VDI: return CephFSFileVDI(self, uuid) @staticmethod @@ -267,7 +277,8 @@ def _is_ceph_available(): return util.find_executable('ceph') class CephFSFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} @@ -275,7 +286,8 @@ def attach(self, sr_uuid, vdi_uuid): return super(CephFSFileVDI, self).attach(sr_uuid, vdi_uuid) - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("SMBFileVDI.generate_config") if not util.pathexists(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -289,15 +301,16 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), "vdi_attach_from_config") return xmlrpc.client.dumps((config,), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: try: if not util.pathexists(self.sr.path): - self.sr.attach(sr_uuid) + return self.sr.attach(sr_uuid) except: util.logException("SMBFileVDI.attach_from_config") raise xs_errors.XenError('SRUnavailable', opterr='Unable to attach from config') - + return '' if __name__ == '__main__': SRCommand.run(CephFSSR, DRIVER_INFO) diff --git a/drivers/DummySR.py b/drivers/DummySR.py index be0298b9..f5674db0 100755 --- a/drivers/DummySR.py +++ b/drivers/DummySR.py @@ -17,7 +17,7 @@ # # DummySR: an example dummy SR for the SDK -from sm_typing import List, Tuple +from sm_typing import Dict, Optional, List, Tuple, override import SR import VDI @@ -49,35 +49,43 @@ class DummySR(SR.SR): """dummy storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if type == TYPE: return True return False - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.sr_vditype = 'phy' - def content_type(self, sr_uuid): + @override + def content_type(self, sr_uuid) -> str: return super(DummySR, self).content_type(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: self._assertValues(['sr_uuid', 'args', 'host_ref', 'session_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.srcmd.params['args']) == 1) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: self._assertValues(['sr_uuid', 'args', 'host_ref', 'session_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.srcmd.params['args']) == 0) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: self._assertValues(['sr_uuid', 'args', 'host_ref', 'session_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.srcmd.params['args']) == 0) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: self._assertValues(['sr_uuid', 'args', 'host_ref', 'session_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.srcmd.params['args']) == 0) - def probe(self): + @override + def probe(self) -> str: # N.B. There are no SR references self._assertValues(['args', 'host_ref', 'session_ref', 'device_config', 'command']) assert(len(self.srcmd.params['args']) == 0) @@ -91,10 +99,12 @@ def probe(self): # Return the Probe XML return util.SRtoXML(SRlist) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return DummyVDI(self, uuid) - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: self._assertValues(['sr_uuid', 'args', 'host_ref', 'session_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.srcmd.params['args']) == 0) @@ -108,7 +118,7 @@ def scan(self, sr_uuid): self.physical_size = 2000000000000 self.physical_utilisation = 0 self.virtual_allocation = 0 - return super(DummySR, self).scan(sr_uuid) + super(DummySR, self).scan(sr_uuid) def _assertValues(self, vals): for attr in vals: @@ -134,15 +144,17 @@ def _getallVDIrecords(self): class DummyVDI(VDI.VDI): - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: self.path = "/dev/null" # returned on attach self.uuid = vdi_uuid self.size = 0 self.utilisation = 0 self.location = vdi_uuid - self.sm_config = {} + self.sm_config: Dict[str, str] = {} - def create(self, sr_uuid, vdi_uuid, size): + @override + def create(self, sr_uuid, vdi_uuid, size) -> str: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_sm_config']) assert(len(self.sr.srcmd.params['args']) == 8) @@ -161,7 +173,8 @@ def create(self, sr_uuid, vdi_uuid, size): self.run_corner_cases_tests() return self.get_params() - def delete(self, sr_uuid, vdi_uuid, data_only=False): + @override + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) assert(len(self.sr.srcmd.params['args']) == 0) @@ -170,7 +183,8 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False): self.run_corner_cases_tests() self._db_forget() - def introduce(self, sr_uuid, vdi_uuid): + @override + def introduce(self, sr_uuid, vdi_uuid) -> str: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_sm_config', 'new_uuid']) assert(len(self.sr.srcmd.params['args']) == 0) self.vdi_sm_config = self.sr.srcmd.params['vdi_sm_config'] @@ -186,19 +200,22 @@ def introduce(self, sr_uuid, vdi_uuid): self.run_corner_cases_tests() return super(DummyVDI, self).get_params() - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) assert(len(self.sr.srcmd.params['args']) == 1) vdi = super(DummyVDI, self).attach(sr_uuid, vdi_uuid) self.run_corner_cases_tests() return vdi - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) self.run_corner_cases_tests() assert(len(self.sr.srcmd.params['args']) == 0) - def activate(self, sr_uuid, vdi_uuid): + @override + def activate(self, sr_uuid, vdi_uuid) -> Optional[Dict[str, str]]: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) assert(len(self.sr.srcmd.params['args']) == 1) self.vdi_ref = self.sr.srcmd.params['vdi_ref'] @@ -206,13 +223,16 @@ def activate(self, sr_uuid, vdi_uuid): self.run_corner_cases_tests() for key in self.other_config.keys(): util.SMlog("\tvdi_other_config: [%s:%s]" % (key, self.other_config[key])) + return None - def deactivate(self, sr_uuid, vdi_uuid): + @override + def deactivate(self, sr_uuid, vdi_uuid) -> None: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) self.run_corner_cases_tests() assert(len(self.sr.srcmd.params['args']) == 0) - def resize(self, sr_uuid, vdi_uuid, size): + @override + def resize(self, sr_uuid, vdi_uuid, size) -> str: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref', 'vdi_ref', 'vdi_location', 'vdi_uuid']) assert(len(self.sr.srcmd.params['args']) == 1) @@ -222,7 +242,8 @@ def resize(self, sr_uuid, vdi_uuid, size): self.run_corner_cases_tests() return super(DummyVDI, self).get_params() - def snapshot(self, sr_uuid, vdi_uuid): + @override + def snapshot(self, sr_uuid, vdi_uuid) -> str: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.sr.srcmd.params['args']) == 0) @@ -236,7 +257,8 @@ def snapshot(self, sr_uuid, vdi_uuid): self.run_corner_cases_tests() return vdi.get_params() - def clone(self, sr_uuid, vdi_uuid): + @override + def clone(self, sr_uuid, vdi_uuid) -> str: self.sr._assertValues(['sr_uuid', 'args', 'host_ref', 'device_config', 'command', 'sr_ref']) assert(len(self.sr.srcmd.params['args']) == 0) diff --git a/drivers/EXTSR.py b/drivers/EXTSR.py index 16481620..30aa3832 100755 --- a/drivers/EXTSR.py +++ b/drivers/EXTSR.py @@ -17,8 +17,11 @@ # # EXTSR: Based on local-file storage repository, mounts ext3 partition +from sm_typing import override + import SR from SR import deviceCheck +import VDI import SRCommand import FileSR import util @@ -57,11 +60,13 @@ class EXTSR(FileSR.FileSR): """EXT3 Local file storage repository""" - def handles(srtype): + @override + @staticmethod + def handles(srtype) -> bool: return srtype == 'ext' - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.ops_exclusive = FileSR.OPS_EXCLUSIVE self.lock = Lock(vhdutil.LOCK_TYPE_SR, self.uuid) self.sr_vditype = SR.DEFAULT_TAP @@ -72,7 +77,8 @@ def load(self, sr_uuid): self.attached = self._checkmount() self.driver_config = DRIVER_CONFIG - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: super(EXTSR, self).delete(sr_uuid) # Check PVs match VG @@ -102,7 +108,8 @@ def delete(self, sr_uuid): raise xs_errors.XenError('LVMDelete', opterr='errno is %d' % inst.code) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self._checkmount(): try: #Activate LV @@ -136,7 +143,8 @@ def attach(self, sr_uuid): for dev in self.dconf['device'].split(','): self.block_setscheduler(dev) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: super(EXTSR, self).detach(sr_uuid) try: # deactivate SR @@ -147,13 +155,15 @@ def detach(self, sr_uuid): 'LVMUnMount', opterr='lvm -an failed errno is %d' % inst.code) + @override @deviceCheck - def probe(self): + def probe(self) -> str: return lvutil.srlist_toxml(lvutil.scan_srlist(EXT_PREFIX, self.dconf['device']), EXT_PREFIX) + @override @deviceCheck - def create(self, sr_uuid, size): + def create(self, sr_uuid, size) -> None: if self._checkmount(): raise xs_errors.XenError('SRExists') @@ -214,12 +224,14 @@ def create(self, sr_uuid, size): self.session, self.sr_ref, scsiutil.devlist_to_serialstring(self.dconf['device'].split(','))) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return EXTFileVDI(self, uuid) class EXTFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} diff --git a/drivers/FileSR.py b/drivers/FileSR.py index 9f2ec28f..60699a3e 100755 --- a/drivers/FileSR.py +++ b/drivers/FileSR.py @@ -17,7 +17,7 @@ # # FileSR: local-file storage repository -from sm_typing import Dict, List +from sm_typing import Dict, Optional, List, override import SR import VDI @@ -73,9 +73,10 @@ class FileSR(SR.SR): SR_TYPE = "file" - def handles(srtype): + @override + @staticmethod + def handles(srtype) -> bool: return srtype == 'file' - handles = staticmethod(handles) def _check_o_direct(self): if self.sr_ref and self.session is not None: @@ -91,7 +92,8 @@ def __init__(self, srcmd, sr_uuid): SR.SR.__init__(self, srcmd, sr_uuid) self._check_o_direct() - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.ops_exclusive = OPS_EXCLUSIVE self.lock = Lock(vhdutil.LOCK_TYPE_SR, self.uuid) self.sr_vditype = vhdutil.VDI_TYPE_VHD @@ -104,7 +106,8 @@ def load(self, sr_uuid): self.attached = False self.driver_config = DRIVER_CONFIG - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: """ Create the SR. The path must not already exist, or if it does, it must be empty. (This accounts for the case where the user has mounted a device onto a directory manually and want to use this as the @@ -126,7 +129,8 @@ def create(self, sr_uuid, size): except: raise xs_errors.XenError('FileSRCreate') - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: self.attach(sr_uuid) cleanup.gc_force(self.session, self.uuid) @@ -158,10 +162,11 @@ def delete(self, sr_uuid): raise xs_errors.XenError('FileSRDelete', \ opterr='error %d' % inst.code) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: self.attach_and_bind(sr_uuid) - def attach_and_bind(self, sr_uuid, bind=True): + def attach_and_bind(self, sr_uuid, bind=True) -> None: if not self._checkmount(): try: util.ioretry(lambda: util.makedirs(self.path, mode=0o700)) @@ -180,7 +185,8 @@ def attach_and_bind(self, sr_uuid, bind=True): opterr='fail to mount FileSR. Errno is %s' % inst.code) self.attached = True - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: if self._checkmount(): try: util.SMlog("Aborting GC/coalesce") @@ -192,7 +198,8 @@ def detach(self, sr_uuid): raise xs_errors.XenError('SRInUse', opterr=str(e)) self.attached = False - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: if not self._checkmount(): raise xs_errors.XenError('SRUnavailable', \ opterr='no such directory %s' % self.path) @@ -226,7 +233,8 @@ def scan(self, sr_uuid): # default behaviour from here on super(FileSR, self).scan(sr_uuid) - def update(self, sr_uuid): + @override + def update(self, sr_uuid) -> None: if not self._checkmount(): raise xs_errors.XenError('SRUnavailable', \ opterr='no such directory %s' % self.path) @@ -239,10 +247,12 @@ def _update(self, sr_uuid, virt_alloc_delta): self.physical_utilisation = self._getutilisation() self._db_update() - def content_type(self, sr_uuid): + @override + def content_type(self, sr_uuid) -> str: return super(FileSR, self).content_type(sr_uuid) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return FileVDI(self, uuid) def added_vdi(self, vdi): @@ -252,7 +262,8 @@ def deleted_vdi(self, uuid): if uuid in self.vdis: del self.vdis[uuid] - def replay(self, uuid): + @override + def replay(self, uuid) -> None: try: file = open(self.path + "/filelog.txt", "r") data = file.readlines() @@ -392,7 +403,7 @@ def _isbind(self): st2 = os.stat(self.remotepath) return st1.st_dev == st2.st_dev and st1.st_ino == st2.st_ino - def _checkmount(self): + def _checkmount(self) -> bool: mount_path = self.path if self.handles("smb"): mount_path = self.mountpoint @@ -402,7 +413,7 @@ def _checkmount(self): util.pathexists(self.remotepath) and self._isbind())) # Override in SharedFileSR. - def _check_hardlinks(self): + def _check_hardlinks(self) -> bool: return True class FileVDI(VDI.VDI): @@ -445,7 +456,8 @@ def _find_path_with_retries(self, vdi_uuid, maxretry=5, period=2.0): return found - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: self.lock = self.sr.lock self.sr.srcmd.params['o_direct'] = self.sr.o_direct @@ -554,13 +566,15 @@ def load(self, vdi_uuid): raise xs_errors.XenError('VDILoad', \ opterr='Failed load VDI information %s' % self.path) - def update(self, sr_uuid, vdi_location): + @override + def update(self, sr_uuid, vdi_location) -> None: self.load(vdi_location) vdi_ref = self.sr.srcmd.params['vdi_ref'] self.sm_config = self.session.xenapi.VDI.get_sm_config(vdi_ref) self._db_update() - def create(self, sr_uuid, vdi_uuid, size): + @override + def create(self, sr_uuid, vdi_uuid, size) -> str: if util.ioretry(lambda: util.pathexists(self.path)): raise xs_errors.XenError('VDIExists') @@ -591,7 +605,8 @@ def create(self, sr_uuid, vdi_uuid, size): self.sr._update(self.sr.uuid, self.size) return super(FileVDI, self).get_params() - def delete(self, sr_uuid, vdi_uuid, data_only=False): + @override + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: if not util.ioretry(lambda: util.pathexists(self.path)): return super(FileVDI, self).delete(sr_uuid, vdi_uuid, data_only) @@ -614,7 +629,8 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False): self.sr._kickGC() return super(FileVDI, self).delete(sr_uuid, vdi_uuid, data_only) - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if self.path is None: self._find_path_with_retries(vdi_uuid) if not self._checkpath(self.path): @@ -638,10 +654,12 @@ def attach(self, sr_uuid, vdi_uuid): except util.CommandException as inst: raise xs_errors.XenError('VDILoad', opterr='error %d' % inst.code) - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: self.attached = False - def resize(self, sr_uuid, vdi_uuid, size): + @override + def resize(self, sr_uuid, vdi_uuid, size) -> str: if not self.exists: raise xs_errors.XenError('VDIUnavailable', \ opterr='VDI %s unavailable %s' % (vdi_uuid, self.path)) @@ -681,10 +699,12 @@ def resize(self, sr_uuid, vdi_uuid, size): super(FileVDI, self).resize_cbt(self.sr.uuid, self.uuid, self.size) return VDI.VDI.get_params(self) - def clone(self, sr_uuid, vdi_uuid): + @override + def clone(self, sr_uuid, vdi_uuid) -> str: return self._do_snapshot(sr_uuid, vdi_uuid, VDI.SNAPSHOT_DOUBLE) - def compose(self, sr_uuid, vdi1, vdi2): + @override + def compose(self, sr_uuid, vdi1, vdi2) -> None: if self.vdi_type != vhdutil.VDI_TYPE_VHD: raise xs_errors.XenError('Unimplemented') parent_fn = vdi1 + vhdutil.FILE_EXTN[vhdutil.VDI_TYPE_VHD] @@ -711,8 +731,9 @@ def reset_leaf(self, sr_uuid, vdi_uuid): vhdutil.killData(self.path) + @override def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, - cloneOp=False, secondary=None, cbtlog=None): + cloneOp=False, secondary=None, cbtlog=None) -> str: # If cbt enabled, save file consistency state if cbtlog is not None: if blktap2.VDI.tap_status(self.session, vdi_uuid): @@ -734,7 +755,8 @@ def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, finally: blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid, secondary) - def _rename(self, src, dst): + @override + def _rename(self, src, dst) -> None: util.SMlog("FileVDI._rename %s to %s" % (src, dst)) util.ioretry(lambda: os.rename(src, dst)) @@ -915,7 +937,8 @@ def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None): ret_vdi = self return ret_vdi.get_params() - def get_params(self): + @override + def get_params(self) -> str: if not self._checkpath(self.path): raise xs_errors.XenError('VDIUnavailable', \ opterr='VDI %s unavailable %s' % (self.uuid, self.path)) @@ -995,7 +1018,8 @@ def extractUuid(path): return uuid extractUuid = staticmethod(extractUuid) - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: """ Generate the XML config required to attach and activate a VDI for use when XAPI is not running. Attach and @@ -1014,7 +1038,8 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), "vdi_attach_from_config") return xmlrpc.client.dumps((config, ), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: """ Attach and activate a VDI using config generated by vdi_generate_config above. This is used for cases such as @@ -1023,15 +1048,17 @@ def attach_from_config(self, sr_uuid, vdi_uuid): util.SMlog("FileVDI.attach_from_config") try: if not util.pathexists(self.sr.path): - self.sr.attach(sr_uuid) + return self.sr.attach(sr_uuid) except: util.logException("FileVDI.attach_from_config") raise xs_errors.XenError( 'SRUnavailable', opterr='Unable to attach from config' ) + return '' - def _create_cbt_log(self): + @override + def _create_cbt_log(self) -> str: # Create CBT log file # Name: .cbtlog #Handle if file already exists @@ -1040,7 +1067,8 @@ def _create_cbt_log(self): open_file.close() return super(FileVDI, self)._create_cbt_log() - def _delete_cbt_log(self): + @override + def _delete_cbt_log(self) -> None: logPath = self._get_cbt_logpath(self.uuid) try: os.remove(logPath) @@ -1048,7 +1076,8 @@ def _delete_cbt_log(self): if e.errno != errno.ENOENT: raise - def _cbt_log_exists(self, logpath): + @override + def _cbt_log_exists(self, logpath) -> bool: return util.pathexists(logpath) @@ -1074,7 +1103,8 @@ def _check_writable(self): def _raise_hardlink_error(self): raise OSError(524, "Unknown error 524") - def _check_hardlinks(self): + @override + def _check_hardlinks(self) -> bool: hardlink_conf = self._read_hardlink_conf() if hardlink_conf is not None: return hardlink_conf @@ -1116,7 +1146,7 @@ def _check_hardlinks(self): def _get_hardlink_conf_path(self): return os.path.join(self.path, 'sm-hardlink.conf') - def _read_hardlink_conf(self): + def _read_hardlink_conf(self) -> Optional[bool]: try: with open(self._get_hardlink_conf_path(), 'r') as f: try: diff --git a/drivers/GlusterFSSR.py b/drivers/GlusterFSSR.py index 041a9142..7b515dc8 100644 --- a/drivers/GlusterFSSR.py +++ b/drivers/GlusterFSSR.py @@ -16,6 +16,8 @@ # along with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from sm_typing import override + import errno import os import syslog as _syslog @@ -29,6 +31,7 @@ import SRCommand import FileSR # end of careful +import VDI import cleanup import util import vhdutil @@ -77,13 +80,14 @@ class GlusterFSSR(FileSR.FileSR): DRIVER_TYPE = 'glusterfs' - def handles(sr_type): + @override + @staticmethod + def handles(sr_type) -> bool: # fudge, because the parent class (FileSR) checks for smb to alter its behavior return sr_type == GlusterFSSR.DRIVER_TYPE or sr_type == 'smb' - handles = staticmethod(handles) - - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not self._is_glusterfs_available(): raise xs_errors.XenError( 'SRUnavailable', @@ -160,7 +164,8 @@ def unmount(self, mountpoint, rmmountpoint): except OSError as inst: raise GlusterFSException("rmdir failed with error '%s'" % inst.strerror) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self.checkmount(): try: self.mount() @@ -169,7 +174,8 @@ def attach(self, sr_uuid): raise xs_errors.SROSError(12, exc.errstr) self.attached = True - def probe(self): + @override + def probe(self) -> str: try: self.mount(PROBE_MOUNTPOINT) sr_list = filter(util.match_uuid, util.listdir(PROBE_MOUNTPOINT)) @@ -179,7 +185,8 @@ def probe(self): # Create a dictionary from the SR uuids to feed SRtoXML() return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: if not self.checkmount(): return util.SMlog("Aborting GC/coalesce") @@ -190,7 +197,8 @@ def detach(self, sr_uuid): os.unlink(self.path) self.attached = False - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: if self.checkmount(): raise xs_errors.SROSError(113, 'GlusterFS mount point already attached') @@ -224,7 +232,8 @@ def create(self, sr_uuid, size): os.strerror(inst.code))) self.detach(sr_uuid) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: # try to remove/delete non VDI contents first super(GlusterFSSR, self).delete(sr_uuid) try: @@ -239,7 +248,8 @@ def delete(self, sr_uuid): if inst.code != errno.ENOENT: raise xs_errors.SROSError(114, "Failed to remove GlusterFS mount point") - def vdi(self, uuid, loadLocked=False): + @override + def vdi(self, uuid, loadLocked=False) -> VDI.VDI: return GlusterFSFileVDI(self, uuid) @staticmethod @@ -248,7 +258,8 @@ def _is_glusterfs_available(): class GlusterFSFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} @@ -256,7 +267,8 @@ def attach(self, sr_uuid, vdi_uuid): return super(GlusterFSFileVDI, self).attach(sr_uuid, vdi_uuid) - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("SMBFileVDI.generate_config") if not util.pathexists(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -270,15 +282,16 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), "vdi_attach_from_config") return xmlrpc.client.dumps((config,), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: try: if not util.pathexists(self.sr.path): - self.sr.attach(sr_uuid) + return self.sr.attach(sr_uuid) except: util.logException("SMBFileVDI.attach_from_config") raise xs_errors.XenError('SRUnavailable', opterr='Unable to attach from config') - + return '' if __name__ == '__main__': SRCommand.run(GlusterFSSR, DRIVER_INFO) diff --git a/drivers/HBASR.py b/drivers/HBASR.py index 66ca3ee9..1740f42e 100755 --- a/drivers/HBASR.py +++ b/drivers/HBASR.py @@ -19,8 +19,11 @@ # hardware based iSCSI # +from sm_typing import Dict, List, override + import SR import SRCommand +import VDI import devscan import scsiutil import util @@ -50,20 +53,22 @@ class HBASR(SR.SR): """HBA storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if type == "hba": return True return False - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.sr_vditype = 'phy' self.type = "any" if 'type' in self.dconf and self.dconf['type']: self.type = self.dconf['type'] self.attached = False self.procname = "" - self.devs = {} + self.devs: Dict[str, List[str]] = {} def _init_hbadict(self): if not hasattr(self, "hbas"): @@ -174,15 +179,18 @@ def _probe_hba(self): raise xs_errors.XenError('XMLParse', \ opterr='HBA probe failed') - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: self._mpathHandle() - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: if util._containsVDIinuse(self): return return - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: # Check whether an SR already exists SRs = self.session.xenapi.SR.get_all_records() for sr in SRs: @@ -211,11 +219,13 @@ def create(self, sr_uuid, size): self.sm_config['multipathable'] = 'true' self.session.xenapi.SR.set_sm_config(self.sr_ref, self.sm_config) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: self.detach(sr_uuid) return - def probe(self): + @override + def probe(self) -> str: self._init_hbadict() self.attach("") SRs = self.session.xenapi.SR.get_all_records() @@ -228,7 +238,8 @@ def probe(self): Recs[record["uuid"]] = sm_config return self.srlist_toxml(Recs) - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: self._init_hbadict() if not self.passthrough: if not self.attached: @@ -242,7 +253,7 @@ def scan(self, sr_uuid): if vdi.managed: self.physical_utilisation += vdi.size self.virtual_allocation = self.physical_utilisation - return super(HBASR, self).scan(sr_uuid) + super(HBASR, self).scan(sr_uuid) def print_devs(self): self.attach("") @@ -273,7 +284,8 @@ def _loadvdis(self): def _getLUNbySMconfig(self, sm_config): raise xs_errors.XenError('VDIUnavailable') - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return LUNperVDI.RAWVDI(self, uuid) def srlist_toxml(self, SRs): diff --git a/drivers/ISOSR.py b/drivers/ISOSR.py index 64f99b48..401461c6 100755 --- a/drivers/ISOSR.py +++ b/drivers/ISOSR.py @@ -17,6 +17,8 @@ # # ISOSR: remote iso storage repository +from sm_typing import override + import SR import VDI import SRCommand @@ -152,7 +154,7 @@ class ISOSR(SR.SR): """Local file storage repository""" # Some helper functions: - def _checkmount(self): + def _checkmount(self) -> bool: """Checks that the mountpoint exists and is mounted""" if not util.pathexists(self.mountpoint): return False @@ -221,21 +223,24 @@ def _loadvdis(self): vdi.read_only = False # Now for the main functions: - def handles(type): + @override + @staticmethod + def handles(type) -> bool: """Do we handle this type?""" if type == TYPE: return True return False - handles = staticmethod(handles) - def content_type(self, sr_uuid): + @override + def content_type(self, sr_uuid) -> str: """Returns the content_type XML""" return super(ISOSR, self).content_type(sr_uuid) # pylint: disable=no-member vdi_path_regex = re.compile(r"[a-z0-9.-]+\.(iso|img)", re.I) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: """Create a VDI class. If the VDI does not exist, we determine here what its filename should be.""" @@ -267,7 +272,8 @@ def vdi(self, uuid): return ISOVDI(self, filename) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: """Initialises the SR""" # First of all, check we've got the correct keys in dconf if 'location' not in self.dconf: @@ -302,10 +308,12 @@ def load(self, sr_uuid): # Some info we need: self.sr_vditype = 'phy' - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: pass - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: """Std. attach""" # Very-Legacy mode means the ISOs are in the local fs - so no need to attach. if 'legacy_mode' in self.dconf: @@ -475,8 +483,8 @@ def _check_nfs_server(self, location): except nfs.NfsException as e: raise xs_errors.XenError('NFSTarget', opterr=str(e.errstr)) - - def after_master_attach(self, uuid): + @override + def after_master_attach(self, uuid) -> None: """Perform actions required after attaching on the pool master Return: None @@ -568,7 +576,8 @@ def getCacheOptions(self): """Pass cache options to mount.cifs""" return "cache=none" - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: """Std. detach""" if 'legacy_mode' in self.dconf or not self._checkmount(): return @@ -579,7 +588,8 @@ def detach(self, sr_uuid): raise xs_errors.XenError('NFSUnMount', \ opterr='error is %d' % inst.code) - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: """Scan: see _loadvdis""" if not util.isdir(self.path): raise xs_errors.XenError('SRUnavailable', \ @@ -660,9 +670,10 @@ def scan(self, sr_uuid): self.session.xenapi.VDI.remove_from_sm_config(vdi, 'xs-tools') else: - return super(ISOSR, self).scan(sr_uuid) + super(ISOSR, self).scan(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: self.attach(sr_uuid) if 'type' in self.dconf: smconfig = self.session.xenapi.SR.get_sm_config(self.sr_ref) @@ -681,9 +692,10 @@ def create(self, sr_uuid, size): self.detach(sr_uuid) - + class ISOVDI(VDI.VDI): - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: # Nb, in the vdi_create call, the filename is unset, so the following # will fail. self.vdi_type = "iso" @@ -725,17 +737,20 @@ def __init__(self, mysr, filename): self.sm_config['xs-tools-version'] = product_version self.sm_config['xs-tools-build'] = build_number - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: pass - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: try: os.stat(self.path) return super(ISOVDI, self).attach(sr_uuid, vdi_uuid) except: raise xs_errors.XenError('VDIMissing') - def create(self, sr_uuid, vdi_uuid, size): + @override + def create(self, sr_uuid, vdi_uuid, size) -> str: self.uuid = vdi_uuid self.path = os.path.join(self.sr.path, self.filename) self.size = size @@ -758,7 +773,8 @@ def create(self, sr_uuid, vdi_uuid, size): raise xs_errors.XenError('VDICreate', \ opterr='could not create file: "%s"' % self.path) - def delete(self, sr_uuid, vdi_uuid, data_only=False): + @override + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: util.SMlog("Deleting...") self.uuid = vdi_uuid diff --git a/drivers/LUNperVDI.py b/drivers/LUNperVDI.py index 497ba6b5..e3583a59 100755 --- a/drivers/LUNperVDI.py +++ b/drivers/LUNperVDI.py @@ -18,6 +18,8 @@ # LUNperVDI: Generic Raw LUN handler, used by HBASR and ISCSISR # +from sm_typing import override + import os import VDI import util @@ -28,7 +30,8 @@ class RAWVDI(VDI.VDI): - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: if not self.sr.attached: raise xs_errors.XenError('SRUnavailable') @@ -60,7 +63,8 @@ def _query(self, path, id): sm_config['backend-kind'] = 'vbd' self.sm_config = sm_config - def introduce(self, sr_uuid, vdi_uuid): + @override + def introduce(self, sr_uuid, vdi_uuid) -> str: self.sm_config = self.sr.srcmd.params['vdi_sm_config'] vdi_path = self.sr._getLUNbySMconfig(self.sm_config) self._query(vdi_path, self.sm_config['LUNid']) @@ -78,7 +82,8 @@ def introduce(self, sr_uuid, vdi_uuid): self.sr.vdis[vdi_uuid]._db_introduce() return super(RAWVDI, self).get_params() - def create(self, sr_uuid, vdi_uuid, size): + @override + def create(self, sr_uuid, vdi_uuid, size) -> str: VDIs = util._getVDIs(self.sr) self.sr._loadvdis() smallest = 0 @@ -98,7 +103,8 @@ def create(self, sr_uuid, vdi_uuid, size): return super(RAWVDI, self.sr.vdis[v['uuid']]).get_params() raise xs_errors.XenError('SRNoSpace') - def delete(self, sr_uuid, vdi_uuid, data_only=False): + @override + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: try: vdi = util._getVDI(self.sr, vdi_uuid) if not vdi['managed']: @@ -108,7 +114,8 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False): except: pass - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: self.sr._loadvdis() if vdi_uuid not in self.sr.vdis: raise xs_errors.XenError('VDIUnavailable') @@ -126,7 +133,8 @@ def attach(self, sr_uuid, vdi_uuid): raise xs_errors.XenError('VDIUnavailable') return super(RAWVDI, self).attach(sr_uuid, vdi_uuid) - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: self.sr._loadvdis() if 'SCSIid' in self.sm_config: self.sr.mpathmodule.reset(self.sm_config['SCSIid'], True) # explicitly unmap diff --git a/drivers/LVHDSR.py b/drivers/LVHDSR.py index 0cb86bea..3e9afcde 100755 --- a/drivers/LVHDSR.py +++ b/drivers/LVHDSR.py @@ -18,7 +18,7 @@ # LVHDSR: VHD on LVM storage repository # -from sm_typing import Dict, List +from sm_typing import Dict, List, override import SR from SR import deviceCheck @@ -135,7 +135,9 @@ class LVHDSR(SR.SR): legacyMode = True - def handles(type): + @override + @staticmethod + def handles(type) -> bool: """Returns True if this SR class understands the given dconf string""" # we can pose as LVMSR or EXTSR for compatibility purposes if __name__ == '__main__': @@ -147,9 +149,9 @@ def handles(type): elif name.endswith("EXTSR"): return type == "ext" return type == LVHDSR.DRIVER_TYPE - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.ops_exclusive = OPS_EXCLUSIVE self.isMaster = False @@ -238,7 +240,8 @@ def load(self, sr_uuid): except: pass - def cleanup(self): + @override + def cleanup(self) -> None: # we don't need to hold the lock to dec refcounts of activated LVs if not self.lvActivator.deactivateAll(): raise util.SMException("failed to deactivate LVs") @@ -481,8 +484,9 @@ def _expand_size(self): util.logException("LVHDSR._expand_size for %s failed to resize" " the PV" % self.uuid) + @override @deviceCheck - def create(self, uuid, size): + def create(self, uuid, size) -> None: util.SMlog("LVHDSR.create for %s" % self.uuid) if not self.isMaster: util.SMlog('sr_create blocked for non-master') @@ -510,7 +514,8 @@ def create(self, uuid, size): self.session.xenapi.SR.add_to_sm_config(self.sr_ref, \ self.FLAG_USE_VHD, 'true') - def delete(self, uuid): + @override + def delete(self, uuid) -> None: util.SMlog("LVHDSR.delete for %s" % self.uuid) if not self.isMaster: raise xs_errors.XenError('LVMMaster') @@ -564,7 +569,8 @@ def delete(self, uuid): lvutil.removeVG(self.dconf['device'], self.vgname) self._cleanup() - def attach(self, uuid): + @override + def attach(self, uuid) -> None: util.SMlog("LVHDSR.attach for %s" % self.uuid) self._cleanup(True) # in case of host crashes, if detach wasn't called @@ -602,7 +608,8 @@ def attach(self, uuid): for dev in self.dconf['device'].split(','): self.block_setscheduler(dev) - def detach(self, uuid): + @override + def detach(self, uuid) -> None: util.SMlog("LVHDSR.detach for %s" % self.uuid) cleanup.abort(self.uuid) @@ -660,12 +667,14 @@ def detach(self, uuid): # only place to do so. self._cleanup(self.isMaster) - def forget_vdi(self, uuid): + @override + def forget_vdi(self, uuid) -> None: if not self.legacyMode: LVMMetadataHandler(self.mdpath).deleteVdiFromMetadata(uuid) super(LVHDSR, self).forget_vdi(uuid) - def scan(self, uuid): + @override + def scan(self, uuid) -> None: activated = True try: lvname = '' @@ -686,7 +695,7 @@ def scan(self, uuid): # Now check if there are any VDIs in the metadata, which are not in # XAPI if self.mdexists: - vdiToSnaps = {} + vdiToSnaps: Dict[str, List[str]] = {} # get VDIs from XAPI vdis = self.session.xenapi.SR.get_VDIs(self.sr_ref) vdi_uuids = set([]) @@ -813,7 +822,8 @@ def scan(self, uuid): if lvname != '' and activated: self.lvmCache.deactivateNoRefcount(lvname) - def update(self, uuid): + @override + def update(self, uuid) -> None: if not lvutil._checkVG(self.vgname): return self._updateStats(uuid, 0) @@ -842,15 +852,17 @@ def _updateStats(self, uuid, virtAllocDelta): self.physical_utilisation = stats['physical_utilisation'] self._db_update() + @override @deviceCheck - def probe(self): + def probe(self) -> str: return lvutil.srlist_toxml( lvutil.scan_srlist(lvhdutil.VG_PREFIX, self.dconf['device']), lvhdutil.VG_PREFIX, ('metadata' in self.srcmd.params['sr_sm_config'] and \ self.srcmd.params['sr_sm_config']['metadata'] == 'true')) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return LVHDVDI(self, uuid) def _loadvdis(self): @@ -1307,12 +1319,13 @@ def ensureCBTSpace(self): # Ensure we have space for at least one LV self._ensureSpaceAvailable(self.journaler.LV_SIZE) - + class LVHDVDI(VDI.VDI): JRN_CLONE = "clone" # journal entry type for the clone operation - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: self.lock = self.sr.lock self.lvActivator = self.sr.lvActivator self.loaded = False @@ -1352,7 +1365,8 @@ def load(self, vdi_uuid): self.lvname = "%s%s" % (lvhdutil.LV_PREFIX[self.vdi_type], vdi_uuid) self.path = os.path.join(self.sr.path, self.lvname) - def create(self, sr_uuid, vdi_uuid, size): + @override + def create(self, sr_uuid, vdi_uuid, size) -> str: util.SMlog("LVHDVDI.create for %s" % self.uuid) if not self.sr.isMaster: raise xs_errors.XenError('LVMMaster') @@ -1416,7 +1430,8 @@ def create(self, sr_uuid, vdi_uuid, size): return VDI.VDI.get_params(self) - def delete(self, sr_uuid, vdi_uuid, data_only=False): + @override + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: util.SMlog("LVHDVDI.delete for %s" % self.uuid) try: self._loadThis() @@ -1461,7 +1476,8 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False): self.sr._kickGC() return super(LVHDVDI, self).delete(sr_uuid, vdi_uuid, data_only) - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: util.SMlog("LVHDVDI.attach for %s" % self.uuid) if self.sr.journaler.hasJournals(self.uuid): raise xs_errors.XenError('VDIUnavailable', @@ -1491,7 +1507,8 @@ def attach(self, sr_uuid, vdi_uuid): if not self.sr.lvActivator.deactivateAll(): util.SMlog("Failed to deactivate LVs back (%s)" % self.uuid) - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: util.SMlog("LVHDVDI.detach for %s" % self.uuid) self._loadThis() already_deflated = (self.utilisation < \ @@ -1522,7 +1539,8 @@ def detach(self, sr_uuid, vdi_uuid): raise xs_errors.XenError("SMGeneral", opterr="deactivation") # We only support offline resize - def resize(self, sr_uuid, vdi_uuid, size): + @override + def resize(self, sr_uuid, vdi_uuid, size) -> str: util.SMlog("LVHDVDI.resize for %s" % self.uuid) if not self.sr.isMaster: raise xs_errors.XenError('LVMMaster') @@ -1575,11 +1593,13 @@ def resize(self, sr_uuid, vdi_uuid, size): super(LVHDVDI, self).resize_cbt(self.sr.uuid, self.uuid, self.size) return VDI.VDI.get_params(self) - def clone(self, sr_uuid, vdi_uuid): + @override + def clone(self, sr_uuid, vdi_uuid) -> str: return self._do_snapshot( sr_uuid, vdi_uuid, VDI.SNAPSHOT_DOUBLE, cloneOp=True) - def compose(self, sr_uuid, vdi1, vdi2): + @override + def compose(self, sr_uuid, vdi1, vdi2) -> None: util.SMlog("LVHDSR.compose for %s -> %s" % (vdi2, vdi1)) if self.vdi_type != vhdutil.VDI_TYPE_VHD: raise xs_errors.XenError('Unimplemented') @@ -1639,8 +1659,9 @@ def _detach(self): self._chainSetActive(False, True) self.attached = False + @override def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, - cloneOp=False, secondary=None, cbtlog=None): + cloneOp=False, secondary=None, cbtlog=None) -> str: # If cbt enabled, save file consistency state if cbtlog is not None: if blktap2.VDI.tap_status(self.session, vdi_uuid): @@ -2169,7 +2190,8 @@ def _prepareThin(self, attach): self.session.xenapi.SR.set_physical_utilisation(self.sr.sr_ref, str(sr_utilisation)) - def update(self, sr_uuid, vdi_uuid): + @override + def update(self, sr_uuid, vdi_uuid) -> None: if self.sr.legacyMode: return @@ -2189,28 +2211,33 @@ def update(self, sr_uuid, vdi_uuid): self.session.xenapi.VDI.get_metadata_of_pool(vdi_ref) LVMMetadataHandler(self.sr.mdpath).updateMetadata(update_map) - def _ensure_cbt_space(self): + @override + def _ensure_cbt_space(self) -> None: self.sr.ensureCBTSpace() - def _create_cbt_log(self): + @override + def _create_cbt_log(self) -> str: logname = self._get_cbt_logname(self.uuid) self.sr.lvmCache.create(logname, self.sr.journaler.LV_SIZE, CBTLOG_TAG) logpath = super(LVHDVDI, self)._create_cbt_log() self.sr.lvmCache.deactivateNoRefcount(logname) return logpath - def _delete_cbt_log(self): + @override + def _delete_cbt_log(self) -> None: logpath = self._get_cbt_logpath(self.uuid) if self._cbt_log_exists(logpath): logname = self._get_cbt_logname(self.uuid) self.sr.lvmCache.remove(logname) - def _rename(self, oldpath, newpath): + @override + def _rename(self, oldpath, newpath) -> None: oldname = os.path.basename(oldpath) newname = os.path.basename(newpath) self.sr.lvmCache.rename(oldname, newname) - def _activate_cbt_log(self, lv_name): + @override + def _activate_cbt_log(self, lv_name) -> bool: self.sr.lvmCache.refresh() if not self.sr.lvmCache.is_active(lv_name): try: @@ -2223,14 +2250,16 @@ def _activate_cbt_log(self, lv_name): else: return False - def _deactivate_cbt_log(self, lv_name): + @override + def _deactivate_cbt_log(self, lv_name) -> None: try: self.sr.lvmCache.deactivateNoRefcount(lv_name) except Exception as e: util.SMlog("Exception in _deactivate_cbt_log, Error: %s." % str(e)) raise - def _cbt_log_exists(self, logpath): + @override + def _cbt_log_exists(self, logpath) -> bool: return lvutil.exists(logpath) if __name__ == '__main__': diff --git a/drivers/LVHDoFCoESR.py b/drivers/LVHDoFCoESR.py index 766d83f4..11bf298e 100755 --- a/drivers/LVHDoFCoESR.py +++ b/drivers/LVHDoFCoESR.py @@ -18,7 +18,10 @@ # LVHDoFCoESR: LVHD over Fibre Channel over Ethernet driver # +from sm_typing import override + import SR +import VDI import LVHDoHBASR import LVHDSR import SRCommand @@ -54,7 +57,9 @@ class LVHDoFCoESR(LVHDoHBASR.LVHDoHBASR): """LVHD over FCoE storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if __name__ == '__main__': name = sys.argv[0] else: @@ -64,9 +69,9 @@ def handles(type): if type == "lvhdofcoe": return True return False - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: driver = SR.driver('hba') if 'type' not in self.original_srcmd.params['device_config'] or \ 'type' in self.original_srcmd.params['device_config'] and \ @@ -86,7 +91,8 @@ def load(self, sr_uuid): self.SCSIid = self.dconf['SCSIid'] LVHDSR.LVHDSR.load(self, sr_uuid) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return LVHDoFCoEVDI(self, uuid) diff --git a/drivers/LVHDoHBASR.py b/drivers/LVHDoHBASR.py index 1eb8fb63..78451273 100755 --- a/drivers/LVHDoHBASR.py +++ b/drivers/LVHDoHBASR.py @@ -19,9 +19,12 @@ # hardware based iSCSI # +from sm_typing import override + import SR import LVHDSR import SRCommand +import VDI import lvutil import HBASR import os @@ -58,7 +61,9 @@ class LVHDoHBASR(LVHDSR.LVHDSR): """LVHD over HBA storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if __name__ == '__main__': name = sys.argv[0] else: @@ -68,9 +73,9 @@ def handles(type): if type == "lvhdohba": return True return False - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: driver = SR.driver('hba') self.hbasr = driver(self.original_srcmd, sr_uuid) @@ -108,7 +113,8 @@ def load(self, sr_uuid): self.SCSIid = self.dconf['SCSIid'] super(LVHDoHBASR, self).load(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: self.hbasr.attach(sr_uuid) if self.mpath == "true": self.mpathmodule.refresh(self.SCSIid, 0) @@ -121,7 +127,8 @@ def create(self, sr_uuid, size): util.remove_mpathcount_field(self.session, self.host_ref, \ self.sr_ref, self.SCSIid) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: self.hbasr.attach(sr_uuid) if self.mpath == "true": self.mpathmodule.refresh(self.SCSIid, 0) @@ -140,7 +147,8 @@ def attach(self, sr_uuid): LVHDSR.LVHDSR.attach(self, sr_uuid) self._setMultipathableFlag(SCSIid=self.SCSIid) - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: # During a reboot, scan is called ahead of attach, which causes the MGT # to point of the wrong device instead of dm-x. Running multipathing will # take care of this scenario. @@ -154,7 +162,8 @@ def scan(self, sr_uuid): self._pathrefresh(LVHDoHBASR) LVHDSR.LVHDSR.scan(self, sr_uuid) - def probe(self): + @override + def probe(self) -> str: if self.mpath == "true" and 'SCSIid' in self.dconf: # When multipathing is enabled, since we don't refcount the multipath maps, # we should not attempt to do the iscsi.attach/detach when the map is already present, @@ -181,7 +190,8 @@ def probe(self): self.mpathmodule.reset(self.SCSIid, explicit_unmap=True) raise - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: LVHDSR.LVHDSR.detach(self, sr_uuid) self.mpathmodule.reset(self.SCSIid, explicit_unmap=True) try: @@ -205,7 +215,8 @@ def _remove_device_nodes(self): (os.path.basename(node)), 'w') as f: f.write('1\n') - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: self._pathrefresh(LVHDoHBASR) try: LVHDSR.LVHDSR.delete(self, sr_uuid) @@ -214,12 +225,14 @@ def delete(self, sr_uuid): self.mpathmodule.reset(self.SCSIid, explicit_unmap=True) self._remove_device_nodes() - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return LVHDoHBAVDI(self, uuid) class LVHDoHBAVDI(LVHDSR.LVHDVDI): - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("LVHDoHBAVDI.generate_config") if not lvutil._checkLV(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -235,7 +248,8 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([dict]), "vdi_attach_from_config") return xmlrpc.client.dumps((config, ), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("LVHDoHBAVDI.attach_from_config") self.sr.hbasr.attach(sr_uuid) if self.sr.mpath == "true": diff --git a/drivers/LVHDoISCSISR.py b/drivers/LVHDoISCSISR.py index 442ec30f..799e89d6 100755 --- a/drivers/LVHDoISCSISR.py +++ b/drivers/LVHDoISCSISR.py @@ -18,7 +18,10 @@ # LVHDoISCSISR: LVHD over ISCSI software initiator SR driver # +from sm_typing import override + import SR +import VDI import LVHDSR import BaseISCSI import SRCommand @@ -71,7 +74,9 @@ class LVHDoISCSISR(LVHDSR.LVHDSR): """LVHD over ISCSI storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if __name__ == '__main__': name = sys.argv[0] else: @@ -81,9 +86,9 @@ def handles(type): if type == "lvhdoiscsi": return True return False - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not sr_uuid: # This is a probe call, generate a temp sr_uuid sr_uuid = util.gen_uuid() @@ -426,7 +431,8 @@ def _LUNprint(self, sr_uuid): self.iscsi.print_LUNs() self.iscsi.detach(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: # Check SCSIid not already in use by other PBDs if util.test_SCSIid(self.session, sr_uuid, self.SCSIid): raise xs_errors.XenError('SRInUse') @@ -441,13 +447,15 @@ def create(self, sr_uuid, size): raise xs_errors.XenError("SRUnavailable", opterr=inst) self.iscsi.detach(sr_uuid) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: self._pathrefresh(LVHDoISCSISR) LVHDSR.LVHDSR.delete(self, sr_uuid) for i in self.iscsiSRs: i.detach(sr_uuid) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: try: connected = False stored_exception = None @@ -485,12 +493,14 @@ def attach(self, sr_uuid): raise xs_errors.XenError("SRUnavailable", opterr=inst) self._setMultipathableFlag(SCSIid=self.SCSIid) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: LVHDSR.LVHDSR.detach(self, sr_uuid) for i in self.iscsiSRs: i.detach(sr_uuid) - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: self._pathrefresh(LVHDoISCSISR) if self.mpath == "true": for i in self.iscsiSRs: @@ -500,7 +510,8 @@ def scan(self, sr_uuid): util.SMlog("Connection failed for target %s, continuing.." % i.target) LVHDSR.LVHDSR.scan(self, sr_uuid) - def probe(self): + @override + def probe(self) -> str: self.uuid = util.gen_uuid() # When multipathing is enabled, since we don't refcount the multipath maps, @@ -523,7 +534,8 @@ def probe(self): self.iscsi.detach(self.uuid) return out - def check_sr(self, sr_uuid): + @override + def check_sr(self, sr_uuid) -> None: """Hook to check SR health""" pbdref = util.find_my_pbd(self.session, self.host_ref, self.sr_ref) if pbdref: @@ -536,12 +548,14 @@ def check_sr(self, sr_uuid): except xs_errors.SROSError: util.SMlog("Failed to attach iSCSI target") - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return LVHDoISCSIVDI(self, uuid) class LVHDoISCSIVDI(LVHDSR.LVHDVDI): - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("LVHDoISCSIVDI.generate_config") if not lvutil._checkLV(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -562,7 +576,8 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([dict]), "vdi_attach_from_config") return xmlrpc.client.dumps((config, ), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("LVHDoISCSIVDI.attach_from_config") try: self.sr.iscsi.attach(sr_uuid) diff --git a/drivers/LargeBlockSR.py b/drivers/LargeBlockSR.py index ba0ac1d1..fa66cfce 100644 --- a/drivers/LargeBlockSR.py +++ b/drivers/LargeBlockSR.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from sm_typing import override + import SR from SR import deviceCheck import SRCommand @@ -52,17 +54,20 @@ class LargeBlockSR(EXTSR.EXTSR): DRIVER_TYPE = "largeblock" LOOP_SECTOR_SIZE = 512 + @override @staticmethod - def handles(srtype): + def handles(srtype) -> bool: return srtype == LargeBlockSR.DRIVER_TYPE - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: super(LargeBlockSR, self).load(sr_uuid) self.is_deleting = False self.vgname = LARGEBLOCK_PREFIX + sr_uuid self.remotepath = os.path.join("/dev", self.vgname, sr_uuid) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self.is_deleting: vg_device = self._get_device() self.dconf["device"] = ",".join(vg_device) @@ -71,7 +76,8 @@ def attach(self, sr_uuid): self._redo_vg_connection() # Call redo VG connection to connect it correctly to the loop device instead of the real 4KiB block device super(LargeBlockSR, self).attach(sr_uuid) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: if not self.is_deleting: vg_device = self._get_device() self.dconf["device"] = ",".join(vg_device) @@ -79,8 +85,9 @@ def detach(self, sr_uuid): if not self.is_deleting: self._destroy_emulated_device() + @override @deviceCheck - def create(self, sr_uuid, size): + def create(self, sr_uuid, size) -> None: base_devices = self.dconf["device"].split(",") if len(base_devices) > 1: raise xs_errors.XenError("ConfigDeviceInvalid", opterr="Multiple devices configuration is not supported") @@ -96,7 +103,8 @@ def create(self, sr_uuid, size): finally: self._destroy_emulated_device(base_devices) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: base_devices = self._get_device() self.dconf["device"] = ",".join(self._get_loopdev_from_device(base_devices)) @@ -112,8 +120,9 @@ def delete(self, sr_uuid): self._destroy_emulated_device(base_devices) self.is_deleting = False + @override @deviceCheck - def probe(self): + def probe(self) -> str: # We override EXTSR.probe because it uses EXT_PREFIX in this call return lvutil.srlist_toxml( lvutil.scan_srlist(LARGEBLOCK_PREFIX, self.dconf['device']), diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index 22c1b37e..459ed420 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from sm_typing import Optional, override + from constants import CBTLOG_TAG try: @@ -302,11 +304,13 @@ class LinstorSR(SR.SR): # SR methods. # -------------------------------------------------------------------------- + @override @staticmethod - def handles(type): + def handles(type) -> bool: return type == LinstorSR.DRIVER_TYPE - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not LINSTOR_AVAILABLE: raise util.SMException( 'Can\'t load LinstorSR: LINSTOR libraries are missing' @@ -541,12 +545,14 @@ def wrap(self, *args, **kwargs): return wrap - def cleanup(self): + @override + def cleanup(self) -> None: if self._vdi_shared_time: self._shared_lock_vdi(self.srcmd.params['vdi_uuid'], locked=False) + @override @_locked_load - def create(self, uuid, size): + def create(self, uuid, size) -> None: util.SMlog('LinstorSR.create for {}'.format(self.uuid)) host_adresses = util.get_host_addresses(self.session) @@ -635,8 +641,9 @@ def create(self, uuid, size): ) raise e + @override @_locked_load - def delete(self, uuid): + def delete(self, uuid) -> None: util.SMlog('LinstorSR.delete for {}'.format(self.uuid)) cleanup.gc_force(self.session, self.uuid) @@ -698,8 +705,9 @@ def delete(self, uuid): Lock.cleanupAll(self.uuid) + @override @_locked_load - def update(self, uuid): + def update(self, uuid) -> None: util.SMlog('LinstorSR.update for {}'.format(self.uuid)) # Well, how can we update a SR if it doesn't exist? :thinking: @@ -722,8 +730,9 @@ def update(self, uuid): ) } + @override @_locked_load - def attach(self, uuid): + def attach(self, uuid) -> None: util.SMlog('LinstorSR.attach for {}'.format(self.uuid)) if not self._linstor: @@ -732,18 +741,22 @@ def attach(self, uuid): opterr='no such group: {}'.format(self._group_name) ) + @override @_locked_load - def detach(self, uuid): + def detach(self, uuid) -> None: util.SMlog('LinstorSR.detach for {}'.format(self.uuid)) cleanup.abort(self.uuid) + @override @_locked_load - def probe(self): + def probe(self) -> str: util.SMlog('LinstorSR.probe for {}'.format(self.uuid)) # TODO + return '' + @override @_locked_load - def scan(self, uuid): + def scan(self, uuid) -> None: if self._init_status == self.INIT_STATUS_FAIL: return @@ -792,8 +805,9 @@ def is_master(self): return self._is_master + @override @_locked_load - def vdi(self, uuid): + def vdi(self, uuid) -> VDI.VDI: return LinstorVDI(self, uuid) # To remove in python 3.10 @@ -1565,7 +1579,8 @@ class LinstorVDI(VDI.VDI): # VDI methods. # -------------------------------------------------------------------------- - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: self._lock = self.sr.lock self._exists = True self._linstor = self.sr._linstor @@ -1636,7 +1651,8 @@ def raise_bad_load(e): except Exception as e: raise_bad_load(e) - def create(self, sr_uuid, vdi_uuid, size): + @override + def create(self, sr_uuid, vdi_uuid, size) -> str: # Usage example: # xe vdi-create sr-uuid=39a5826b-5a90-73eb-dd09-51e3a116f937 # name-label="linstor-vdi-1" virtual-size=4096MiB sm-config:type=vhd @@ -1744,7 +1760,8 @@ def create(self, sr_uuid, vdi_uuid, size): return VDI.VDI.get_params(self) - def delete(self, sr_uuid, vdi_uuid, data_only=False): + @override + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: util.SMlog('LinstorVDI.delete for {}'.format(self.uuid)) if self.attached: raise xs_errors.XenError('VDIInUse') @@ -1790,7 +1807,8 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False): self.sr._kick_gc() return super(LinstorVDI, self).delete(sr_uuid, vdi_uuid, data_only) - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: util.SMlog('LinstorVDI.attach for {}'.format(self.uuid)) attach_from_config = self.sr.srcmd.cmd == 'vdi_attach_from_config' if ( @@ -1845,7 +1863,8 @@ def attach(self, sr_uuid, vdi_uuid): self.attached = True return VDI.VDI.attach(self, self.sr.uuid, self.uuid) - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: util.SMlog('LinstorVDI.detach for {}'.format(self.uuid)) detach_from_config = self.sr.srcmd.cmd == 'vdi_detach_from_config' self.attached = False @@ -1907,7 +1926,8 @@ def detach(self, sr_uuid, vdi_uuid): util.SMlog('Failed to clean VDI {} during detach: {}'.format(vdi_uuid, e)) vdi_uuid = parent_vdi_uuid - def resize(self, sr_uuid, vdi_uuid, size): + @override + def resize(self, sr_uuid, vdi_uuid, size) -> str: util.SMlog('LinstorVDI.resize for {}'.format(self.uuid)) if not self.sr.is_master(): raise xs_errors.XenError( @@ -1973,10 +1993,12 @@ def resize(self, sr_uuid, vdi_uuid, size): self.sr._update_stats(self.size - old_size) return VDI.VDI.get_params(self) - def clone(self, sr_uuid, vdi_uuid): + @override + def clone(self, sr_uuid, vdi_uuid) -> str: return self._do_snapshot(sr_uuid, vdi_uuid, VDI.SNAPSHOT_DOUBLE) - def compose(self, sr_uuid, vdi1, vdi2): + @override + def compose(self, sr_uuid, vdi1, vdi2) -> None: util.SMlog('VDI.compose for {} -> {}'.format(vdi2, vdi1)) if self.vdi_type != vhdutil.VDI_TYPE_VHD: raise xs_errors.XenError('Unimplemented') @@ -2007,7 +2029,8 @@ def compose(self, sr_uuid, vdi1, vdi2): util.SMlog('Compose done') - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: """ Generate the XML config required to attach and activate a VDI for use when XAPI is not running. Attach and @@ -2053,7 +2076,8 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), 'vdi_attach_from_config') return xmlrpc.client.dumps((config,), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: """ Attach and activate a VDI using config generated by vdi_generate_config above. This is used for cases such as @@ -2074,6 +2098,7 @@ def attach_from_config(self, sr_uuid, vdi_uuid): 'SRUnavailable', opterr='Unable to attach from config' ) + return '' def reset_leaf(self, sr_uuid, vdi_uuid): if self.vdi_type != vhdutil.VDI_TYPE_VHD: @@ -2140,7 +2165,8 @@ def _mark_hidden(self, hidden=True): }) self.hidden = hidden - def update(self, sr_uuid, vdi_uuid): + @override + def update(self, sr_uuid, vdi_uuid) -> None: xenapi = self.session.xenapi vdi_ref = xenapi.VDI.get_by_uuid(self.uuid) @@ -2316,13 +2342,15 @@ def _create_snapshot(self, snap_uuid, snap_of_uuid=None): # Implement specific SR methods. # -------------------------------------------------------------------------- - def _rename(self, oldpath, newpath): + @override + def _rename(self, oldpath, newpath) -> None: # TODO: I'm not sure... Used by CBT. volume_uuid = self._linstor.get_volume_uuid_from_device_path(oldpath) self._linstor.update_volume_name(volume_uuid, newpath) + @override def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, - cloneOp=False, secondary=None, cbtlog=None): + cloneOp=False, secondary=None, cbtlog=None) -> str: # If cbt enabled, save file consistency state. if cbtlog is not None: if blktap2.VDI.tap_status(self.session, vdi_uuid): diff --git a/drivers/MooseFSSR.py b/drivers/MooseFSSR.py index 8fc4a4ad..4ebd7bd8 100755 --- a/drivers/MooseFSSR.py +++ b/drivers/MooseFSSR.py @@ -18,6 +18,8 @@ # # MooseFSSR: Based on CEPHFSSR and FileSR, mounts MooseFS share +from sm_typing import override + import errno import os import syslog as _syslog @@ -32,6 +34,7 @@ import SRCommand import FileSR # end of careful +import VDI import cleanup import util import vhdutil @@ -79,13 +82,14 @@ class MooseFSSR(FileSR.FileSR): DRIVER_TYPE = 'moosefs' - def handles(sr_type): + @override + @staticmethod + def handles(sr_type) -> bool: # fudge, because the parent class (FileSR) checks for smb to alter its behavior return sr_type == MooseFSSR.DRIVER_TYPE or sr_type == 'smb' - handles = staticmethod(handles) - - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not self._is_moosefs_available(): raise xs_errors.XenError( 'SRUnavailable', @@ -176,7 +180,8 @@ def unmount(self, mountpoint, rmmountpoint): except OSError as inst: raise MooseFSException("Command rmdir failed with error '%s'" % inst.strerror) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self.checkmount(): try: self.mount() @@ -184,7 +189,8 @@ def attach(self, sr_uuid): raise xs_errors.SROSError(12, exc.errstr) self.attached = True - def probe(self): + @override + def probe(self) -> str: try: self.mount(PROBE_MOUNTPOINT) sr_list = filter(util.match_uuid, util.listdir(PROBE_MOUNTPOINT)) @@ -194,7 +200,8 @@ def probe(self): # Create a dictionary from the SR uuids to feed SRtoXML() return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: if not self.checkmount(): return util.SMlog("Aborting GC/coalesce") @@ -204,7 +211,8 @@ def detach(self, sr_uuid): self.unmount(self.mountpoint, True) self.attached = False - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: if self.checkmount(): raise xs_errors.SROSError(113, 'MooseFS mount point already attached') @@ -248,7 +256,8 @@ def create(self, sr_uuid, size): finally: self.detach(sr_uuid) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: # try to remove/delete non VDI contents first super(MooseFSSR, self).delete(sr_uuid) try: @@ -268,7 +277,8 @@ def delete(self, sr_uuid): if inst.code != errno.ENOENT: raise xs_errors.SROSError(114, "Failed to remove MooseFS mount point") - def vdi(self, uuid, loadLocked=False): + @override + def vdi(self, uuid, loadLocked=False) -> VDI.VDI: return MooseFSFileVDI(self, uuid) @staticmethod @@ -276,7 +286,8 @@ def _is_moosefs_available(): return util.find_executable('mfsmount') class MooseFSFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} @@ -284,7 +295,8 @@ def attach(self, sr_uuid, vdi_uuid): return super(MooseFSFileVDI, self).attach(sr_uuid, vdi_uuid) - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("MooseFSFileVDI.generate_config") if not util.pathexists(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -298,15 +310,16 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), "vdi_attach_from_config") return xmlrpc.client.dumps((config,), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: try: if not util.pathexists(self.sr.path): - self.sr.attach(sr_uuid) + return self.sr.attach(sr_uuid) except: util.logException("MooseFSFileVDI.attach_from_config") raise xs_errors.XenError('SRUnavailable', opterr='Unable to attach from config') - + return '' if __name__ == '__main__': SRCommand.run(MooseFSSR, DRIVER_INFO) diff --git a/drivers/NFSSR.py b/drivers/NFSSR.py index ef73e1b4..6eeedeb7 100755 --- a/drivers/NFSSR.py +++ b/drivers/NFSSR.py @@ -17,9 +17,12 @@ # # FileSR: local-file storage repository +from sm_typing import override + import socket import SR +import VDI import SRCommand import FileSR import util @@ -68,11 +71,13 @@ class NFSSR(FileSR.SharedFileSR): """NFS file-based storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: return type == 'nfs' - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.ops_exclusive = FileSR.OPS_EXCLUSIVE self.lock = Lock(vhdutil.LOCK_TYPE_SR, self.uuid) self.sr_vditype = SR.DEFAULT_TAP @@ -137,7 +142,8 @@ def mount(self, mountpoint, remotepath, timeout=None, retrans=None): except nfs.NfsException as exc: raise xs_errors.XenError('NFSMount', opterr=exc.errstr) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self._checkmount(): try: self.validate_remotepath(False) @@ -164,7 +170,8 @@ def mount_remotepath(self, sr_uuid): self.mount(self.path, self.remotepath, timeout=io_timeout, retrans=io_retrans) - def probe(self): + @override + def probe(self) -> str: # Verify NFS target and port util._testHost(self.dconf['server'], NFSPORT, 'NFSTarget') @@ -182,7 +189,8 @@ def probe(self): except: pass - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: """Detach the SR: Unmounts and removes the mountpoint""" if not self._checkmount(): return @@ -199,7 +207,8 @@ def detach(self, sr_uuid): self.attached = False - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: util._testHost(self.dconf['server'], NFSPORT, 'NFSTarget') self.validate_remotepath(True) if self._checkmount(): @@ -239,7 +248,8 @@ def create(self, sr_uuid, size): % inst.code) self.detach(sr_uuid) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: # try to remove/delete non VDI contents first super(NFSSR, self).delete(sr_uuid) try: @@ -260,7 +270,8 @@ def delete(self, sr_uuid): if inst.code != errno.ENOENT: raise xs_errors.XenError('NFSDelete') - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return NFSFileVDI(self, uuid) def scan_exports(self, target): @@ -287,7 +298,8 @@ def set_transport(self): class NFSFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} @@ -295,7 +307,8 @@ def attach(self, sr_uuid, vdi_uuid): return super(NFSFileVDI, self).attach(sr_uuid, vdi_uuid) - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("NFSFileVDI.generate_config") if not util.pathexists(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -311,12 +324,13 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), "vdi_attach_from_config") return xmlrpc.client.dumps((config, ), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: """Used for HA State-file only. Will not just attach the VDI but also start a tapdisk on the file""" util.SMlog("NFSFileVDI.attach_from_config") try: - self.sr.attach(sr_uuid) + return self.sr.attach(sr_uuid) except: util.logException("NFSFileVDI.attach_from_config") raise xs_errors.XenError('SRUnavailable', \ diff --git a/drivers/RawISCSISR.py b/drivers/RawISCSISR.py index 1df1c7a2..a4848a88 100644 --- a/drivers/RawISCSISR.py +++ b/drivers/RawISCSISR.py @@ -18,8 +18,11 @@ # ISCSISR: ISCSI software initiator SR driver # +from sm_typing import override + import SR import SRCommand +import VDI import BaseISCSI import LUNperVDI import util @@ -53,25 +56,30 @@ class RawISCSISR(BaseISCSI.BaseISCSISR): """Raw ISCSI storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if type == "iscsi": return True return False - handles = staticmethod(handles) - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: super(RawISCSISR, self).load(vdi_uuid) self.managed = True - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: super(RawISCSISR, self).detach_and_delete(sr_uuid) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return ISCSIVDI(self, uuid) class ISCSIVDI(LUNperVDI.RAWVDI): - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: super(ISCSIVDI, self).load(vdi_uuid) self.managed = True diff --git a/drivers/SHMSR.py b/drivers/SHMSR.py index 5e3ef7f4..250d5813 100644 --- a/drivers/SHMSR.py +++ b/drivers/SHMSR.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from sm_typing import override + import SR import VDI import SRCommand @@ -54,25 +56,29 @@ def _loadvdis(self): except: pass - def handles(type): + @override + @staticmethod + def handles(type) -> bool: """Do we handle this type?""" if type == TYPE: return True return False - handles = staticmethod(handles) - def content_type(self, sr_uuid): + @override + def content_type(self, sr_uuid) -> str: """Returns the content_type XML""" return super(SHMSR, self).content_type(sr_uuid) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: """Create a VDI class""" if 'vdi_location' in self.srcmd.params: return SHMVDI(self, uuid, self.srcmd.params['vdi_location']) else: return SHMVDI(self, uuid, self.srcmd.params['device_config']['location']) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: """Initialises the SR""" if 'location' not in self.dconf: raise xs_errors.XenError('ConfigLocationMissing') @@ -82,26 +88,31 @@ def load(self, sr_uuid): self.physical_utilisation = 0 self.virtual_allocation = 0 - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: """Std. attach""" self._loadvdis() - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: """Std. detach""" pass - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: """Scan""" self._loadvdis() - return super(SHMSR, self).scan(sr_uuid) + super(SHMSR, self).scan(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: self.attach(sr_uuid) self.detach(sr_uuid) class SHMVDI(VDI.VDI): - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: try: stat = os.stat(self.path) self.utilisation = int(stat.st_size) @@ -120,13 +131,16 @@ def __init__(self, mysr, uuid, filename): self.shareable = True self.sm_config = {} - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: pass - def clone(self, sr_uuid, vdi_uuid): + @override + def clone(self, sr_uuid, vdi_uuid) -> str: return self.get_params() - def snapshot(self, sr_uuid, vdi_uuid): + @override + def snapshot(self, sr_uuid, vdi_uuid) -> str: return self.get_params() if __name__ == '__main__': diff --git a/drivers/SMBSR.py b/drivers/SMBSR.py index 962060cd..34ba40d6 100755 --- a/drivers/SMBSR.py +++ b/drivers/SMBSR.py @@ -17,8 +17,11 @@ # # SMBSR: SMB filesystem based storage repository +from sm_typing import override + import SR import SRCommand +import VDI import FileSR import util import errno @@ -71,11 +74,13 @@ def __init__(self, errstr): class SMBSR(FileSR.SharedFileSR): """SMB file-based storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: return type == 'smb' - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: self.ops_exclusive = FileSR.OPS_EXCLUSIVE self.lock = Lock(vhdutil.LOCK_TYPE_SR, self.uuid) self.sr_vditype = SR.DEFAULT_TAP @@ -190,7 +195,8 @@ def __check_license(self): restrictions['restrict_cifs'] == "true": raise xs_errors.XenError('NoSMBLicense') - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self.checkmount(): try: self.mount() @@ -208,7 +214,8 @@ def attach(self, sr_uuid): self.attached = True - def probe(self): + @override + def probe(self) -> str: err = "SMBMount" try: self.mount(PROBE_MOUNTPOINT) @@ -223,7 +230,8 @@ def probe(self): # Create a dictionary from the SR uuids to feed SRtoXML() return util.SRtoXML({sr_uuid: {} for sr_uuid in sr_list}) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: """Detach the SR: Unmounts and removes the mountpoint""" if not self.checkmount(): return @@ -241,7 +249,8 @@ def detach(self, sr_uuid): self.attached = False - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: self.__check_license() if self.checkmount(): @@ -283,7 +292,8 @@ def create(self, sr_uuid, size): .format(os.strerror(inst.code))) from inst self.detach(sr_uuid) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: # try to remove/delete non VDI contents first super(SMBSR, self).delete(sr_uuid) try: @@ -299,12 +309,14 @@ def delete(self, sr_uuid): if inst.code != errno.ENOENT: raise xs_errors.XenError('SMBDelete') - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: return SMBFileVDI(self, uuid) class SMBFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} @@ -312,7 +324,8 @@ def attach(self, sr_uuid, vdi_uuid): return super(SMBFileVDI, self).attach(sr_uuid, vdi_uuid) - def generate_config(self, sr_uuid, vdi_uuid): + @override + def generate_config(self, sr_uuid, vdi_uuid) -> str: util.SMlog("SMBFileVDI.generate_config") if not util.pathexists(self.path): raise xs_errors.XenError('VDIUnavailable') @@ -327,17 +340,19 @@ def generate_config(self, sr_uuid, vdi_uuid): config = xmlrpc.client.dumps(tuple([resp]), "vdi_attach_from_config") return xmlrpc.client.dumps((config, ), "", True) - def attach_from_config(self, sr_uuid, vdi_uuid): + @override + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: """Used for HA State-file only. Will not just attach the VDI but also start a tapdisk on the file""" util.SMlog("SMBFileVDI.attach_from_config") try: if not util.pathexists(self.sr.path): - self.sr.attach(sr_uuid) + return self.sr.attach(sr_uuid) except: util.logException("SMBFileVDI.attach_from_config") raise xs_errors.XenError('SRUnavailable', \ opterr='Unable to attach from config') + return '' if __name__ == '__main__': diff --git a/drivers/SR.py b/drivers/SR.py index d1aacdb3..fb2798ea 100755 --- a/drivers/SR.py +++ b/drivers/SR.py @@ -80,10 +80,10 @@ class SR(object): sr_vditype: string, repository type """ - def handles(type): + @staticmethod + def handles(type) -> bool: """Returns True if this SR class understands the given dconf string""" return False - handles = staticmethod(handles) def __init__(self, srcmd, sr_uuid): """Base class initializer. All subclasses should call SR.__init__ @@ -229,7 +229,7 @@ def _addLUNperVDIkey(self): except: pass - def create(self, uuid, size): + def create(self, uuid, size) -> None: """Create this repository. This operation may delete existing data. @@ -243,7 +243,7 @@ def create(self, uuid, size): """ raise xs_errors.XenError('Unimplemented') - def delete(self, uuid): + def delete(self, uuid) -> None: """Delete this repository and its contents. This operation IS idempotent -- it will succeed if the repository @@ -259,7 +259,7 @@ def delete(self, uuid): """ raise xs_errors.XenError('Unimplemented') - def update(self, uuid): + def update(self, uuid) -> None: """Refresh the fields in the SR object Returns: @@ -270,7 +270,7 @@ def update(self, uuid): # no-op unless individual backends implement it return - def attach(self, uuid): + def attach(self, uuid) -> None: """Initiate local access to the SR. Initialises any device state required to access the substrate. @@ -283,7 +283,7 @@ def attach(self, uuid): """ raise xs_errors.XenError('Unimplemented') - def after_master_attach(self, uuid): + def after_master_attach(self, uuid) -> None: """Perform actions required after attaching on the pool master Return: None @@ -298,7 +298,7 @@ def after_master_attach(self, uuid): self.session.xenapi.message.create( msg_name, 2, "SR", uuid, msg_body) - def detach(self, uuid): + def detach(self, uuid) -> None: """Remove local access to the SR. Destroys any device state initiated by the sr_attach() operation. @@ -312,7 +312,7 @@ def detach(self, uuid): """ raise xs_errors.XenError('Unimplemented') - def probe(self): + def probe(self) -> str: """Perform a backend-specific scan, using the current dconf. If the dconf is complete, then this will return a list of the SRs present of this type on the device, if any. If the dconf is partial, then a @@ -332,7 +332,7 @@ def probe(self): """ raise xs_errors.XenError('Unimplemented') - def scan(self, uuid): + def scan(self, uuid) -> None: """ Returns: """ @@ -342,7 +342,7 @@ def scan(self, uuid): scanrecord = ScanRecord(self) scanrecord.synchronise() - def replay(self, uuid): + def replay(self, uuid) -> None: """Replay a multi-stage log entry Returns: @@ -352,30 +352,27 @@ def replay(self, uuid): """ raise xs_errors.XenError('Unimplemented') - def content_type(self, uuid): + def content_type(self, uuid) -> str: """Returns the 'content_type' of an SR as a string""" return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True) - def load(self, sr_uuid): + def load(self, sr_uuid) -> None: """Post-init hook""" pass - def check_sr(self, sr_uuid): + def check_sr(self, sr_uuid) -> None: """Hook to check SR health""" pass - def vdi(self, uuid): + def vdi(self, uuid) -> 'VDI.VDI': """Return VDI object owned by this repository""" - if uuid not in self.vdis: - self.vdis[uuid] = VDI.VDI(self, uuid) raise xs_errors.XenError('Unimplemented') - return self.vdis[uuid] - def forget_vdi(self, uuid): + def forget_vdi(self, uuid) -> None: vdi = self.session.xenapi.VDI.get_by_uuid(uuid) self.session.xenapi.VDI.db_forget(vdi) - def cleanup(self): + def cleanup(self) -> None: # callback after the op is done pass diff --git a/drivers/VDI.py b/drivers/VDI.py index bc2ae5f2..c396b80e 100755 --- a/drivers/VDI.py +++ b/drivers/VDI.py @@ -16,6 +16,8 @@ # VDI: Base class for virtual disk instances # +from sm_typing import Dict, Optional + import SR import xmlrpc.client import xs_errors @@ -132,7 +134,7 @@ def from_uuid(session, vdi_uuid): sr.srcmd.params['vdi_ref'] = vdi_ref return sr.vdi(vdi_uuid) - def create(self, sr_uuid, vdi_uuid, size): + def create(self, sr_uuid, vdi_uuid, size) -> str: """Create a VDI of size MB on the given SR. This operation IS NOT idempotent and will fail if the UUID @@ -145,7 +147,7 @@ def create(self, sr_uuid, vdi_uuid, size): """ raise xs_errors.XenError('Unimplemented') - def update(self, sr_uuid, vdi_uuid): + def update(self, sr_uuid, vdi_uuid) -> None: """Query and update the configuration of a particular VDI. Given an SR and VDI UUID, this operation returns summary statistics @@ -155,7 +157,7 @@ def update(self, sr_uuid, vdi_uuid): # no-op unless individual backends implement it return - def introduce(self, sr_uuid, vdi_uuid): + def introduce(self, sr_uuid, vdi_uuid) -> str: """Explicitly introduce a particular VDI. Given an SR and VDI UUID and a disk location (passed in via the @@ -164,7 +166,7 @@ def introduce(self, sr_uuid, vdi_uuid): """ raise xs_errors.XenError('Unimplemented') - def attach(self, sr_uuid, vdi_uuid): + def attach(self, sr_uuid, vdi_uuid) -> str: """Initiate local access to the VDI. Initialises any device state required to access the VDI. @@ -178,7 +180,7 @@ def attach(self, sr_uuid, vdi_uuid): 'xenstore_data': (self.xenstore_data or {})} return xmlrpc.client.dumps((struct, ), "", True) - def detach(self, sr_uuid, vdi_uuid): + def detach(self, sr_uuid, vdi_uuid) -> None: """Remove local access to the VDI. Destroys any device state initialised via the vdi.attach() command. @@ -186,7 +188,7 @@ def detach(self, sr_uuid, vdi_uuid): """ raise xs_errors.XenError('Unimplemented') - def clone(self, sr_uuid, vdi_uuid): + def clone(self, sr_uuid, vdi_uuid) -> str: """Create a mutable instance of the referenced VDI. This operation is not idempotent and will fail if the UUID @@ -207,14 +209,14 @@ def resize_online(self, sr_uuid, vdi_uuid, size): been paused for the duration of this call.""" raise xs_errors.XenError('Unimplemented') - def generate_config(self, sr_uuid, vdi_uuid): + def generate_config(self, sr_uuid, vdi_uuid) -> str: """Generate the XML config required to activate a VDI for use when XAPI is not running. Activation is handled by the vdi_attach_from_config() SMAPI call. """ raise xs_errors.XenError('Unimplemented') - def compose(self, sr_uuid, vdi1, vdi2): + def compose(self, sr_uuid, vdi1, vdi2) -> None: """Layer the updates from [vdi2] onto [vdi1], calling the result [vdi2]. @@ -223,7 +225,7 @@ def compose(self, sr_uuid, vdi1, vdi2): """ raise xs_errors.XenError('Unimplemented') - def attach_from_config(self, sr_uuid, vdi_uuid): + def attach_from_config(self, sr_uuid, vdi_uuid) -> str: """Activate a VDI based on the config passed in on the CLI. For use when XAPI is not running. The config is generated by the Activation is handled by the vdi_generate_config() SMAPI call. @@ -231,23 +233,23 @@ def attach_from_config(self, sr_uuid, vdi_uuid): raise xs_errors.XenError('Unimplemented') def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, - cloneOp=False, secondary=None, cbtlog=None): + cloneOp=False, secondary=None, cbtlog=None) -> str: raise xs_errors.XenError('Unimplemented') - def _delete_cbt_log(self): + def _delete_cbt_log(self) -> None: raise xs_errors.XenError('Unimplemented') - def _rename(self, old, new): + def _rename(self, old, new) -> None: raise xs_errors.XenError('Unimplemented') - def _cbt_log_exists(self, logpath): + def _cbt_log_exists(self, logpath) -> bool: """Check if CBT log file exists Must be implemented by all classes inheriting from base VDI class """ raise xs_errors.XenError('Unimplemented') - def resize(self, sr_uuid, vdi_uuid, size): + def resize(self, sr_uuid, vdi_uuid, size) -> str: """Resize the given VDI to size MB. Size can be any valid disk size greater than [or smaller than] the current value. @@ -293,7 +295,7 @@ def resize_cbt(self, sr_uuid, vdi_uuid, size): % vdi_uuid) self._disable_cbt_on_error(alert_name, alert_str) - def delete(self, sr_uuid, vdi_uuid, data_only=False): + def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None: """Delete this VDI. This operation IS idempotent and should succeed if the VDI @@ -369,7 +371,7 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False): lock.release() lock.cleanup("cbtlog", str(vdi_uuid)) - def snapshot(self, sr_uuid, vdi_uuid): + def snapshot(self, sr_uuid, vdi_uuid) -> str: """Save an immutable copy of the referenced VDI. This operation IS NOT idempotent and will fail if the UUID @@ -403,7 +405,7 @@ def snapshot(self, sr_uuid, vdi_uuid): return self._do_snapshot(sr_uuid, vdi_uuid, snapType, secondary=secondary, cbtlog=cbtlog) - def activate(self, sr_uuid, vdi_uuid): + def activate(self, sr_uuid, vdi_uuid) -> Optional[Dict[str, str]]: """Activate VDI - called pre tapdisk open""" if self._get_blocktracking_status(): if 'args' in self.sr.srcmd.params: @@ -441,7 +443,7 @@ def activate(self, sr_uuid, vdi_uuid): return {'cbtlog': logpath} return None - def deactivate(self, sr_uuid, vdi_uuid): + def deactivate(self, sr_uuid, vdi_uuid) -> None: """Deactivate VDI - called post tapdisk close""" if self._get_blocktracking_status(): from lock import Lock @@ -457,7 +459,7 @@ def deactivate(self, sr_uuid, vdi_uuid): finally: lock.release() - def get_params(self): + def get_params(self) -> str: """ Returns: XMLRPC response containing a single struct with fields @@ -467,7 +469,7 @@ def get_params(self): 'uuid': self.uuid} return xmlrpc.client.dumps((struct, ), "", True) - def load(self, vdi_uuid): + def load(self, vdi_uuid) -> None: """Post-init hook""" pass @@ -804,7 +806,7 @@ def _cbt_snapshot(self, snapshot_uuid, consistency_state): % self.uuid) self._disable_cbt_on_error(alert_name, alert_str) - def _get_blocktracking_status(self, uuid=None): + def _get_blocktracking_status(self, uuid=None) -> bool: """ Get blocktracking status """ if not uuid: uuid = self.uuid @@ -825,7 +827,7 @@ def _set_blocktracking_status(self, vdi_ref, enable): self.session.xenapi.VDI.add_to_other_config( vdi_ref, "cbt_enabled", enable) - def _ensure_cbt_space(self): + def _ensure_cbt_space(self) -> None: """ Ensure enough CBT space """ pass @@ -834,12 +836,12 @@ def _get_cbt_logname(self, uuid): logName = "%s.%s" % (uuid, CBTLOG_TAG) return logName - def _get_cbt_logpath(self, uuid): + def _get_cbt_logpath(self, uuid) -> str: """ Get CBT logpath """ logName = self._get_cbt_logname(uuid) return os.path.join(self.sr.path, logName) - def _create_cbt_log(self): + def _create_cbt_log(self) -> str: """ Create CBT log """ try: logpath = self._get_cbt_logpath(self.uuid) @@ -858,7 +860,7 @@ def _create_cbt_log(self): return logpath - def _activate_cbt_log(self, logname): + def _activate_cbt_log(self, logname) -> bool: """Activate CBT log file SR specific Implementation required for VDIs on block-based SRs. @@ -866,7 +868,7 @@ def _activate_cbt_log(self, logname): """ return False - def _deactivate_cbt_log(self, logname): + def _deactivate_cbt_log(self, logname) -> None: """Deactivate CBT log file SR specific Implementation required for VDIs on block-based SRs. diff --git a/drivers/XFSSR.py b/drivers/XFSSR.py index ad4aca74..5bd73267 100755 --- a/drivers/XFSSR.py +++ b/drivers/XFSSR.py @@ -18,9 +18,12 @@ # # XFSSR: Based on local-file storage repository, mounts xfs partition +from sm_typing import override + import SR from SR import deviceCheck import SRCommand +import VDI import FileSR import util import lvutil @@ -60,11 +63,13 @@ class XFSSR(FileSR.FileSR): DRIVER_TYPE = 'xfs' - def handles(srtype): + @override + @staticmethod + def handles(srtype) -> bool: return srtype == XFSSR.DRIVER_TYPE - handles = staticmethod(handles) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not self._is_xfs_available(): raise xs_errors.XenError( 'SRUnavailable', @@ -81,7 +86,8 @@ def load(self, sr_uuid): self.attached = self._checkmount() self.driver_config = DRIVER_CONFIG - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: super(XFSSR, self).delete(sr_uuid) # Check PVs match VG @@ -111,7 +117,8 @@ def delete(self, sr_uuid): raise xs_errors.XenError('LVMDelete', \ opterr='errno is %d' % inst.code) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not self._checkmount(): try: #Activate LV @@ -150,7 +157,8 @@ def attach(self, sr_uuid): for dev in self.dconf['device'].split(','): self.block_setscheduler(dev) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: super(XFSSR, self).detach(sr_uuid) try: # deactivate SR @@ -160,13 +168,15 @@ def detach(self, sr_uuid): raise xs_errors.XenError('LVMUnMount', \ opterr='lvm -an failed errno is %d' % inst.code) + @override @deviceCheck - def probe(self): + def probe(self) -> str: return lvutil.srlist_toxml(lvutil.scan_srlist(EXT_PREFIX, self.dconf['device']), EXT_PREFIX) + @override @deviceCheck - def create(self, sr_uuid, size): + def create(self, sr_uuid, size) -> None: if self._checkmount(): raise xs_errors.XenError('SRExists') @@ -224,7 +234,8 @@ def create(self, sr_uuid, size): scsiutil.add_serial_record(self.session, self.sr_ref, \ scsiutil.devlist_to_serialstring(self.dconf['device'].split(','))) - def vdi(self, uuid, loadLocked = False): + @override + def vdi(self, uuid, loadLocked = False) -> VDI.VDI: return XFSFileVDI(self, uuid) @staticmethod @@ -233,7 +244,8 @@ def _is_xfs_available(): class XFSFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} diff --git a/drivers/ZFSSR.py b/drivers/ZFSSR.py index cf5eb12d..13a895a2 100644 --- a/drivers/ZFSSR.py +++ b/drivers/ZFSSR.py @@ -14,8 +14,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from sm_typing import override + import SR import SRCommand +import VDI import FileSR @@ -71,11 +74,13 @@ def is_zfs_path(path): class ZFSSR(FileSR.FileSR): DRIVER_TYPE = 'zfs' + @override @staticmethod - def handles(type): + def handles(type) -> bool: return type == ZFSSR.DRIVER_TYPE - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: if not is_zfs_available(): raise xs_errors.XenError( 'SRUnavailable', @@ -83,7 +88,8 @@ def load(self, sr_uuid): ) return super(ZFSSR, self).load(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: if not is_zfs_path(self.remotepath): raise xs_errors.XenError( 'ZFSSRCreate', @@ -91,7 +97,8 @@ def create(self, sr_uuid, size): ) return super(ZFSSR, self).create(sr_uuid, size) - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: if not self._checkmount(): raise xs_errors.XenError( 'ZFSSRDelete', @@ -99,28 +106,33 @@ def delete(self, sr_uuid): ) return super(ZFSSR, self).delete(sr_uuid) - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: if not is_zfs_path(self.remotepath): raise xs_errors.XenError( 'SRUnavailable', opterr='Invalid ZFS path' ) - return super(ZFSSR, self).attach(sr_uuid) + super(ZFSSR, self).attach(sr_uuid) - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: return super(ZFSSR, self).detach(sr_uuid) - def vdi(self, uuid, loadLocked=False): + @override + def vdi(self, uuid, loadLocked=False) -> VDI.VDI: return ZFSFileVDI(self, uuid) # Ensure _checkmount is overridden to prevent bad behaviors in FileSR. - def _checkmount(self): + @override + def _checkmount(self) -> bool: return super(ZFSSR, self)._checkmount() and \ is_zfs_path(self.remotepath) class ZFSFileVDI(FileSR.FileVDI): - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if not hasattr(self, 'xenstore_data'): self.xenstore_data = {} diff --git a/drivers/blktap2.py b/drivers/blktap2.py index 672f7448..bb2cbcac 100755 --- a/drivers/blktap2.py +++ b/drivers/blktap2.py @@ -18,7 +18,9 @@ # blktap2: blktap/tapdisk management layer # -from sm_typing import Any, Callable, ClassVar, Dict +from sm_typing import Any, Callable, ClassVar, Dict, override + +from abc import abstractmethod import grp import os @@ -152,7 +154,8 @@ def __init__(self, cmd, **info): self.cmd = cmd self.info = info - def __str__(self): + @override + def __str__(self) -> str: items = self.info.items() info = ", ".join("%s=%s" % item for item in items) @@ -448,7 +451,8 @@ class TapdiskExists(Exception): def __init__(self, tapdisk): self.tapdisk = tapdisk - def __str__(self): + @override + def __str__(self) -> str: return "%s already running" % self.tapdisk @@ -458,7 +462,8 @@ class TapdiskNotRunning(Exception): def __init__(self, **attrs): self.attrs = attrs - def __str__(self): + @override + def __str__(self) -> str: items = iter(self.attrs.items()) attrs = ", ".join("%s=%s" % attr for attr in items) @@ -471,7 +476,8 @@ class TapdiskNotUnique(Exception): def __init__(self, tapdisks): self.tapdisks = tapdisks - def __str__(self): + @override + def __str__(self) -> str: tapdisks = map(str, self.tapdisks) return "Found multiple tapdisks: %s" % tapdisks @@ -483,7 +489,8 @@ def __init__(self, arg, err): self.arg = arg self.err = err - def __str__(self): + @override + def __str__(self) -> str: return "Tapdisk(%s): %s" % (self.arg, self.err) def get_error(self): @@ -496,7 +503,8 @@ class TapdiskInvalidState(Exception): def __init__(self, tapdisk): self.tapdisk = tapdisk - def __str__(self): + @override + def __str__(self) -> str: return str(self.tapdisk) @@ -518,8 +526,9 @@ class KObject(object): SYSFS_CLASSTYPE: ClassVar[str] = "" - def sysfs_devname(self): - raise NotImplementedError("sysfs_devname is undefined") + @abstractmethod + def sysfs_devname(self) -> str: + pass class Attribute(object): @@ -538,7 +547,8 @@ class NoSuchAttribute(Exception): def __init__(self, name): self.name = name - def __str__(self): + @override + def __str__(self) -> str: return "No such attribute: %s" % self.name def _open(self, mode='r'): @@ -595,10 +605,12 @@ def allocate(cls): def free(self): TapCtl.free(self.minor) - def __str__(self): + @override + def __str__(self) -> str: return "%s(minor=%d)" % (self.__class__.__name__, self.minor) - def sysfs_devname(self): + @override + def sysfs_devname(self) -> str: return "blktap!blktap%d" % self.minor class Pool(Attribute): @@ -666,7 +678,8 @@ def __init__(self, pid, minor, _type, path, state): self._dirty = False self._blktap = None - def __str__(self): + @override + def __str__(self) -> str: state = self.pause_state() return "Tapdisk(%s, pid=%d, minor=%s, state=%s)" % \ (self.get_arg(), self.pid, self.minor, state) @@ -752,7 +765,8 @@ def __init__(self, _type, path): self.type = _type self.path = path - def __str__(self): + @override + def __str__(self) -> str: return "%s:%s" % (self.type, self.path) @classmethod @@ -772,14 +786,16 @@ class InvalidType(Exception): def __init__(self, _type): self.type = _type - def __str__(self): + @override + def __str__(self) -> str: return "Not a Tapdisk type: %s" % self.type class InvalidArgument(Exception): def __init__(self, arg): self.arg = arg - def __str__(self): + @override + def __str__(self) -> str: return "Not a Tapdisk image: %s" % self.arg def get_arg(self): @@ -921,7 +937,8 @@ def _refresh(self, __get): t = self.from_minor(__get('minor')) self.__init__(t.pid, t.minor, t.type, t.path, t.state) - def __getattribute__(self, name): + @override + def __getattribute__(self, name) -> Any: def __get(name): # NB. avoid(rec(ursion) return object.__getattribute__(self, name) @@ -1091,7 +1108,8 @@ def __init__(self, vdi_type, target): self.vdi_type = vdi_type self.target = target - def __str__(self): + @override + def __str__(self) -> str: return \ "Target %s has unexpected VDI type '%s'" % \ (type(self.target), self.vdi_type) @@ -1176,11 +1194,12 @@ class Link(object): BASEDIR: ClassVar[str] = "" - def _mklink(self, target): - raise NotImplementedError("_mklink is not defined") + def _mklink(self, target) -> None: + pass - def _equals(self, target): - raise NotImplementedError("_equals is not defined") + @abstractmethod + def _equals(self, target) -> bool: + pass def __init__(self, path): self._path = path @@ -1201,7 +1220,7 @@ def path(self): def stat(self): return os.stat(self.path()) - def mklink(self, target): + def mklink(self, target) -> None: path = self.path() util.SMlog("%s -> %s" % (self, target)) @@ -1224,7 +1243,8 @@ def unlink(self): if e.errno != errno.ENOENT: raise - def __str__(self): + @override + def __str__(self) -> str: path = self.path() return "%s(%s)" % (self.__class__.__name__, path) @@ -1237,10 +1257,12 @@ def readlink(self): def symlink(self): return self.path() - def _mklink(self, target): + @override + def _mklink(self, target) -> None: os.symlink(target, self.path()) - def _equals(self, target): + @override + def _equals(self, target) -> bool: return self.readlink() == target class DeviceNode(Link): @@ -1257,7 +1279,8 @@ def is_block(cls, target): """Whether @target refers to a block device.""" return S_ISBLK(cls._real_stat(target).st_mode) - def _mklink(self, target): + @override + def _mklink(self, target) -> None: st = self._real_stat(target) if not S_ISBLK(st.st_mode): @@ -1267,7 +1290,8 @@ def _mklink(self, target): os.mknod(self.path(), st.st_mode | stat.S_IRGRP, st.st_rdev) os.chown(self.path(), st.st_uid, grp.getgrnam("disk").gr_gid) - def _equals(self, target): + @override + def _equals(self, target) -> bool: target_rdev = self._real_stat(target).st_rdev return self.stat().st_rdev == target_rdev @@ -1282,7 +1306,8 @@ def __init__(self, path, st): self.path = path self.st = st - def __str__(self): + @override + def __str__(self) -> str: return "%s is not a block device: %s" % (self.path, self.st) class Hybrid(Link): @@ -1298,14 +1323,16 @@ def rdev(self): return self._devnode.rdev() raise self._devnode.NotABlockDevice(self.path(), st) - def mklink(self, target): + @override + def mklink(self, target) -> None: if self._devnode.is_block(target): self._obj = self._devnode else: self._obj = self._symlink self._obj.mklink(target) - def _equals(self, target): + @override + def _equals(self, target) -> bool: return self._obj._equals(target) class PhyLink(SymLink): @@ -2106,7 +2133,8 @@ def __init__(self, args): super().__init__(args) self.key = args[0] - def __str__(self): + @override + def __str__(self) -> str: return \ "Key '%s' missing in environment. " % self.key + \ "Not called in udev context?" @@ -2129,7 +2157,8 @@ def __init__(self, event, handler): self.event = event self.handler = handler - def __str__(self): + @override + def __str__(self) -> str: return "Uevent '%s' not handled by %s" % \ (self.event, self.handler.__class__.__name__) @@ -2145,7 +2174,8 @@ def run(self): return fn(self) - def __str__(self): + @override + def __str__(self) -> str: try: action = self.get_action() except: @@ -2160,7 +2190,8 @@ def __init__(self): ClassDevice.__init__(self) self._default_pool = None - def sysfs_devname(self): + @override + def sysfs_devname(self) -> str: return "blktap!control" class DefaultPool(Attribute): @@ -2187,7 +2218,8 @@ class NoSuchPool(Exception): def __init__(self, name): self.name = name - def __str__(self): + @override + def __str__(self) -> str: return "No such pool: {}".format(self.name) def get_pool(self, name): @@ -2207,6 +2239,10 @@ def __init__(self, path): self.path = path self._size = None + @override + def sysfs_devname(self) -> str: + return '' + def sysfs_path(self): return self.path @@ -2347,11 +2383,13 @@ def signal_hotplug(self, online=True): self._xs_rm_path(xapi_path) self._xs_rm_path(upstream_path) - def sysfs_devname(self): + @override + def sysfs_devname(self) -> str: return "%s-%d-%d" % (self.XENBUS_DEVTYPE, self.domid, self.devid) - def __str__(self): + @override + def __str__(self) -> str: return self.sysfs_devname() @classmethod @@ -2402,7 +2440,8 @@ def __init__(self, vbd, _str): self.vbd = vbd self.str = _str - def __str__(self): + @override + def __str__(self) -> str: return "Backend %s " % self.vbd + \ "has %s = %s" % (self.KEY, self.str) @@ -2435,10 +2474,12 @@ def makedev(self): def is_tap(self): return self.major == Tapdisk.major() - def __str__(self): + @override + def __str__(self) -> str: return "%s:%s" % (self.major, self.minor) - def __eq__(self, other): + @override + def __eq__(self, other) -> bool: return \ self.major == other.major and \ self.minor == other.minor @@ -2554,14 +2595,16 @@ def __init__(self, ident=None, action=None): UEventHandler.__init__(self) - def run(self): + @override + def run(self) -> None: self.xs_path = self.getenv('XENBUS_PATH') openlog(str(self), 0, self.LOG_FACILITY) UEventHandler.run(self) - def __str__(self): + @override + def __str__(self) -> str: try: path = self.xs_path diff --git a/drivers/cleanup.py b/drivers/cleanup.py index 940e6802..f1341dc9 100755 --- a/drivers/cleanup.py +++ b/drivers/cleanup.py @@ -18,6 +18,8 @@ # Script to coalesce and garbage collect VHD-based SR's in the background # +from sm_typing import Optional, override + import os import os.path import sys @@ -113,7 +115,7 @@ class Util: PREFIX = {"G": 1024 * 1024 * 1024, "M": 1024 * 1024, "K": 1024} @staticmethod - def log(text): + def log(text) -> None: util.SMlog(text, ident="SMGC") @staticmethod @@ -552,11 +554,11 @@ def __init__(self, sr, uuid, raw): def extractUuid(path): raise NotImplementedError("Implement in sub class") - def load(self, info=None): + def load(self, info=None) -> None: """Load VDI info""" - pass # abstract + pass - def getDriverName(self): + def getDriverName(self) -> str: return self.DRIVER_NAME_VHD def getRef(self): @@ -588,7 +590,7 @@ def ensureUnpaused(self): Util.log("Unpausing VDI %s" % self) self.unpause() - def pause(self, failfast=False): + def pause(self, failfast=False) -> None: if not blktap2.VDI.tap_pause(self.sr.xapi.session, self.sr.uuid, self.uuid, failfast): raise util.SMException("Failed to pause VDI %s" % self) @@ -709,10 +711,10 @@ def getAllPrunable(self): vdiList.append(self) return vdiList - def getSizeVHD(self): + def getSizeVHD(self) -> int: return self._sizeVHD - def getAllocatedSize(self): + def getAllocatedSize(self) -> int: return self._sizeAllocated def getTreeRoot(self): @@ -745,12 +747,12 @@ def getAllLeaves(self): leaves.extend(child.getAllLeaves()) return leaves - def updateBlockInfo(self): + def updateBlockInfo(self) -> Optional[str]: val = base64.b64encode(self._queryVHDBlocks()).decode() self.setConfig(VDI.DB_VHD_BLOCKS, val) return val - def rename(self, uuid): + def rename(self, uuid) -> None: "Rename the VDI file" assert(not self.sr.vdis.get(uuid)) self._clearRef() @@ -761,19 +763,20 @@ def rename(self, uuid): del self.sr.vdis[oldUuid] self.sr.vdis[self.uuid] = self - def delete(self): + def delete(self) -> None: "Physically delete the VDI" lock.Lock.cleanup(self.uuid, lvhdutil.NS_PREFIX_LVM + self.sr.uuid) lock.Lock.cleanupAll(self.uuid) self._clear() - def getParent(self): + def getParent(self) -> str: return vhdutil.getParent(self.path, lambda x: x.strip()) - def repair(self, parent): + def repair(self, parent) -> None: vhdutil.repair(parent) - def __str__(self): + @override + def __str__(self) -> str: strHidden = "" if self.hidden: strHidden = "*" @@ -794,7 +797,7 @@ def __str__(self): return "%s%s(%s%s%s)%s" % (strHidden, self.uuid[0:8], strSizeVirt, strSizeVHD, strSizeAllocated, strType) - def validate(self, fast=False): + def validate(self, fast=False) -> None: if not vhdutil.check(self.path, fast=fast): raise util.SMException("VHD %s corrupted" % self) @@ -808,7 +811,7 @@ def _clear(self): def _clearRef(self): self._vdiRef = None - def _doCoalesce(self): + def _doCoalesce(self) -> None: """Coalesce self onto parent. Only perform the actual coalescing of VHD, but not the subsequent relinking. We'll do that as the next step, after reloading the entire SR in case things have changed while we @@ -890,7 +893,7 @@ def _reportCoalesceError(vdi, ce): if xcmsg: xapi.message.create(msg_name, "3", "SR", vdi.sr.uuid, msg_body) - def coalesce(self): + def coalesce(self) -> int: # size is returned in sectors return vhdutil.coalesce(self.path) * 512 @@ -951,7 +954,7 @@ def _coalesceVHD(self, timeOut): util.fistpoint.activate("LVHDRT_coalescing_VHD_data", self.sr.uuid) - def _relinkSkip(self): + def _relinkSkip(self) -> None: """Relink children of this VDI to point to the parent of this VDI""" abortFlag = IPCFlag(self.sr.uuid) for child in self.children: @@ -1022,7 +1025,7 @@ def _loadInfoParent(self): if ret: self.parentUuid = ret - def _setParent(self, parent): + def _setParent(self, parent) -> None: vhdutil.setParent(self.path, parent.path, False) self.parent = parent self.parentUuid = parent.uuid @@ -1035,15 +1038,15 @@ def _setParent(self, parent): Util.log("Failed to update %s with vhd-parent field %s" % \ (self.uuid, self.parentUuid)) - def _loadInfoHidden(self): + def _loadInfoHidden(self) -> None: hidden = vhdutil.getHidden(self.path) self.hidden = (hidden != 0) - def _setHidden(self, hidden=True): + def _setHidden(self, hidden=True) -> None: vhdutil.setHidden(self.path, hidden) self.hidden = hidden - def _increaseSizeVirt(self, size, atomic=True): + def _increaseSizeVirt(self, size, atomic=True) -> None: """ensure the virtual size of 'self' is at least 'size'. Note that resizing a VHD must always be offline and atomically: the file must not be open by anyone and no concurrent operations may take place. @@ -1075,14 +1078,14 @@ def _increaseSizeVirt(self, size, atomic=True): self.sizeVirt = vhdutil.getSizeVirt(self.path) - def _setSizeVirt(self, size): + def _setSizeVirt(self, size) -> None: """WARNING: do not call this method directly unless all VDIs in the subtree are guaranteed to be unplugged (and remain so for the duration of the operation): this operation is only safe for offline VHDs""" jFile = os.path.join(self.sr.path, self.uuid) vhdutil.setSizeVirt(self.path, size, jFile) - def _queryVHDBlocks(self): + def _queryVHDBlocks(self) -> bytes: return vhdutil.getBlockBitmap(self.path) def _getCoalescedSizeData(self): @@ -1101,20 +1104,20 @@ def _getCoalescedSizeData(self): assert(sizeData <= self.sizeVirt) return sizeData - def _calcExtraSpaceForCoalescing(self): + def _calcExtraSpaceForCoalescing(self) -> int: sizeData = self._getCoalescedSizeData() sizeCoalesced = sizeData + vhdutil.calcOverheadBitmap(sizeData) + \ vhdutil.calcOverheadEmpty(self.sizeVirt) Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) return sizeCoalesced - self.parent.getSizeVHD() - def _calcExtraSpaceForLeafCoalescing(self): + def _calcExtraSpaceForLeafCoalescing(self) -> int: """How much extra space in the SR will be required to [live-]leaf-coalesce this VDI""" # the space requirements are the same as for inline coalesce return self._calcExtraSpaceForCoalescing() - def _calcExtraSpaceForSnapshotCoalescing(self): + def _calcExtraSpaceForSnapshotCoalescing(self) -> int: """How much extra space in the SR will be required to snapshot-coalesce this VDI""" return self._calcExtraSpaceForCoalescing() + \ @@ -1149,7 +1152,8 @@ def __init__(self, sr, uuid, raw): else: self.fileName = "%s%s" % (self.uuid, vhdutil.FILE_EXTN_VHD) - def load(self, info=None): + @override + def load(self, info=None) -> None: if not info: if not util.pathexists(self.path): raise util.SMException("%s not found" % self.path) @@ -1169,7 +1173,8 @@ def load(self, info=None): self.path = os.path.join(self.sr.path, "%s%s" % \ (self.uuid, vhdutil.FILE_EXTN_VHD)) - def rename(self, uuid): + @override + def rename(self, uuid) -> None: oldPath = self.path VDI.rename(self, uuid) self.fileName = "%s%s" % (self.uuid, vhdutil.FILE_EXTN_VHD) @@ -1178,7 +1183,8 @@ def rename(self, uuid): Util.log("Renaming %s -> %s" % (oldPath, self.path)) os.rename(oldPath, self.path) - def delete(self): + @override + def delete(self) -> None: if len(self.children) > 0: raise util.SMException("VDI %s has children, can't delete" % \ self.uuid) @@ -1193,7 +1199,8 @@ def delete(self): raise util.SMException("os.unlink(%s) failed" % self.path) VDI.delete(self) - def getAllocatedSize(self): + @override + def getAllocatedSize(self) -> int: if self._sizeAllocated == -1: self._sizeAllocated = vhdutil.getAllocatedSize(self.path) return self._sizeAllocated @@ -1205,7 +1212,8 @@ class LVHDVDI(VDI): JRN_ZERO = "zero" # journal entry type for zeroing out end of parent DRIVER_NAME_RAW = "aio" - def load(self, info=None): + @override + def load(self, info=None) -> None: # `info` is always set. `None` default value is only here to match parent method. assert info, "No info given to LVHDVDI.load" self.parent = None @@ -1227,7 +1235,8 @@ def load(self, info=None): def extractUuid(path): return lvhdutil.extractUuid(path) - def getDriverName(self): + @override + def getDriverName(self) -> str: if self.raw: return self.DRIVER_NAME_RAW return self.DRIVER_NAME_VHD @@ -1274,11 +1283,14 @@ def inflateParentForCoalesce(self): util.fistpoint.activate("LVHDRT_coalescing_before_inflate_grandparent", self.sr.uuid) self.parent.inflate(self.parent.sizeLV + inc) - def updateBlockInfo(self): + @override + def updateBlockInfo(self) -> Optional[str]: if not self.raw: return VDI.updateBlockInfo(self) + return None - def rename(self, uuid): + @override + def rename(self, uuid) -> None: oldUuid = self.uuid oldLVName = self.fileName VDI.rename(self, uuid) @@ -1297,7 +1309,8 @@ def rename(self, uuid): RefCounter.set(self.uuid, cnt, bcnt, ns) RefCounter.reset(oldUuid, ns) - def delete(self): + @override + def delete(self) -> None: if len(self.children) > 0: raise util.SMException("VDI %s has children, can't delete" % \ self.uuid) @@ -1310,7 +1323,8 @@ def delete(self): RefCounter.reset(self.uuid, lvhdutil.NS_PREFIX_LVM + self.sr.uuid) VDI.delete(self) - def getSizeVHD(self): + @override + def getSizeVHD(self) -> int: if self._sizeVHD == -1: self._loadInfoSizeVHD() return self._sizeVHD @@ -1328,7 +1342,8 @@ def _loadInfoSizeVHD(self): raise util.SMException("phys size of %s = %d" % \ (self, self._sizeVHD)) - def getAllocatedSize(self): + @override + def getAllocatedSize(self) -> int: if self._sizeAllocated == -1: self._loadInfoSizeAllocated() return self._sizeAllocated @@ -1342,20 +1357,23 @@ def _loadInfoSizeAllocated(self): self._activate() self._sizeAllocated = vhdutil.getAllocatedSize(self.path) - def _loadInfoHidden(self): + @override + def _loadInfoHidden(self) -> None: if self.raw: self.hidden = self.sr.lvmCache.getHidden(self.fileName) else: VDI._loadInfoHidden(self) - def _setHidden(self, hidden=True): + @override + def _setHidden(self, hidden=True) -> None: if self.raw: self.sr.lvmCache.setHidden(self.fileName, hidden) self.hidden = hidden else: VDI._setHidden(self, hidden) - def __str__(self): + @override + def __str__(self) -> str: strType = "VHD" if self.raw: strType = "RAW" @@ -1377,11 +1395,13 @@ def __str__(self): Util.num2str(self.sizeVirt), strSizeVHD, strSizeAllocated, Util.num2str(self.sizeLV), strActive) - def validate(self, fast=False): + @override + def validate(self, fast=False) -> None: if not self.raw: VDI.validate(self, fast) - def _doCoalesce(self): + @override + def _doCoalesce(self) -> None: """LVHD parents must first be activated, inflated, and made writable""" try: self._activateChain() @@ -1394,7 +1414,8 @@ def _doCoalesce(self): self.parent.deflate() self.sr.lvmCache.setReadonly(self.parent.fileName, True) - def _setParent(self, parent): + @override + def _setParent(self, parent) -> None: self._activate() if self.lvReadonly: self.sr.lvmCache.setReadonly(self.fileName, False) @@ -1428,7 +1449,8 @@ def _activateChain(self): def _deactivate(self): self.sr.lvActivator.deactivate(self.uuid, False) - def _increaseSizeVirt(self, size, atomic=True): + @override + def _increaseSizeVirt(self, size, atomic=True) -> None: "ensure the virtual size of 'self' is at least 'size'" self._activate() if not self.raw: @@ -1464,7 +1486,8 @@ def _increaseSizeVirt(self, size, atomic=True): VDI.POLL_INTERVAL, 0) self.sr.journaler.remove(self.JRN_ZERO, self.uuid) - def _setSizeVirt(self, size): + @override + def _setSizeVirt(self, size) -> None: """WARNING: do not call this method directly unless all VDIs in the subtree are guaranteed to be unplugged (and remain so for the duration of the operation): this operation is only safe for offline VHDs""" @@ -1477,25 +1500,29 @@ def _setSizeVirt(self, size): finally: lvhdutil.deleteVHDJournalLV(self.sr.lvmCache, self.uuid) - def _queryVHDBlocks(self): + @override + def _queryVHDBlocks(self) -> bytes: self._activate() return VDI._queryVHDBlocks(self) - def _calcExtraSpaceForCoalescing(self): + @override + def _calcExtraSpaceForCoalescing(self) -> int: if self.parent.raw: return 0 # raw parents are never deflated in the first place sizeCoalesced = lvhdutil.calcSizeVHDLV(self._getCoalescedSizeData()) Util.log("Coalesced size = %s" % Util.num2str(sizeCoalesced)) return sizeCoalesced - self.parent.sizeLV - def _calcExtraSpaceForLeafCoalescing(self): + @override + def _calcExtraSpaceForLeafCoalescing(self) -> int: """How much extra space in the SR will be required to [live-]leaf-coalesce this VDI""" # we can deflate the leaf to minimize the space requirements deflateDiff = self.sizeLV - lvhdutil.calcSizeLV(self.getSizeVHD()) return self._calcExtraSpaceForCoalescing() - deflateDiff - def _calcExtraSpaceForSnapshotCoalescing(self): + @override + def _calcExtraSpaceForSnapshotCoalescing(self) -> int: return self._calcExtraSpaceForCoalescing() + \ lvhdutil.calcSizeLV(self.getSizeVHD()) @@ -1505,7 +1532,8 @@ class LinstorVDI(VDI): VOLUME_LOCK_TIMEOUT = 30 - def load(self, info=None): + @override + def load(self, info=None) -> None: self.parentUuid = info.parentUuid self.scanError = True self.parent = None @@ -1532,7 +1560,8 @@ def load(self, info=None): self.scanError = False self.vdi_type = vhdutil.VDI_TYPE_VHD - def getSizeVHD(self, fetch=False): + @override + def getSizeVHD(self, fetch=False) -> int: if self._sizeVHD < 0 or fetch: self._sizeVHD = self.sr._vhdutil.get_size_phys(self.uuid) return self._sizeVHD @@ -1542,7 +1571,8 @@ def getDrbdSize(self, fetch=False): self.drbd_size = self.sr._vhdutil.get_drbd_size(self.uuid) return self.drbd_size - def getAllocatedSize(self): + @override + def getAllocatedSize(self) -> int: if self._sizeAllocated == -1: if not self.raw: self._sizeAllocated = self.sr._vhdutil.get_allocated_size(self.uuid) @@ -1582,14 +1612,16 @@ def inflateFully(self): if not self.raw: self.inflate(LinstorVhdUtil.compute_volume_size(self.sizeVirt, self.vdi_type)) - def rename(self, uuid): + @override + def rename(self, uuid) -> None: Util.log('Renaming {} -> {} (path={})'.format( self.uuid, uuid, self.path )) self.sr._linstor.update_volume_uuid(self.uuid, uuid) VDI.rename(self, uuid) - def delete(self): + @override + def delete(self) -> None: if len(self.children) > 0: raise util.SMException( 'VDI {} has children, can\'t delete'.format(self.uuid) @@ -1602,32 +1634,38 @@ def delete(self): self.sr.unlock() VDI.delete(self) - def validate(self, fast=False): + @override + def validate(self, fast=False) -> None: if not self.raw and not self.sr._vhdutil.check(self.uuid, fast=fast): raise util.SMException('VHD {} corrupted'.format(self)) - def pause(self, failfast=False): + @override + def pause(self, failfast=False) -> None: self.sr._linstor.ensure_volume_is_not_locked( self.uuid, timeout=self.VOLUME_LOCK_TIMEOUT ) return super(LinstorVDI, self).pause(failfast) - def coalesce(self): + @override + def coalesce(self) -> int: # Note: We raise `SMException` here to skip the current coalesce in case of failure. # Using another exception we can't execute the next coalesce calls. return self.sr._vhdutil.force_coalesce(self.path) * 512 - def getParent(self): + @override + def getParent(self) -> str: return self.sr._vhdutil.get_parent( self.sr._linstor.get_volume_uuid_from_device_path(self.path) ) - def repair(self, parent_uuid): + @override + def repair(self, parent_uuid) -> None: self.sr._vhdutil.force_repair( self.sr._linstor.get_device_path(parent_uuid) ) - def _relinkSkip(self): + @override + def _relinkSkip(self) -> None: abortFlag = IPCFlag(self.sr.uuid) for child in self.children: if abortFlag.test(FLAG_TYPE_ABORT): @@ -1651,7 +1689,8 @@ def _relinkSkip(self): blktap2.VDI.tap_unpause(session, sr_uuid, vdi_uuid) self.children = [] - def _setParent(self, parent): + @override + def _setParent(self, parent) -> None: self.sr._linstor.get_device_path(self.uuid) self.sr._vhdutil.force_parent(self.path, parent.path) self.parent = parent @@ -1665,7 +1704,8 @@ def _setParent(self, parent): Util.log("Failed to update %s with vhd-parent field %s" % \ (self.uuid, self.parentUuid)) - def _doCoalesce(self): + @override + def _doCoalesce(self) -> None: try: self._activateChain() self.parent.validate() @@ -1685,7 +1725,8 @@ def _activateChain(self): raise util.SMException(str(e)) vdi = vdi.parent - def _setHidden(self, hidden=True): + @override + def _setHidden(self, hidden=True) -> None: HIDDEN_TAG = 'hidden' if self.raw: @@ -1696,7 +1737,8 @@ def _setHidden(self, hidden=True): else: VDI._setHidden(self, hidden) - def _setSizeVirt(self, size): + @override + def _setSizeVirt(self, size) -> None: jfile = self.uuid + '-jvhd' self.sr._linstor.create_volume( jfile, vhdutil.MAX_VHD_JOURNAL_SIZE, persistent=False, volume_name=jfile @@ -1711,7 +1753,8 @@ def _setSizeVirt(self, size): # We can ignore it, in any case this volume is not persistent. pass - def _queryVHDBlocks(self): + @override + def _queryVHDBlocks(self) -> bytes: return self.sr._vhdutil.get_block_bitmap(self.uuid) def _inflateParentForCoalesce(self): @@ -1721,7 +1764,8 @@ def _inflateParentForCoalesce(self): if inc > 0: self.parent.inflate(self.parent.getDrbdSize() + inc) - def _calcExtraSpaceForCoalescing(self): + @override + def _calcExtraSpaceForCoalescing(self) -> int: if self.parent.raw: return 0 size_coalesced = LinstorVhdUtil.compute_volume_size( @@ -1730,14 +1774,16 @@ def _calcExtraSpaceForCoalescing(self): Util.log("Coalesced size = %s" % Util.num2str(size_coalesced)) return size_coalesced - self.parent.getDrbdSize() - def _calcExtraSpaceForLeafCoalescing(self): + @override + def _calcExtraSpaceForLeafCoalescing(self) -> int: assert self.getDrbdSize() > 0 assert self.getSizeVHD() > 0 deflate_diff = self.getDrbdSize() - LinstorVolumeManager.round_up_volume_size(self.getSizeVHD()) assert deflate_diff >= 0 return self._calcExtraSpaceForCoalescing() - deflate_diff - def _calcExtraSpaceForSnapshotCoalescing(self): + @override + def _calcExtraSpaceForSnapshotCoalescing(self) -> int: assert self.getSizeVHD() > 0 return self._calcExtraSpaceForCoalescing() + \ LinstorVolumeManager.round_up_volume_size(self.getSizeVHD()) @@ -1869,10 +1915,10 @@ def gcEnabled(self, refresh=True): return False return True - def scan(self, force=False): + def scan(self, force=False) -> None: """Scan the SR and load VDI info for each VDI. If called repeatedly, update VDI objects if they already exist""" - pass # abstract + pass def scanLocked(self, force=False): self.lock() @@ -2078,14 +2124,14 @@ def findGarbage(self): vdiList.extend(vdi.getAllPrunable()) return vdiList - def deleteVDIs(self, vdiList): + def deleteVDIs(self, vdiList) -> None: for vdi in vdiList: if IPCFlag(self.uuid).test(FLAG_TYPE_ABORT): raise AbortException("Aborting due to signal") Util.log("Deleting unlinked VDI %s" % vdi) self.deleteVDI(vdi) - def deleteVDI(self, vdi): + def deleteVDI(self, vdi) -> None: assert(len(vdi.children) == 0) del self.vdis[vdi.uuid] if vdi.parent: @@ -2094,10 +2140,10 @@ def deleteVDI(self, vdi): self.vdiTrees.remove(vdi) vdi.delete() - def forgetVDI(self, vdiUuid): + def forgetVDI(self, vdiUuid) -> None: self.xapi.forgetVDI(self.uuid, vdiUuid) - def pauseVDIs(self, vdiList): + def pauseVDIs(self, vdiList) -> None: paused = [] failed = False for vdi in vdiList: @@ -2124,14 +2170,15 @@ def unpauseVDIs(self, vdiList): if failed: raise util.SMException("Failed to unpause VDIs") - def getFreeSpace(self): + def getFreeSpace(self) -> int: return 0 def cleanup(self): Util.log("In cleanup") return - def __str__(self): + @override + def __str__(self) -> str: if self.name: ret = "%s ('%s')" % (self.uuid[0:4], self.name) else: @@ -2166,7 +2213,7 @@ def unlock(self): if self._locked == 0: self._srLock.release() - def needUpdateBlockInfo(self): + def needUpdateBlockInfo(self) -> bool: for vdi in self.vdis.values(): if vdi.scanError or len(vdi.children) == 0: continue @@ -2174,7 +2221,7 @@ def needUpdateBlockInfo(self): return True return False - def updateBlockInfo(self): + def updateBlockInfo(self) -> None: for vdi in self.vdis.values(): if vdi.scanError or len(vdi.children) == 0: continue @@ -2203,7 +2250,7 @@ def cleanupJournals(self, dryRun=False): if not dryRun: self.journaler.remove(t, uuid) - def cleanupCache(self, maxAge=-1): + def cleanupCache(self, maxAge=-1) -> int: return 0 def _coalesce(self, vdi): @@ -2436,7 +2483,7 @@ def _snapshotCoalesce(self, vdi): return False return True - def _liveLeafCoalesce(self, vdi): + def _liveLeafCoalesce(self, vdi) -> bool: util.fistpoint.activate("LVHDRT_coaleaf_delay_3", self.uuid) self.lock() try: @@ -2537,44 +2584,43 @@ def _doCoalesceLeaf(self, vdi): self._finishCoalesceLeaf(parent) self._updateSlavesOnResize(parent) - def _calcExtraSpaceNeeded(self, child, parent): + def _calcExtraSpaceNeeded(self, child, parent) -> int: assert(not parent.raw) # raw parents not supported extra = child.getSizeVHD() - parent.getSizeVHD() if extra < 0: extra = 0 return extra - def _prepareCoalesceLeaf(self, vdi): + def _prepareCoalesceLeaf(self, vdi) -> None: pass - def _updateNode(self, vdi): + def _updateNode(self, vdi) -> None: pass - def _finishCoalesceLeaf(self, parent): + def _finishCoalesceLeaf(self, parent) -> None: pass - def _updateSlavesOnUndoLeafCoalesce(self, parent, child): + def _updateSlavesOnUndoLeafCoalesce(self, parent, child) -> None: pass - def _updateSlavesOnRename(self, vdi, oldName, origParentUuid): + def _updateSlavesOnRename(self, vdi, oldName, origParentUuid) -> None: pass - def _updateSlavesOnResize(self, vdi): + def _updateSlavesOnResize(self, vdi) -> None: pass - def _removeStaleVDIs(self, uuidsPresent): + def _removeStaleVDIs(self, uuidsPresent) -> None: for uuid in list(self.vdis.keys()): if not uuid in uuidsPresent: Util.log("VDI %s disappeared since last scan" % \ self.vdis[uuid]) del self.vdis[uuid] - def _handleInterruptedCoalesceLeaf(self): + def _handleInterruptedCoalesceLeaf(self) -> None: """An interrupted leaf-coalesce operation may leave the VHD tree in an inconsistent state. If the old-leaf VDI is still present, we revert the operation (in case the original error is persistent); otherwise we must finish the operation""" - # abstract pass def _buildTree(self, force): @@ -2613,7 +2659,8 @@ def __init__(self, uuid, xapi, createLock, force): self.path = "/var/run/sr-mount/%s" % self.uuid self.journaler = fjournaler.Journaler(self.path) - def scan(self, force=False): + @override + def scan(self, force=False) -> None: if not util.pathexists(self.path): raise util.SMException("directory %s not found!" % self.uuid) vhds = self._scan(force) @@ -2639,10 +2686,12 @@ def scan(self, force=False): self.logFilter.logState() self._handleInterruptedCoalesceLeaf() - def getFreeSpace(self): + @override + def getFreeSpace(self) -> int: return util.get_fs_size(self.path) - util.get_fs_utilisation(self.path) - def deleteVDIs(self, vdiList): + @override + def deleteVDIs(self, vdiList) -> None: rootDeleted = False for vdi in vdiList: if not vdi.parent: @@ -2652,7 +2701,8 @@ def deleteVDIs(self, vdiList): if self.xapi.srRecord["type"] == "nfs" and rootDeleted: self.xapi.markCacheSRsDirty() - def cleanupCache(self, maxAge=-1): + @override + def cleanupCache(self, maxAge=-1) -> int: """Clean up IntelliCache cache files. Caches for leaf nodes are removed when the leaf node no longer exists or its allow-caching attribute is not set. Caches for parent nodes are removed when the @@ -2741,7 +2791,8 @@ def _scan(self, force): return vhds raise util.SMException("Scan error") - def deleteVDI(self, vdi): + @override + def deleteVDI(self, vdi) -> None: self._checkSlaves(vdi) SR.deleteVDI(self, vdi) @@ -2766,7 +2817,8 @@ def _checkSlave(self, hostRef, vdi): _host = self.xapi.session.xenapi.host text = _host.call_plugin( * call) - def _handleInterruptedCoalesceLeaf(self): + @override + def _handleInterruptedCoalesceLeaf(self) -> None: entries = self.journaler.getAll(VDI.JRN_LEAF) for uuid, parentUuid in entries.items(): fileList = os.listdir(self.path) @@ -2850,26 +2902,31 @@ def __init__(self, uuid, xapi, createLock, force): self.lvActivator = LVActivator(self.uuid, self.lvmCache) self.journaler = journaler.Journaler(self.lvmCache) - def deleteVDI(self, vdi): + @override + def deleteVDI(self, vdi) -> None: if self.lvActivator.get(vdi.uuid, False): self.lvActivator.deactivate(vdi.uuid, False) self._checkSlaves(vdi) SR.deleteVDI(self, vdi) - def forgetVDI(self, vdiUuid): + @override + def forgetVDI(self, vdiUuid) -> None: SR.forgetVDI(self, vdiUuid) mdpath = os.path.join(self.path, lvutil.MDVOLUME_NAME) LVMMetadataHandler(mdpath).deleteVdiFromMetadata(vdiUuid) - def getFreeSpace(self): + @override + def getFreeSpace(self) -> int: stats = lvutil._getVGstats(self.vgName) return stats['physical_size'] - stats['physical_utilisation'] + @override def cleanup(self): if not self.lvActivator.deactivateAll(): Util.log("ERROR deactivating LVs while cleaning up") - def needUpdateBlockInfo(self): + @override + def needUpdateBlockInfo(self) -> bool: for vdi in self.vdis.values(): if vdi.scanError or vdi.raw or len(vdi.children) == 0: continue @@ -2877,7 +2934,8 @@ def needUpdateBlockInfo(self): return True return False - def updateBlockInfo(self): + @override + def updateBlockInfo(self) -> None: numUpdated = 0 for vdi in self.vdis.values(): if vdi.scanError or vdi.raw or len(vdi.children) == 0: @@ -2892,7 +2950,8 @@ def updateBlockInfo(self): # inherit the refcount value and preventing the correct decrement self.cleanup() - def scan(self, force=False): + @override + def scan(self, force=False) -> None: vdis = self._scan(force) for uuid, vdiInfo in vdis.items(): vdi = self.getVDI(uuid) @@ -2923,7 +2982,8 @@ def _scan(self, force): return vdis raise util.SMException("Scan error") - def _removeStaleVDIs(self, uuidsPresent): + @override + def _removeStaleVDIs(self, uuidsPresent) -> None: for uuid in list(self.vdis.keys()): if not uuid in uuidsPresent: Util.log("VDI %s disappeared since last scan" % \ @@ -2932,7 +2992,8 @@ def _removeStaleVDIs(self, uuidsPresent): if self.lvActivator.get(uuid, False): self.lvActivator.remove(uuid, False) - def _liveLeafCoalesce(self, vdi): + @override + def _liveLeafCoalesce(self, vdi) -> bool: """If the parent is raw and the child was resized (virt. size), then we'll need to resize the parent, which can take a while due to zeroing out of the extended portion of the LV. Do it before pausing the child @@ -2943,13 +3004,15 @@ def _liveLeafCoalesce(self, vdi): return SR._liveLeafCoalesce(self, vdi) - def _prepareCoalesceLeaf(self, vdi): + @override + def _prepareCoalesceLeaf(self, vdi) -> None: vdi._activateChain() self.lvmCache.setReadonly(vdi.parent.fileName, False) vdi.deflate() vdi.inflateParentForCoalesce() - def _updateNode(self, vdi): + @override + def _updateNode(self, vdi) -> None: # fix the refcounts: the remaining node should inherit the binary # refcount from the leaf (because if it was online, it should remain # refcounted as such), but the normal refcount from the parent (because @@ -2963,16 +3026,19 @@ def _updateNode(self, vdi): assert(pCnt >= 0) RefCounter.set(vdi.parent.uuid, pCnt, cBcnt, ns) - def _finishCoalesceLeaf(self, parent): + @override + def _finishCoalesceLeaf(self, parent) -> None: if not parent.isSnapshot() or parent.isAttachedRW(): parent.inflateFully() else: parent.deflate() - def _calcExtraSpaceNeeded(self, child, parent): + @override + def _calcExtraSpaceNeeded(self, child, parent) -> int: return lvhdutil.calcSizeVHDLV(parent.sizeVirt) - parent.sizeLV - def _handleInterruptedCoalesceLeaf(self): + @override + def _handleInterruptedCoalesceLeaf(self) -> None: entries = self.journaler.getAll(VDI.JRN_LEAF) for uuid, parentUuid in entries.items(): childLV = lvhdutil.LV_PREFIX[vhdutil.VDI_TYPE_VHD] + uuid @@ -3083,7 +3149,8 @@ def _checkSlaves(self, vdi): if hostRef in onlineHosts: raise - def _updateSlavesOnUndoLeafCoalesce(self, parent, child): + @override + def _updateSlavesOnUndoLeafCoalesce(self, parent, child) -> None: slaves = util.get_slaves_attached_on(self.xapi.session, [child.uuid]) if not slaves: Util.log("Update-on-leaf-undo: VDI %s not attached on any slave" % \ @@ -3109,7 +3176,8 @@ def _updateSlavesOnUndoLeafCoalesce(self, parent, child): slave, self.xapi.PLUGIN_ON_SLAVE, "multi", args) Util.log("call-plugin returned: '%s'" % text) - def _updateSlavesOnRename(self, vdi, oldNameLV, origParentUuid): + @override + def _updateSlavesOnRename(self, vdi, oldNameLV, origParentUuid) -> None: slaves = util.get_slaves_attached_on(self.xapi.session, [vdi.uuid]) if not slaves: Util.log("Update-on-rename: VDI %s not attached on any slave" % vdi) @@ -3131,7 +3199,8 @@ def _updateSlavesOnRename(self, vdi, oldNameLV, origParentUuid): slave, self.xapi.PLUGIN_ON_SLAVE, "multi", args) Util.log("call-plugin returned: '%s'" % text) - def _updateSlavesOnResize(self, vdi): + @override + def _updateSlavesOnResize(self, vdi) -> None: uuids = [x.uuid for x in vdi.getAllLeaves()] slaves = util.get_slaves_attached_on(self.xapi.session, uuids) if not slaves: @@ -3154,14 +3223,17 @@ def __init__(self, uuid, xapi, createLock, force): self.path = LinstorVolumeManager.DEV_ROOT_PATH self._reloadLinstor() - def deleteVDI(self, vdi): + @override + def deleteVDI(self, vdi) -> None: self._checkSlaves(vdi) SR.deleteVDI(self, vdi) - def getFreeSpace(self): + @override + def getFreeSpace(self) -> int: return self._linstor.max_volume_size_allowed - def scan(self, force=False): + @override + def scan(self, force=False) -> None: all_vdi_info = self._scan(force) for uuid, vdiInfo in all_vdi_info.items(): # When vdiInfo is None, the VDI is RAW. @@ -3177,7 +3249,8 @@ def scan(self, force=False): self.logFilter.logState() self._handleInterruptedCoalesceLeaf() - def pauseVDIs(self, vdiList): + @override + def pauseVDIs(self, vdiList) -> None: self._linstor.ensure_volume_list_is_not_locked( vdiList, timeout=LinstorVDI.VOLUME_LOCK_TIMEOUT ) @@ -3294,18 +3367,21 @@ def _load_vdi_info(self): return all_vdi_info - def _prepareCoalesceLeaf(self, vdi): + @override + def _prepareCoalesceLeaf(self, vdi) -> None: vdi._activateChain() vdi.deflate() vdi._inflateParentForCoalesce() - def _finishCoalesceLeaf(self, parent): + @override + def _finishCoalesceLeaf(self, parent) -> None: if not parent.isSnapshot() or parent.isAttachedRW(): parent.inflateFully() else: parent.deflate() - def _calcExtraSpaceNeeded(self, child, parent): + @override + def _calcExtraSpaceNeeded(self, child, parent) -> int: return LinstorVhdUtil.compute_volume_size(parent.sizeVirt, parent.vdi_type) - parent.getDrbdSize() def _hasValidDevicePath(self, uuid): @@ -3316,7 +3392,8 @@ def _hasValidDevicePath(self, uuid): return False return True - def _liveLeafCoalesce(self, vdi): + @override + def _liveLeafCoalesce(self, vdi) -> bool: self.lock() try: self._linstor.ensure_volume_is_not_locked( @@ -3326,7 +3403,8 @@ def _liveLeafCoalesce(self, vdi): finally: self.unlock() - def _handleInterruptedCoalesceLeaf(self): + @override + def _handleInterruptedCoalesceLeaf(self) -> None: entries = self.journaler.get_all(VDI.JRN_LEAF) for uuid, parentUuid in entries.items(): if self._hasValidDevicePath(parentUuid) or \ diff --git a/drivers/flock.py b/drivers/flock.py index 2d295ec4..a853da27 100644 --- a/drivers/flock.py +++ b/drivers/flock.py @@ -23,7 +23,7 @@ got to grow our own. """ -from sm_typing import ClassVar +from sm_typing import ClassVar, override import os import fcntl @@ -62,7 +62,8 @@ def __getattr__(self, name): idx = self.FIELDS[name] return self.fields[idx] - def __setattr__(self, name, value): + @override + def __setattr__(self, name, value) -> None: idx = self.FIELDS.get(name) if idx is None: self.__dict__[name] = value diff --git a/drivers/lcache.py b/drivers/lcache.py index f1e25c36..1b7c78f2 100755 --- a/drivers/lcache.py +++ b/drivers/lcache.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from sm_typing import override + import os import blktap2 import glob @@ -71,7 +73,8 @@ def __init__(self, tapdisk, stats): self.tapdisk = tapdisk self.stats = stats - def __str__(self): + @override + def __str__(self) -> str: return \ "Tapdisk %s in state '%s' not found caching." % \ (self.tapdisk, self.stats) @@ -116,7 +119,8 @@ def vdi_stats_total(self): return rd_hits, rd_miss, wr_rdir - def __str__(self): + @override + def __str__(self) -> str: return "%s(%s, minor=%s)" % \ (self.__class__.__name__, self.tapdisk.path, self.tapdisk.minor) @@ -144,7 +148,8 @@ def vdi_stats(self): return rd_hits, rd_miss, wr_rdir - def __str__(self): + @override + def __str__(self) -> str: return "%s(%s, minor=%s)" % \ (self.__class__.__name__, self.tapdisk.path, self.tapdisk.minor) @@ -166,7 +171,8 @@ class NotAMountPoint(Exception): def __init__(self, path): self.path = path - def __str__(self): + @override + def __str__(self) -> str: return "Not a mount point: %s" % self.path @classmethod diff --git a/drivers/linstorvhdutil.py b/drivers/linstorvhdutil.py index 6ad4787d..c33c24c0 100644 --- a/drivers/linstorvhdutil.py +++ b/drivers/linstorvhdutil.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from sm_typing import override + from linstorjournaler import LinstorJournaler from linstorvolumemanager import LinstorVolumeManager import base64 @@ -60,7 +62,8 @@ class LinstorCallException(util.SMException): def __init__(self, cmd_err): self.cmd_err = cmd_err - def __str__(self): + @override + def __str__(self) -> str: return str(self.cmd_err) diff --git a/drivers/linstorvolumemanager.py b/drivers/linstorvolumemanager.py index 55327254..5b7da10a 100755 --- a/drivers/linstorvolumemanager.py +++ b/drivers/linstorvolumemanager.py @@ -15,6 +15,7 @@ # along with this program. If not, see . # +from sm_typing import override import errno import json @@ -375,7 +376,8 @@ def __init__(self, name): self.virtual_size = 0 self.diskful = [] - def __repr__(self): + @override + def __repr__(self) -> str: return 'VolumeInfo("{}", {}, {}, {})'.format( self.name, self.allocated_size, self.virtual_size, self.diskful diff --git a/drivers/lock.py b/drivers/lock.py index 2e6e2c9c..6792d7b0 100755 --- a/drivers/lock.py +++ b/drivers/lock.py @@ -194,7 +194,7 @@ def _open(self): fd = self.lockfile.fileno() self.lock = flock.WriteLock(fd) - def _open_lockfile(self): + def _open_lockfile(self) -> None: """Provide a seam, so extreme situations could be tested""" util.SMlog("lock: opening lock file %s" % self.lockpath) self.lockfile = open(self.lockpath, "w+") diff --git a/drivers/mpath_cli.py b/drivers/mpath_cli.py index 4c7ce54a..357e84a0 100755 --- a/drivers/mpath_cli.py +++ b/drivers/mpath_cli.py @@ -15,6 +15,8 @@ # # Talk to the multipathd cli +from sm_typing import override + import util import re import time @@ -25,7 +27,8 @@ class MPathCLIFail(Exception): def __init__(self): return - def __str__(self): + @override + def __str__(self) -> str: return "MPath CLI failed" mpathcmd = ["/usr/sbin/multipathd", "-k"] diff --git a/drivers/nfs-on-slave b/drivers/nfs-on-slave index 3e2ee8d3..71837724 100644 --- a/drivers/nfs-on-slave +++ b/drivers/nfs-on-slave @@ -18,6 +18,9 @@ # A plugin for synchronizing slaves when something changes on the Master import sys; sys.path.append("/opt/xensource/sm/") + +from sm_typing import override + import util import os, glob, errno @@ -31,7 +34,8 @@ class NfsCheckException(Exception): except: self.exe = None - def __str__(self): + @override + def __str__(self) -> str: return "File %s in use by pid %d (%s), fd %d" % \ (self.path, self.pid, self.exe, self.fd) diff --git a/drivers/srmetadata.py b/drivers/srmetadata.py index c80fb6d7..34aa5f16 100755 --- a/drivers/srmetadata.py +++ b/drivers/srmetadata.py @@ -16,7 +16,9 @@ # Functions to read and write SR metadata # -from sm_typing import ClassVar +from sm_typing import ClassVar, override + +from abc import abstractmethod from io import SEEK_SET @@ -275,8 +277,9 @@ def __del__(self): def vdi_info_size(self): return self.VDI_INFO_SIZE_IN_SECTORS * SECTOR_SIZE - def spaceAvailableForVdis(self, count): - raise NotImplementedError("spaceAvailableForVdis is undefined") + @abstractmethod + def spaceAvailableForVdis(self, count) -> None: + pass # common utility functions def getMetadata(self, params={}): @@ -660,10 +663,10 @@ def getMetadataToWrite(self, sr_info, vdi_info, lower, upper, update_map, \ raise # specific functions, to be implement by the child classes - def getVdiInfo(self, Dict, generateSector=0): + def getVdiInfo(self, Dict, generateSector=0) -> bytes: return b"" - def getSRInfoForSectors(self, sr_info, range): + def getSRInfoForSectors(self, sr_info, range) -> bytes: return b"" @@ -676,7 +679,8 @@ def __init__(self, path=None, write=True): lvutil.ensurePathExists(path) MetadataHandler.__init__(self, path, write) - def spaceAvailableForVdis(self, count): + @override + def spaceAvailableForVdis(self, count) -> None: created = False try: # The easiest way to do this, is to create a dummy vdi and write it @@ -707,7 +711,8 @@ def spaceAvailableForVdis(self, count): # it also takes in a parameter to determine whether both the sector # or only one sector needs to be generated, and which one # generateSector - can be 1 or 2, defaults to 0 and generates both sectors - def getVdiInfo(self, Dict, generateSector=0): + @override + def getVdiInfo(self, Dict, generateSector=0) -> bytes: util.SMlog("Entering VDI info") try: vdi_info = b"" @@ -763,7 +768,8 @@ def getVdiInfo(self, Dict, generateSector=0): (Dict, str(e))) raise - def getSRInfoForSectors(self, sr_info, range): + @override + def getSRInfoForSectors(self, sr_info, range) -> bytes: srinfo = b"" try: diff --git a/drivers/udevSR.py b/drivers/udevSR.py index 4862ef8b..a8442764 100755 --- a/drivers/udevSR.py +++ b/drivers/udevSR.py @@ -18,6 +18,8 @@ # udevSR: represents VDIs which are hotplugged into dom0 via udev e.g. # USB CDROM/disk devices +from sm_typing import override + import SR import VDI import SRCommand @@ -50,16 +52,19 @@ class udevSR(SR.SR): """udev-driven storage repository""" - def handles(type): + @override + @staticmethod + def handles(type) -> bool: if type == TYPE: return True return False - handles = staticmethod(handles) - def content_type(self, sr_uuid): + @override + def content_type(self, sr_uuid) -> str: return super(udevSR, self).content_type(sr_uuid) - def vdi(self, uuid): + @override + def vdi(self, uuid) -> VDI.VDI: util.SMlog("params = %s" % (self.srcmd.params.keys())) if 'vdi_location' in self.srcmd.params: @@ -74,7 +79,8 @@ def get_vdi_location(self, uuid): vdi_ref = vdi.get_by_uuid(uuid) return vdi.get_location(vdi_ref) - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: # First of all, check we've got the correct keys in dconf if 'location' not in self.dconf: raise xs_errors.XenError('ConfigLocationMissing') @@ -82,7 +88,8 @@ def load(self, sr_uuid): # Cache the sm_config self.sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref) - def update(self, sr_uuid): + @override + def update(self, sr_uuid) -> None: # Return as much information as we have sr_root = self.dconf['location'] @@ -102,22 +109,27 @@ def update(self, sr_uuid): self._db_update() - def scan(self, sr_uuid): + @override + def scan(self, sr_uuid) -> None: self.update(sr_uuid) # base class scan does all the work: - return super(udevSR, self).scan(sr_uuid) + super(udevSR, self).scan(sr_uuid) - def create(self, sr_uuid, size): + @override + def create(self, sr_uuid, size) -> None: pass - def delete(self, sr_uuid): + @override + def delete(self, sr_uuid) -> None: pass - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: pass - def detach(self, sr_uuid): + @override + def detach(self, sr_uuid) -> None: pass @@ -134,7 +146,8 @@ def __init__(self, sr, location): self.location = location VDI.VDI.__init__(self, sr, None) - def load(self, location): + @override + def load(self, location) -> None: self.path = self.location self.size = 0 self.utilisation = 0 @@ -149,7 +162,7 @@ def load(self, location): self.sm_config['hotplugged_at'] = iso8601 self.path = os.path.realpath(self.path) - + dev = os.path.basename(self.path) info = sysdevice.stat(dev) if "size" in info.keys(): @@ -176,7 +189,8 @@ def load(self, location): except OSError as e: self.deleted = True - def introduce(self, sr_uuid, vdi_uuid): + @override + def introduce(self, sr_uuid, vdi_uuid) -> str: self.uuid = vdi_uuid self.location = self.sr.srcmd.params['vdi_location'] self._db_introduce() @@ -184,7 +198,8 @@ def introduce(self, sr_uuid, vdi_uuid): self.sr.update(sr_uuid) return super(udevVDI, self).get_params() - def update(self, sr_uuid, vdi_location): + @override + def update(self, sr_uuid, vdi_location) -> None: self.load(vdi_location) # _db_update requires self.uuid to be set self.uuid = self.sr.srcmd.params['vdi_uuid'] @@ -198,13 +213,15 @@ def update(self, sr_uuid, vdi_location): #self.sr.session.xenapi.VDI.set_name_label(vdi, self.label) #self.sr.session.xenapi.VDI.set_name_description(vdi, self.description) - def attach(self, sr_uuid, vdi_uuid): + @override + def attach(self, sr_uuid, vdi_uuid) -> str: if self.deleted: raise xs_errors.XenError('VDIUnavailable') return super(udevVDI, self).attach(sr_uuid, vdi_uuid) - def detach(self, sr_uuid, vdi_uuid): + @override + def detach(self, sr_uuid, vdi_uuid) -> None: pass if __name__ == '__main__': diff --git a/misc/fairlock/fairlock.py b/misc/fairlock/fairlock.py index 391fbea3..1a3352e4 100644 --- a/misc/fairlock/fairlock.py +++ b/misc/fairlock/fairlock.py @@ -1,4 +1,4 @@ -from sm_typing import Any, Callable, Dict, Optional +from sm_typing import Any, Callable, Dict, Optional, override import os import socket @@ -15,10 +15,11 @@ class SingletonWithArgs(type): def __init__(cls, name, bases, dct): cls._init[cls] = dct.get('__init__', None) - def __call__(cls, *args, **kwargs): + @override + def __call__(cls, *args, **kwargs) -> Any: init = cls._init[cls] if init is not None: - key = (cls, frozenset( + key: Any = (cls, frozenset( inspect.getcallargs(init, None, *args, **kwargs).items())) else: key = cls diff --git a/sm_typing/__init__.py b/sm_typing/__init__.py index c515056c..2042dea6 100644 --- a/sm_typing/__init__.py +++ b/sm_typing/__init__.py @@ -1,2 +1,14 @@ import typing from typing import * + +if not hasattr(typing, 'override'): + def override(method): # type: ignore + try: + # Set internal attr `__override__` like described in PEP 698. + method.__override__ = True + except (AttributeError, TypeError): + pass + return method + +if not hasattr(typing, 'Never'): + Never = None # type: ignore diff --git a/stubs/XenAPIPlugin.pyi b/stubs/XenAPIPlugin.pyi index ca5f52ca..72bad09f 100644 --- a/stubs/XenAPIPlugin.pyi +++ b/stubs/XenAPIPlugin.pyi @@ -1,5 +1,5 @@ class Failure(Exception): def __init__(self, code, params) -> None: ... - def __str__(self) -> str: ... + def __str__(self) -> str: ... # type: ignore def dispatch(fn_table) -> None: ... diff --git a/tests/lvmlib.py b/tests/lvmlib.py index c57b32e0..6ec8d2c7 100644 --- a/tests/lvmlib.py +++ b/tests/lvmlib.py @@ -1,3 +1,5 @@ +from sm_typing import Never, override + import argparse import sys @@ -7,10 +9,12 @@ class TestArgParse(argparse.ArgumentParser): to stderr during the tests """ - def exit(self, status=0, msg=None): + @override + def exit(self, status=0, msg=None) -> Never: sys.exit(status) - def error(self, msg): + @override + def error(self, msg) -> Never: """error(msg : string)""" self.exit(2, "%s: error: %s\n" % (self.prog, msg)) diff --git a/tests/shared_iscsi_test_base.py b/tests/shared_iscsi_test_base.py index a6555547..b224d30d 100644 --- a/tests/shared_iscsi_test_base.py +++ b/tests/shared_iscsi_test_base.py @@ -1,3 +1,5 @@ +from sm_typing import Dict, List, Tuple, override + import unittest from unittest import mock @@ -6,8 +8,11 @@ class ISCSITestCase(unittest.TestCase): + # Declared in subclasses. + TEST_CLASS: str - def setUp(self): + @override + def setUp(self) -> None: iscsilib_patcher = mock.patch(f'{self.TEST_CLASS}.iscsilib', autospec=True) self.mock_iscsilib = iscsilib_patcher.start() @@ -15,8 +20,8 @@ def setUp(self): self.mock_iscsilib._checkTGT.side_effect = self._checkTGT self.mock_iscsilib.login.side_effect = self.iscsi_login self.mock_iscsilib.parse_IP_port = iscsilib.parse_IP_port - self.discovery_data = {} - self.sessions = [] + self.discovery_data: Dict[str, Tuple[str, int, str]] = {} + self.sessions: List[str] = [] sleep_patcher = mock.patch(f'{self.TEST_CLASS}.time.sleep', autospec=True) diff --git a/tests/test_BaseISCSI.py b/tests/test_BaseISCSI.py index cc489a46..7c4ed83f 100644 --- a/tests/test_BaseISCSI.py +++ b/tests/test_BaseISCSI.py @@ -2,6 +2,8 @@ Unit tests for the Base ISCSI SR """ +from sm_typing import override + from unittest import mock from uuid import uuid4 @@ -15,7 +17,8 @@ class TestBaseISCSI(ISCSITestCase): TEST_CLASS = 'BaseISCSI' - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) util_patcher = mock.patch('BaseISCSI.util', autospec=True) diff --git a/tests/test_FileSR.py b/tests/test_FileSR.py index a82d0e5b..109f6d30 100644 --- a/tests/test_FileSR.py +++ b/tests/test_FileSR.py @@ -1,3 +1,5 @@ +from sm_typing import Any, Optional, Set, override + import errno import os import stat @@ -18,7 +20,8 @@ class FakeFileVDI(FileSR.FileVDI): - def load(self, uuid): + @override + def load(self, uuid) -> None: self.vdi_type = vhdutil.VDI_TYPE_VHD self.hidden = False self.path = os.path.join(self.sr.path, '%s.%s' % ( @@ -27,7 +30,8 @@ def load(self, uuid): class TestFileVDI(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: startlog_patcher = mock.patch('FileSR.util.start_log_entry', autospec=True) self.mock_startlog = startlog_patcher.start() @@ -50,7 +54,7 @@ def setUp(self): fist_patcher = mock.patch('FileSR.util.FistPoint.is_active', autospec=True) self.mock_fist = fist_patcher.start() - self.active_fists = set() + self.active_fists: Set[Any] = set() def active_fists(): return self.active_fists @@ -419,15 +423,19 @@ class FakeSharedFileSR(FileSR.SharedFileSR): """ Test SR class for SharedFileSR """ - def load(self, sr_uuid): + + @override + def load(self, sr_uuid) -> None: self.path = os.path.join(SR.MOUNT_BASE, sr_uuid) - self.lock = None + self.lock = None # type: ignore - def attach(self, sr_uuid): + @override + def attach(self, sr_uuid) -> None: self._check_writable() self._check_hardlinks() - def _read_hardlink_conf(self): + @override + def _read_hardlink_conf(self) -> Optional[bool]: return None class TestShareFileSR(unittest.TestCase): @@ -437,7 +445,8 @@ class TestShareFileSR(unittest.TestCase): TEST_SR_REF = "test_sr_ref" ERROR_524 = "Unknown error 524" - def setUp(self): + @override + def setUp(self) -> None: util_patcher = mock.patch('FileSR.util', autospec=True) self.mock_util = util_patcher.start() @@ -561,7 +570,8 @@ def test_scan_load_vdis_scan_list_differ(self): self.assertEqual(1, len(test_sr.vdis)) class TestFileSR(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: pread_patcher = mock.patch('FileSR.util.pread') self.mock_pread = pread_patcher.start() diff --git a/tests/test_ISCSISR.py b/tests/test_ISCSISR.py index e71ac268..9af3d4bc 100644 --- a/tests/test_ISCSISR.py +++ b/tests/test_ISCSISR.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest import BaseISCSI import unittest.mock as mock @@ -8,17 +10,20 @@ class TestBase(unittest.TestCase): """ Provides errorcodes.xml, so exceptions are sensible """ - def setUp(self): + @override + def setUp(self) -> None: self._xmldefs = xs_errors.XML_DEFS xs_errors.XML_DEFS = os.path.join( os.path.dirname(__file__), 'XE_SR_ERRORCODES.xml') - def tearDown(self): + @override + def tearDown(self) -> None: xs_errors.XML_DEFS = self._xmldefs class NonLoadingISCSISR(BaseISCSI.BaseISCSISR): - def load(self, sr_uuid): + @override + def load(self, sr_uuid) -> None: pass @@ -82,10 +87,12 @@ def __init__(self, node1, node2): extra_adapter: None } - def _synchroniseAddrList(self, *args, **kwargs): + @override + def _synchroniseAddrList(self, *args, **kwargs) -> None: pass - def _init_adapters(self): + @override + def _init_adapters(self) -> None: pass @@ -115,8 +122,8 @@ def test_vdi_type_modified_by_force_tapdisk(self): class TestMultiLUNISCSISR(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: self.node1 = { 'ip': '127.0.0.1', 'port': 3260, diff --git a/tests/test_LVHDSR.py b/tests/test_LVHDSR.py index 94a390d3..5dc5066c 100644 --- a/tests/test_LVHDSR.py +++ b/tests/test_LVHDSR.py @@ -1,3 +1,5 @@ +from sm_typing import override + import os import unittest import unittest.mock as mock @@ -34,11 +36,12 @@ def remove_stubs(self): class TestLVHDSR(unittest.TestCase, Stubs): - - def setUp(self): + @override + def setUp(self) -> None: self.init_stubs() - def tearDown(self): + @override + def tearDown(self) -> None: self.remove_stubs() def create_LVHDSR(self, master=False, command='foo', sr_uuid=None): @@ -249,8 +252,8 @@ def cmd(args): class TestLVHDVDI(unittest.TestCase, Stubs): - - def setUp(self): + @override + def setUp(self) -> None: self.init_stubs() lvhdutil_patcher = mock.patch('LVHDSR.lvhdutil', autospec=True) @@ -287,7 +290,8 @@ def setUp(self): self.addCleanup(mock.patch.stopall) - def tearDown(self): + @override + def tearDown(self) -> None: self.remove_stubs() def create_LVHDSR(self): diff --git a/tests/test_LVHDoHBASR.py b/tests/test_LVHDoHBASR.py index 3bc7196b..da88e461 100644 --- a/tests/test_LVHDoHBASR.py +++ b/tests/test_LVHDoHBASR.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest.mock as mock import LVHDoHBASR import unittest @@ -72,8 +74,8 @@ def test_generate_config_bad_path_assert(self, class TestLVHDoHBASR(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: self.host_ref = str(uuid4()) self.session_ref = str(uuid4()) self.sr_ref = str(uuid4()) diff --git a/tests/test_LVHDoISCSISR.py b/tests/test_LVHDoISCSISR.py index 3b5e1c42..ec71f20d 100644 --- a/tests/test_LVHDoISCSISR.py +++ b/tests/test_LVHDoISCSISR.py @@ -1,3 +1,5 @@ +from sm_typing import Set, override + import os import unittest import unittest.mock as mock @@ -62,7 +64,8 @@ class TestLVHDoISCSISR_load(unittest.TestCase): Tests for 'LVHDoISCSISR.load()' """ - def setUp(self): + @override + def setUp(self) -> None: patchers = [ mock.patch( 'BaseISCSI.BaseISCSISR', @@ -124,14 +127,15 @@ class TestLVHDoISCSISR(ISCSITestCase): TEST_CLASS = 'LVHDoISCSISR' - def setUp(self): + @override + def setUp(self) -> None: util_patcher = mock.patch('LVHDoISCSISR.util', autospec=True) self.mock_util = util_patcher.start() # self.mock_util.SMlog.side_effect = print self.mock_util.isVDICommand = util.isVDICommand self.mock_util.sessions_less_than_targets = util.sessions_less_than_targets - self.base_srs = set() + self.base_srs: Set[BaseISCSISR] = set() baseiscsi_patcher = mock.patch('LVHDoISCSISR.BaseISCSI.BaseISCSISR', autospec=True) patched_baseiscsi = baseiscsi_patcher.start() diff --git a/tests/test_SMBSR.py b/tests/test_SMBSR.py index 4cfd2733..42ce035e 100644 --- a/tests/test_SMBSR.py +++ b/tests/test_SMBSR.py @@ -1,3 +1,5 @@ +from sm_typing import Dict, override + import unittest import unittest.mock as mock import uuid @@ -25,14 +27,14 @@ def __init__(self, srcmd, none): class Test_SMBSR(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) pread_patcher = mock.patch('SMBSR.util.pread', autospec=True) self.mock_pread = pread_patcher.start() self.mock_pread.side_effect = self.pread - self.pread_results = {} + self.pread_results: Dict[str, str] = {} listdir_patcher = mock.patch('SMBSR.util.listdir', autospec=True) self.mock_list_dir = listdir_patcher.start() diff --git a/tests/test_SR.py b/tests/test_SR.py index b139f4c5..4e0de361 100644 --- a/tests/test_SR.py +++ b/tests/test_SR.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest import unittest.mock as mock import SR @@ -18,10 +20,12 @@ def __init__(self, device=None): def verify(self): pass - def setUp(self): + @override + def setUp(self) -> None: pass - def tearDown(self): + @override + def tearDown(self) -> None: pass def create_SR(self, cmd, dconf, cmd_params=None): diff --git a/tests/test_blktap2.py b/tests/test_blktap2.py index d0e36daf..81311fb9 100644 --- a/tests/test_blktap2.py +++ b/tests/test_blktap2.py @@ -1,3 +1,5 @@ +from sm_typing import override + import errno import json from io import StringIO @@ -23,7 +25,8 @@ class TestTapdisk(unittest.TestCase): # hence no usual decorator mocks and the monkey patching. # https://bugs.python.org/issue23078 # - def setUp(self): + @override + def setUp(self) -> None: subprocess_patcher = mock.patch("blktap2.subprocess") self.mock_subprocess = subprocess_patcher.start() @@ -110,7 +113,8 @@ def no_medium(pid, minor, type, path, options): class TestVDI(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) lock_patcher = mock.patch('blktap2.Lock', autospec=True) @@ -337,8 +341,8 @@ def test_activate_relink_while_tagging( class TestTapCtl(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: subprocess_patcher = mock.patch("blktap2.subprocess") self.mock_subprocess = subprocess_patcher.start() diff --git a/tests/test_cbt.py b/tests/test_cbt.py index 9093f216..18bdc8a0 100644 --- a/tests/test_cbt.py +++ b/tests/test_cbt.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest.mock as mock import SR import testlib @@ -15,7 +17,8 @@ class TestVDI(VDI.VDI): - def load(self, vdi_uuid): + @override + def load(self, vdi_uuid) -> None: self.vdi_type = vhdutil.VDI_TYPE_VHD self._state_mock = mock.Mock() self.path = "/mock/sr_path/" + str(vdi_uuid) @@ -25,46 +28,56 @@ def load(self, vdi_uuid): def state_mock(self): return self._state_mock - def _get_blocktracking_status(self, uuid=None): + @override + def _get_blocktracking_status(self, uuid=None) -> bool: return self.block_tracking_state - def _ensure_cbt_space(self): + @override + def _ensure_cbt_space(self) -> None: super(TestVDI, self)._ensure_cbt_space() self.state_mock._ensure_cbt_space() - def _get_cbt_logpath(self, uuid): + @override + def _get_cbt_logpath(self, uuid) -> str: super(TestVDI, self)._get_cbt_logpath(uuid) self.state_mock._get_cbt_logpath(uuid) return "/mock/sr_path/{0}.log".format(uuid) - def _create_cbt_log(self): + @override + def _create_cbt_log(self) -> str: logpath = super(TestVDI, self)._create_cbt_log() self.state_mock._create_cbt_log() self.block_tracking_state = True return logpath - def _delete_cbt_log(self): + @override + def _delete_cbt_log(self) -> None: self.state_mock._delete_cbt_log() self.block_tracking_state = False - def _rename(self, from_path, to_path): + @override + def _rename(self, from_path, to_path) -> None: self.state_mock._rename(from_path, to_path) + @override def _do_snapshot(self, sr_uuid, vdi_uuid, snapType, - cloneOp=False, secondary=None, cbtlog=None): - self.state_mock._do_snapshot(sr_uuid, vdi_uuid, snapType, cloneOp, - secondary, cbtlog) + cloneOp=False, secondary=None, cbtlog=None) -> str: + return self.state_mock._do_snapshot( + sr_uuid, vdi_uuid, snapType, cloneOp, secondary, cbtlog + ) - def _activate_cbt_log(self, logname): - self.state_mock._activate_cbt_log(logname) + @override + def _activate_cbt_log(self, logname) -> bool: + return self.state_mock._activate_cbt_log(logname) - def _deactivate_cbt_log(self, logname): + @override + def _deactivate_cbt_log(self, logname) -> None: self.state_mock._deactivate_cbt_log(logname) class TestCBT(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: self.sr = mock.MagicMock() self.vdi_uuid = uuid.uuid4() self.sr_uuid = uuid.uuid4() diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index bd789515..d8c8055b 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -1,4 +1,4 @@ -from sm_typing import List +from sm_typing import Dict, List, override import errno import unittest @@ -54,7 +54,8 @@ def create_cleanup_sr(xapi, uuid=None): class TestSR(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: time_sleep_patcher = mock.patch('cleanup.time.sleep') self.mock_time_sleep = time_sleep_patcher.start() @@ -1788,14 +1789,14 @@ def test_not_plugged_retry(self): class TestLockGCActive(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) self.lock_patcher = mock.patch('cleanup.lock.Lock') patched_lock = self.lock_patcher.start() patched_lock.side_effect = self.create_lock - self.locks = {} + self.locks: Dict[str, TestLockGCActive.DummyLock] = {} self.sr_uuid = str(uuid4()) diff --git a/tests/test_fairlock.py b/tests/test_fairlock.py index d0e1f61b..e003cc37 100644 --- a/tests/test_fairlock.py +++ b/tests/test_fairlock.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest import unittest.mock as mock @@ -5,7 +7,8 @@ from fairlock import Fairlock, FairlockServiceTimeout, FairlockDeadlock class TestFairlock(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: sock_patcher = mock.patch('fairlock.socket', autospec=True) self.mock_socket = sock_patcher.start() os_patcher = mock.patch('fairlock.os', autospec=True) diff --git a/tests/test_fjournaler.py b/tests/test_fjournaler.py index 115aaee9..67835875 100644 --- a/tests/test_fjournaler.py +++ b/tests/test_fjournaler.py @@ -1,3 +1,5 @@ +from sm_typing import Any, Dict, override + from io import BytesIO as StringIO import os import unittest @@ -32,9 +34,9 @@ def readline(self): class TestFjournaler(unittest.TestCase): - - def setUp(self): - self.files = {} + @override + def setUp(self) -> None: + self.files: Dict[str, Any] = {} self.open_handlers = {TEST_DIR_PATH: self.__fake_open} self.subject = fjournaler.Journaler(TEST_DIR_PATH) diff --git a/tests/test_keymanagerutil.py b/tests/test_keymanagerutil.py index d97bb172..70edbe5d 100644 --- a/tests/test_keymanagerutil.py +++ b/tests/test_keymanagerutil.py @@ -1,6 +1,9 @@ """ Test the "fake" keymanager for testing VHD encryption """ + +from sm_typing import Dict, override + import base64 import copy import io @@ -15,8 +18,8 @@ class TestKeymanagerutil(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) log_patcher = mock.patch('plugins.keymanagerutil.util.SMlog', autospec=True) @@ -27,7 +30,7 @@ def setUp(self): self.mock_open = open_patcher.start() self.mock_open.side_effect = self.open - self.files = {} + self.files: Dict[str, io.StringIO] = {} isfile_patcher = mock.patch( 'plugins.keymanagerutil.os.path.isfile', autospec=True) diff --git a/tests/test_lock.py b/tests/test_lock.py index f90d3bb6..bee447a0 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -1,3 +1,5 @@ +from sm_typing import override + import fcntl import unittest import unittest.mock as mock @@ -12,8 +14,8 @@ class TestLock(unittest.TestCase): - - def tearDown(self): + @override + def tearDown(self) -> None: lock.Lock.INSTANCES = {} lock.Lock.BASE_INSTANCES = {} @@ -197,7 +199,8 @@ def create_lock_class_that_fails_to_create_file(number_of_failures): class LockThatFailsToCreateFile(lock.LockImplementation): _failures = number_of_failures - def _open_lockfile(self): + @override + def _open_lockfile(self) -> None: if self._failures > 0: error = IOError('No such file') error.errno = errno.ENOENT diff --git a/tests/test_lock_queue.py b/tests/test_lock_queue.py index edd733dc..e9622cff 100644 --- a/tests/test_lock_queue.py +++ b/tests/test_lock_queue.py @@ -1,3 +1,5 @@ +from sm_typing import override + import builtins import copy import os @@ -22,7 +24,8 @@ def mock_pickle_load_fn(*args): class Test_LockQueue(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: # Re-initialize queue to empty for each test global saved_queue saved_queue = [] diff --git a/tests/test_lvutil.py b/tests/test_lvutil.py index 2df8300b..439c1fbe 100644 --- a/tests/test_lvutil.py +++ b/tests/test_lvutil.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest.mock as mock import os import syslog @@ -25,7 +27,8 @@ def decorated(self, context, *args, **kwargs): class TestCreate(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: lock_patcher = mock.patch('lvutil.Fairlock', autospec=True) self.addCleanup(lock_patcher.stop) self.mock_lock = lock_patcher.start() @@ -98,7 +101,8 @@ def test_create_percentage_has_precedence_over_size(self, mock_pread): class TestRemove(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: lock_patcher = mock.patch('lvutil.Fairlock', autospec=True) self.addCleanup(lock_patcher.stop) self.mock_lock = lock_patcher.start() @@ -123,8 +127,8 @@ def test_remove_additional_config_param(self, mock_pread, _bugCleanup): class TestDeactivate(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: lock_patcher = mock.patch('lvutil.Fairlock', autospec=True) pathexists_patcher = mock.patch('lvutil.util.pathexists', autospec=True) lexists_patcher = mock.patch('lvutil.os.path.lexists', autospec=True) @@ -208,7 +212,8 @@ def test_deactivate_noref_withbugcleanup_retry_fail( class TestActivate(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) lock_patcher = mock.patch('lvutil.Fairlock', autospec=True) diff --git a/tests/test_mpath_dmp.py b/tests/test_mpath_dmp.py index 134b40e0..312b55da 100644 --- a/tests/test_mpath_dmp.py +++ b/tests/test_mpath_dmp.py @@ -1,6 +1,9 @@ """ Unit tests for mpath dmp """ + +from sm_typing import override + import errno import os import unittest @@ -22,7 +25,8 @@ class TestMpathDmp(unittest.TestCase): Unit tests for mpath dmp """ - def setUp(self): + @override + def setUp(self) -> None: time_patcher = mock.patch('mpath_dmp.time', autospec=True) self.mock_time = time_patcher.start() diff --git a/tests/test_on_slave.py b/tests/test_on_slave.py index 4b59f632..b55e24aa 100644 --- a/tests/test_on_slave.py +++ b/tests/test_on_slave.py @@ -1,3 +1,5 @@ +from sm_typing import override + import errno import unittest import unittest.mock as mock @@ -27,7 +29,8 @@ def fake_import(self, *args, **kwargs): print('Asked to import {}'.format(args[0])) return self.mocks.get(args[0], self.real_import(*args, **kwargs)) - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) self.mocks = {x: mock.MagicMock() for x in self.MOCK_IMPORTS} @@ -132,7 +135,8 @@ class Test_on_slave_refresh_lun(unittest.TestCase): Tests for refresh_lun_size_by_SCSIid """ - def setUp(self): + @override + def setUp(self) -> None: self.mock_session = mock.MagicMock() @mock.patch('on_slave.scsiutil') @@ -164,7 +168,8 @@ class Test_on_slave_multi(unittest.TestCase): TMP_RENAME_PREFIX = "TEST_OLD_" - def setUp(self): + @override + def setUp(self) -> None: self.session = mock.MagicMock() lvmcache_patcher = mock.patch('on_slave.LVMCache', autospec=True) diff --git a/tests/test_sr_health_check.py b/tests/test_sr_health_check.py index c104e962..22e78378 100644 --- a/tests/test_sr_health_check.py +++ b/tests/test_sr_health_check.py @@ -1,3 +1,5 @@ +from sm_typing import override + import unittest import unittest.mock as mock @@ -10,8 +12,8 @@ class TestSrHealthCheck(unittest.TestCase): - - def setUp(self): + @override + def setUp(self) -> None: util_patcher = mock.patch('sr_health_check.util') self.mock_util = util_patcher.start() self.mock_session = mock.MagicMock() diff --git a/tests/test_srmetadata.py b/tests/test_srmetadata.py index 720f12ff..fbce6244 100644 --- a/tests/test_srmetadata.py +++ b/tests/test_srmetadata.py @@ -1,3 +1,5 @@ +from sm_typing import Generator, override + import io import random import string @@ -453,14 +455,17 @@ def __init__(self): super().__init__() self._metadata_file_content = b'\x00' * 4 * 1024 * 1024 - def start(self): + @override + def start(self) -> None: super().start() self.patch("util.gen_uuid", new=genuuid) - def generate_device_paths(self): + @override + def generate_device_paths(self) -> Generator[str, None, None]: yield self.METADATA_PATH - def fake_open(self, fname, mode='r'): + @override + def fake_open(self, fname, mode='r') -> io.TextIOBase: if fname != self.METADATA_PATH: # pragma: no cover return super().fake_open(fname, mode) else: diff --git a/tests/test_storage_init.py b/tests/test_storage_init.py index d91d9089..38e46570 100644 --- a/tests/test_storage_init.py +++ b/tests/test_storage_init.py @@ -1,3 +1,5 @@ +from sm_typing import DefaultDict, Dict, List, override + import json import os import re @@ -31,7 +33,8 @@ class TestStorageInit(unittest.TestCase): storage after installation. """ - def setUp(self): + @override + def setUp(self) -> None: self.test_dir = tempfile.TemporaryDirectory() # There are tweaks we need to make the to storage-init: @@ -103,11 +106,12 @@ def setUp(self): self.copy_command("touch") self.script_exited = False - self.created_srs = defaultdict(list) - self.misc_xe_calls = [] - self.unanticipated_xe_calls = [] + self.created_srs: DefaultDict[str, List[Dict[str, str]]] = defaultdict(list) + self.misc_xe_calls: List[List[str]] = [] + self.unanticipated_xe_calls: List[List[str]] = [] - def tearDown(self): + @override + def tearDown(self) -> None: self.socket.close() self.test_dir.cleanup() diff --git a/tests/test_util.py b/tests/test_util.py index 965eec15..26912f1f 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,3 +1,5 @@ +from sm_typing import Any, Dict, List, Set, override + import copy import errno import io @@ -43,7 +45,8 @@ class TestUtil(unittest.TestCase): Tests for the util module methods """ - def setUp(self): + @override + def setUp(self) -> None: # OS Patchers statvfs_patcher = mock.patch("util.os.statvfs", autospec=True) self.mock_statvfs = statvfs_patcher.start() @@ -53,7 +56,7 @@ def setUp(self): self.mock_mkdir = mkdir_patcher.start() unlink_patcher = mock.patch('util.os.unlink', autospec=True) self.mock_unlink = unlink_patcher.start() - self.dir_contents = {} + self.dir_contents: Dict[str, List[str]] = {} listdir_patcher = mock.patch('util.os.listdir', autospec=True) self.mock_listdir = listdir_patcher.start() self.mock_listdir.side_effect = self.list_dir @@ -77,12 +80,12 @@ def setUp(self): self.mock_session = mock.MagicMock() self.mock_xenapi.xapi_local.return_value = self.mock_session - self.processes = {} + self.processes: Dict[str, Any] = {} popen_patcher = mock.patch('util.subprocess.Popen', autospec=True) self.mock_popen = popen_patcher.start() self.mock_popen.side_effect = self.popen - self.mock_files = {} + self.mock_files: Dict[str, Any] = {} self.addCleanup(mock.patch.stopall) @@ -714,7 +717,8 @@ def test_unictrunc(self): class TestFistPoints(unittest.TestCase): - def setUp(self): + @override + def setUp(self) -> None: self.addCleanup(mock.patch.stopall) sleep_patcher = mock.patch('util.time.sleep', autospec=True) self.mock_sleep = sleep_patcher.start() @@ -725,7 +729,7 @@ def setUp(self): exists_patcher = mock.patch('util.os.path.exists', autospec=True) self.mock_exists = exists_patcher.start() self.mock_exists.side_effect = self.exists - self.existing_files = set() + self.existing_files: Set[str] = set() xenapi_patcher = mock.patch('util.XenAPI', autospec=True) patched_xenapi = xenapi_patcher.start() diff --git a/tests/testlib.py b/tests/testlib.py index 167bb172..066188f4 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,3 +1,5 @@ +from sm_typing import Any, Generator, override + import re import unittest.mock as mock import os @@ -49,12 +51,13 @@ def add_disk(self): def add_parameter(self, host_class, values): self.parameters.append((host_class, values)) - def adapter_device_paths(self, host_id): + def adapter_device_paths(self, host_id) -> Generator[str, None, None]: yield '/sys/class/scsi_host/host%s' % host_id class AdapterWithNonBlockDevice(SCSIAdapter): - def adapter_device_paths(self, host_id): + @override + def adapter_device_paths(self, host_id) -> Generator[str, None, None]: for adapter_device_path in super(AdapterWithNonBlockDevice, self).adapter_device_paths(host_id): yield adapter_device_path @@ -115,7 +118,7 @@ def patch(self, *args, **kwargs): self.patchers.append(patcher) patcher.start() - def start(self): + def start(self) -> None: self.patch('builtins.open', new=self.fake_open) self.patch('fcntl.fcntl', new=self.fake_fcntl) self.patch('os.path.exists', new=self.fake_exists) @@ -190,7 +193,7 @@ def fake_uname(self): 'x86_64' ) - def fake_open(self, fname, mode='r'): + def fake_open(self, fname, mode='r') -> Any: if fname == '/etc/xensource-inventory': return io.StringIO(self.generate_inventory_contents()) @@ -264,7 +267,7 @@ def generate_path_content(self): for path, value in self._path_content.items(): yield (path, value) - def generate_device_paths(self): + def generate_device_paths(self) -> Generator[str, None, None]: actual_disk_letter = 'a' for host_id, adapter in enumerate(self.scsi_adapters): for adapter_device_path in adapter.adapter_device_paths(host_id): From 59ed5e3e899df39cc993aefee2edc6e5e80bb140 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Thu, 7 Nov 2024 15:50:22 +0100 Subject: [PATCH 28/29] Makefile fix: don't execute precheck during installation Due to mypy modifications, we can't build the sm RPM in Koji without a recent pylint version. So the precheck target is only executed in a github workflow now. Signed-off-by: Ronan Abhamon --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 84cbb5a7..fbe2dec5 100755 --- a/Makefile +++ b/Makefile @@ -127,7 +127,7 @@ precheck: build echo "Precheck succeeded with no outstanding issues found." .PHONY: install -install: precheck +install: build mkdir -p $(SM_STAGING) $(call mkdir_clean,$(SM_STAGING)) mkdir -p $(SM_STAGING)$(SM_DEST) From 85b9b23330607ecc90ff0d7301ee07565ef301cb Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Wed, 8 Jan 2025 11:12:13 +0100 Subject: [PATCH 29/29] Fix LVHDSR.load: test other_conf to prevent mypy error Avoid: ``` drivers/LVHDSR.py:195: error: Item "None" of "Any | None" has no attribute "get" [union-attr] drivers/LVHDSR.py:196: error: Value of type "Any | None" is not indexable [index] ``` Signed-off-by: Ronan Abhamon --- drivers/LVHDSR.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/LVHDSR.py b/drivers/LVHDSR.py index 3e9afcde..c5ed1907 100755 --- a/drivers/LVHDSR.py +++ b/drivers/LVHDSR.py @@ -167,8 +167,7 @@ def load(self, sr_uuid) -> None: self.provision = self.PROVISIONING_DEFAULT self.other_conf = None - has_sr_ref = self.srcmd.params.get("sr_ref") - if has_sr_ref: + if self.srcmd.params.get("sr_ref"): self.other_conf = self.session.xenapi.SR.get_other_config(self.sr_ref) self.lvm_conf = None @@ -182,7 +181,7 @@ def load(self, sr_uuid) -> None: opterr='Failed to initialise the LVMCache') self.lvActivator = LVActivator(self.uuid, self.lvmCache) self.journaler = Journaler(self.lvmCache) - if not has_sr_ref: + if not self.other_conf: return # must be a probe call # Test for thick vs thin provisioning conf parameter if 'allocation' in self.dconf: