Merge branch 'stackhpc/zed' into upstream/zed-2024-09-02

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @stackhpc/openstack

.github/workflows/tag-and-release.yml

@@ -0,0 +1,12 @@
+---
+name: Tag & Release
+'on':
+  push:
+    branches:
+      - stackhpc/zed
+permissions:
+  actions: read
+  contents: write
+jobs:
+  tag-and-release:
+    uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main

.github/workflows/tox.yml

@@ -0,0 +1,7 @@
+---
+name: Tox Continuous Integration
+'on':
+  pull_request:
+jobs:
+  tox:
+    uses: stackhpc/.github/.github/workflows/tox.yml@main

glance/async_/flows/plugins/image_conversion.py

@@ -25,6 +25,7 @@
 from taskflow import task
 
 from glance.async_ import utils
+from glance.common import format_inspector
 from glance.i18n import _, _LI
 
 LOG = logging.getLogger(__name__)
@@ -87,8 +88,40 @@ def _execute(self, action, file_path, **kwargs):
                                              'target': target_format}
         self.dest_path = dest_path
 
+        source_format = action.image_disk_format
+        inspector_cls = format_inspector.get_inspector(source_format)
+        if not inspector_cls:
+            # We cannot convert from disk_format types that qemu-img doesn't
+            # support (like iso, ploop, etc). The ones it supports overlaps
+            # with the ones we have inspectors for, so reject conversion for
+            # any format we don't have an inspector for.
+            raise RuntimeError(
+                'Unable to convert from format %s' % source_format)
+
+        # Use our own cautious inspector module (if we have one for this
+        # format) to make sure a file is the format the submitter claimed
+        # it is and that it passes some basic safety checks _before_ we run
+        # qemu-img on it.
+        # See https://bugs.launchpad.net/nova/+bug/2059809 for details.
+        try:
+            inspector = inspector_cls.from_file(src_path)
+            if not inspector.safety_check():
+                LOG.error('Image failed %s safety check; aborting conversion',
+                          source_format)
+                raise RuntimeError('Image has disallowed configuration')
+        except RuntimeError:
+            raise
+        except format_inspector.ImageFormatError as e:
+            LOG.error('Image claimed to be %s format failed format '
+                      'inspection: %s', source_format, e)
+            raise RuntimeError('Image format detection failed')
+        except Exception as e:
+            LOG.exception('Unknown error inspecting image format: %s', e)
+            raise RuntimeError('Unable to inspect image')
+
         try:
             stdout, stderr = putils.trycmd("qemu-img", "info",
+                                           "-f", source_format,
                                            "--output=json",
                                            src_path,
                                            prlimit=utils.QEMU_IMG_PROC_LIMITS,
@@ -105,13 +138,10 @@ def _execute(self, action, file_path, **kwargs):
             raise RuntimeError(stderr)
 
         metadata = json.loads(stdout)
-        try:
-            source_format = metadata['format']
-        except KeyError:
-            msg = ("Failed to do introspection as part of image "
-                   "conversion for %(iid)s: Source format not reported")
-            LOG.error(msg, {'iid': self.image_id})
-            raise RuntimeError(msg)
+        if metadata.get('format') != source_format:
+            LOG.error('Image claiming to be %s reported as %s by qemu-img',
+                      source_format, metadata.get('format', 'unknown'))
+            raise RuntimeError('Image metadata disagrees about format')
 
         virtual_size = metadata.get('virtual-size', 0)
         action.set_image_attribute(virtual_size=virtual_size)

glance/common/format_inspector.py

@@ -866,19 +866,52 @@ def close(self):
             self._source.close()
 
 
+ALL_FORMATS = {
+    'raw': FileInspector,
+    'qcow2': QcowInspector,
+    'vhd': VHDInspector,
+    'vhdx': VHDXInspector,
+    'vmdk': VMDKInspector,
+    'vdi': VDIInspector,
+    'qed': QEDInspector,
+}
+
+
 def get_inspector(format_name):
     """Returns a FormatInspector class based on the given name.
 
     :param format_name: The name of the disk_format (raw, qcow2, etc).
     :returns: A FormatInspector or None if unsupported.
     """
-    formats = {
-        'raw': FileInspector,
-        'qcow2': QcowInspector,
-        'vhd': VHDInspector,
-        'vhdx': VHDXInspector,
-        'vmdk': VMDKInspector,
-        'vdi': VDIInspector,
-    }
-
-    return formats.get(format_name)
+
+    return ALL_FORMATS.get(format_name)
+
+
+def detect_file_format(filename):
+    """Attempts to detect the format of a file.
+
+    This runs through a file one time, running all the known inspectors in
+    parallel. It stops reading the file once one of them matches or all of
+    them are sure they don't match.
+
+    Returns the FileInspector that matched, if any. None if 'raw'.
+    """
+    inspectors = {k: v() for k, v in ALL_FORMATS.items()}
+    with open(filename, 'rb') as f:
+        for chunk in chunked_reader(f):
+            for format, inspector in list(inspectors.items()):
+                try:
+                    inspector.eat_chunk(chunk)
+                except ImageFormatError:
+                    # No match, so stop considering this format
+                    inspectors.pop(format)
+                    continue
+                if (inspector.format_match and inspector.complete and
+                        format != 'raw'):
+                    # First complete match (other than raw) wins
+                    return inspector
+            if all(i.complete for i in inspectors.values()):
+                # If all the inspectors are sure they are not a match, avoid
+                # reading to the end of the file to settle on 'raw'.
+                break
+    return inspectors['raw']

glance/tests/unit/async_/flows/plugins/test_image_conversion.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import fixtures
 import json
 import os
 from unittest import mock
@@ -24,6 +25,7 @@
 import glance.async_.flows.api_image_import as import_flow
 import glance.async_.flows.plugins.image_conversion as image_conversion
 from glance.async_ import utils as async_utils
+from glance.common import format_inspector
 from glance.common import utils
 from glance import domain
 from glance import gateway
@@ -90,6 +92,11 @@ def setUp(self):
                                                        self.image_id,
                                                        self.task.task_id)
 
+        self.inspector_mock = mock.MagicMock()
+        self.useFixture(fixtures.MockPatch('glance.common.format_inspector.'
+                                           'get_inspector',
+                                           self.inspector_mock))
+
     @mock.patch.object(os, 'stat')
     @mock.patch.object(os, 'remove')
     def test_image_convert_success(self, mock_os_remove, mock_os_stat):
@@ -104,7 +111,7 @@ def test_image_convert_success(self, mock_os_remove, mock_os_stat):
         image = mock.MagicMock(image_id=self.image_id, virtual_size=None,
                                extra_properties={
                                    'os_glance_import_task': self.task.task_id},
-                               disk_format='qcow2')
+                               disk_format='raw')
         self.img_repo.get.return_value = image
 
         with mock.patch.object(processutils, 'execute') as exc_mock:
@@ -126,7 +133,7 @@ def test_image_convert_success(self, mock_os_remove, mock_os_stat):
         self.assertEqual(456, image.virtual_size)
         self.assertEqual(123, image.size)
 
-    def _setup_image_convert_info_fail(self):
+    def _setup_image_convert_info_fail(self, disk_format='qcow2'):
         image_convert = image_conversion._ConvertImage(self.context,
                                                        self.task.task_id,
                                                        self.task_type,
@@ -136,7 +143,7 @@ def _setup_image_convert_info_fail(self):
         image = mock.MagicMock(image_id=self.image_id, virtual_size=None,
                                extra_properties={
                                    'os_glance_import_task': self.task.task_id},
-                               disk_format='qcow2')
+                               disk_format=disk_format)
         self.img_repo.get.return_value = image
         return image_convert
 
@@ -148,6 +155,7 @@ def test_image_convert_fails_inspection(self):
                               convert.execute, 'file:///test/path.raw')
             exc_mock.assert_called_once_with(
                 'qemu-img', 'info',
+                '-f', 'qcow2',
                 '--output=json',
                 '/test/path.raw',
                 prlimit=async_utils.QEMU_IMG_PROC_LIMITS,
@@ -164,6 +172,7 @@ def test_image_convert_inspection_reports_error(self):
                               convert.execute, 'file:///test/path.raw')
             exc_mock.assert_called_once_with(
                 'qemu-img', 'info',
+                '-f', 'qcow2',
                 '--output=json',
                 '/test/path.raw',
                 prlimit=async_utils.QEMU_IMG_PROC_LIMITS,
@@ -207,7 +216,7 @@ def _test_image_convert_invalid_vmdk(self):
                         'create-type': 'monolithicFlat',
                     }}}
 
-        convert = self._setup_image_convert_info_fail()
+        convert = self._setup_image_convert_info_fail(disk_format='vmdk')
         with mock.patch.object(processutils, 'execute') as exc_mock:
             exc_mock.return_value = json.dumps(data), ''
             convert.execute('file:///test/path.vmdk')
@@ -236,14 +245,15 @@ def test_image_convert_valid_vmdk(self):
                               self._test_image_convert_invalid_vmdk)
 
     def test_image_convert_fails(self):
-        convert = self._setup_image_convert_info_fail()
+        convert = self._setup_image_convert_info_fail(disk_format='raw')
         with mock.patch.object(processutils, 'execute') as exc_mock:
             exc_mock.side_effect = [('{"format":"raw"}', ''),
                                     OSError('convert_fail')]
             self.assertRaises(OSError,
                               convert.execute, 'file:///test/path.raw')
             exc_mock.assert_has_calls(
                 [mock.call('qemu-img', 'info',
+                           '-f', 'raw',
                            '--output=json',
                            '/test/path.raw',
                            prlimit=async_utils.QEMU_IMG_PROC_LIMITS,
@@ -256,14 +266,15 @@ def test_image_convert_fails(self):
         self.img_repo.save.assert_not_called()
 
     def test_image_convert_reports_fail(self):
-        convert = self._setup_image_convert_info_fail()
+        convert = self._setup_image_convert_info_fail(disk_format='raw')
         with mock.patch.object(processutils, 'execute') as exc_mock:
             exc_mock.side_effect = [('{"format":"raw"}', ''),
                                     ('', 'some error')]
             self.assertRaises(RuntimeError,
                               convert.execute, 'file:///test/path.raw')
             exc_mock.assert_has_calls(
                 [mock.call('qemu-img', 'info',
+                           '-f', 'raw',
                            '--output=json',
                            '/test/path.raw',
                            prlimit=async_utils.QEMU_IMG_PROC_LIMITS,
@@ -281,9 +292,10 @@ def test_image_convert_fails_source_format(self):
             exc_mock.return_value = ('{}', '')
             exc = self.assertRaises(RuntimeError,
                                     convert.execute, 'file:///test/path.raw')
-            self.assertIn('Source format not reported', str(exc))
+            self.assertIn('Image metadata disagrees about format', str(exc))
             exc_mock.assert_called_once_with(
                 'qemu-img', 'info',
+                '-f', 'qcow2',
                 '--output=json',
                 '/test/path.raw',
                 prlimit=async_utils.QEMU_IMG_PROC_LIMITS,
@@ -301,6 +313,7 @@ def test_image_convert_same_format_does_nothing(self):
             # Make sure we only called qemu-img for inspection, not conversion
             exc_mock.assert_called_once_with(
                 'qemu-img', 'info',
+                '-f', 'qcow2',
                 '--output=json',
                 '/test/path.qcow',
                 prlimit=async_utils.QEMU_IMG_PROC_LIMITS,

glance/tests/unit/v2/test_tasks_resource.py

@@ -415,8 +415,8 @@ def test_create_with_wrong_import_form(self):
                        "non-local source of image data.")
             else:
                 supported = ['http', ]
-                msg = ("The given uri is not valid. Please specify a "
-                       "valid uri from the following list of supported uri "
+                msg = ("The given URI is not valid. Please specify a "
+                       "valid URI from the following list of supported URI "
                        "%(supported)s") % {'supported': supported}
             self.assertEqual(msg, final_task.message)