Skip to content

Commit

Permalink
Convert images to raw if ceph image backend (#1078)
Browse files Browse the repository at this point in the history 
Convert images to raw if ceph image backend

We are currently uploading qcow2 images, and
Nova is converting them to raw when running
the tests, sometimes timing out the tests.

With this change we are pre-converting the
images and uploading them as raw, so Nova
does not have to convert them.

(cherry picked from commit e0498d6)
(cherry picked from commit e28139b)
(cherry picked from commit fb856f7)
  • Loading branch information
@rodrigogansobarbieri
rodrigogansobarbieri committed Aug 29, 2023
1 parent a4a8cf7 commit bee2d31
Show file tree
Hide file tree
Showing 3 changed files with 198 additions and 3 deletions.
127 changes: 127 additions & 0 deletions unit_tests/utilities/test_zaza_utilities_openstack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,10 +544,99 @@ def test_upload_image_to_glance(self):
expected_status='active',
msg='Image status wait')

def test_is_ceph_image_backend_True(self):
self.patch_object(openstack_utils.juju_utils, "get_full_juju_status",
return_value={
"applications": {
"glance": {
"relations": {
"ceph": [
"ceph-mon"
]
}
}
}
})
self.assertTrue(openstack_utils.is_ceph_image_backend())
self.get_full_juju_status.assert_called_once_with(model_name=None)

def test_is_ceph_image_backend_False(self):
self.patch_object(openstack_utils.juju_utils, "get_full_juju_status",
return_value={
"applications": {
"glance": {
"relations": {
}
}
}
})
self.assertFalse(openstack_utils.is_ceph_image_backend('foo'))
self.get_full_juju_status.assert_called_once_with(model_name='foo')

def test_convert_image_format_to_raw_if_qcow2_qemu_cmd_error(self):
self.patch_object(openstack_utils.subprocess, "check_output")
self.check_output.side_effect = subprocess.CalledProcessError(
returncode=42, cmd='mycmd')
self.assertEqual("/tmp/original_path",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_called_once_with([
"qemu-img", "info", "--output=json", '/tmp/original_path'])

def test_convert_image_format_to_raw_if_qcow2_raw_error(self):
self.patch_object(openstack_utils.os.path, "exists",
return_value=False)
self.patch_object(openstack_utils.subprocess, "check_output",
side_effect=[
'{"format": "qcow2"}',
subprocess.CalledProcessError(
returncode=42, cmd='mycmd')])
self.assertEqual("/tmp/original_path",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_has_calls([
mock.call(["qemu-img", "info", "--output=json",
'/tmp/original_path']),
mock.call(["qemu-img", "convert", '/tmp/original_path',
'/tmp/original_path.raw'])
])

def test_convert_image_format_to_raw_if_qcow2_raw_success(self):
self.patch_object(openstack_utils.os.path, "exists",
return_value=False)
self.patch_object(openstack_utils.subprocess, "check_output",
side_effect=['{"format": "qcow2"}', 'success'])

self.assertEqual("/tmp/original_path.raw",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_has_calls([
mock.call(["qemu-img", "info", "--output=json",
'/tmp/original_path']),
mock.call(["qemu-img", "convert", '/tmp/original_path',
'/tmp/original_path.raw'])
])

def test_convert_image_format_to_raw_if_qcow2_raw_already_exists(self):
self.patch_object(openstack_utils.pathlib.Path, "exists",
return_value=True)
self.patch_object(openstack_utils.subprocess, "check_output",
return_value='{"format": "qcow2"}')
self.assertEqual("/tmp/original_path.raw",
openstack_utils.convert_image_format_to_raw_if_qcow2(
"/tmp/original_path"))
self.check_output.assert_called_once_with([
"qemu-img", "info", "--output=json", '/tmp/original_path'])

def test_create_image_use_tempdir(self):
glance_mock = mock.MagicMock()
self.patch_object(openstack_utils.os.path, "exists")
self.patch_object(openstack_utils, "download_image")
self.patch_object(openstack_utils, "is_ceph_image_backend",
return_value=True)
self.patch_object(openstack_utils,
"convert_image_format_to_raw_if_qcow2",
return_value='wibbly/c.img.raw')
self.patch_object(openstack_utils, "upload_image_to_glance")
self.patch_object(openstack_utils.tempfile, "gettempdir")
self.gettempdir.return_value = "wibbly"
Expand All @@ -556,6 +645,37 @@ def test_create_image_use_tempdir(self):
'http://cirros/c.img',
'bob')
self.exists.return_value = False
self.download_image.assert_called_once_with(
'http://cirros/c.img',
'wibbly/c.img')
self.upload_image_to_glance.assert_called_once_with(
glance_mock,
'wibbly/c.img.raw',
'bob',
backend=None,
disk_format='raw',
visibility='public',
container_format='bare',
force_import=False)
self.convert_image_format_to_raw_if_qcow2.assert_called_once_with(
'wibbly/c.img')

def test_create_image_not_convert(self):
glance_mock = mock.MagicMock()
self.patch_object(openstack_utils.os.path, "exists")
self.patch_object(openstack_utils, "download_image")
self.patch_object(openstack_utils, "is_ceph_image_backend")
self.patch_object(openstack_utils,
"convert_image_format_to_raw_if_qcow2")
self.patch_object(openstack_utils, "upload_image_to_glance")
self.patch_object(openstack_utils.tempfile, "gettempdir")
self.gettempdir.return_value = "wibbly"
openstack_utils.create_image(
glance_mock,
'http://cirros/c.img',
'bob',
convert_image_to_raw_if_ceph_used=False)
self.exists.return_value = False
self.download_image.assert_called_once_with(
'http://cirros/c.img',
'wibbly/c.img')
Expand All @@ -568,11 +688,17 @@ def test_create_image_use_tempdir(self):
visibility='public',
container_format='bare',
force_import=False)
self.is_ceph_image_backend.assert_not_called()
self.convert_image_format_to_raw_if_qcow2.assert_not_called()

def test_create_image_pass_directory(self):
glance_mock = mock.MagicMock()
self.patch_object(openstack_utils.os.path, "exists")
self.patch_object(openstack_utils, "download_image")
self.patch_object(openstack_utils,
"convert_image_format_to_raw_if_qcow2")
self.patch_object(openstack_utils, "is_ceph_image_backend",
return_value=False)
self.patch_object(openstack_utils, "upload_image_to_glance")
self.patch_object(openstack_utils.tempfile, "gettempdir")
openstack_utils.create_image(
Expand All @@ -594,6 +720,7 @@ def test_create_image_pass_directory(self):
container_format='bare',
force_import=False)
self.gettempdir.assert_not_called()
self.convert_image_format_to_raw_if_qcow2.assert_not_called()

def test_create_ssh_key(self):
nova_mock = mock.MagicMock()
Expand Down
3 changes: 2 additions & 1 deletion zaza/openstack/charm_tests/glance/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ def test_412_image_conversion(self):
self.glance_client,
image_url,
'cirros-test-import',
force_import=True)
force_import=True,
convert_image_to_raw_if_ceph_used=False)

disk_format = self.glance_client.images.get(image.id).disk_format
self.assertEqual('raw', disk_format)
Expand Down
71 changes: 69 additions & 2 deletions zaza/openstack/utilities/openstack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
import enum
import io
import itertools
import json
import juju_wait
import logging
import os
import paramiko
import pathlib
import re
import shutil
import six
Expand Down Expand Up @@ -2498,7 +2500,7 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
:type local_path: str
:param image_name: The label to give the image in glance
:type image_name: str
:param disk_format: The of the underlying disk image.
:param disk_format: The format of the underlying disk image.
:type disk_format: str
:param visibility: Who can access image
:type visibility: str (public, private, shared or community)
Expand Down Expand Up @@ -2537,10 +2539,65 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2',
return image


def is_ceph_image_backend(model_name=None):
"""Check if glance is related to ceph, therefore using ceph as backend.
:returns: True if glance is related to Ceph, otherwise False
:rtype: bool
"""
status = juju_utils.get_full_juju_status(model_name=model_name)
result = False
try:
result = 'ceph-mon' in (
status['applications']['glance']['relations']['ceph'])
except KeyError:
pass
logging.debug("Detected Ceph related to Glance?: {}".format(result))
return result


def convert_image_format_to_raw_if_qcow2(local_path):
"""Convert the image format to raw if the detected format is qcow2.
:param local_path: The path to the original image file.
:type local_path: str
:returns: The path to the final image file
:rtype: str
"""
try:
output = subprocess.check_output([
"qemu-img", "info", "--output=json", local_path])
except subprocess.CalledProcessError:
logging.error("Image conversion: Failed to detect image format "
"of file {}".format(local_path))
return local_path
result = json.loads(output)
if result['format'] == 'qcow2':
logging.info("Image conversion: Detected qcow2 vs desired raw format"
" of file {}".format(local_path))
converted_path = pathlib.Path(local_path).resolve().with_suffix('.raw')
converted_path_str = str(converted_path)
if converted_path.exists():
logging.info("Image conversion: raw converted file already"
" exists: {}".format(converted_path_str))
return converted_path_str
logging.info("Image conversion: Converting image {} to raw".format(
local_path))
try:
output = subprocess.check_output([
"qemu-img", "convert", local_path, converted_path_str])
except subprocess.CalledProcessError:
logging.error("Image conversion: Failed to convert image"
" {} to raw".format(local_path))
return local_path
return converted_path_str
return local_path


def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
properties=None, backend=None, disk_format='qcow2',
visibility='public', container_format='bare',
force_import=False):
force_import=False, convert_image_to_raw_if_ceph_used=True):
"""Download the image and upload it to glance.
Download an image from image_url and upload it to glance labelling
Expand All @@ -2562,6 +2619,10 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
:param force_import: Force the use of glance image import
instead of direct upload
:type force_import: boolean
:param convert_image_to_raw_if_ceph_used: force conversion of requested
image to raw upon download if Ceph is present in the model and
has a relation to Glance
:type convert_image_to_raw_if_ceph_used: boolean
:returns: glance image pointer
:rtype: glanceclient.common.utils.RequestIdProxy
"""
Expand All @@ -2581,6 +2642,12 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[],
logging.info('Cached image found at {} - Skipping download'.format(
local_path))

if convert_image_to_raw_if_ceph_used and is_ceph_image_backend():
logging.info("Image conversion: Detected ceph backend, forcing"
" use of raw image format")
disk_format = 'raw'
local_path = convert_image_format_to_raw_if_qcow2(local_path)

image = upload_image_to_glance(
glance, local_path, image_name, backend=backend,
disk_format=disk_format, visibility=visibility,
Expand Down

0 comments on commit bee2d31

Please sign in to comment.