Merge "Move to a more canonicalized output from qemu-img info."

6 files changed, 302 insertions, 57 deletions

diff --git a/nova/tests/test_image_utils.py b/nova/tests/test_image_utils.py
index 711f1c202..fac0422bf 100644
--- a/nova/tests/test_image_utils.py
+++ b/nova/tests/test_image_utils.py
@@ -21,7 +21,7 @@ from nova.virt import images
 
 
 class ImageUtilsTestCase(test.TestCase):
-    def test_qemu_info(self):
+    def test_qemu_info_canon(self):
         path = "disk.config"
         example_output = """image: disk.config
 file format: raw
@@ -35,14 +35,35 @@ blah BLAH: bb
                       'qemu-img', 'info', path).AndReturn((example_output, ''))
         self.mox.ReplayAll()
         image_info = images.qemu_img_info(path)
-        self.assertEquals('disk.config', image_info['image'])
-        self.assertEquals('raw', image_info['file format'])
-        self.assertEquals('64M (67108864 bytes)', image_info['virtual size'])
-        self.assertEquals('96K', image_info['disk size'])
-        self.assertEquals('bb', image_info['blah blah'])
-        self.assertEquals("65536", image_info['cluster_size'])
+        self.assertEquals('disk.config', image_info.image)
+        self.assertEquals('raw', image_info.file_format)
+        self.assertEquals(67108864, image_info.virtual_size)
+        self.assertEquals(98304, image_info.disk_size)
+        self.assertEquals(65536, image_info.cluster_size)
 
-    def test_qemu_info_snap(self):
+    def test_qemu_info_canon2(self):
+        path = "disk.config"
+        example_output = """image: disk.config
+file format: QCOW2
+virtual size: 67108844
+cluster_size: 65536
+disk size: 963434
+backing file: /var/lib/nova/a328c7998805951a_2
+"""
+        self.mox.StubOutWithMock(utils, 'execute')
+        utils.execute('env', 'LC_ALL=C', 'LANG=C',
+                      'qemu-img', 'info', path).AndReturn((example_output, ''))
+        self.mox.ReplayAll()
+        image_info = images.qemu_img_info(path)
+        self.assertEquals('disk.config', image_info.image)
+        self.assertEquals('qcow2', image_info.file_format)
+        self.assertEquals(67108844, image_info.virtual_size)
+        self.assertEquals(963434, image_info.disk_size)
+        self.assertEquals(65536, image_info.cluster_size)
+        self.assertEquals('/var/lib/nova/a328c7998805951a_2',
+                          image_info.backing_file)
+
+    def test_qemu_backing_file_actual(self):
         path = "disk.config"
         example_output = """image: disk.config
 file format: raw
@@ -52,18 +73,63 @@ disk size: 96K
 Snapshot list:
 ID        TAG                 VM SIZE                DATE       VM CLOCK
 1     d9a9784a500742a7bb95627bb3aace38      0 2012-08-20 10:52:46 00:00:00.000
+backing file: /var/lib/nova/a328c7998805951a_2 (actual path: /b/3a988059e51a_2)
+"""
+        self.mox.StubOutWithMock(utils, 'execute')
+        utils.execute('env', 'LC_ALL=C', 'LANG=C',
+                      'qemu-img', 'info', path).AndReturn((example_output, ''))
+        self.mox.ReplayAll()
+        image_info = images.qemu_img_info(path)
+        self.assertEquals('disk.config', image_info.image)
+        self.assertEquals('raw', image_info.file_format)
+        self.assertEquals(67108864, image_info.virtual_size)
+        self.assertEquals(98304, image_info.disk_size)
+        self.assertEquals(1, len(image_info.snapshots))
+        self.assertEquals('/b/3a988059e51a_2',
+                          image_info.backing_file)
+
+    def test_qemu_info_convert(self):
+        path = "disk.config"
+        example_output = """image: disk.config
+file format: raw
+virtual size: 64M
+disk size: 96K
+Snapshot list:
+ID        TAG                 VM SIZE                DATE       VM CLOCK
+1        d9a9784a500742a7bb95627bb3aace38    0 2012-08-20 10:52:46 00:00:00.000
+3        d9a9784a500742a7bb95627bb3aace38    0 2012-08-20 10:52:46 00:00:00.000
+4        d9a9784a500742a7bb95627bb3aace38    0 2012-08-20 10:52:46 00:00:00.000
+junk stuff: bbb
+"""
+        self.mox.StubOutWithMock(utils, 'execute')
+        utils.execute('env', 'LC_ALL=C', 'LANG=C',
+                      'qemu-img', 'info', path).AndReturn((example_output, ''))
+        self.mox.ReplayAll()
+        image_info = images.qemu_img_info(path)
+        self.assertEquals('disk.config', image_info.image)
+        self.assertEquals('raw', image_info.file_format)
+        self.assertEquals(67108864, image_info.virtual_size)
+        self.assertEquals(98304, image_info.disk_size)
+
+    def test_qemu_info_snaps(self):
+        path = "disk.config"
+        example_output = """image: disk.config
+file format: raw
+virtual size: 64M (67108864 bytes)
+disk size: 96K
+Snapshot list:
+ID        TAG                 VM SIZE                DATE       VM CLOCK
+1        d9a9784a500742a7bb95627bb3aace38    0 2012-08-20 10:52:46 00:00:00.000
+3        d9a9784a500742a7bb95627bb3aace38    0 2012-08-20 10:52:46 00:00:00.000
+4        d9a9784a500742a7bb95627bb3aace38    0 2012-08-20 10:52:46 00:00:00.000
 """
         self.mox.StubOutWithMock(utils, 'execute')
         utils.execute('env', 'LC_ALL=C', 'LANG=C',
                       'qemu-img', 'info', path).AndReturn((example_output, ''))
         self.mox.ReplayAll()
         image_info = images.qemu_img_info(path)
-        self.assertEquals('disk.config', image_info['image'])
-        self.assertEquals('raw', image_info['file format'])
-        self.assertEquals('64M (67108864 bytes)', image_info['virtual size'])
-        self.assertEquals('96K', image_info['disk size'])
-        self.assertEquals("65536", image_info['cluster_size'])
-        # This would be triggered if the split encountered this section
-        self.assertNotIn('snapshot list', image_info)
-        bad_cap = '1     d9a9784a500742a7bb95627bb3aace38      0 2012-08-20 10'
-        self.assertNotIn(bad_cap, image_info)
+        self.assertEquals('disk.config', image_info.image)
+        self.assertEquals('raw', image_info.file_format)
+        self.assertEquals(67108864, image_info.virtual_size)
+        self.assertEquals(98304, image_info.disk_size)
+        self.assertEquals(3, len(image_info.snapshots))
diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py
index f97d6eeb7..1b84b858d 100644
--- a/nova/tests/test_utils.py
+++ b/nova/tests/test_utils.py
@@ -37,6 +37,41 @@ from nova import utils
 FLAGS = flags.FLAGS
 
 
+class ByteConversionTest(test.TestCase):
+    def test_string_conversions(self):
+        working_examples = {
+            '1024KB': 1048576,
+            '1024TB': 1125899906842624,
+            '1024K': 1048576,
+            '1024T': 1125899906842624,
+            '1TB': 1099511627776,
+            '1T': 1099511627776,
+            '1KB': 1024,
+            '1K': 1024,
+            '1B': 1,
+            '1B': 1,
+            '1': 1,
+            '1MB': 1048576,
+            '7MB': 7340032,
+            '0MB': 0,
+            '0KB': 0,
+            '0TB': 0,
+            '': 0,
+        }
+        for (in_value, expected_value) in working_examples.items():
+            b_value = utils.to_bytes(in_value)
+            self.assertEquals(expected_value, b_value)
+            if len(in_value):
+                in_value = "-" + in_value
+                b_value = utils.to_bytes(in_value)
+                self.assertEquals(expected_value * -1, b_value)
+        breaking_examples = [
+            'junk1KB', '1023BBBB',
+        ]
+        for v in breaking_examples:
+            self.assertRaises(TypeError, utils.to_bytes, v)
+
+
 class ExecuteTestCase(test.TestCase):
     def test_retry_on_failure(self):
         fd, tmpfilename = tempfile.mkstemp()
diff --git a/nova/utils.py b/nova/utils.py
index 26637a489..284d72b55 100644
--- a/nova/utils.py
+++ b/nova/utils.py
@@ -63,6 +63,16 @@ FLAGS.register_opt(
     cfg.BoolOpt('disable_process_locking', default=False,
                 help='Whether to disable inter-process locks'))
 
+# Used for looking up extensions of text
+# to their 'multiplied' byte amount
+BYTE_MULTIPLIERS = {
+    '': 1,
+    't': 1024 ** 4,
+    'g': 1024 ** 3,
+    'm': 1024 ** 2,
+    'k': 1024,
+}
+
 
 def vpn_ping(address, port, timeout=0.05, session_id=None):
     """Sends a vpn negotiation packet and returns the server session.
@@ -574,6 +584,34 @@ def utf8(value):
     return value
 
 
+def to_bytes(text, default=0):
+    """Try to turn a string into a number of bytes. Looks at the last
+    characters of the text to determine what conversion is needed to
+    turn the input text into a byte number.
+
+    Supports: B/b, K/k, M/m, G/g, T/t (or the same with b/B on the end)
+
+    """
+    # Take off everything not number 'like' (which should leave
+    # only the byte 'identifier' left)
+    mult_key_org = text.lstrip('-1234567890')
+    mult_key = mult_key_org.lower()
+    mult_key_len = len(mult_key)
+    if mult_key.endswith("b"):
+        mult_key = mult_key[0:-1]
+    try:
+        multiplier = BYTE_MULTIPLIERS[mult_key]
+        if mult_key_len:
+            # Empty cases shouldn't cause text[0:-0]
+            text = text[0:-mult_key_len]
+        return int(text) * multiplier
+    except KeyError:
+        msg = _('Unknown byte multiplier: %s') % mult_key_org
+        raise TypeError(msg)
+    except ValueError:
+        return default
+
+
 def delete_if_exists(pathname):
     """delete a file, but ignore file not found error"""
 
diff --git a/nova/virt/disk/api.py b/nova/virt/disk/api.py
index 0cddcfa69..e113391a5 100644
--- a/nova/virt/disk/api.py
+++ b/nova/virt/disk/api.py
@@ -112,9 +112,7 @@ def get_disk_size(path):
     :returns: Size (in bytes) of the given disk image as it would be seen
               by a virtual machine.
     """
-    size = images.qemu_img_info(path)['virtual size']
-    size = size.split('(')[1].split()[0]
-    return int(size)
+    return images.qemu_img_info(path).virtual_size
 
 
 def extend(image, size):
diff --git a/nova/virt/images.py b/nova/virt/images.py
index 133f5e25b..5b631a0da 100644
--- a/nova/virt/images.py
+++ b/nova/virt/images.py
@@ -22,6 +22,7 @@ Handling of VM disk images.
 """
 
 import os
+import re
 
 from nova import exception
 from nova import flags
@@ -43,31 +44,142 @@ FLAGS = flags.FLAGS
 FLAGS.register_opts(image_opts)
 
 
-def qemu_img_info(path):
-    """Return a dict containing the parsed output from qemu-img info."""
+class QemuImgInfo(object):
+    BACKING_FILE_RE = re.compile((r"^(.*?)\s*\(actual\s+path\s*:"
+                                  r"\s+(.*?)\)\s*$"), re.I)
+    TOP_LEVEL_RE = re.compile(r"^([\w\d\s\_\-]+):(.*)$")
+    SIZE_RE = re.compile(r"\(\s*(\d+)\s+bytes\s*\)", re.I)
+
+    def __init__(self, cmd_output):
+        details = self._parse(cmd_output)
+        self.image = details.get('image')
+        self.backing_file = details.get('backing_file')
+        self.file_format = details.get('file_format')
+        self.virtual_size = details.get('virtual_size')
+        self.cluster_size = details.get('cluster_size')
+        self.disk_size = details.get('disk_size')
+        self.snapshots = details.get('snapshot_list', [])
+        self.encryption = details.get('encryption')
+
+    def __str__(self):
+        lines = [
+            'image: %s' % self.image,
+            'file_format: %s' % self.file_format,
+            'virtual_size: %s' % self.virtual_size,
+            'disk_size: %s' % self.disk_size,
+            'cluster_size: %s' % self.cluster_size,
+            'backing_file: %s' % self.backing_file,
+        ]
+        if self.snapshots:
+            lines.append("snapshots: %s" % self.snapshots)
+        return "\n".join(lines)
+
+    def _canonicalize(self, field):
+        # Standardize on underscores/lc/no dash and no spaces
+        # since qemu seems to have mixed outputs here... and
+        # this format allows for better integration with python
+        # - ie for usage in kwargs and such...
+        field = field.lower().strip()
+        for c in (" ", "-"):
+            field = field.replace(c, '_')
+        return field
+
+    def _extract_bytes(self, details):
+        # Replace it with the byte amount
+        real_size = self.SIZE_RE.search(details)
+        if real_size:
+            details = real_size.group(1)
+        try:
+            details = utils.to_bytes(details)
+        except (TypeError, ValueError):
+            pass
+        return details
+
+    def _extract_details(self, root_cmd, root_details, lines_after):
+        consumed_lines = 0
+        real_details = root_details
+        if root_cmd == 'backing_file':
+            # Replace it with the real backing file
+            backing_match = self.BACKING_FILE_RE.match(root_details)
+            if backing_match:
+                real_details = backing_match.group(2).strip()
+        elif root_cmd in ['virtual_size', 'cluster_size', 'disk_size']:
+            # Replace it with the byte amount (if we can convert it)
+            real_details = self._extract_bytes(root_details)
+        elif root_cmd == 'file_format':
+            real_details = real_details.strip().lower()
+        elif root_cmd == 'snapshot_list':
+            # Next line should be a header, starting with 'ID'
+            if not lines_after or not lines_after[0].startswith("ID"):
+                msg = _("Snapshot list encountered but no header found!")
+                raise ValueError(msg)
+            consumed_lines += 1
+            possible_contents = lines_after[1:]
+            real_details = []
+            # This is the sprintf pattern we will try to match
+            # "%-10s%-20s%7s%20s%15s"
+            # ID TAG VM SIZE DATE VM CLOCK (current header)
+            for line in possible_contents:
+                line_pieces = line.split(None)
+                if len(line_pieces) != 6:
+                    break
+                else:
+                    # Check against this pattern occuring in the final position
+                    # "%02d:%02d:%02d.%03d"
+                    date_pieces = line_pieces[5].split(":")
+                    if len(date_pieces) != 3:
+                        break
+                    real_details.append({
+                        'id': line_pieces[0],
+                        'tag': line_pieces[1],
+                        'vm_size': line_pieces[2],
+                        'date': line_pieces[3],
+                        'vm_clock': line_pieces[4] + " " + line_pieces[5],
+                    })
+                    consumed_lines += 1
+        return (real_details, consumed_lines)
+
+    def _parse(self, cmd_output):
+        # Analysis done of qemu-img.c to figure out what is going on here
+        # Find all points start with some chars and then a ':' then a newline
+        # and then handle the results of those 'top level' items in a separate
+        # function.
+        #
+        # TODO(harlowja): newer versions might have a json output format
+        #                 we should switch to that whenever possible.
+        #                 see: http://bit.ly/XLJXDX
+        if not cmd_output:
+            cmd_output = ''
+        contents = {}
+        lines = cmd_output.splitlines()
+        i = 0
+        line_am = len(lines)
+        while i < line_am:
+            line = lines[i]
+            if not line.strip():
+                i += 1
+                continue
+            consumed_lines = 0
+            top_level = self.TOP_LEVEL_RE.match(line)
+            if top_level:
+                root = self._canonicalize(top_level.group(1))
+                if not root:
+                    i += 1
+                    continue
+                root_details = top_level.group(2).strip()
+                details, consumed_lines = self._extract_details(root,
+                                                                root_details,
+                                                                lines[i + 1:])
+                contents[root] = details
+            i += consumed_lines + 1
+        return contents
+
 
+def qemu_img_info(path):
+    """Return a object containing the parsed output from qemu-img info."""
     out, err = utils.execute('env', 'LC_ALL=C', 'LANG=C',
                              'qemu-img', 'info', path)
-
-    # output of qemu-img is 'field: value'
-    # except when its in the snapshot listing mode
-    data = {}
-    for line in out.splitlines():
-        pieces = line.split(':', 1)
-        if len(pieces) != 2:
-            continue
-        (field, val) = pieces
-        field = field.strip().lower()
-        val = val.strip()
-        if field == 'snapshot list':
-            # Skip everything after the snapshot list
-            # which is safe to do since the code prints
-            # these out at the end and nobody currently
-            # uses this information in openstack as-is.
-            break
-        data[field] = val
-
-    return data
+    return QemuImgInfo(out)
 
 
 def convert_image(source, dest, out_format):
@@ -95,13 +207,13 @@ def fetch_to_raw(context, image_href, path, user_id, project_id):
     with utils.remove_path_on_error(path_tmp):
         data = qemu_img_info(path_tmp)
 
-        fmt = data.get('file format')
+        fmt = data.file_format
         if fmt is None:
             raise exception.ImageUnacceptable(
                 reason=_("'qemu-img info' parsing failed."),
                 image_id=image_href)
 
-        backing_file = data.get('backing file')
+        backing_file = data.backing_file
         if backing_file is not None:
             raise exception.ImageUnacceptable(image_id=image_href,
                 reason=_("fmt=%(fmt)s backed by: %(backing_file)s") % locals())
@@ -113,10 +225,10 @@ def fetch_to_raw(context, image_href, path, user_id, project_id):
                 convert_image(path_tmp, staged, 'raw')
 
                 data = qemu_img_info(staged)
-                if data.get('file format') != "raw":
+                if data.file_format != "raw":
                     raise exception.ImageUnacceptable(image_id=image_href,
                         reason=_("Converted to raw, but format is now %s") %
-                        data.get('file format'))
+                        data.file_format)
 
                 os.rename(staged, path)
 
diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py
index fe54cacec..5da0aa6fb 100644
--- a/nova/virt/libvirt/utils.py
+++ b/nova/virt/libvirt/utils.py
@@ -94,18 +94,18 @@ def create_cow_image(backing_file, path):
         cow_opts += ['backing_file=%s' % backing_file]
         base_details = images.qemu_img_info(backing_file)
     else:
-        base_details = {}
+        base_details = None
     # This doesn't seem to get inherited so force it to...
     # http://paste.ubuntu.com/1213295/
     # TODO(harlowja) probably file a bug against qemu-img/qemu
-    if 'cluster_size' in base_details:
-        cow_opts += ['cluster_size=%s' % base_details['cluster_size']]
+    if base_details and base_details.cluster_size is not None:
+        cow_opts += ['cluster_size=%s' % base_details.cluster_size]
     # For now don't inherit this due the following discussion...
     # See: http://www.gossamer-threads.com/lists/openstack/dev/10592
     # if 'preallocation' in base_details:
     #     cow_opts += ['preallocation=%s' % base_details['preallocation']]
-    if 'encryption' in base_details:
-        cow_opts += ['encryption=%s' % base_details['encryption']]
+    if base_details and base_details.encryption:
+        cow_opts += ['encryption=%s' % base_details.encryption]
     if cow_opts:
         # Format as a comma separated list
         csv_opts = ",".join(cow_opts)
@@ -228,8 +228,7 @@ def get_disk_size(path):
     :returns: Size (in bytes) of the given disk image as it would be seen
               by a virtual machine.
     """
-    size = images.qemu_img_info(path)['virtual size']
-    size = size.split('(')[1].split()[0]
+    size = images.qemu_img_info(path).virtual_size
     return int(size)
 
 
@@ -239,11 +238,8 @@ def get_disk_backing_file(path):
     :param path: Path to the disk image
     :returns: a path to the image's backing store
     """
-    backing_file = images.qemu_img_info(path).get('backing file')
-
+    backing_file = images.qemu_img_info(path).backing_file
     if backing_file:
-        if 'actual path: ' in backing_file:
-            backing_file = backing_file.split('actual path: ')[1][:-1]
         backing_file = os.path.basename(backing_file)
 
     return backing_file