From 0aee8864f19334211e2c52af9cf62e7fd3eecd1f Mon Sep 17 00:00:00 2001 From: Michael Still Date: Thu, 19 Apr 2012 02:27:56 +1000 Subject: Move image checksums into a generic file. This will allow us to store other information about an image on a single compute node. This is required for prefetching to be implemented. Change-Id: Ic65b04cc1ef2101a1ac588595b943b813abdd50d --- nova/virt/libvirt/imagecache.py | 73 ++++++------------------------ nova/virt/libvirt/utils.py | 98 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 59 deletions(-) (limited to 'nova/virt') diff --git a/nova/virt/libvirt/imagecache.py b/nova/virt/libvirt/imagecache.py index 5c8eba66f..3a92b6253 100644 --- a/nova/virt/libvirt/imagecache.py +++ b/nova/virt/libvirt/imagecache.py @@ -55,10 +55,6 @@ imagecache_opts = [ cfg.BoolOpt('checksum_base_images', default=False, help='Write a checksum for files in _base to disk'), - cfg.StrOpt('image_info_filename_pattern', - default='$instances_path/$base_dir_name/%(image)s.sha1', - help='Allows image information files to be stored in ' - 'non-standard locations'), ] flags.DECLARE('instances_path', 'nova.compute.manager') @@ -67,65 +63,23 @@ FLAGS = flags.FLAGS FLAGS.register_opts(imagecache_opts) -def get_info_filename(base_path): - """Construct a filename for storing addtional information about a base - image. - - Returns a filename. - """ - - base_file = os.path.basename(base_path) - return (FLAGS.image_info_filename_pattern - % {'image': base_file}) - - -def is_valid_info_file(path): - """Test if a given path matches the pattern for info files.""" - - digest_size = hashlib.sha1().digestsize * 2 - regexp = (FLAGS.image_info_filename_pattern - % {'instances_path': FLAGS.instances_path, - 'image': ('([0-9a-f]{%(digest_size)d}|' - '[0-9a-f]{%(digest_size)d}_sm|' - '[0-9a-f]{%(digest_size)d}_[0-9]+)' - % {'digest_size': digest_size})}) - m = re.match(regexp, path) - if m: - return True - return False - - -def read_stored_checksum(base_path): - """Read the checksum which was created at image fetch time. +def read_stored_checksum(target): + """Read the checksum. Returns the checksum (as hex) or None. """ - - checksum_file = get_info_filename(base_path) - if not os.path.exists(checksum_file): - return None - - f = open(checksum_file, 'r') - stored_checksum = f.read().rstrip() - f.close() - return stored_checksum + return virtutils.read_stored_info(target, field='sha1') def write_stored_checksum(target): """Write a checksum to disk for a file in _base.""" - checksum_filename = get_info_filename(target) - if os.path.exists(target) and not os.path.exists(checksum_filename): - # NOTE(mikal): Create the checksum file first to exclude possible - # overlap in checksum operations. An empty checksum file is ignored if - # encountered during verification. - sum_file = open(checksum_filename, 'w') + if not read_stored_checksum(target): img_file = open(target, 'r') checksum = utils.hash_file(img_file) img_file.close() - sum_file.write(checksum) - sum_file.close() + virtutils.write_stored_info(target, field='sha1', value=checksum) class ImageCacheManager(object): @@ -170,7 +124,8 @@ class ImageCacheManager(object): elif (len(ent) > digest_size + 2 and ent[digest_size] == '_' and - not is_valid_info_file(os.path.join(base_dir, ent))): + not virtutils.is_valid_info_file(os.path.join(base_dir, + ent))): self._store_image(base_dir, ent, original=False) def _list_running_instances(self, context): @@ -254,7 +209,7 @@ class ImageCacheManager(object): if m: yield img, False, True - def _verify_checksum(self, img_id, base_file): + def _verify_checksum(self, img_id, base_file, create_if_missing=True): """Compare the checksum stored on disk with the current file. Note that if the checksum fails to verify this is logged, but no actual @@ -279,15 +234,15 @@ class ImageCacheManager(object): return True else: - LOG.debug(_('%(id)s (%(base_file)s): image verification skipped, ' - 'no hash stored'), - {'id': img_id, - 'base_file': base_file}) + LOG.info(_('%(id)s (%(base_file)s): image verification skipped, ' + 'no hash stored'), + {'id': img_id, + 'base_file': base_file}) # NOTE(mikal): If the checksum file is missing, then we should # create one. We don't create checksums when we download images # from glance because that would delay VM startup. - if FLAGS.checksum_base_images: + if FLAGS.checksum_base_images and create_if_missing: write_stored_checksum(base_file) return None @@ -316,7 +271,7 @@ class ImageCacheManager(object): LOG.info(_('Removing base file: %s'), base_file) try: os.remove(base_file) - signature = get_info_filename(base_file) + signature = virtutils.get_info_filename(base_file) if os.path.exists(signature): os.remove(signature) except OSError, e: diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py index db7e11b26..f4c6d20a5 100644 --- a/nova/virt/libvirt/utils.py +++ b/nova/virt/libvirt/utils.py @@ -19,16 +19,34 @@ # License for the specific language governing permissions and limitations # under the License. +import hashlib +import json import os import random +import re from nova import exception from nova import flags +from nova import log as logging from nova import utils +from nova.openstack.common import cfg from nova.virt import images +LOG = logging.getLogger(__name__) + + +util_opts = [ + cfg.StrOpt('image_info_filename_pattern', + default='$instances_path/$base_dir_name/%(image)s.info', + help='Allows image information files to be stored in ' + 'non-standard locations') + ] + +flags.DECLARE('instances_path', 'nova.compute.manager') +flags.DECLARE('base_dir_name', 'nova.compute.manager') FLAGS = flags.FLAGS +FLAGS.register_opts(util_opts) def execute(*args, **kwargs): @@ -286,3 +304,83 @@ def get_fs_info(path): def fetch_image(context, target, image_id, user_id, project_id): """Grab image""" images.fetch_to_raw(context, image_id, target, user_id, project_id) + + +def get_info_filename(base_path): + """Construct a filename for storing addtional information about a base + image. + + Returns a filename. + """ + + base_file = os.path.basename(base_path) + return (FLAGS.image_info_filename_pattern + % {'image': base_file}) + + +def is_valid_info_file(path): + """Test if a given path matches the pattern for info files.""" + + digest_size = hashlib.sha1().digestsize * 2 + regexp = (FLAGS.image_info_filename_pattern + % {'image': ('([0-9a-f]{%(digest_size)d}|' + '[0-9a-f]{%(digest_size)d}_sm|' + '[0-9a-f]{%(digest_size)d}_[0-9]+)' + % {'digest_size': digest_size})}) + m = re.match(regexp, path) + if m: + return True + return False + + +def read_stored_info(base_path, field=None): + """Read information about an image. + + Returns an empty dictionary if there is no info, just the field value if + a field is requested, or the entire dictionary otherwise. + """ + + info_file = get_info_filename(base_path) + if not os.path.exists(info_file): + d = {} + + else: + LOG.info(_('Reading image info file: %s'), info_file) + f = open(info_file, 'r') + serialized = f.read().rstrip() + f.close() + LOG.info(_('Read: %s'), serialized) + + try: + d = json.loads(serialized) + + except ValueError, e: + LOG.error(_('Error reading image info file %(filename)s: ' + '%(error)s'), + {'filename': info_file, + 'error': e}) + d = {} + + if field: + return d.get(field, None) + return d + + +def write_stored_info(target, field=None, value=None): + """Write information about an image.""" + + if not field: + return + + info_file = get_info_filename(target) + ensure_tree(os.path.dirname(info_file)) + + d = read_stored_info(info_file) + d[field] = value + serialized = json.dumps(d) + + LOG.info(_('Writing image info file: %s'), info_file) + LOG.info(_('Wrote: %s'), serialized) + f = open(info_file, 'w') + f.write(serialized) + f.close() -- cgit