Move image checksums into a generic file.

This will allow us to store other information about an image on a
single compute node. This is required for prefetching to be
implemented.

Change-Id: Ic65b04cc1ef2101a1ac588595b943b813abdd50d

1 files changed, 150 insertions, 51 deletions

diff --git a/nova/tests/test_imagecache.py b/nova/tests/test_imagecache.py
index 6de3722eb..410f4b1a8 100644
--- a/nova/tests/test_imagecache.py
+++ b/nova/tests/test_imagecache.py
@@ -53,7 +53,6 @@ class ImageCacheManagerTestCase(test.TestCase):
 
     def test_read_stored_checksum_missing(self):
         self.stubs.Set(os.path, 'exists', lambda x: False)
-
         csum = imagecache.read_stored_checksum('/tmp/foo')
         self.assertEquals(csum, None)
 
@@ -61,30 +60,31 @@ class ImageCacheManagerTestCase(test.TestCase):
         with utils.tempdir() as tmpdir:
             self.flags(instances_path=tmpdir)
             self.flags(image_info_filename_pattern=('$instances_path/'
-                                                    '%(image)s.sha1'))
+                                                    '%(image)s.info'))
 
+            csum_input = '{"sha1": "fdghkfhkgjjksfdgjksjkghsdf"}\n'
             fname = os.path.join(tmpdir, 'aaa')
-
-            csum_input = 'fdghkfhkgjjksfdgjksjkghsdf'
-            f = open('%s.sha1' % fname, 'w')
-            f.write('%s\n' % csum_input)
+            info_fname = virtutils.get_info_filename(fname)
+            f = open(info_fname, 'w')
+            f.write(csum_input)
             f.close()
 
             csum_output = imagecache.read_stored_checksum(fname)
-
-            self.assertEquals(csum_input, csum_output)
+            self.assertEquals(csum_input.rstrip(),
+                              '{"sha1": "%s"}' % csum_output)
 
     def test_list_base_images(self):
         listing = ['00000001',
                    'ephemeral_0_20_None',
-                   'e97222e91fc4241f49a7f520d1dcf446751129b3_sm',
-                   'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm',
-                   'e97222e91fc4241f49a7f520d1dcf446751129b3',
-                   '17d1b00b81642842e514494a78e804e9a511637c',
-                   '17d1b00b81642842e514494a78e804e9a511637c_5368709120',
-                   '17d1b00b81642842e514494a78e804e9a511637c_5368709120.sha1',
-                   '17d1b00b81642842e514494a78e804e9a511637c_10737418240',
-                   '00000004']
+                   '17d1b00b81642842e514494a78e804e9a511637c_5368709120.info',
+                    '00000004']
+        images = ['e97222e91fc4241f49a7f520d1dcf446751129b3_sm',
+                  'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm',
+                  'e97222e91fc4241f49a7f520d1dcf446751129b3',
+                  '17d1b00b81642842e514494a78e804e9a511637c',
+                  '17d1b00b81642842e514494a78e804e9a511637c_5368709120',
+                  '17d1b00b81642842e514494a78e804e9a511637c_10737418240']
+        listing.extend(images)
 
         self.stubs.Set(os, 'listdir', lambda x: listing)
         self.stubs.Set(os.path, 'isfile', lambda x: True)
@@ -95,7 +95,13 @@ class ImageCacheManagerTestCase(test.TestCase):
         image_cache_manager = imagecache.ImageCacheManager()
         image_cache_manager._list_base_images(base_dir)
 
-        self.assertEquals(len(image_cache_manager.unexplained_images), 6)
+        sanitized = []
+        for ent in image_cache_manager.unexplained_images:
+            sanitized.append(ent.replace(base_dir + '/', ''))
+
+        sanitized = sanitized.sort()
+        images = images.sort()
+        self.assertEquals(sanitized, images)
 
         expected = os.path.join(base_dir,
                                 'e97222e91fc4241f49a7f520d1dcf446751129b3')
@@ -296,9 +302,20 @@ class ImageCacheManagerTestCase(test.TestCase):
         finally:
             mylog.logger.removeHandler(handler)
 
-    def test_verify_checksum(self):
+    def _make_checksum(self, tmpdir):
         testdata = ('OpenStack Software delivers a massively scalable cloud '
                     'operating system.')
+
+        fname = os.path.join(tmpdir, 'aaa')
+        info_fname = virtutils.get_info_filename(fname)
+
+        f = open(fname, 'w')
+        f.write(testdata)
+        f.close()
+
+        return fname, info_fname, testdata
+
+    def test_verify_checksum(self):
         img = {'container_format': 'ami', 'id': '42'}
 
         self.flags(checksum_base_images=True)
@@ -307,45 +324,117 @@ class ImageCacheManagerTestCase(test.TestCase):
             with utils.tempdir() as tmpdir:
                 self.flags(instances_path=tmpdir)
                 self.flags(image_info_filename_pattern=('$instances_path/'
-                                                        '%(image)s.sha1'))
-
-                fname = os.path.join(tmpdir, 'aaa')
-
-                f = open(fname, 'w')
-                f.write(testdata)
-                f.close()
+                                                        '%(image)s.info'))
+                fname, info_fname, testdata = self._make_checksum(tmpdir)
 
                 # Checksum is valid
-                f = open('%s.sha1' % fname, 'w')
+                f = open(info_fname, 'w')
                 csum = hashlib.sha1()
                 csum.update(testdata)
-                f.write(csum.hexdigest())
+                f.write('{"sha1": "%s"}\n' % csum.hexdigest())
                 f.close()
 
                 image_cache_manager = imagecache.ImageCacheManager()
                 res = image_cache_manager._verify_checksum(img, fname)
                 self.assertTrue(res)
 
-                # Checksum is invalid
-                f = open('%s.sha1' % fname, 'w')
+    def test_verify_checksum_invalid_json(self):
+        img = {'container_format': 'ami', 'id': '42'}
+
+        self.flags(checksum_base_images=True)
+
+        with self._intercept_log_messages() as stream:
+            with utils.tempdir() as tmpdir:
+                self.flags(instances_path=tmpdir)
+                self.flags(image_info_filename_pattern=('$instances_path/'
+                                                        '%(image)s.info'))
+
+                fname, info_fname, testdata = self._make_checksum(tmpdir)
+
+                # Checksum is invalid, and not json
+                f = open(info_fname, 'w')
                 f.write('banana')
                 f.close()
 
                 image_cache_manager = imagecache.ImageCacheManager()
+                res = image_cache_manager._verify_checksum(
+                    img, fname, create_if_missing=False)
+                self.assertFalse(res)
+                log = stream.getvalue()
+
+                # NOTE(mikal): this is a skip not a fail because the file is
+                # present, but is not in valid json format and therefore is
+                # skipped.
+                self.assertNotEqual(log.find('image verification skipped'), -1)
+
+    def test_verify_checksum_invalid_repaired(self):
+        img = {'container_format': 'ami', 'id': '42'}
+
+        self.flags(checksum_base_images=True)
+
+        with utils.tempdir() as tmpdir:
+            self.flags(instances_path=tmpdir)
+            self.flags(image_info_filename_pattern=('$instances_path/'
+                                                    '%(image)s.info'))
+
+            fname, info_fname, testdata = self._make_checksum(tmpdir)
+
+            # Checksum is invalid, and not json
+            f = open(info_fname, 'w')
+            f.write('banana')
+            f.close()
+
+            image_cache_manager = imagecache.ImageCacheManager()
+            res = image_cache_manager._verify_checksum(
+                img, fname, create_if_missing=True)
+            self.assertTrue(res == None)
+
+    def test_verify_checksum_invalid(self):
+        img = {'container_format': 'ami', 'id': '42'}
+
+        self.flags(checksum_base_images=True)
+
+        with self._intercept_log_messages() as stream:
+            with utils.tempdir() as tmpdir:
+                self.flags(instances_path=tmpdir)
+                self.flags(image_info_filename_pattern=('$instances_path/'
+                                                        '%(image)s.info'))
+
+                fname, info_fname, testdata = self._make_checksum(tmpdir)
+
+                # Checksum is invalid, but is in valid json
+                f = open(info_fname, 'w')
+                f.write('{"sha1": "banana"}')
+                f.close()
+
+                image_cache_manager = imagecache.ImageCacheManager()
                 res = image_cache_manager._verify_checksum(img, fname)
                 self.assertFalse(res)
                 log = stream.getvalue()
+
                 self.assertNotEqual(log.find('image verification failed'), -1)
 
+    def test_verify_checksum_file_missing(self):
+        img = {'container_format': 'ami', 'id': '42'}
+
+        self.flags(checksum_base_images=True)
+
+        with self._intercept_log_messages() as stream:
+            with utils.tempdir() as tmpdir:
+                self.flags(instances_path=tmpdir)
+                self.flags(image_info_filename_pattern=('$instances_path/'
+                                                        '%(image)s.info'))
+
+                fname, info_fname, testdata = self._make_checksum(tmpdir)
+
                 # Checksum file missing
-                os.remove('%s.sha1' % fname)
                 image_cache_manager = imagecache.ImageCacheManager()
                 res = image_cache_manager._verify_checksum(img, fname)
                 self.assertEquals(res, None)
 
                 # Checksum requests for a file with no checksum now have the
                 # side effect of creating the checksum
-                self.assertTrue(os.path.exists('%s.sha1' % fname))
+                self.assertTrue(os.path.exists(info_fname))
 
     @contextlib.contextmanager
     def _make_base_file(self, checksum=True):
@@ -354,8 +443,7 @@ class ImageCacheManagerTestCase(test.TestCase):
         with utils.tempdir() as tmpdir:
             self.flags(instances_path=tmpdir)
             self.flags(image_info_filename_pattern=('$instances_path/'
-                                                    '%(image)s.sha1'))
-
+                                                    '%(image)s.info'))
             fname = os.path.join(tmpdir, 'aaa')
 
             base_file = open(fname, 'w')
@@ -364,9 +452,7 @@ class ImageCacheManagerTestCase(test.TestCase):
             base_file = open(fname, 'r')
 
             if checksum:
-                checksum_file = open('%s.sha1' % fname, 'w')
-                checksum_file.write(utils.hash_file(base_file))
-                checksum_file.close()
+                imagecache.write_stored_checksum(fname)
 
             base_file.close()
             yield fname
@@ -375,46 +461,52 @@ class ImageCacheManagerTestCase(test.TestCase):
         with self._make_base_file() as fname:
             image_cache_manager = imagecache.ImageCacheManager()
             image_cache_manager._remove_base_file(fname)
+            info_fname = virtutils.get_info_filename(fname)
 
             # Files are initially too new to delete
             self.assertTrue(os.path.exists(fname))
-            self.assertTrue(os.path.exists('%s.sha1' % fname))
+            self.assertTrue(os.path.exists(info_fname))
 
             # Old files get cleaned up though
             os.utime(fname, (-1, time.time() - 3601))
             image_cache_manager._remove_base_file(fname)
 
             self.assertFalse(os.path.exists(fname))
-            self.assertFalse(os.path.exists('%s.sha1' % fname))
+            self.assertFalse(os.path.exists(info_fname))
 
     def test_remove_base_file_original(self):
         with self._make_base_file() as fname:
             image_cache_manager = imagecache.ImageCacheManager()
             image_cache_manager.originals = [fname]
             image_cache_manager._remove_base_file(fname)
+            info_fname = virtutils.get_info_filename(fname)
 
             # Files are initially too new to delete
             self.assertTrue(os.path.exists(fname))
-            self.assertTrue(os.path.exists('%s.sha1' % fname))
+            self.assertTrue(os.path.exists(info_fname))
 
             # This file should stay longer than a resized image
             os.utime(fname, (-1, time.time() - 3601))
             image_cache_manager._remove_base_file(fname)
 
             self.assertTrue(os.path.exists(fname))
-            self.assertTrue(os.path.exists('%s.sha1' % fname))
+            self.assertTrue(os.path.exists(info_fname))
 
             # Originals don't stay forever though
             os.utime(fname, (-1, time.time() - 3600 * 25))
             image_cache_manager._remove_base_file(fname)
 
             self.assertFalse(os.path.exists(fname))
-            self.assertFalse(os.path.exists('%s.sha1' % fname))
+            self.assertFalse(os.path.exists(info_fname))
 
     def test_remove_base_file_dne(self):
         # This test is solely to execute the "does not exist" code path. We
         # don't expect the method being tested to do anything in this case.
         with utils.tempdir() as tmpdir:
+            self.flags(instances_path=tmpdir)
+            self.flags(image_info_filename_pattern=('$instances_path/'
+                                                    '%(image)s.info'))
+
             fname = os.path.join(tmpdir, 'aaa')
             image_cache_manager = imagecache.ImageCacheManager()
             image_cache_manager._remove_base_file(fname)
@@ -422,6 +514,10 @@ class ImageCacheManagerTestCase(test.TestCase):
     def test_remove_base_file_oserror(self):
         with self._intercept_log_messages() as stream:
             with utils.tempdir() as tmpdir:
+                self.flags(instances_path=tmpdir)
+                self.flags(image_info_filename_pattern=('$instances_path/'
+                                                        '%(image)s.info'))
+
                 fname = os.path.join(tmpdir, 'aaa')
 
                 os.mkdir(fname)
@@ -500,6 +596,10 @@ class ImageCacheManagerTestCase(test.TestCase):
         img = '123'
 
         with utils.tempdir() as tmpdir:
+            self.flags(instances_path=tmpdir)
+            self.flags(image_info_filename_pattern=('$instances_path/'
+                                                    '%(image)s.info'))
+
             fname = os.path.join(tmpdir, 'aaa')
 
             image_cache_manager = imagecache.ImageCacheManager()
@@ -565,13 +665,13 @@ class ImageCacheManagerTestCase(test.TestCase):
                         '/instance_path/instance-1/disk',
                         '/instance_path/instance-2/disk',
                         '/instance_path/instance-3/disk',
-                        '/instance_path/_base/%s.sha1' % hashed_42]:
+                        '/instance_path/_base/%s.info' % hashed_42]:
                 return True
 
             for p in base_file_list:
                 if path == fq_path(p):
                     return True
-                if path == fq_path(p) + '.sha1':
+                if path == fq_path(p) + '.info':
                     return False
 
             if path in ['/instance_path/_base/%s_sm' % hashed_1,
@@ -702,13 +802,12 @@ class ImageCacheManagerTestCase(test.TestCase):
                                                 '%(image)s.info'))
         base_filename = os.path.join(FLAGS.instances_path, '_base', hashed)
 
-        self.assertFalse(imagecache.is_valid_info_file('banana'))
-        self.assertFalse(imagecache.is_valid_info_file(
+        self.assertFalse(virtutils.is_valid_info_file('banana'))
+        self.assertFalse(virtutils.is_valid_info_file(
                 os.path.join(FLAGS.instances_path, '_base', '00000001')))
-        self.assertFalse(imagecache.is_valid_info_file(base_filename))
-        self.assertFalse(imagecache.is_valid_info_file(base_filename +
-                                                       '.sha1'))
-        self.assertTrue(imagecache.is_valid_info_file(base_filename + '.info'))
+        self.assertFalse(virtutils.is_valid_info_file(base_filename))
+        self.assertFalse(virtutils.is_valid_info_file(base_filename + '.sha1'))
+        self.assertTrue(virtutils.is_valid_info_file(base_filename + '.info'))
 
     def test_configured_checksum_path(self):
         with utils.tempdir() as tmpdir:
@@ -742,8 +841,8 @@ class ImageCacheManagerTestCase(test.TestCase):
             touch(base_filename)
             touch(base_filename + '.info')
             os.utime(base_filename + '.info', (old, old))
-            touch(base_filename + '.sha1')
-            os.utime(base_filename + '.sha1', (old, old))
+            touch(base_filename + '.info')
+            os.utime(base_filename + '.info', (old, old))
 
             image_cache_manager = imagecache.ImageCacheManager()
             image_cache_manager.verify_base_images(None)