Implement backup with rotation and expose this functionality in the OS API.

7 files changed, 291 insertions, 37 deletions

diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py
index d43340e10..7ebf58023 100644
--- a/nova/api/openstack/images.py
+++ b/nova/api/openstack/images.py
@@ -90,31 +90,67 @@ class Controller(object):
         return webob.exc.HTTPNoContent()
 
     def create(self, req, body):
-        """Snapshot a server instance and save the image.
+        """Snapshot or backup a server instance and save the image.
+
+        Images now have an `image_type` associated with them, which can be
+        'snapshot' or the backup type, like 'daily' or 'weekly'.
+
+        If the image_type is backup-like, then the rotation factor can be
+        included and that will cause the oldest backups that exceed the
+        rotation factor to be deleted.
 
         :param req: `wsgi.Request` object
         """
+        def get_param(param):
+            try:
+                return body["image"][param]
+            except KeyError:
+                raise webob.exc.HTTPBadRequest(explanation="Missing required "
+                        "param: %s" % param)
+
         context = req.environ['nova.context']
         content_type = req.get_content_type()
 
         if not body:
             raise webob.exc.HTTPBadRequest()
 
+        image_type = body["image"].get("image_type", "snapshot")
+
         try:
             server_id = self._server_id_from_req(req, body)
-            image_name = body["image"]["name"]
         except KeyError:
             raise webob.exc.HTTPBadRequest()
 
+        image_name = get_param("name")
         props = self._get_extra_properties(req, body)
 
-        image = self._compute_service.snapshot(context, server_id,
-                                               image_name, props)
+        if image_type == "snapshot":
+            image = self._compute_service.snapshot(
+                        context, server_id, image_name,
+                        extra_properties=props)
+        elif image_type == "backup":
+            # NOTE(sirp): Unlike snapshot, backup is not a customer facing
+            # API call; rather, it's used by the internal backup scheduler
+            if not FLAGS.allow_admin_api:
+                raise webob.exc.HTTPBadRequest(
+                        explanation="Admin API Required")
+
+            backup_type = get_param("backup_type")
+            rotation = int(get_param("rotation"))
+
+            image = self._compute_service.backup(
+                        context, server_id, image_name,
+                        backup_type, rotation, extra_properties=props)
+        else:
+            LOG.error(_("Invalid image_type '%s' passed") % image_type)
+            raise webob.exc.HTTPBadRequest(explanation="Invalue image_type: "
+                   "%s" % image_type)
+
         return dict(image=self.get_builder(req).build(image, detail=True))
 
     def get_builder(self, request):
         """Indicates that you must use a Controller subclass."""
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def _server_id_from_req(self, req, data):
         raise NotImplementedError()
diff --git a/nova/compute/api.py b/nova/compute/api.py
index af18741b6..59a392e7a 100644
--- a/nova/compute/api.py
+++ b/nova/compute/api.py
@@ -711,19 +711,60 @@ class API(base.Base):
         raise exception.Error(_("Unable to find host for Instance %s")
                                 % instance_id)
 
+    def backup(self, context, instance_id, name, backup_type, rotation,
+               extra_properties=None):
+        """Backup the given instance
+
+        :param instance_id: nova.db.sqlalchemy.models.Instance.Id
+        :param name: name of the backup or snapshot
+            name = backup_type  # daily backups are called 'daily'
+        :param rotation: int representing how many backups to keep around;
+            None if rotation shouldn't be used (as in the case of snapshots)
+        :param extra_properties: dict of extra image properties to include
+        """
+        recv_meta = self._create_image(context, instance_id, name, 'backup',
+                            backup_type=backup_type, rotation=rotation,
+                            extra_properties=extra_properties)
+        return recv_meta
+
     def snapshot(self, context, instance_id, name, extra_properties=None):
         """Snapshot the given instance.
 
+        :param instance_id: nova.db.sqlalchemy.models.Instance.Id
+        :param name: name of the backup or snapshot
+        :param extra_properties: dict of extra image properties to include
+
         :returns: A dict containing image metadata
         """
-        properties = {'instance_id': str(instance_id),
+        return self._create_image(context, instance_id, name, 'snapshot',
+                                  extra_properties=extra_properties)
+
+    def _create_image(self, context, instance_id, name, image_type,
+                      backup_type=None, rotation=None, extra_properties=None):
+        """Create snapshot or backup for an instance on this host.
+
+        :param context: security context
+        :param instance_id: nova.db.sqlalchemy.models.Instance.Id
+        :param name: string for name of the snapshot
+        :param image_type: snapshot | backup
+        :param backup_type: daily | weekly
+        :param rotation: int representing how many backups to keep around;
+            None if rotation shouldn't be used (as in the case of snapshots)
+        :param extra_properties: dict of extra image properties to include
+
+        """
+        instance = db.api.instance_get(context, instance_id)
+        properties = {'instance_uuid': instance['uuid'],
                       'user_id': str(context.user_id),
-                      'image_state': 'creating'}
+                      'image_state': 'creating',
+                      'image_type': image_type,
+                      'backup_type': backup_type}
         properties.update(extra_properties or {})
         sent_meta = {'name': name, 'is_public': False,
                      'status': 'creating', 'properties': properties}
         recv_meta = self.image_service.create(context, sent_meta)
-        params = {'image_id': recv_meta['id']}
+        params = {'image_id': recv_meta['id'], 'image_type': image_type,
+                  'backup_type': backup_type, 'rotation': rotation}
         self._cast_compute_message('snapshot_instance', context, instance_id,
                                    params=params)
         return recv_meta
diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 98e02f5b2..40a640083 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -46,6 +46,7 @@ from eventlet import greenthread
 
 from nova import exception
 from nova import flags
+import nova.image
 from nova import log as logging
 from nova import manager
 from nova import network
@@ -506,8 +507,19 @@ class ComputeManager(manager.SchedulerDependentManager):
         self._update_state(context, instance_id)
 
     @exception.wrap_exception
-    def snapshot_instance(self, context, instance_id, image_id):
-        """Snapshot an instance on this host."""
+    def snapshot_instance(self, context, instance_id, image_id,
+                          image_type='snapshot', backup_type=None,
+                          rotation=None):
+        """Snapshot an instance on this host.
+
+        :param context: security context
+        :param instance_id: nova.db.sqlalchemy.models.Instance.Id
+        :param image_id: glance.db.sqlalchemy.models.Image.Id
+        :param image_type: snapshot | backup
+        :param backup_type: daily | weekly
+        :param rotation: int representing how many backups to keep around;
+            None if rotation shouldn't be used (as in the case of snapshots)
+        """
         context = context.elevated()
         instance_ref = self.db.instance_get(context, instance_id)
 
@@ -527,6 +539,65 @@ class ComputeManager(manager.SchedulerDependentManager):
 
         self.driver.snapshot(instance_ref, image_id)
 
+        if image_type == 'snapshot':
+            if rotation:
+                raise exception.ImageRotationNotAllowed()
+        elif image_type == 'backup':
+            if rotation:
+                instance_uuid = instance_ref['uuid']
+                self.rotate_backups(context, instance_uuid, backup_type,
+                                    rotation)
+            else:
+                raise exception.RotationRequiredForBackup()
+        else:
+            raise Exception(_('Image type not recognized %s') % image_type)
+
+    def rotate_backups(self, context, instance_uuid, backup_type, rotation):
+        """Delete excess backups associated to an instance.
+
+        Instances are allowed a fixed number of backups (the rotation number);
+        this method deletes the oldest backups that exceed the rotation
+        threshold.
+
+        :param context: security context
+        :param instance_uuid: string representing uuid of instance
+        :param backup_type: daily | weekly
+        :param rotation: int representing how many backups to keep around;
+            None if rotation shouldn't be used (as in the case of snapshots)
+        """
+        # NOTE(jk0): Eventually extract this out to the ImageService?
+        def fetch_images():
+            images = []
+            marker = None
+            while True:
+                batch = image_service.detail(context, filters=filters,
+                        marker=marker, sort_key='created_at', sort_dir='desc')
+                if not batch:
+                    break
+                images += batch
+                marker = batch[-1]['id']
+            return images
+
+        image_service = nova.image.get_default_image_service()
+        filters = {'property-image_type': 'backup',
+                   'property-backup_type': backup_type,
+                   'property-instance_uuid': instance_uuid}
+
+        images = fetch_images()
+        num_images = len(images)
+        LOG.debug(_("Found %(num_images)d images (rotation: %(rotation)d)"
+                    % locals()))
+        if num_images > rotation:
+            # NOTE(sirp): this deletes all backups that exceed the rotation
+            # limit
+            excess = len(images) - rotation
+            LOG.debug(_("Rotating out %d backups" % excess))
+            for i in xrange(excess):
+                image = images.pop()
+                image_id = image['id']
+                LOG.debug(_("Deleting image %d" % image_id))
+                image_service.delete(context, image_id)
+
     @exception.wrap_exception
     @checks_instance_lock
     def set_admin_password(self, context, instance_id, new_pass=None):
diff --git a/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py b/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py
index cb3c73170..7e51d93b7 100644
--- a/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py
+++ b/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py
@@ -58,7 +58,7 @@ provider_fw_rules = Table('provider_fw_rules', meta,
         Column('to_port', Integer()),
         Column('cidr',
                String(length=255, convert_unicode=False, assert_unicode=None,
-                      unicode_error=None, _warn_on_bytestring=False)))
+                    unicode_error=None, _warn_on_bytestring=False)))
 
 
 def upgrade(migrate_engine):
diff --git a/nova/exception.py b/nova/exception.py
index ef50b144f..5d02e3179 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -558,6 +558,14 @@ class GlobalRoleNotAllowed(NotAllowed):
     message = _("Unable to use global role %(role_id)s")
 
 
+class ImageRotationNotAllowed(NovaException):
+    message = _("Rotation is not allowed for snapshots")
+
+
+class RotationRequiredForBackup(NovaException):
+    message = _("Rotation param is required for backup image_type")
+
+
 #TODO(bcwaldon): EOL this exception!
 class Duplicate(NovaException):
     pass
diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py
index eeb96cb12..26b1de818 100644
--- a/nova/tests/api/openstack/fakes.py
+++ b/nova/tests/api/openstack/fakes.py
@@ -147,6 +147,16 @@ def stub_out_compute_api_snapshot(stubs):
     stubs.Set(nova.compute.API, 'snapshot', snapshot)
 
 
+def stub_out_compute_api_backup(stubs):
+    def backup(self, context, instance_id, name, backup_type, rotation,
+               extra_properties=None):
+        props = dict(instance_id=instance_id, instance_ref=instance_id,
+                     backup_type=backup_type, rotation=rotation)
+        props.update(extra_properties or {})
+        return dict(id='123', status='ACTIVE', name=name, properties=props)
+    stubs.Set(nova.compute.API, 'backup', backup)
+
+
 def stub_out_glance_add_image(stubs, sent_to_glance):
     """
     We return the metadata sent to glance by modifying the sent_to_glance dict
diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py
index 446d68e9e..f6115c01e 100644
--- a/nova/tests/api/openstack/test_images.py
+++ b/nova/tests/api/openstack/test_images.py
@@ -340,6 +340,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         self.fixtures = self._make_image_fixtures()
         fakes.stub_out_glance(self.stubs, initial_fixtures=self.fixtures)
         fakes.stub_out_compute_api_snapshot(self.stubs)
+        fakes.stub_out_compute_api_backup(self.stubs)
 
     def tearDown(self):
         """Run after each test."""
@@ -364,10 +365,10 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         response_list = response_dict["images"]
 
         expected = [{'id': 123, 'name': 'public image'},
-                    {'id': 124, 'name': 'queued backup'},
-                    {'id': 125, 'name': 'saving backup'},
-                    {'id': 126, 'name': 'active backup'},
-                    {'id': 127, 'name': 'killed backup'},
+                    {'id': 124, 'name': 'queued snapshot'},
+                    {'id': 125, 'name': 'saving snapshot'},
+                    {'id': 126, 'name': 'active snapshot'},
+                    {'id': 127, 'name': 'killed snapshot'},
                     {'id': 129, 'name': None}]
 
         self.assertDictListMatch(response_list, expected)
@@ -617,14 +618,14 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         },
         {
             'id': 124,
-            'name': 'queued backup',
+            'name': 'queued snapshot',
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
             'status': 'QUEUED',
         },
         {
             'id': 125,
-            'name': 'saving backup',
+            'name': 'saving snapshot',
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
             'status': 'SAVING',
@@ -632,14 +633,14 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         },
         {
             'id': 126,
-            'name': 'active backup',
+            'name': 'active snapshot',
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
             'status': 'ACTIVE'
         },
         {
             'id': 127,
-            'name': 'killed backup',
+            'name': 'killed snapshot',
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
             'status': 'FAILED',
@@ -684,7 +685,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         },
         {
             'id': 124,
-            'name': 'queued backup',
+            'name': 'queued snapshot',
             'serverRef': "http://localhost:8774/v1.1/servers/42",
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
@@ -706,7 +707,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         },
         {
             'id': 125,
-            'name': 'saving backup',
+            'name': 'saving snapshot',
             'serverRef': "http://localhost:8774/v1.1/servers/42",
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
@@ -729,7 +730,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         },
         {
             'id': 126,
-            'name': 'active backup',
+            'name': 'active snapshot',
             'serverRef': "http://localhost:8774/v1.1/servers/42",
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
@@ -751,7 +752,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         },
         {
             'id': 127,
-            'name': 'killed backup',
+            'name': 'killed snapshot',
             'serverRef': "http://localhost:8774/v1.1/servers/42",
             'updated': self.NOW_API_FORMAT,
             'created': self.NOW_API_FORMAT,
@@ -969,8 +970,48 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         self.assertEqual(res.status_int, 404)
 
     def test_create_image(self):
+        body = dict(image=dict(serverId='123', name='Snapshot 1'))
+        req = webob.Request.blank('/v1.0/images')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers["content-type"] = "application/json"
+        response = req.get_response(fakes.wsgi_app())
+        self.assertEqual(200, response.status_int)
+
+    def test_create_snapshot_no_name(self):
+        """Name is required for snapshots"""
+        body = dict(image=dict(serverId='123'))
+        req = webob.Request.blank('/v1.0/images')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers["content-type"] = "application/json"
+        response = req.get_response(fakes.wsgi_app())
+        self.assertEqual(400, response.status_int)
+
+    def test_create_backup_no_name(self):
+        """Name is also required for backups"""
+        body = dict(image=dict(serverId='123', image_type='backup',
+                               backup_type='daily', rotation=1))
+        req = webob.Request.blank('/v1.0/images')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers["content-type"] = "application/json"
+        response = req.get_response(fakes.wsgi_app())
+        self.assertEqual(400, response.status_int)
+
+    def test_create_backup_with_rotation_and_backup_type(self):
+        """The happy path for creating backups
 
-        body = dict(image=dict(serverId='123', name='Backup 1'))
+        Creating a backup is an admin-only operation, as opposed to snapshots
+        which are available to anybody.
+        """
+        # FIXME(sirp): teardown needed?
+        FLAGS.allow_admin_api = True
+
+        # FIXME(sirp): should the fact that backups are admin_only be a FLAG
+        body = dict(image=dict(serverId='123', image_type='backup',
+                               name='Backup 1',
+                               backup_type='daily', rotation=1))
         req = webob.Request.blank('/v1.0/images')
         req.method = 'POST'
         req.body = json.dumps(body)
@@ -978,9 +1019,54 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
         response = req.get_response(fakes.wsgi_app())
         self.assertEqual(200, response.status_int)
 
+    def test_create_backup_no_rotation(self):
+        """Rotation is required for backup requests"""
+        # FIXME(sirp): teardown needed?
+        FLAGS.allow_admin_api = True
+
+        # FIXME(sirp): should the fact that backups are admin_only be a FLAG
+        body = dict(image=dict(serverId='123', name='daily',
+                               image_type='backup', backup_type='daily'))
+        req = webob.Request.blank('/v1.0/images')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers["content-type"] = "application/json"
+        response = req.get_response(fakes.wsgi_app())
+        self.assertEqual(400, response.status_int)
+
+    def test_create_backup_no_backup_type(self):
+        """Backup Type (daily or weekly) is required for backup requests"""
+        # FIXME(sirp): teardown needed?
+        FLAGS.allow_admin_api = True
+
+        # FIXME(sirp): should the fact that backups are admin_only be a FLAG
+        body = dict(image=dict(serverId='123', name='daily',
+                               image_type='backup', rotation=1))
+        req = webob.Request.blank('/v1.0/images')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers["content-type"] = "application/json"
+        response = req.get_response(fakes.wsgi_app())
+        self.assertEqual(400, response.status_int)
+
+    def test_create_image_with_invalid_image_type(self):
+        """Valid image_types are snapshot | daily | weekly"""
+        # FIXME(sirp): teardown needed?
+        FLAGS.allow_admin_api = True
+
+        # FIXME(sirp): should the fact that backups are admin_only be a FLAG
+        body = dict(image=dict(serverId='123', image_type='monthly',
+                               rotation=1))
+        req = webob.Request.blank('/v1.0/images')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers["content-type"] = "application/json"
+        response = req.get_response(fakes.wsgi_app())
+        self.assertEqual(400, response.status_int)
+
     def test_create_image_no_server_id(self):
 
-        body = dict(image=dict(name='Backup 1'))
+        body = dict(image=dict(name='Snapshot 1'))
         req = webob.Request.blank('/v1.0/images')
         req.method = 'POST'
         req.body = json.dumps(body)
@@ -990,7 +1076,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
 
     def test_create_image_v1_1(self):
 
-        body = dict(image=dict(serverRef='123', name='Backup 1'))
+        body = dict(image=dict(serverRef='123', name='Snapshot 1'))
         req = webob.Request.blank('/v1.1/images')
         req.method = 'POST'
         req.body = json.dumps(body)
@@ -1024,7 +1110,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
 
     def test_create_image_v1_1_xml_serialization(self):
 
-        body = dict(image=dict(serverRef='123', name='Backup 1'))
+        body = dict(image=dict(serverRef='123', name='Snapshot 1'))
         req = webob.Request.blank('/v1.1/images')
         req.method = 'POST'
         req.body = json.dumps(body)
@@ -1038,7 +1124,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
             <image
                    created="None"
                    id="123"
-                   name="Backup 1"
+                   name="Snapshot 1"
                    serverRef="http://localhost/v1.1/servers/123"
                    status="ACTIVE"
                    updated="None"
@@ -1057,7 +1143,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
 
     def test_create_image_v1_1_no_server_ref(self):
 
-        body = dict(image=dict(name='Backup 1'))
+        body = dict(image=dict(name='Snapshot 1'))
         req = webob.Request.blank('/v1.1/images')
         req.method = 'POST'
         req.body = json.dumps(body)
@@ -1084,19 +1170,21 @@ class ImageControllerWithGlanceServiceTest(test.TestCase):
                     status='active', properties={})
         image_id += 1
 
-        # Backup for User 1
+        # Snapshot for User 1
         server_ref = 'http://localhost:8774/v1.1/servers/42'
-        backup_properties = {'instance_ref': server_ref, 'user_id': '1'}
+        snapshot_properties = {'instance_ref': server_ref, 'user_id': '1'}
         for status in ('queued', 'saving', 'active', 'killed'):
-            add_fixture(id=image_id, name='%s backup' % status,
+            add_fixture(id=image_id, name='%s snapshot' % status,
                         is_public=False, status=status,
-                        properties=backup_properties)
+                        properties=snapshot_properties)
             image_id += 1
 
-        # Backup for User 2
-        other_backup_properties = {'instance_id': '43', 'user_id': '2'}
-        add_fixture(id=image_id, name='someone elses backup', is_public=False,
-                    status='active', properties=other_backup_properties)
+        # Snapshot for User 2
+        other_snapshot_properties = {'instance_id': '43', 'user_id': '2'}
+        add_fixture(id=image_id, name='someone elses snapshot',
+                    is_public=False, status='active',
+                    properties=other_snapshot_properties)
+
         image_id += 1
 
         # Image without a name