Re-merging code for generating system-usages to get around bzr merge braindeadness.

6 files changed, 349 insertions, 0 deletions

diff --git a/bin/instance-usage-audit b/bin/instance-usage-audit
new file mode 100755
index 000000000..1124cf550
--- /dev/null
+++ b/bin/instance-usage-audit
@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Cron script to generate usage notifications for instances neither created
+   nor destroyed in a given time period. 
+   
+   Together with the notifications generated by compute on instance
+   create/delete/resize, over that ime period, this allows an external 
+   system consuming usage notification feeds to calculate instance usage 
+   for each tenant.
+
+   Time periods are specified like so:
+   <number>[mdy]   
+
+   1m = previous month. If the script is run April 1, it will generate usages
+        for March 1 thry March 31.
+   3m = 3 previous months.
+   90d = previous 90 days. 
+   1y = previous year. If run on Jan 1, it generates usages for 
+        Jan 1 thru Dec 31 of the previous year.      
+"""
+
+import datetime
+import gettext
+import os
+import sys
+import time
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')):
+    sys.path.insert(0, POSSIBLE_TOPDIR)
+
+gettext.install('nova', unicode=1)
+
+
+from nova import context
+from nova import db
+from nova import exception
+from nova import flags
+from nova import log as logging
+from nova import utils
+
+from nova.notifier import api as notifier_api
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('instance_usage_audit_period', '1m',
+                    'time period to generate instance usages for.')
+
+def time_period(period):
+    today = datetime.date.today()
+    unit = period[-1]
+    if unit not in 'mdy':
+        raise ValueError('Time period must be m, d, or y')
+    n = int(period[:-1])
+    if unit == 'm':
+       year = today.year - (n//12)
+       n = n % 12 
+       if n >= today.month:
+           year -= 1 
+           month = 12 + (today.month - n)
+       else:
+           month = today.month - n 
+       begin = datetime.datetime(day=1, month=month, year=year)
+       end = datetime.datetime(day=1, month=today.month, year=today.year)
+
+    elif unit == 'y':
+       begin = datetime.datetime(day=1, month=1, year=today.year - n)
+       end = datetime.datetime(day=1, month=1, year=today.year)
+      
+    elif unit == 'd':
+       b = today - datetime.timedelta(days=n)
+       begin = datetime.datetime(day=b.day, month=b.month, year=b.year)
+       end = datetime.datetime(day=today.day, 
+                               month=today.month, 
+                               year=today.year)
+ 
+    return (begin, end)
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    flags.FLAGS(sys.argv)
+    logging.setup()
+    begin, end = time_period(FLAGS.instance_usage_audit_period)
+    print "Creating usages for %s until %s" % (str(begin), str(end))
+    instances = db.instance_get_active_by_window(context.get_admin_context(),
+                                                 begin,
+                                                 end)
+    print "%s instances" % len(instances)
+    for instance_ref in instances:
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_ref['instance_type']['name'],
+              instance_type_id=instance_ref['instance_type_id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=instance_ref['image_id'],
+              audit_period_begining=str(begin),
+              audit_period_ending=str(end))
+        notifier_api.notify('compute.%s' % FLAGS.host,
+                            'compute.instance.exists',
+                            notifier_api.INFO,
+                            usage_info)
+
+     
diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 923feaa59..d0e9cdf95 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -50,10 +50,12 @@ from nova import flags
 from nova import log as logging
 from nova import manager
 from nova import network
+from nova import notifier
 from nova import rpc
 from nova import utils
 from nova import volume
 from nova.compute import power_state
+from nova.notifier import api as notifier_api
 from nova.virt import driver
 
 
@@ -275,6 +277,21 @@ class ComputeManager(manager.SchedulerDependentManager):
 
         self._update_launched_at(context, instance_id)
         self._update_state(context, instance_id)
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_ref['instance_type']['name'],
+              instance_type_id=instance_ref['instance_type_id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=instance_ref['image_id'])
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.create',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -327,6 +344,21 @@ class ComputeManager(manager.SchedulerDependentManager):
 
         # TODO(ja): should we keep it in a terminated state for a bit?
         self.db.instance_destroy(context, instance_id)
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_ref['instance_type']['name'],
+              instance_type_id=instance_ref['instance_type_id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=instance_ref['image_id'])
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.delete',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -354,6 +386,21 @@ class ComputeManager(manager.SchedulerDependentManager):
         self._update_image_id(context, instance_id, image_id)
         self._update_launched_at(context, instance_id)
         self._update_state(context, instance_id)
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_ref['instance_type']['name'],
+              instance_type_id=instance_ref['instance_type_id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=image_id)
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.rebuild',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -501,6 +548,21 @@ class ComputeManager(manager.SchedulerDependentManager):
         context = context.elevated()
         instance_ref = self.db.instance_get(context, instance_id)
         self.driver.destroy(instance_ref)
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_ref['instance_type']['name'],
+              instance_type_id=instance_ref['instance_type_id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=instance_ref['image_id'])
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.resize.confirm',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -548,6 +610,21 @@ class ComputeManager(manager.SchedulerDependentManager):
         self.driver.revert_resize(instance_ref)
         self.db.migration_update(context, migration_id,
                 {'status': 'reverted'})
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_type['name'],
+              instance_type_id=instance_type['id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=instance_ref['image_id'])
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.resize.revert',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -584,6 +661,23 @@ class ComputeManager(manager.SchedulerDependentManager):
                        'migration_id': migration_ref['id'],
                        'instance_id': instance_id, },
                 })
+        usage_info = dict(
+              tenant_id=instance_ref['project_id'],
+              user_id=instance_ref['user_id'],
+              instance_id=instance_ref['id'],
+              instance_type=instance_ref['instance_type']['name'],
+              instance_type_id=instance_ref['instance_type_id'],
+              new_instance_type=instance_type['name'],
+              new_instance_type_id=instance_type['id'],
+              display_name=instance_ref['display_name'],
+              created_at=str(instance_ref['created_at']),
+              launched_at=str(instance_ref['launched_at']) \
+                          if instance_ref['launched_at'] else '',
+              image_id=instance_ref['image_id'])
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.resize.prep',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
diff --git a/nova/db/api.py b/nova/db/api.py
index ef8aa1143..aca403856 100644
--- a/nova/db/api.py
+++ b/nova/db/api.py
@@ -422,6 +422,11 @@ def instance_get_all(context):
     return IMPL.instance_get_all(context)
 
 
+def instance_get_active_by_window(context, begin, end=None):
+    """Get instances active during a certain time window."""
+    return IMPL.instance_get_active_by_window(context, begin, end)
+
+
 def instance_get_all_by_user(context, user_id):
     """Get all instances."""
     return IMPL.instance_get_all_by_user(context, user_id)
diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py
index 3681f30db..873cfe4d1 100644
--- a/nova/db/sqlalchemy/api.py
+++ b/nova/db/sqlalchemy/api.py
@@ -879,6 +879,24 @@ def instance_get_all(context):
 
 
 @require_admin_context
+def instance_get_active_by_window(context, begin, end=None):
+    """Return instances that were continuously active over the given window"""
+    session = get_session()
+    query = session.query(models.Instance).\
+                   options(joinedload_all('fixed_ip.floating_ips')).\
+                   options(joinedload('security_groups')).\
+                   options(joinedload_all('fixed_ip.network')).\
+                   options(joinedload('instance_type')).\
+                   filter(models.Instance.launched_at < begin)
+    if end:
+        query = query.filter(or_(models.Instance.terminated_at == None,
+                                 models.Instance.terminated_at > end))
+    else:
+        query = query.filter(models.Instance.terminated_at == None)
+    return query.all()
+
+
+@require_admin_context
 def instance_get_all_by_user(context, user_id):
     session = get_session()
     return session.query(models.Instance).\
diff --git a/nova/notifier/test_notifier.py b/nova/notifier/test_notifier.py
new file mode 100644
index 000000000..d43f43e48
--- /dev/null
+++ b/nova/notifier/test_notifier.py
@@ -0,0 +1,28 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+from nova import flags
+from nova import log as logging
+
+FLAGS = flags.FLAGS
+
+NOTIFICATIONS = []
+
+
+def notify(message):
+    """Test notifier, stores notifications in memory for unittests."""
+    NOTIFICATIONS.append(message)
diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 55e7ae0c4..30a65a4b1 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -38,6 +38,7 @@ from nova.compute import manager as compute_manager
 from nova.compute import power_state
 from nova.db.sqlalchemy import models
 from nova.image import local
+from nova.notifier import test_notifier
 
 LOG = logging.getLogger('nova.tests.compute')
 FLAGS = flags.FLAGS
@@ -63,6 +64,7 @@ class ComputeTestCase(test.TestCase):
         super(ComputeTestCase, self).setUp()
         self.flags(connection_type='fake',
                    stub_network=True,
+                   notification_driver='nova.notifier.test_notifier',
                    network_manager='nova.network.manager.FlatManager')
         self.compute = utils.import_object(FLAGS.compute_manager)
         self.compute_api = compute.API()
@@ -70,6 +72,7 @@ class ComputeTestCase(test.TestCase):
         self.user = self.manager.create_user('fake', 'fake', 'fake')
         self.project = self.manager.create_project('fake', 'fake', 'fake')
         self.context = context.RequestContext('fake', 'fake', False)
+        test_notifier.NOTIFICATIONS = []
 
         def fake_show(meh, context, id):
             return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}}
@@ -305,6 +308,50 @@ class ComputeTestCase(test.TestCase):
         self.assert_(console)
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_run_instance_usage_notification(self):
+        """Ensure run instance generates apropriate usage notification"""
+        instance_id = self._create_instance()
+        self.compute.run_instance(self.context, instance_id)
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.create')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_id'], '1')
+        self.compute.terminate_instance(self.context, instance_id)
+
+    def test_terminate_usage_notification(self):
+        """Ensure terminate_instance generates apropriate usage notification"""
+        instance_id = self._create_instance()
+        self.compute.run_instance(self.context, instance_id)
+        test_notifier.NOTIFICATIONS = []
+        self.compute.terminate_instance(self.context, instance_id)
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.delete')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_id'], '1')
+
     def test_run_instance_existing(self):
         """Ensure failure when running an instance that already exists"""
         instance_id = self._create_instance()
@@ -334,6 +381,36 @@ class ComputeTestCase(test.TestCase):
 
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_resize_instance_notification(self):
+        """Ensure instance can be migrated/resized"""
+        instance_id = self._create_instance()
+        context = self.context.elevated()
+
+        self.compute.run_instance(self.context, instance_id)
+        test_notifier.NOTIFICATIONS = []
+
+        db.instance_update(self.context, instance_id, {'host': 'foo'})
+        self.compute.prep_resize(context, instance_id, 1)
+        migration_ref = db.migration_get_by_instance_and_status(context,
+                instance_id, 'pre-migrating')
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.resize.prep')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_id'], '1')
+        self.compute.terminate_instance(context, instance_id)
+
     def test_resize_instance(self):
         """Ensure instance can be migrated/resized"""
         instance_id = self._create_instance()