From a4cb456631f0e70685fbc361eced9baa477a8c56 Mon Sep 17 00:00:00 2001 From: Michael Kerrin Date: Tue, 16 Apr 2013 09:43:28 +0000 Subject: Volume IO usage gets reset to 0 after a reboot / crash Check for reset of block device stats before updating volume usage cache. If an instance is reboot or crashes, the results of block device stats for that domain get reset. Add check that if block device stats are lower than the current stats in the database, then add current values to totals and update. Change-Id: I3011872f8d80a1519771f8423001650dcbcc7f5b Fixes bug: 1169097 --- nova/tests/test_db_api.py | 156 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 137 insertions(+), 19 deletions(-) (limited to 'nova/tests') diff --git a/nova/tests/test_db_api.py b/nova/tests/test_db_api.py index 6dc3287bf..3c817a6a0 100644 --- a/nova/tests/test_db_api.py +++ b/nova/tests/test_db_api.py @@ -2664,16 +2664,18 @@ class InstanceDestroyConstraints(test.TestCase): class VolumeUsageDBApiTestCase(test.TestCase): + def setUp(self): super(VolumeUsageDBApiTestCase, self).setUp() self.user_id = 'fake' self.project_id = 'fake' self.context = context.RequestContext(self.user_id, self.project_id) + self.useFixture(test.TimeOverride()) + def test_vol_usage_update_no_totals_update(self): ctxt = context.get_admin_context() now = timeutils.utcnow() - timeutils.set_time_override(now) start_time = now - datetime.timedelta(seconds=10) refreshed_time = now - datetime.timedelta(seconds=5) @@ -2684,7 +2686,11 @@ class VolumeUsageDBApiTestCase(test.TestCase): 'curr_reads': 1000, 'curr_read_bytes': 2000, 'curr_writes': 3000, - 'curr_write_bytes': 4000}, + 'curr_write_bytes': 4000, + 'tot_reads': 0, + 'tot_read_bytes': 0, + 'tot_writes': 0, + 'tot_write_bytes': 0}, {'volume_id': u'2', 'instance_uuid': 'fake-instance-uuid2', 'project_id': 'fake-project-uuid2', @@ -2692,7 +2698,11 @@ class VolumeUsageDBApiTestCase(test.TestCase): 'curr_reads': 100, 'curr_read_bytes': 200, 'curr_writes': 300, - 'curr_write_bytes': 400}] + 'curr_write_bytes': 400, + 'tot_reads': 0, + 'tot_read_bytes': 0, + 'tot_writes': 0, + 'tot_write_bytes': 0}] def _compare(vol_usage, expected): for key, value in expected.items(): @@ -2725,26 +2735,11 @@ class VolumeUsageDBApiTestCase(test.TestCase): self.assertEqual(len(vol_usages), 2) _compare(vol_usages[0], expected_vol_usages[0]) _compare(vol_usages[1], expected_vol_usages[1]) - timeutils.clear_time_override() def test_vol_usage_update_totals_update(self): ctxt = context.get_admin_context() now = timeutils.utcnow() - timeutils.set_time_override(now) start_time = now - datetime.timedelta(seconds=10) - expected_vol_usages = {'volume_id': u'1', - 'project_id': 'fake-project-uuid', - 'user_id': 'fake-user-uuid', - 'instance_uuid': 'fake-instance-uuid', - 'availability_zone': 'fake-az', - 'tot_reads': 600, - 'tot_read_bytes': 800, - 'tot_writes': 1000, - 'tot_write_bytes': 1200, - 'curr_reads': 0, - 'curr_read_bytes': 0, - 'curr_writes': 0, - 'curr_write_bytes': 0} vol_usage = db.vol_usage_update(ctxt, 1, rd_req=100, rd_bytes=200, wr_req=300, wr_bytes=400, @@ -2752,6 +2747,10 @@ class VolumeUsageDBApiTestCase(test.TestCase): project_id='fake-project-uuid', user_id='fake-user-uuid', availability_zone='fake-az') + current_usage = db.vol_get_usage_by_time(ctxt, start_time)[0] + self.assertEqual(current_usage['tot_reads'], 0) + self.assertEqual(current_usage['curr_reads'], 100) + vol_usage = db.vol_usage_update(ctxt, 1, rd_req=200, rd_bytes=300, wr_req=400, wr_bytes=500, instance_id='fake-instance-uuid', @@ -2759,12 +2758,20 @@ class VolumeUsageDBApiTestCase(test.TestCase): user_id='fake-user-uuid', availability_zone='fake-az', update_totals=True) + current_usage = db.vol_get_usage_by_time(ctxt, start_time)[0] + self.assertEqual(current_usage['tot_reads'], 200) + self.assertEqual(current_usage['curr_reads'], 0) + vol_usage = db.vol_usage_update(ctxt, 1, rd_req=300, rd_bytes=400, wr_req=500, wr_bytes=600, instance_id='fake-instance-uuid', project_id='fake-project-uuid', availability_zone='fake-az', user_id='fake-user-uuid') + current_usage = db.vol_get_usage_by_time(ctxt, start_time)[0] + self.assertEqual(current_usage['tot_reads'], 200) + self.assertEqual(current_usage['curr_reads'], 300) + vol_usage = db.vol_usage_update(ctxt, 1, rd_req=400, rd_bytes=500, wr_req=600, wr_bytes=700, instance_id='fake-instance-uuid', @@ -2775,10 +2782,121 @@ class VolumeUsageDBApiTestCase(test.TestCase): vol_usages = db.vol_get_usage_by_time(ctxt, start_time) + expected_vol_usages = {'volume_id': u'1', + 'project_id': 'fake-project-uuid', + 'user_id': 'fake-user-uuid', + 'instance_uuid': 'fake-instance-uuid', + 'availability_zone': 'fake-az', + 'tot_reads': 600, + 'tot_read_bytes': 800, + 'tot_writes': 1000, + 'tot_write_bytes': 1200, + 'curr_reads': 0, + 'curr_read_bytes': 0, + 'curr_writes': 0, + 'curr_write_bytes': 0} + self.assertEquals(1, len(vol_usages)) for key, value in expected_vol_usages.items(): self.assertEqual(vol_usages[0][key], value) - timeutils.clear_time_override() + + def test_vol_usage_update_when_blockdevicestats_reset(self): + ctxt = context.get_admin_context() + now = timeutils.utcnow() + start_time = now - datetime.timedelta(seconds=10) + + vol_usages = db.vol_get_usage_by_time(ctxt, start_time) + self.assertEqual(len(vol_usages), 0) + + db.vol_usage_update(ctxt, 1, + rd_req=10000, rd_bytes=20000, + wr_req=30000, wr_bytes=40000, + instance_id='fake-instance-uuid1', + project_id='fake-project-uuid1', + availability_zone='fake-az', + user_id='fake-user-uuid1') + + # Instance rebooted or crashed. block device stats were reset and are + # less then the previous values + db.vol_usage_update(ctxt, 1, + rd_req=100, rd_bytes=200, + wr_req=300, wr_bytes=400, + instance_id='fake-instance-uuid1', + project_id='fake-project-uuid1', + availability_zone='fake-az', + user_id='fake-user-uuid1') + + db.vol_usage_update(ctxt, 1, + rd_req=200, rd_bytes=300, + wr_req=400, wr_bytes=500, + instance_id='fake-instance-uuid1', + project_id='fake-project-uuid1', + availability_zone='fake-az', + user_id='fake-user-uuid1') + + vol_usage = db.vol_get_usage_by_time(ctxt, start_time)[0] + expected_vol_usage = {'volume_id': u'1', + 'instance_uuid': 'fake-instance-uuid1', + 'project_id': 'fake-project-uuid1', + 'availability_zone': 'fake-az', + 'user_id': 'fake-user-uuid1', + 'curr_reads': 200, + 'curr_read_bytes': 300, + 'curr_writes': 400, + 'curr_write_bytes': 500, + 'tot_reads': 10000, + 'tot_read_bytes': 20000, + 'tot_writes': 30000, + 'tot_write_bytes': 40000} + for key, value in expected_vol_usage.items(): + self.assertEqual(vol_usage[key], value, key) + + def test_vol_usage_update_totals_update_when_blockdevicestats_reset(self): + # This is unlikely to happen, but could when a volume is detached + # right after a instance has rebooted / recovered and before + # the system polled and updated the volume usage cache table. + ctxt = context.get_admin_context() + now = timeutils.utcnow() + start_time = now - datetime.timedelta(seconds=10) + + vol_usages = db.vol_get_usage_by_time(ctxt, start_time) + self.assertEqual(len(vol_usages), 0) + + db.vol_usage_update(ctxt, 1, + rd_req=10000, rd_bytes=20000, + wr_req=30000, wr_bytes=40000, + instance_id='fake-instance-uuid1', + project_id='fake-project-uuid1', + availability_zone='fake-az', + user_id='fake-user-uuid1') + + # Instance rebooted or crashed. block device stats were reset and are + # less then the previous values + db.vol_usage_update(ctxt, 1, + rd_req=100, rd_bytes=200, + wr_req=300, wr_bytes=400, + instance_id='fake-instance-uuid1', + project_id='fake-project-uuid1', + availability_zone='fake-az', + user_id='fake-user-uuid1', + update_totals=True) + + vol_usage = db.vol_get_usage_by_time(ctxt, start_time)[0] + expected_vol_usage = {'volume_id': u'1', + 'instance_uuid': 'fake-instance-uuid1', + 'project_id': 'fake-project-uuid1', + 'availability_zone': 'fake-az', + 'user_id': 'fake-user-uuid1', + 'curr_reads': 0, + 'curr_read_bytes': 0, + 'curr_writes': 0, + 'curr_write_bytes': 0, + 'tot_reads': 10100, + 'tot_read_bytes': 20200, + 'tot_writes': 30300, + 'tot_write_bytes': 40400} + for key, value in expected_vol_usage.items(): + self.assertEqual(vol_usage[key], value, key) class TaskLogTestCase(test.TestCase): -- cgit