From 0ed62fb7affbda4a701c2175e95aa6f92038604c Mon Sep 17 00:00:00 2001 From: Peter Feiner Date: Wed, 19 Jun 2013 21:14:43 +0000 Subject: db.compute_node_update: ignore values['update_at'] When individual instances are updated (e.g., during spawn and terminate), ResourceTracker (in nova.compute.resource_tracker) calls compute_node_update with values=self.compute_node. Since self.compute_node is an instance of ComputeNode that was retrieved from the database, it has updated_at set. Since updated_at is in values, sqlalchemy doesn't automatically change the record's updated_at column (see nova.openstack.common.db.sqlalchemy.models.TimestampMixin). Moreover, since updated_at is set to the last value's updated_at, updated_at effectively doesn't change until values without updated_at are sent, which only happens during the periodic task that calls ResourceTracker.update_available_resources. Nova-scheduler relies on ComputeNode.updated_at to keep its model of available resources up-to-date. In particular, nova-scheduler doesn't play a role in instance termination, so it doesn't account for freed resources until ComputeNode.updated_at changes. Thus, between nova-compute's periodic calls to ResourceTracker.update_available_resources, nova-scheduler's model of available resources monotonically decreases. If, for example, a node has resources for 10 instances, and you manage to boot 10, terminate 10, then attempt to boot another before the end of the period, nova-scheduler won't schedule the new instance on the vacant node. Fixes bug #1194900. Note that f398b9e195cda582bad57396b097dec274384c07 fixed a separate issue (bug #1153778) related to ComputeNode.update_at being stale. Change-Id: Ifd1e56edfd811241816970715071876857de80d3 --- nova/db/sqlalchemy/api.py | 3 +-- nova/tests/db/test_db_api.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index fd79ae215..12d1a582f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -555,8 +555,7 @@ def compute_node_update(context, compute_id, values, prune_stats=False): # Always update this, even if there's going to be no other # changes in data. This ensures that we invalidate the # scheduler cache of compute node data in case of races. - if 'updated_at' not in values: - values['updated_at'] = timeutils.utcnow() + values['updated_at'] = timeutils.utcnow() convert_datetimes(values, 'created_at', 'deleted_at', 'updated_at') compute_ref.update(values) return compute_ref diff --git a/nova/tests/db/test_db_api.py b/nova/tests/db/test_db_api.py index 279481d87..48f448d22 100644 --- a/nova/tests/db/test_db_api.py +++ b/nova/tests/db/test_db_api.py @@ -4926,6 +4926,19 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin): self.assertNotEqual(self.item['updated_at'], item_updated['updated_at']) + def test_compute_node_update_override_updated_at(self): + # Update the record once so updated_at is set. + first = db.compute_node_update(self.ctxt, self.item['id'], + {'free_ram_mb': '12'}) + self.assertIsNotNone(first['updated_at']) + + # Update a second time. Make sure that the updated_at value we send + # is overridden. + second = db.compute_node_update(self.ctxt, self.item['id'], + {'updated_at': first.updated_at, + 'free_ram_mb': '13'}) + self.assertNotEqual(first['updated_at'], second['updated_at']) + def test_compute_node_stat_unchanged(self): # don't update unchanged stat values: stats = self.item['stats'] -- cgit