From c3229871853ad98cb99eba5036aef0c6a5d51275 Mon Sep 17 00:00:00 2001 From: Boris Pavlovic Date: Mon, 17 Dec 2012 13:48:28 +0400 Subject: Database metadata performance optimizations In instance_metadata_update(): Fix bug: there was no transaction in method. Performance optimization: let N is number of metadata items. Was: *) 1 request to get all (M) existing in DB entries *) (M-L) requests to get all entries that don't exist in metadata *) (M-L) requests to soft delete all entries *) N request to try to get get metadata by key from DB *) N request to save Now: *) 1 requst to soft delete entries that exist in DB and don't exist in metadata *) 1 requst to get all(L) entries that exist in metadata and DB *) L request to update all L entries that exist in DB and metadata *) N-L request to create new entries. result 1 + 2*(M - L + N) -> 2 + N In instance_system_metadata_update(): Fix bug: there was no transaction in method. Same performance situation. In aggregate_metadata_get: Fix bug: there was no transaction at all in method. Performance optimization: Let N is number of metadata items. All request have check that Aggregate exist so each time there are 2 request. Was: *) 1 + 1 request to get all (M) entries that exist in DB *) (1 + 1)*(M-L) request to get each entry that exist in DB and doesn't exist in metadata *) (1 + 1)*(M-L) to soft delete entries *) N * (1 + 1) try to get each item that exist in metadata *) N * (1 + 1) update or save Now: *) 1 request to get query object if exist Aggregate *) 1 request to soft delete or entries that exist in DB and doesn't exit in metadata *) 1 request to get elements that exist in DB and in metadata *) L request to update existing metadata *) N - L request to create new metadata 2 + 4*(M - L) + 4*N -> 3 + N blueprint db-session-cleanup Change-Id: Ie14f89c567c971dece34cc3659ad3a48cf82acf6 --- nova/db/sqlalchemy/api.py | 156 ++++++++++++++++++++++++---------------------- 1 file changed, 80 insertions(+), 76 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 9cc3b64a1..541674d2f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -3794,43 +3794,42 @@ def instance_metadata_get_item(context, instance_uuid, key, session=None): @require_context def instance_metadata_update(context, instance_uuid, metadata, delete, session=None): + all_keys = metadata.keys() + synchronize_session = "fetch" if session is None: session = get_session() - # Set existing metadata to deleted if delete argument is True - if delete: - original_metadata = instance_metadata_get(context, instance_uuid, - session=session) - for meta_key, meta_value in original_metadata.iteritems(): - if meta_key not in metadata: - meta_ref = instance_metadata_get_item(context, instance_uuid, - meta_key, session) - meta_ref.update({'deleted': True}) - meta_ref.save(session=session) - - meta_ref = None - - # Now update all existing items with new values, or create new meta objects - for meta_key, meta_value in metadata.iteritems(): + synchronize_session = False + with session.begin(subtransactions=True): + if delete: + _instance_metadata_get_query(context, instance_uuid, + session=session).\ + filter(~models.InstanceMetadata.key.in_(all_keys)).\ + soft_delete(synchronize_session=synchronize_session) + + already_existing_keys = [] + meta_refs = _instance_metadata_get_query(context, instance_uuid, + session=session).\ + filter(models.InstanceMetadata.key.in_(all_keys)).\ + all() - # update the value whether it exists or not - item = {"value": meta_value} + for meta_ref in meta_refs: + already_existing_keys.append(meta_ref.key) + meta_ref.update({"value": metadata[meta_ref.key]}) - try: - meta_ref = instance_metadata_get_item(context, instance_uuid, - meta_key, session) - except exception.InstanceMetadataNotFound: + new_keys = set(all_keys) - set(already_existing_keys) + for key in new_keys: meta_ref = models.InstanceMetadata() - item.update({"key": meta_key, "instance_uuid": instance_uuid}) - - meta_ref.update(item) - meta_ref.save(session=session) + meta_ref.update({"key": key, "value": metadata[key], + "instance_uuid": instance_uuid}) + session.add(meta_ref) - return metadata + return metadata ####################### # System-owned metadata + def _instance_system_metadata_get_query(context, instance_uuid, session=None): return model_query(context, models.InstanceSystemMetadata, session=session).\ @@ -3866,39 +3865,36 @@ def _instance_system_metadata_get_item(context, instance_uuid, key, @require_context def instance_system_metadata_update(context, instance_uuid, metadata, delete, session=None): + all_keys = metadata.keys() + synchronize_session = "fetch" if session is None: session = get_session() + synchronize_session = False + with session.begin(subtransactions=True): + if delete: + _instance_system_metadata_get_query(context, instance_uuid, + session=session).\ + filter(~models.InstanceSystemMetadata.key.in_(all_keys)).\ + soft_delete(synchronize_session=synchronize_session) + + already_existing_keys = [] + meta_refs = _instance_system_metadata_get_query(context, instance_uuid, + session=session).\ + filter(models.InstanceSystemMetadata.key.in_(all_keys)).\ + all() - # Set existing metadata to deleted if delete argument is True - if delete: - original_metadata = instance_system_metadata_get( - context, instance_uuid, session=session) - for meta_key, meta_value in original_metadata.iteritems(): - if meta_key not in metadata: - meta_ref = _instance_system_metadata_get_item( - context, instance_uuid, meta_key, session) - meta_ref.update({'deleted': True}) - meta_ref.save(session=session) - - meta_ref = None - - # Now update all existing items with new values, or create new meta objects - for meta_key, meta_value in metadata.iteritems(): - - # update the value whether it exists or not - item = {"value": meta_value} + for meta_ref in meta_refs: + already_existing_keys.append(meta_ref.key) + meta_ref.update({"value": metadata[meta_ref.key]}) - try: - meta_ref = _instance_system_metadata_get_item( - context, instance_uuid, meta_key, session) - except exception.InstanceSystemMetadataNotFound: + new_keys = set(all_keys) - set(already_existing_keys) + for key in new_keys: meta_ref = models.InstanceSystemMetadata() - item.update({"key": meta_key, "instance_uuid": instance_uuid}) + meta_ref.update({"key": key, "value": metadata[key], + "instance_uuid": instance_uuid}) + session.add(meta_ref) - meta_ref.update(item) - meta_ref.save(session=session) - - return metadata + return metadata #################### @@ -4340,6 +4336,16 @@ def aggregate_get_all(context): return _aggregate_get_query(context, models.Aggregate).all() +@require_admin_context +def aggregate_metadata_get_query(context, aggregate_id, session=None, + read_deleted="yes"): + return model_query(context, + models.AggregateMetadata, + read_deleted=read_deleted, + session=session).\ + filter_by(aggregate_id=aggregate_id) + + @require_admin_context @require_aggregate_exists def aggregate_metadata_get(context, aggregate_id): @@ -4385,33 +4391,31 @@ def aggregate_metadata_get_item(context, aggregate_id, key, session=None): @require_aggregate_exists def aggregate_metadata_add(context, aggregate_id, metadata, set_delete=False): session = get_session() + all_keys = metadata.keys() + with session.begin(): + query = aggregate_metadata_get_query(context, aggregate_id, + session=session) + if set_delete: + query.filter(~models.AggregateMetadata.key.in_(all_keys)).\ + soft_delete(synchronize_session=False) - if set_delete: - original_metadata = aggregate_metadata_get(context, aggregate_id) - for meta_key, meta_value in original_metadata.iteritems(): - if meta_key not in metadata: - meta_ref = aggregate_metadata_get_item(context, aggregate_id, - meta_key, session) - meta_ref.update({'deleted': True}) - meta_ref.save(session=session) - - meta_ref = None + query = query.filter(models.AggregateMetadata.key.in_(all_keys)) + already_existing_keys = [] + for meta_ref in query.all(): + key = meta_ref.key + meta_ref.update({"value": metadata[key], + "deleted": False, + "deleted_at": None}) + already_existing_keys.append(key) - for meta_key, meta_value in metadata.iteritems(): - item = {"value": meta_value} - try: - meta_ref = aggregate_metadata_get_item(context, aggregate_id, - meta_key, session) - if meta_ref.deleted: - item.update({'deleted': False, 'deleted_at': None}) - except exception.AggregateMetadataNotFound: + for key in set(all_keys) - set(already_existing_keys): meta_ref = models.AggregateMetadata() - item.update({"key": meta_key, "aggregate_id": aggregate_id}) - - meta_ref.update(item) - meta_ref.save(session=session) + meta_ref.update({"key": key, + "value": metadata[key], + "aggregate_id": aggregate_id}) + session.add(meta_ref) - return metadata + return metadata @require_admin_context -- cgit