Recover from a lost data store connection

1 files changed, 12 insertions, 0 deletions

diff --git a/nova/service.py b/nova/service.py
index f1f90742f..7203430c6 100644
--- a/nova/service.py
+++ b/nova/service.py
@@ -24,17 +24,21 @@ import inspect
 import logging
 import os
 import sys
+import time
 
 from eventlet import event
 from eventlet import greenthread
 from eventlet import greenpool
 
+from sqlalchemy.exc import OperationalError
+
 from nova import context
 from nova import db
 from nova import exception
 from nova import flags
 from nova import rpc
 from nova import utils
+from nova.db.sqlalchemy import models
 
 
 FLAGS = flags.FLAGS
@@ -204,6 +208,14 @@ class Service(object):
                 self.model_disconnected = True
                 logging.exception(_("model server went away"))
 
+                try:
+                    models.register_models()
+                except OperationalError:
+                    logging.exception(_("Data store is unreachable."
+                        " Trying again in %d seconds.") %
+                            FLAGS.sql_retry_interval)
+                    time.sleep(FLAGS.sql_retry_interval)
+
 
 def serve(*services):
     argv = FLAGS(sys.argv)