Use InnoDB for MySQL

This change adds a migration to convert any non-InnoDB tables to
InnoDB.

On some systems, the default engine is MyISAM, which doesn't
support features used by Keystone (foreign keys).

The approach is the same as what's used in Nova. A test is added
to ensure that all tables use InnoDB after migration. The test
passes when all the tables are mysql_engine='InnoDB'. This is
accomplished by adding a new migration that migrates all the
tables that aren't InnoDB to InnoDB.

Fixes bug 1191110.

Change-Id: I220f7642f5468c5cf4194f248210f90ff983b6e5

3 files changed, 183 insertions, 0 deletions

diff --git a/keystone/common/sql/migrate_repo/versions/027_set_engine_mysql_innodb.py b/keystone/common/sql/migrate_repo/versions/027_set_engine_mysql_innodb.py
new file mode 100644
index 00000000..ca8ccd08
--- /dev/null
+++ b/keystone/common/sql/migrate_repo/versions/027_set_engine_mysql_innodb.py
@@ -0,0 +1,143 @@
+
+import sqlalchemy as sql
+from sqlalchemy import MetaData
+
+from keystone.common.sql import migration_helpers
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine;
+    # bind migrate_engine to your metadata
+
+    if migrate_engine.name != 'mysql':
+        # InnoDB / MyISAM only applies to MySQL.
+        return
+
+    # This is a list of all the tables that might have been created with MyISAM
+    # rather than InnoDB.
+    tables = [
+        'credential',
+        'domain',
+        'ec2_credential',
+        'endpoint',
+        'group',
+        'group_domain_metadata',
+        'group_project_metadata',
+        'policy',
+        'project',
+        'role',
+        'service',
+        'token',
+        'trust',
+        'trust_role',
+        'user',
+        'user_domain_metadata',
+        'user_group_membership',
+        'user_project_metadata',
+    ]
+
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    domain_table = sql.Table('domain', meta, autoload=True)
+    endpoint_table = sql.Table('endpoint', meta, autoload=True)
+    group_table = sql.Table('group', meta, autoload=True)
+    group_domain_metadata_table = sql.Table('group_domain_metadata', meta,
+                                            autoload=True)
+    group_project_metadata_table = sql.Table('group_project_metadata', meta,
+                                             autoload=True)
+    project_table = sql.Table('project', meta, autoload=True)
+    service_table = sql.Table('service', meta, autoload=True)
+    user_table = sql.Table('user', meta, autoload=True)
+    user_domain_metadata_table = sql.Table('user_domain_metadata', meta,
+                                           autoload=True)
+    user_group_membership_table = sql.Table('user_group_membership', meta,
+                                            autoload=True)
+
+    # Mapping of table name to the constraints on that table,
+    # so we can create them.
+    table_constraints = {
+        'endpoint': [{'table': endpoint_table,
+                      'fk_column': 'service_id',
+                      'ref_column': service_table.c.id},
+                     ],
+        'group': [{'table': group_table,
+                   'fk_column': 'domain_id',
+                   'ref_column': domain_table.c.id},
+                  ],
+        'group_domain_metadata': [{'table': group_domain_metadata_table,
+                                   'fk_column': 'domain_id',
+                                   'ref_column': domain_table.c.id},
+                                  ],
+        'group_project_metadata': [{'table': group_project_metadata_table,
+                                    'fk_column': 'project_id',
+                                    'ref_column': project_table.c.id},
+                                   ],
+        'project': [{'table': project_table,
+                     'fk_column': 'domain_id',
+                     'ref_column': domain_table.c.id},
+                    ],
+        'user': [{'table': user_table,
+                  'fk_column': 'domain_id',
+                  'ref_column': domain_table.c.id},
+                 ],
+        'user_domain_metadata': [{'table': user_domain_metadata_table,
+                                  'fk_column': 'domain_id',
+                                  'ref_column': domain_table.c.id},
+                                 ],
+        'user_group_membership': [{'table': user_group_membership_table,
+                                   'fk_column': 'user_id',
+                                   'ref_column': user_table.c.id},
+                                  {'table': user_group_membership_table,
+                                   'fk_column': 'group_id',
+                                   'ref_column': group_table.c.id},
+                                  ],
+        'user_project_metadata': [{'table': group_project_metadata_table,
+                                   'fk_column': 'project_id',
+                                   'ref_column': project_table.c.id},
+                                  ],
+    }
+
+    # Maps a table name to the tables that reference it as a FK constraint
+    # (See the map above).
+    ref_tables_map = {
+        'service': ['endpoint', ],
+        'domain': ['group', 'group_domain_metadata', 'project', 'user',
+                   'user_domain_metadata', ],
+        'project': ['group_project_metadata', 'user_project_metadata', ],
+        'user': ['user_group_membership', ],
+        'group': ['user_group_membership', ],
+    }
+
+    # The names of tables that need to have their FKs added.
+    fk_table_names = set()
+
+    d = migrate_engine.execute("SHOW TABLE STATUS WHERE Engine!='InnoDB';")
+    for row in d.fetchall():
+        table_name = row[0]
+
+        if table_name not in tables:
+            # Skip this table since it's not a Keystone table.
+            continue
+
+        migrate_engine.execute("ALTER TABLE `%s` Engine=InnoDB" % table_name)
+
+        # Will add the FKs to the table if any of
+        # a) the table itself was converted
+        # b) the tables that the table referenced were converted
+
+        if table_name in table_constraints:
+            fk_table_names.add(table_name)
+
+        ref_tables = ref_tables_map.get(table_name, [])
+        for other_table_name in ref_tables:
+            fk_table_names.add(other_table_name)
+
+    # Now add all the FK constraints to those tables
+    for table_name in fk_table_names:
+        constraints = table_constraints.get(table_name)
+        migration_helpers.add_constraints(constraints)
+
+
+def downgrade(migrate_engine):
+    pass
diff --git a/keystone/common/sql/migration_helpers.py b/keystone/common/sql/migration_helpers.py
index fb2c8095..8a581924 100644
--- a/keystone/common/sql/migration_helpers.py
+++ b/keystone/common/sql/migration_helpers.py
@@ -52,6 +52,19 @@ def remove_constraints(constraints):
 
 def add_constraints(constraints):
     for constraint_def in constraints:
+
+        if constraint_def['table'].kwargs.get('mysql_engine') == 'MyISAM':
+            # Don't try to create constraint when using MyISAM because it's
+            # not supported.
+            continue
+
+        ref_col = constraint_def['ref_column']
+        ref_engine = ref_col.table.kwargs.get('mysql_engine')
+        if ref_engine == 'MyISAM':
+            # Don't try to create constraint when using MyISAM because it's
+            # not supported.
+            continue
+
         migrate.ForeignKeyConstraint(
             columns=[getattr(constraint_def['table'].c,
                              constraint_def['fk_column'])],
diff --git a/tests/test_sql_upgrade.py b/tests/test_sql_upgrade.py
index 21db6ade..e7e34b4b 100644
--- a/tests/test_sql_upgrade.py
+++ b/tests/test_sql_upgrade.py
@@ -506,6 +506,10 @@ class SqlUpgradeTests(test.TestCase):
 
     def test_downgrade_to_0(self):
         self.upgrade(self.max_version)
+
+        if self.engine.name == 'mysql':
+            self._mysql_check_all_tables_innodb()
+
         self.downgrade(0)
         for table_name in ["user", "token", "role", "user_tenant_membership",
                            "metadata"]:
@@ -961,3 +965,26 @@ class SqlUpgradeTests(test.TestCase):
         for ver, change in changeset:
             self.schema.runchange(ver, change, changeset.step)
         self.assertEqual(self.schema.version, version)
+
+    def _mysql_check_all_tables_innodb(self):
+        database = self.engine.url.database
+
+        connection = self.engine.connect()
+        # sanity check
+        total = connection.execute("SELECT count(*) "
+                                   "from information_schema.TABLES "
+                                   "where TABLE_SCHEMA='%(database)s'" %
+                                   locals())
+        self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?")
+
+        noninnodb = connection.execute("SELECT table_name "
+                                       "from information_schema.TABLES "
+                                       "where TABLE_SCHEMA='%(database)s' "
+                                       "and ENGINE!='InnoDB' "
+                                       "and TABLE_NAME!='migrate_version'" %
+                                       locals())
+        names = [x[0] for x in noninnodb]
+        self.assertEqual(names, [],
+                         "Non-InnoDB tables exist")
+
+        connection.close()