Split endpoint records in SQL by interface

This migrates the SQL backend such that v2 endpoints containing up to 3
URL's (public, internal and admin) stored in 'extra' are split into
unique endpoints.

Because legacy "endpoints" (each having publicUrl, internalUrl and
adminUrl) are no longer conceptually identical to v3's "endpoints" (each
having an interface and a url), new ID's are assigned to each entity and
each API continues to operate using with independent sets of endpoint
ID's.

Endpoints created on the v3 API are not exposed on the v2 API.

Change-Id: I2ba59d55907313ae65e908585fc49be0c4ce899a

1 files changed, 163 insertions, 28 deletions

diff --git a/tests/test_sql_upgrade.py b/tests/test_sql_upgrade.py
index fac9986f..9dcaaeff 100644
--- a/tests/test_sql_upgrade.py
+++ b/tests/test_sql_upgrade.py
@@ -16,10 +16,10 @@
 
 import copy
 import json
+import uuid
 
 from migrate.versioning import api as versioning_api
 import sqlalchemy
-from sqlalchemy.orm import sessionmaker
 
 from keystone.common import sql
 from keystone import config
@@ -27,6 +27,7 @@ from keystone import test
 from keystone.common.sql import migration
 import default_fixtures
 
+
 CONF = config.CONF
 
 
@@ -38,7 +39,11 @@ class SqlUpgradeTests(test.TestCase):
                      test.testsdir('backend_sql.conf')])
 
         # create and share a single sqlalchemy engine for testing
-        self.engine = sql.Base().get_engine(allow_global_engine=False)
+        base = sql.Base()
+        self.engine = base.get_engine(allow_global_engine=False)
+        self.Session = base.get_sessionmaker(
+            engine=self.engine,
+            autocommit=False)
         self.metadata = sqlalchemy.MetaData()
 
         # populate the engine with tables & fixtures
@@ -64,9 +69,7 @@ class SqlUpgradeTests(test.TestCase):
         self.assertEqual(expected_cols, actual_cols, '%s table' % table_name)
 
     def test_upgrade_0_to_1(self):
-        self.assertEqual(self.schema.version, 0, "DB is at version 0")
-        self._migrate(self.repo_path, 1)
-        self.assertEqual(self.schema.version, 1, "DB is at version 1")
+        self.upgrade(1)
         self.assertTableColumns("user", ["id", "name", "extra"])
         self.assertTableColumns("tenant", ["id", "name", "extra"])
         self.assertTableColumns("role", ["id", "name"])
@@ -76,23 +79,18 @@ class SqlUpgradeTests(test.TestCase):
         self.populate_user_table()
 
     def test_upgrade_5_to_6(self):
-        self._migrate(self.repo_path, 5)
-        self.assertEqual(self.schema.version, 5)
+        self.upgrade(5)
         self.assertTableDoesNotExist('policy')
 
-        self._migrate(self.repo_path, 6)
-        self.assertEqual(self.schema.version, 6)
+        self.upgrade(6)
         self.assertTableExists('policy')
         self.assertTableColumns('policy', ['id', 'type', 'blob', 'extra'])
 
     def test_upgrade_7_to_9(self):
-
-        self.assertEqual(self.schema.version, 0)
-        self._migrate(self.repo_path, 7)
+        self.upgrade(7)
         self.populate_user_table()
         self.populate_tenant_table()
-        self._migrate(self.repo_path, 9)
-        self.assertEqual(self.schema.version, 9)
+        self.upgrade(9)
         self.assertTableColumns("user",
                                 ["id", "name", "extra", "password",
                                  "enabled"])
@@ -103,8 +101,7 @@ class SqlUpgradeTests(test.TestCase):
         self.assertTableColumns("user_tenant_membership",
                                 ["user_id", "tenant_id"])
         self.assertTableColumns("metadata", ["user_id", "tenant_id", "data"])
-        maker = sessionmaker(bind=self.engine)
-        session = maker()
+        session = self.Session()
         user_table = sqlalchemy.Table("user",
                                       self.metadata,
                                       autoload=True)
@@ -120,25 +117,155 @@ class SqlUpgradeTests(test.TestCase):
         session.commit()
 
     def test_downgrade_9_to_7(self):
-        self.assertEqual(self.schema.version, 0)
-        self._migrate(self.repo_path, 9)
-        self._migrate(self.repo_path, 7, False)
+        self.upgrade(9)
+        self.downgrade(7)
+
+    def test_upgrade_9_to_12(self):
+        self.upgrade(9)
+
+        service_extra = {
+            'name': uuid.uuid4().hex,
+        }
+        service = {
+            'id': uuid.uuid4().hex,
+            'type': uuid.uuid4().hex,
+            'extra': json.dumps(service_extra),
+        }
+        endpoint_extra = {
+            'publicurl': uuid.uuid4().hex,
+            'internalurl': uuid.uuid4().hex,
+            'adminurl': uuid.uuid4().hex,
+        }
+        endpoint = {
+            'id': uuid.uuid4().hex,
+            'region': uuid.uuid4().hex,
+            'service_id': service['id'],
+            'extra': json.dumps(endpoint_extra),
+        }
+
+        session = self.Session()
+        self.insert_dict(session, 'service', service)
+        self.insert_dict(session, 'endpoint', endpoint)
+        session.commit()
+
+        self.upgrade(12)
+
+        self.assertTableColumns(
+            'service',
+            ['id', 'type', 'extra'])
+        self.assertTableColumns(
+            'endpoint',
+            ['id', 'legacy_endpoint_id', 'interface', 'region', 'service_id',
+             'url', 'extra'])
+
+        endpoint_table = sqlalchemy.Table(
+            'endpoint', self.metadata, autoload=True)
+
+        session = self.Session()
+        self.assertEqual(session.query(endpoint_table).count(), 3)
+        for interface in ['public', 'internal', 'admin']:
+            q = session.query(endpoint_table)
+            q = q.filter_by(legacy_endpoint_id=endpoint['id'])
+            q = q.filter_by(interface=interface)
+            ref = q.one()
+            self.assertNotEqual(ref.id, endpoint['id'])
+            self.assertEqual(ref.legacy_endpoint_id, endpoint['id'])
+            self.assertEqual(ref.interface, interface)
+            self.assertEqual(ref.region, endpoint['region'])
+            self.assertEqual(ref.service_id, endpoint['service_id'])
+            self.assertEqual(ref.url, endpoint_extra['%surl' % interface])
+            self.assertEqual(ref.extra, '{}')
+
+    def test_downgrade_12_to_9(self):
+        self.upgrade(12)
+
+        service_extra = {
+            'name': uuid.uuid4().hex,
+        }
+        service = {
+            'id': uuid.uuid4().hex,
+            'type': uuid.uuid4().hex,
+            'extra': json.dumps(service_extra),
+        }
+
+        common_endpoint_attrs = {
+            'legacy_endpoint_id': uuid.uuid4().hex,
+            'region': uuid.uuid4().hex,
+            'service_id': service['id'],
+            'extra': json.dumps({}),
+        }
+        endpoints = {
+            'public': {
+                'id': uuid.uuid4().hex,
+                'interface': 'public',
+                'url': uuid.uuid4().hex,
+            },
+            'internal': {
+                'id': uuid.uuid4().hex,
+                'interface': 'internal',
+                'url': uuid.uuid4().hex,
+            },
+            'admin': {
+                'id': uuid.uuid4().hex,
+                'interface': 'admin',
+                'url': uuid.uuid4().hex,
+            },
+        }
+
+        session = self.Session()
+        self.insert_dict(session, 'service', service)
+        for endpoint in endpoints.values():
+            endpoint.update(common_endpoint_attrs)
+            self.insert_dict(session, 'endpoint', endpoint)
+        session.commit()
+
+        self.downgrade(8)
+
+        self.assertTableColumns(
+            'service',
+            ['id', 'type', 'extra'])
+        self.assertTableColumns(
+            'endpoint',
+            ['id', 'region', 'service_id', 'extra'])
+
+        endpoint_table = sqlalchemy.Table(
+            'endpoint', self.metadata, autoload=True)
+
+        session = self.Session()
+        self.assertEqual(session.query(endpoint_table).count(), 1)
+        q = session.query(endpoint_table)
+        q = q.filter_by(id=common_endpoint_attrs['legacy_endpoint_id'])
+        ref = q.one()
+        self.assertEqual(ref.id, common_endpoint_attrs['legacy_endpoint_id'])
+        self.assertEqual(ref.region, endpoint['region'])
+        self.assertEqual(ref.service_id, endpoint['service_id'])
+        extra = json.loads(ref.extra)
+        for interface in ['public', 'internal', 'admin']:
+            expected_url = endpoints[interface]['url']
+            self.assertEqual(extra['%surl' % interface], expected_url)
+
+    def insert_dict(self, session, table_name, d):
+        """Naively inserts key-value pairs into a table, given a dictionary."""
+        session.execute(
+            'INSERT INTO `%s` (%s) VALUES (%s)' % (
+                table_name,
+                ', '.join('%s' % k for k in d.keys()),
+                ', '.join("'%s'" % v for v in d.values())))
 
     def test_downgrade_to_0(self):
-        self._migrate(self.repo_path, 9)
-        self._migrate(self.repo_path, 0, False)
+        self.upgrade(12)
+        self.downgrade(0)
         for table_name in ["user", "token", "role", "user_tenant_membership",
                            "metadata"]:
             self.assertTableDoesNotExist(table_name)
 
     def test_upgrade_6_to_7(self):
-        self._migrate(self.repo_path, 6)
-        self.assertEqual(self.schema.version, 6, "DB is at version 6")
+        self.upgrade(6)
         self.assertTableDoesNotExist('credential')
         self.assertTableDoesNotExist('domain')
         self.assertTableDoesNotExist('user_domain_metadata')
-        self._migrate(self.repo_path, 7)
-        self.assertEqual(self.schema.version, 7, "DB is at version 7")
+
+        self.upgrade(7)
         self.assertTableExists('credential')
         self.assertTableColumns('credential', ['id', 'user_id', 'project_id',
                                                'blob', 'type', 'extra'])
@@ -191,12 +318,20 @@ class SqlUpgradeTests(test.TestCase):
         else:
             raise AssertionError('Table "%s" already exists' % table_name)
 
-    def _migrate(self, repository, version, upgrade=True):
-        err = ""
+    def upgrade(self, *args, **kwargs):
+        self._migrate(*args, **kwargs)
+
+    def downgrade(self, *args, **kwargs):
+        self._migrate(*args, downgrade=True, **kwargs)
+
+    def _migrate(self, version, repository=None, downgrade=False):
+        repository = repository or self.repo_path
+        err = ''
         version = versioning_api._migrate_version(self.schema,
                                                   version,
-                                                  upgrade,
+                                                  not downgrade,
                                                   err)
         changeset = self.schema.changeset(version)
         for ver, change in changeset:
             self.schema.runchange(ver, change, changeset.step)
+        self.assertEqual(self.schema.version, version)