Merge "Split endpoint records in SQL by interface"

4 files changed, 199 insertions, 74 deletions

diff --git a/tests/test_backend.py b/tests/test_backend.py
index e20011b9..5bcdfbe3 100644
--- a/tests/test_backend.py
+++ b/tests/test_backend.py
@@ -885,6 +885,9 @@ class CatalogTests(object):
         # create an endpoint attached to the service
         endpoint = {
             'id': uuid.uuid4().hex,
+            'region': uuid.uuid4().hex,
+            'interface': uuid.uuid4().hex,
+            'url': uuid.uuid4().hex,
             'service_id': service['id'],
         }
         self.catalog_api.create_endpoint(endpoint['id'], endpoint)
diff --git a/tests/test_backend_sql.py b/tests/test_backend_sql.py
index c163acba..8306c65e 100644
--- a/tests/test_backend_sql.py
+++ b/tests/test_backend_sql.py
@@ -217,66 +217,52 @@ class SqlToken(SqlTests, test_backend.TokenTests):
 
 class SqlCatalog(SqlTests, test_backend.CatalogTests):
     def test_malformed_catalog_throws_error(self):
-        self.catalog_api.create_service('a', {"id": "a", "desc": "a1",
-                                        "name": "b"})
-        badurl = "http://192.168.1.104:$(compute_port)s/v2/$(tenant)s"
-        self.catalog_api.create_endpoint('b', {"id": "b", "region": "b1",
-                                         "service_id": "a", "adminurl": badurl,
-                                         "internalurl": badurl,
-                                         "publicurl": badurl})
+        service = {
+            'id': uuid.uuid4().hex,
+            'type': uuid.uuid4().hex,
+            'name': uuid.uuid4().hex,
+            'description': uuid.uuid4().hex,
+        }
+        self.catalog_api.create_service(service['id'], service.copy())
+
+        malformed_url = "http://192.168.1.104:$(compute_port)s/v2/$(tenant)s"
+        endpoint = {
+            'id': uuid.uuid4().hex,
+            'region': uuid.uuid4().hex,
+            'service_id': service['id'],
+            'interface': 'public',
+            'url': malformed_url,
+        }
+        self.catalog_api.create_endpoint(endpoint['id'], endpoint.copy())
+
         with self.assertRaises(exception.MalformedEndpoint):
             self.catalog_api.get_catalog('fake-user', 'fake-tenant')
 
-    def test_get_catalog_without_endpoint(self):
-        new_service = {
+    def test_get_catalog_with_empty_public_url(self):
+        service = {
             'id': uuid.uuid4().hex,
             'type': uuid.uuid4().hex,
             'name': uuid.uuid4().hex,
             'description': uuid.uuid4().hex,
         }
-        self.catalog_api.create_service(
-            new_service['id'],
-            new_service.copy())
-        service_id = new_service['id']
+        self.catalog_api.create_service(service['id'], service.copy())
 
-        new_endpoint = {
+        endpoint = {
             'id': uuid.uuid4().hex,
             'region': uuid.uuid4().hex,
-            'service_id': service_id,
+            'interface': 'public',
+            'url': '',
+            'service_id': service['id'],
         }
-
-        self.catalog_api.create_endpoint(
-            new_endpoint['id'],
-            new_endpoint.copy())
+        self.catalog_api.create_endpoint(endpoint['id'], endpoint.copy())
 
         catalog = self.catalog_api.get_catalog('user', 'tenant')
-
-        service_type = new_service['type']
-        region = new_endpoint['region']
-
-        self.assertEqual(catalog[region][service_type]['name'],
-                         new_service['name'])
-        self.assertEqual(catalog[region][service_type]['id'],
-                         new_endpoint['id'])
-        self.assertEqual(catalog[region][service_type]['publicURL'],
-                         "")
-        self.assertEqual(catalog[region][service_type]['adminURL'],
-                         None)
-        self.assertEqual(catalog[region][service_type]['internalURL'],
-                         None)
-
-    def test_delete_service_with_endpoints(self):
-        self.catalog_api.create_service('c', {"id": "c", "desc": "a1",
-                                        "name": "d"})
-        self.catalog_api.create_endpoint('d', {"id": "d", "region": None,
-                                         "service_id": "c", "adminurl": None,
-                                         "internalurl": None,
-                                         "publicurl": None})
-        self.catalog_api.delete_service("c")
-        self.assertRaises(exception.ServiceNotFound,
-                          self.catalog_man.delete_service, {}, "c")
-        self.assertRaises(exception.EndpointNotFound,
-                          self.catalog_man.delete_endpoint, {}, "d")
+        catalog_endpoint = catalog[endpoint['region']][service['type']]
+        self.assertEqual(catalog_endpoint['name'], service['name'])
+        self.assertEqual(catalog_endpoint['id'], endpoint['id'])
+        self.assertEqual(catalog_endpoint['publicURL'], '')
+        self.assertIsNone(catalog_endpoint.get('adminURL'))
+        self.assertIsNone(catalog_endpoint.get('internalURL'))
 
 
 class SqlPolicy(SqlTests, test_backend.PolicyTests):
diff --git a/tests/test_sql_upgrade.py b/tests/test_sql_upgrade.py
index fac9986f..9dcaaeff 100644
--- a/tests/test_sql_upgrade.py
+++ b/tests/test_sql_upgrade.py
@@ -16,10 +16,10 @@
 
 import copy
 import json
+import uuid
 
 from migrate.versioning import api as versioning_api
 import sqlalchemy
-from sqlalchemy.orm import sessionmaker
 
 from keystone.common import sql
 from keystone import config
@@ -27,6 +27,7 @@ from keystone import test
 from keystone.common.sql import migration
 import default_fixtures
 
+
 CONF = config.CONF
 
 
@@ -38,7 +39,11 @@ class SqlUpgradeTests(test.TestCase):
                      test.testsdir('backend_sql.conf')])
 
         # create and share a single sqlalchemy engine for testing
-        self.engine = sql.Base().get_engine(allow_global_engine=False)
+        base = sql.Base()
+        self.engine = base.get_engine(allow_global_engine=False)
+        self.Session = base.get_sessionmaker(
+            engine=self.engine,
+            autocommit=False)
         self.metadata = sqlalchemy.MetaData()
 
         # populate the engine with tables & fixtures
@@ -64,9 +69,7 @@ class SqlUpgradeTests(test.TestCase):
         self.assertEqual(expected_cols, actual_cols, '%s table' % table_name)
 
     def test_upgrade_0_to_1(self):
-        self.assertEqual(self.schema.version, 0, "DB is at version 0")
-        self._migrate(self.repo_path, 1)
-        self.assertEqual(self.schema.version, 1, "DB is at version 1")
+        self.upgrade(1)
         self.assertTableColumns("user", ["id", "name", "extra"])
         self.assertTableColumns("tenant", ["id", "name", "extra"])
         self.assertTableColumns("role", ["id", "name"])
@@ -76,23 +79,18 @@ class SqlUpgradeTests(test.TestCase):
         self.populate_user_table()
 
     def test_upgrade_5_to_6(self):
-        self._migrate(self.repo_path, 5)
-        self.assertEqual(self.schema.version, 5)
+        self.upgrade(5)
         self.assertTableDoesNotExist('policy')
 
-        self._migrate(self.repo_path, 6)
-        self.assertEqual(self.schema.version, 6)
+        self.upgrade(6)
         self.assertTableExists('policy')
         self.assertTableColumns('policy', ['id', 'type', 'blob', 'extra'])
 
     def test_upgrade_7_to_9(self):
-
-        self.assertEqual(self.schema.version, 0)
-        self._migrate(self.repo_path, 7)
+        self.upgrade(7)
         self.populate_user_table()
         self.populate_tenant_table()
-        self._migrate(self.repo_path, 9)
-        self.assertEqual(self.schema.version, 9)
+        self.upgrade(9)
         self.assertTableColumns("user",
                                 ["id", "name", "extra", "password",
                                  "enabled"])
@@ -103,8 +101,7 @@ class SqlUpgradeTests(test.TestCase):
         self.assertTableColumns("user_tenant_membership",
                                 ["user_id", "tenant_id"])
         self.assertTableColumns("metadata", ["user_id", "tenant_id", "data"])
-        maker = sessionmaker(bind=self.engine)
-        session = maker()
+        session = self.Session()
         user_table = sqlalchemy.Table("user",
                                       self.metadata,
                                       autoload=True)
@@ -120,25 +117,155 @@ class SqlUpgradeTests(test.TestCase):
         session.commit()
 
     def test_downgrade_9_to_7(self):
-        self.assertEqual(self.schema.version, 0)
-        self._migrate(self.repo_path, 9)
-        self._migrate(self.repo_path, 7, False)
+        self.upgrade(9)
+        self.downgrade(7)
+
+    def test_upgrade_9_to_12(self):
+        self.upgrade(9)
+
+        service_extra = {
+            'name': uuid.uuid4().hex,
+        }
+        service = {
+            'id': uuid.uuid4().hex,
+            'type': uuid.uuid4().hex,
+            'extra': json.dumps(service_extra),
+        }
+        endpoint_extra = {
+            'publicurl': uuid.uuid4().hex,
+            'internalurl': uuid.uuid4().hex,
+            'adminurl': uuid.uuid4().hex,
+        }
+        endpoint = {
+            'id': uuid.uuid4().hex,
+            'region': uuid.uuid4().hex,
+            'service_id': service['id'],
+            'extra': json.dumps(endpoint_extra),
+        }
+
+        session = self.Session()
+        self.insert_dict(session, 'service', service)
+        self.insert_dict(session, 'endpoint', endpoint)
+        session.commit()
+
+        self.upgrade(12)
+
+        self.assertTableColumns(
+            'service',
+            ['id', 'type', 'extra'])
+        self.assertTableColumns(
+            'endpoint',
+            ['id', 'legacy_endpoint_id', 'interface', 'region', 'service_id',
+             'url', 'extra'])
+
+        endpoint_table = sqlalchemy.Table(
+            'endpoint', self.metadata, autoload=True)
+
+        session = self.Session()
+        self.assertEqual(session.query(endpoint_table).count(), 3)
+        for interface in ['public', 'internal', 'admin']:
+            q = session.query(endpoint_table)
+            q = q.filter_by(legacy_endpoint_id=endpoint['id'])
+            q = q.filter_by(interface=interface)
+            ref = q.one()
+            self.assertNotEqual(ref.id, endpoint['id'])
+            self.assertEqual(ref.legacy_endpoint_id, endpoint['id'])
+            self.assertEqual(ref.interface, interface)
+            self.assertEqual(ref.region, endpoint['region'])
+            self.assertEqual(ref.service_id, endpoint['service_id'])
+            self.assertEqual(ref.url, endpoint_extra['%surl' % interface])
+            self.assertEqual(ref.extra, '{}')
+
+    def test_downgrade_12_to_9(self):
+        self.upgrade(12)
+
+        service_extra = {
+            'name': uuid.uuid4().hex,
+        }
+        service = {
+            'id': uuid.uuid4().hex,
+            'type': uuid.uuid4().hex,
+            'extra': json.dumps(service_extra),
+        }
+
+        common_endpoint_attrs = {
+            'legacy_endpoint_id': uuid.uuid4().hex,
+            'region': uuid.uuid4().hex,
+            'service_id': service['id'],
+            'extra': json.dumps({}),
+        }
+        endpoints = {
+            'public': {
+                'id': uuid.uuid4().hex,
+                'interface': 'public',
+                'url': uuid.uuid4().hex,
+            },
+            'internal': {
+                'id': uuid.uuid4().hex,
+                'interface': 'internal',
+                'url': uuid.uuid4().hex,
+            },
+            'admin': {
+                'id': uuid.uuid4().hex,
+                'interface': 'admin',
+                'url': uuid.uuid4().hex,
+            },
+        }
+
+        session = self.Session()
+        self.insert_dict(session, 'service', service)
+        for endpoint in endpoints.values():
+            endpoint.update(common_endpoint_attrs)
+            self.insert_dict(session, 'endpoint', endpoint)
+        session.commit()
+
+        self.downgrade(8)
+
+        self.assertTableColumns(
+            'service',
+            ['id', 'type', 'extra'])
+        self.assertTableColumns(
+            'endpoint',
+            ['id', 'region', 'service_id', 'extra'])
+
+        endpoint_table = sqlalchemy.Table(
+            'endpoint', self.metadata, autoload=True)
+
+        session = self.Session()
+        self.assertEqual(session.query(endpoint_table).count(), 1)
+        q = session.query(endpoint_table)
+        q = q.filter_by(id=common_endpoint_attrs['legacy_endpoint_id'])
+        ref = q.one()
+        self.assertEqual(ref.id, common_endpoint_attrs['legacy_endpoint_id'])
+        self.assertEqual(ref.region, endpoint['region'])
+        self.assertEqual(ref.service_id, endpoint['service_id'])
+        extra = json.loads(ref.extra)
+        for interface in ['public', 'internal', 'admin']:
+            expected_url = endpoints[interface]['url']
+            self.assertEqual(extra['%surl' % interface], expected_url)
+
+    def insert_dict(self, session, table_name, d):
+        """Naively inserts key-value pairs into a table, given a dictionary."""
+        session.execute(
+            'INSERT INTO `%s` (%s) VALUES (%s)' % (
+                table_name,
+                ', '.join('%s' % k for k in d.keys()),
+                ', '.join("'%s'" % v for v in d.values())))
 
     def test_downgrade_to_0(self):
-        self._migrate(self.repo_path, 9)
-        self._migrate(self.repo_path, 0, False)
+        self.upgrade(12)
+        self.downgrade(0)
         for table_name in ["user", "token", "role", "user_tenant_membership",
                            "metadata"]:
             self.assertTableDoesNotExist(table_name)
 
     def test_upgrade_6_to_7(self):
-        self._migrate(self.repo_path, 6)
-        self.assertEqual(self.schema.version, 6, "DB is at version 6")
+        self.upgrade(6)
         self.assertTableDoesNotExist('credential')
         self.assertTableDoesNotExist('domain')
         self.assertTableDoesNotExist('user_domain_metadata')
-        self._migrate(self.repo_path, 7)
-        self.assertEqual(self.schema.version, 7, "DB is at version 7")
+
+        self.upgrade(7)
         self.assertTableExists('credential')
         self.assertTableColumns('credential', ['id', 'user_id', 'project_id',
                                                'blob', 'type', 'extra'])
@@ -191,12 +318,20 @@ class SqlUpgradeTests(test.TestCase):
         else:
             raise AssertionError('Table "%s" already exists' % table_name)
 
-    def _migrate(self, repository, version, upgrade=True):
-        err = ""
+    def upgrade(self, *args, **kwargs):
+        self._migrate(*args, **kwargs)
+
+    def downgrade(self, *args, **kwargs):
+        self._migrate(*args, downgrade=True, **kwargs)
+
+    def _migrate(self, version, repository=None, downgrade=False):
+        repository = repository or self.repo_path
+        err = ''
         version = versioning_api._migrate_version(self.schema,
                                                   version,
-                                                  upgrade,
+                                                  not downgrade,
                                                   err)
         changeset = self.schema.changeset(version)
         for ver, change in changeset:
             self.schema.runchange(ver, change, changeset.step)
+        self.assertEqual(self.schema.version, version)
diff --git a/tests/test_v3.py b/tests/test_v3.py
index 81a088e8..9a999585 100644
--- a/tests/test_v3.py
+++ b/tests/test_v3.py
@@ -44,6 +44,7 @@ class RestfulTestCase(test_content_types.RestfulTestCase):
         ref = self.new_ref()
         ref['interface'] = uuid.uuid4().hex
         ref['service_id'] = service_id
+        ref['url'] = uuid.uuid4().hex
         return ref
 
     def new_domain_ref(self):