Add cloudpipe/vpn api to openstack api contrib.

blueprint cloudpipe-extension

Updates:
    2011-12-19 #1: * Remove unused imports
                   * return uuid as the instance_id
                   * change state with bad config to "invalid"
                   * whitespace cleanup
                   * change top-level key on index to "cloudpipes"
    2011-12-22 #1: * add serializer
                   * change post body to be cloudpipe/project_id
                   * change admin api method
    2011-12-23 #1: * Change extension namespace
    2011-12-23 #2: * Fix failing extension test
    2011-12-23 #3: * Add xtoddx@gmail.com to .mailmap
    2011-12-27 #1: * pep-8
    2012-01-02 #1: * fix test stubs to not cause later test failures
    2012-01-03 #1: * fix test self.app to not traverse middlewares
                   * don't use not in for a single item list

Change-Id: I5710f8cea710fa09e5405c30d565144a7c10e112

3 files changed, 421 insertions, 0 deletions

diff --git a/nova/api/openstack/v2/contrib/cloudpipe.py b/nova/api/openstack/v2/contrib/cloudpipe.py
new file mode 100644
index 000000000..33c23c1d0
--- /dev/null
+++ b/nova/api/openstack/v2/contrib/cloudpipe.py
@@ -0,0 +1,183 @@
+#   Copyright 2011 Openstack, LLC.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+"""Connect your vlan to the world."""
+
+import os
+
+from nova.api.openstack import wsgi
+from nova.api.openstack import xmlutil
+from nova.api.openstack.v2 import extensions
+from nova.auth import manager
+from nova.cloudpipe import pipelib
+from nova import compute
+from nova.compute import vm_states
+from nova import db
+from nova import exception
+from nova import flags
+from nova import log as logging
+from nova import utils
+
+
+FLAGS = flags.FLAGS
+LOG = logging.getLogger("nova.api.openstack.v2.contrib.cloudpipe")
+
+
+class CloudpipeController(object):
+    """Handle creating and listing cloudpipe instances."""
+
+    def __init__(self):
+        self.compute_api = compute.API()
+        self.auth_manager = manager.AuthManager()
+        self.cloudpipe = pipelib.CloudPipe()
+        self.setup()
+
+    def setup(self):
+        """Ensure the keychains and folders exist."""
+        # TODO(todd): this was copyed from api.ec2.cloud
+        # FIXME(ja): this should be moved to a nova-manage command,
+        # if not setup throw exceptions instead of running
+        # Create keys folder, if it doesn't exist
+        if not os.path.exists(FLAGS.keys_path):
+            os.makedirs(FLAGS.keys_path)
+        # Gen root CA, if we don't have one
+        root_ca_path = os.path.join(FLAGS.ca_path, FLAGS.ca_file)
+        if not os.path.exists(root_ca_path):
+            genrootca_sh_path = os.path.join(os.path.dirname(__file__),
+                                             os.path.pardir,
+                                             os.path.pardir,
+                                             'CA',
+                                             'genrootca.sh')
+
+            start = os.getcwd()
+            if not os.path.exists(FLAGS.ca_path):
+                os.makedirs(FLAGS.ca_path)
+            os.chdir(FLAGS.ca_path)
+            # TODO(vish): Do this with M2Crypto instead
+            utils.runthis(_("Generating root CA: %s"), "sh", genrootca_sh_path)
+            os.chdir(start)
+
+    def _get_cloudpipe_for_project(self, context, project_id):
+        """Get the cloudpipe instance for a project ID."""
+        # NOTE(todd): this should probably change to compute_api.get_all
+        #             or db.instance_get_project_vpn
+        for instance in db.instance_get_all_by_project(context, project_id):
+            if (instance['image_id'] == str(FLAGS.vpn_image_id)
+                and instance['vm_state'] != vm_states.DELETED):
+                return instance
+
+    def _vpn_dict(self, project, vpn_instance):
+        rv = {'project_id': project.id,
+              'public_ip': project.vpn_ip,
+              'public_port': project.vpn_port}
+        if vpn_instance:
+            rv['instance_id'] = vpn_instance['uuid']
+            rv['created_at'] = utils.isotime(vpn_instance['created_at'])
+            address = vpn_instance.get('fixed_ip', None)
+            if address:
+                rv['internal_ip'] = address['address']
+            if project.vpn_ip and project.vpn_port:
+                if utils.vpn_ping(project.vpn_ip, project.vpn_port):
+                    rv['state'] = 'running'
+                else:
+                    rv['state'] = 'down'
+            else:
+                rv['state'] = 'invalid'
+        else:
+            rv['state'] = 'pending'
+        return rv
+
+    def create(self, req, body):
+        """Create a new cloudpipe instance, if none exists.
+
+        Parameters: {cloudpipe: {project_id: XYZ}}
+        """
+
+        ctxt = req.environ['nova.context']
+        params = body.get('cloudpipe', {})
+        project_id = params.get('project_id', ctxt.project_id)
+        instance = self._get_cloudpipe_for_project(ctxt, project_id)
+        if not instance:
+            proj = self.auth_manager.get_project(project_id)
+            user_id = proj.project_manager_id
+            try:
+                self.cloudpipe.launch_vpn_instance(project_id, user_id)
+            except db.NoMoreNetworks:
+                msg = _("Unable to claim IP for VPN instances, ensure it "
+                        "isn't running, and try again in a few minutes")
+                raise exception.ApiError(msg)
+            instance = self._get_cloudpipe_for_project(ctxt, proj)
+        return {'instance_id': instance['uuid']}
+
+    def index(self, req):
+        """Show admins the list of running cloudpipe instances."""
+        context = req.environ['nova.context']
+        vpns = []
+        # TODO(todd): could use compute_api.get_all with admin context?
+        for project in self.auth_manager.get_projects():
+            instance = self._get_cloudpipe_for_project(context, project.id)
+            vpns.append(self._vpn_dict(project, instance))
+        return {'cloudpipes': vpns}
+
+
+class Cloudpipe(extensions.ExtensionDescriptor):
+    """Adds actions to create cloudpipe instances.
+
+    When running with the Vlan network mode, you need a mechanism to route
+    from the public Internet to your vlans.  This mechanism is known as a
+    cloudpipe.
+
+    At the time of creating this class, only OpenVPN is supported.  Support for
+    a SSH Bastion host is forthcoming.
+    """
+
+    name = "Cloudpipe"
+    alias = "os-cloudpipe"
+    namespace = "http://docs.openstack.org/compute/ext/cloudpipe/api/v1.1"
+    updated = "2011-12-16T00:00:00+00:00"
+    admin_only = True
+
+    def get_resources(self):
+        resources = []
+        body_serializers = {
+            'application/xml': CloudpipeSerializer(),
+            }
+        serializer = wsgi.ResponseSerializer(body_serializers)
+        res = extensions.ResourceExtension('os-cloudpipe',
+                                           CloudpipeController(),
+                                           serializer=serializer)
+        resources.append(res)
+        return resources
+
+
+class CloudpipeSerializer(xmlutil.XMLTemplateSerializer):
+    def index(self):
+        return CloudpipesTemplate()
+
+    def default(self):
+        return CloudpipeTemplate()
+
+
+class CloudpipeTemplate(xmlutil.TemplateBuilder):
+    def construct(self):
+        return xmlutil.MasterTemplate(xmlutil.make_flat_dict('cloudpipe'), 1)
+
+
+class CloudpipesTemplate(xmlutil.TemplateBuilder):
+    def construct(self):
+        root = xmlutil.TemplateElement('cloudpipes')
+        elem = xmlutil.make_flat_dict('cloudpipe', selector='cloudpipes',
+                                      subselector='cloudpipe')
+        root.append(elem)
+        return xmlutil.MasterTemplate(root, 1)
diff --git a/nova/tests/api/openstack/v2/contrib/test_cloudpipe.py b/nova/tests/api/openstack/v2/contrib/test_cloudpipe.py
new file mode 100644
index 000000000..8dfac4765
--- /dev/null
+++ b/nova/tests/api/openstack/v2/contrib/test_cloudpipe.py
@@ -0,0 +1,237 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+import json
+
+import webob
+from lxml import etree
+
+from nova.api import auth
+from nova.api.openstack import v2
+from nova.api.openstack.v2 import wsgi
+from nova.api.openstack.v2.contrib import cloudpipe
+from nova.auth import manager
+from nova.cloudpipe import pipelib
+from nova import context
+from nova import crypto
+from nova import db
+from nova import flags
+from nova import test
+from nova.tests.api.openstack import fakes
+from nova import utils
+
+
+EMPTY_INSTANCE_LIST = True
+FLAGS = flags.FLAGS
+
+
+class FakeProject(object):
+    def __init__(self, id, name, manager, desc, members, ip, port):
+        self.id = id
+        self.name = name
+        self.project_manager_id = manager
+        self.description = desc
+        self.member_ids = members
+        self.vpn_ip = ip
+        self.vpn_port = port
+
+
+def fake_vpn_instance():
+    return {'id': 7, 'image_id': FLAGS.vpn_image_id, 'vm_state': 'active',
+            'created_at': utils.parse_strtime('1981-10-20T00:00:00.000000'),
+            'uuid': 7777}
+
+
+def fake_vpn_instance_low_id():
+    return {'id': 4, 'image_id': FLAGS.vpn_image_id, 'vm_state': 'active',
+            'created_at': utils.parse_strtime('1981-10-20T00:00:00.000000')}
+
+
+def fake_project():
+    proj = FakeProject(1, '1', 'fakeuser', '', [1], '127.0.0.1', 22)
+    return proj
+
+
+def db_instance_get_all_by_project(self, project_id):
+    if EMPTY_INSTANCE_LIST:
+        return []
+    else:
+        return [fake_vpn_instance()]
+
+
+def db_security_group_exists(context, project_id, group_name):
+    # used in pipelib
+    return True
+
+
+def pipelib_launch_vpn_instance(self, project_id, user_id):
+    global EMPTY_INSTANCE_LIST
+    EMPTY_INSTANCE_LIST = False
+
+
+def auth_manager_get_project(self, project_id):
+    return fake_project()
+
+
+def auth_manager_get_projects(self):
+    return [fake_project()]
+
+
+def utils_vpn_ping(addr, port, timoeout=0.05, session_id=None):
+    return True
+
+
+def better_not_call_this(*args, **kwargs):
+    raise Exception("You should not have done that")
+
+
+class FakeAuthManager(object):
+    def get_projects(self):
+        return [fake_project()]
+
+    def get_project(self, project_id):
+        return fake_project()
+
+
+class CloudpipeTest(test.TestCase):
+
+    def setUp(self):
+        super(CloudpipeTest, self).setUp()
+        self.flags(allow_admin_api=True)
+        self.app = fakes.wsgi_app()
+        inner_app = v2.APIRouter()
+        adm_ctxt = context.get_admin_context()
+        self.app = auth.InjectContext(adm_ctxt, inner_app)
+        route = inner_app.map.match('/1234/os-cloudpipe')
+        self.controller = route['controller'].controller
+        fakes.stub_out_networking(self.stubs)
+        fakes.stub_out_rate_limiting(self.stubs)
+        self.stubs.Set(db, "instance_get_all_by_project",
+                       db_instance_get_all_by_project)
+        self.stubs.Set(db, "security_group_exists",
+                       db_security_group_exists)
+        self.stubs.SmartSet(self.controller.cloudpipe, "launch_vpn_instance",
+                            pipelib_launch_vpn_instance)
+        #self.stubs.SmartSet(self.controller.auth_manager, "get_project",
+        #                    auth_manager_get_project)
+        #self.stubs.SmartSet(self.controller.auth_manager, "get_projects",
+        #                    auth_manager_get_projects)
+        # NOTE(todd): The above code (just setting the stub, not invoking it)
+        # causes failures in AuthManagerLdapTestCase.  So use a fake object.
+        self.controller.auth_manager = FakeAuthManager()
+        self.stubs.Set(utils, 'vpn_ping', utils_vpn_ping)
+        self.context = context.get_admin_context()
+        global EMPTY_INSTANCE_LIST
+        EMPTY_INSTANCE_LIST = True
+
+    def test_cloudpipe_list_none_running(self):
+        """Should still get an entry per-project, just less descriptive."""
+        req = webob.Request.blank('/123/os-cloudpipe')
+        res = req.get_response(self.app)
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+        response = {'cloudpipes': [{'project_id': 1, 'public_ip': '127.0.0.1',
+                     'public_port': 22, 'state': 'pending'}]}
+        self.assertEqual(res_dict, response)
+
+    def test_cloudpipe_list(self):
+        global EMPTY_INSTANCE_LIST
+        EMPTY_INSTANCE_LIST = False
+        req = webob.Request.blank('/123/os-cloudpipe')
+        res = req.get_response(self.app)
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+        response = {'cloudpipes': [{'project_id': 1, 'public_ip': '127.0.0.1',
+                     'public_port': 22, 'state': 'running',
+                     'instance_id': 7777,
+                     'created_at': '1981-10-20T00:00:00Z'}]}
+        self.assertEqual(res_dict, response)
+
+    def test_cloudpipe_create(self):
+        body = {'cloudpipe': {'project_id': 1}}
+        req = webob.Request.blank('/123/os-cloudpipe')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers['Content-Type'] = 'application/json'
+        res = req.get_response(self.app)
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+        response = {'instance_id': 7777}
+        self.assertEqual(res_dict, response)
+
+    def test_cloudpipe_create_already_running(self):
+        global EMPTY_INSTANCE_LIST
+        EMPTY_INSTANCE_LIST = False
+        self.stubs.SmartSet(self.controller.cloudpipe, 'launch_vpn_instance',
+                            better_not_call_this)
+        body = {'cloudpipe': {'project_id': 1}}
+        req = webob.Request.blank('/123/os-cloudpipe')
+        req.method = 'POST'
+        req.body = json.dumps(body)
+        req.headers['Content-Type'] = 'application/json'
+        res = req.get_response(self.app)
+        self.assertEqual(res.status_int, 200)
+        res_dict = json.loads(res.body)
+        response = {'instance_id': 7777}
+        self.assertEqual(res_dict, response)
+
+
+class CloudpipesXMLSerializerTest(test.TestCase):
+    def setUp(self):
+        super(CloudpipesXMLSerializerTest, self).setUp()
+        self.serializer = cloudpipe.CloudpipeSerializer()
+        self.deserializer = wsgi.XMLDeserializer()
+
+    def test_default_serializer(self):
+        exemplar = dict(cloudpipe=dict(instance_id='1234-1234-1234-1234'))
+        text = self.serializer.serialize(exemplar)
+        tree = etree.fromstring(text)
+        self.assertEqual('cloudpipe', tree.tag)
+        for child in tree:
+            self.assertTrue(child.tag in exemplar['cloudpipe'])
+            self.assertEqual(child.text, exemplar['cloudpipe'][child.tag])
+
+    def test_index_serializer(self):
+        exemplar = dict(cloudpipes=[
+                dict(cloudpipe=dict(
+                        project_id='1234',
+                        public_ip='1.2.3.4',
+                        public_port='321',
+                        instance_id='1234-1234-1234-1234',
+                        created_at=utils.isotime(datetime.datetime.utcnow()),
+                        state='running')),
+                dict(cloudpipe=dict(
+                        project_id='4321',
+                        public_ip='4.3.2.1',
+                        public_port='123',
+                        state='pending'))])
+        text = self.serializer.serialize(exemplar, 'index')
+        tree = etree.fromstring(text)
+        self.assertEqual('cloudpipes', tree.tag)
+        self.assertEqual(len(exemplar['cloudpipes']), len(tree))
+        for idx, cloudpipe in enumerate(tree):
+            self.assertEqual('cloudpipe', cloudpipe.tag)
+            kp_data = exemplar['cloudpipes'][idx]['cloudpipe']
+            for child in cloudpipe:
+                self.assertTrue(child.tag in kp_data)
+                self.assertEqual(child.text, kp_data[child.tag])
+
+    def test_deserializer(self):
+        exemplar = dict(cloudpipe=dict(project_id='4321'))
+        intext = ("<?xml version='1.0' encoding='UTF-8'?>\n"
+                  '<cloudpipe><project_id>4321</project_id></cloudpipe>')
+        result = self.deserializer.deserialize(intext)['body']
+        self.assertEqual(result, exemplar)
diff --git a/nova/tests/api/openstack/v2/test_extensions.py b/nova/tests/api/openstack/v2/test_extensions.py
index 302095fe8..51af89da3 100644
--- a/nova/tests/api/openstack/v2/test_extensions.py
+++ b/nova/tests/api/openstack/v2/test_extensions.py
@@ -100,6 +100,7 @@ class ExtensionControllerTest(ExtensionTestCase):
         self.ext_list = [
             "Accounts",
             "AdminActions",
+            "Cloudpipe",
             "Console_output",
             "Createserverext",
             "DeferredDelete",