From f127d85d7790585d6e735648dfab13416d79fbde Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 6 Nov 2010 00:02:36 +0000 Subject: Per-project vpns, certificates, and revocation --- CA/INTER/.gitignore | 1 - CA/INTER/.placeholder | 0 CA/geninter.sh | 26 +++-- CA/genrootca.sh | 1 + CA/openssl.cnf.tmpl | 3 +- CA/projects/.gitignore | 1 + CA/projects/.placeholder | 0 bin/nova-manage | 75 +++++++++----- nova/api/__init__.py | 2 - nova/api/cloudpipe/__init__.py | 69 ------------- nova/auth/manager.py | 73 +++++++------- nova/cloudpipe/bootscript.sh | 63 ------------ nova/cloudpipe/bootscript.template | 50 ++++++++++ nova/cloudpipe/pipelib.py | 123 +++++++++++++++-------- nova/crypto.py | 195 +++++++++++++++++++++++++++++-------- nova/db/api.py | 39 ++++++++ nova/db/sqlalchemy/api.py | 78 +++++++++++++++ nova/db/sqlalchemy/models.py | 12 ++- nova/tests/auth_unittest.py | 17 ++-- 19 files changed, 533 insertions(+), 295 deletions(-) delete mode 100644 CA/INTER/.gitignore delete mode 100644 CA/INTER/.placeholder create mode 100644 CA/projects/.gitignore create mode 100644 CA/projects/.placeholder delete mode 100644 nova/api/cloudpipe/__init__.py delete mode 100755 nova/cloudpipe/bootscript.sh create mode 100755 nova/cloudpipe/bootscript.template diff --git a/CA/INTER/.gitignore b/CA/INTER/.gitignore deleted file mode 100644 index 72e8ffc0d..000000000 --- a/CA/INTER/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/CA/INTER/.placeholder b/CA/INTER/.placeholder deleted file mode 100644 index e69de29bb..000000000 diff --git a/CA/geninter.sh b/CA/geninter.sh index 7d6c280d5..1fbcc9e73 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -16,16 +16,24 @@ # License for the specific language governing permissions and limitations # under the License. -# ARG is the id of the user -export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$1" -mkdir INTER/$1 -cd INTER/$1 +# $1 is the id of the project and $2 is the subject of the cert +NAME=$1 +SUBJ=$2 +mkdir -p projects/$NAME +cd projects/$NAME cp ../../openssl.cnf.tmpl openssl.cnf -sed -i -e s/%USERNAME%/$1/g openssl.cnf +sed -i -e s/%USERNAME%/$NAME/g openssl.cnf mkdir certs crl newcerts private +openssl req -new -x509 -extensions v3_ca -keyout private/cakey.pem -out cacert.pem -days 365 -config ./openssl.cnf -batch -nodes echo "10" > serial touch index.txt -openssl genrsa -out private/cakey.pem 1024 -config ./openssl.cnf -batch -nodes -openssl req -new -sha2 -key private/cakey.pem -out ../../reqs/inter$1.csr -batch -subj "$SUBJ" -cd ../../ -openssl ca -extensions v3_ca -days 365 -out INTER/$1/cacert.pem -in reqs/inter$1.csr -config openssl.cnf -batch +# NOTE(vish): Disabling intermediate ca's because we don't actually need them. +# It makes more sense to have each project have its own root ca. +# openssl genrsa -out private/cakey.pem 1024 -config ./openssl.cnf -batch -nodes +# openssl req -new -sha256 -key private/cakey.pem -out ../../reqs/inter$NAME.csr -batch -subj "$SUBJ" +openssl ca -gencrl -config ./openssl.cnf -out crl.pem +if [ "`id -u`" != "`grep nova /etc/passwd | cut -d':' -f3`" ]; then + sudo chown -R nova:nogroup . +fi +# cd ../../ +# openssl ca -extensions v3_ca -days 365 -out INTER/$NAME/cacert.pem -in reqs/inter$NAME.csr -config openssl.cnf -batch diff --git a/CA/genrootca.sh b/CA/genrootca.sh index 31976092e..8f2c3ee3f 100755 --- a/CA/genrootca.sh +++ b/CA/genrootca.sh @@ -25,4 +25,5 @@ else openssl req -new -x509 -extensions v3_ca -keyout private/cakey.pem -out cacert.pem -days 365 -config ./openssl.cnf -batch -nodes touch index.txt echo "10" > serial + openssl ca -gencrl -config ./openssl.cnf -out crl.pem fi diff --git a/CA/openssl.cnf.tmpl b/CA/openssl.cnf.tmpl index 639b8e80a..dd81f1c2b 100644 --- a/CA/openssl.cnf.tmpl +++ b/CA/openssl.cnf.tmpl @@ -24,7 +24,6 @@ dir = . [ ca ] default_ca = CA_default -unique_subject = no [ CA_default ] serial = $dir/serial @@ -32,6 +31,8 @@ database = $dir/index.txt new_certs_dir = $dir/newcerts certificate = $dir/cacert.pem private_key = $dir/private/cakey.pem +unique_subject = no +default_crl_days = 365 default_days = 365 default_md = md5 preserve = no diff --git a/CA/projects/.gitignore b/CA/projects/.gitignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/CA/projects/.gitignore @@ -0,0 +1 @@ +* diff --git a/CA/projects/.placeholder b/CA/projects/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/bin/nova-manage b/bin/nova-manage index 08b3da123..b788ee62d 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -69,6 +69,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) from nova import context +from nova import crypto from nova import db from nova import exception from nova import flags @@ -93,32 +94,36 @@ class VpnCommands(object): self.manager = manager.AuthManager() self.pipe = pipelib.CloudPipe() - def list(self): - """Print a listing of the VPNs for all projects.""" + def list(self, project=None): + """Print a listing of the VPN data for one or all projects. + + args: [project=all]""" print "%-12s\t" % 'project', print "%-20s\t" % 'ip:port', + print "%-20s\t" % 'private_ip', print "%s" % 'state' - for project in self.manager.get_projects(): + if project: + projects = [self.manager.get_project(project)] + else: + projects = self.manager.get_projects() + for project in projects: print "%-12s\t" % project.name, - - try: - s = "%s:%s" % (project.vpn_ip, project.vpn_port) - except exception.NotFound: - s = "None" - print "%-20s\t" % s, - + ipport = "%s:%s" % (project.vpn_ip, project.vpn_port) + print "%-20s\t" % ipport, vpn = self._vpn_for(project.id) if vpn: - command = "ping -c1 -w1 %s > /dev/null; echo $?" - out, _err = utils.execute(command % vpn['private_dns_name'], - check_exit_code=False) - if out.strip() == '0': - net = 'up' - else: - net = 'down' - print vpn['private_dns_name'], - print vpn['node_name'], - print vpn['instance_id'], + net = 'down' + address = None + if vpn.get('fixed_ip', None): + address = vpn['fixed_ip']['address'] + command = "ping -c1 -w1 %s > /dev/null; echo $?" + out, _err = utils.execute(command % address, + check_exit_code=False) + if out.strip() == '0': + net = 'up' + print address, + print vpn['host'], + print vpn['ec2_id'], print vpn['state_description'], print net @@ -127,11 +132,11 @@ class VpnCommands(object): def _vpn_for(self, project_id): """Get the VPN instance for a project ID.""" - for instance in db.instance_get_all(context.get_admin_context()): + ctxt = context.get_admin_context() + for instance in db.instance_get_all_by_project(ctxt, project_id): if (instance['image_id'] == FLAGS.vpn_image_id and not instance['state_description'] in - ['shutting_down', 'shutdown'] - and instance['project_id'] == project_id): + ['shutting_down', 'shutdown']): return instance def spawn(self): @@ -146,6 +151,22 @@ class VpnCommands(object): """Start the VPN for a given project.""" self.pipe.launch_vpn_instance(project_id) + def change(self, project_id, ip, port): + """Change the ip and port for a vpn. + + args: project, ip, port""" + project = self.manager.get_project(project_id) + if not project: + print 'No project %s' % (project_id) + return + admin = context.get_admin_context() + network_ref = db.project_get_network(admin, project_id) + db.network_update(admin, + network_ref['id'], + {'vpn_public_address': ip, + 'vpn_public_port': int(port)}) + + class ShellCommands(object): def bpython(self): @@ -292,6 +313,14 @@ class UserCommands(object): is_admin = False self.manager.modify_user(name, access_key, secret_key, is_admin) + def revoke(self, user_id, project_id=None): + """revoke certs for a user + arguments: user_id [project_id]""" + if project_id: + crypto.revoke_certs_by_user_and_project(user_id, project_id) + else: + crypto.revoke_certs_by_user(user_id) + class ProjectCommands(object): """Class for managing projects.""" diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 707c1623e..176d571a8 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -25,7 +25,6 @@ import webob.dec from nova import flags from nova import wsgi -from nova.api import cloudpipe from nova.api import ec2 from nova.api import openstack from nova.api.ec2 import metadatarequesthandler @@ -74,7 +73,6 @@ class API(wsgi.Router): mapper.connect('%s/{path_info:.*}' % s, controller=mrh, conditions=ec2api_subdomain) - mapper.connect("/cloudpipe/{path_info:.*}", controller=cloudpipe.API()) super(API, self).__init__(mapper) @webob.dec.wsgify diff --git a/nova/api/cloudpipe/__init__.py b/nova/api/cloudpipe/__init__.py deleted file mode 100644 index 6d40990a8..000000000 --- a/nova/api/cloudpipe/__init__.py +++ /dev/null @@ -1,69 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -REST API Request Handlers for CloudPipe -""" - -import logging -import urllib -import webob -import webob.dec -import webob.exc - -from nova import crypto -from nova import wsgi -from nova.auth import manager -from nova.api.ec2 import cloud - - -_log = logging.getLogger("api") -_log.setLevel(logging.DEBUG) - - -class API(wsgi.Application): - - def __init__(self): - self.controller = cloud.CloudController() - - @webob.dec.wsgify - def __call__(self, req): - if req.method == 'POST': - return self.sign_csr(req) - _log.debug("Cloudpipe path is %s" % req.path_info) - if req.path_info.endswith("/getca/"): - return self.send_root_ca(req) - return webob.exc.HTTPNotFound() - - def get_project_id_from_ip(self, ip): - # TODO(eday): This was removed with the ORM branch, fix! - instance = self.controller.get_instance_by_ip(ip) - return instance['project_id'] - - def send_root_ca(self, req): - _log.debug("Getting root ca") - project_id = self.get_project_id_from_ip(req.remote_addr) - res = webob.Response() - res.headers["Content-Type"] = "text/plain" - res.body = crypto.fetch_ca(project_id) - return res - - def sign_csr(self, req): - project_id = self.get_project_id_from_ip(req.remote_addr) - cert = self.str_params['cert'] - return crypto.sign_csr(urllib.unquote(cert), project_id) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 001a96875..c6d4b6e53 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -64,12 +64,8 @@ flags.DEFINE_string('credential_key_file', 'pk.pem', 'Filename of private key in credentials zip') flags.DEFINE_string('credential_cert_file', 'cert.pem', 'Filename of certificate in credentials zip') -flags.DEFINE_string('credential_rc_file', 'novarc', +flags.DEFINE_string('credential_rc_file', '%src', 'Filename of rc in credentials zip') -flags.DEFINE_string('credential_cert_subject', - '/C=US/ST=California/L=MountainView/O=AnsoLabs/' - 'OU=NovaDev/CN=%s-%s', - 'Subject for certificate for users') flags.DEFINE_string('auth_driver', 'nova.auth.dbdriver.DbDriver', 'Driver that auth manager uses') @@ -625,27 +621,37 @@ class AuthManager(object): with self.driver() as drv: drv.modify_user(uid, access_key, secret_key, admin) - def get_credentials(self, user, project=None): + def get_credentials(self, user, project=None, use_dmz=True): """Get credential zip for user in project""" if not isinstance(user, User): user = self.get_user(user) if project is None: project = user.id pid = Project.safe_id(project) - rc = self.__generate_rc(user.access, user.secret, pid) - private_key, signed_cert = self._generate_x509_cert(user.id, pid) + private_key, signed_cert = crypto.generate_x509_cert(user.id, pid) tmpdir = tempfile.mkdtemp() zf = os.path.join(tmpdir, "temp.zip") zippy = zipfile.ZipFile(zf, 'w') - zippy.writestr(FLAGS.credential_rc_file, rc) + if use_dmz and FLAGS.region_list: + regions = {} + for item in FLAGS.region_list: + region, _sep, region_host = item.partition("=") + regions[region] = region_host + else: + regions = {'nova': FLAGS.cc_host} + for region, host in regions.iteritems(): + rc = self.__generate_rc(user.access, + user.secret, + pid, + use_dmz, + host) + zippy.writestr(FLAGS.credential_rc_file % region, rc) + zippy.writestr(FLAGS.credential_key_file, private_key) zippy.writestr(FLAGS.credential_cert_file, signed_cert) - try: - (vpn_ip, vpn_port) = self.get_project_vpn_data(project) - except exception.NotFound: - vpn_ip = None + (vpn_ip, vpn_port) = self.get_project_vpn_data(project) if vpn_ip: configfile = open(FLAGS.vpn_client_template, "r") s = string.Template(configfile.read()) @@ -656,10 +662,9 @@ class AuthManager(object): port=vpn_port) zippy.writestr(FLAGS.credential_vpn_file, config) else: - logging.warn("No vpn data for project %s" % - pid) + LOG.warn("No vpn data for project %s", pid) - zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(user.id)) + zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(pid)) zippy.close() with open(zf, 'rb') as f: read_buffer = f.read() @@ -667,38 +672,38 @@ class AuthManager(object): shutil.rmtree(tmpdir) return read_buffer - def get_environment_rc(self, user, project=None): + def get_environment_rc(self, user, project=None, use_dmz=True): """Get credential zip for user in project""" if not isinstance(user, User): user = self.get_user(user) if project is None: project = user.id pid = Project.safe_id(project) - return self.__generate_rc(user.access, user.secret, pid) + return self.__generate_rc(user.access, user.secret, pid, use_dmz) @staticmethod - def __generate_rc(access, secret, pid): + def __generate_rc(access, secret, pid, use_dmz=True, host=None): """Generate rc file for user""" + if use_dmz: + cc_host = FLAGS.cc_dmz + else: + cc_host = FLAGS.cc_host + # NOTE(vish): Always use the dmz since it is used from inside the + # instance + s3_host = FLAGS.s3_dmz + if host: + s3_host = host + cc_host = host rc = open(FLAGS.credentials_template).read() rc = rc % {'access': access, 'project': pid, 'secret': secret, - 'ec2': FLAGS.ec2_url, - 's3': 'http://%s:%s' % (FLAGS.s3_host, FLAGS.s3_port), + 'ec2': '%s://%s:%s%s' % (FLAGS.ec2_prefix, + cc_host, + FLAGS.cc_port, + FLAGS.ec2_suffix), + 's3': 'http://%s:%s' % (s3_host, FLAGS.s3_port), 'nova': FLAGS.ca_file, 'cert': FLAGS.credential_cert_file, 'key': FLAGS.credential_key_file} return rc - - def _generate_x509_cert(self, uid, pid): - """Generate x509 cert for user""" - (private_key, csr) = crypto.generate_x509_cert( - self.__cert_subject(uid)) - # TODO(joshua): This should be async call back to the cloud controller - signed_cert = crypto.sign_csr(csr, pid) - return (private_key, signed_cert) - - @staticmethod - def __cert_subject(uid): - """Helper to generate cert subject""" - return FLAGS.credential_cert_subject % (uid, utils.isotime()) diff --git a/nova/cloudpipe/bootscript.sh b/nova/cloudpipe/bootscript.sh deleted file mode 100755 index 30d9ad102..000000000 --- a/nova/cloudpipe/bootscript.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This gets zipped and run on the cloudpipe-managed OpenVPN server - -export SUPERVISOR="http://10.255.255.1:8773/cloudpipe" -export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{print $1}'` -export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $1}'` -export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $1}'` -export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` -export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-vpn-$VPN_IP" - -DHCP_LOWER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 10 }'` -DHCP_UPPER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 1 }'` - -# generate a server DH -openssl dhparam -out /etc/openvpn/dh1024.pem 1024 - -# generate a server priv key -openssl genrsa -out /etc/openvpn/server.key 2048 - -# generate a server CSR -openssl req -new -key /etc/openvpn/server.key -out /etc/openvpn/server.csr -batch -subj "$SUBJ" - -# URLEncode the CSR -CSRTEXT=`cat /etc/openvpn/server.csr` -CSRTEXT=$(python -c "import urllib; print urllib.quote('''$CSRTEXT''')") - -# SIGN the csr and save as server.crt -# CURL fetch to the supervisor, POSTing the CSR text, saving the result as the CRT file -curl --fail $SUPERVISOR -d "cert=$CSRTEXT" > /etc/openvpn/server.crt -curl --fail $SUPERVISOR/getca/ > /etc/openvpn/ca.crt - -# Customize the server.conf.template -cd /etc/openvpn - -sed -e s/VPN_IP/$VPN_IP/g server.conf.template > server.conf -sed -i -e s/DHCP_SUBNET/$DHCP_MASK/g server.conf -sed -i -e s/DHCP_LOWER/$DHCP_LOWER/g server.conf -sed -i -e s/DHCP_UPPER/$DHCP_UPPER/g server.conf -sed -i -e s/max-clients\ 1/max-clients\ 10/g server.conf - -echo "\npush \"route 10.255.255.1 255.255.255.255 $GATEWAY\"\n" >> server.conf -echo "\npush \"route 10.255.255.253 255.255.255.255 $GATEWAY\"\n" >> server.conf -echo "\nduplicate-cn\n" >> server.conf - -/etc/init.d/openvpn start diff --git a/nova/cloudpipe/bootscript.template b/nova/cloudpipe/bootscript.template new file mode 100755 index 000000000..11578c134 --- /dev/null +++ b/nova/cloudpipe/bootscript.template @@ -0,0 +1,50 @@ +#!/bin/bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This gets zipped and run on the cloudpipe-managed OpenVPN server + +export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{print $$1}'` +export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $$1}'` +export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $$1}'` +export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` + +DHCP_LOWER=`echo $$BROADCAST | awk -F. '{print $$1"."$$2"."$$3"." $$4 - ${num_vpn} }'` +DHCP_UPPER=`echo $$BROADCAST | awk -F. '{print $$1"."$$2"."$$3"." $$4 - 1 }'` + +# generate a server DH +openssl dhparam -out /etc/openvpn/dh1024.pem 1024 + +cp crl.pem /etc/openvpn/ +cp server.key /etc/openvpn/ +cp ca.crt /etc/openvpn/ +cp server.crt /etc/openvpn/ +# Customize the server.conf.template +cd /etc/openvpn + +sed -e s/VPN_IP/$$VPN_IP/g server.conf.template > server.conf +sed -i -e s/DHCP_SUBNET/$$DHCP_MASK/g server.conf +sed -i -e s/DHCP_LOWER/$$DHCP_LOWER/g server.conf +sed -i -e s/DHCP_UPPER/$$DHCP_UPPER/g server.conf +sed -i -e s/max-clients\ 1/max-clients\ 10/g server.conf + +echo "push \"route ${dmz_net} ${dmz_mask} $$GATEWAY\"" >> server.conf +echo "duplicate-cn" >> server.conf +echo "crl-verify /etc/openvpn/crl.pem" >> server.conf + +/etc/init.d/openvpn start diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 3472201cd..147b9dd7b 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -22,13 +22,15 @@ an instance with it. """ -import base64 import logging import os +import string import tempfile import zipfile from nova import context +from nova import crypto +from nova import db from nova import exception from nova import flags from nova import utils @@ -39,8 +41,17 @@ from nova.api.ec2 import cloud FLAGS = flags.FLAGS flags.DEFINE_string('boot_script_template', - utils.abspath('cloudpipe/bootscript.sh'), + utils.abspath('cloudpipe/bootscript.template'), 'Template for script to run on cloudpipe instance boot') +flags.DEFINE_string('dmz_net', + '10.0.0.0', + 'Network to push into openvpn config') +flags.DEFINE_string('dmz_mask', + '255.255.255.0', + 'Netmask to push into openvpn config') + + +LOG = logging.getLogger('nova-cloudpipe') class CloudPipe(object): @@ -48,64 +59,96 @@ class CloudPipe(object): self.controller = cloud.CloudController() self.manager = manager.AuthManager() - def launch_vpn_instance(self, project_id): - logging.debug("Launching VPN for %s" % (project_id)) - project = self.manager.get_project(project_id) + def get_encoded_zip(self, project_id): # Make a payload.zip tmpfolder = tempfile.mkdtemp() filename = "payload.zip" zippath = os.path.join(tmpfolder, filename) z = zipfile.ZipFile(zippath, "w", zipfile.ZIP_DEFLATED) - - z.write(FLAGS.boot_script_template, 'autorun.sh') + shellfile = open(FLAGS.boot_script_template, "r") + s = string.Template(shellfile.read()) + shellfile.close() + boot_script = s.substitute(cc_dmz=FLAGS.cc_dmz, + cc_port=FLAGS.cc_port, + dmz_net=FLAGS.dmz_net, + dmz_mask=FLAGS.dmz_mask, + num_vpn=FLAGS.cnt_vpn_clients) + # genvpn, sign csr + crypto.generate_vpn_files(project_id) + z.writestr('autorun.sh', boot_script) + crl = os.path.join(crypto.ca_folder(project_id), 'crl.pem') + z.write(crl, 'crl.pem') + server_key = os.path.join(crypto.ca_folder(project_id), 'server.key') + z.write(server_key, 'server.key') + ca_crt = os.path.join(crypto.ca_path(project_id)) + z.write(ca_crt, 'ca.crt') + server_crt = os.path.join(crypto.ca_folder(project_id), 'server.crt') + z.write(server_crt, 'server.crt') z.close() - - key_name = self.setup_key_pair(project.project_manager_id, project_id) zippy = open(zippath, "r") - context = context.RequestContext(user=project.project_manager, - project=project) - - reservation = self.controller.run_instances(context, - # Run instances expects encoded userdata, it is decoded in the - # get_metadata_call. autorun.sh also decodes the zip file, hence - # the double encoding. - user_data=zippy.read().encode("base64").encode("base64"), + # NOTE(vish): run instances expects encoded userdata, it is decoded + # in the get_metadata_call. autorun.sh also decodes the zip file, + # hence the double encoding. + encoded = zippy.read().encode("base64").encode("base64") + zippy.close() + return encoded + + def launch_vpn_instance(self, project_id): + LOG.debug("Launching VPN for %s" % (project_id)) + project = self.manager.get_project(project_id) + ctxt = context.RequestContext(user=project.project_manager, + project=project) + key_name = self.setup_key_pair(ctxt) + group_name = self.setup_security_group(ctxt) + + reservation = self.controller.run_instances(ctxt, + user_data=self.get_encoded_zip(project_id), max_count=1, min_count=1, instance_type='m1.tiny', image_id=FLAGS.vpn_image_id, key_name=key_name, - security_groups=["vpn-secgroup"]) - zippy.close() + security_group=[group_name]) + + def setup_security_group(self, context): + group_name = '%s%s' % (context.project.id, FLAGS.vpn_key_suffix) + if db.security_group_exists(context, context.project.id, group_name): + return group_name + group = {'user_id': context.user.id, + 'project_id': context.project.id, + 'name': group_name, + 'description': 'Group for vpn'} + group_ref = db.security_group_create(context, group) + rule = {'parent_group_id': group_ref['id'], + 'cidr': '0.0.0.0/0', + 'protocol': 'udp', + 'from_port': 1194, + 'to_port': 1194} + db.security_group_rule_create(context, rule) + rule = {'parent_group_id': group_ref['id'], + 'cidr': '0.0.0.0/0', + 'protocol': 'icmp', + 'from_port': -1, + 'to_port': -1} + db.security_group_rule_create(context, rule) + # NOTE(vish): No need to trigger the group since the instance + # has not been run yet. + return group_name - def setup_key_pair(self, user_id, project_id): - key_name = '%s%s' % (project_id, FLAGS.vpn_key_suffix) + def setup_key_pair(self, context): + key_name = '%s%s' % (context.project.id, FLAGS.vpn_key_suffix) try: - private_key, fingerprint = self.manager.generate_key_pair(user_id, - key_name) + result = cloud._gen_key(context, context.user.id, key_name) + private_key = result['private_key'] try: - key_dir = os.path.join(FLAGS.keys_path, user_id) + key_dir = os.path.join(FLAGS.keys_path, context.user.id) if not os.path.exists(key_dir): os.makedirs(key_dir) - file_name = os.path.join(key_dir, '%s.pem' % key_name) - with open(file_name, 'w') as f: + key_path = os.path.join(key_dir, '%s.pem' % key_name) + with open(key_path, 'w') as f: f.write(private_key) except: pass except exception.Duplicate: pass return key_name - - # def setup_secgroups(self, username): - # conn = self.euca.connection_for(username) - # try: - # secgroup = conn.create_security_group("vpn-secgroup", - # "vpn-secgroup") - # secgroup.authorize(ip_protocol = "udp", from_port = "1194", - # to_port = "1194", cidr_ip = "0.0.0.0/0") - # secgroup.authorize(ip_protocol = "tcp", from_port = "80", - # to_port = "80", cidr_ip = "0.0.0.0/0") - # secgroup.authorize(ip_protocol = "tcp", from_port = "22", - # to_port = "22", cidr_ip = "0.0.0.0/0") - # except: - # pass diff --git a/nova/crypto.py b/nova/crypto.py index 16b4f5e1f..9e29f0b8d 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -17,7 +17,7 @@ # under the License. """ -Wrappers around standard crypto, including root and intermediate CAs, +Wrappers around standard crypto, including root and project CAs, SSH key_pairs and x509 certificates. """ @@ -33,28 +33,57 @@ import utils import M2Crypto -from nova import exception +from nova import context +from nova import db from nova import flags FLAGS = flags.FLAGS flags.DEFINE_string('ca_file', 'cacert.pem', 'Filename of root CA') +flags.DEFINE_string('key_file', + os.path.join('private', 'cakey.pem'), + 'Filename of private key') +flags.DEFINE_string('crl_file', 'crl.pem', + 'Filename of root Certificate Revokation List') flags.DEFINE_string('keys_path', utils.abspath('../keys'), 'Where we keep our keys') flags.DEFINE_string('ca_path', utils.abspath('../CA'), 'Where we keep our root CA') -flags.DEFINE_boolean('use_intermediate_ca', False, - 'Should we use intermediate CAs for each project?') +flags.DEFINE_boolean('use_project_ca', False, + 'Should we use a CA for each project?') +flags.DEFINE_string('user_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=%s-%s-%s', + 'Subject for certificate for users, ' + '%s for project, user, timestamp') +flags.DEFINE_string('project_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=project-ca-%s-%s', + 'Subject for certificate for projects, ' + '%s for project, timestamp') +flags.DEFINE_string('vpn_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=project-vpn-%s-%s', + 'Subject for certificate for vpns, ' + '%s for project, timestamp') -def ca_path(project_id): - if project_id: - return "%s/INTER/%s/cacert.pem" % (FLAGS.ca_path, project_id) - return "%s/cacert.pem" % (FLAGS.ca_path) +def ca_folder(project_id=None): + if FLAGS.use_project_ca and project_id: + return os.path.join(FLAGS.ca_path, 'projects', project_id) + return FLAGS.ca_path + + +def ca_path(project_id=None): + return os.path.join(ca_folder(project_id), FLAGS.ca_file) + + +def key_path(project_id=None): + return os.path.join(ca_folder(project_id), FLAGS.key_file) def fetch_ca(project_id=None, chain=True): - if not FLAGS.use_intermediate_ca: + if not FLAGS.use_project_ca: project_id = None buffer = "" if project_id: @@ -91,8 +120,8 @@ def generate_key_pair(bits=1024): def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'): - pub_key_buffer = M2Crypto.BIO.MemoryBuffer(ssl_public_key) - rsa_key = M2Crypto.RSA.load_pub_key_bio(pub_key_buffer) + buf = M2Crypto.BIO.MemoryBuffer(ssl_public_key) + rsa_key = M2Crypto.RSA.load_pub_key_bio(buf) e, n = rsa_key.pub() key_type = 'ssh-rsa' @@ -105,53 +134,137 @@ def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'): return '%s %s %s@%s\n' % (key_type, b64_blob, name, suffix) -def generate_x509_cert(subject, bits=1024): +def revoke_cert(project_id, file_name): + """Revoke a cert by file name""" + start = os.getcwd() + os.chdir(ca_folder(project_id)) + # NOTE(vish): potential race condition here + utils.execute("openssl ca -config ./openssl.cnf -revoke '%s'" % file_name) + utils.execute("openssl ca -gencrl -config ./openssl.cnf -out '%s'" % + FLAGS.crl_file) + os.chdir(start) + + +def revoke_certs_by_user(user_id): + """Revoke all user certs""" + admin = context.get_admin_context() + for cert in db.certificate_get_all_by_user(admin, user_id): + revoke_cert(cert['project_id'], cert['file_name']) + + +def revoke_certs_by_project(project_id): + """Revoke all project certs""" + # NOTE(vish): This is somewhat useless because we can just shut down + # the vpn. + admin = context.get_admin_context() + for cert in db.certificate_get_all_by_project(admin, project_id): + revoke_cert(cert['project_id'], cert['file_name']) + + +def revoke_certs_by_user_and_project(user_id, project_id): + """Revoke certs for user in project""" + admin = context.get_admin_context() + for cert in db.certificate_get_all_by_user(admin, user_id, project_id): + revoke_cert(cert['project_id'], cert['file_name']) + + +def _project_cert_subject(project_id): + """Helper to generate user cert subject""" + return FLAGS.project_cert_subject % (project_id, utils.isotime()) + + +def _vpn_cert_subject(project_id): + """Helper to generate user cert subject""" + return FLAGS.vpn_cert_subject % (project_id, utils.isotime()) + + +def _user_cert_subject(user_id, project_id): + """Helper to generate user cert subject""" + return FLAGS.user_cert_subject % (project_id, user_id, utils.isotime()) + + +def generate_x509_cert(user_id, project_id, bits=1024): + """Generate and sign a cert for user in project""" + subject = _user_cert_subject(user_id, project_id) tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - logging.debug("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.runthis("Generating private key: %s", - "openssl genrsa -out %s %s" % (keyfile, bits)) - utils.runthis("Generating CSR: %s", - "openssl req -new -key %s -out %s -batch -subj %s" % + utils.execute("openssl genrsa -out %s %s" % (keyfile, bits)) + utils.execute("openssl req -new -key %s -out %s -batch -subj %s" % (keyfile, csrfile, subject)) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) - return (private_key, csr) - - -def sign_csr(csr_text, intermediate=None): - if not FLAGS.use_intermediate_ca: - intermediate = None - if not intermediate: - return _sign_csr(csr_text, FLAGS.ca_path) - user_ca = "%s/INTER/%s" % (FLAGS.ca_path, intermediate) - if not os.path.exists(user_ca): + (serial, signed_csr) = sign_csr(csr, project_id) + strserial = "%X" % serial + if(len(strserial) % 2): + strserial = "0%s" % strserial + fname = os.path.join(ca_folder(project_id), "newcerts/%s.pem" % strserial) + cert = {'user_id': user_id, + 'project_id': project_id, + 'file_name': fname} + db.certificate_create(context.get_admin_context(), cert) + return (private_key, signed_csr) + + +def _ensure_project_folder(project_id): + if not os.path.exists(ca_path(project_id)): start = os.getcwd() - os.chdir(FLAGS.ca_path) - utils.runthis("Generating intermediate CA: %s", - "sh geninter.sh %s" % (intermediate)) + os.chdir(ca_folder()) + utils.execute("sh geninter.sh %s %s" % + (project_id, _project_cert_subject(project_id))) os.chdir(start) - return _sign_csr(csr_text, user_ca) + + +def generate_vpn_files(project_id): + project_folder = ca_folder(project_id) + csr_fn = os.path.join(project_folder, "server.csr") + crt_fn = os.path.join(project_folder, "server.crt") + + if os.path.exists(crt_fn): + return + _ensure_project_folder(project_id) + start = os.getcwd() + os.chdir(ca_folder()) + # TODO(vish): the shell scripts could all be done in python + utils.execute("sh genvpn.sh %s %s" % + (project_id, _vpn_cert_subject(project_id))) + with open(csr_fn, "r") as csrfile: + csr_text = csrfile.read() + (serial, signed_csr) = sign_csr(csr_text, project_id) + with open(crt_fn, "w") as crtfile: + crtfile.write(signed_csr) + os.chdir(start) + + +def sign_csr(csr_text, project_id=None): + if not FLAGS.use_project_ca: + project_id = None + if not project_id: + return _sign_csr(csr_text, ca_folder()) + _ensure_project_folder(project_id) + project_folder = ca_folder(project_id) + return _sign_csr(csr_text, ca_folder(project_id)) def _sign_csr(csr_text, ca_folder): tmpfolder = tempfile.mkdtemp() - csrfile = open("%s/inbound.csr" % (tmpfolder), "w") + inbound = os.path.join(tmpfolder, "inbound.csr") + outbound = os.path.join(tmpfolder, "outbound.csr") + csrfile = open(inbound, "w") csrfile.write(csr_text) csrfile.close() - logging.debug("Flags path: %s" % ca_folder) + logging.debug("Flags path: %s", ca_folder) start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.runthis("Signing cert: %s", - "openssl ca -batch -out %s/outbound.crt " - "-config ./openssl.cnf -infiles %s/inbound.csr" % - (tmpfolder, tmpfolder)) + utils.execute("openssl ca -batch -out %s -config " + "./openssl.cnf -infiles %s" % (outbound, inbound)) + out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) + serial = int(out.rpartition("=")[2]) os.chdir(start) - with open("%s/outbound.crt" % (tmpfolder), "r") as crtfile: - return crtfile.read() + with open(outbound, "r") as crtfile: + return (serial, crtfile.read()) def mkreq(bits, subject="foo", ca=0): @@ -159,8 +272,7 @@ def mkreq(bits, subject="foo", ca=0): req = M2Crypto.X509.Request() rsa = M2Crypto.RSA.gen_key(bits, 65537, callback=lambda: None) pk.assign_rsa(rsa) - # Should not be freed here - rsa = None + rsa = None # should not be freed here req.set_pubkey(pk) req.set_subject(subject) req.sign(pk, 'sha512') @@ -224,7 +336,6 @@ def mkcacert(subject='nova', years=1): # IN THE SOFTWARE. # http://code.google.com/p/boto - def compute_md5(fp): """ @type fp: file diff --git a/nova/db/api.py b/nova/db/api.py index 80563c452..aa77dd2ea 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -117,6 +117,45 @@ def service_update(context, service_id, values): ################### +def certificate_create(context, values): + """Create a certificate from the values dictionary.""" + return IMPL.certificate_create(context, values) + + +def certificate_destroy(context, certificate_id): + """Destroy the certificate or raise if it does not exist.""" + return IMPL.certificate_destroy(context, certificate_id) + + +def certificate_get_all_by_project(context, project_id): + """Get all certificates for a project.""" + return IMPL.certificate_get_all_by_project(context, project_id) + + +def certificate_get_all_by_user(context, user_id): + """Get all certificates for a user.""" + return IMPL.certificate_get_all_by_user(context, user_id) + + +def certificate_get_all_by_user_and_project(context, user_id, project_id): + """Get all certificates for a user and project.""" + return IMPL.certificate_get_all_by_user_and_project(context, + user_id, + project_id) + + +def certificate_update(context, certificate_id, values): + """Set the given properties on an certificate and update it. + + Raises NotFound if service does not exist. + + """ + return IMPL.service_update(context, certificate_id, values) + + +################### + + def floating_ip_allocate_address(context, host, project_id): """Allocate free floating ip and return the address. diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index db4d9f68f..9ae12f178 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -253,6 +253,84 @@ def service_update(context, service_id, values): ################### +@require_admin_context +def certificate_get(context, certificate_id, session=None): + if not session: + session = get_session() + + result = session.query(models.Certificate).\ + filter_by(id=certificate_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + + if not result: + raise exception.NotFound('No certificate for id %s' % certificate_id) + + return result + + +@require_admin_context +def certificate_create(context, values): + certificate_ref = models.Certificate() + for (key, value) in values.iteritems(): + certificate_ref[key] = value + certificate_ref.save() + return certificate_ref + + +@require_admin_context +def certificate_destroy(context, certificate_id): + session = get_session() + with session.begin(): + certificate_ref = certificate_get(context, + certificate_id, + session=session) + certificate_ref.delete(session=session) + + +@require_admin_context +def certificate_get_all_by_project(context, project_id): + session = get_session() + return session.query(models.Certificate).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def certificate_get_all_by_user(context, user_id): + session = get_session() + return session.query(models.Certificate).\ + filter_by(user_id=user_id).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def certificate_get_all_by_user_and_project(_context, user_id, project_id): + session = get_session() + return session.query(models.Certificate).\ + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def certificate_update(context, certificate_id, values): + session = get_session() + with session.begin(): + certificate_ref = certificate_get(context, + certificate_id, + session=session) + for (key, value) in values.iteritems(): + certificate_ref[key] = value + certificate_ref.save(session=session) + + +################### + + @require_context def floating_ip_allocate_address(context, host, project_id): authorize_project_context(context, project_id) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 1111b5cbd..434f88e5e 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -151,6 +151,16 @@ class Service(BASE, NovaBase): disabled = Column(Boolean, default=False) +class Certificate(BASE, NovaBase): + """Represents a an x509 certificate""" + __tablename__ = 'certificates' + id = Column(Integer, primary_key=True) + + user_id = Column(String(255)) + project_id = Column(String(255)) + file_name = Column(String(255)) + + class Instance(BASE, NovaBase): """Represents a guest vm""" __tablename__ = 'instances' @@ -521,7 +531,7 @@ def register_models(): """Register Models and create metadata""" from sqlalchemy import create_engine models = (Service, Instance, Volume, ExportDevice, IscsiTarget, FixedIp, - FloatingIp, Network, SecurityGroup, + FloatingIp, Network, SecurityGroup, Certificate, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, Project) # , Image, Host engine = create_engine(FLAGS.sql_connection, echo=False) diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index fe891beee..0d2082bdb 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -208,17 +208,13 @@ class AuthManagerTestCase(object): # so it probably belongs in crypto_unittest # but I'm leaving it where I found it. with user_and_project_generator(self.manager) as (user, project): - # NOTE(todd): Should mention why we must setup controller first - # (somebody please clue me in) - cloud_controller = cloud.CloudController() - cloud_controller.setup() - _key, cert_str = self.manager._generate_x509_cert('test1', - 'testproj') + # NOTE(vish): Setup runs genroot.sh if it hasn't been run + cloud.CloudController().setup() + _key, cert_str = crypto.generate_x509_cert(user.id, project.id) logging.debug(cert_str) - # Need to verify that it's signed by the right intermediate CA - full_chain = crypto.fetch_ca(project_id='testproj', chain=True) - int_cert = crypto.fetch_ca(project_id='testproj', chain=False) + full_chain = crypto.fetch_ca(project_id=project.id, chain=True) + int_cert = crypto.fetch_ca(project_id=project.id, chain=False) cloud_cert = crypto.fetch_ca() logging.debug("CA chain:\n\n =====\n%s\n\n=====" % full_chain) signed_cert = X509.load_cert_string(cert_str) @@ -227,7 +223,8 @@ class AuthManagerTestCase(object): cloud_cert = X509.load_cert_string(cloud_cert) self.assertTrue(signed_cert.verify(chain_cert.get_pubkey())) self.assertTrue(signed_cert.verify(int_cert.get_pubkey())) - if not FLAGS.use_intermediate_ca: + + if not FLAGS.use_project_ca: self.assertTrue(signed_cert.verify(cloud_cert.get_pubkey())) else: self.assertFalse(signed_cert.verify(cloud_cert.get_pubkey())) -- cgit From 79fa23620a1846eae28eaed26cd79973571d6b99 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 6 Nov 2010 00:58:05 +0000 Subject: add dmz to flags and change a couple defaults --- nova/flags.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 4ae86d9b2..acdfc6f6d 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -180,7 +180,8 @@ DEFINE_list('region_list', 'list of region=url pairs separated by commas') DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') -DEFINE_string('s3_host', '127.0.0.1', 's3 host') +DEFINE_string('s3_host', '127.0.0.1', 's3 host (for infrastructure)') +DEFINE_string('s3_dmz', '127.0.0.1', 's3 dmz ip (for instances)') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') @@ -197,7 +198,8 @@ DEFINE_string('rabbit_userid', 'guest', 'rabbit userid') DEFINE_string('rabbit_password', 'guest', 'rabbit password') DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('cc_host', '127.0.0.1', 'ip of api server') +DEFINE_string('cc_host', '127.0.0.1', 'ip of api server (for infrastructure') +DEFINE_string('cc_dmz', '127.0.0.1', 'ip of api server (for instances)') DEFINE_integer('cc_port', 8773, 'cloud controller port') DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', 'Url to ec2 api server') @@ -211,10 +213,10 @@ DEFINE_string('default_ramdisk', 'ari-11111', DEFINE_string('default_instance_type', 'm1.small', 'default instance type to use, testing only') -DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server') +DEFINE_string('vpn_image_id', 'ami-cloudpipe', 'AMI for cloudpipe vpn server') DEFINE_string('vpn_key_suffix', - '-key', - 'Suffix to add to project name for vpn key') + '-vpn', + 'Suffix to add to project name for vpn key and secgroups') DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') -- cgit From e59e7e9c3ad3e25545555986dcce0c384f9a7b6e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 17 Nov 2010 02:33:09 +0000 Subject: remove extra line and ref. to LOG that doesn't exist --- bin/nova-manage | 1 - nova/auth/manager.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b788ee62d..4ab2e9832 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -167,7 +167,6 @@ class VpnCommands(object): 'vpn_public_port': int(port)}) - class ShellCommands(object): def bpython(self): """Runs a bpython shell. diff --git a/nova/auth/manager.py b/nova/auth/manager.py index c6d4b6e53..252c5e65b 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -662,7 +662,7 @@ class AuthManager(object): port=vpn_port) zippy.writestr(FLAGS.credential_vpn_file, config) else: - LOG.warn("No vpn data for project %s", pid) + logging.warn("No vpn data for project %s", pid) zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(pid)) zippy.close() -- cgit From 1638e71c4442187a315f44840453ad14cafb36ac Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 23 Nov 2010 21:16:25 +0000 Subject: the serial returned by x509 is already formatted in hex --- nova/crypto.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index 9e29f0b8d..a438d232b 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -196,10 +196,7 @@ def generate_x509_cert(user_id, project_id, bits=1024): csr = open(csrfile).read() shutil.rmtree(tmpdir) (serial, signed_csr) = sign_csr(csr, project_id) - strserial = "%X" % serial - if(len(strserial) % 2): - strserial = "0%s" % strserial - fname = os.path.join(ca_folder(project_id), "newcerts/%s.pem" % strserial) + fname = os.path.join(ca_folder(project_id), "newcerts/%s.pem" % serial) cert = {'user_id': user_id, 'project_id': project_id, 'file_name': fname} @@ -261,7 +258,7 @@ def _sign_csr(csr_text, ca_folder): utils.execute("openssl ca -batch -out %s -config " "./openssl.cnf -infiles %s" % (outbound, inbound)) out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) - serial = int(out.rpartition("=")[2]) + serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: return (serial, crtfile.read()) -- cgit From a7a24e2ac54b88ba09afe5966806f42181784e9d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 23 Nov 2010 21:48:32 +0000 Subject: add vpn ping and optimize vpn list --- bin/nova-manage | 27 +++++++++------------------ nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 12 ++++++++++++ nova/utils.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 4ab2e9832..3d0122637 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -106,39 +106,30 @@ class VpnCommands(object): projects = [self.manager.get_project(project)] else: projects = self.manager.get_projects() + # NOTE(vish): This hits the database a lot. We could optimize + # by getting all networks in one query and all vpns + # in aother query, then doing lookups by project for project in projects: print "%-12s\t" % project.name, ipport = "%s:%s" % (project.vpn_ip, project.vpn_port) print "%-20s\t" % ipport, - vpn = self._vpn_for(project.id) + ctxt = context.get_admin_context() + vpn = db.instance_get_project_vpn(ctxt, project.id) if vpn: - net = 'down' address = None + state = 'down' if vpn.get('fixed_ip', None): address = vpn['fixed_ip']['address'] - command = "ping -c1 -w1 %s > /dev/null; echo $?" - out, _err = utils.execute(command % address, - check_exit_code=False) - if out.strip() == '0': - net = 'up' + if utils.vpn_ping(project.vpn_ip, project.vpn_port): + state = 'up' print address, print vpn['host'], print vpn['ec2_id'], print vpn['state_description'], - print net - + print state else: print None - def _vpn_for(self, project_id): - """Get the VPN instance for a project ID.""" - ctxt = context.get_admin_context() - for instance in db.instance_get_all_by_project(ctxt, project_id): - if (instance['image_id'] == FLAGS.vpn_image_id - and not instance['state_description'] in - ['shutting_down', 'shutdown']): - return instance - def spawn(self): """Run all VPNs.""" for p in reversed(self.manager.get_projects()): diff --git a/nova/db/api.py b/nova/db/api.py index aa77dd2ea..5e0852c8f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -326,6 +326,11 @@ def instance_get_floating_address(context, instance_id): return IMPL.instance_get_floating_address(context, instance_id) +def instance_get_project_vpn(context, project_id): + """Get a vpn instance by project or return None.""" + return IMPL.instance_get_project_vpn(context, project_id) + + def instance_get_by_internal_id(context, internal_id): """Get an instance by ec2 id.""" return IMPL.instance_get_by_internal_id(context, internal_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 9ae12f178..01e0b1fd6 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -724,6 +724,18 @@ def instance_get_all_by_reservation(context, reservation_id): all() +@require_admin_context +def instance_get_project_vpn(context, project_id): + session = get_session() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(project_id=project_id).\ + filter_by(image_id=FLAGS.vpn_image_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + + @require_context def instance_get_by_internal_id(context, internal_id): session = get_session() diff --git a/nova/utils.py b/nova/utils.py index e7892a212..b3297acd9 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -27,6 +27,7 @@ import os import random import subprocess import socket +import struct import sys from xml.sax import saxutils @@ -61,6 +62,51 @@ def import_object(import_str): return cls() +def vpn_ping(address, port, timeout=0.05, session_id=None): + """Sends a vpn negotiation packet and returns the server session. + + Returns False on a failure. Basic packet structure is below. + + Client packet (14 bytes):: + 0 1 8 9 13 + +-+--------+-----+ + |x| cli_id |?????| + +-+--------+-----+ + x = packet identifier 0x38 + cli_id = 64 bit identifier + ? = unknown, probably flags/padding + + Server packet (26 bytes):: + 0 1 8 9 13 14 21 2225 + +-+--------+-----+--------+----+ + |x| srv_id |?????| cli_id |????| + +-+--------+-----+--------+----+ + x = packet identifier 0x40 + cli_id = 64 bit identifier + ? = unknown, probably flags/padding + bit 9 was 1 and the rest were 0 in testing + """ + if session_id is None: + session_id = random.randint(0, 0xffffffffffffffff) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + data = struct.pack("!BQxxxxxx", 0x38, session_id) + sock.sendto(data, (address, port)) + sock.settimeout(timeout) + try: + received = sock.recv(2048) + except socket.timeout: + return False + finally: + sock.close() + fmt = "!BQxxxxxQxxxx" + if len(received) != struct.calcsize(fmt): + print struct.calcsize(fmt) + return False + (identifier, server_sess, client_sess) = struct.unpack(fmt, received) + if identifier == 0x40 and client_sess == session_id: + return server_sess + + def fetchfile(url, target): logging.debug("Fetching %s" % url) # c = pycurl.Curl() -- cgit From f74094cd303b21c12e8a83790d405bcae8103be8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 24 Nov 2010 21:40:41 +0000 Subject: don't error on edge case where vpn has been launched but fails to get a network --- bin/nova-manage | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 3d0122637..7c07ce3f0 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -120,7 +120,8 @@ class VpnCommands(object): state = 'down' if vpn.get('fixed_ip', None): address = vpn['fixed_ip']['address'] - if utils.vpn_ping(project.vpn_ip, project.vpn_port): + if project.vpn_ip and utils.vpn_ping(project.vpn_ip, + project.vpn_port): state = 'up' print address, print vpn['host'], -- cgit From 99347717ed2c7e92b3dc3bd33c12a3a05e8e349d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 15 Dec 2010 00:25:04 +0000 Subject: Lockout middleware for ec2 api --- nova/api/ec2/__init__.py | 70 ++++++++++++++++++++++++++++++++- nova/fakememcache.py | 50 ++++++++++++++++++++++++ nova/tests/middleware_unittest.py | 82 +++++++++++++++++++++++++++++++++++++++ run_tests.py | 1 + 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 nova/fakememcache.py create mode 100644 nova/tests/middleware_unittest.py diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index a6ee16c33..19eb666cd 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -22,12 +22,13 @@ Starting point for routing EC2 requests. import logging import routes +import time import webob import webob.dec import webob.exc -from nova import exception from nova import context +from nova import exception from nova import flags from nova import wsgi from nova.api.ec2 import apirequest @@ -37,6 +38,16 @@ from nova.auth import manager FLAGS = flags.FLAGS +flags.DEFINE_boolean('use_lockout', False, + 'Whether or not to use lockout middleware.') +flags.DEFINE_integer('lockout_attempts', 5, + 'Number of failed auths before lockout.') +flags.DEFINE_integer('lockout_minutes', 15, + 'Number of minutes to lockout if triggered.') +flags.DEFINE_list('lockout_memcached_servers', None, + 'Memcached servers or None for in process cache.') + + _log = logging.getLogger("api") _log.setLevel(logging.DEBUG) @@ -47,6 +58,63 @@ class API(wsgi.Middleware): def __init__(self): self.application = Authenticate(Router(Authorizer(Executor()))) + if FLAGS.use_lockout: + self.application = Lockout(self.application) + + +class Lockout(wsgi.Middleware): + """Only allow x failed auths in a y minute period. + + x = lockout_attempts flag + y = lockout_timeout flag + + Uses memcached if lockout_memcached_servers flag is set, otherwise it + uses a very simple in-proccess cache. Due to the simplicity of + the implementation, the timeout window is reset with every failed + request, so it actually blocks if there are x failed logins with no + more than y minutes between any two failures. + + There is a possible race condition where simultaneous requests could + sneak in before the lockout hits, but this is extremely rare and would + only result in a couple of extra failed attempts.""" + + def __init__(self, application, time_fn=time.time): + """The middleware can use a custom time function for testing.""" + self.time_fn = time_fn + if FLAGS.lockout_memcached_servers: + import memcache + else: + from nova import fakememcache as memcache + self.mc = memcache.Client(FLAGS.lockout_memcached_servers, debug=0) + super(Lockout, self).__init__(application) + + @webob.dec.wsgify + def __call__(self, req): + access_key = req.params['AWSAccessKeyId'] + failures_key = "%s-failures" % access_key + last_key = "%s-last" % access_key + now = self.time_fn() + timeout = now - FLAGS.lockout_minutes * 60 + # NOTE(vish): To use incr, failures has to be a string. + failures = int(self.mc.get(failures_key) or 0) + last = self.mc.get(last_key) + if (failures and failures >= FLAGS.lockout_attempts + and last > timeout): + self.mc.set(last_key, now) + detail = "Too many failed authentications." + raise webob.exc.HTTPForbidden(detail=detail) + res = req.get_response(self.application) + if res.status_int == 403: + if last > timeout: + failures = int(self.mc.incr(failures_key)) + if failures >= FLAGS.lockout_attempts: + _log.warn('Access key %s has had %d failed authentications' + ' and will be locked out for %d minutes.' % + (access_key, failures, FLAGS.lockout_minutes)) + else: + self.mc.set(failures_key, '1') + self.mc.set(last_key, now) + return res class Authenticate(wsgi.Middleware): diff --git a/nova/fakememcache.py b/nova/fakememcache.py new file mode 100644 index 000000000..0b2e3b6c1 --- /dev/null +++ b/nova/fakememcache.py @@ -0,0 +1,50 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Super simple fake memcache client.""" + + +class Client(object): + """Replicates a tiny subset of memcached client interface.""" + __cache = {} + + def __init__(self, *args, **kwargs): + """Ignores all constructor params.""" + pass + + def get(self, key): + """Retrieves the value for a key or None.""" + return self.__cache.get(key, None) + + def set(self, key, value): + """Sets the value for a key.""" + self.__cache[key] = value + return True + + def add(self, key, value): + """Sets the value for a key if it doesn't exist.""" + if key in self.__cache: + return False + return self.set(key, value) + + def incr(self, key, delta=1): + """Increments the value for a key.""" + if not key in self.__cache: + return 0 + self.__cache[key] = str(int(self.__cache[key]) + 1) + return self.__cache[key] diff --git a/nova/tests/middleware_unittest.py b/nova/tests/middleware_unittest.py new file mode 100644 index 000000000..bbbd4a5a7 --- /dev/null +++ b/nova/tests/middleware_unittest.py @@ -0,0 +1,82 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import webob +import webob.dec +import webob.exc + +from nova.api import ec2 +from nova import flags +from nova import test + + +FLAGS = flags.FLAGS + + +@webob.dec.wsgify +def conditional_forbid(req): + """Helper wsgi app returns 403 if param 'die' is 1.""" + if 'die' in req.params and req.params['die'] == '1': + raise webob.exc.HTTPForbidden() + return 'OK' + + +class LockoutTestCase(test.TrialTestCase): + """Test case for the Lockout middleware.""" + def setUp(self): # pylint: disable-msg=C0103 + self.local_time = 0 + self.lockout = ec2.Lockout(conditional_forbid, + time_fn=self._constant_time) + super(LockoutTestCase, self).setUp() + + def _constant_time(self): + """Helper method to force timeouts.""" + return self.local_time + + def _trigger_lockout(self, access_key): + """Send x failed requests where x = lockout_attempts.""" + for i in xrange(FLAGS.lockout_attempts): + req = webob.Request.blank('/?AWSAccessKeyId=%s&die=1' % access_key) + self.assertEqual(req.get_response(self.lockout).status_int, 403) + + def _is_locked_out(self, access_key): + """Sends a test request to see if key is locked out.""" + req = webob.Request.blank('/?AWSAccessKeyId=%s' % access_key) + return (req.get_response(self.lockout).status_int == 403) + + def _timeout(self): + """Increment time to 1 second past the lockout.""" + self.local_time = 1 + self.local_time + FLAGS.lockout_minutes * 60 + + def test_lockout(self): + self._trigger_lockout('test') + self.assertTrue(self._is_locked_out('test')) + + def test_timeout(self): + self._trigger_lockout('test') + self.assertTrue(self._is_locked_out('test')) + self._timeout() + self.assertFalse(self._is_locked_out('test')) + + def test_multiple_keys(self): + self._trigger_lockout('test1') + self.assertTrue(self._is_locked_out('test1')) + self.assertFalse(self._is_locked_out('test2')) + self._timeout() + self.assertFalse(self._is_locked_out('test1')) + self.assertFalse(self._is_locked_out('test2')) diff --git a/run_tests.py b/run_tests.py index 37a548e4c..a0ef3fd99 100644 --- a/run_tests.py +++ b/run_tests.py @@ -57,6 +57,7 @@ from nova.tests.auth_unittest import * from nova.tests.cloud_unittest import * from nova.tests.compute_unittest import * from nova.tests.flags_unittest import * +from nova.tests.middleware_unittest import * from nova.tests.misc_unittest import * from nova.tests.network_unittest import * from nova.tests.objectstore_unittest import * -- cgit From b0279030127b7fe8df21db12a8727ea623ca46e2 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 15 Dec 2010 09:38:38 -0800 Subject: clean up code to use timeout instead of two keys --- nova/api/ec2/__init__.py | 58 ++++++++++++++++++--------------------- nova/fakememcache.py | 38 +++++++++++++++---------- nova/tests/middleware_unittest.py | 27 +++++++++++------- 3 files changed, 68 insertions(+), 55 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 19eb666cd..381b0e871 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -22,7 +22,6 @@ Starting point for routing EC2 requests. import logging import routes -import time import webob import webob.dec import webob.exc @@ -44,6 +43,8 @@ flags.DEFINE_integer('lockout_attempts', 5, 'Number of failed auths before lockout.') flags.DEFINE_integer('lockout_minutes', 15, 'Number of minutes to lockout if triggered.') +flags.DEFINE_integer('lockout_window', 15, + 'Number of minutes for lockout window.') flags.DEFINE_list('lockout_memcached_servers', None, 'Memcached servers or None for in process cache.') @@ -53,7 +54,6 @@ _log.setLevel(logging.DEBUG) class API(wsgi.Middleware): - """Routing for all EC2 API requests.""" def __init__(self): @@ -63,57 +63,53 @@ class API(wsgi.Middleware): class Lockout(wsgi.Middleware): - """Only allow x failed auths in a y minute period. + """Lockout for x minutes on y failed auths in a z minute period. - x = lockout_attempts flag - y = lockout_timeout flag + x = lockout_timeout flag + y = lockout_window flag + z = lockout_attempts flag Uses memcached if lockout_memcached_servers flag is set, otherwise it uses a very simple in-proccess cache. Due to the simplicity of - the implementation, the timeout window is reset with every failed - request, so it actually blocks if there are x failed logins with no - more than y minutes between any two failures. + the implementation, the timeout window is started with the first + failed request, so it will block if there are x failed logins within + that period. There is a possible race condition where simultaneous requests could sneak in before the lockout hits, but this is extremely rare and would only result in a couple of extra failed attempts.""" - def __init__(self, application, time_fn=time.time): - """The middleware can use a custom time function for testing.""" - self.time_fn = time_fn + def __init__(self, application, time_fn=None): + """middleware can pass a custom time function to fake for testing.""" if FLAGS.lockout_memcached_servers: import memcache + self.mc = memcache.Client(FLAGS.lockout_memcached_servers, + debug=0) else: - from nova import fakememcache as memcache - self.mc = memcache.Client(FLAGS.lockout_memcached_servers, debug=0) + from nova import fakememcache + self.mc = fakememcache.Client(time_fn=time_fn) super(Lockout, self).__init__(application) @webob.dec.wsgify def __call__(self, req): access_key = req.params['AWSAccessKeyId'] - failures_key = "%s-failures" % access_key - last_key = "%s-last" % access_key - now = self.time_fn() - timeout = now - FLAGS.lockout_minutes * 60 - # NOTE(vish): To use incr, failures has to be a string. + failures_key = "authfailures-%s" % access_key failures = int(self.mc.get(failures_key) or 0) - last = self.mc.get(last_key) - if (failures and failures >= FLAGS.lockout_attempts - and last > timeout): - self.mc.set(last_key, now) + if failures >= FLAGS.lockout_attempts: detail = "Too many failed authentications." raise webob.exc.HTTPForbidden(detail=detail) res = req.get_response(self.application) if res.status_int == 403: - if last > timeout: - failures = int(self.mc.incr(failures_key)) - if failures >= FLAGS.lockout_attempts: - _log.warn('Access key %s has had %d failed authentications' - ' and will be locked out for %d minutes.' % - (access_key, failures, FLAGS.lockout_minutes)) - else: - self.mc.set(failures_key, '1') - self.mc.set(last_key, now) + failures = self.mc.incr(failures_key) + if failures is None: + # NOTE(vish): To use incr, failures has to be a string. + self.mc.set(failures_key, '1', time=FLAGS.lockout_window * 60) + elif failures >= FLAGS.lockout_attempts: + _log.warn('Access key %s has had %d failed authentications' + ' and will be locked out for %d minutes.' % + (access_key, failures, FLAGS.lockout_minutes)) + self.mc.set(failures_key, str(failures), + time=FLAGS.lockout_minutes * 60) return res diff --git a/nova/fakememcache.py b/nova/fakememcache.py index 0b2e3b6c1..0b4037ef6 100644 --- a/nova/fakememcache.py +++ b/nova/fakememcache.py @@ -18,33 +18,43 @@ """Super simple fake memcache client.""" +import time + class Client(object): """Replicates a tiny subset of memcached client interface.""" - __cache = {} - def __init__(self, *args, **kwargs): - """Ignores all constructor params.""" - pass + def __init__(self, time_fn=time.time, *args, **kwargs): + """Time fn is to allow testing through a custom function""" + self.time_fn = time_fn + self.cache = {} def get(self, key): """Retrieves the value for a key or None.""" - return self.__cache.get(key, None) + (timeout, value) = self.cache.get(key, (0, None)) + if timeout == 0 or self.time_fn() < timeout: + return value + return None - def set(self, key, value): + def set(self, key, value, time=0, min_compress_len=0): """Sets the value for a key.""" - self.__cache[key] = value + timeout = 0 + if time != 0: + timeout = self.time_fn() + time + self.cache[key] = (timeout, value) return True - def add(self, key, value): + def add(self, key, value, time=0, min_compress_len=0): """Sets the value for a key if it doesn't exist.""" - if key in self.__cache: + if not self.get(key) is None: return False - return self.set(key, value) + return self.set(key, value, time, min_compress_len) def incr(self, key, delta=1): """Increments the value for a key.""" - if not key in self.__cache: - return 0 - self.__cache[key] = str(int(self.__cache[key]) + 1) - return self.__cache[key] + value = self.get(key) + if value is None: + return None + new_value = int(value) + delta + self.cache[key] = (self.cache[key][0], str(new_value)) + return new_value diff --git a/nova/tests/middleware_unittest.py b/nova/tests/middleware_unittest.py index bbbd4a5a7..61a790c1f 100644 --- a/nova/tests/middleware_unittest.py +++ b/nova/tests/middleware_unittest.py @@ -48,9 +48,9 @@ class LockoutTestCase(test.TrialTestCase): """Helper method to force timeouts.""" return self.local_time - def _trigger_lockout(self, access_key): - """Send x failed requests where x = lockout_attempts.""" - for i in xrange(FLAGS.lockout_attempts): + def _send_bad_attempts(self, access_key, num_attempts=1): + """Fail x.""" + for i in xrange(num_attempts): req = webob.Request.blank('/?AWSAccessKeyId=%s&die=1' % access_key) self.assertEqual(req.get_response(self.lockout).status_int, 403) @@ -59,24 +59,31 @@ class LockoutTestCase(test.TrialTestCase): req = webob.Request.blank('/?AWSAccessKeyId=%s' % access_key) return (req.get_response(self.lockout).status_int == 403) - def _timeout(self): + def _advance_time(self, time): """Increment time to 1 second past the lockout.""" - self.local_time = 1 + self.local_time + FLAGS.lockout_minutes * 60 + self.local_time = self.local_time + time def test_lockout(self): - self._trigger_lockout('test') + self._send_bad_attempts('test', FLAGS.lockout_attempts) self.assertTrue(self._is_locked_out('test')) def test_timeout(self): - self._trigger_lockout('test') + self._send_bad_attempts('test', FLAGS.lockout_attempts) self.assertTrue(self._is_locked_out('test')) - self._timeout() + self._advance_time(FLAGS.lockout_minutes * 60) self.assertFalse(self._is_locked_out('test')) def test_multiple_keys(self): - self._trigger_lockout('test1') + self._send_bad_attempts('test1', FLAGS.lockout_attempts) self.assertTrue(self._is_locked_out('test1')) self.assertFalse(self._is_locked_out('test2')) - self._timeout() + self._advance_time(FLAGS.lockout_minutes * 60) self.assertFalse(self._is_locked_out('test1')) self.assertFalse(self._is_locked_out('test2')) + + def test_window_timeout(self): + self._send_bad_attempts('test', FLAGS.lockout_attempts - 1) + self.assertFalse(self._is_locked_out('test')) + self._advance_time(FLAGS.lockout_window * 60) + self._send_bad_attempts('test', FLAGS.lockout_attempts - 1) + self.assertFalse(self._is_locked_out('test')) -- cgit From dd4ee43cc2042299ed7a56b4690999fa1df120a1 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 15 Dec 2010 11:23:33 -0800 Subject: clean up tests and add overriden time method to utils --- nova/api/ec2/__init__.py | 11 +++++----- nova/fakememcache.py | 11 +++++----- nova/tests/middleware_unittest.py | 23 ++++++++++----------- nova/utils.py | 42 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 60 insertions(+), 27 deletions(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 381b0e871..5ae15f2ae 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -79,15 +79,14 @@ class Lockout(wsgi.Middleware): sneak in before the lockout hits, but this is extremely rare and would only result in a couple of extra failed attempts.""" - def __init__(self, application, time_fn=None): - """middleware can pass a custom time function to fake for testing.""" + def __init__(self, application): + """middleware can use fake for testing.""" if FLAGS.lockout_memcached_servers: import memcache - self.mc = memcache.Client(FLAGS.lockout_memcached_servers, - debug=0) else: - from nova import fakememcache - self.mc = fakememcache.Client(time_fn=time_fn) + from nova import fakememcache as memcache + self.mc = memcache.Client(FLAGS.lockout_memcached_servers, + debug=0) super(Lockout, self).__init__(application) @webob.dec.wsgify diff --git a/nova/fakememcache.py b/nova/fakememcache.py index 0b4037ef6..67f46dbdc 100644 --- a/nova/fakememcache.py +++ b/nova/fakememcache.py @@ -18,21 +18,20 @@ """Super simple fake memcache client.""" -import time +import utils class Client(object): """Replicates a tiny subset of memcached client interface.""" - def __init__(self, time_fn=time.time, *args, **kwargs): - """Time fn is to allow testing through a custom function""" - self.time_fn = time_fn + def __init__(self, *args, **kwargs): + """Ignores the passed in args""" self.cache = {} def get(self, key): """Retrieves the value for a key or None.""" (timeout, value) = self.cache.get(key, (0, None)) - if timeout == 0 or self.time_fn() < timeout: + if timeout == 0 or utils.utcnow_ts() < timeout: return value return None @@ -40,7 +39,7 @@ class Client(object): """Sets the value for a key.""" timeout = 0 if time != 0: - timeout = self.time_fn() + time + timeout = utils.utcnow_ts() + time self.cache[key] = (timeout, value) return True diff --git a/nova/tests/middleware_unittest.py b/nova/tests/middleware_unittest.py index 61a790c1f..0febf52d6 100644 --- a/nova/tests/middleware_unittest.py +++ b/nova/tests/middleware_unittest.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime import webob import webob.dec import webob.exc @@ -23,6 +24,7 @@ import webob.exc from nova.api import ec2 from nova import flags from nova import test +from nova import utils FLAGS = flags.FLAGS @@ -39,14 +41,13 @@ def conditional_forbid(req): class LockoutTestCase(test.TrialTestCase): """Test case for the Lockout middleware.""" def setUp(self): # pylint: disable-msg=C0103 - self.local_time = 0 - self.lockout = ec2.Lockout(conditional_forbid, - time_fn=self._constant_time) super(LockoutTestCase, self).setUp() + utils.set_time_override() + self.lockout = ec2.Lockout(conditional_forbid) - def _constant_time(self): - """Helper method to force timeouts.""" - return self.local_time + def tearDown(self): # pylint: disable-msg=C0103 + utils.clear_time_override() + super(LockoutTestCase, self).tearDown() def _send_bad_attempts(self, access_key, num_attempts=1): """Fail x.""" @@ -59,10 +60,6 @@ class LockoutTestCase(test.TrialTestCase): req = webob.Request.blank('/?AWSAccessKeyId=%s' % access_key) return (req.get_response(self.lockout).status_int == 403) - def _advance_time(self, time): - """Increment time to 1 second past the lockout.""" - self.local_time = self.local_time + time - def test_lockout(self): self._send_bad_attempts('test', FLAGS.lockout_attempts) self.assertTrue(self._is_locked_out('test')) @@ -70,20 +67,20 @@ class LockoutTestCase(test.TrialTestCase): def test_timeout(self): self._send_bad_attempts('test', FLAGS.lockout_attempts) self.assertTrue(self._is_locked_out('test')) - self._advance_time(FLAGS.lockout_minutes * 60) + utils.advance_time_seconds(FLAGS.lockout_minutes * 60) self.assertFalse(self._is_locked_out('test')) def test_multiple_keys(self): self._send_bad_attempts('test1', FLAGS.lockout_attempts) self.assertTrue(self._is_locked_out('test1')) self.assertFalse(self._is_locked_out('test2')) - self._advance_time(FLAGS.lockout_minutes * 60) + utils.advance_time_seconds(FLAGS.lockout_minutes * 60) self.assertFalse(self._is_locked_out('test1')) self.assertFalse(self._is_locked_out('test2')) def test_window_timeout(self): self._send_bad_attempts('test', FLAGS.lockout_attempts - 1) self.assertFalse(self._is_locked_out('test')) - self._advance_time(FLAGS.lockout_window * 60) + utils.advance_time_seconds(FLAGS.lockout_window * 60) self._send_bad_attempts('test', FLAGS.lockout_attempts - 1) self.assertFalse(self._is_locked_out('test')) diff --git a/nova/utils.py b/nova/utils.py index 142584df8..048a9d974 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -21,7 +21,6 @@ System-level utilities and helper functions. """ import datetime -import functools import inspect import logging import os @@ -29,6 +28,7 @@ import random import subprocess import socket import sys +import time from xml.sax import saxutils from twisted.internet.threads import deferToThread @@ -165,13 +165,51 @@ def get_my_ip(): return "127.0.0.1" +def utcnow(): + """Overridable version of datetime.datetime.utcnow.""" + if utcnow.override_time: + return utcnow.override_time + return datetime.datetime.utcnow() + + +utcnow.override_time = None + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return time.mktime(utcnow().timetuple()) + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """Override utils.utcnow to return a constant time.""" + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overriden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overriden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + def isotime(at=None): + """Returns iso formatted utcnow.""" if not at: - at = datetime.datetime.utcnow() + at = utcnow() return at.strftime(TIME_FORMAT) def parse_isotime(timestr): + """Turn an iso formatted time back into a datetime""" return datetime.datetime.strptime(timestr, TIME_FORMAT) -- cgit From e1da5d66b2e33a043e7e9ee357d9769276d6e302 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 15 Dec 2010 13:14:28 -0800 Subject: memcached requires strings not unicode --- nova/api/ec2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5ae15f2ae..def0ee207 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -91,7 +91,7 @@ class Lockout(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): - access_key = req.params['AWSAccessKeyId'] + access_key = str(req.params['AWSAccessKeyId']) failures_key = "authfailures-%s" % access_key failures = int(self.mc.get(failures_key) or 0) if failures >= FLAGS.lockout_attempts: -- cgit From dc29400d104d34c6383132a43e018f7724e85ec3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 16 Dec 2010 19:13:37 +0000 Subject: use getent, update docstring --- CA/genvpn.sh | 36 ++++++++++++++++++++++++++++++++++++ nova/auth/manager.py | 3 ++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100755 CA/genvpn.sh diff --git a/CA/genvpn.sh b/CA/genvpn.sh new file mode 100755 index 000000000..7e7db185d --- /dev/null +++ b/CA/genvpn.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This gets zipped and run on the cloudpipe-managed OpenVPN server +NAME=$1 +SUBJ=$2 + +mkdir -p projects/$NAME +cd projects/$NAME + +# generate a server priv key +openssl genrsa -out server.key 2048 + +# generate a server CSR +openssl req -new -key server.key -out server.csr -batch -subj "$SUBJ" + +novauid=`getent passwd nova | awk -F: '{print $3}'` +if [ ! -z "${novauid}" ] && [ "`id -u`" != "${novauid}" ]; then + sudo chown -R nova:nogroup . +fi diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 783ef51af..735473027 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -65,7 +65,8 @@ flags.DEFINE_string('credential_key_file', 'pk.pem', flags.DEFINE_string('credential_cert_file', 'cert.pem', 'Filename of certificate in credentials zip') flags.DEFINE_string('credential_rc_file', '%src', - 'Filename of rc in credentials zip') + 'Filename of rc in credentials zip, %s will be ' + 'replaced by name of the region (nova by default)') flags.DEFINE_string('auth_driver', 'nova.auth.dbdriver.DbDriver', 'Driver that auth manager uses') -- cgit From a6f90bacda223add276698958b2e7479bb6841e9 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 16 Dec 2010 23:25:21 +0000 Subject: make sure all network data is recreated when nova-network is rebooted --- nova/network/manager.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index 6a30f30b7..815fbfdc2 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -112,6 +112,16 @@ class NetworkManager(manager.Manager): ctxt = context.get_admin_context() for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) + floating_ips = self.db.floating_ip_get_all_by_host(ctxt, + self.host) + for floating_ip in floating_ips: + if floating_ip.get('fixed_ip', None): + fixed_address = floating_ip['fixed_ip']['address'] + # NOTE(vish): The False here is because we ignore the case + # that the ip is already bound. + self.driver.bind_floating_ip(floating_ip['address'], False) + self.driver.ensure_floating_forward(floating_ip['address'], + fixed_address) def set_network_host(self, context, network_id): """Safely sets the host of the network.""" @@ -444,12 +454,7 @@ class VlanManager(NetworkManager): def setup_fixed_ip(self, context, address): """Sets forwarding rules and dhcp for fixed ip.""" - fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) network_ref = self.db.fixed_ip_get_network(context, address) - if self.db.instance_is_vpn(context, fixed_ip_ref['instance_id']): - self.driver.ensure_vlan_forward(network_ref['vpn_public_address'], - network_ref['vpn_public_port'], - network_ref['vpn_private_address']) self.driver.update_dhcp(context, network_ref['id']) def setup_compute_network(self, context, instance_id): @@ -497,13 +502,24 @@ class VlanManager(NetworkManager): def _on_set_network_host(self, context, network_id): """Called when this host becomes the host for a network.""" network_ref = self.db.network_get(context, network_id) - net = {} - net['vpn_public_address'] = FLAGS.vpn_ip - db.network_update(context, network_id, net) + if not network_ref['vpn_public_address']: + net = {} + address = FLAGS.vpn_ip + net['vpn_public_address'] = address + db.network_update(context, network_id, net) + else: + address = network_ref['vpn_public_address'] self.driver.ensure_vlan_bridge(network_ref['vlan'], network_ref['bridge'], network_ref) - self.driver.update_dhcp(context, network_id) + # NOTE(vish): only ensure this forward if the address hasn't been set + # manually. + if address == FLAGS.vpn_ip: + self.driver.ensure_vlan_forward(FLAGS.vpn_ip, + network_ref['vpn_public_port'], + network_ref['vpn_private_address']) + if not FLAGS.fake_network: + self.driver.update_dhcp(context, network_id) @property def _bottom_reserved_ips(self): -- cgit From 0d705117a0d0c04d845c5d146455cd11ba9af88c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 16 Dec 2010 23:47:14 +0000 Subject: add conditional bind to linux net --- nova/network/linux_net.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0fefd9415..40fe7619b 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -77,10 +77,11 @@ def init_host(): {'range': FLAGS.fixed_range}) -def bind_floating_ip(floating_ip): +def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface)) + FLAGS.public_interface), + check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): -- cgit From 86f71493fa5a02762bc7c56308c85b9182913efb Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 17 Dec 2010 00:43:18 +0000 Subject: move some flags around --- nova/api/__init__.py | 2 +- nova/api/ec2/cloud.py | 16 ++++++++++------ nova/flags.py | 12 +++++++----- nova/network/linux_net.py | 5 ++--- nova/utils.py | 5 ----- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 95fc6b145..803470570 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -29,7 +29,6 @@ import routes import webob.dec from nova import flags -from nova import utils from nova import wsgi from nova.api import ec2 from nova.api import openstack @@ -40,6 +39,7 @@ flags.DEFINE_string('osapi_subdomain', 'api', 'subdomain running the OpenStack API') flags.DEFINE_string('ec2api_subdomain', 'ec2', 'subdomain running the EC2 API') + FLAGS = flags.FLAGS diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index ebb13aedc..684e29ee1 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -196,15 +196,19 @@ class CloudController(object): if FLAGS.region_list: regions = [] for region in FLAGS.region_list: - name, _sep, url = region.partition('=') + name, _sep, host = region.partition('=') + endpoint = '%s://%s:%s%s' % (FLAGS.ec2_prefix, + host, + FLAGS.cc_port, + FLAGS.ec2_suffix) regions.append({'regionName': name, - 'regionEndpoint': url}) + 'regionEndpoint': endpoint}) else: regions = [{'regionName': 'nova', - 'regionEndpoint': FLAGS.ec2_url}] - if region_name: - regions = [r for r in regions if r['regionName'] in region_name] - return {'regionInfo': regions} + 'regionEndpoint': '%s://%s:%s%s' % (FLAGS.ec2_prefix, + FLAGS.cc_host, + FLAGS.cc_port, + FLAGS.ec2_suffix)}] def describe_snapshots(self, context, diff --git a/nova/flags.py b/nova/flags.py index 9e99ffb5e..95bfd3773 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -29,6 +29,7 @@ import sys import gflags +from nova import utils class FlagValues(gflags.FlagValues): """Extension of gflags.FlagValues that allows undefined and runtime flags. @@ -211,8 +212,8 @@ DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') DEFINE_integer('s3_port', 3333, 's3 port') -DEFINE_string('s3_host', '127.0.0.1', 's3 host (for infrastructure)') -DEFINE_string('s3_dmz', '127.0.0.1', 's3 dmz ip (for instances)') +DEFINE_string('s3_host', utils.get_my_ip(), 's3 host (for infrastructure)') +DEFINE_string('s3_dmz', utils.get_my_ip(), 's3 dmz ip (for instances)') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') @@ -231,10 +232,11 @@ DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('cc_dmz', '127.0.0.1', 'ip of api server (for instances)') +DEFINE_string('ec2_prefix', 'http', 'prefix for ec2') +DEFINE_string('cc_host', utils.get_my_ip(), 'ip of api server') +DEFINE_string('cc_dmz', utils.get_my_ip(), 'internal ip of api server') DEFINE_integer('cc_port', 8773, 'cloud controller port') -DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', - 'Url to ec2 api server') +DEFINE_string('ec2_suffix', '/services/Cloud', 'suffix for ec2') DEFINE_string('default_image', 'ami-11111', 'default image to use, testing only') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index cb9f01a36..8c7c528b5 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -46,8 +46,7 @@ flags.DEFINE_string('vlan_interface', 'eth0', 'network device for vlans') flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') -flags.DEFINE_string('cc_host', utils.get_my_ip(), 'ip of api server') -flags.DEFINE_string('routing_source_ip', '127.0.0.1', +flags.DEFINE_string('routing_source_ip', utils.get_my_ip(), 'Public IP of network host') flags.DEFINE_bool('use_nova_chains', False, 'use the nova_ routing chains instead of default') @@ -59,7 +58,7 @@ def metadata_forward(): """Create forwarding rule for metadata""" _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.cc_host, FLAGS.cc_port)) + "--to-destination %s:%s" % (FLAGS.cc_dmz, FLAGS.cc_port)) def init_host(): diff --git a/nova/utils.py b/nova/utils.py index 11160c118..082a42acf 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -21,7 +21,6 @@ System-level utilities and helper functions. """ import datetime -import functools import inspect import logging import os @@ -36,11 +35,9 @@ from eventlet import event from eventlet import greenthread from nova import exception -from nova import flags from nova.exception import ProcessExecutionError -FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" @@ -197,8 +194,6 @@ def last_octet(address): def get_my_ip(): """Returns the actual ip of the local machine.""" - if getattr(FLAGS, 'fake_tests', None): - return '127.0.0.1' try: csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) csock.connect(('8.8.8.8', 80)) -- cgit From 9e28957c45c69bf11a414faeb16a068f10a6a73d Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 17 Dec 2010 00:44:08 +0000 Subject: clean up use of iptables chains --- nova/network/linux_net.py | 104 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 26 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0fefd9415..8aca1c80a 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -19,7 +19,6 @@ Implements vlans, bridges, and iptables rules using linux utilities. import logging import os -import signal # TODO(ja): does the definition of network_path belong here? @@ -48,39 +47,88 @@ flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') flags.DEFINE_string('cc_host', utils.get_my_ip(), 'ip of api server') flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -flags.DEFINE_string('routing_source_ip', '127.0.0.1', +flags.DEFINE_string('routing_source_ip', utils.get_my_ip(), 'Public IP of network host') flags.DEFINE_bool('use_nova_chains', False, 'use the nova_ routing chains instead of default') - -DEFAULT_PORTS = [("tcp", 80), ("tcp", 22), ("udp", 1194), ("tcp", 443)] +flags.DEFINE_string('dns_server', None, + 'if set, uses specific dns server for dnsmasq') def metadata_forward(): """Create forwarding rule for metadata""" _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.cc_host, FLAGS.cc_port)) + "--to-destination %s:%s" % (FLAGS.cc_dmz, FLAGS.cc_port)) def init_host(): """Basic networking setup goes here""" + + if FLAGS.use_nova_chains: + _execute("sudo iptables -N nova_input", check_exit_code=False) + _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, + check_exit_code=False) + _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) + + _execute("sudo iptables -N nova_forward", check_exit_code=False) + _execute("sudo iptables -D FORWARD -j nova_forward", + check_exit_code=False) + _execute("sudo iptables -A FORWARD -j nova_forward") + + _execute("sudo iptables -N nova_output", check_exit_code=False) + _execute("sudo iptables -D OUTPUT -j nova_output", + check_exit_code=False) + _execute("sudo iptables -A OUTPUT -j nova_output") + + _execute("sudo iptables -t nat -N nova_prerouting", + check_exit_code=False) + _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", + check_exit_code=False) + _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") + + _execute("sudo iptables -t nat -N nova_postrouting", + check_exit_code=False) + _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", + check_exit_code=False) + _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") + + _execute("sudo iptables -t nat -N nova_snatting", + check_exit_code=False) + _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", + check_exit_code=False) + _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") + + _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) + _execute("sudo iptables -t nat -D OUTPUT -j nova_output", + check_exit_code=False) + _execute("sudo iptables -t nat -A OUTPUT -j nova_output") + else: + # NOTE(vish): This makes it easy to ensure snatting rules always + # come after the accept rules in the postrouting chain + _execute("sudo iptables -t nat -N SNATTING", + check_exit_code=False) + _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", + check_exit_code=False) + _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") + # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("POSTROUTING", "-t nat -s %s " + _confirm_rule("SNATTING", "-t nat -s %s " "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip)) + % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) - _confirm_rule("POSTROUTING", "-t nat -s %s -j MASQUERADE" % - FLAGS.fixed_range) + _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % + (FLAGS.fixed_range, FLAGS.dmz_cidr)) _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % {'range': FLAGS.fixed_range}) -def bind_floating_ip(floating_ip): +def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface)) + FLAGS.public_interface), + check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): @@ -102,27 +150,16 @@ def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" % (floating_ip, fixed_ip)) - _confirm_rule("POSTROUTING", "-t nat -s %s -j SNAT --to %s" + _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" % (fixed_ip, floating_ip)) - # TODO(joshua): Get these from the secgroup datastore entries - _confirm_rule("FORWARD", "-d %s -p icmp -j ACCEPT" - % (fixed_ip)) - for (protocol, port) in DEFAULT_PORTS: - _confirm_rule("FORWARD", "-d %s -p %s --dport %s -j ACCEPT" - % (fixed_ip, protocol, port)) def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" % (floating_ip, fixed_ip)) - _remove_rule("POSTROUTING", "-t nat -s %s -j SNAT --to %s" + _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" % (fixed_ip, floating_ip)) - _remove_rule("FORWARD", "-d %s -p icmp -j ACCEPT" - % (fixed_ip)) - for (protocol, port) in DEFAULT_PORTS: - _remove_rule("FORWARD", "-d %s -p %s --dport %s -j ACCEPT" - % (fixed_ip, protocol, port)) def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): @@ -160,6 +197,15 @@ def ensure_bridge(bridge, interface, net_attrs=None): net_attrs['netmask'])) else: _execute("sudo ifconfig %s up" % bridge) + if FLAGS.use_nova_chains: + (out, err) = _execute("sudo iptables -N nova_forward", + check_exit_code=False) + if err != 'iptables: Chain already exists.\n': + # NOTE(vish): chain didn't exist link chain + _execute("sudo iptables -D FORWARD -j nova_forward", + check_exit_code=False) + _execute("sudo iptables -A FORWARD -j nova_forward") + _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) @@ -236,13 +282,17 @@ def _device_exists(device): return not err -def _confirm_rule(chain, cmd): +def _confirm_rule(chain, cmd, append=False): """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() + if append: + loc = "-A" + else: + loc = "-I" _execute("sudo iptables --delete %s %s" % (chain, cmd), check_exit_code=False) - _execute("sudo iptables -I %s %s" % (chain, cmd)) + _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) def _remove_rule(chain, cmd): @@ -265,6 +315,8 @@ def _dnsmasq_cmd(net): ' --dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), ' --dhcp-script=%s' % FLAGS.dhcpbridge, ' --leasefile-ro'] + if FLAGS.dns_server: + cmd.append(' -h -R --server=%s' % FLAGS.dns_server) return ''.join(cmd) -- cgit From baf0b1db2d4997f0e47277763e8ab393c131b8c8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 17 Dec 2010 00:52:17 +0000 Subject: pep8 --- nova/flags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/flags.py b/nova/flags.py index 95bfd3773..74badf6fd 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -31,6 +31,7 @@ import gflags from nova import utils + class FlagValues(gflags.FlagValues): """Extension of gflags.FlagValues that allows undefined and runtime flags. -- cgit From a17fa6df76a3215d84d99738a1a7752a54cfe914 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 21 Dec 2010 18:43:41 +0000 Subject: don't allocate networks when getting vpn info --- nova/auth/manager.py | 6 +++--- nova/db/sqlalchemy/api.py | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 74da8e045..f9a7dd000 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -540,10 +540,10 @@ class AuthManager(object): """ network_ref = db.project_get_network(context.get_admin_context(), - Project.safe_id(project)) + Project.safe_id(project), False) - if not network_ref['vpn_public_port']: - raise exception.NotFound('project network data has not been set') + if not network_ref: + return (None, None) return (network_ref['vpn_public_address'], network_ref['vpn_public_port']) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 6ecd824e1..35c2c76fd 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1075,24 +1075,26 @@ def network_update(context, network_id, values): @require_context -def project_get_network(context, project_id): +def project_get_network(context, project_id, associate=True): session = get_session() - rv = session.query(models.Network).\ - filter_by(project_id=project_id).\ - filter_by(deleted=False).\ - first() - if not rv: + result = session.query(models.Network).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() + if not result: + if not associate: + return None try: return network_associate(context, project_id) except IntegrityError: # NOTE(vish): We hit this if there is a race and two # processes are attempting to allocate the # network at the same time - rv = session.query(models.Network).\ - filter_by(project_id=project_id).\ - filter_by(deleted=False).\ - first() - return rv + result = session.query(models.Network).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() + return result ################### -- cgit From 902db577ea19459c9b01ed7b262024b900440573 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 21 Dec 2010 18:48:07 +0000 Subject: update db/api.py as well --- nova/db/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/db/api.py b/nova/db/api.py index 339e0a3ae..69cfa3377 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -512,12 +512,14 @@ def network_update(context, network_id, values): ################### -def project_get_network(context, project_id): +def project_get_network(context, project_id, associate=True): """Return the network associated with the project. - Raises NotFound if no such network can be found. + If associate is true, it will attempt to associate a new + network if one is not found, otherwise it returns None. """ + return IMPL.project_get_network(context, project_id) -- cgit From 7af11742b6bab492eb87c212d05bf77c0c13aea9 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Wed, 22 Dec 2010 12:24:53 +0100 Subject: Populate user_data field from run-instances call parameter, default to empty string to avoid metadata base64 decoding failure, LP: #691598 --- nova/api/ec2/cloud.py | 1 + nova/compute/api.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8375c4399..13c2b4574 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -756,6 +756,7 @@ class CloudController(object): display_name=kwargs.get('display_name'), description=kwargs.get('display_description'), key_name=kwargs.get('key_name'), + user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), generate_hostname=internal_id_to_ec2_id) return self._format_run_instances(context, diff --git a/nova/compute/api.py b/nova/compute/api.py index c740814da..8fdc0fc9e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -57,6 +57,7 @@ class ComputeAPI(base.Base): max_count=1, kernel_id=None, ramdisk_id=None, display_name='', description='', key_name=None, key_data=None, security_group='default', + user_data=None, generate_hostname=generate_default_hostname): """Create the number of instances requested if quote and other arguments check out ok.""" @@ -120,6 +121,7 @@ class ComputeAPI(base.Base): 'local_gb': type_data['local_gb'], 'display_name': display_name, 'display_description': description, + 'user_data': user_data or '', 'key_name': key_name, 'key_data': key_data} -- cgit From f783e8ad65c2ba3f605bdc350ac64f4beaf27e9d Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Wed, 22 Dec 2010 13:52:44 +0100 Subject: Adding me in the Authors file --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index fa38ef0b1..0b048becb 100644 --- a/Authors +++ b/Authors @@ -27,6 +27,7 @@ Rick Clark Ryan Lucio Sandy Walsh Soren Hansen +Thierry Carrez Todd Willey Trey Morris Vishvananda Ishaya -- cgit From 56856ac1103ec9f3ba0f2da81832a59e7e773256 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 22 Dec 2010 11:12:20 -0500 Subject: Fix doc building endpoint for gettext. --- doc/ext/nova_autodoc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/ext/nova_autodoc.py b/doc/ext/nova_autodoc.py index 39aa2c2cf..5429bb656 100644 --- a/doc/ext/nova_autodoc.py +++ b/doc/ext/nova_autodoc.py @@ -1,5 +1,8 @@ +import gettext import os +gettext.install('nova') + from nova import utils def setup(app): -- cgit