From 9bd72f56224a8cc980620b17210d9b9b9ede6166 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 18 Jan 2011 18:33:04 -0800 Subject: various fixes to smoketests, including allowing admin tests to run as a user, better timing, and allowing volume tests to run on non-udev linux --- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/smoketests/flags.py b/smoketests/flags.py index 35f432a77..9dc310692 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -35,5 +35,5 @@ DEFINE_bool = DEFINE_bool # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 DEFINE_string('region', 'nova', 'Region to use') -DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') +DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') DEFINE_string('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index d5a3a7556..f73ab7e1c 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -258,10 +258,15 @@ class VolumeTests(UserSmokeTestCase): instance = reservation.instances[0] self.data['instance'] = instance for x in xrange(120): - if self.can_ping(instance.private_dns_name): + time.sleep(1) + instance.update() + #if self.can_ping(instance.private_dns_name): + if instance.state == u'running': break else: self.fail('unable to start instance') + time.sleep(10) + instance.update() def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') @@ -273,10 +278,11 @@ class VolumeTests(UserSmokeTestCase): def test_002_can_attach_volume(self): volume = self.data['volume'] - for x in xrange(10): - if volume.status == u'available': + for x in xrange(30): + print volume.status + if volume.status.startswith('available'): break - time.sleep(5) + time.sleep(1) volume.update() else: self.fail('cannot attach volume with state %s' % volume.status) @@ -285,12 +291,12 @@ class VolumeTests(UserSmokeTestCase): # Volumes seems to report "available" too soon. for x in xrange(10): - if volume.status == u'in-use': + if volume.status.startswith('in-use'): break time.sleep(5) volume.update() - self.assertEqual(volume.status, u'in-use') + self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. time.sleep(5) @@ -298,9 +304,15 @@ class VolumeTests(UserSmokeTestCase): def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name conn = self.connect_ssh(ip, TEST_KEY) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) commands = [] commands.append('mkdir -p /mnt/vol') - commands.append('mkfs.ext2 %s' % self.device) + commands.append('/sbin/mke2fs %s' % self.device) commands.append('mount %s /mnt/vol' % self.device) commands.append('echo success') stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) @@ -327,7 +339,7 @@ class VolumeTests(UserSmokeTestCase): "df -h | grep %s | awk {'print $2'}" % self.device) out = stdout.read() conn.close() - if not out.strip() == '1008M': + if not out.strip() == '1007.9M': self.fail('Volume is not the right size: %s %s' % (out, stderr.read())) -- cgit From 2f4258d99e8d97ec70645cd2df2f4e54dc869e89 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 20 Jan 2011 00:14:42 -0800 Subject: more smoketest fixes --- nova/__init__.py | 2 -- smoketests/base.py | 7 +++++-- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 26 ++++++++++++++++---------- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/nova/__init__.py b/nova/__init__.py index 8745617bc..256db55a9 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -30,5 +30,3 @@ .. moduleauthor:: Manish Singh .. moduleauthor:: Andy Smith """ - -from exception import * diff --git a/smoketests/base.py b/smoketests/base.py index 610270c5c..89ee92840 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -17,12 +17,10 @@ # under the License. import boto -import boto_v6 import commands import httplib import os import paramiko -import random import sys import unittest from boto.ec2.regioninfo import RegionInfo @@ -30,6 +28,8 @@ from boto.ec2.regioninfo import RegionInfo from smoketests import flags FLAGS = flags.FLAGS +boto_v6 = None + class SmokeTestCase(unittest.TestCase): @@ -146,6 +146,9 @@ class SmokeTestCase(unittest.TestCase): def run_tests(suites): argv = FLAGS(sys.argv) + if FLAGS.use_ipv6: + global boto_v6 + boto_v6 = __import__('boto_v6') if not os.getenv('EC2_ACCESS_KEY'): print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ diff --git a/smoketests/flags.py b/smoketests/flags.py index 9dc310692..dc756347b 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -36,4 +36,4 @@ DEFINE_bool = DEFINE_bool DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') -DEFINE_string('use_ipv6', True, 'use the ipv6 or not') +DEFINE_bool('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index f73ab7e1c..e5bc98ede 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -189,8 +189,8 @@ class InstanceTests(UserSmokeTestCase): try: conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) conn.close() - except Exception: - time.sleep(1) + except Exception, e: + time.sleep(5) else: break else: @@ -224,7 +224,7 @@ class InstanceTests(UserSmokeTestCase): try: conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) conn.close() - except socket.error: + except Exception: time.sleep(1) else: break @@ -256,17 +256,24 @@ class VolumeTests(UserSmokeTestCase): instance_type='m1.tiny', key_name=TEST_KEY) instance = reservation.instances[0] - self.data['instance'] = instance for x in xrange(120): time.sleep(1) instance.update() - #if self.can_ping(instance.private_dns_name): - if instance.state == u'running': + if self.can_ping(instance.private_dns_name): break else: self.fail('unable to start instance') - time.sleep(10) - instance.update() + self.data['instance'] = instance + for x in xrange(30): + try: + conn = self.connect_ssh(instance.private_dns_name, TEST_KEY) + conn.close() + except Exception: + time.sleep(5) + else: + break + else: + self.fail('could not ssh to instance') def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') @@ -279,7 +286,6 @@ class VolumeTests(UserSmokeTestCase): volume = self.data['volume'] for x in xrange(30): - print volume.status if volume.status.startswith('available'): break time.sleep(1) @@ -438,7 +444,7 @@ class SecurityGroupTests(UserSmokeTestCase): if __name__ == "__main__": suites = {'image': unittest.makeSuite(ImageTests), 'instance': unittest.makeSuite(InstanceTests), - 'security_group': unittest.makeSuite(SecurityGroupTests), + #'security_group': unittest.makeSuite(SecurityGroupTests), 'volume': unittest.makeSuite(VolumeTests) } sys.exit(base.run_tests(suites)) -- cgit From e6b7fa7ae31e90f2d7322445da3843281fff9a70 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 22 Jan 2011 16:20:36 -0800 Subject: fixes and refactoring of smoketests --- smoketests/base.py | 43 +++++++++++--- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 133 +++++++++++++----------------------------- 3 files changed, 77 insertions(+), 101 deletions(-) diff --git a/smoketests/base.py b/smoketests/base.py index 89ee92840..afc618074 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -22,6 +22,7 @@ import httplib import os import paramiko import sys +import time import unittest from boto.ec2.regioninfo import RegionInfo @@ -31,7 +32,6 @@ FLAGS = flags.FLAGS boto_v6 = None - class SmokeTestCase(unittest.TestCase): def connect_ssh(self, ip, key_name): # TODO(devcamcar): set a more reasonable connection timeout time @@ -39,12 +39,10 @@ class SmokeTestCase(unittest.TestCase): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip, username='root', pkey=key) - stdin, stdout, stderr = client.exec_command('uptime') - print 'uptime: ', stdout.read() return client - def can_ping(self, ip): - """ Attempt to ping the specified IP, and give up after 1 second. """ + def can_ping(self, ip, command="ping"): + """Attempt to ping the specified IP, and give up after 1 second.""" # NOTE(devcamcar): ping timeout flag is different in OSX. if sys.platform == 'darwin': @@ -52,10 +50,41 @@ class SmokeTestCase(unittest.TestCase): else: timeout_flag = 'w' - status, output = commands.getstatusoutput('ping -c1 -%s1 %s' % - (timeout_flag, ip)) + status, output = commands.getstatusoutput('%s -c1 -%s1 %s' % + (command, timeout_flag, ip)) return status == 0 + def wait_for_running(self, instance, tries=60, wait=1): + """Wait for instance to be running""" + for x in xrange(tries): + instance.update() + if instance.state.startswith('running'): + return True + time.sleep(wait) + else: + return False + + def wait_for_ping(self, ip, command="ping", tries=120): + """Wait for ip to be pingable""" + for x in xrange(tries): + if self.can_ping(ip, command): + return True + else: + return False + + def wait_for_ssh(self, ip, key_name, tries=30, wait=5): + """Wait for ip to be sshable""" + for x in xrange(tries): + try: + conn = self.connect_ssh(ip, key_name) + conn.close() + except Exception, e: + time.sleep(wait) + else: + return True + else: + return False + def connection_for_env(self, **kwargs): """ Returns a boto ec2 connection for the current environment. diff --git a/smoketests/flags.py b/smoketests/flags.py index dc756347b..5f3c8505e 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -36,4 +36,4 @@ DEFINE_bool = DEFINE_bool DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') -DEFINE_bool('use_ipv6', True, 'use the ipv6 or not') +DEFINE_bool('use_ipv6', False, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index e5bc98ede..26f6344f7 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -19,7 +19,6 @@ import commands import os import random -import socket import sys import time import unittest @@ -91,7 +90,6 @@ class ImageTests(UserSmokeTestCase): break time.sleep(1) else: - print image.state self.assert_(False) # wasn't available within 10 seconds self.assert_(image.type == 'machine') @@ -143,70 +141,36 @@ class InstanceTests(UserSmokeTestCase): key_name=TEST_KEY, instance_type='m1.tiny') self.assertEqual(len(reservation.instances), 1) - self.data['instance_id'] = reservation.instances[0].id + self.data['instance'] = reservation.instances[0] def test_003_instance_runs_within_60_seconds(self): - reservations = self.conn.get_all_instances([self.data['instance_id']]) - instance = reservations[0].instances[0] + instance = self.data['instance'] # allow 60 seconds to exit pending with IP - for x in xrange(60): - instance.update() - if instance.state == u'running': - break - time.sleep(1) - else: + if not self.wait_for_running(self.data['instance']): self.fail('instance failed to start') - ip = reservations[0].instances[0].private_dns_name + self.data['instance'].update() + ip = self.data['instance'].private_dns_name self.failIf(ip == '0.0.0.0') - self.data['private_ip'] = ip if FLAGS.use_ipv6: - ipv6 = reservations[0].instances[0].dns_name_v6 + ipv6 = self.data['instance'].dns_name_v6 self.failIf(ipv6 is None) - self.data['ip_v6'] = ipv6 def test_004_can_ping_private_ip(self): - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 %s' % self.data['private_ip']) - if status == 0: - break - else: + if not self.wait_for_ping(self.data['instance'].private_dns_name): self.fail('could not ping instance') if FLAGS.use_ipv6: - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping6 -c1 %s' % self.data['ip_v6']) - if status == 0: - break - else: - self.fail('could not ping instance') + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') def test_005_can_ssh_to_private_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) - conn.close() - except Exception, e: - time.sleep(5) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') if FLAGS.use_ipv6: - for x in xrange(30): - try: - conn = self.connect_ssh( - self.data['ip_v6'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): self.fail('could not ssh to instance v6') def test_006_can_allocate_elastic_ip(self): @@ -215,21 +179,13 @@ class InstanceTests(UserSmokeTestCase): self.data['public_ip'] = result.public_ip def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance_id'], + result = self.conn.associate_address(self.data['instance'].id, self.data['public_ip']) self.assertTrue(result) def test_008_can_ssh_with_public_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: - self.fail('could not ssh to instance') + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') def test_009_can_disassociate_ip_from_instance(self): result = self.conn.disassociate_address(self.data['public_ip']) @@ -241,8 +197,7 @@ class InstanceTests(UserSmokeTestCase): def test_999_tearDown(self): self.delete_key_pair(self.conn, TEST_KEY) - if self.data.has_key('instance_id'): - self.conn.terminate_instances([self.data['instance_id']]) + self.conn.terminate_instances([self.data['instance'].id]) class VolumeTests(UserSmokeTestCase): @@ -255,24 +210,14 @@ class VolumeTests(UserSmokeTestCase): reservation = self.conn.run_instances(FLAGS.test_image, instance_type='m1.tiny', key_name=TEST_KEY) - instance = reservation.instances[0] - for x in xrange(120): - time.sleep(1) - instance.update() - if self.can_ping(instance.private_dns_name): - break - else: - self.fail('unable to start instance') - self.data['instance'] = instance - for x in xrange(30): - try: - conn = self.connect_ssh(instance.private_dns_name, TEST_KEY) - conn.close() - except Exception: - time.sleep(5) - else: - break - else: + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') def test_001_can_create_volume(self): @@ -280,32 +225,34 @@ class VolumeTests(UserSmokeTestCase): self.assertEqual(volume.size, 1) self.data['volume'] = volume # Give network time to find volume. - time.sleep(5) + time.sleep(10) def test_002_can_attach_volume(self): volume = self.data['volume'] - for x in xrange(30): + for x in xrange(10): + volume.update() if volume.status.startswith('available'): break time.sleep(1) - volume.update() else: self.fail('cannot attach volume with state %s' % volume.status) volume.attach(self.data['instance'].id, self.device) - # Volumes seems to report "available" too soon. + # wait for x in xrange(10): + volume.update() if volume.status.startswith('in-use'): break - time.sleep(5) - volume.update() + time.sleep(1) + else: + self.fail('volume never got to in use') self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. - time.sleep(5) + time.sleep(10) def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name @@ -316,12 +263,12 @@ class VolumeTests(UserSmokeTestCase): 'grep %s /proc/partitions | ' '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' % self.device.rpartition('/')[2]) - commands = [] - commands.append('mkdir -p /mnt/vol') - commands.append('/sbin/mke2fs %s' % self.device) - commands.append('mount %s /mnt/vol' % self.device) - commands.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) out = stdout.read() conn.close() if not out.strip().endswith('success'): -- cgit From 745b7b22f7b22a09e6c3bbc1cd8591eb3aa7f554 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 15 Feb 2011 21:38:47 +0100 Subject: removed flag --pidfile from nova/services.py --- nova/service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/service.py b/nova/service.py index 59648adf2..8b1b91e90 100644 --- a/nova/service.py +++ b/nova/service.py @@ -50,10 +50,6 @@ flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) -flags.DEFINE_string('pidfile', None, - 'pidfile to use for this service') - - flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) -- cgit From 9a7213b615bcaa2127f76146d594f5247ea0d0a4 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 17 Feb 2011 15:00:18 -0800 Subject: Initial support for per-instance metadata, though the OpenStack API. Key/value pairs can be specified at instance creation time and are returned in the details view. Support limits based on quota system. --- nova/api/ec2/cloud.py | 6 +- nova/api/openstack/servers.py | 30 +++++++-- nova/compute/api.py | 29 +++++++- nova/db/sqlalchemy/api.py | 2 + .../versions/004_add_instance_metadata.py | 78 ++++++++++++++++++++++ nova/db/sqlalchemy/models.py | 18 ++++- nova/quota.py | 14 +++- nova/tests/api/openstack/test_servers.py | 11 ++- nova/tests/test_quota.py | 24 +++++++ run_tests.sh | 4 +- 10 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6919cd8d2..33eba5028 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -783,6 +783,9 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) + # NOTE(justinsb): the EC2 API doesn't support metadata here, but this + # is needed for the unit tests. Maybe the unit tests shouldn't be + # calling the EC2 code instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), @@ -797,7 +800,8 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone')) + 'AvailabilityZone'), + metadata=kwargs.get('metadata', [])) return self._format_run_instances(context, instances[0]['reservation_id']) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 009ef6db1..49611703a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -78,9 +78,14 @@ def _translate_detail_keys(inst): except KeyError: LOG.debug(_("Failed to read public ip(s)")) - inst_dict['metadata'] = {} inst_dict['hostId'] = '' + # Return the metadata as a dictionary + metadata = {} + for item in inst['metadata']: + metadata[item['key']] = item['value'] + inst_dict['metadata'] = metadata + return dict(server=inst_dict) @@ -162,14 +167,26 @@ class Controller(wsgi.Controller): if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - key_pair = auth_manager.AuthManager.get_key_pairs( - req.environ['nova.context'])[0] + context = req.environ['nova.context'] + + key_pair = auth_manager.AuthManager.get_key_pairs(context)[0] image_id = common.get_image_id_from_image_hash(self._image_service, - req.environ['nova.context'], env['server']['imageId']) + context, env['server']['imageId']) kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( req, image_id) + + # Metadata is a list, not a Dictionary, because we allow duplicate keys + # (even though JSON can't encode this) + # In future, we may not allow duplicate keys. + # However, the CloudServers API is not definitive on this front, + # and we want to be compatible. + metadata = [] + if env['server']['metadata']: + for k, v in env['server']['metadata'].items(): + metadata.append({'key': k, 'value': v}) + instances = self.compute_api.create( - req.environ['nova.context'], + context, instance_types.get_by_flavor_id(env['server']['flavorId']), image_id, kernel_id=kernel_id, @@ -177,7 +194,8 @@ class Controller(wsgi.Controller): display_name=env['server']['name'], display_description=env['server']['name'], key_name=key_pair['name'], - key_data=key_pair['public_key']) + key_data=key_pair['public_key'], + metadata=metadata) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index ed6f0e34a..cad167f4d 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -85,7 +85,7 @@ class API(base.Base): min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', - availability_zone=None, user_data=None): + availability_zone=None, user_data=None, metadata=[]): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -99,6 +99,30 @@ class API(base.Base): "run %s more instances of this type.") % num_instances, "InstanceLimitExceeded") + num_metadata = len(metadata) + quota_metadata = quota.allowed_metadata_items(context, num_metadata) + if quota_metadata < num_metadata: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " tried to set %(num_metadata)s metadata properties") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + + # Because metadata is stored in the DB, we hard-code the size limits + # In future, we may support more variable length strings, so we act + # as if this is quota-controlled for forwards compatibility + for metadata_item in metadata: + k = metadata_item['key'] + v = metadata_item['value'] + if len(k) > 255 or len(v) > 255: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " metadata property key or value too long") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + is_vpn = image_id == FLAGS.vpn_image_id if not is_vpn: image = self.image_service.show(context, image_id) @@ -155,7 +179,8 @@ class API(base.Base): 'key_name': key_name, 'key_data': key_data, 'locked': False, - 'availability_zone': availability_zone} + 'availability_zone': availability_zone, + 'metadata': metadata} elevated = context.elevated() instances = [] diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..a6b8066b9 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -715,6 +715,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ filter_by(id=instance_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -723,6 +724,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ + options(joinedload('metadata')).\ filter_by(project_id=context.project_id).\ filter_by(id=instance_id).\ filter_by(deleted=False).\ diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 40a96fc17..a842e4cc4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -256,6 +256,7 @@ class Quota(BASE, NovaBase): volumes = Column(Integer) gigabytes = Column(Integer) floating_ips = Column(Integer) + metadata_items = Column(Integer) class ExportDevice(BASE, NovaBase): @@ -536,6 +537,20 @@ class Console(BASE, NovaBase): pool = relationship(ConsolePool, backref=backref('consoles')) +class InstanceMetadata(BASE, NovaBase): + """Represents a metadata key/value pair for an instance""" + __tablename__ = 'instance_metadata' + id = Column(Integer, primary_key=True) + key = Column(String(255)) + value = Column(String(255)) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False) + instance = relationship(Instance, backref="metadata", + foreign_keys=instance_id, + primaryjoin='and_(' + 'InstanceMetadata.instance_id == Instance.id,' + 'InstanceMetadata.deleted == False)') + + class Zone(BASE, NovaBase): """Represents a child zone of this zone.""" __tablename__ = 'zones' @@ -557,7 +572,8 @@ def register_models(): Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, - Project, Certificate, ConsolePool, Console, Zone) + Project, Certificate, ConsolePool, Console, Zone, + InstanceMetadata) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/quota.py b/nova/quota.py index 3884eb308..6b52a97fa 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -35,6 +35,8 @@ flags.DEFINE_integer('quota_gigabytes', 1000, 'number of volume gigabytes allowed per project') flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') +flags.DEFINE_integer('quota_metadata_items', 128, + 'number of metadata items allowed per instance') def get_quota(context, project_id): @@ -42,7 +44,8 @@ def get_quota(context, project_id): 'cores': FLAGS.quota_cores, 'volumes': FLAGS.quota_volumes, 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips} + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items} try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -94,6 +97,15 @@ def allowed_floating_ips(context, num_floating_ips): return min(num_floating_ips, allowed_floating_ips) +def allowed_metadata_items(context, num_metadata_items): + """Check quota; return min(num_metadata_items,allowed_metadata_items)""" + project_id = context.project_id + context = context.elevated() + quota = get_quota(context, project_id) + num_allowed_metadata_items = quota['metadata_items'] + return min(num_metadata_items, num_allowed_metadata_items) + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..7eb81c2b8 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -28,6 +28,7 @@ import nova.api.openstack from nova.api.openstack import servers import nova.db.api from nova.db.sqlalchemy.models import Instance +from nova.db.sqlalchemy.models import InstanceMetadata import nova.rpc from nova.tests.api.openstack import fakes @@ -64,6 +65,9 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, private_address=None, public_addresses=None): + metadata = [] + metadata.append(InstanceMetadata(key='seq', value=id)) + if public_addresses == None: public_addresses = list() @@ -95,7 +99,8 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "availability_zone": "", "display_name": "server%s" % id, "display_description": "", - "locked": False} + "locked": False, + "metadata": metadata} instance["fixed_ip"] = { "address": private_address, @@ -214,7 +219,8 @@ class ServersTest(unittest.TestCase): "get_image_id_from_image_hash", image_id_from_hash) body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, metadata={}, + name='server_test', imageId=2, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, personality={})) req = webob.Request.blank('/v1.0/servers') req.method = 'POST' @@ -291,6 +297,7 @@ class ServersTest(unittest.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) + self.assertEqual(s['metadata']['seq'], i) i += 1 def test_server_pause(self): diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 9548a8c13..36ccc273e 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -87,6 +87,18 @@ class QuotaTestCase(test.TestCase): num_instances = quota.allowed_instances(self.context, 100, instance_types.INSTANCE_TYPES['m1.small']) self.assertEqual(num_instances, 10) + + # metadata_items + too_many_items = FLAGS.quota_metadata_items + 1000 + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, FLAGS.quota_metadata_items) + db.quota_update(self.context, self.project.id, {'metadata_items': 5}) + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, 5) + + # Cleanup db.quota_destroy(self.context, self.project.id) def test_too_many_instances(self): @@ -151,3 +163,15 @@ class QuotaTestCase(test.TestCase): self.assertRaises(quota.QuotaError, self.cloud.allocate_address, self.context) db.floating_ip_destroy(context.get_admin_context(), address) + + def test_too_many_metadata_items(self): + metadata = {} + for i in range(FLAGS.quota_metadata_items + 1): + metadata['key%s' % i] = 'value%s' % i + self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.context, + min_count=1, + max_count=1, + instance_type='m1.small', + image_id='fake', + metadata=metadata) diff --git a/run_tests.sh b/run_tests.sh index 4e21fe945..58e92c06b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -73,7 +73,9 @@ fi if [ -z "$noseargs" ]; then - run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py bin/* nova setup.py || exit 1 + srcfiles=`find bin -type f ! -name "nova.conf*"` + srcfiles+=" nova setup.py" + run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} || exit 1 else run_tests fi -- cgit From bef44d7621db516a0f5d407655f5e76adfd5c06d Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 10:14:56 -0800 Subject: Rename migration 004 => 005 --- .../versions/004_add_instance_metadata.py | 78 ---------------------- .../versions/005_add_instance_metadata.py | 78 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 78 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py deleted file mode 100644 index 4cb07e0d8..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - - -# Just for the ForeignKey and column creation to succeed, these are not the -# actual definitions of instances or services. -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -quotas = Table('quotas', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - - -# -# New Tables -# - -instance_metadata_table = Table('instance_metadata', meta, - Column('created_at', DateTime(timezone=False)), - Column('updated_at', DateTime(timezone=False)), - Column('deleted_at', DateTime(timezone=False)), - Column('deleted', Boolean(create_constraint=True, name=None)), - Column('id', Integer(), primary_key=True, nullable=False), - Column('instance_id', - Integer(), - ForeignKey('instances.id'), - nullable=False), - Column('key', - String(length=255, convert_unicode=False, assert_unicode=None, - unicode_error=None, _warn_on_bytestring=False)), - Column('value', - String(length=255, convert_unicode=False, assert_unicode=None, - unicode_error=None, _warn_on_bytestring=False))) - - -# -# New columns -# -quota_metadata_items = Column('metadata_items', Integer()) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - for table in (instance_metadata_table, ): - try: - table.create() - except Exception: - logging.info(repr(table)) - logging.exception('Exception while creating table') - raise - - quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) -- cgit From 8684eb3aa638883ea82bbaf8eb59076f1d7e6a05 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 17:17:51 -0800 Subject: ObjectStore doesn't use properties collection; kernel_id and ramdisk_id aren't required anyway --- nova/api/openstack/servers.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 486eca508..11a84687d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,13 +144,11 @@ class Controller(wsgi.Controller): metadata stored in Glance as 'image_properties' """ def lookup(param): - _image_id = image_id - try: - return image['properties'][param] - except KeyError: - raise exception.NotFound( - _("%(param)s property not found for image %(_image_id)s") % - locals()) + properties = image.get('properties') + if properties: + return properties.get(param) + else: + return image.get(param) image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From aeab8eeb038ca1d1dde05705028144a78552c4f7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 17:27:25 -0800 Subject: Don't crash if there's no 'fixed_ip' attribute (was returning None, which was unsubscriptable) --- nova/api/openstack/servers.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 486eca508..b54e28c0c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,20 +63,22 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - # grab single private fixed ip - try: - private_ip = inst['fixed_ip']['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) - - # grab all public floating ips - try: - for floating in inst['fixed_ip']['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + fixed_ip = inst['fixed_ip'] + if fixed_ip: + # grab single private fixed ip + try: + private_ip = fixed_ip['address'] + if private_ip: + inst_dict['addresses']['private'].append(private_ip) + except KeyError: + LOG.debug(_("Failed to read private ip")) + + # grab all public floating ips + try: + for floating in fixed_ip['floating_ips']: + inst_dict['addresses']['public'].append(floating['address']) + except KeyError: + LOG.debug(_("Failed to read public ip(s)")) inst_dict['metadata'] = {} inst_dict['hostId'] = '' -- cgit From a7eed42c57fe7eaf6f2981a88a74a81a6890198c Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 20 Feb 2011 20:56:14 +0100 Subject: puppet scripts only there as an example, should be moved to some other place if they are still necessary --- contrib/puppet/files/etc/default/nova-compute | 1 - contrib/puppet/files/etc/default/nova-volume | 1 - contrib/puppet/files/etc/issue | 5 - contrib/puppet/files/etc/libvirt/qemu.conf | 170 -------- contrib/puppet/files/etc/lvm/lvm.conf | 463 -------------------- contrib/puppet/files/etc/nova.conf | 28 -- contrib/puppet/files/production/boto.cfg | 3 - contrib/puppet/files/production/genvpn.sh | 35 -- .../files/production/libvirt.qemu.xml.template | 35 -- contrib/puppet/files/production/my.cnf | 137 ------ contrib/puppet/files/production/nova-iptables | 187 --------- contrib/puppet/files/production/nova-iscsi-dev.sh | 19 - contrib/puppet/files/production/setup_data.sh | 6 - contrib/puppet/files/production/slap.sh | 261 ------------ contrib/puppet/fileserver.conf | 8 - contrib/puppet/manifests/classes/apt.pp | 1 - contrib/puppet/manifests/classes/issue.pp | 14 - contrib/puppet/manifests/classes/kern_module.pp | 34 -- contrib/puppet/manifests/classes/loopback.pp | 6 - contrib/puppet/manifests/classes/lvm.pp | 8 - contrib/puppet/manifests/classes/lvmconf.pp | 8 - contrib/puppet/manifests/classes/nova.pp | 464 --------------------- contrib/puppet/manifests/classes/swift.pp | 7 - contrib/puppet/manifests/site.pp | 120 ------ contrib/puppet/manifests/templates.pp | 21 - contrib/puppet/puppet.conf | 11 - contrib/puppet/templates/haproxy.cfg.erb | 39 -- contrib/puppet/templates/monitrc-nova-api.erb | 138 ------ contrib/puppet/templates/nova-iptables.erb | 10 - .../templates/production/nova-common.conf.erb | 55 --- .../puppet/templates/production/nova-nova.conf.erb | 21 - 31 files changed, 2316 deletions(-) delete mode 100644 contrib/puppet/files/etc/default/nova-compute delete mode 100644 contrib/puppet/files/etc/default/nova-volume delete mode 100644 contrib/puppet/files/etc/issue delete mode 100644 contrib/puppet/files/etc/libvirt/qemu.conf delete mode 100644 contrib/puppet/files/etc/lvm/lvm.conf delete mode 100644 contrib/puppet/files/etc/nova.conf delete mode 100644 contrib/puppet/files/production/boto.cfg delete mode 100644 contrib/puppet/files/production/genvpn.sh delete mode 100644 contrib/puppet/files/production/libvirt.qemu.xml.template delete mode 100644 contrib/puppet/files/production/my.cnf delete mode 100755 contrib/puppet/files/production/nova-iptables delete mode 100644 contrib/puppet/files/production/nova-iscsi-dev.sh delete mode 100755 contrib/puppet/files/production/setup_data.sh delete mode 100755 contrib/puppet/files/production/slap.sh delete mode 100644 contrib/puppet/fileserver.conf delete mode 100644 contrib/puppet/manifests/classes/apt.pp delete mode 100644 contrib/puppet/manifests/classes/issue.pp delete mode 100644 contrib/puppet/manifests/classes/kern_module.pp delete mode 100644 contrib/puppet/manifests/classes/loopback.pp delete mode 100644 contrib/puppet/manifests/classes/lvm.pp delete mode 100644 contrib/puppet/manifests/classes/lvmconf.pp delete mode 100644 contrib/puppet/manifests/classes/nova.pp delete mode 100644 contrib/puppet/manifests/classes/swift.pp delete mode 100644 contrib/puppet/manifests/site.pp delete mode 100644 contrib/puppet/manifests/templates.pp delete mode 100644 contrib/puppet/puppet.conf delete mode 100644 contrib/puppet/templates/haproxy.cfg.erb delete mode 100644 contrib/puppet/templates/monitrc-nova-api.erb delete mode 100644 contrib/puppet/templates/nova-iptables.erb delete mode 100644 contrib/puppet/templates/production/nova-common.conf.erb delete mode 100644 contrib/puppet/templates/production/nova-nova.conf.erb diff --git a/contrib/puppet/files/etc/default/nova-compute b/contrib/puppet/files/etc/default/nova-compute deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-compute +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/default/nova-volume b/contrib/puppet/files/etc/default/nova-volume deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-volume +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/issue b/contrib/puppet/files/etc/issue deleted file mode 100644 index 8c567221b..000000000 --- a/contrib/puppet/files/etc/issue +++ /dev/null @@ -1,5 +0,0 @@ ------------------------------------------------ - - Welcome to your OpenStack installation! - ------------------------------------------------ diff --git a/contrib/puppet/files/etc/libvirt/qemu.conf b/contrib/puppet/files/etc/libvirt/qemu.conf deleted file mode 100644 index 7839f12e5..000000000 --- a/contrib/puppet/files/etc/libvirt/qemu.conf +++ /dev/null @@ -1,170 +0,0 @@ -# Master configuration file for the QEMU driver. -# All settings described here are optional - if omitted, sensible -# defaults are used. - -# VNC is configured to listen on 127.0.0.1 by default. -# To make it listen on all public interfaces, uncomment -# this next option. -# -# NB, strong recommendation to enable TLS + x509 certificate -# verification when allowing public access -# -# vnc_listen = "0.0.0.0" - - -# Enable use of TLS encryption on the VNC server. This requires -# a VNC client which supports the VeNCrypt protocol extension. -# Examples include vinagre, virt-viewer, virt-manager and vencrypt -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to setup CA and issue a server certificate -# before enabling this. -# -# vnc_tls = 1 - - -# Use of TLS requires that x509 certificates be issued. The -# default it to keep them in /etc/pki/libvirt-vnc. This directory -# must contain -# -# ca-cert.pem - the CA master certificate -# server-cert.pem - the server certificate signed with ca-cert.pem -# server-key.pem - the server private key -# -# This option allows the certificate directory to be changed -# -# vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc" - - -# The default TLS configuration only uses certificates for the server -# allowing the client to verify the server's identity and establish -# and encrypted channel. -# -# It is possible to use x509 certificates for authentication too, by -# issuing a x509 certificate to every client who needs to connect. -# -# Enabling this option will reject any client who does not have a -# certificate signed by the CA in /etc/pki/libvirt-vnc/ca-cert.pem -# -# vnc_tls_x509_verify = 1 - - -# The default VNC password. Only 8 letters are significant for -# VNC passwords. This parameter is only used if the per-domain -# XML config does not already provide a password. To allow -# access without passwords, leave this commented out. An empty -# string will still enable passwords, but be rejected by QEMU -# effectively preventing any use of VNC. Obviously change this -# example here before you set this -# -# vnc_password = "XYZ12345" - - -# Enable use of SASL encryption on the VNC server. This requires -# a VNC client which supports the SASL protocol extension. -# Examples include vinagre, virt-viewer and virt-manager -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to configure /etc/sasl2/qemu.conf to choose -# the desired SASL plugin (eg, GSSPI for Kerberos) -# -# vnc_sasl = 1 - - -# The default SASL configuration file is located in /etc/sasl2/ -# When running libvirtd unprivileged, it may be desirable to -# override the configs in this location. Set this parameter to -# point to the directory, and create a qemu.conf in that location -# -# vnc_sasl_dir = "/some/directory/sasl2" - - - - -# The default security driver is SELinux. If SELinux is disabled -# on the host, then the security driver will automatically disable -# itself. If you wish to disable QEMU SELinux security driver while -# leaving SELinux enabled for the host in general, then set this -# to 'none' instead -# -# security_driver = "selinux" - - -# The user ID for QEMU processes run by the system instance -user = "root" - -# The group ID for QEMU processes run by the system instance -group = "root" - -# Whether libvirt should dynamically change file ownership -# to match the configured user/group above. Defaults to 1. -# Set to 0 to disable file ownership changes. -#dynamic_ownership = 1 - - -# What cgroup controllers to make use of with QEMU guests -# -# - 'cpu' - use for schedular tunables -# - 'devices' - use for device whitelisting -# -# NB, even if configured here, they won't be used unless -# the adminsitrator has mounted cgroups. eg -# -# mkdir /dev/cgroup -# mount -t cgroup -o devices,cpu none /dev/cgroup -# -# They can be mounted anywhere, and different controlers -# can be mounted in different locations. libvirt will detect -# where they are located. -# -# cgroup_controllers = [ "cpu", "devices" ] - -# This is the basic set of devices allowed / required by -# all virtual machines. -# -# As well as this, any configured block backed disks, -# all sound device, and all PTY devices are allowed. -# -# This will only need setting if newer QEMU suddenly -# wants some device we don't already know a bout. -# -#cgroup_device_acl = [ -# "/dev/null", "/dev/full", "/dev/zero", -# "/dev/random", "/dev/urandom", -# "/dev/ptmx", "/dev/kvm", "/dev/kqemu", -# "/dev/rtc", "/dev/hpet", "/dev/net/tun", -#] - -# The default format for Qemu/KVM guest save images is raw; that is, the -# memory from the domain is dumped out directly to a file. If you have -# guests with a large amount of memory, however, this can take up quite -# a bit of space. If you would like to compress the images while they -# are being saved to disk, you can also set "lzop", "gzip", "bzip2", or "xz" -# for save_image_format. Note that this means you slow down the process of -# saving a domain in order to save disk space; the list above is in descending -# order by performance and ascending order by compression ratio. -# -# save_image_format = "raw" - -# If provided by the host and a hugetlbfs mount point is configured, -# a guest may request huge page backing. When this mount point is -# unspecified here, determination of a host mount point in /proc/mounts -# will be attempted. Specifying an explicit mount overrides detection -# of the same in /proc/mounts. Setting the mount point to "" will -# disable guest hugepage backing. -# -# NB, within this mount point, guests will create memory backing files -# in a location of $MOUNTPOINT/libvirt/qemu - -# hugetlbfs_mount = "/dev/hugepages" - -# mac_filter enables MAC addressed based filtering on bridge ports. -# This currently requires ebtables to be installed. -# -# mac_filter = 1 - -# By default, PCI devices below non-ACS switch are not allowed to be assigned -# to guests. By setting relaxed_acs_check to 1 such devices will be allowed to -# be assigned to guests. -# -# relaxed_acs_check = 1 diff --git a/contrib/puppet/files/etc/lvm/lvm.conf b/contrib/puppet/files/etc/lvm/lvm.conf deleted file mode 100644 index 4e814ad49..000000000 --- a/contrib/puppet/files/etc/lvm/lvm.conf +++ /dev/null @@ -1,463 +0,0 @@ -# This is an example configuration file for the LVM2 system. -# It contains the default settings that would be used if there was no -# /etc/lvm/lvm.conf file. -# -# Refer to 'man lvm.conf' for further information including the file layout. -# -# To put this file in a different directory and override /etc/lvm set -# the environment variable LVM_SYSTEM_DIR before running the tools. - - -# This section allows you to configure which block devices should -# be used by the LVM system. -devices { - - # Where do you want your volume groups to appear ? - dir = "/dev" - - # An array of directories that contain the device nodes you wish - # to use with LVM2. - scan = [ "/dev" ] - - # If several entries in the scanned directories correspond to the - # same block device and the tools need to display a name for device, - # all the pathnames are matched against each item in the following - # list of regular expressions in turn and the first match is used. - preferred_names = [ ] - - # Try to avoid using undescriptive /dev/dm-N names, if present. - # preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ] - - # A filter that tells LVM2 to only use a restricted set of devices. - # The filter consists of an array of regular expressions. These - # expressions can be delimited by a character of your choice, and - # prefixed with either an 'a' (for accept) or 'r' (for reject). - # The first expression found to match a device name determines if - # the device will be accepted or rejected (ignored). Devices that - # don't match any patterns are accepted. - - # Be careful if there there are symbolic links or multiple filesystem - # entries for the same device as each name is checked separately against - # the list of patterns. The effect is that if any name matches any 'a' - # pattern, the device is accepted; otherwise if any name matches any 'r' - # pattern it is rejected; otherwise it is accepted. - - # Don't have more than one filter line active at once: only one gets used. - - # Run vgscan after you change this parameter to ensure that - # the cache file gets regenerated (see below). - # If it doesn't do what you expect, check the output of 'vgscan -vvvv'. - - - # By default we accept every block device: - filter = [ "r|/dev/etherd/.*|", "r|/dev/block/.*|", "a/.*/" ] - - # Exclude the cdrom drive - # filter = [ "r|/dev/cdrom|" ] - - # When testing I like to work with just loopback devices: - # filter = [ "a/loop/", "r/.*/" ] - - # Or maybe all loops and ide drives except hdc: - # filter =[ "a|loop|", "r|/dev/hdc|", "a|/dev/ide|", "r|.*|" ] - - # Use anchors if you want to be really specific - # filter = [ "a|^/dev/hda8$|", "r/.*/" ] - - # The results of the filtering are cached on disk to avoid - # rescanning dud devices (which can take a very long time). - # By default this cache is stored in the /etc/lvm/cache directory - # in a file called '.cache'. - # It is safe to delete the contents: the tools regenerate it. - # (The old setting 'cache' is still respected if neither of - # these new ones is present.) - cache_dir = "/etc/lvm/cache" - cache_file_prefix = "" - - # You can turn off writing this cache file by setting this to 0. - write_cache_state = 1 - - # Advanced settings. - - # List of pairs of additional acceptable block device types found - # in /proc/devices with maximum (non-zero) number of partitions. - # types = [ "fd", 16 ] - - # If sysfs is mounted (2.6 kernels) restrict device scanning to - # the block devices it believes are valid. - # 1 enables; 0 disables. - sysfs_scan = 1 - - # By default, LVM2 will ignore devices used as components of - # software RAID (md) devices by looking for md superblocks. - # 1 enables; 0 disables. - md_component_detection = 1 - - # By default, if a PV is placed directly upon an md device, LVM2 - # will align its data blocks with the md device's stripe-width. - # 1 enables; 0 disables. - md_chunk_alignment = 1 - - # By default, the start of a PV's data area will be a multiple of - # the 'minimum_io_size' or 'optimal_io_size' exposed in sysfs. - # - minimum_io_size - the smallest request the device can perform - # w/o incurring a read-modify-write penalty (e.g. MD's chunk size) - # - optimal_io_size - the device's preferred unit of receiving I/O - # (e.g. MD's stripe width) - # minimum_io_size is used if optimal_io_size is undefined (0). - # If md_chunk_alignment is enabled, that detects the optimal_io_size. - # This setting takes precedence over md_chunk_alignment. - # 1 enables; 0 disables. - data_alignment_detection = 1 - - # Alignment (in KB) of start of data area when creating a new PV. - # If a PV is placed directly upon an md device and md_chunk_alignment or - # data_alignment_detection is enabled this parameter is ignored. - # Set to 0 for the default alignment of 64KB or page size, if larger. - data_alignment = 0 - - # By default, the start of the PV's aligned data area will be shifted by - # the 'alignment_offset' exposed in sysfs. This offset is often 0 but - # may be non-zero; e.g.: certain 4KB sector drives that compensate for - # windows partitioning will have an alignment_offset of 3584 bytes - # (sector 7 is the lowest aligned logical block, the 4KB sectors start - # at LBA -1, and consequently sector 63 is aligned on a 4KB boundary). - # 1 enables; 0 disables. - data_alignment_offset_detection = 1 - - # If, while scanning the system for PVs, LVM2 encounters a device-mapper - # device that has its I/O suspended, it waits for it to become accessible. - # Set this to 1 to skip such devices. This should only be needed - # in recovery situations. - ignore_suspended_devices = 0 -} - -# This section that allows you to configure the nature of the -# information that LVM2 reports. -log { - - # Controls the messages sent to stdout or stderr. - # There are three levels of verbosity, 3 being the most verbose. - verbose = 0 - - # Should we send log messages through syslog? - # 1 is yes; 0 is no. - syslog = 1 - - # Should we log error and debug messages to a file? - # By default there is no log file. - #file = "/var/log/lvm2.log" - - # Should we overwrite the log file each time the program is run? - # By default we append. - overwrite = 0 - - # What level of log messages should we send to the log file and/or syslog? - # There are 6 syslog-like log levels currently in use - 2 to 7 inclusive. - # 7 is the most verbose (LOG_DEBUG). - level = 0 - - # Format of output messages - # Whether or not (1 or 0) to indent messages according to their severity - indent = 1 - - # Whether or not (1 or 0) to display the command name on each line output - command_names = 0 - - # A prefix to use before the message text (but after the command name, - # if selected). Default is two spaces, so you can see/grep the severity - # of each message. - prefix = " " - - # To make the messages look similar to the original LVM tools use: - # indent = 0 - # command_names = 1 - # prefix = " -- " - - # Set this if you want log messages during activation. - # Don't use this in low memory situations (can deadlock). - # activation = 0 -} - -# Configuration of metadata backups and archiving. In LVM2 when we -# talk about a 'backup' we mean making a copy of the metadata for the -# *current* system. The 'archive' contains old metadata configurations. -# Backups are stored in a human readeable text format. -backup { - - # Should we maintain a backup of the current metadata configuration ? - # Use 1 for Yes; 0 for No. - # Think very hard before turning this off! - backup = 1 - - # Where shall we keep it ? - # Remember to back up this directory regularly! - backup_dir = "/etc/lvm/backup" - - # Should we maintain an archive of old metadata configurations. - # Use 1 for Yes; 0 for No. - # On by default. Think very hard before turning this off. - archive = 1 - - # Where should archived files go ? - # Remember to back up this directory regularly! - archive_dir = "/etc/lvm/archive" - - # What is the minimum number of archive files you wish to keep ? - retain_min = 10 - - # What is the minimum time you wish to keep an archive file for ? - retain_days = 30 -} - -# Settings for the running LVM2 in shell (readline) mode. -shell { - - # Number of lines of history to store in ~/.lvm_history - history_size = 100 -} - - -# Miscellaneous global LVM2 settings -global { - - # The file creation mask for any files and directories created. - # Interpreted as octal if the first digit is zero. - umask = 077 - - # Allow other users to read the files - #umask = 022 - - # Enabling test mode means that no changes to the on disk metadata - # will be made. Equivalent to having the -t option on every - # command. Defaults to off. - test = 0 - - # Default value for --units argument - units = "h" - - # Since version 2.02.54, the tools distinguish between powers of - # 1024 bytes (e.g. KiB, MiB, GiB) and powers of 1000 bytes (e.g. - # KB, MB, GB). - # If you have scripts that depend on the old behaviour, set this to 0 - # temporarily until you update them. - si_unit_consistency = 1 - - # Whether or not to communicate with the kernel device-mapper. - # Set to 0 if you want to use the tools to manipulate LVM metadata - # without activating any logical volumes. - # If the device-mapper kernel driver is not present in your kernel - # setting this to 0 should suppress the error messages. - activation = 1 - - # If we can't communicate with device-mapper, should we try running - # the LVM1 tools? - # This option only applies to 2.4 kernels and is provided to help you - # switch between device-mapper kernels and LVM1 kernels. - # The LVM1 tools need to be installed with .lvm1 suffices - # e.g. vgscan.lvm1 and they will stop working after you start using - # the new lvm2 on-disk metadata format. - # The default value is set when the tools are built. - # fallback_to_lvm1 = 0 - - # The default metadata format that commands should use - "lvm1" or "lvm2". - # The command line override is -M1 or -M2. - # Defaults to "lvm2". - # format = "lvm2" - - # Location of proc filesystem - proc = "/proc" - - # Type of locking to use. Defaults to local file-based locking (1). - # Turn locking off by setting to 0 (dangerous: risks metadata corruption - # if LVM2 commands get run concurrently). - # Type 2 uses the external shared library locking_library. - # Type 3 uses built-in clustered locking. - # Type 4 uses read-only locking which forbids any operations that might - # change metadata. - locking_type = 1 - - # Set to 0 to fail when a lock request cannot be satisfied immediately. - wait_for_locks = 1 - - # If using external locking (type 2) and initialisation fails, - # with this set to 1 an attempt will be made to use the built-in - # clustered locking. - # If you are using a customised locking_library you should set this to 0. - fallback_to_clustered_locking = 1 - - # If an attempt to initialise type 2 or type 3 locking failed, perhaps - # because cluster components such as clvmd are not running, with this set - # to 1 an attempt will be made to use local file-based locking (type 1). - # If this succeeds, only commands against local volume groups will proceed. - # Volume Groups marked as clustered will be ignored. - fallback_to_local_locking = 1 - - # Local non-LV directory that holds file-based locks while commands are - # in progress. A directory like /tmp that may get wiped on reboot is OK. - locking_dir = "/var/lock/lvm" - - # Whenever there are competing read-only and read-write access requests for - # a volume group's metadata, instead of always granting the read-only - # requests immediately, delay them to allow the read-write requests to be - # serviced. Without this setting, write access may be stalled by a high - # volume of read-only requests. - # NB. This option only affects locking_type = 1 viz. local file-based - # locking. - prioritise_write_locks = 1 - - # Other entries can go here to allow you to load shared libraries - # e.g. if support for LVM1 metadata was compiled as a shared library use - # format_libraries = "liblvm2format1.so" - # Full pathnames can be given. - - # Search this directory first for shared libraries. - # library_dir = "/lib/lvm2" - - # The external locking library to load if locking_type is set to 2. - # locking_library = "liblvm2clusterlock.so" -} - -activation { - # Set to 0 to disable udev syncronisation (if compiled into the binaries). - # Processes will not wait for notification from udev. - # They will continue irrespective of any possible udev processing - # in the background. You should only use this if udev is not running - # or has rules that ignore the devices LVM2 creates. - # The command line argument --nodevsync takes precedence over this setting. - # If set to 1 when udev is not running, and there are LVM2 processes - # waiting for udev, run 'dmsetup udevcomplete_all' manually to wake them up. - udev_sync = 1 - - # How to fill in missing stripes if activating an incomplete volume. - # Using "error" will make inaccessible parts of the device return - # I/O errors on access. You can instead use a device path, in which - # case, that device will be used to in place of missing stripes. - # But note that using anything other than "error" with mirrored - # or snapshotted volumes is likely to result in data corruption. - missing_stripe_filler = "error" - - # How much stack (in KB) to reserve for use while devices suspended - reserved_stack = 256 - - # How much memory (in KB) to reserve for use while devices suspended - reserved_memory = 8192 - - # Nice value used while devices suspended - process_priority = -18 - - # If volume_list is defined, each LV is only activated if there is a - # match against the list. - # "vgname" and "vgname/lvname" are matched exactly. - # "@tag" matches any tag set in the LV or VG. - # "@*" matches if any tag defined on the host is also set in the LV or VG - # - # volume_list = [ "vg1", "vg2/lvol1", "@tag1", "@*" ] - - # Size (in KB) of each copy operation when mirroring - mirror_region_size = 512 - - # Setting to use when there is no readahead value stored in the metadata. - # - # "none" - Disable readahead. - # "auto" - Use default value chosen by kernel. - readahead = "auto" - - # 'mirror_image_fault_policy' and 'mirror_log_fault_policy' define - # how a device failure affecting a mirror is handled. - # A mirror is composed of mirror images (copies) and a log. - # A disk log ensures that a mirror does not need to be re-synced - # (all copies made the same) every time a machine reboots or crashes. - # - # In the event of a failure, the specified policy will be used to determine - # what happens. This applies to automatic repairs (when the mirror is being - # monitored by dmeventd) and to manual lvconvert --repair when - # --use-policies is given. - # - # "remove" - Simply remove the faulty device and run without it. If - # the log device fails, the mirror would convert to using - # an in-memory log. This means the mirror will not - # remember its sync status across crashes/reboots and - # the entire mirror will be re-synced. If a - # mirror image fails, the mirror will convert to a - # non-mirrored device if there is only one remaining good - # copy. - # - # "allocate" - Remove the faulty device and try to allocate space on - # a new device to be a replacement for the failed device. - # Using this policy for the log is fast and maintains the - # ability to remember sync state through crashes/reboots. - # Using this policy for a mirror device is slow, as it - # requires the mirror to resynchronize the devices, but it - # will preserve the mirror characteristic of the device. - # This policy acts like "remove" if no suitable device and - # space can be allocated for the replacement. - # - # "allocate_anywhere" - Not yet implemented. Useful to place the log device - # temporarily on same physical volume as one of the mirror - # images. This policy is not recommended for mirror devices - # since it would break the redundant nature of the mirror. This - # policy acts like "remove" if no suitable device and space can - # be allocated for the replacement. - - mirror_log_fault_policy = "allocate" - mirror_device_fault_policy = "remove" -} - - -#################### -# Advanced section # -#################### - -# Metadata settings -# -# metadata { - # Default number of copies of metadata to hold on each PV. 0, 1 or 2. - # You might want to override it from the command line with 0 - # when running pvcreate on new PVs which are to be added to large VGs. - - # pvmetadatacopies = 1 - - # Approximate default size of on-disk metadata areas in sectors. - # You should increase this if you have large volume groups or - # you want to retain a large on-disk history of your metadata changes. - - # pvmetadatasize = 255 - - # List of directories holding live copies of text format metadata. - # These directories must not be on logical volumes! - # It's possible to use LVM2 with a couple of directories here, - # preferably on different (non-LV) filesystems, and with no other - # on-disk metadata (pvmetadatacopies = 0). Or this can be in - # addition to on-disk metadata areas. - # The feature was originally added to simplify testing and is not - # supported under low memory situations - the machine could lock up. - # - # Never edit any files in these directories by hand unless you - # you are absolutely sure you know what you are doing! Use - # the supplied toolset to make changes (e.g. vgcfgrestore). - - # dirs = [ "/etc/lvm/metadata", "/mnt/disk2/lvm/metadata2" ] -#} - -# Event daemon -# -dmeventd { - # mirror_library is the library used when monitoring a mirror device. - # - # "libdevmapper-event-lvm2mirror.so" attempts to recover from - # failures. It removes failed devices from a volume group and - # reconfigures a mirror as necessary. If no mirror library is - # provided, mirrors are not monitored through dmeventd. - - mirror_library = "libdevmapper-event-lvm2mirror.so" - - # snapshot_library is the library used when monitoring a snapshot device. - # - # "libdevmapper-event-lvm2snapshot.so" monitors the filling of - # snapshots and emits a warning through syslog, when the use of - # snapshot exceedes 80%. The warning is repeated when 85%, 90% and - # 95% of the snapshot are filled. - - snapshot_library = "libdevmapper-event-lvm2snapshot.so" -} diff --git a/contrib/puppet/files/etc/nova.conf b/contrib/puppet/files/etc/nova.conf deleted file mode 100644 index a0d64078c..000000000 --- a/contrib/puppet/files/etc/nova.conf +++ /dev/null @@ -1,28 +0,0 @@ ---ec2_url=http://192.168.255.1:8773/services/Cloud ---rabbit_host=192.168.255.1 ---redis_host=192.168.255.1 ---s3_host=192.168.255.1 ---vpn_ip=192.168.255.1 ---datastore_path=/var/lib/nova/keeper ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vlan_start=2000 ---vlan_end=3000 ---private_range=192.168.0.0/16 ---public_range=10.0.0.0/24 ---volume_group=vgdata ---storage_dev=/dev/sdc ---bridge_dev=eth2 ---aoe_eth_dev=eth2 ---public_interface=vlan0 ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---vpn_image_id=ami-cloudpipe ---daemonize ---verbose ---syslog ---prefix=nova diff --git a/contrib/puppet/files/production/boto.cfg b/contrib/puppet/files/production/boto.cfg deleted file mode 100644 index f4a2de2b6..000000000 --- a/contrib/puppet/files/production/boto.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[Boto] -debug = 0 -num_retries = 1 diff --git a/contrib/puppet/files/production/genvpn.sh b/contrib/puppet/files/production/genvpn.sh deleted file mode 100644 index 538c3cd33..000000000 --- a/contrib/puppet/files/production/genvpn.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This gets zipped and run on the cloudpipe-managed OpenVPN server -NAME=$1 -SUBJ=$2 - -mkdir -p projects/$NAME -cd projects/$NAME - -# generate a server priv key -openssl genrsa -out server.key 2048 - -# generate a server CSR -openssl req -new -key server.key -out server.csr -batch -subj "$SUBJ" - -if [ "`id -u`" != "`grep nova /etc/passwd | cut -d':' -f3`" ]; then - sudo chown -R nova:nogroup . -fi diff --git a/contrib/puppet/files/production/libvirt.qemu.xml.template b/contrib/puppet/files/production/libvirt.qemu.xml.template deleted file mode 100644 index 114dfdc01..000000000 --- a/contrib/puppet/files/production/libvirt.qemu.xml.template +++ /dev/null @@ -1,35 +0,0 @@ - - %(name)s - - hvm - %(basepath)s/kernel - %(basepath)s/ramdisk - root=/dev/vda1 console=ttyS0 - - - - - %(memory_kb)s - %(vcpus)s - - - - - - - - - - - - - - - - - diff --git a/contrib/puppet/files/production/my.cnf b/contrib/puppet/files/production/my.cnf deleted file mode 100644 index 8777bc480..000000000 --- a/contrib/puppet/files/production/my.cnf +++ /dev/null @@ -1,137 +0,0 @@ -# -# The MySQL database server configuration file. -# -# You can copy this to one of: -# - "/etc/mysql/my.cnf" to set global options, -# - "~/.my.cnf" to set user-specific options. -# -# One can use all long options that the program supports. -# Run program with --help to get a list of available options and with -# --print-defaults to see which it would actually understand and use. -# -# For explanations see -# http://dev.mysql.com/doc/mysql/en/server-system-variables.html - -# This will be passed to all mysql clients -# It has been reported that passwords should be enclosed with ticks/quotes -# escpecially if they contain "#" chars... -# Remember to edit /etc/mysql/debian.cnf when changing the socket location. -[client] -port = 3306 -socket = /var/run/mysqld/mysqld.sock - -# Here is entries for some specific programs -# The following values assume you have at least 32M ram - -# This was formally known as [safe_mysqld]. Both versions are currently parsed. -[mysqld_safe] -socket = /var/run/mysqld/mysqld.sock -nice = 0 - -[mysqld] -# -# * Basic Settings -# - -# -# * IMPORTANT -# If you make changes to these settings and your system uses apparmor, you may -# also need to also adjust /etc/apparmor.d/usr.sbin.mysqld. -# - -user = mysql -socket = /var/run/mysqld/mysqld.sock -port = 3306 -basedir = /usr -datadir = /var/lib/mysql -tmpdir = /tmp -skip-external-locking -# -# Instead of skip-networking the default is now to listen only on -# localhost which is more compatible and is not less secure. -# bind-address = 127.0.0.1 -# -# * Fine Tuning -# -innodb_buffer_pool_size = 12G -#innodb_log_file_size = 256M -innodb_log_buffer_size=4M -innodb_flush_log_at_trx_commit=2 -innodb_thread_concurrency=8 -innodb_flush_method=O_DIRECT -key_buffer = 128M -max_allowed_packet = 256M -thread_stack = 8196K -thread_cache_size = 32 -# This replaces the startup script and checks MyISAM tables if needed -# the first time they are touched -myisam-recover = BACKUP -max_connections = 1000 -table_cache = 1024 -#thread_concurrency = 10 -# -# * Query Cache Configuration -# -query_cache_limit = 32M -query_cache_size = 256M -# -# * Logging and Replication -# -# Both location gets rotated by the cronjob. -# Be aware that this log type is a performance killer. -# As of 5.1 you can enable the log at runtime! -#general_log_file = /var/log/mysql/mysql.log -#general_log = 1 - -log_error = /var/log/mysql/error.log - -# Here you can see queries with especially long duration -log_slow_queries = /var/log/mysql/mysql-slow.log -long_query_time = 2 -#log-queries-not-using-indexes -# -# The following can be used as easy to replay backup logs or for replication. -# note: if you are setting up a replication slave, see README.Debian about -# other settings you may need to change. -server-id = 1 -log_bin = /var/log/mysql/mysql-bin.log -expire_logs_days = 10 -max_binlog_size = 50M -#binlog_do_db = include_database_name -#binlog_ignore_db = include_database_name -# -# * InnoDB -# -sync_binlog=1 -# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. -# Read the manual for more InnoDB related options. There are many! -# -# * Security Features -# -# Read the manual, too, if you want chroot! -# chroot = /var/lib/mysql/ -# -# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". -# -# ssl-ca=/etc/mysql/cacert.pem -# ssl-cert=/etc/mysql/server-cert.pem -# ssl-key=/etc/mysql/server-key.pem - - - -[mysqldump] -quick -quote-names -max_allowed_packet = 256M - -[mysql] -#no-auto-rehash # faster start of mysql but no tab completition - -[isamchk] -key_buffer = 128M - -# -# * IMPORTANT: Additional settings that can override those from this file! -# The files must end with '.cnf', otherwise they'll be ignored. -# -!includedir /etc/mysql/conf.d/ diff --git a/contrib/puppet/files/production/nova-iptables b/contrib/puppet/files/production/nova-iptables deleted file mode 100755 index 61e2ca2b9..000000000 --- a/contrib/puppet/files/production/nova-iptables +++ /dev/null @@ -1,187 +0,0 @@ -#! /bin/sh - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# NOTE(vish): This script sets up some reasonable defaults for iptables and -# creates nova-specific chains. If you use this script you should -# run nova-network and nova-compute with --use_nova_chains=True - - -# NOTE(vish): If you run public nova-api on a different port, make sure to -# change the port here - -if [ -f /etc/default/nova-iptables ] ; then - . /etc/default/nova-iptables -fi - -export LC_ALL=C - -API_PORT=${API_PORT:-"8773"} - -if [ ! -n "$IP" ]; then - # NOTE(vish): IP address is what address the services ALLOW on. - # This will just get the first ip in the list, so if you - # have more than one eth device set up, this will fail, and - # you should explicitly pass in the ip of the instance - IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` -fi - -if [ ! -n "$PRIVATE_RANGE" ]; then - #NOTE(vish): PRIVATE_RANGE: range is ALLOW to access DHCP - PRIVATE_RANGE="192.168.0.0/12" -fi - -if [ ! -n "$MGMT_IP" ]; then - # NOTE(vish): Management IP is the ip over which to allow ssh traffic. It - # will also allow traffic to nova-api - MGMT_IP="$IP" -fi - -if [ ! -n "$DMZ_IP" ]; then - # NOTE(vish): DMZ IP is the ip over which to allow api & objectstore access - DMZ_IP="$IP" -fi - -clear_nova_iptables() { - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - iptables -F - iptables -t nat -F - iptables -F services - iptables -X services - # HACK: re-adding fail2ban rules :( - iptables -N fail2ban-ssh - iptables -A INPUT -p tcp -m multiport --dports 22 -j fail2ban-ssh - iptables -A fail2ban-ssh -j RETURN -} - -load_nova_iptables() { - - iptables -P INPUT DROP - iptables -A INPUT -m state --state INVALID -j DROP - iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - # NOTE(ja): allow localhost for everything - iptables -A INPUT -d 127.0.0.1/32 -j ACCEPT - # NOTE(ja): 22 only allowed MGMT_IP before, but we widened it to any - # address, since ssh should be listening only on internal - # before we re-add this rule we will need to add - # flexibility for RSYNC between omega/stingray - iptables -A INPUT -m tcp -p tcp --dport 22 -j ACCEPT - iptables -A INPUT -m udp -p udp --dport 123 -j ACCEPT - iptables -A INPUT -p icmp -j ACCEPT - iptables -N services - iptables -A INPUT -j services - iptables -A INPUT -p tcp -j REJECT --reject-with tcp-reset - iptables -A INPUT -j REJECT --reject-with icmp-port-unreachable - - iptables -P FORWARD DROP - iptables -A FORWARD -m state --state INVALID -j DROP - iptables -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT - iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu - - # NOTE(vish): DROP on output is too restrictive for now. We need to add - # in a bunch of more specific output rules to use it. - # iptables -P OUTPUT DROP - iptables -A OUTPUT -m state --state INVALID -j DROP - iptables -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - - if [ -n "$GANGLIA" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 8649 -j ACCEPT - iptables -A services -m udp -p udp -d $IP --dport 8649 -j ACCEPT - fi - - # if [ -n "$WEB" ] || [ -n "$ALL" ]; then - # # NOTE(vish): This opens up ports for web access, allowing web-based - # # dashboards to work. - # iptables -A services -m tcp -p tcp -d $IP --dport 80 -j ACCEPT - # iptables -A services -m tcp -p tcp -d $IP --dport 443 -j ACCEPT - # fi - - if [ -n "$OBJECTSTORE" ] || [ -n "$ALL" ]; then - # infrastructure - iptables -A services -m tcp -p tcp -d $IP --dport 3333 -j ACCEPT - # clients - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport 3333 -j ACCEPT - fi - - if [ -n "$API" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport $API_PORT -j ACCEPT - if [ "$IP" != "$DMZ_IP" ]; then - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport $API_PORT -j ACCEPT - fi - if [ "$IP" != "$MGMT_IP" ] && [ "$DMZ_IP" != "$MGMT_IP" ]; then - iptables -A services -m tcp -p tcp -d $MGMT_IP --dport $API_PORT -j ACCEPT - fi - fi - - if [ -n "$REDIS" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 6379 -j ACCEPT - fi - - if [ -n "$MYSQL" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3306 -j ACCEPT - fi - - if [ -n "$RABBITMQ" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 4369 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 5672 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 53284 -j ACCEPT - fi - - if [ -n "$DNSMASQ" ] || [ -n "$ALL" ]; then - # NOTE(vish): this could theoretically be setup per network - # for each host, but it seems like overkill - iptables -A services -m tcp -p tcp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp --dport 67 -j ACCEPT - fi - - if [ -n "$LDAP" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 389 -j ACCEPT - fi - - if [ -n "$ISCSI" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3260 -j ACCEPT - iptables -A services -m tcp -p tcp -d 127.0.0.0/16 --dport 3260 -j ACCEPT - fi -} - - -case "$1" in - start) - echo "Starting nova-iptables: " - load_nova_iptables - ;; - stop) - echo "Clearing nova-iptables: " - clear_nova_iptables - ;; - restart) - echo "Restarting nova-iptables: " - clear_nova_iptables - load_nova_iptables - ;; - *) - echo "Usage: $NAME {start|stop|restart}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/contrib/puppet/files/production/nova-iscsi-dev.sh b/contrib/puppet/files/production/nova-iscsi-dev.sh deleted file mode 100644 index 8eda10d2e..000000000 --- a/contrib/puppet/files/production/nova-iscsi-dev.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# FILE: /etc/udev/scripts/iscsidev.sh - -BUS=${1} -HOST=${BUS%%:*} - -[ -e /sys/class/iscsi_host ] || exit 1 - -file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" - -target_name=$(cat ${file}) - -# This is not an open-scsi drive -if [ -z "${target_name}" ]; then - exit 1 -fi - -echo "${target_name##*:}" diff --git a/contrib/puppet/files/production/setup_data.sh b/contrib/puppet/files/production/setup_data.sh deleted file mode 100755 index 1fbbac41c..000000000 --- a/contrib/puppet/files/production/setup_data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -/root/slap.sh -mysql -e "DROP DATABASE nova" -mysql -e "CREATE DATABASE nova" -mysql -e "GRANT ALL on nova.* to nova@'%' identified by 'TODO:CHANGEME:CMON'" -touch /root/installed diff --git a/contrib/puppet/files/production/slap.sh b/contrib/puppet/files/production/slap.sh deleted file mode 100755 index f8ea16949..000000000 --- a/contrib/puppet/files/production/slap.sh +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# LDAP INSTALL SCRIPT - SHOULD BE IDEMPOTENT, but it SCRUBS all USERS - -apt-get install -y slapd ldap-utils python-ldap - -cat >/etc/ldap/schema/openssh-lpk_openldap.schema < -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) -LPK_SCHEMA_EOF - -cat >/etc/ldap/schema/nova.schema < -# -# - -# using internet experimental oid arc as per BP64 3.1 -objectidentifier novaSchema 1.3.6.1.3.1.666.666 -objectidentifier novaAttrs novaSchema:3 -objectidentifier novaOCs novaSchema:4 - -attributetype ( - novaAttrs:1 - NAME 'accessKey' - DESC 'Key for accessing data' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:2 - NAME 'secretKey' - DESC 'Secret key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - -objectClass ( - novaOCs:1 - NAME 'novaUser' - DESC 'access and secret keys' - AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) - -NOVA_SCHEMA_EOF - -mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig -cat >/etc/ldap/slapd.conf </etc/ldap/ldap.conf </etc/ldap/base.ldif < "/usr/bin/apt-get update" } diff --git a/contrib/puppet/manifests/classes/issue.pp b/contrib/puppet/manifests/classes/issue.pp deleted file mode 100644 index 8bb37ee3f..000000000 --- a/contrib/puppet/manifests/classes/issue.pp +++ /dev/null @@ -1,14 +0,0 @@ -class issue { - file { "/etc/issue": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } - file { "/etc/issue.net": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } -} diff --git a/contrib/puppet/manifests/classes/kern_module.pp b/contrib/puppet/manifests/classes/kern_module.pp deleted file mode 100644 index 00ec0636c..000000000 --- a/contrib/puppet/manifests/classes/kern_module.pp +++ /dev/null @@ -1,34 +0,0 @@ -# via http://projects.puppetlabs.com/projects/puppet/wiki/Kernel_Modules_Patterns - -define kern_module ($ensure) { - $modulesfile = $operatingsystem ? { ubuntu => "/etc/modules", redhat => "/etc/rc.modules" } - case $operatingsystem { - redhat: { file { "/etc/rc.modules": ensure => file, mode => 755 } } - } - case $ensure { - present: { - exec { "insert_module_${name}": - command => $operatingsystem ? { - ubuntu => "/bin/echo '${name}' >> '${modulesfile}'", - redhat => "/bin/echo '/sbin/modprobe ${name}' >> '${modulesfile}' " - }, - unless => "/bin/grep -qFx '${name}' '${modulesfile}'" - } - exec { "/sbin/modprobe ${name}": unless => "/bin/grep -q '^${name} ' '/proc/modules'" } - } - absent: { - exec { "/sbin/modprobe -r ${name}": onlyif => "/bin/grep -q '^${name} ' '/proc/modules'" } - exec { "remove_module_${name}": - command => $operatingsystem ? { - ubuntu => "/usr/bin/perl -ni -e 'print unless /^\\Q${name}\\E\$/' '${modulesfile}'", - redhat => "/usr/bin/perl -ni -e 'print unless /^\\Q/sbin/modprobe ${name}\\E\$/' '${modulesfile}'" - }, - onlyif => $operatingsystem ? { - ubuntu => "/bin/grep -qFx '${name}' '${modulesfile}'", - redhat => "/bin/grep -q '^/sbin/modprobe ${name}' '${modulesfile}'" - } - } - } - default: { err ( "unknown ensure value ${ensure}" ) } - } -} diff --git a/contrib/puppet/manifests/classes/loopback.pp b/contrib/puppet/manifests/classes/loopback.pp deleted file mode 100644 index e0fa9d541..000000000 --- a/contrib/puppet/manifests/classes/loopback.pp +++ /dev/null @@ -1,6 +0,0 @@ -define loopback($num) { - exec { "mknod -m 0660 /dev/loop${num} b 7 ${num}; chown root:disk /dev/loop${num}": - creates => "/dev/loop${num}", - path => ["/usr/bin", "/usr/sbin", "/bin"] - } -} diff --git a/contrib/puppet/manifests/classes/lvm.pp b/contrib/puppet/manifests/classes/lvm.pp deleted file mode 100644 index 5a407abcb..000000000 --- a/contrib/puppet/manifests/classes/lvm.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvm { - file { "/etc/lvm/lvm.conf": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/lvm.conf", - } -} diff --git a/contrib/puppet/manifests/classes/lvmconf.pp b/contrib/puppet/manifests/classes/lvmconf.pp deleted file mode 100644 index 4aa7ddfdc..000000000 --- a/contrib/puppet/manifests/classes/lvmconf.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvmconf { - file { "/etc/lvm/lvm.conf": - owner => "root", group => "root", mode => 644, - source => "puppet://${puppet_server}/files/etc/lvm/lvm.conf", - ensure => present - } -} - diff --git a/contrib/puppet/manifests/classes/nova.pp b/contrib/puppet/manifests/classes/nova.pp deleted file mode 100644 index e942860f4..000000000 --- a/contrib/puppet/manifests/classes/nova.pp +++ /dev/null @@ -1,464 +0,0 @@ -import "kern_module" -import "apt" -import "loopback" - -#$head_node_ip = "undef" -#$rabbit_ip = "undef" -#$vpn_ip = "undef" -#$public_interface = "undef" -#$vlan_start = "5000" -#$vlan_end = "6000" -#$private_range = "10.0.0.0/16" -#$public_range = "192.168.177.0/24" - -define nova_iptables($services, $ip="", $private_range="", $mgmt_ip="", $dmz_ip="") { - file { "/etc/init.d/nova-iptables": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iptables", - } - - file { "/etc/default/nova-iptables": - owner => "root", mode => 644, - content => template("nova-iptables.erb") - } -} - -define nova_conf_pointer($name) { - file { "/etc/nova/nova-${name}.conf": - owner => "nova", mode => 400, - content => "--flagfile=/etc/nova/nova.conf" - } -} - -class novaconf { - file { "/etc/nova/nova.conf": - owner => "nova", mode => 400, - content => template("production/nova-common.conf.erb", "production/nova-${cluster_name}.conf.erb") - } - nova_conf_pointer{'manage': name => 'manage'} -} - -class novadata { - package { "rabbitmq-server": ensure => present } - - file { "/etc/rabbitmq/rabbitmq.conf": - owner => "root", mode => 644, - content => "NODENAME=rabbit@localhost", - } - - service { "rabbitmq-server": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/rabbitmq/rabbitmq.conf"], - Package["rabbitmq-server"] - ] - } - - package { "mysql-server": ensure => present } - - file { "/etc/mysql/my.cnf": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/my.cnf", - } - - service { "mysql": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/mysql/my.cnf"], - Package["mysql-server"] - ] - } - - file { "/root/slap.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/slap.sh", - } - - file { "/root/setup_data.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/setup_data.sh", - } - - # setup compute data - exec { "setup_data": - command => "/root/setup_data.sh", - path => "/usr/bin:/bin", - unless => "test -f /root/installed", - require => [ - Service["mysql"], - File["/root/slap.sh"], - File["/root/setup_data.sh"] - ] - } -} - -define nscheduler($version) { - package { "nova-scheduler": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'scheduler': name => 'scheduler'} - exec { "update-rc.d -f nova-scheduler remove; update-rc.d nova-scheduler defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-scheduler", - unless => "test -f /etc/rc2.d/S50nova-scheduler" - } - service { "nova-scheduler": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-scheduler"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-scheduler.conf"] - ] - } - -} - -define napi($version, $api_servers, $api_base_port) { - file { "/etc/boto.cfg": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/boto.cfg", - } - - file { "/var/lib/nova/CA/genvpn.sh": - owner => "nova", mode => 755, - source => "puppet://${puppet_server}/files/production/genvpn.sh", - } - - package { "python-greenlet": ensure => present } - package { "nova-api": ensure => $version, require => [Exec["update-apt"], Package["python-greenlet"]] } - nova_conf_pointer{'api': name => 'api'} - - exec { "update-rc.d -f nova-api remove; update-rc.d nova-api defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-api", - unless => "test -f /etc/rc2.d/S50nova-api" - } - - service { "nova-netsync": - start => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock start", - stop => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock stop", - ensure => running, - hasstatus => false, - pattern => "nova-netsync", - require => Service["nova-api"], - subscribe => File["/etc/nova/nova.conf"] - } - service { "nova-api": - start => "monit start all -g nova_api", - stop => "monit stop all -g nova_api", - restart => "monit restart all -g nova_api", - # ensure => running, - # hasstatus => true, - require => Service["monit"], - subscribe => [ - Package["nova-objectstore"], - File["/etc/boto.cfg"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } - - # the haproxy & monit's template use $api_servers and $api_base_port - - package { "haproxy": ensure => present } - file { "/etc/default/haproxy": - owner => "root", mode => 644, - content => "ENABLED=1", - require => Package['haproxy'] - } - file { "/etc/haproxy/haproxy.cfg": - owner => "root", mode => 644, - content => template("/srv/cloud/puppet/templates/haproxy.cfg.erb"), - require => Package['haproxy'] - } - service { "haproxy": - ensure => true, - enable => true, - hasstatus => true, - subscribe => [ - Package["haproxy"], - File["/etc/default/haproxy"], - File["/etc/haproxy/haproxy.cfg"], - ] - } - - package { "socat": ensure => present } - - file { "/usr/local/bin/gmetric_haproxy.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/ganglia/gmetric_scripts/gmetric_haproxy.sh", - } - - cron { "gmetric_haproxy": - command => "/usr/local/bin/gmetric_haproxy.sh", - user => root, - minute => "*/3", - } - - package { "monit": ensure => present } - - file { "/etc/default/monit": - owner => "root", mode => 644, - content => "startup=1", - require => Package['monit'] - } - file { "/etc/monit/monitrc": - owner => "root", mode => 600, - content => template("/srv/cloud/puppet/templates/monitrc-nova-api.erb"), - require => Package['monit'] - } - service { "monit": - ensure => true, - pattern => "sbin/monit", - subscribe => [ - Package["monit"], - File["/etc/default/monit"], - File["/etc/monit/monitrc"], - ] - } - -} - - -define nnetwork($version) { - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - # EVIL HACK: custom binary because dnsmasq 2.52 segfaulted accessing dereferenced object - file { "/usr/sbin/dnsmasq": - owner => "root", group => "root", - source => "puppet://${puppet_server}/files/production/dnsmasq", - } - - package { "nova-network": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'dhcpbridge': name => 'dhcpbridge'} - nova_conf_pointer{'network': name => "network" } - - exec { "update-rc.d -f nova-network remove; update-rc.d nova-network defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-network", - unless => "test -f /etc/rc2.d/S50nova-network" - } - service { "nova-network": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-network"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-network.conf"] - ] - } -} - -define nobjectstore($version) { - package { "nova-objectstore": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'objectstore': name => 'objectstore'} - exec { "update-rc.d -f nova-objectstore remove; update-rc.d nova-objectstore defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-objectstore", - unless => "test -f /etc/rc2.d/S50nova-objectstore" - } - service { "nova-objectstore": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-objectstore"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } -} - -define ncompute($version) { - include ganglia-python - include ganglia-compute - - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - - # LIBVIRT has to be restarted when ebtables / gawk is installed - service { "libvirt-bin": - ensure => running, - pattern => "sbin/libvirtd", - subscribe => [ - Package["ebtables"], - Kern_module["kvm_intel"] - ], - require => [ - Package["libvirt-bin"], - Package["ebtables"], - Package["gawk"], - Kern_module["kvm_intel"], - File["/dev/kvm"] - ] - } - - package { "libvirt-bin": ensure => "0.8.3-1ubuntu14~ppalucid2" } - package { "ebtables": ensure => present } - package { "gawk": ensure => present } - - # ensure proper permissions on /dev/kvm - file { "/dev/kvm": - owner => "root", - group => "kvm", - mode => 660 - } - - # require hardware virt - kern_module { "kvm_intel": - ensure => present, - } - - # increase loopback devices - file { "/etc/modprobe.d/loop.conf": - owner => "root", mode => 644, - content => "options loop max_loop=40" - } - - nova_conf_pointer{'compute': name => 'compute'} - - loopback{loop0: num => 0} - loopback{loop1: num => 1} - loopback{loop2: num => 2} - loopback{loop3: num => 3} - loopback{loop4: num => 4} - loopback{loop5: num => 5} - loopback{loop6: num => 6} - loopback{loop7: num => 7} - loopback{loop8: num => 8} - loopback{loop9: num => 9} - loopback{loop10: num => 10} - loopback{loop11: num => 11} - loopback{loop12: num => 12} - loopback{loop13: num => 13} - loopback{loop14: num => 14} - loopback{loop15: num => 15} - loopback{loop16: num => 16} - loopback{loop17: num => 17} - loopback{loop18: num => 18} - loopback{loop19: num => 19} - loopback{loop20: num => 20} - loopback{loop21: num => 21} - loopback{loop22: num => 22} - loopback{loop23: num => 23} - loopback{loop24: num => 24} - loopback{loop25: num => 25} - loopback{loop26: num => 26} - loopback{loop27: num => 27} - loopback{loop28: num => 28} - loopback{loop29: num => 29} - loopback{loop30: num => 30} - loopback{loop31: num => 31} - loopback{loop32: num => 32} - loopback{loop33: num => 33} - loopback{loop34: num => 34} - loopback{loop35: num => 35} - loopback{loop36: num => 36} - loopback{loop37: num => 37} - loopback{loop38: num => 38} - loopback{loop39: num => 39} - - package { "python-libvirt": ensure => "0.8.3-1ubuntu14~ppalucid2" } - - package { "nova-compute": - ensure => "$version", - require => Package["python-libvirt"] - } - - #file { "/usr/share/nova/libvirt.qemu.xml.template": - # owner => "nova", mode => 400, - # source => "puppet://${puppet_server}/files/production/libvirt.qemu.xml.template", - #} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-compute remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-compute" - } - service { "nova-compute": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-compute"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-compute.conf"], - #File["/usr/share/nova/libvirt.qemu.xml.template"], - Service["libvirt-bin"], - Kern_module["kvm_intel"] - ] - } -} - -define nvolume($version) { - - package { "nova-volume": ensure => $version, require => Exec["update-apt"] } - - nova_conf_pointer{'volume': name => 'volume'} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-volume remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-volume" - } - - file { "/etc/default/iscsitarget": - owner => "root", mode => 644, - content => "ISCSITARGET_ENABLE=true" - } - - package { "iscsitarget": ensure => present } - - file { "/dev/iscsi": ensure => directory } # FIXME(vish): owner / mode? - file { "/usr/sbin/nova-iscsi-dev.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iscsi-dev.sh" - } - file { "/etc/udev/rules.d/55-openiscsi.rules": - owner => "root", mode => 644, - content => 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/usr/sbin/nova-iscsi-dev.sh %b",SYMLINK+="iscsi/%c%n"' - } - - service { "iscsitarget": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/default/iscsitarget"], - Package["iscsitarget"] - ] - } - - service { "nova-volume": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-volume"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-volume.conf"] - ] - } -} - -class novaspool { - # This isn't in release yet - #cron { logspool: - # command => "/usr/bin/nova-logspool /var/log/nova.log /var/lib/nova/spool", - # user => "nova" - #} - #cron { spoolsentry: - # command => "/usr/bin/nova-spoolsentry ${sentry_url} ${sentry_key} /var/lib/nova/spool", - # user => "nova" - #} -} diff --git a/contrib/puppet/manifests/classes/swift.pp b/contrib/puppet/manifests/classes/swift.pp deleted file mode 100644 index 64ffb6fa3..000000000 --- a/contrib/puppet/manifests/classes/swift.pp +++ /dev/null @@ -1,7 +0,0 @@ -class swift { - package { "memcached": ensure => present } - service { "memcached": require => Package['memcached'] } - - package { "swift-proxy": ensure => present } -} - diff --git a/contrib/puppet/manifests/site.pp b/contrib/puppet/manifests/site.pp deleted file mode 100644 index ca07a34ad..000000000 --- a/contrib/puppet/manifests/site.pp +++ /dev/null @@ -1,120 +0,0 @@ -# site.pp - -import "templates" -import "classes/*" - -node novabase inherits default { -# $puppet_server = "192.168.0.10" - $cluster_name = "openstack001" - $ganglia_udp_send_channel = "openstack001.example.com" - $syslog = "192.168.0.10" - - # THIS STUFF ISN'T IN RELEASE YET - #$sentry_url = "http://192.168.0.19/sentry/store/" - #$sentry_key = "TODO:SENTRYPASS" - - $local_network = "192.168.0.0/16" - $vpn_ip = "192.168.0.2" - $public_interface = "eth0" - include novanode -# include nova-common - include opsmetrics - -# non-nova stuff such as nova-dash inherit from novanode -# novaspool needs a better home -# include novaspool -} - -# Builder -node "nova000.example.com" inherits novabase { - $syslog = "server" - include ntp - include syslog-server -} - -# Non-Nova nodes - -node - "blog.example.com", - "wiki.example.com" -inherits novabase { - include ganglia-python - include ganglia-apache - include ganglia-mysql -} - - -node "nova001.example.com" -inherits novabase { - include novabase - - nova_iptables { nova: - services => [ - "ganglia", - "mysql", - "rabbitmq", - "ldap", - "api", - "objectstore", - "nrpe", - ], - ip => "192.168.0.10", - } - - nobjectstore { nova: version => "0.9.0" } - nscheduler { nova: version => "0.9.0" } - napi { nova: - version => "0.9.0", - api_servers => 10, - api_base_port => 8000 - } -} - -node "nova002.example.com" -inherits novabase { - include novaconf - - nova_iptables { nova: - services => [ - "ganglia", - "dnsmasq", - "nrpe" - ], - ip => "192.168.4.2", - private_range => "192.168.0.0/16", - } - - nnetwork { nova: version => "0.9.0" } -} - -node - "nova003.example.com", - "nova004.example.com", - "nova005.example.com", - "nova006.example.com", - "nova007.example.com", - "nova008.example.com", - "nova009.example.com", - "nova010.example.com", - "nova011.example.com", - "nova012.example.com", - "nova013.example.com", - "nova014.example.com", - "nova015.example.com", - "nova016.example.com", - "nova017.example.com", - "nova018.example.com", - "nova019.example.com", -inherits novabase { - include novaconf - ncompute { nova: version => "0.9.0" } - nvolume { nova: version => "0.9.0" } -} - -#node -# "nova020.example.com" -# "nova021.example.com" -#inherits novanode { -# include novaconf - #ncompute { nova: version => "0.9.0" } -#} diff --git a/contrib/puppet/manifests/templates.pp b/contrib/puppet/manifests/templates.pp deleted file mode 100644 index 90e433013..000000000 --- a/contrib/puppet/manifests/templates.pp +++ /dev/null @@ -1,21 +0,0 @@ -# templates.pp - -import "classes/*" - -class baseclass { -# include dns-client # FIXME: missing resolv.conf.erb?? - include issue -} - -node default { - $nova_site = "undef" - $nova_ns1 = "undef" - $nova_ns2 = "undef" -# include baseclass -} - -# novanode handles the system-level requirements for Nova/Swift nodes -class novanode { - include baseclass - include lvmconf -} diff --git a/contrib/puppet/puppet.conf b/contrib/puppet/puppet.conf deleted file mode 100644 index 92af920e3..000000000 --- a/contrib/puppet/puppet.conf +++ /dev/null @@ -1,11 +0,0 @@ -[main] -logdir=/var/log/puppet -vardir=/var/lib/puppet -ssldir=/var/lib/puppet/ssl -rundir=/var/run/puppet -factpath=$vardir/lib/facter -pluginsync=false - -[puppetmasterd] -templatedir=/var/lib/nova/contrib/puppet/templates -autosign=true diff --git a/contrib/puppet/templates/haproxy.cfg.erb b/contrib/puppet/templates/haproxy.cfg.erb deleted file mode 100644 index bd9991de7..000000000 --- a/contrib/puppet/templates/haproxy.cfg.erb +++ /dev/null @@ -1,39 +0,0 @@ -# this config needs haproxy-1.1.28 or haproxy-1.2.1 - -global - log 127.0.0.1 local0 - log 127.0.0.1 local1 notice - #log loghost local0 info - maxconn 4096 - #chroot /usr/share/haproxy - stats socket /var/run/haproxy.sock - user haproxy - group haproxy - daemon - #debug - #quiet - -defaults - log global - mode http - option httplog - option dontlognull - retries 3 - option redispatch - stats enable - stats uri /haproxy - maxconn 2000 - contimeout 5000 - clitimeout 50000 - srvtimeout 50000 - - -listen nova-api 0.0.0.0:8773 - option httpchk GET / HTTP/1.0\r\nHost:\ example.com - option forwardfor - reqidel ^X-Forwarded-For:.* - balance roundrobin -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset -%> - server api_<%= port %> 127.0.0.1:<%= port %> maxconn 1 check -<% end -%> - option httpclose # disable keep-alive diff --git a/contrib/puppet/templates/monitrc-nova-api.erb b/contrib/puppet/templates/monitrc-nova-api.erb deleted file mode 100644 index fe2626327..000000000 --- a/contrib/puppet/templates/monitrc-nova-api.erb +++ /dev/null @@ -1,138 +0,0 @@ -############################################################################### -## Monit control file -############################################################################### -## -## Comments begin with a '#' and extend through the end of the line. Keywords -## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. -## -## Below you will find examples of some frequently used statements. For -## information about the control file, a complete list of statements and -## options please have a look in the monit manual. -## -## -############################################################################### -## Global section -############################################################################### -## -## Start monit in the background (run as a daemon): -# -set daemon 60 # check services at 1-minute intervals - with start delay 30 # optional: delay the first check by half a minute - # (by default check immediately after monit start) - - -## Set syslog logging with the 'daemon' facility. If the FACILITY option is -## omitted, monit will use 'user' facility by default. If you want to log to -## a stand alone log file instead, specify the path to a log file -# -set logfile syslog facility log_daemon -# -# -### Set the location of monit id file which saves the unique id specific for -### given monit. The id is generated and stored on first monit start. -### By default the file is placed in $HOME/.monit.id. -# -# set idfile /var/.monit.id -# -### Set the location of monit state file which saves the monitoring state -### on each cycle. By default the file is placed in $HOME/.monit.state. If -### state file is stored on persistent filesystem, monit will recover the -### monitoring state across reboots. If it is on temporary filesystem, the -### state will be lost on reboot. -# -# set statefile /var/.monit.state -# -## Set the list of mail servers for alert delivery. Multiple servers may be -## specified using comma separator. By default monit uses port 25 - this -## is possible to override with the PORT option. -# -# set mailserver mail.bar.baz, # primary mailserver -# backup.bar.baz port 10025, # backup mailserver on port 10025 -# localhost # fallback relay -# -# -## By default monit will drop alert events if no mail servers are available. -## If you want to keep the alerts for a later delivery retry, you can use the -## EVENTQUEUE statement. The base directory where undelivered alerts will be -## stored is specified by the BASEDIR option. You can limit the maximal queue -## size using the SLOTS option (if omitted, the queue is limited by space -## available in the back end filesystem). -# -# set eventqueue -# basedir /var/monit # set the base directory where events will be stored -# slots 100 # optionaly limit the queue size -# -# -## Send status and events to M/Monit (Monit central management: for more -## informations about M/Monit see http://www.tildeslash.com/mmonit). -# -# set mmonit http://monit:monit@192.168.1.10:8080/collector -# -# -## Monit by default uses the following alert mail format: -## -## --8<-- -## From: monit@$HOST # sender -## Subject: monit alert -- $EVENT $SERVICE # subject -## -## $EVENT Service $SERVICE # -## # -## Date: $DATE # -## Action: $ACTION # -## Host: $HOST # body -## Description: $DESCRIPTION # -## # -## Your faithful employee, # -## monit # -## --8<-- -## -## You can override this message format or parts of it, such as subject -## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. -## are expanded at runtime. For example, to override the sender: -# -# set mail-format { from: monit@foo.bar } -# -# -## You can set alert recipients here whom will receive alerts if/when a -## service defined in this file has errors. Alerts may be restricted on -## events by using a filter as in the second example below. -# -# set alert sysadm@foo.bar # receive all alerts -# set alert manager@foo.bar only on { timeout } # receive just service- -# # timeout alert -# -# -## Monit has an embedded web server which can be used to view status of -## services monitored, the current configuration, actual services parameters -## and manage services from a web interface. -# - set httpd port 2812 and - use address localhost # only accept connection from localhost - allow localhost # allow localhost to connect to the server and -# allow admin:monit # require user 'admin' with password 'monit' -# allow @monit # allow users of group 'monit' to connect (rw) -# allow @users readonly # allow users of group 'users' to connect readonly -# -# -############################################################################### -## Services -############################################################################### - -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset %> - -check process nova_api_<%= port %> with pidfile /var/run/nova/nova-api-<%= port %>.pid - group nova_api - start program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock start" - as uid nova - stop program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock stop" - as uid nova - if failed port <%= port %> protocol http - with timeout 15 seconds - for 4 cycles - then restart - if totalmem > 300 Mb then restart - if cpu is greater than 60% for 2 cycles then alert - if cpu > 80% for 3 cycles then restart - if 3 restarts within 5 cycles then timeout - -<% end %> diff --git a/contrib/puppet/templates/nova-iptables.erb b/contrib/puppet/templates/nova-iptables.erb deleted file mode 100644 index 2fc066305..000000000 --- a/contrib/puppet/templates/nova-iptables.erb +++ /dev/null @@ -1,10 +0,0 @@ -<% services.each do |service| -%> -<%= service.upcase %>=1 -<% end -%> -<% if ip && ip != "" %>IP="<%=ip%>"<% end %> -<% if private_range && private_range != "" %>PRIVATE_RANGE="<%=private_range%>"<% end %> -<% if mgmt_ip && mgmt_ip != "" %>MGMT_IP="<%=mgmt_ip%>"<% end %> -<% if dmz_ip && dmz_ip != "" %>DMZ_IP="<%=dmz_ip%>"<% end %> - -# warning: this file is auto-generated by puppet - diff --git a/contrib/puppet/templates/production/nova-common.conf.erb b/contrib/puppet/templates/production/nova-common.conf.erb deleted file mode 100644 index 23ee0c5e8..000000000 --- a/contrib/puppet/templates/production/nova-common.conf.erb +++ /dev/null @@ -1,55 +0,0 @@ -# global ---dmz_net=192.168.0.0 ---dmz_mask=255.255.0.0 ---dmz_cidr=192.168.0.0/16 ---ldap_user_dn=cn=Administrators,dc=example,dc=com ---ldap_user_unit=Users ---ldap_user_subtree=ou=Users,dc=example,dc=com ---ldap_project_subtree=ou=Groups,dc=example,dc=com ---role_project_subtree=ou=Groups,dc=example,dc=com ---ldap_cloudadmin=cn=NovaAdmins,ou=Groups,dc=example,dc=com ---ldap_itsec=cn=NovaSecurity,ou=Groups,dc=example,dc=com ---ldap_sysadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_netadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_developer=cn=developers,ou=Groups,dc=example,dc=com ---verbose ---daemonize ---syslog ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---scheduler_driver=nova.scheduler.simple.SimpleScheduler ---libvirt_xml_template=/usr/share/nova/libvirt.qemu.xml.template ---credentials_template=/usr/share/nova/novarc.template ---boot_script_template=/usr/share/nova/bootscript.template ---vpn_client_template=/usr/share/nova/client.ovpn.template ---max_cores=40 ---max_gigabytes=2000 ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vpn_start=11000 ---volume_group=vgdata ---volume_manager=nova.volume.manager.ISCSIManager ---volume_driver=nova.volume.driver.ISCSIDriver ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---dhcpbridge=/usr/bin/nova-dhcpbridge ---vpn_image_id=ami-cloudpipe ---dhcpbridge_flagfile=/etc/nova/nova.conf ---credential_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=NOVA/CN=%s-%s ---auth_driver=nova.auth.ldapdriver.LdapDriver ---quota_cores=17 ---quota_floating_ips=5 ---quota_instances=6 ---quota_volumes=10 ---quota_gigabytes=100 ---use_nova_chains=True ---input_chain=services ---use_project_ca=True ---fixed_ip_disassociate_timeout=300 ---api_max_requests=1 ---api_listen_ip=127.0.0.1 ---user_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=%s-%s-%s ---project_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-ca-%s-%s ---vpn_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-vpn-%s-%s diff --git a/contrib/puppet/templates/production/nova-nova.conf.erb b/contrib/puppet/templates/production/nova-nova.conf.erb deleted file mode 100644 index 8683fefde..000000000 --- a/contrib/puppet/templates/production/nova-nova.conf.erb +++ /dev/null @@ -1,21 +0,0 @@ ---fixed_range=192.168.0.0/16 ---iscsi_ip_prefix=192.168.4 ---floating_range=10.0.0.0/24 ---rabbit_host=192.168.0.10 ---s3_host=192.168.0.10 ---cc_host=192.168.0.10 ---cc_dmz=192.168.24.10 ---s3_dmz=192.168.24.10 ---ec2_url=http://192.168.0.1:8773/services/Cloud ---vpn_ip=192.168.0.2 ---ldap_url=ldap://192.168.0.10 ---sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---other_sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---routing_source_ip=192.168.0.2 ---bridge_dev=eth1 ---public_interface=eth0 ---vlan_start=3100 ---num_networks=700 ---rabbit_userid=TODO:RABBIT ---rabbit_password=TODO:CHANGEME ---ldap_password=TODO:CHANGEME -- cgit From 5b2ec209d07d7df45f9b7ca6eebfcbc9443de94e Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 22 Feb 2011 17:10:34 -0800 Subject: don't make a syslog handler if we didn't ask for one --- nova/log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/log.py b/nova/log.py index 10c14d74b..591d26c63 100644 --- a/nova/log.py +++ b/nova/log.py @@ -236,16 +236,17 @@ class NovaRootLogger(NovaLogger): def __init__(self, name, level=NOTSET): self.logpath = None self.filelog = None - self.syslog = SysLogHandler(address='/dev/log') self.streamlog = StreamHandler() + self.syslog = None NovaLogger.__init__(self, name, level) def setup_from_flags(self): """Setup logger from flags""" global _filelog if FLAGS.use_syslog: + self.syslog = SysLogHandler(address='/dev/log') self.addHandler(self.syslog) - else: + elif self.syslog: self.removeHandler(self.syslog) logpath = _get_log_file_path() if logpath: -- cgit From 18793c2e184713d33bc93306d464cf443584ffd6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 17:44:07 -0800 Subject: test that shows error on filtering groups --- nova/tests/test_cloud.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 445cc6e8b..2bce64353 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -133,6 +133,20 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_security_groups(self): + """Makes sure describe_security_groups works and filters results.""" + sec = db.security_group_create(self.context, {'name': 'test'}) + result = self.cloud.describe_security_groups(self.context) + # NOTE(vish): should have the default group as well + self.assertEqual(len(result['securityGroupInfo']), 2) + result = self.cloud.describe_security_groups(self.context, + group_name=[sec['name']]) + self.assertEqual(len(result['securityGroupInfo']), 1) + self.assertEqual( + cloud.ec2_id_to_id(result['securityGroupInfo'][0]['name']), + sec['name']) + db.security_group_destroy(self.context, sec['id']) + def test_describe_volumes(self): """Makes sure describe_volumes works and filters results.""" vol1 = db.volume_create(self.context, {}) @@ -286,19 +300,6 @@ class CloudTestCase(test.TestCase): LOG.debug(_("Terminating instance %s"), instance_id) rv = self.compute.terminate_instance(instance_id) - def test_describe_instances(self): - """Makes sure describe_instances works.""" - instance1 = db.instance_create(self.context, {'host': 'host2'}) - comp1 = db.service_create(self.context, {'host': 'host2', - 'availability_zone': 'zone1', - 'topic': "compute"}) - result = self.cloud.describe_instances(self.context) - self.assertEqual(result['reservationSet'][0] - ['instancesSet'][0] - ['placement']['availabilityZone'], 'zone1') - db.instance_destroy(self.context, instance1['id']) - db.service_destroy(self.context, comp1['id']) - def test_instance_update_state(self): # TODO(termie): what is this code even testing? def instance(num): -- cgit From 2610a522d26351686612058a6da0300bce731112 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 17:49:38 -0800 Subject: fix test --- nova/tests/test_cloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 2bce64353..afdbb80a9 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -135,7 +135,9 @@ class CloudTestCase(test.TestCase): def test_describe_security_groups(self): """Makes sure describe_security_groups works and filters results.""" - sec = db.security_group_create(self.context, {'name': 'test'}) + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, + 'name': 'test'}) result = self.cloud.describe_security_groups(self.context) # NOTE(vish): should have the default group as well self.assertEqual(len(result['securityGroupInfo']), 2) @@ -143,7 +145,7 @@ class CloudTestCase(test.TestCase): group_name=[sec['name']]) self.assertEqual(len(result['securityGroupInfo']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['securityGroupInfo'][0]['name']), + result['securityGroupInfo'][0]['groupName'], sec['name']) db.security_group_destroy(self.context, sec['id']) -- cgit From 828e3ea3f29f57767a4e25ad40b275c886cb7968 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 18:02:25 -0800 Subject: fix and optimize security group filtering --- nova/api/ec2/cloud.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..fc9c13d91 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -318,14 +318,19 @@ class CloudController(object): def describe_security_groups(self, context, group_name=None, **kwargs): self.compute_api.ensure_default_security_group(context) - if context.is_admin: + if group_name: + groups = [] + for name in group_name: + group = db.security_group_get_by_name(context, + context.project_id, + name) + groups.append(group) + elif context.is_admin: groups = db.security_group_get_all(context) else: groups = db.security_group_get_by_project(context, context.project_id) groups = [self._format_security_group(context, g) for g in groups] - if not group_name is None: - groups = [g for g in groups if g.name in group_name] return {'securityGroupInfo': list(sorted(groups, -- cgit From 2fd33bdd50b933dc14fea065c823f5a73324129b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 18:04:08 -0800 Subject: separate out smoketests and add updated nova.sh --- contrib/nova.sh | 12 +- smoketests/base.py | 12 + smoketests/public_network_smoketests.py | 11 +- smoketests/sysadmin_smoketests.py | 293 +++++++++++++++++++++++ smoketests/user_smoketests.py | 397 -------------------------------- 5 files changed, 321 insertions(+), 404 deletions(-) create mode 100644 smoketests/sysadmin_smoketests.py delete mode 100644 smoketests/user_smoketests.py diff --git a/contrib/nova.sh b/contrib/nova.sh index 9259035ca..1187f2728 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -66,7 +66,7 @@ if [ "$CMD" == "install" ]; then sudo apt-get install -y user-mode-linux kvm libvirt-bin sudo apt-get install -y screen euca2ools vlan curl rabbitmq-server sudo apt-get install -y lvm2 iscsitarget open-iscsi - sudo apt-get install -y socat + sudo apt-get install -y socat unzip echo "ISCSITARGET_ENABLE=true" | sudo tee /etc/default/iscsitarget sudo /etc/init.d/iscsitarget restart sudo modprobe kvm @@ -111,8 +111,7 @@ if [ "$CMD" == "run" ]; then --nodaemon --dhcpbridge_flagfile=$NOVA_DIR/bin/nova.conf --network_manager=nova.network.manager.$NET_MAN ---cc_host=$HOST_IP ---routing_source_ip=$HOST_IP +--my_ip=$HOST_IP --sql_connection=$SQL_CONN --auth_driver=nova.auth.$AUTH --libvirt_type=$LIBVIRT_TYPE @@ -151,7 +150,6 @@ NOVA_CONF_EOF mkdir -p $NOVA_DIR/instances rm -rf $NOVA_DIR/networks mkdir -p $NOVA_DIR/networks - $NOVA_DIR/tools/clean-vlans if [ ! -d "$NOVA_DIR/images" ]; then ln -s $DIR/images $NOVA_DIR/images fi @@ -169,10 +167,14 @@ NOVA_CONF_EOF # create a project called 'admin' with project manager of 'admin' $NOVA_DIR/bin/nova-manage project create admin admin # export environment variables for project 'admin' and user 'admin' - $NOVA_DIR/bin/nova-manage project environment admin admin $NOVA_DIR/novarc + $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip + unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ # create a small network $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 1 32 + # create some floating ips + $NOVA_DIR/bin/nova-manage floating create `hostname` 10.6.0.0/27 + # nova api crashes if we start it with a regular screen command, # so send the start command by forcing text into the window. screen_it api "$NOVA_DIR/bin/nova-api" diff --git a/smoketests/base.py b/smoketests/base.py index afc618074..204b4a1eb 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -28,7 +28,9 @@ from boto.ec2.regioninfo import RegionInfo from smoketests import flags +SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) boto_v6 = None @@ -173,6 +175,16 @@ class SmokeTestCase(unittest.TestCase): return True +TEST_DATA = {} + + +class UserSmokeTestCase(SmokeTestCase): + def setUp(self): + global TEST_DATA + self.conn = self.connection_for_env() + self.data = TEST_DATA + + def run_tests(suites): argv = FLAGS(sys.argv) if FLAGS.use_ipv6: diff --git a/smoketests/public_network_smoketests.py b/smoketests/public_network_smoketests.py index bfc2b20ba..5a4c67642 100644 --- a/smoketests/public_network_smoketests.py +++ b/smoketests/public_network_smoketests.py @@ -24,9 +24,16 @@ import sys import time import unittest +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from smoketests import flags from smoketests import base -from smoketests import user_smoketests #Note that this test should run from #public network (outside of private network segments) @@ -42,7 +49,7 @@ TEST_KEY2 = '%s_key2' % TEST_PREFIX TEST_DATA = {} -class InstanceTestsFromPublic(user_smoketests.UserSmokeTestCase): +class InstanceTestsFromPublic(base.UserSmokeTestCase): def test_001_can_create_keypair(self): key = self.create_key_pair(self.conn, TEST_KEY) self.assertEqual(key.name, TEST_KEY) diff --git a/smoketests/sysadmin_smoketests.py b/smoketests/sysadmin_smoketests.py new file mode 100644 index 000000000..e3b84d3d3 --- /dev/null +++ b/smoketests/sysadmin_smoketests.py @@ -0,0 +1,293 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + + +FLAGS = flags.FLAGS +flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', + 'Local kernel file to use for bundling tests') +flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', + 'Local image file to use for bundling tests') + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX +class ImageTests(base.UserSmokeTestCase): + def test_001_can_bundle_image(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_image)) + + def test_002_can_upload_image(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) + + def test_003_can_register_image(self): + image_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_image)) + self.assert_(image_id is not None) + self.data['image_id'] = image_id + + def test_004_can_bundle_kernel(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) + + def test_005_can_upload_kernel(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) + + def test_006_can_register_kernel(self): + kernel_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_kernel)) + self.assert_(kernel_id is not None) + self.data['kernel_id'] = kernel_id + + def test_007_images_are_available_within_10_seconds(self): + for i in xrange(10): + image = self.conn.get_image(self.data['image_id']) + if image and image.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(image.type == 'machine') + + for i in xrange(10): + kernel = self.conn.get_image(self.data['kernel_id']) + if kernel and kernel.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(kernel.type == 'kernel') + + def test_008_can_describe_image_attribute(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + + def test_009_can_modify_image_launch_permission(self): + self.conn.modify_image_attribute(image_id=self.data['image_id'], + operation='add', + attribute='launchPermission', + groups='all') + image = self.conn.get_image(self.data['image_id']) + self.assertEqual(image.id, self.data['image_id']) + + def test_010_can_see_launch_permission(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + self.assert_(attrs.attrs['groups'][0], 'all') + + def test_011_user_can_deregister_kernel(self): + self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) + + def test_012_can_deregister_image(self): + self.assertTrue(self.conn.deregister_image(self.data['image_id'])) + + def test_013_can_delete_bundle(self): + self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) + + +class InstanceTests(base.UserSmokeTestCase): + def test_001_can_create_keypair(self): + key = self.create_key_pair(self.conn, TEST_KEY) + self.assertEqual(key.name, TEST_KEY) + + def test_002_can_create_instance_with_keypair(self): + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + instance_type='m1.tiny') + self.assertEqual(len(reservation.instances), 1) + self.data['instance'] = reservation.instances[0] + + def test_003_instance_runs_within_60_seconds(self): + instance = self.data['instance'] + # allow 60 seconds to exit pending with IP + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + ip = self.data['instance'].private_dns_name + self.failIf(ip == '0.0.0.0') + if FLAGS.use_ipv6: + ipv6 = self.data['instance'].dns_name_v6 + self.failIf(ipv6 is None) + + def test_004_can_ping_private_ip(self): + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + + if FLAGS.use_ipv6: + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') + + def test_005_can_ssh_to_private_ip(self): + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + if FLAGS.use_ipv6: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): + self.fail('could not ssh to instance v6') + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class VolumeTests(base.UserSmokeTestCase): + def setUp(self): + super(VolumeTests, self).setUp() + self.device = '/dev/vdb' + + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_create_volume(self): + volume = self.conn.create_volume(1, 'nova') + self.assertEqual(volume.size, 1) + self.data['volume'] = volume + # Give network time to find volume. + time.sleep(10) + + def test_002_can_attach_volume(self): + volume = self.data['volume'] + + for x in xrange(10): + volume.update() + if volume.status.startswith('available'): + break + time.sleep(1) + else: + self.fail('cannot attach volume with state %s' % volume.status) + + volume.attach(self.data['instance'].id, self.device) + + # wait + for x in xrange(10): + volume.update() + if volume.status.startswith('in-use'): + break + time.sleep(1) + else: + self.fail('volume never got to in use') + + self.assertTrue(volume.status.startswith('in-use')) + + # Give instance time to recognize volume. + time.sleep(10) + + def test_003_can_mount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) + out = stdout.read() + conn.close() + if not out.strip().endswith('success'): + self.fail('Unable to mount: %s %s' % (out, stderr.read())) + + def test_004_can_write_to_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted + stdin, stdout, stderr = conn.exec_command( + 'echo hello > /mnt/vol/test.txt') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to write to mount: %s' % (err)) + + def test_005_volume_is_correct_size(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command( + "df -h | grep %s | awk {'print $2'}" % self.device) + out = stdout.read() + conn.close() + if not out.strip() == '1007.9M': + self.fail('Volume is not the right size: %s %s' % + (out, stderr.read())) + + def test_006_me_can_umount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to unmount: %s' % (err)) + + def test_007_me_can_detach_volume(self): + result = self.conn.detach_volume(volume_id=self.data['volume'].id) + self.assertTrue(result) + time.sleep(5) + + def test_008_me_can_delete_volume(self): + result = self.conn.delete_volume(self.data['volume'].id) + self.assertTrue(result) + + def test_999_tearDown(self): + self.conn.terminate_instances([self.data['instance'].id]) + self.conn.delete_key_pair(TEST_KEY) + + +if __name__ == "__main__": + suites = {'image': unittest.makeSuite(ImageTests), + 'instance': unittest.makeSuite(InstanceTests), + 'volume': unittest.makeSuite(VolumeTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py deleted file mode 100644 index 26f6344f7..000000000 --- a/smoketests/user_smoketests.py +++ /dev/null @@ -1,397 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import commands -import os -import random -import sys -import time -import unittest - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -from smoketests import flags -from smoketests import base - - -SUITE_NAMES = '[image, instance, volume]' - -FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) -flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', - 'Local kernel file to use for bundling tests') -flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', - 'Local image file to use for bundling tests') - -TEST_PREFIX = 'test%s' % int(random.random() * 1000000) -TEST_BUCKET = '%s_bucket' % TEST_PREFIX -TEST_KEY = '%s_key' % TEST_PREFIX -TEST_GROUP = '%s_group' % TEST_PREFIX -TEST_DATA = {} - - -class UserSmokeTestCase(base.SmokeTestCase): - def setUp(self): - global TEST_DATA - self.conn = self.connection_for_env() - self.data = TEST_DATA - - -class ImageTests(UserSmokeTestCase): - def test_001_can_bundle_image(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_image)) - - def test_002_can_upload_image(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) - - def test_003_can_register_image(self): - image_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_image)) - self.assert_(image_id is not None) - self.data['image_id'] = image_id - - def test_004_can_bundle_kernel(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) - - def test_005_can_upload_kernel(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) - - def test_006_can_register_kernel(self): - kernel_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_kernel)) - self.assert_(kernel_id is not None) - self.data['kernel_id'] = kernel_id - - def test_007_images_are_available_within_10_seconds(self): - for i in xrange(10): - image = self.conn.get_image(self.data['image_id']) - if image and image.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(image.type == 'machine') - - for i in xrange(10): - kernel = self.conn.get_image(self.data['kernel_id']) - if kernel and kernel.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(kernel.type == 'kernel') - - def test_008_can_describe_image_attribute(self): - attrs = self.conn.get_image_attribute(self.data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - - def test_009_can_modify_image_launch_permission(self): - self.conn.modify_image_attribute(image_id=self.data['image_id'], - operation='add', - attribute='launchPermission', - groups='all') - image = self.conn.get_image(self.data['image_id']) - self.assertEqual(image.id, self.data['image_id']) - - def test_010_can_see_launch_permission(self): - attrs = self.conn.get_image_attribute(self.data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - self.assert_(attrs.attrs['groups'][0], 'all') - - def test_011_user_can_deregister_kernel(self): - self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) - - def test_012_can_deregister_image(self): - self.assertTrue(self.conn.deregister_image(self.data['image_id'])) - - def test_013_can_delete_bundle(self): - self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) - - -class InstanceTests(UserSmokeTestCase): - def test_001_can_create_keypair(self): - key = self.create_key_pair(self.conn, TEST_KEY) - self.assertEqual(key.name, TEST_KEY) - - def test_002_can_create_instance_with_keypair(self): - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - instance_type='m1.tiny') - self.assertEqual(len(reservation.instances), 1) - self.data['instance'] = reservation.instances[0] - - def test_003_instance_runs_within_60_seconds(self): - instance = self.data['instance'] - # allow 60 seconds to exit pending with IP - if not self.wait_for_running(self.data['instance']): - self.fail('instance failed to start') - self.data['instance'].update() - ip = self.data['instance'].private_dns_name - self.failIf(ip == '0.0.0.0') - if FLAGS.use_ipv6: - ipv6 = self.data['instance'].dns_name_v6 - self.failIf(ipv6 is None) - - def test_004_can_ping_private_ip(self): - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - - if FLAGS.use_ipv6: - if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): - self.fail('could not ping instance v6') - - def test_005_can_ssh_to_private_ip(self): - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') - - if FLAGS.use_ipv6: - if not self.wait_for_ssh(self.data['instance'].ip_v6, - TEST_KEY): - self.fail('could not ssh to instance v6') - - def test_006_can_allocate_elastic_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance'].id, - self.data['public_ip']) - self.assertTrue(result) - - def test_008_can_ssh_with_public_ip(self): - if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): - self.fail('could not ssh to public ip') - - def test_009_can_disassociate_ip_from_instance(self): - result = self.conn.disassociate_address(self.data['public_ip']) - self.assertTrue(result) - - def test_010_can_deallocate_elastic_ip(self): - result = self.conn.release_address(self.data['public_ip']) - self.assertTrue(result) - - def test_999_tearDown(self): - self.delete_key_pair(self.conn, TEST_KEY) - self.conn.terminate_instances([self.data['instance'].id]) - - -class VolumeTests(UserSmokeTestCase): - def setUp(self): - super(VolumeTests, self).setUp() - self.device = '/dev/vdb' - - def test_000_setUp(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - instance_type='m1.tiny', - key_name=TEST_KEY) - self.data['instance'] = reservation.instances[0] - if not self.wait_for_running(self.data['instance']): - self.fail('instance failed to start') - self.data['instance'].update() - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') - - def test_001_can_create_volume(self): - volume = self.conn.create_volume(1, 'nova') - self.assertEqual(volume.size, 1) - self.data['volume'] = volume - # Give network time to find volume. - time.sleep(10) - - def test_002_can_attach_volume(self): - volume = self.data['volume'] - - for x in xrange(10): - volume.update() - if volume.status.startswith('available'): - break - time.sleep(1) - else: - self.fail('cannot attach volume with state %s' % volume.status) - - volume.attach(self.data['instance'].id, self.device) - - # wait - for x in xrange(10): - volume.update() - if volume.status.startswith('in-use'): - break - time.sleep(1) - else: - self.fail('volume never got to in use') - - self.assertTrue(volume.status.startswith('in-use')) - - # Give instance time to recognize volume. - time.sleep(10) - - def test_003_can_mount_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - # NOTE(vish): this will create an dev for images that don't have - # udev rules - stdin, stdout, stderr = conn.exec_command( - 'grep %s /proc/partitions | ' - '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' - % self.device.rpartition('/')[2]) - exec_list = [] - exec_list.append('mkdir -p /mnt/vol') - exec_list.append('/sbin/mke2fs %s' % self.device) - exec_list.append('mount %s /mnt/vol' % self.device) - exec_list.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) - out = stdout.read() - conn.close() - if not out.strip().endswith('success'): - self.fail('Unable to mount: %s %s' % (out, stderr.read())) - - def test_004_can_write_to_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted - stdin, stdout, stderr = conn.exec_command( - 'echo hello > /mnt/vol/test.txt') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to write to mount: %s' % (err)) - - def test_005_volume_is_correct_size(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - stdin, stdout, stderr = conn.exec_command( - "df -h | grep %s | awk {'print $2'}" % self.device) - out = stdout.read() - conn.close() - if not out.strip() == '1007.9M': - self.fail('Volume is not the right size: %s %s' % - (out, stderr.read())) - - def test_006_me_can_umount_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to unmount: %s' % (err)) - - def test_007_me_can_detach_volume(self): - result = self.conn.detach_volume(volume_id=self.data['volume'].id) - self.assertTrue(result) - time.sleep(5) - - def test_008_me_can_delete_volume(self): - result = self.conn.delete_volume(self.data['volume'].id) - self.assertTrue(result) - - def test_999_tearDown(self): - self.conn.terminate_instances([self.data['instance'].id]) - self.conn.delete_key_pair(TEST_KEY) - - -class SecurityGroupTests(UserSmokeTestCase): - - def __public_instance_is_accessible(self): - id_url = "latest/meta-data/instance-id" - options = "-s --max-time 1" - command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) - instance_id = commands.getoutput(command).strip() - if not instance_id: - return False - if instance_id != self.data['instance_id']: - raise Exception("Wrong instance id") - return True - - def test_001_can_create_security_group(self): - self.conn.create_security_group(TEST_GROUP, description='test') - - groups = self.conn.get_all_security_groups() - self.assertTrue(TEST_GROUP in [group.name for group in groups]) - - def test_002_can_launch_instance_in_security_group(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - security_groups=[TEST_GROUP], - instance_type='m1.tiny') - - self.data['instance_id'] = reservation.instances[0].id - - def test_003_can_authorize_security_group_ingress(self): - self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - - def test_004_can_access_instance_over_public_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - result = self.conn.associate_address(self.data['instance_id'], - self.data['public_ip']) - start_time = time.time() - while not self.__public_instance_is_accessible(): - # 1 minute to launch - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_005_can_revoke_security_group_ingress(self): - self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - start_time = time.time() - while self.__public_instance_is_accessible(): - # 1 minute to teardown - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_999_tearDown(self): - self.conn.delete_key_pair(TEST_KEY) - self.conn.delete_security_group(TEST_GROUP) - groups = self.conn.get_all_security_groups() - self.assertFalse(TEST_GROUP in [group.name for group in groups]) - self.conn.terminate_instances([self.data['instance_id']]) - self.assertTrue(self.conn.release_address(self.data['public_ip'])) - - -if __name__ == "__main__": - suites = {'image': unittest.makeSuite(ImageTests), - 'instance': unittest.makeSuite(InstanceTests), - #'security_group': unittest.makeSuite(SecurityGroupTests), - 'volume': unittest.makeSuite(VolumeTests) - } - sys.exit(base.run_tests(suites)) -- cgit From ef37833e6f45f99b1d16143d29685974a191c387 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 18:04:32 -0800 Subject: add netadmin smoketests --- smoketests/netadmin_smoketests.py | 194 ++++++++++++++++++++++++++++++++++++++ smoketests/proxy.sh | 22 +++++ 2 files changed, 216 insertions(+) create mode 100644 smoketests/netadmin_smoketests.py create mode 100755 smoketests/proxy.sh diff --git a/smoketests/netadmin_smoketests.py b/smoketests/netadmin_smoketests.py new file mode 100644 index 000000000..38beb8fdc --- /dev/null +++ b/smoketests/netadmin_smoketests.py @@ -0,0 +1,194 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + +FLAGS = flags.FLAGS + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX + + +class AddressTests(base.UserSmokeTestCase): + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_allocate_floating_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + def test_002_can_associate_ip_with_instance(self): + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + self.assertTrue(result) + + def test_003_can_ssh_with_public_ip(self): + ssh_authorized = False + groups = self.conn.get_all_security_groups(['default']) + for rule in groups[0].rules: + if (rule.ip_protocol == 'tcp' and + rule.from_port <= 22 and rule.to_port >= 22): + ssh_authorized = True + if not ssh_authorized: + self.conn.authorize_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + try: + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') + finally: + if not ssh_authorized: + self.conn.revoke_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + + def test_004_can_disassociate_ip_from_instance(self): + result = self.conn.disassociate_address(self.data['public_ip']) + self.assertTrue(result) + + def test_005_can_deallocate_floating_ip(self): + result = self.conn.release_address(self.data['public_ip']) + self.assertTrue(result) + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class SecurityGroupTests(base.UserSmokeTestCase): + + def __public_instance_is_accessible(self): + id_url = "latest/meta-data/instance-id" + options = "-s --max-time 1" + command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) + instance_id = commands.getoutput(command).strip() + if not instance_id: + return False + if instance_id != self.data['instance'].id: + raise Exception("Wrong instance id") + return True + + def test_001_can_create_security_group(self): + self.conn.create_security_group(TEST_GROUP, description='test') + + groups = self.conn.get_all_security_groups() + self.assertTrue(TEST_GROUP in [group.name for group in groups]) + + def test_002_can_launch_instance_in_security_group(self): + with open("proxy.sh") as f: + user_data = f.read() + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + security_groups=[TEST_GROUP], + user_data=user_data, + instance_type='m1.tiny') + + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_003_can_authorize_security_group_ingress(self): + self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + + def test_004_can_access_metadata_over_public_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + start_time = time.time() + try: + while not self.__public_instance_is_accessible(): + # 1 minute to launch + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + finally: + result = self.conn.disassociate_address(self.data['public_ip']) + + def test_005_can_revoke_security_group_ingress(self): + self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + start_time = time.time() + while self.__public_instance_is_accessible(): + # 1 minute to teardown + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + + def test_999_tearDown(self): + self.conn.delete_key_pair(TEST_KEY) + self.conn.delete_security_group(TEST_GROUP) + groups = self.conn.get_all_security_groups() + self.assertFalse(TEST_GROUP in [group.name for group in groups]) + self.conn.terminate_instances([self.data['instance'].id]) + self.assertTrue(self.conn.release_address(self.data['public_ip'])) + + +if __name__ == "__main__": + suites = {'address': unittest.makeSuite(AddressTests), + 'security_group': unittest.makeSuite(SecurityGroupTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/proxy.sh b/smoketests/proxy.sh new file mode 100755 index 000000000..9b3f3108a --- /dev/null +++ b/smoketests/proxy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This is a simple shell script that uses netcat to set up a proxy to the +# metadata server on port 80 and to a google ip on port 8080. This is meant +# to be passed in by a script to an instance via user data, so that +# automatic testing of network connectivity can be performed. + +# Example usage: +# euca-run-instances -t m1.tiny -f proxy.sh ami-tty + +mkfifo backpipe1 +mkfifo backpipe2 + +# NOTE(vish): proxy metadata on port 80 +while true; do + nc -l -p 80 0backpipe1 +done & + +# NOTE(vish): proxy google on port 8080 +while true; do + nc -l -p 8080 0backpipe2 +done & -- cgit From 9f169fdef93898097e33b5e1c0318f543ced672e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:41:41 -0800 Subject: Reverted change to focus on the core bug - kernel_id and ramdisk_id are optional --- nova/api/openstack/servers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 11a84687d..41b05cbb4 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,11 +144,13 @@ class Controller(wsgi.Controller): metadata stored in Glance as 'image_properties' """ def lookup(param): - properties = image.get('properties') - if properties: - return properties.get(param) - else: - return image.get(param) + _image_id = image_id + try: + return image['properties'][param] + except KeyError: + LOG.debug( + _("%(param)s property not found for image %(_image_id)s") % + locals()) image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From 3ef3dfc2f6c8b9cc14119793df4990432ff74ea2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:42:23 -0800 Subject: Return null if no kernel_id / ramdisk_id --- nova/api/openstack/servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 41b05cbb4..d83bd34ab 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -151,6 +151,7 @@ class Controller(wsgi.Controller): LOG.debug( _("%(param)s property not found for image %(_image_id)s") % locals()) + return None image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From 943b863bef09a4e2b3de36c26a3fabbcc6093411 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 23:21:01 -0800 Subject: Lots of test fixing --- nova/api/ec2/cloud.py | 5 ++- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 3 +- nova/tests/api/openstack/test_servers.py | 2 +- nova/tests/test_cloud.py | 75 ++++++++------------------------ nova/tests/test_network.py | 3 ++ nova/tests/test_scheduler.py | 3 ++ nova/tests/test_virt.py | 3 ++ nova/virt/fake.py | 4 +- 9 files changed, 36 insertions(+), 64 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..99b6d5cb6 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -529,8 +529,9 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - internal_id = ec2_id_to_id(ec2_id) - return self.compute_api.get_ajax_console(context, internal_id) + instance_id = ec2_id_to_id(ec2_id) + return self.compute_api.get_ajax_console(context, + instance_id=instance_id) def describe_volumes(self, context, volume_id=None, **kwargs): if volume_id: diff --git a/nova/compute/api.py b/nova/compute/api.py index 81ea6dc53..0caadc32e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -447,7 +447,7 @@ class API(base.Base): {'method': 'authorize_ajax_console', 'args': {'token': output['token'], 'host': output['host'], 'port': output['port']}}) - return {'url': '%s?token=%s' % (FLAGS.ajax_console_proxy_url, + return {'url': '%s/?token=%s' % (FLAGS.ajax_console_proxy_url, output['token'])} def get_console_output(self, context, instance_id): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..2ab402e1c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1046,7 +1046,8 @@ def network_create_safe(context, values): @require_admin_context def network_disassociate(context, network_id): - network_update(context, network_id, {'project_id': None}) + network_update(context, network_id, {'project_id': None, + 'host': None}) @require_admin_context diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..589f3d3eb 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -84,7 +84,7 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "vcpus": 0, "local_gb": 0, "hostname": "", - "host": "", + "host": None, "instance_type": "", "user_data": "", "reservation_id": "", diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 1824d24bc..2c6dc5973 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -66,6 +66,7 @@ class CloudTestCase(test.TestCase): # set up services self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') self.network = self.start_service('network') self.manager = manager.AuthManager() @@ -73,8 +74,12 @@ class CloudTestCase(test.TestCase): self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, project=self.project) + host = self.network.get_network_host(self.context.elevated()) def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.compute.kill() @@ -201,27 +206,32 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) + greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] output = self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_ajax_console(self): + image_id = FLAGS.default_image kwargs = {'image_id': image_id} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) - self.assertEquals(b64decode(output['output']), - 'http://fakeajaxconsole.com/?token=FAKETOKEN') + greenthread.sleep(0.3) + output = self.cloud.get_ajax_console(context=self.context, + instance_id=[instance_id]) + self.assertEquals(output['url'], + '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -297,57 +307,6 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, instance1['id']) db.service_destroy(self.context, comp1['id']) - def test_instance_update_state(self): - # TODO(termie): what is this code even testing? - def instance(num): - return { - 'reservation_id': 'r-1', - 'instance_id': 'i-%s' % num, - 'image_id': 'ami-%s' % num, - 'private_dns_name': '10.0.0.%s' % num, - 'dns_name': '10.0.0%s' % num, - 'ami_launch_index': str(num), - 'instance_type': 'fake', - 'availability_zone': 'fake', - 'key_name': None, - 'kernel_id': 'fake', - 'ramdisk_id': 'fake', - 'groups': ['default'], - 'product_codes': None, - 'state': 0x01, - 'user_data': ''} - rv = self.cloud._format_describe_instances(self.context) - logging.error(str(rv)) - self.assertEqual(len(rv['reservationSet']), 0) - - # simulate launch of 5 instances - # self.cloud.instances['pending'] = {} - #for i in xrange(5): - # inst = instance(i) - # self.cloud.instances['pending'][inst['instance_id']] = inst - - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - # report 4 nodes each having 1 of the instances - #for i in xrange(4): - # self.cloud.update_state('instances', - # {('node-%s' % i): {('i-%s' % i): - # instance(i)}}) - - # one instance should be pending still - #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) - - # check that the reservations collapse - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - - # check that we can get metadata for each instance - #for i in xrange(4): - # data = self.cloud.get_metadata(instance(i)['private_dns_name']) - # self.assert_(data['meta-data']['ami-id'] == 'ami-%s' % i) - @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..53cfea276 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -117,6 +117,9 @@ class NetworkTestCase(test.TestCase): utils.to_global_ipv6( network_ref['cidr_v6'], instance_ref['mac_address'])) + self._deallocate_address(0, address) + db.instance_destroy(context.get_admin_context(), + instance_ref['id']) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 250170072..8e4a4daf5 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -118,6 +118,7 @@ class ZoneSchedulerTestCase(test.TestCase): arg = IgnoreArg() db.service_get_all_by_topic(arg, arg).AndReturn(service_list) self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + self.mox.StubOutWithMock(db, 'instance_create', use_mock_anything=True) rpc.cast(ctxt, 'compute.host1', {'method': 'run_instance', @@ -150,6 +151,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" @@ -270,6 +272,7 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.driver.schedule_run_instance, self.context, instance_id) + db.instance_destroy(self.context, instance_id) for instance_id in instance_ids1: compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..5b3247df9 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -204,6 +204,7 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) + db.instance_destroy(user_context, instance_ref['id']) def tearDown(self): super(LibvirtConnTestCase, self).tearDown() @@ -365,6 +366,7 @@ class IptablesFirewallTestCase(test.TestCase): '--dports 80:81 -j ACCEPT' % security_group_chain \ in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") + db.instance_destroy(admin_ctxt, instance_ref['id']) class NWFilterTestCase(test.TestCase): @@ -514,3 +516,4 @@ class NWFilterTestCase(test.TestCase): self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 92749f38a..4346dffc1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -319,7 +319,9 @@ class FakeConnection(object): return 'FAKE CONSOLE OUTPUT' def get_ajax_console(self, instance): - return 'http://fakeajaxconsole.com/?token=FAKETOKEN' + return {'token': 'FAKETOKEN', + 'host': 'fakeajaxconsole.com', + 'port': 6969} def get_console_pool_info(self, console_type): return {'address': '127.0.0.1', -- cgit From 015900b215805808d8cc3138b0f4deb2c0941f76 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 23:30:52 -0800 Subject: remove unnecessary stubout --- nova/tests/test_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 8e4a4daf5..b6888c4d2 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -118,7 +118,6 @@ class ZoneSchedulerTestCase(test.TestCase): arg = IgnoreArg() db.service_get_all_by_topic(arg, arg).AndReturn(service_list) self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) - self.mox.StubOutWithMock(db, 'instance_create', use_mock_anything=True) rpc.cast(ctxt, 'compute.host1', {'method': 'run_instance', -- cgit From a9075d4edc126b95910258face7f00073449073d Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Wed, 23 Feb 2011 10:27:30 +0000 Subject: FlatManager.init_host now inhibits call to method in superclass. Floating IP methods have been redefined in FlatManager to raise NotImplementedError --- nova/network/manager.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nova/network/manager.py b/nova/network/manager.py index c6eba225e..a7f263daa 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -322,6 +322,17 @@ class FlatManager(NetworkManager): """ timeout_fixed_ips = False + def init_host(self): + """Do any initialization that needs to be run if this is a + standalone service. + """ + #Fix for bug 723298 - do not call init_host on superclass + #Following code has been copied for NetworkManager.init_host + ctxt = context.get_admin_context() + for network in self.db.host_get_networks(ctxt, self.host): + self._on_set_network_host(ctxt, network['id']) + + def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute @@ -406,6 +417,22 @@ class FlatManager(NetworkManager): net['dns'] = FLAGS.flat_network_dns self.db.network_update(context, network_id, net) + def allocate_floating_ip(self, context, project_id): + #Fix for bug 723298 + raise NotImplementedError() + + def associate_floating_ip(self, context, floating_address, fixed_address): + #Fix for bug 723298 + raise NotImplementedError() + + def disassociate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + + def deallocate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + class FlatDHCPManager(FlatManager): """Flat networking with dhcp. -- cgit From 79a4c527fbb75bc563721fa23be4ea4aa97b39ee Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Wed, 23 Feb 2011 11:49:47 +0000 Subject: Fixed pep8 errors --- nova/network/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index a7f263daa..1df193be0 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -332,7 +332,6 @@ class FlatManager(NetworkManager): for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) - def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute -- cgit From d160455b77d7e180f252f4b412e3f65d7286b51f Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Feb 2011 08:45:27 -0800 Subject: allow users to omit 'nova.tests' with run_tests --- run_tests.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 6d96454b9..47e3ee317 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + import gettext import os import unittest @@ -62,6 +79,15 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('nova.tests.%s' % x) + else: + argv.append(x) + c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, @@ -70,4 +96,4 @@ if __name__ == '__main__': runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(not core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) -- cgit From 2bbbfc5af62db57158a8d6aa26912ba234d0296e Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Feb 2011 08:46:11 -0800 Subject: dump error output directly on short import errors --- run_tests.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 70212cc6a..4e8159e7b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -40,7 +40,18 @@ done function run_tests { # Just run the test suites in current environment ${wrapper} rm -f nova.sqlite - ${wrapper} $NOSETESTS + ${wrapper} $NOSETESTS 2> run_tests.err.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.err.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.err.log + fi + fi + return $RESULT } NOSETESTS="python run_tests.py $noseargs" -- cgit From ef0dfb6809f31cfe8ca8056892fc9dcc2f00a0d7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 09:40:43 -0800 Subject: Changed unit test to refer to compute API, per Todd's suggestion. Avoids needing to extend our implementation of the EC2 API. --- nova/api/ec2/cloud.py | 6 +----- nova/tests/test_quota.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 5db865b02..882cdcfc9 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -783,9 +783,6 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) - # NOTE(justinsb): the EC2 API doesn't support metadata here, but this - # is needed for the unit tests. Maybe the unit tests shouldn't be - # calling the EC2 code instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), @@ -800,8 +797,7 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone'), - metadata=kwargs.get('metadata', [])) + 'AvailabilityZone')) return self._format_run_instances(context, instances[0]['reservation_id']) diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 36ccc273e..1e42fddf3 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nova import compute from nova import context from nova import db from nova import flags @@ -168,7 +169,7 @@ class QuotaTestCase(test.TestCase): metadata = {} for i in range(FLAGS.quota_metadata_items + 1): metadata['key%s' % i] = 'value%s' % i - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, -- cgit From 48d4054e093a2faccbd819de8e9e02c03d28cda0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 11:26:38 -0800 Subject: fix for failing describe_instances test --- nova/api/ec2/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 1b96567eb..e219fb30c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -675,7 +675,8 @@ class CloudController(object): instances = [] for ec2_id in instance_id: internal_id = ec2_id_to_id(ec2_id) - instance = self.compute_api.get(context, internal_id) + instance = self.compute_api.get(context, + instance_id=internal_id) instances.append(instance) else: instances = self.compute_api.get_all(context, **kwargs) -- cgit From 5283e1c131a21ea4963c702a7137536f7b894bb6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:05:49 -0800 Subject: Created mini XPath implementation, to simplify mapping logic --- nova/api/openstack/servers.py | 21 ++----- nova/tests/test_minixpath.py | 141 ++++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 45 ++++++++++++++ 3 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 nova/tests/test_minixpath.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index b54e28c0c..794705306 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -30,7 +30,7 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack - +import types LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) @@ -63,22 +63,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - fixed_ip = inst['fixed_ip'] - if fixed_ip: - # grab single private fixed ip - try: - private_ip = fixed_ip['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) + private_ips = utils.minixpath_select(inst, 'fixed_ip/address') + inst_dict['addresses']['private'] = private_ips - # grab all public floating ips - try: - for floating in fixed_ip['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + public_ips = utils.minixpath_select(inst, 'fixed_ip/floating_ips/address') + inst_dict['addresses']['public'] = public_ips inst_dict['metadata'] = {} inst_dict['hostId'] = '' diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py new file mode 100644 index 000000000..7fddcf9e9 --- /dev/null +++ b/nova/tests/test_minixpath.py @@ -0,0 +1,141 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class MiniXPathTestCase(test.TestCase): + def test_tolerates_nones(self): + xp = utils.minixpath_select + + input = [] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [None] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + def test_does_select(self): + xp = utils.minixpath_select + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], + xp(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) + + self.assertEquals([], xp(input, "a/b/c/d")) + self.assertEquals([], xp(input, "c/a/b/d")) + self.assertEquals([], xp(input, "i/r/t")) + + def test_flattens_lists(self): + xp = utils.minixpath_select + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + def test_bad_xpath(self): + xp = utils.minixpath_select + + self.assertRaises(exception.Error, xp, [], None) + self.assertRaises(exception.Error, xp, [], "") + self.assertRaises(exception.Error, xp, [], "/") + self.assertRaises(exception.Error, xp, [], "/a") + self.assertRaises(exception.Error, xp, [], "/a/") + self.assertRaises(exception.Error, xp, [], "//") + self.assertRaises(exception.Error, xp, [], "//a") + self.assertRaises(exception.Error, xp, [], "a//a") + self.assertRaises(exception.Error, xp, [], "a//a/") + self.assertRaises(exception.Error, xp, [], "a/a/") diff --git a/nova/utils.py b/nova/utils.py index 42efa0008..2f926bd82 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -34,6 +35,7 @@ import time from xml.sax import saxutils import re import netaddr +import types from eventlet import event from eventlet import greenthread @@ -499,3 +501,46 @@ def ensure_b64_encoding(val): return val except TypeError: return base64.b64encode(val) + + +def minixpath_select(items, minixpath): + """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each + item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of + the intermediate results are lists it will treat each list item + individually. A 'None' in items or any child expressions will be ignored, + this function will not throw because of None (anywhere) in items""" + + if minixpath is None: + raise exception.Error("Invalid mini_xpath") + + (first_token, sep, remainder) = minixpath.partition("/") + + if first_token == "": + raise exception.Error("Invalid mini_xpath") + + results = [] + + if items is None: + return results + + for item in items: + if item is None: + continue + get_method = getattr(item, "get", None) + if get_method is None: + continue + child = get_method(first_token) + if child is None: + continue + if isinstance(child, types.ListType): + # Flatten intermediate lists + for x in child: + results.append(x) + else: + results.append(child) + + if not sep: + # No more tokens + return results + else: + return minixpath_select(results, remainder) -- cgit From b3b005f50de54b5ef6c62e387dcec5a123f93cf6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:36:09 -0800 Subject: Cope when we pass a non-list to xpath_select - wrap it in a list --- nova/tests/test_minixpath.py | 38 ++++++++++++++++++++++++++++++++++++++ nova/utils.py | 8 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 7fddcf9e9..3b1bdf40b 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -139,3 +139,41 @@ class MiniXPathTestCase(test.TestCase): self.assertRaises(exception.Error, xp, [], "a//a") self.assertRaises(exception.Error, xp, [], "a//a/") self.assertRaises(exception.Error, xp, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + xp = utils.minixpath_select + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = xp(inst, 'fixed_ip/address') + public_ips = xp(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + xp = utils.minixpath_select + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + diff --git a/nova/utils.py b/nova/utils.py index 2f926bd82..c2cbeb2a7 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -508,7 +508,8 @@ def minixpath_select(items, minixpath): item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of the intermediate results are lists it will treat each list item individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items""" + this function will not throw because of None (anywhere) in items. The + returned list will contain no None values.""" if minixpath is None: raise exception.Error("Invalid mini_xpath") @@ -523,6 +524,10 @@ def minixpath_select(items, minixpath): if items is None: return results + if not isinstance(items, types.ListType): + # Wrap single objects in a list + items = [items] + for item in items: if item is None: continue @@ -532,6 +537,7 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue + #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: -- cgit From 21ebea24b4b77f8bd1fd42152454f1b0189843d4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 12:54:46 -0800 Subject: fix describe_availability_zones --- nova/api/ec2/cloud.py | 5 +++-- nova/db/api.py | 4 ++-- nova/db/sqlalchemy/api.py | 9 +++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 99b6d5cb6..9e8764836 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -198,8 +198,9 @@ class CloudController(object): return self._describe_availability_zones(context, **kwargs) def _describe_availability_zones(self, context, **kwargs): - enabled_services = db.service_get_all(context) - disabled_services = db.service_get_all(context, True) + ctxt = context.elevated() + enabled_services = db.service_get_all(ctxt) + disabled_services = db.service_get_all(ctxt, True) available_zones = [] for zone in [service.availability_zone for service in enabled_services]: diff --git a/nova/db/api.py b/nova/db/api.py index d7f3746d2..0a010e727 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -85,8 +85,8 @@ def service_get(context, service_id): def service_get_all(context, disabled=False): - """Get all service.""" - return IMPL.service_get_all(context, None, disabled) + """Get all services.""" + return IMPL.service_get_all(context, disabled) def service_get_all_by_topic(context, topic): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index be29fe2a0..d8751bef4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -136,15 +136,12 @@ def service_get(context, service_id, session=None): @require_admin_context -def service_get_all(context, session=None, disabled=False): - if not session: - session = get_session() - - result = session.query(models.Service).\ +def service_get_all(context, disabled=False): + session = get_session() + return session.query(models.Service).\ filter_by(deleted=can_read_deleted(context)).\ filter_by(disabled=disabled).\ all() - return result @require_admin_context -- cgit From 89ade95d2eaabf77f9c81a8d50c7cc11aa175464 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:55:07 -0800 Subject: Fix pep8 violation (trailing whitespace) --- nova/tests/test_minixpath.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 3b1bdf40b..cc4a35ef3 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -175,5 +175,3 @@ class MiniXPathTestCase(test.TestCase): input = {'a': [1, 2, {'b': 'b_1'}]} self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) self.assertEquals(['b_1'], xp(input, "a/b")) - - -- cgit From 1183c9e11b12984b1f5007ace831864e80483712 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:07:08 -0800 Subject: Rename minixpath_select to get_from_path --- nova/api/openstack/servers.py | 5 +- nova/tests/test_minixpath.py | 177 ------------------------------------------ nova/tests/test_utils.py | 174 +++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 24 +++--- 4 files changed, 188 insertions(+), 192 deletions(-) delete mode 100644 nova/tests/test_minixpath.py create mode 100644 nova/tests/test_utils.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 794705306..ce4a6256a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -30,7 +30,6 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack -import types LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) @@ -63,10 +62,10 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - private_ips = utils.minixpath_select(inst, 'fixed_ip/address') + private_ips = utils.get_from_path(inst, 'fixed_ip/address') inst_dict['addresses']['private'] = private_ips - public_ips = utils.minixpath_select(inst, 'fixed_ip/floating_ips/address') + public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips inst_dict['metadata'] = {} diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py deleted file mode 100644 index cc4a35ef3..000000000 --- a/nova/tests/test_minixpath.py +++ /dev/null @@ -1,177 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import test -from nova import utils -from nova import exception - - -class MiniXPathTestCase(test.TestCase): - def test_tolerates_nones(self): - xp = utils.minixpath_select - - input = [] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [None] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': None}] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': None}}] - self.assertEquals([{'b': None}], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': None}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] - self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - def test_does_select(self): - xp = utils.minixpath_select - - input = [{'a': 'a_1'}] - self.assertEquals(['a_1'], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': 'b_1'}}] - self.assertEquals([{'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] - self.assertEquals([{'b': {'c': 'c_1'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': None}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': {'c': 'c_2'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], - xp(input, "a/b")) - self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) - - self.assertEquals([], xp(input, "a/b/c/d")) - self.assertEquals([], xp(input, "c/a/b/d")) - self.assertEquals([], xp(input, "i/r/t")) - - def test_flattens_lists(self): - xp = utils.minixpath_select - - input = [{'a': [1, 2, 3]}] - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}] - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [1, 2, {'b': 'b_1'}]}] - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - - def test_bad_xpath(self): - xp = utils.minixpath_select - - self.assertRaises(exception.Error, xp, [], None) - self.assertRaises(exception.Error, xp, [], "") - self.assertRaises(exception.Error, xp, [], "/") - self.assertRaises(exception.Error, xp, [], "/a") - self.assertRaises(exception.Error, xp, [], "/a/") - self.assertRaises(exception.Error, xp, [], "//") - self.assertRaises(exception.Error, xp, [], "//a") - self.assertRaises(exception.Error, xp, [], "a//a") - self.assertRaises(exception.Error, xp, [], "a//a/") - self.assertRaises(exception.Error, xp, [], "a/a/") - - def test_real_failure1(self): - # Real world failure case... - # We weren't coping when the input was a Dictionary instead of a List - # This led to test_accepts_dictionaries - xp = utils.minixpath_select - - inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], - 'address': '192.168.0.3'}, - 'hostname': ''} - - private_ips = xp(inst, 'fixed_ip/address') - public_ips = xp(inst, 'fixed_ip/floating_ips/address') - self.assertEquals(['192.168.0.3'], private_ips) - self.assertEquals(['1.2.3.4'], public_ips) - - def test_accepts_dictionaries(self): - xp = utils.minixpath_select - - input = {'a': [1, 2, 3]} - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': {'b': [1, 2, 3]}} - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [1, 2, {'b': 'b_1'}]} - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py new file mode 100644 index 000000000..34a407f1a --- /dev/null +++ b/nova/tests/test_utils.py @@ -0,0 +1,174 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class GetFromPathTestCase(test.TestCase): + def test_tolerates_nones(self): + f = utils.get_from_path + + input = [] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [None] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + def test_does_select(self): + f = utils.get_from_path + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + f(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], f(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], f(input, "a/b/c")) + + self.assertEquals([], f(input, "a/b/c/d")) + self.assertEquals([], f(input, "c/a/b/d")) + self.assertEquals([], f(input, "i/r/t")) + + def test_flattens_lists(self): + f = utils.get_from_path + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + + def test_bad_xpath(self): + f = utils.get_from_path + + self.assertRaises(exception.Error, f, [], None) + self.assertRaises(exception.Error, f, [], "") + self.assertRaises(exception.Error, f, [], "/") + self.assertRaises(exception.Error, f, [], "/a") + self.assertRaises(exception.Error, f, [], "/a/") + self.assertRaises(exception.Error, f, [], "//") + self.assertRaises(exception.Error, f, [], "//a") + self.assertRaises(exception.Error, f, [], "a//a") + self.assertRaises(exception.Error, f, [], "a//a/") + self.assertRaises(exception.Error, f, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + f = utils.get_from_path + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = f(inst, 'fixed_ip/address') + public_ips = f(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + f = utils.get_from_path + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) diff --git a/nova/utils.py b/nova/utils.py index c2cbeb2a7..65e28c648 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -32,10 +32,10 @@ import string import struct import sys import time +import types from xml.sax import saxutils import re import netaddr -import types from eventlet import event from eventlet import greenthread @@ -503,18 +503,19 @@ def ensure_b64_encoding(val): return base64.b64encode(val) -def minixpath_select(items, minixpath): - """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each - item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of - the intermediate results are lists it will treat each list item - individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items. The - returned list will contain no None values.""" +def get_from_path(items, path): + """ Returns a list of items matching the specified path. Takes an + XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, + looks up items[prop1][prop2][prop3]. Like XPath, if any of the + intermediate results are lists it will treat each list item individually. + A 'None' in items or any child expressions will be ignored, this function + will not throw because of None (anywhere) in items. The returned list + will contain no None values.""" - if minixpath is None: + if path is None: raise exception.Error("Invalid mini_xpath") - (first_token, sep, remainder) = minixpath.partition("/") + (first_token, sep, remainder) = path.partition("/") if first_token == "": raise exception.Error("Invalid mini_xpath") @@ -537,7 +538,6 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue - #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: @@ -549,4 +549,4 @@ def minixpath_select(items, minixpath): # No more tokens return results else: - return minixpath_select(results, remainder) + return get_from_path(results, remainder) -- cgit From a508e2dce781b98db5a719df75a451d9a2727fca Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:12:43 -0800 Subject: Make sure there are two blank links after the import --- nova/api/openstack/servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ce4a6256a..6c227d71a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -31,6 +31,7 @@ from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack + LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) -- cgit From 8c007b56b586257d048b6db4ecfbed8f502381fd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:16:31 -0800 Subject: Put back the comments I accidentally removed --- nova/api/openstack/servers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 6c227d71a..97323f66f 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,9 +63,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) + # grab single private fixed ip private_ips = utils.get_from_path(inst, 'fixed_ip/address') inst_dict['addresses']['private'] = private_ips + # grab all public floating ips public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips -- cgit