From 9bd72f56224a8cc980620b17210d9b9b9ede6166 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 18 Jan 2011 18:33:04 -0800 Subject: various fixes to smoketests, including allowing admin tests to run as a user, better timing, and allowing volume tests to run on non-udev linux --- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/smoketests/flags.py b/smoketests/flags.py index 35f432a77..9dc310692 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -35,5 +35,5 @@ DEFINE_bool = DEFINE_bool # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 DEFINE_string('region', 'nova', 'Region to use') -DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') +DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') DEFINE_string('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index d5a3a7556..f73ab7e1c 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -258,10 +258,15 @@ class VolumeTests(UserSmokeTestCase): instance = reservation.instances[0] self.data['instance'] = instance for x in xrange(120): - if self.can_ping(instance.private_dns_name): + time.sleep(1) + instance.update() + #if self.can_ping(instance.private_dns_name): + if instance.state == u'running': break else: self.fail('unable to start instance') + time.sleep(10) + instance.update() def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') @@ -273,10 +278,11 @@ class VolumeTests(UserSmokeTestCase): def test_002_can_attach_volume(self): volume = self.data['volume'] - for x in xrange(10): - if volume.status == u'available': + for x in xrange(30): + print volume.status + if volume.status.startswith('available'): break - time.sleep(5) + time.sleep(1) volume.update() else: self.fail('cannot attach volume with state %s' % volume.status) @@ -285,12 +291,12 @@ class VolumeTests(UserSmokeTestCase): # Volumes seems to report "available" too soon. for x in xrange(10): - if volume.status == u'in-use': + if volume.status.startswith('in-use'): break time.sleep(5) volume.update() - self.assertEqual(volume.status, u'in-use') + self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. time.sleep(5) @@ -298,9 +304,15 @@ class VolumeTests(UserSmokeTestCase): def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name conn = self.connect_ssh(ip, TEST_KEY) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) commands = [] commands.append('mkdir -p /mnt/vol') - commands.append('mkfs.ext2 %s' % self.device) + commands.append('/sbin/mke2fs %s' % self.device) commands.append('mount %s /mnt/vol' % self.device) commands.append('echo success') stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) @@ -327,7 +339,7 @@ class VolumeTests(UserSmokeTestCase): "df -h | grep %s | awk {'print $2'}" % self.device) out = stdout.read() conn.close() - if not out.strip() == '1008M': + if not out.strip() == '1007.9M': self.fail('Volume is not the right size: %s %s' % (out, stderr.read())) -- cgit From 2f4258d99e8d97ec70645cd2df2f4e54dc869e89 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 20 Jan 2011 00:14:42 -0800 Subject: more smoketest fixes --- nova/__init__.py | 2 -- smoketests/base.py | 7 +++++-- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 26 ++++++++++++++++---------- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/nova/__init__.py b/nova/__init__.py index 8745617bc..256db55a9 100644 --- a/nova/__init__.py +++ b/nova/__init__.py @@ -30,5 +30,3 @@ .. moduleauthor:: Manish Singh .. moduleauthor:: Andy Smith """ - -from exception import * diff --git a/smoketests/base.py b/smoketests/base.py index 610270c5c..89ee92840 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -17,12 +17,10 @@ # under the License. import boto -import boto_v6 import commands import httplib import os import paramiko -import random import sys import unittest from boto.ec2.regioninfo import RegionInfo @@ -30,6 +28,8 @@ from boto.ec2.regioninfo import RegionInfo from smoketests import flags FLAGS = flags.FLAGS +boto_v6 = None + class SmokeTestCase(unittest.TestCase): @@ -146,6 +146,9 @@ class SmokeTestCase(unittest.TestCase): def run_tests(suites): argv = FLAGS(sys.argv) + if FLAGS.use_ipv6: + global boto_v6 + boto_v6 = __import__('boto_v6') if not os.getenv('EC2_ACCESS_KEY'): print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ diff --git a/smoketests/flags.py b/smoketests/flags.py index 9dc310692..dc756347b 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -36,4 +36,4 @@ DEFINE_bool = DEFINE_bool DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') -DEFINE_string('use_ipv6', True, 'use the ipv6 or not') +DEFINE_bool('use_ipv6', True, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index f73ab7e1c..e5bc98ede 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -189,8 +189,8 @@ class InstanceTests(UserSmokeTestCase): try: conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) conn.close() - except Exception: - time.sleep(1) + except Exception, e: + time.sleep(5) else: break else: @@ -224,7 +224,7 @@ class InstanceTests(UserSmokeTestCase): try: conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) conn.close() - except socket.error: + except Exception: time.sleep(1) else: break @@ -256,17 +256,24 @@ class VolumeTests(UserSmokeTestCase): instance_type='m1.tiny', key_name=TEST_KEY) instance = reservation.instances[0] - self.data['instance'] = instance for x in xrange(120): time.sleep(1) instance.update() - #if self.can_ping(instance.private_dns_name): - if instance.state == u'running': + if self.can_ping(instance.private_dns_name): break else: self.fail('unable to start instance') - time.sleep(10) - instance.update() + self.data['instance'] = instance + for x in xrange(30): + try: + conn = self.connect_ssh(instance.private_dns_name, TEST_KEY) + conn.close() + except Exception: + time.sleep(5) + else: + break + else: + self.fail('could not ssh to instance') def test_001_can_create_volume(self): volume = self.conn.create_volume(1, 'nova') @@ -279,7 +286,6 @@ class VolumeTests(UserSmokeTestCase): volume = self.data['volume'] for x in xrange(30): - print volume.status if volume.status.startswith('available'): break time.sleep(1) @@ -438,7 +444,7 @@ class SecurityGroupTests(UserSmokeTestCase): if __name__ == "__main__": suites = {'image': unittest.makeSuite(ImageTests), 'instance': unittest.makeSuite(InstanceTests), - 'security_group': unittest.makeSuite(SecurityGroupTests), + #'security_group': unittest.makeSuite(SecurityGroupTests), 'volume': unittest.makeSuite(VolumeTests) } sys.exit(base.run_tests(suites)) -- cgit From e6b7fa7ae31e90f2d7322445da3843281fff9a70 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sat, 22 Jan 2011 16:20:36 -0800 Subject: fixes and refactoring of smoketests --- smoketests/base.py | 43 +++++++++++--- smoketests/flags.py | 2 +- smoketests/user_smoketests.py | 133 +++++++++++++----------------------------- 3 files changed, 77 insertions(+), 101 deletions(-) diff --git a/smoketests/base.py b/smoketests/base.py index 89ee92840..afc618074 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -22,6 +22,7 @@ import httplib import os import paramiko import sys +import time import unittest from boto.ec2.regioninfo import RegionInfo @@ -31,7 +32,6 @@ FLAGS = flags.FLAGS boto_v6 = None - class SmokeTestCase(unittest.TestCase): def connect_ssh(self, ip, key_name): # TODO(devcamcar): set a more reasonable connection timeout time @@ -39,12 +39,10 @@ class SmokeTestCase(unittest.TestCase): client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip, username='root', pkey=key) - stdin, stdout, stderr = client.exec_command('uptime') - print 'uptime: ', stdout.read() return client - def can_ping(self, ip): - """ Attempt to ping the specified IP, and give up after 1 second. """ + def can_ping(self, ip, command="ping"): + """Attempt to ping the specified IP, and give up after 1 second.""" # NOTE(devcamcar): ping timeout flag is different in OSX. if sys.platform == 'darwin': @@ -52,10 +50,41 @@ class SmokeTestCase(unittest.TestCase): else: timeout_flag = 'w' - status, output = commands.getstatusoutput('ping -c1 -%s1 %s' % - (timeout_flag, ip)) + status, output = commands.getstatusoutput('%s -c1 -%s1 %s' % + (command, timeout_flag, ip)) return status == 0 + def wait_for_running(self, instance, tries=60, wait=1): + """Wait for instance to be running""" + for x in xrange(tries): + instance.update() + if instance.state.startswith('running'): + return True + time.sleep(wait) + else: + return False + + def wait_for_ping(self, ip, command="ping", tries=120): + """Wait for ip to be pingable""" + for x in xrange(tries): + if self.can_ping(ip, command): + return True + else: + return False + + def wait_for_ssh(self, ip, key_name, tries=30, wait=5): + """Wait for ip to be sshable""" + for x in xrange(tries): + try: + conn = self.connect_ssh(ip, key_name) + conn.close() + except Exception, e: + time.sleep(wait) + else: + return True + else: + return False + def connection_for_env(self, **kwargs): """ Returns a boto ec2 connection for the current environment. diff --git a/smoketests/flags.py b/smoketests/flags.py index dc756347b..5f3c8505e 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -36,4 +36,4 @@ DEFINE_bool = DEFINE_bool DEFINE_string('region', 'nova', 'Region to use') DEFINE_string('test_image', 'ami-tty', 'Image to use for launch tests') -DEFINE_bool('use_ipv6', True, 'use the ipv6 or not') +DEFINE_bool('use_ipv6', False, 'use the ipv6 or not') diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py index e5bc98ede..26f6344f7 100644 --- a/smoketests/user_smoketests.py +++ b/smoketests/user_smoketests.py @@ -19,7 +19,6 @@ import commands import os import random -import socket import sys import time import unittest @@ -91,7 +90,6 @@ class ImageTests(UserSmokeTestCase): break time.sleep(1) else: - print image.state self.assert_(False) # wasn't available within 10 seconds self.assert_(image.type == 'machine') @@ -143,70 +141,36 @@ class InstanceTests(UserSmokeTestCase): key_name=TEST_KEY, instance_type='m1.tiny') self.assertEqual(len(reservation.instances), 1) - self.data['instance_id'] = reservation.instances[0].id + self.data['instance'] = reservation.instances[0] def test_003_instance_runs_within_60_seconds(self): - reservations = self.conn.get_all_instances([self.data['instance_id']]) - instance = reservations[0].instances[0] + instance = self.data['instance'] # allow 60 seconds to exit pending with IP - for x in xrange(60): - instance.update() - if instance.state == u'running': - break - time.sleep(1) - else: + if not self.wait_for_running(self.data['instance']): self.fail('instance failed to start') - ip = reservations[0].instances[0].private_dns_name + self.data['instance'].update() + ip = self.data['instance'].private_dns_name self.failIf(ip == '0.0.0.0') - self.data['private_ip'] = ip if FLAGS.use_ipv6: - ipv6 = reservations[0].instances[0].dns_name_v6 + ipv6 = self.data['instance'].dns_name_v6 self.failIf(ipv6 is None) - self.data['ip_v6'] = ipv6 def test_004_can_ping_private_ip(self): - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 %s' % self.data['private_ip']) - if status == 0: - break - else: + if not self.wait_for_ping(self.data['instance'].private_dns_name): self.fail('could not ping instance') if FLAGS.use_ipv6: - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping6 -c1 %s' % self.data['ip_v6']) - if status == 0: - break - else: - self.fail('could not ping instance') + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') def test_005_can_ssh_to_private_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) - conn.close() - except Exception, e: - time.sleep(5) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') if FLAGS.use_ipv6: - for x in xrange(30): - try: - conn = self.connect_ssh( - self.data['ip_v6'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): self.fail('could not ssh to instance v6') def test_006_can_allocate_elastic_ip(self): @@ -215,21 +179,13 @@ class InstanceTests(UserSmokeTestCase): self.data['public_ip'] = result.public_ip def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance_id'], + result = self.conn.associate_address(self.data['instance'].id, self.data['public_ip']) self.assertTrue(result) def test_008_can_ssh_with_public_ip(self): - for x in xrange(30): - try: - conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) - conn.close() - except Exception: - time.sleep(1) - else: - break - else: - self.fail('could not ssh to instance') + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') def test_009_can_disassociate_ip_from_instance(self): result = self.conn.disassociate_address(self.data['public_ip']) @@ -241,8 +197,7 @@ class InstanceTests(UserSmokeTestCase): def test_999_tearDown(self): self.delete_key_pair(self.conn, TEST_KEY) - if self.data.has_key('instance_id'): - self.conn.terminate_instances([self.data['instance_id']]) + self.conn.terminate_instances([self.data['instance'].id]) class VolumeTests(UserSmokeTestCase): @@ -255,24 +210,14 @@ class VolumeTests(UserSmokeTestCase): reservation = self.conn.run_instances(FLAGS.test_image, instance_type='m1.tiny', key_name=TEST_KEY) - instance = reservation.instances[0] - for x in xrange(120): - time.sleep(1) - instance.update() - if self.can_ping(instance.private_dns_name): - break - else: - self.fail('unable to start instance') - self.data['instance'] = instance - for x in xrange(30): - try: - conn = self.connect_ssh(instance.private_dns_name, TEST_KEY) - conn.close() - except Exception: - time.sleep(5) - else: - break - else: + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): self.fail('could not ssh to instance') def test_001_can_create_volume(self): @@ -280,32 +225,34 @@ class VolumeTests(UserSmokeTestCase): self.assertEqual(volume.size, 1) self.data['volume'] = volume # Give network time to find volume. - time.sleep(5) + time.sleep(10) def test_002_can_attach_volume(self): volume = self.data['volume'] - for x in xrange(30): + for x in xrange(10): + volume.update() if volume.status.startswith('available'): break time.sleep(1) - volume.update() else: self.fail('cannot attach volume with state %s' % volume.status) volume.attach(self.data['instance'].id, self.device) - # Volumes seems to report "available" too soon. + # wait for x in xrange(10): + volume.update() if volume.status.startswith('in-use'): break - time.sleep(5) - volume.update() + time.sleep(1) + else: + self.fail('volume never got to in use') self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. - time.sleep(5) + time.sleep(10) def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name @@ -316,12 +263,12 @@ class VolumeTests(UserSmokeTestCase): 'grep %s /proc/partitions | ' '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' % self.device.rpartition('/')[2]) - commands = [] - commands.append('mkdir -p /mnt/vol') - commands.append('/sbin/mke2fs %s' % self.device) - commands.append('mount %s /mnt/vol' % self.device) - commands.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) out = stdout.read() conn.close() if not out.strip().endswith('success'): -- cgit From 8001f334221a16d8328289f6954ef549844f76f3 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Mon, 14 Feb 2011 14:26:32 +0100 Subject: Fix DescribeRegion answer by using specific 'listen' configuration parameter instead of overloading ec2_host --- bin/nova-api | 4 ++-- bin/nova-combined | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 11176a021..eb59d0191 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -59,12 +59,12 @@ def run_app(paste_config_file): LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) wsgi.paste_config_to_flags(config, { "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_host" % api: getattr(FLAGS, "%s_host" % api), "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + config.get('listen', '0.0.0.0'))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) diff --git a/bin/nova-combined b/bin/nova-combined index 913c866bf..889600eb7 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -67,11 +67,11 @@ if __name__ == '__main__': continue wsgi.paste_config_to_flags(config, { "verbose": FLAGS.verbose, - "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_host" % api: getattr(FLAGS, "%s_host" % api), "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) app = wsgi.load_paste_app(paste_config_file, api) apps.append((app, getattr(FLAGS, "%s_port" % api), - getattr(FLAGS, "%s_host" % api))) + config.get('listen', '0.0.0.0'))) if len(apps) > 0: logging.basicConfig() server = wsgi.Server() -- cgit From 745b7b22f7b22a09e6c3bbc1cd8591eb3aa7f554 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 15 Feb 2011 21:38:47 +0100 Subject: removed flag --pidfile from nova/services.py --- nova/service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/service.py b/nova/service.py index 59648adf2..8b1b91e90 100644 --- a/nova/service.py +++ b/nova/service.py @@ -50,10 +50,6 @@ flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) -flags.DEFINE_string('pidfile', None, - 'pidfile to use for this service') - - flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) -- cgit From 9a7213b615bcaa2127f76146d594f5247ea0d0a4 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 17 Feb 2011 15:00:18 -0800 Subject: Initial support for per-instance metadata, though the OpenStack API. Key/value pairs can be specified at instance creation time and are returned in the details view. Support limits based on quota system. --- nova/api/ec2/cloud.py | 6 +- nova/api/openstack/servers.py | 30 +++++++-- nova/compute/api.py | 29 +++++++- nova/db/sqlalchemy/api.py | 2 + .../versions/004_add_instance_metadata.py | 78 ++++++++++++++++++++++ nova/db/sqlalchemy/models.py | 18 ++++- nova/quota.py | 14 +++- nova/tests/api/openstack/test_servers.py | 11 ++- nova/tests/test_quota.py | 24 +++++++ run_tests.sh | 4 +- 10 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6919cd8d2..33eba5028 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -783,6 +783,9 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) + # NOTE(justinsb): the EC2 API doesn't support metadata here, but this + # is needed for the unit tests. Maybe the unit tests shouldn't be + # calling the EC2 code instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), @@ -797,7 +800,8 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone')) + 'AvailabilityZone'), + metadata=kwargs.get('metadata', [])) return self._format_run_instances(context, instances[0]['reservation_id']) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 009ef6db1..49611703a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -78,9 +78,14 @@ def _translate_detail_keys(inst): except KeyError: LOG.debug(_("Failed to read public ip(s)")) - inst_dict['metadata'] = {} inst_dict['hostId'] = '' + # Return the metadata as a dictionary + metadata = {} + for item in inst['metadata']: + metadata[item['key']] = item['value'] + inst_dict['metadata'] = metadata + return dict(server=inst_dict) @@ -162,14 +167,26 @@ class Controller(wsgi.Controller): if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - key_pair = auth_manager.AuthManager.get_key_pairs( - req.environ['nova.context'])[0] + context = req.environ['nova.context'] + + key_pair = auth_manager.AuthManager.get_key_pairs(context)[0] image_id = common.get_image_id_from_image_hash(self._image_service, - req.environ['nova.context'], env['server']['imageId']) + context, env['server']['imageId']) kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( req, image_id) + + # Metadata is a list, not a Dictionary, because we allow duplicate keys + # (even though JSON can't encode this) + # In future, we may not allow duplicate keys. + # However, the CloudServers API is not definitive on this front, + # and we want to be compatible. + metadata = [] + if env['server']['metadata']: + for k, v in env['server']['metadata'].items(): + metadata.append({'key': k, 'value': v}) + instances = self.compute_api.create( - req.environ['nova.context'], + context, instance_types.get_by_flavor_id(env['server']['flavorId']), image_id, kernel_id=kernel_id, @@ -177,7 +194,8 @@ class Controller(wsgi.Controller): display_name=env['server']['name'], display_description=env['server']['name'], key_name=key_pair['name'], - key_data=key_pair['public_key']) + key_data=key_pair['public_key'], + metadata=metadata) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index ed6f0e34a..cad167f4d 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -85,7 +85,7 @@ class API(base.Base): min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', - availability_zone=None, user_data=None): + availability_zone=None, user_data=None, metadata=[]): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -99,6 +99,30 @@ class API(base.Base): "run %s more instances of this type.") % num_instances, "InstanceLimitExceeded") + num_metadata = len(metadata) + quota_metadata = quota.allowed_metadata_items(context, num_metadata) + if quota_metadata < num_metadata: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " tried to set %(num_metadata)s metadata properties") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + + # Because metadata is stored in the DB, we hard-code the size limits + # In future, we may support more variable length strings, so we act + # as if this is quota-controlled for forwards compatibility + for metadata_item in metadata: + k = metadata_item['key'] + v = metadata_item['value'] + if len(k) > 255 or len(v) > 255: + pid = context.project_id + msg = (_("Quota exceeeded for %(pid)s," + " metadata property key or value too long") + % locals()) + LOG.warn(msg) + raise quota.QuotaError(msg, "MetadataLimitExceeded") + is_vpn = image_id == FLAGS.vpn_image_id if not is_vpn: image = self.image_service.show(context, image_id) @@ -155,7 +179,8 @@ class API(base.Base): 'key_name': key_name, 'key_data': key_data, 'locked': False, - 'availability_zone': availability_zone} + 'availability_zone': availability_zone, + 'metadata': metadata} elevated = context.elevated() instances = [] diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..a6b8066b9 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -715,6 +715,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ filter_by(id=instance_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -723,6 +724,7 @@ def instance_get(context, instance_id, session=None): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload_all('security_groups.rules')).\ options(joinedload('volumes')).\ + options(joinedload('metadata')).\ filter_by(project_id=context.project_id).\ filter_by(id=instance_id).\ filter_by(deleted=False).\ diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 40a96fc17..a842e4cc4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -256,6 +256,7 @@ class Quota(BASE, NovaBase): volumes = Column(Integer) gigabytes = Column(Integer) floating_ips = Column(Integer) + metadata_items = Column(Integer) class ExportDevice(BASE, NovaBase): @@ -536,6 +537,20 @@ class Console(BASE, NovaBase): pool = relationship(ConsolePool, backref=backref('consoles')) +class InstanceMetadata(BASE, NovaBase): + """Represents a metadata key/value pair for an instance""" + __tablename__ = 'instance_metadata' + id = Column(Integer, primary_key=True) + key = Column(String(255)) + value = Column(String(255)) + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False) + instance = relationship(Instance, backref="metadata", + foreign_keys=instance_id, + primaryjoin='and_(' + 'InstanceMetadata.instance_id == Instance.id,' + 'InstanceMetadata.deleted == False)') + + class Zone(BASE, NovaBase): """Represents a child zone of this zone.""" __tablename__ = 'zones' @@ -557,7 +572,8 @@ def register_models(): Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, - Project, Certificate, ConsolePool, Console, Zone) + Project, Certificate, ConsolePool, Console, Zone, + InstanceMetadata) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/quota.py b/nova/quota.py index 3884eb308..6b52a97fa 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -35,6 +35,8 @@ flags.DEFINE_integer('quota_gigabytes', 1000, 'number of volume gigabytes allowed per project') flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') +flags.DEFINE_integer('quota_metadata_items', 128, + 'number of metadata items allowed per instance') def get_quota(context, project_id): @@ -42,7 +44,8 @@ def get_quota(context, project_id): 'cores': FLAGS.quota_cores, 'volumes': FLAGS.quota_volumes, 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips} + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items} try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -94,6 +97,15 @@ def allowed_floating_ips(context, num_floating_ips): return min(num_floating_ips, allowed_floating_ips) +def allowed_metadata_items(context, num_metadata_items): + """Check quota; return min(num_metadata_items,allowed_metadata_items)""" + project_id = context.project_id + context = context.elevated() + quota = get_quota(context, project_id) + num_allowed_metadata_items = quota['metadata_items'] + return min(num_metadata_items, num_allowed_metadata_items) + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..7eb81c2b8 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -28,6 +28,7 @@ import nova.api.openstack from nova.api.openstack import servers import nova.db.api from nova.db.sqlalchemy.models import Instance +from nova.db.sqlalchemy.models import InstanceMetadata import nova.rpc from nova.tests.api.openstack import fakes @@ -64,6 +65,9 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, private_address=None, public_addresses=None): + metadata = [] + metadata.append(InstanceMetadata(key='seq', value=id)) + if public_addresses == None: public_addresses = list() @@ -95,7 +99,8 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "availability_zone": "", "display_name": "server%s" % id, "display_description": "", - "locked": False} + "locked": False, + "metadata": metadata} instance["fixed_ip"] = { "address": private_address, @@ -214,7 +219,8 @@ class ServersTest(unittest.TestCase): "get_image_id_from_image_hash", image_id_from_hash) body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, metadata={}, + name='server_test', imageId=2, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, personality={})) req = webob.Request.blank('/v1.0/servers') req.method = 'POST' @@ -291,6 +297,7 @@ class ServersTest(unittest.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) + self.assertEqual(s['metadata']['seq'], i) i += 1 def test_server_pause(self): diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 9548a8c13..36ccc273e 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -87,6 +87,18 @@ class QuotaTestCase(test.TestCase): num_instances = quota.allowed_instances(self.context, 100, instance_types.INSTANCE_TYPES['m1.small']) self.assertEqual(num_instances, 10) + + # metadata_items + too_many_items = FLAGS.quota_metadata_items + 1000 + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, FLAGS.quota_metadata_items) + db.quota_update(self.context, self.project.id, {'metadata_items': 5}) + num_metadata_items = quota.allowed_metadata_items(self.context, + too_many_items) + self.assertEqual(num_metadata_items, 5) + + # Cleanup db.quota_destroy(self.context, self.project.id) def test_too_many_instances(self): @@ -151,3 +163,15 @@ class QuotaTestCase(test.TestCase): self.assertRaises(quota.QuotaError, self.cloud.allocate_address, self.context) db.floating_ip_destroy(context.get_admin_context(), address) + + def test_too_many_metadata_items(self): + metadata = {} + for i in range(FLAGS.quota_metadata_items + 1): + metadata['key%s' % i] = 'value%s' % i + self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.context, + min_count=1, + max_count=1, + instance_type='m1.small', + image_id='fake', + metadata=metadata) diff --git a/run_tests.sh b/run_tests.sh index 4e21fe945..58e92c06b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -73,7 +73,9 @@ fi if [ -z "$noseargs" ]; then - run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py bin/* nova setup.py || exit 1 + srcfiles=`find bin -type f ! -name "nova.conf*"` + srcfiles+=" nova setup.py" + run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} || exit 1 else run_tests fi -- cgit From debfca945627323c160b4ad9aa9b63b364deff99 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:02:55 +0100 Subject: Switch to API_listen and API_listen_port, drop wsgi.paste_config_to_flags --- bin/nova-api | 18 ++++++++++++------ etc/nova-api.conf | 3 --- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index eb59d0191..e7ee6f6fe 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -47,6 +47,12 @@ FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] +for api in API_ENDPOINTS: + flags.DEFINE_string("%s_api_listen" % api, "0.0.0.0", + "IP address to listen to for API %s" % api) + flags.DEFINE_integer("%s_api_listen_port" % api, + getattr(FLAGS, "%s_port" % api), + "Port to listen to for API %s" % api) def run_app(paste_config_file): LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) @@ -57,14 +63,10 @@ def run_app(paste_config_file): LOG.debug(_("No paste configuration for app: %s"), api) continue LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: getattr(FLAGS, "%s_host" % api), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - config.get('listen', '0.0.0.0'))) + apps.append((app, getattr(FLAGS, "%s_api_listen_port" % api), + getattr(FLAGS, "%s_api_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) @@ -82,6 +84,10 @@ if __name__ == '__main__': FLAGS(sys.argv) LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) + LOG.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + flag_get = FLAGS.get(flag, None) + LOG.debug("%(flag)s : %(flag_get)s" % locals()) conf = wsgi.paste_config_file('nova-api.conf') if conf: run_app(conf) diff --git a/etc/nova-api.conf b/etc/nova-api.conf index f0e749805..9f7e93d4c 100644 --- a/etc/nova-api.conf +++ b/etc/nova-api.conf @@ -1,6 +1,3 @@ -[DEFAULT] -verbose = 1 - ####### # EC2 # ####### -- cgit From a0145eed239a7afb545def17f25a08e8e4c68824 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:08:33 +0100 Subject: Set up logging once FLAGS properly read, no need to redo logging config anymore (was inoperant anyway) --- bin/nova-api | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index e7ee6f6fe..46f695248 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -39,10 +39,6 @@ from nova import log as logging from nova import version from nova import wsgi -logging.basicConfig() -LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) - FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] @@ -72,8 +68,6 @@ def run_app(paste_config_file): paste_config_file) return - # NOTE(todd): redo logging config, verbose could be set in paste config - logging.basicConfig() server = wsgi.Server() for app in apps: server.start(*app) @@ -82,6 +76,8 @@ def run_app(paste_config_file): if __name__ == '__main__': FLAGS(sys.argv) + logging.basicConfig() + LOG = logging.getLogger('nova.api') LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) LOG.debug(_("Full set of FLAGS:")) -- cgit From 27c2de313a41bced77f7a4769deae089a70f5385 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:21:14 +0100 Subject: Port changes to nova-combined, rename flags to API_listen and API_listen_port --- bin/nova-api | 8 ++++---- bin/nova-combined | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 46f695248..8d47a656e 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -44,9 +44,9 @@ FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] for api in API_ENDPOINTS: - flags.DEFINE_string("%s_api_listen" % api, "0.0.0.0", + flags.DEFINE_string("%s_listen" % api, "0.0.0.0", "IP address to listen to for API %s" % api) - flags.DEFINE_integer("%s_api_listen_port" % api, + flags.DEFINE_integer("%s_listen_port" % api, getattr(FLAGS, "%s_port" % api), "Port to listen to for API %s" % api) @@ -61,8 +61,8 @@ def run_app(paste_config_file): LOG.debug(_("App Config: %(api)s\n%(config)r") % locals()) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_api_listen_port" % api), - getattr(FLAGS, "%s_api_listen" % api))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), paste_config_file) diff --git a/bin/nova-combined b/bin/nova-combined index 889600eb7..40dc2945d 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -45,6 +45,14 @@ from nova import wsgi FLAGS = flags.FLAGS +API_ENDPOINTS = ['ec2', 'osapi'] + +for api in API_ENDPOINTS: + flags.DEFINE_string("%s_listen" % api, "0.0.0.0", + "IP address to listen to for API %s" % api) + flags.DEFINE_integer("%s_listen_port" % api, + getattr(FLAGS, "%s_port" % api), + "Port to listen to for API %s" % api) if __name__ == '__main__': utils.default_flagfile() @@ -57,21 +65,17 @@ if __name__ == '__main__': scheduler = service.Service.create(binary='nova-scheduler') #objectstore = service.Service.create(binary='nova-objectstore') - service.serve(compute, network, volume, scheduler) +# service.serve(compute, network, volume, scheduler) apps = [] paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in ['osapi', 'ec2']: + for api in API_ENDPOINTS: config = wsgi.load_paste_configuration(paste_config_file, api) if config is None: continue - wsgi.paste_config_to_flags(config, { - "verbose": FLAGS.verbose, - "%s_host" % api: getattr(FLAGS, "%s_host" % api), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), - config.get('listen', '0.0.0.0'))) + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), + getattr(FLAGS, "%s_listen" % api))) if len(apps) > 0: logging.basicConfig() server = wsgi.Server() -- cgit From a02af158f781b61dba67c454afb59b34f27ca5d7 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:23:15 +0100 Subject: Remove paste_config_to_flags since it's now unused --- nova/wsgi.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/nova/wsgi.py b/nova/wsgi.py index e01cc1e1e..d9b1cae86 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -515,10 +515,3 @@ def load_paste_app(filename, appname): except LookupError: pass return app - - -def paste_config_to_flags(config, mixins): - for k, v in mixins.iteritems(): - value = config.get(k, v) - converted_value = FLAGS[k].parser.Parse(value) - setattr(FLAGS, k, converted_value) -- cgit From 15d140992a980d959be6633b59d178a5ea273cdc Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:23:52 +0100 Subject: PEP8 fix --- bin/nova-api | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nova-api b/bin/nova-api index 8d47a656e..1228f723c 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -50,6 +50,7 @@ for api in API_ENDPOINTS: getattr(FLAGS, "%s_port" % api), "Port to listen to for API %s" % api) + def run_app(paste_config_file): LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) apps = [] -- cgit From bf570ca5f199091d505d96b91a3dc3acfbfc9fc7 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Fri, 18 Feb 2011 16:37:00 +0100 Subject: Fixed testing mode leftover --- bin/nova-combined | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-combined b/bin/nova-combined index 40dc2945d..dd21c8df4 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -65,7 +65,7 @@ if __name__ == '__main__': scheduler = service.Service.create(binary='nova-scheduler') #objectstore = service.Service.create(binary='nova-objectstore') -# service.serve(compute, network, volume, scheduler) + service.serve(compute, network, volume, scheduler) apps = [] paste_config_file = wsgi.paste_config_file('nova-api.conf') -- cgit From bef44d7621db516a0f5d407655f5e76adfd5c06d Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 10:14:56 -0800 Subject: Rename migration 004 => 005 --- .../versions/004_add_instance_metadata.py | 78 ---------------------- .../versions/005_add_instance_metadata.py | 78 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 78 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py deleted file mode 100644 index 4cb07e0d8..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - - -# Just for the ForeignKey and column creation to succeed, these are not the -# actual definitions of instances or services. -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -quotas = Table('quotas', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - - -# -# New Tables -# - -instance_metadata_table = Table('instance_metadata', meta, - Column('created_at', DateTime(timezone=False)), - Column('updated_at', DateTime(timezone=False)), - Column('deleted_at', DateTime(timezone=False)), - Column('deleted', Boolean(create_constraint=True, name=None)), - Column('id', Integer(), primary_key=True, nullable=False), - Column('instance_id', - Integer(), - ForeignKey('instances.id'), - nullable=False), - Column('key', - String(length=255, convert_unicode=False, assert_unicode=None, - unicode_error=None, _warn_on_bytestring=False)), - Column('value', - String(length=255, convert_unicode=False, assert_unicode=None, - unicode_error=None, _warn_on_bytestring=False))) - - -# -# New columns -# -quota_metadata_items = Column('metadata_items', Integer()) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - for table in (instance_metadata_table, ): - try: - table.create() - except Exception: - logging.info(repr(table)) - logging.exception('Exception while creating table') - raise - - quotas.create_column(quota_metadata_items) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py new file mode 100644 index 000000000..4cb07e0d8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_instance_metadata.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +quotas = Table('quotas', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# + +instance_metadata_table = Table('instance_metadata', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('value', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +# +# New columns +# +quota_metadata_items = Column('metadata_items', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (instance_metadata_table, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + quotas.create_column(quota_metadata_items) -- cgit From f66840390bf6f4660a091663d306417e760735bc Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 10:57:19 -0800 Subject: Support for HP SAN --- .../versions/006_add_provider_data_to_volumes.py | 72 ++++++ nova/db/sqlalchemy/models.py | 3 + nova/volume/driver.py | 144 +++++++++-- nova/volume/manager.py | 8 +- nova/volume/san.py | 287 ++++++++++++++++++--- 5 files changed, 459 insertions(+), 55 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py b/nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py new file mode 100644 index 000000000..705fc8ff3 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/006_add_provider_data_to_volumes.py @@ -0,0 +1,72 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Table stub-definitions +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +# +volumes = Table('volumes', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# +# None + +# +# Tables to alter +# +# None + +# +# Columns to add to existing tables +# + +volumes_provider_location = Column('provider_location', + String(length=256, + convert_unicode=False, + assert_unicode=None, + unicode_error=None, + _warn_on_bytestring=False)) + +volumes_provider_auth = Column('provider_auth', + String(length=256, + convert_unicode=False, + assert_unicode=None, + unicode_error=None, + _warn_on_bytestring=False)) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + # Add columns to existing tables + volumes.create_column(volumes_provider_location) + volumes.create_column(volumes_provider_auth) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 40a96fc17..4485ee9e4 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -243,6 +243,9 @@ class Volume(BASE, NovaBase): display_name = Column(String(255)) display_description = Column(String(255)) + provider_location = Column(String(255)) + provider_auth = Column(String(255)) + class Quota(BASE, NovaBase): """Represents quota overrides for a project.""" diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 82f4c2f54..f172e2fdc 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -21,6 +21,7 @@ Drivers for volumes. """ import time +import os from nova import exception from nova import flags @@ -36,6 +37,8 @@ flags.DEFINE_string('aoe_eth_dev', 'eth0', 'Which device to export the volumes on') flags.DEFINE_string('num_shell_tries', 3, 'number of times to attempt to run flakey shell commands') +flags.DEFINE_string('num_iscsi_scan_tries', 3, + 'number of times to rescan iSCSI target to find volume') flags.DEFINE_integer('num_shelves', 100, 'Number of vblade shelves') @@ -294,40 +297,133 @@ class ISCSIDriver(VolumeDriver): self._execute("sudo ietadm --op delete --tid=%s" % iscsi_target) - def _get_name_and_portal(self, volume): - """Gets iscsi name and portal from volume name and host.""" + def _do_iscsi_discovery(self, volume): + #TODO(justinsb): Deprecate discovery and use stored info + #NOTE(justinsb): Discovery won't work with CHAP-secured targets (?) + LOG.warn(_("ISCSI provider_location not stored, using discovery")) + volume_name = volume['name'] - host = volume['host'] + (out, _err) = self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % host) + "sendtargets -p %s" % (volume['host'])) for target in out.splitlines(): if FLAGS.iscsi_ip_prefix in target and volume_name in target: - (location, _sep, iscsi_name) = target.partition(" ") - break - iscsi_portal = location.split(",")[0] - return (iscsi_name, iscsi_portal) + return target + return None + + def _get_iscsi_properties(self, volume): + """Gets iscsi configuration, ideally from saved information in the + volume entity, but falling back to discovery if need be.""" + + properties = {} + + location = volume['provider_location'] + + if location: + # provider_location is the same format as iSCSI discovery output + properties['target_discovered'] = False + else: + location = self._do_iscsi_discovery(volume) + + if not location: + raise exception.Error(_("Could not find iSCSI export " + " for volume %s") % + (volume['name'])) + + LOG.debug(_("ISCSI Discovery: Found %s") % (location)) + properties['target_discovered'] = True + + (iscsi_target, _sep, iscsi_name) = location.partition(" ") + + iscsi_portal = iscsi_target.split(",")[0] + + properties['target_iqn'] = iscsi_name + properties['target_portal'] = iscsi_portal + + auth = volume['provider_auth'] + + if auth: + (auth_method, auth_username, auth_secret) = auth.split() + + properties['auth_method'] = auth_method + properties['auth_username'] = auth_username + properties['auth_password'] = auth_secret + + return properties + + def _run_iscsiadm(self, iscsi_properties, iscsi_command): + command = ("sudo iscsiadm -m node -T %s -p %s %s" % + (iscsi_properties['target_iqn'], + iscsi_properties['target_portal'], + iscsi_command)) + (out, err) = self._execute(command) + LOG.debug("iscsiadm %s: stdout=%s stderr=%s" % + (iscsi_command, out, err)) + return (out, err) + + def _iscsiadm_update(self, iscsi_properties, property_key, property_value): + iscsi_command = ("--op update -n %s -v %s" % + (property_key, property_value)) + return self._run_iscsiadm(iscsi_properties, iscsi_command) def discover_volume(self, volume): """Discover volume on a remote host.""" - iscsi_name, iscsi_portal = self._get_name_and_portal(volume) - self._execute("sudo iscsiadm -m node -T %s -p %s --login" % - (iscsi_name, iscsi_portal)) - self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v automatic" % - (iscsi_name, iscsi_portal)) - return "/dev/disk/by-path/ip-%s-iscsi-%s-lun-0" % (iscsi_portal, - iscsi_name) + iscsi_properties = self._get_iscsi_properties(volume) + + if not iscsi_properties['target_discovered']: + self._run_iscsiadm(iscsi_properties, "--op new") + + if iscsi_properties.get('auth_method'): + self._iscsiadm_update(iscsi_properties, + "node.session.auth.authmethod", + iscsi_properties['auth_method']) + self._iscsiadm_update(iscsi_properties, + "node.session.auth.username", + iscsi_properties['auth_username']) + self._iscsiadm_update(iscsi_properties, + "node.session.auth.password", + iscsi_properties['auth_password']) + + self._run_iscsiadm(iscsi_properties, "--login") + + self._iscsiadm_update(iscsi_properties, "node.startup", "automatic") + + mount_device = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-0" % + (iscsi_properties['target_portal'], + iscsi_properties['target_iqn'])) + + # The /dev/disk/by-path/... node is not always present immediately + # TODO(justinsb): This retry-with-delay is a pattern, move to utils? + tries = 0 + while not os.path.exists(mount_device): + if tries >= FLAGS.num_iscsi_scan_tries: + raise exception.Error(_("iSCSI device not found at %s") % + (mount_device)) + + LOG.warn(_("ISCSI volume not yet found at: %(mount_device)s. " + "Will rescan & retry. Try number: %(tries)s") % + locals()) + + # The rescan isn't documented as being necessary(?), but it helps + self._run_iscsiadm(iscsi_properties, "--rescan") + + tries = tries + 1 + if not os.path.exists(mount_device): + time.sleep(tries ** 2) + + if tries != 0: + LOG.debug(_("Found iSCSI node %(mount_device)s " + "(after %(tries)s rescans)") % + locals()) + + return mount_device def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" - iscsi_name, iscsi_portal = self._get_name_and_portal(volume) - self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v manual" % - (iscsi_name, iscsi_portal)) - self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % - (iscsi_name, iscsi_portal)) - self._execute("sudo iscsiadm -m node --op delete " - "--targetname %s" % iscsi_name) + iscsi_properties = self._get_iscsi_properties(volume) + self._iscsiadm_update(iscsi_properties, "node.startup", "manual") + self._run_iscsiadm(iscsi_properties, "--logout") + self._run_iscsiadm(iscsi_properties, "--op delete") class FakeISCSIDriver(ISCSIDriver): diff --git a/nova/volume/manager.py b/nova/volume/manager.py index d2f02e4e0..7193ece14 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -107,10 +107,14 @@ class VolumeManager(manager.Manager): vol_size = volume_ref['size'] LOG.debug(_("volume %(vol_name)s: creating lv of" " size %(vol_size)sG") % locals()) - self.driver.create_volume(volume_ref) + db_update = self.driver.create_volume(volume_ref) + if db_update: + self.db.volume_update(context, volume_ref['id'], db_update) LOG.debug(_("volume %s: creating export"), volume_ref['name']) - self.driver.create_export(context, volume_ref) + db_update = self.driver.create_export(context, volume_ref) + if db_update: + self.db.volume_update(context, volume_ref['id'], db_update) except Exception: self.db.volume_update(context, volume_ref['id'], {'status': 'error'}) diff --git a/nova/volume/san.py b/nova/volume/san.py index 26d6125e7..911ad096f 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -23,6 +23,8 @@ The unique thing about a SAN is that we don't expect that we can run the volume import os import paramiko +from xml.etree import ElementTree + from nova import exception from nova import flags from nova import log as logging @@ -41,37 +43,15 @@ flags.DEFINE_string('san_password', '', 'Password for SAN controller') flags.DEFINE_string('san_privatekey', '', 'Filename of private key to use for SSH authentication') +flags.DEFINE_string('san_clustername', '', + 'Cluster name to use for creating volumes') +flags.DEFINE_integer('san_ssh_port', 22, + 'SSH port to use with SAN') class SanISCSIDriver(ISCSIDriver): """ Base class for SAN-style storage volumes (storage providers we access over SSH)""" - #Override because SAN ip != host ip - def _get_name_and_portal(self, volume): - """Gets iscsi name and portal from volume name and host.""" - volume_name = volume['name'] - - # TODO(justinsb): store in volume, remerge with generic iSCSI code - host = FLAGS.san_ip - - (out, _err) = self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % host) - - location = None - find_iscsi_name = self._build_iscsi_target_name(volume) - for target in out.splitlines(): - if find_iscsi_name in target: - (location, _sep, iscsi_name) = target.partition(" ") - break - if not location: - raise exception.Error(_("Could not find iSCSI export " - " for volume %s") % - volume_name) - - iscsi_portal = location.split(",")[0] - LOG.debug("iscsi_name=%s, iscsi_portal=%s" % - (iscsi_name, iscsi_portal)) - return (iscsi_name, iscsi_portal) def _build_iscsi_target_name(self, volume): return "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) @@ -85,6 +65,7 @@ class SanISCSIDriver(ISCSIDriver): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if FLAGS.san_password: ssh.connect(FLAGS.san_ip, + port=FLAGS.san_ssh_port, username=FLAGS.san_login, password=FLAGS.san_password) elif FLAGS.san_privatekey: @@ -92,10 +73,11 @@ class SanISCSIDriver(ISCSIDriver): # It sucks that paramiko doesn't support DSA keys privatekey = paramiko.RSAKey.from_private_key_file(privatekeyfile) ssh.connect(FLAGS.san_ip, + port=FLAGS.san_ssh_port, username=FLAGS.san_login, pkey=privatekey) else: - raise exception.Error("Specify san_password or san_privatekey") + raise exception.Error(_("Specify san_password or san_privatekey")) return ssh def _run_ssh(self, command, check_exit_code=True): @@ -124,10 +106,10 @@ class SanISCSIDriver(ISCSIDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" if not (FLAGS.san_password or FLAGS.san_privatekey): - raise exception.Error("Specify san_password or san_privatekey") + raise exception.Error(_("Specify san_password or san_privatekey")) if not (FLAGS.san_ip): - raise exception.Error("san_ip must be set") + raise exception.Error(_("san_ip must be set")) def _collect_lines(data): @@ -306,6 +288,17 @@ class SolarisISCSIDriver(SanISCSIDriver): self._run_ssh("pfexec /usr/sbin/stmfadm add-view -t %s %s" % (target_group_name, luid)) + #TODO(justinsb): Is this always 1? Does it matter? + iscsi_portal_interface = '1' + iscsi_portal = FLAGS.san_ip + ":3260," + iscsi_portal_interface + + db_update = {} + db_update['provider_location'] = ("%s %s" % + (iscsi_portal, + iscsi_name)) + + return db_update + def remove_export(self, context, volume): """Removes an export for a logical volume.""" @@ -333,3 +326,239 @@ class SolarisISCSIDriver(SanISCSIDriver): if self._is_lu_created(volume): self._run_ssh("pfexec /usr/sbin/sbdadm delete-lu %s" % (luid)) + + +class HpSanISCSIDriver(SanISCSIDriver): + """Executes commands relating to HP/Lefthand SAN ISCSI volumes. + We use the CLIQ interface, over SSH. + + Rough overview of CLIQ commands used: + CLIQ createVolume (creates the volume) + CLIQ getVolumeInfo (to discover the IQN etc) + CLIQ getClusterInfo (to discover the iSCSI target IP address) + CLIQ assignVolumeChap (exports it with CHAP security) + + The 'trick' here is that the HP SAN enforces security by default, so + normally a volume mount would need both to configure the SAN in the volume + layer and do the mount on the compute layer. Multi-layer operations are + not catered for at the moment in the nova architecture, so instead we + share the volume using CHAP at volume creation time. Then the mount need + only use those CHAP credentials, so can take place exclusively in the + compute layer""" + + def _cliq_run(self, verb, cliq_args): + """Runs a CLIQ command over SSH, without doing any result parsing""" + cliq_arg_strings = [] + for k, v in cliq_args.items(): + cliq_arg_strings.append(" %s=%s" % (k, v)) + cmd = verb + ''.join(cliq_arg_strings) + + return self._run_ssh(cmd) + + def _cliq_run_xml(self, verb, cliq_args, check_cliq_result=True): + """Runs a CLIQ command over SSH, parsing and checking the output""" + cliq_args['output'] = 'XML' + (out, _err) = self._cliq_run(verb, cliq_args) + + LOG.debug(_("CLIQ command returned %s"), out) + + result_xml = ElementTree.fromstring(out) + if check_cliq_result: + response_node = result_xml.find("response") + if response_node is None: + msg = (_("Malformed response to CLIQ command " + "%(verb)s %(cliq_args)s. Result=%(out)s") % + locals()) + raise exception.Error(msg) + + result_code = response_node.attrib.get("result") + + if result_code != "0": + msg = (_("Error running CLIQ command %(verb)s %(cliq_args)s. " + " Result=%(out)s") % + locals()) + raise exception.Error(msg) + + return result_xml + + def _cliq_get_cluster_info(self, cluster_name): + """Queries for info about the cluster (including IP)""" + cliq_args = {} + cliq_args['clusterName'] = cluster_name + cliq_args['searchDepth'] = '1' + cliq_args['verbose'] = '0' + + result_xml = self._cliq_run_xml("getClusterInfo", cliq_args) + + return result_xml + + def _cliq_get_cluster_vip(self, cluster_name): + """Gets the IP on which a cluster shares iSCSI volumes""" + cluster_xml = self._cliq_get_cluster_info(cluster_name) + + vips = [] + for vip in cluster_xml.findall("response/cluster/vip"): + vips.append(vip.attrib.get('ipAddress')) + + if len(vips) == 1: + return vips[0] + + _xml = ElementTree.tostring(cluster_xml) + msg = (_("Unexpected number of virtual ips for cluster " + " %(cluster_name)s. Result=%(_xml)s") % + locals()) + raise exception.Error(msg) + + def _cliq_get_volume_info(self, volume_name): + """Gets the volume info, including IQN""" + cliq_args = {} + cliq_args['volumeName'] = volume_name + result_xml = self._cliq_run_xml("getVolumeInfo", cliq_args) + + # Result looks like this: + # + # + # + # + # + # + # + # + + # Flatten the nodes into a dictionary; use prefixes to avoid collisions + volume_attributes = {} + + volume_node = result_xml.find("response/volume") + for k, v in volume_node.attrib.items(): + volume_attributes["volume." + k] = v + + status_node = volume_node.find("status") + if not status_node is None: + for k, v in status_node.attrib.items(): + volume_attributes["status." + k] = v + + # We only consider the first permission node + permission_node = volume_node.find("permission") + if not permission_node is None: + for k, v in status_node.attrib.items(): + volume_attributes["permission." + k] = v + + LOG.debug(_("Volume info: %(volume_name)s => %(volume_attributes)s") % + locals()) + return volume_attributes + + def create_volume(self, volume): + """Creates a volume.""" + cliq_args = {} + cliq_args['clusterName'] = FLAGS.san_clustername + #TODO(justinsb): Should we default to inheriting thinProvision? + cliq_args['thinProvision'] = '1' if FLAGS.san_thin_provision else '0' + cliq_args['volumeName'] = volume['name'] + if int(volume['size']) == 0: + cliq_args['size'] = '100MB' + else: + cliq_args['size'] = '%sGB' % volume['size'] + + self._cliq_run_xml("createVolume", cliq_args) + + volume_info = self._cliq_get_volume_info(volume['name']) + cluster_name = volume_info['volume.clusterName'] + iscsi_iqn = volume_info['volume.iscsiIqn'] + + #TODO(justinsb): Is this always 1? Does it matter? + cluster_interface = '1' + + cluster_vip = self._cliq_get_cluster_vip(cluster_name) + iscsi_portal = cluster_vip + ":3260," + cluster_interface + + db_update = {} + db_update['provider_location'] = ("%s %s" % + (iscsi_portal, + iscsi_iqn)) + + return db_update + + def delete_volume(self, volume): + """Deletes a volume.""" + cliq_args = {} + cliq_args['volumeName'] = volume['name'] + cliq_args['prompt'] = 'false' # Don't confirm + + self._cliq_run_xml("deleteVolume", cliq_args) + + def local_path(self, volume): + # TODO(justinsb): Is this needed here? + raise exception.Error(_("local_path not supported")) + + def ensure_export(self, context, volume): + """Synchronously recreates an export for a logical volume.""" + return self._do_export(context, volume, force_create=False) + + def create_export(self, context, volume): + return self._do_export(context, volume, force_create=True) + + def _do_export(self, context, volume, force_create): + """Supports ensure_export and create_export""" + volume_info = self._cliq_get_volume_info(volume['name']) + + is_shared = 'permission.authGroup' in volume_info + + db_update = {} + + should_export = False + + if force_create or not is_shared: + should_export = True + # Check that we have a project_id + project_id = volume['project_id'] + if not project_id: + project_id = context.project_id + + if project_id: + #TODO(justinsb): Use a real per-project password here + chap_username = 'proj_' + project_id + # HP/Lefthand requires that the password be >= 12 characters + chap_password = 'project_secret_' + project_id + else: + msg = (_("Could not determine project for volume %s, " + "can't export") % + (volume['name'])) + if force_create: + raise exception.Error(msg) + else: + LOG.warn(msg) + should_export = False + + if should_export: + cliq_args = {} + cliq_args['volumeName'] = volume['name'] + cliq_args['chapName'] = chap_username + cliq_args['targetSecret'] = chap_password + + self._cliq_run_xml("assignVolumeChap", cliq_args) + + db_update['provider_auth'] = ("CHAP %s %s" % + (chap_username, chap_password)) + + return db_update + + def remove_export(self, context, volume): + """Removes an export for a logical volume.""" + cliq_args = {} + cliq_args['volumeName'] = volume['name'] + + self._cliq_run_xml("unassignVolume", cliq_args) -- cgit From 8684eb3aa638883ea82bbaf8eb59076f1d7e6a05 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 17:17:51 -0800 Subject: ObjectStore doesn't use properties collection; kernel_id and ramdisk_id aren't required anyway --- nova/api/openstack/servers.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 486eca508..11a84687d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,13 +144,11 @@ class Controller(wsgi.Controller): metadata stored in Glance as 'image_properties' """ def lookup(param): - _image_id = image_id - try: - return image['properties'][param] - except KeyError: - raise exception.NotFound( - _("%(param)s property not found for image %(_image_id)s") % - locals()) + properties = image.get('properties') + if properties: + return properties.get(param) + else: + return image.get(param) image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From aeab8eeb038ca1d1dde05705028144a78552c4f7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 18 Feb 2011 17:27:25 -0800 Subject: Don't crash if there's no 'fixed_ip' attribute (was returning None, which was unsubscriptable) --- nova/api/openstack/servers.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 486eca508..b54e28c0c 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,20 +63,22 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - # grab single private fixed ip - try: - private_ip = inst['fixed_ip']['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) - - # grab all public floating ips - try: - for floating in inst['fixed_ip']['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + fixed_ip = inst['fixed_ip'] + if fixed_ip: + # grab single private fixed ip + try: + private_ip = fixed_ip['address'] + if private_ip: + inst_dict['addresses']['private'].append(private_ip) + except KeyError: + LOG.debug(_("Failed to read private ip")) + + # grab all public floating ips + try: + for floating in fixed_ip['floating_ips']: + inst_dict['addresses']['public'].append(floating['address']) + except KeyError: + LOG.debug(_("Failed to read public ip(s)")) inst_dict['metadata'] = {} inst_dict['hostId'] = '' -- cgit From a7eed42c57fe7eaf6f2981a88a74a81a6890198c Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 20 Feb 2011 20:56:14 +0100 Subject: puppet scripts only there as an example, should be moved to some other place if they are still necessary --- contrib/puppet/files/etc/default/nova-compute | 1 - contrib/puppet/files/etc/default/nova-volume | 1 - contrib/puppet/files/etc/issue | 5 - contrib/puppet/files/etc/libvirt/qemu.conf | 170 -------- contrib/puppet/files/etc/lvm/lvm.conf | 463 -------------------- contrib/puppet/files/etc/nova.conf | 28 -- contrib/puppet/files/production/boto.cfg | 3 - contrib/puppet/files/production/genvpn.sh | 35 -- .../files/production/libvirt.qemu.xml.template | 35 -- contrib/puppet/files/production/my.cnf | 137 ------ contrib/puppet/files/production/nova-iptables | 187 --------- contrib/puppet/files/production/nova-iscsi-dev.sh | 19 - contrib/puppet/files/production/setup_data.sh | 6 - contrib/puppet/files/production/slap.sh | 261 ------------ contrib/puppet/fileserver.conf | 8 - contrib/puppet/manifests/classes/apt.pp | 1 - contrib/puppet/manifests/classes/issue.pp | 14 - contrib/puppet/manifests/classes/kern_module.pp | 34 -- contrib/puppet/manifests/classes/loopback.pp | 6 - contrib/puppet/manifests/classes/lvm.pp | 8 - contrib/puppet/manifests/classes/lvmconf.pp | 8 - contrib/puppet/manifests/classes/nova.pp | 464 --------------------- contrib/puppet/manifests/classes/swift.pp | 7 - contrib/puppet/manifests/site.pp | 120 ------ contrib/puppet/manifests/templates.pp | 21 - contrib/puppet/puppet.conf | 11 - contrib/puppet/templates/haproxy.cfg.erb | 39 -- contrib/puppet/templates/monitrc-nova-api.erb | 138 ------ contrib/puppet/templates/nova-iptables.erb | 10 - .../templates/production/nova-common.conf.erb | 55 --- .../puppet/templates/production/nova-nova.conf.erb | 21 - 31 files changed, 2316 deletions(-) delete mode 100644 contrib/puppet/files/etc/default/nova-compute delete mode 100644 contrib/puppet/files/etc/default/nova-volume delete mode 100644 contrib/puppet/files/etc/issue delete mode 100644 contrib/puppet/files/etc/libvirt/qemu.conf delete mode 100644 contrib/puppet/files/etc/lvm/lvm.conf delete mode 100644 contrib/puppet/files/etc/nova.conf delete mode 100644 contrib/puppet/files/production/boto.cfg delete mode 100644 contrib/puppet/files/production/genvpn.sh delete mode 100644 contrib/puppet/files/production/libvirt.qemu.xml.template delete mode 100644 contrib/puppet/files/production/my.cnf delete mode 100755 contrib/puppet/files/production/nova-iptables delete mode 100644 contrib/puppet/files/production/nova-iscsi-dev.sh delete mode 100755 contrib/puppet/files/production/setup_data.sh delete mode 100755 contrib/puppet/files/production/slap.sh delete mode 100644 contrib/puppet/fileserver.conf delete mode 100644 contrib/puppet/manifests/classes/apt.pp delete mode 100644 contrib/puppet/manifests/classes/issue.pp delete mode 100644 contrib/puppet/manifests/classes/kern_module.pp delete mode 100644 contrib/puppet/manifests/classes/loopback.pp delete mode 100644 contrib/puppet/manifests/classes/lvm.pp delete mode 100644 contrib/puppet/manifests/classes/lvmconf.pp delete mode 100644 contrib/puppet/manifests/classes/nova.pp delete mode 100644 contrib/puppet/manifests/classes/swift.pp delete mode 100644 contrib/puppet/manifests/site.pp delete mode 100644 contrib/puppet/manifests/templates.pp delete mode 100644 contrib/puppet/puppet.conf delete mode 100644 contrib/puppet/templates/haproxy.cfg.erb delete mode 100644 contrib/puppet/templates/monitrc-nova-api.erb delete mode 100644 contrib/puppet/templates/nova-iptables.erb delete mode 100644 contrib/puppet/templates/production/nova-common.conf.erb delete mode 100644 contrib/puppet/templates/production/nova-nova.conf.erb diff --git a/contrib/puppet/files/etc/default/nova-compute b/contrib/puppet/files/etc/default/nova-compute deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-compute +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/default/nova-volume b/contrib/puppet/files/etc/default/nova-volume deleted file mode 100644 index 8bd7d091c..000000000 --- a/contrib/puppet/files/etc/default/nova-volume +++ /dev/null @@ -1 +0,0 @@ -ENABLED=true diff --git a/contrib/puppet/files/etc/issue b/contrib/puppet/files/etc/issue deleted file mode 100644 index 8c567221b..000000000 --- a/contrib/puppet/files/etc/issue +++ /dev/null @@ -1,5 +0,0 @@ ------------------------------------------------ - - Welcome to your OpenStack installation! - ------------------------------------------------ diff --git a/contrib/puppet/files/etc/libvirt/qemu.conf b/contrib/puppet/files/etc/libvirt/qemu.conf deleted file mode 100644 index 7839f12e5..000000000 --- a/contrib/puppet/files/etc/libvirt/qemu.conf +++ /dev/null @@ -1,170 +0,0 @@ -# Master configuration file for the QEMU driver. -# All settings described here are optional - if omitted, sensible -# defaults are used. - -# VNC is configured to listen on 127.0.0.1 by default. -# To make it listen on all public interfaces, uncomment -# this next option. -# -# NB, strong recommendation to enable TLS + x509 certificate -# verification when allowing public access -# -# vnc_listen = "0.0.0.0" - - -# Enable use of TLS encryption on the VNC server. This requires -# a VNC client which supports the VeNCrypt protocol extension. -# Examples include vinagre, virt-viewer, virt-manager and vencrypt -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to setup CA and issue a server certificate -# before enabling this. -# -# vnc_tls = 1 - - -# Use of TLS requires that x509 certificates be issued. The -# default it to keep them in /etc/pki/libvirt-vnc. This directory -# must contain -# -# ca-cert.pem - the CA master certificate -# server-cert.pem - the server certificate signed with ca-cert.pem -# server-key.pem - the server private key -# -# This option allows the certificate directory to be changed -# -# vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc" - - -# The default TLS configuration only uses certificates for the server -# allowing the client to verify the server's identity and establish -# and encrypted channel. -# -# It is possible to use x509 certificates for authentication too, by -# issuing a x509 certificate to every client who needs to connect. -# -# Enabling this option will reject any client who does not have a -# certificate signed by the CA in /etc/pki/libvirt-vnc/ca-cert.pem -# -# vnc_tls_x509_verify = 1 - - -# The default VNC password. Only 8 letters are significant for -# VNC passwords. This parameter is only used if the per-domain -# XML config does not already provide a password. To allow -# access without passwords, leave this commented out. An empty -# string will still enable passwords, but be rejected by QEMU -# effectively preventing any use of VNC. Obviously change this -# example here before you set this -# -# vnc_password = "XYZ12345" - - -# Enable use of SASL encryption on the VNC server. This requires -# a VNC client which supports the SASL protocol extension. -# Examples include vinagre, virt-viewer and virt-manager -# itself. UltraVNC, RealVNC, TightVNC do not support this -# -# It is necessary to configure /etc/sasl2/qemu.conf to choose -# the desired SASL plugin (eg, GSSPI for Kerberos) -# -# vnc_sasl = 1 - - -# The default SASL configuration file is located in /etc/sasl2/ -# When running libvirtd unprivileged, it may be desirable to -# override the configs in this location. Set this parameter to -# point to the directory, and create a qemu.conf in that location -# -# vnc_sasl_dir = "/some/directory/sasl2" - - - - -# The default security driver is SELinux. If SELinux is disabled -# on the host, then the security driver will automatically disable -# itself. If you wish to disable QEMU SELinux security driver while -# leaving SELinux enabled for the host in general, then set this -# to 'none' instead -# -# security_driver = "selinux" - - -# The user ID for QEMU processes run by the system instance -user = "root" - -# The group ID for QEMU processes run by the system instance -group = "root" - -# Whether libvirt should dynamically change file ownership -# to match the configured user/group above. Defaults to 1. -# Set to 0 to disable file ownership changes. -#dynamic_ownership = 1 - - -# What cgroup controllers to make use of with QEMU guests -# -# - 'cpu' - use for schedular tunables -# - 'devices' - use for device whitelisting -# -# NB, even if configured here, they won't be used unless -# the adminsitrator has mounted cgroups. eg -# -# mkdir /dev/cgroup -# mount -t cgroup -o devices,cpu none /dev/cgroup -# -# They can be mounted anywhere, and different controlers -# can be mounted in different locations. libvirt will detect -# where they are located. -# -# cgroup_controllers = [ "cpu", "devices" ] - -# This is the basic set of devices allowed / required by -# all virtual machines. -# -# As well as this, any configured block backed disks, -# all sound device, and all PTY devices are allowed. -# -# This will only need setting if newer QEMU suddenly -# wants some device we don't already know a bout. -# -#cgroup_device_acl = [ -# "/dev/null", "/dev/full", "/dev/zero", -# "/dev/random", "/dev/urandom", -# "/dev/ptmx", "/dev/kvm", "/dev/kqemu", -# "/dev/rtc", "/dev/hpet", "/dev/net/tun", -#] - -# The default format for Qemu/KVM guest save images is raw; that is, the -# memory from the domain is dumped out directly to a file. If you have -# guests with a large amount of memory, however, this can take up quite -# a bit of space. If you would like to compress the images while they -# are being saved to disk, you can also set "lzop", "gzip", "bzip2", or "xz" -# for save_image_format. Note that this means you slow down the process of -# saving a domain in order to save disk space; the list above is in descending -# order by performance and ascending order by compression ratio. -# -# save_image_format = "raw" - -# If provided by the host and a hugetlbfs mount point is configured, -# a guest may request huge page backing. When this mount point is -# unspecified here, determination of a host mount point in /proc/mounts -# will be attempted. Specifying an explicit mount overrides detection -# of the same in /proc/mounts. Setting the mount point to "" will -# disable guest hugepage backing. -# -# NB, within this mount point, guests will create memory backing files -# in a location of $MOUNTPOINT/libvirt/qemu - -# hugetlbfs_mount = "/dev/hugepages" - -# mac_filter enables MAC addressed based filtering on bridge ports. -# This currently requires ebtables to be installed. -# -# mac_filter = 1 - -# By default, PCI devices below non-ACS switch are not allowed to be assigned -# to guests. By setting relaxed_acs_check to 1 such devices will be allowed to -# be assigned to guests. -# -# relaxed_acs_check = 1 diff --git a/contrib/puppet/files/etc/lvm/lvm.conf b/contrib/puppet/files/etc/lvm/lvm.conf deleted file mode 100644 index 4e814ad49..000000000 --- a/contrib/puppet/files/etc/lvm/lvm.conf +++ /dev/null @@ -1,463 +0,0 @@ -# This is an example configuration file for the LVM2 system. -# It contains the default settings that would be used if there was no -# /etc/lvm/lvm.conf file. -# -# Refer to 'man lvm.conf' for further information including the file layout. -# -# To put this file in a different directory and override /etc/lvm set -# the environment variable LVM_SYSTEM_DIR before running the tools. - - -# This section allows you to configure which block devices should -# be used by the LVM system. -devices { - - # Where do you want your volume groups to appear ? - dir = "/dev" - - # An array of directories that contain the device nodes you wish - # to use with LVM2. - scan = [ "/dev" ] - - # If several entries in the scanned directories correspond to the - # same block device and the tools need to display a name for device, - # all the pathnames are matched against each item in the following - # list of regular expressions in turn and the first match is used. - preferred_names = [ ] - - # Try to avoid using undescriptive /dev/dm-N names, if present. - # preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ] - - # A filter that tells LVM2 to only use a restricted set of devices. - # The filter consists of an array of regular expressions. These - # expressions can be delimited by a character of your choice, and - # prefixed with either an 'a' (for accept) or 'r' (for reject). - # The first expression found to match a device name determines if - # the device will be accepted or rejected (ignored). Devices that - # don't match any patterns are accepted. - - # Be careful if there there are symbolic links or multiple filesystem - # entries for the same device as each name is checked separately against - # the list of patterns. The effect is that if any name matches any 'a' - # pattern, the device is accepted; otherwise if any name matches any 'r' - # pattern it is rejected; otherwise it is accepted. - - # Don't have more than one filter line active at once: only one gets used. - - # Run vgscan after you change this parameter to ensure that - # the cache file gets regenerated (see below). - # If it doesn't do what you expect, check the output of 'vgscan -vvvv'. - - - # By default we accept every block device: - filter = [ "r|/dev/etherd/.*|", "r|/dev/block/.*|", "a/.*/" ] - - # Exclude the cdrom drive - # filter = [ "r|/dev/cdrom|" ] - - # When testing I like to work with just loopback devices: - # filter = [ "a/loop/", "r/.*/" ] - - # Or maybe all loops and ide drives except hdc: - # filter =[ "a|loop|", "r|/dev/hdc|", "a|/dev/ide|", "r|.*|" ] - - # Use anchors if you want to be really specific - # filter = [ "a|^/dev/hda8$|", "r/.*/" ] - - # The results of the filtering are cached on disk to avoid - # rescanning dud devices (which can take a very long time). - # By default this cache is stored in the /etc/lvm/cache directory - # in a file called '.cache'. - # It is safe to delete the contents: the tools regenerate it. - # (The old setting 'cache' is still respected if neither of - # these new ones is present.) - cache_dir = "/etc/lvm/cache" - cache_file_prefix = "" - - # You can turn off writing this cache file by setting this to 0. - write_cache_state = 1 - - # Advanced settings. - - # List of pairs of additional acceptable block device types found - # in /proc/devices with maximum (non-zero) number of partitions. - # types = [ "fd", 16 ] - - # If sysfs is mounted (2.6 kernels) restrict device scanning to - # the block devices it believes are valid. - # 1 enables; 0 disables. - sysfs_scan = 1 - - # By default, LVM2 will ignore devices used as components of - # software RAID (md) devices by looking for md superblocks. - # 1 enables; 0 disables. - md_component_detection = 1 - - # By default, if a PV is placed directly upon an md device, LVM2 - # will align its data blocks with the md device's stripe-width. - # 1 enables; 0 disables. - md_chunk_alignment = 1 - - # By default, the start of a PV's data area will be a multiple of - # the 'minimum_io_size' or 'optimal_io_size' exposed in sysfs. - # - minimum_io_size - the smallest request the device can perform - # w/o incurring a read-modify-write penalty (e.g. MD's chunk size) - # - optimal_io_size - the device's preferred unit of receiving I/O - # (e.g. MD's stripe width) - # minimum_io_size is used if optimal_io_size is undefined (0). - # If md_chunk_alignment is enabled, that detects the optimal_io_size. - # This setting takes precedence over md_chunk_alignment. - # 1 enables; 0 disables. - data_alignment_detection = 1 - - # Alignment (in KB) of start of data area when creating a new PV. - # If a PV is placed directly upon an md device and md_chunk_alignment or - # data_alignment_detection is enabled this parameter is ignored. - # Set to 0 for the default alignment of 64KB or page size, if larger. - data_alignment = 0 - - # By default, the start of the PV's aligned data area will be shifted by - # the 'alignment_offset' exposed in sysfs. This offset is often 0 but - # may be non-zero; e.g.: certain 4KB sector drives that compensate for - # windows partitioning will have an alignment_offset of 3584 bytes - # (sector 7 is the lowest aligned logical block, the 4KB sectors start - # at LBA -1, and consequently sector 63 is aligned on a 4KB boundary). - # 1 enables; 0 disables. - data_alignment_offset_detection = 1 - - # If, while scanning the system for PVs, LVM2 encounters a device-mapper - # device that has its I/O suspended, it waits for it to become accessible. - # Set this to 1 to skip such devices. This should only be needed - # in recovery situations. - ignore_suspended_devices = 0 -} - -# This section that allows you to configure the nature of the -# information that LVM2 reports. -log { - - # Controls the messages sent to stdout or stderr. - # There are three levels of verbosity, 3 being the most verbose. - verbose = 0 - - # Should we send log messages through syslog? - # 1 is yes; 0 is no. - syslog = 1 - - # Should we log error and debug messages to a file? - # By default there is no log file. - #file = "/var/log/lvm2.log" - - # Should we overwrite the log file each time the program is run? - # By default we append. - overwrite = 0 - - # What level of log messages should we send to the log file and/or syslog? - # There are 6 syslog-like log levels currently in use - 2 to 7 inclusive. - # 7 is the most verbose (LOG_DEBUG). - level = 0 - - # Format of output messages - # Whether or not (1 or 0) to indent messages according to their severity - indent = 1 - - # Whether or not (1 or 0) to display the command name on each line output - command_names = 0 - - # A prefix to use before the message text (but after the command name, - # if selected). Default is two spaces, so you can see/grep the severity - # of each message. - prefix = " " - - # To make the messages look similar to the original LVM tools use: - # indent = 0 - # command_names = 1 - # prefix = " -- " - - # Set this if you want log messages during activation. - # Don't use this in low memory situations (can deadlock). - # activation = 0 -} - -# Configuration of metadata backups and archiving. In LVM2 when we -# talk about a 'backup' we mean making a copy of the metadata for the -# *current* system. The 'archive' contains old metadata configurations. -# Backups are stored in a human readeable text format. -backup { - - # Should we maintain a backup of the current metadata configuration ? - # Use 1 for Yes; 0 for No. - # Think very hard before turning this off! - backup = 1 - - # Where shall we keep it ? - # Remember to back up this directory regularly! - backup_dir = "/etc/lvm/backup" - - # Should we maintain an archive of old metadata configurations. - # Use 1 for Yes; 0 for No. - # On by default. Think very hard before turning this off. - archive = 1 - - # Where should archived files go ? - # Remember to back up this directory regularly! - archive_dir = "/etc/lvm/archive" - - # What is the minimum number of archive files you wish to keep ? - retain_min = 10 - - # What is the minimum time you wish to keep an archive file for ? - retain_days = 30 -} - -# Settings for the running LVM2 in shell (readline) mode. -shell { - - # Number of lines of history to store in ~/.lvm_history - history_size = 100 -} - - -# Miscellaneous global LVM2 settings -global { - - # The file creation mask for any files and directories created. - # Interpreted as octal if the first digit is zero. - umask = 077 - - # Allow other users to read the files - #umask = 022 - - # Enabling test mode means that no changes to the on disk metadata - # will be made. Equivalent to having the -t option on every - # command. Defaults to off. - test = 0 - - # Default value for --units argument - units = "h" - - # Since version 2.02.54, the tools distinguish between powers of - # 1024 bytes (e.g. KiB, MiB, GiB) and powers of 1000 bytes (e.g. - # KB, MB, GB). - # If you have scripts that depend on the old behaviour, set this to 0 - # temporarily until you update them. - si_unit_consistency = 1 - - # Whether or not to communicate with the kernel device-mapper. - # Set to 0 if you want to use the tools to manipulate LVM metadata - # without activating any logical volumes. - # If the device-mapper kernel driver is not present in your kernel - # setting this to 0 should suppress the error messages. - activation = 1 - - # If we can't communicate with device-mapper, should we try running - # the LVM1 tools? - # This option only applies to 2.4 kernels and is provided to help you - # switch between device-mapper kernels and LVM1 kernels. - # The LVM1 tools need to be installed with .lvm1 suffices - # e.g. vgscan.lvm1 and they will stop working after you start using - # the new lvm2 on-disk metadata format. - # The default value is set when the tools are built. - # fallback_to_lvm1 = 0 - - # The default metadata format that commands should use - "lvm1" or "lvm2". - # The command line override is -M1 or -M2. - # Defaults to "lvm2". - # format = "lvm2" - - # Location of proc filesystem - proc = "/proc" - - # Type of locking to use. Defaults to local file-based locking (1). - # Turn locking off by setting to 0 (dangerous: risks metadata corruption - # if LVM2 commands get run concurrently). - # Type 2 uses the external shared library locking_library. - # Type 3 uses built-in clustered locking. - # Type 4 uses read-only locking which forbids any operations that might - # change metadata. - locking_type = 1 - - # Set to 0 to fail when a lock request cannot be satisfied immediately. - wait_for_locks = 1 - - # If using external locking (type 2) and initialisation fails, - # with this set to 1 an attempt will be made to use the built-in - # clustered locking. - # If you are using a customised locking_library you should set this to 0. - fallback_to_clustered_locking = 1 - - # If an attempt to initialise type 2 or type 3 locking failed, perhaps - # because cluster components such as clvmd are not running, with this set - # to 1 an attempt will be made to use local file-based locking (type 1). - # If this succeeds, only commands against local volume groups will proceed. - # Volume Groups marked as clustered will be ignored. - fallback_to_local_locking = 1 - - # Local non-LV directory that holds file-based locks while commands are - # in progress. A directory like /tmp that may get wiped on reboot is OK. - locking_dir = "/var/lock/lvm" - - # Whenever there are competing read-only and read-write access requests for - # a volume group's metadata, instead of always granting the read-only - # requests immediately, delay them to allow the read-write requests to be - # serviced. Without this setting, write access may be stalled by a high - # volume of read-only requests. - # NB. This option only affects locking_type = 1 viz. local file-based - # locking. - prioritise_write_locks = 1 - - # Other entries can go here to allow you to load shared libraries - # e.g. if support for LVM1 metadata was compiled as a shared library use - # format_libraries = "liblvm2format1.so" - # Full pathnames can be given. - - # Search this directory first for shared libraries. - # library_dir = "/lib/lvm2" - - # The external locking library to load if locking_type is set to 2. - # locking_library = "liblvm2clusterlock.so" -} - -activation { - # Set to 0 to disable udev syncronisation (if compiled into the binaries). - # Processes will not wait for notification from udev. - # They will continue irrespective of any possible udev processing - # in the background. You should only use this if udev is not running - # or has rules that ignore the devices LVM2 creates. - # The command line argument --nodevsync takes precedence over this setting. - # If set to 1 when udev is not running, and there are LVM2 processes - # waiting for udev, run 'dmsetup udevcomplete_all' manually to wake them up. - udev_sync = 1 - - # How to fill in missing stripes if activating an incomplete volume. - # Using "error" will make inaccessible parts of the device return - # I/O errors on access. You can instead use a device path, in which - # case, that device will be used to in place of missing stripes. - # But note that using anything other than "error" with mirrored - # or snapshotted volumes is likely to result in data corruption. - missing_stripe_filler = "error" - - # How much stack (in KB) to reserve for use while devices suspended - reserved_stack = 256 - - # How much memory (in KB) to reserve for use while devices suspended - reserved_memory = 8192 - - # Nice value used while devices suspended - process_priority = -18 - - # If volume_list is defined, each LV is only activated if there is a - # match against the list. - # "vgname" and "vgname/lvname" are matched exactly. - # "@tag" matches any tag set in the LV or VG. - # "@*" matches if any tag defined on the host is also set in the LV or VG - # - # volume_list = [ "vg1", "vg2/lvol1", "@tag1", "@*" ] - - # Size (in KB) of each copy operation when mirroring - mirror_region_size = 512 - - # Setting to use when there is no readahead value stored in the metadata. - # - # "none" - Disable readahead. - # "auto" - Use default value chosen by kernel. - readahead = "auto" - - # 'mirror_image_fault_policy' and 'mirror_log_fault_policy' define - # how a device failure affecting a mirror is handled. - # A mirror is composed of mirror images (copies) and a log. - # A disk log ensures that a mirror does not need to be re-synced - # (all copies made the same) every time a machine reboots or crashes. - # - # In the event of a failure, the specified policy will be used to determine - # what happens. This applies to automatic repairs (when the mirror is being - # monitored by dmeventd) and to manual lvconvert --repair when - # --use-policies is given. - # - # "remove" - Simply remove the faulty device and run without it. If - # the log device fails, the mirror would convert to using - # an in-memory log. This means the mirror will not - # remember its sync status across crashes/reboots and - # the entire mirror will be re-synced. If a - # mirror image fails, the mirror will convert to a - # non-mirrored device if there is only one remaining good - # copy. - # - # "allocate" - Remove the faulty device and try to allocate space on - # a new device to be a replacement for the failed device. - # Using this policy for the log is fast and maintains the - # ability to remember sync state through crashes/reboots. - # Using this policy for a mirror device is slow, as it - # requires the mirror to resynchronize the devices, but it - # will preserve the mirror characteristic of the device. - # This policy acts like "remove" if no suitable device and - # space can be allocated for the replacement. - # - # "allocate_anywhere" - Not yet implemented. Useful to place the log device - # temporarily on same physical volume as one of the mirror - # images. This policy is not recommended for mirror devices - # since it would break the redundant nature of the mirror. This - # policy acts like "remove" if no suitable device and space can - # be allocated for the replacement. - - mirror_log_fault_policy = "allocate" - mirror_device_fault_policy = "remove" -} - - -#################### -# Advanced section # -#################### - -# Metadata settings -# -# metadata { - # Default number of copies of metadata to hold on each PV. 0, 1 or 2. - # You might want to override it from the command line with 0 - # when running pvcreate on new PVs which are to be added to large VGs. - - # pvmetadatacopies = 1 - - # Approximate default size of on-disk metadata areas in sectors. - # You should increase this if you have large volume groups or - # you want to retain a large on-disk history of your metadata changes. - - # pvmetadatasize = 255 - - # List of directories holding live copies of text format metadata. - # These directories must not be on logical volumes! - # It's possible to use LVM2 with a couple of directories here, - # preferably on different (non-LV) filesystems, and with no other - # on-disk metadata (pvmetadatacopies = 0). Or this can be in - # addition to on-disk metadata areas. - # The feature was originally added to simplify testing and is not - # supported under low memory situations - the machine could lock up. - # - # Never edit any files in these directories by hand unless you - # you are absolutely sure you know what you are doing! Use - # the supplied toolset to make changes (e.g. vgcfgrestore). - - # dirs = [ "/etc/lvm/metadata", "/mnt/disk2/lvm/metadata2" ] -#} - -# Event daemon -# -dmeventd { - # mirror_library is the library used when monitoring a mirror device. - # - # "libdevmapper-event-lvm2mirror.so" attempts to recover from - # failures. It removes failed devices from a volume group and - # reconfigures a mirror as necessary. If no mirror library is - # provided, mirrors are not monitored through dmeventd. - - mirror_library = "libdevmapper-event-lvm2mirror.so" - - # snapshot_library is the library used when monitoring a snapshot device. - # - # "libdevmapper-event-lvm2snapshot.so" monitors the filling of - # snapshots and emits a warning through syslog, when the use of - # snapshot exceedes 80%. The warning is repeated when 85%, 90% and - # 95% of the snapshot are filled. - - snapshot_library = "libdevmapper-event-lvm2snapshot.so" -} diff --git a/contrib/puppet/files/etc/nova.conf b/contrib/puppet/files/etc/nova.conf deleted file mode 100644 index a0d64078c..000000000 --- a/contrib/puppet/files/etc/nova.conf +++ /dev/null @@ -1,28 +0,0 @@ ---ec2_url=http://192.168.255.1:8773/services/Cloud ---rabbit_host=192.168.255.1 ---redis_host=192.168.255.1 ---s3_host=192.168.255.1 ---vpn_ip=192.168.255.1 ---datastore_path=/var/lib/nova/keeper ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vlan_start=2000 ---vlan_end=3000 ---private_range=192.168.0.0/16 ---public_range=10.0.0.0/24 ---volume_group=vgdata ---storage_dev=/dev/sdc ---bridge_dev=eth2 ---aoe_eth_dev=eth2 ---public_interface=vlan0 ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---vpn_image_id=ami-cloudpipe ---daemonize ---verbose ---syslog ---prefix=nova diff --git a/contrib/puppet/files/production/boto.cfg b/contrib/puppet/files/production/boto.cfg deleted file mode 100644 index f4a2de2b6..000000000 --- a/contrib/puppet/files/production/boto.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[Boto] -debug = 0 -num_retries = 1 diff --git a/contrib/puppet/files/production/genvpn.sh b/contrib/puppet/files/production/genvpn.sh deleted file mode 100644 index 538c3cd33..000000000 --- a/contrib/puppet/files/production/genvpn.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This gets zipped and run on the cloudpipe-managed OpenVPN server -NAME=$1 -SUBJ=$2 - -mkdir -p projects/$NAME -cd projects/$NAME - -# generate a server priv key -openssl genrsa -out server.key 2048 - -# generate a server CSR -openssl req -new -key server.key -out server.csr -batch -subj "$SUBJ" - -if [ "`id -u`" != "`grep nova /etc/passwd | cut -d':' -f3`" ]; then - sudo chown -R nova:nogroup . -fi diff --git a/contrib/puppet/files/production/libvirt.qemu.xml.template b/contrib/puppet/files/production/libvirt.qemu.xml.template deleted file mode 100644 index 114dfdc01..000000000 --- a/contrib/puppet/files/production/libvirt.qemu.xml.template +++ /dev/null @@ -1,35 +0,0 @@ - - %(name)s - - hvm - %(basepath)s/kernel - %(basepath)s/ramdisk - root=/dev/vda1 console=ttyS0 - - - - - %(memory_kb)s - %(vcpus)s - - - - - - - - - - - - - - - - - diff --git a/contrib/puppet/files/production/my.cnf b/contrib/puppet/files/production/my.cnf deleted file mode 100644 index 8777bc480..000000000 --- a/contrib/puppet/files/production/my.cnf +++ /dev/null @@ -1,137 +0,0 @@ -# -# The MySQL database server configuration file. -# -# You can copy this to one of: -# - "/etc/mysql/my.cnf" to set global options, -# - "~/.my.cnf" to set user-specific options. -# -# One can use all long options that the program supports. -# Run program with --help to get a list of available options and with -# --print-defaults to see which it would actually understand and use. -# -# For explanations see -# http://dev.mysql.com/doc/mysql/en/server-system-variables.html - -# This will be passed to all mysql clients -# It has been reported that passwords should be enclosed with ticks/quotes -# escpecially if they contain "#" chars... -# Remember to edit /etc/mysql/debian.cnf when changing the socket location. -[client] -port = 3306 -socket = /var/run/mysqld/mysqld.sock - -# Here is entries for some specific programs -# The following values assume you have at least 32M ram - -# This was formally known as [safe_mysqld]. Both versions are currently parsed. -[mysqld_safe] -socket = /var/run/mysqld/mysqld.sock -nice = 0 - -[mysqld] -# -# * Basic Settings -# - -# -# * IMPORTANT -# If you make changes to these settings and your system uses apparmor, you may -# also need to also adjust /etc/apparmor.d/usr.sbin.mysqld. -# - -user = mysql -socket = /var/run/mysqld/mysqld.sock -port = 3306 -basedir = /usr -datadir = /var/lib/mysql -tmpdir = /tmp -skip-external-locking -# -# Instead of skip-networking the default is now to listen only on -# localhost which is more compatible and is not less secure. -# bind-address = 127.0.0.1 -# -# * Fine Tuning -# -innodb_buffer_pool_size = 12G -#innodb_log_file_size = 256M -innodb_log_buffer_size=4M -innodb_flush_log_at_trx_commit=2 -innodb_thread_concurrency=8 -innodb_flush_method=O_DIRECT -key_buffer = 128M -max_allowed_packet = 256M -thread_stack = 8196K -thread_cache_size = 32 -# This replaces the startup script and checks MyISAM tables if needed -# the first time they are touched -myisam-recover = BACKUP -max_connections = 1000 -table_cache = 1024 -#thread_concurrency = 10 -# -# * Query Cache Configuration -# -query_cache_limit = 32M -query_cache_size = 256M -# -# * Logging and Replication -# -# Both location gets rotated by the cronjob. -# Be aware that this log type is a performance killer. -# As of 5.1 you can enable the log at runtime! -#general_log_file = /var/log/mysql/mysql.log -#general_log = 1 - -log_error = /var/log/mysql/error.log - -# Here you can see queries with especially long duration -log_slow_queries = /var/log/mysql/mysql-slow.log -long_query_time = 2 -#log-queries-not-using-indexes -# -# The following can be used as easy to replay backup logs or for replication. -# note: if you are setting up a replication slave, see README.Debian about -# other settings you may need to change. -server-id = 1 -log_bin = /var/log/mysql/mysql-bin.log -expire_logs_days = 10 -max_binlog_size = 50M -#binlog_do_db = include_database_name -#binlog_ignore_db = include_database_name -# -# * InnoDB -# -sync_binlog=1 -# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. -# Read the manual for more InnoDB related options. There are many! -# -# * Security Features -# -# Read the manual, too, if you want chroot! -# chroot = /var/lib/mysql/ -# -# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". -# -# ssl-ca=/etc/mysql/cacert.pem -# ssl-cert=/etc/mysql/server-cert.pem -# ssl-key=/etc/mysql/server-key.pem - - - -[mysqldump] -quick -quote-names -max_allowed_packet = 256M - -[mysql] -#no-auto-rehash # faster start of mysql but no tab completition - -[isamchk] -key_buffer = 128M - -# -# * IMPORTANT: Additional settings that can override those from this file! -# The files must end with '.cnf', otherwise they'll be ignored. -# -!includedir /etc/mysql/conf.d/ diff --git a/contrib/puppet/files/production/nova-iptables b/contrib/puppet/files/production/nova-iptables deleted file mode 100755 index 61e2ca2b9..000000000 --- a/contrib/puppet/files/production/nova-iptables +++ /dev/null @@ -1,187 +0,0 @@ -#! /bin/sh - -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# NOTE(vish): This script sets up some reasonable defaults for iptables and -# creates nova-specific chains. If you use this script you should -# run nova-network and nova-compute with --use_nova_chains=True - - -# NOTE(vish): If you run public nova-api on a different port, make sure to -# change the port here - -if [ -f /etc/default/nova-iptables ] ; then - . /etc/default/nova-iptables -fi - -export LC_ALL=C - -API_PORT=${API_PORT:-"8773"} - -if [ ! -n "$IP" ]; then - # NOTE(vish): IP address is what address the services ALLOW on. - # This will just get the first ip in the list, so if you - # have more than one eth device set up, this will fail, and - # you should explicitly pass in the ip of the instance - IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` -fi - -if [ ! -n "$PRIVATE_RANGE" ]; then - #NOTE(vish): PRIVATE_RANGE: range is ALLOW to access DHCP - PRIVATE_RANGE="192.168.0.0/12" -fi - -if [ ! -n "$MGMT_IP" ]; then - # NOTE(vish): Management IP is the ip over which to allow ssh traffic. It - # will also allow traffic to nova-api - MGMT_IP="$IP" -fi - -if [ ! -n "$DMZ_IP" ]; then - # NOTE(vish): DMZ IP is the ip over which to allow api & objectstore access - DMZ_IP="$IP" -fi - -clear_nova_iptables() { - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT - iptables -F - iptables -t nat -F - iptables -F services - iptables -X services - # HACK: re-adding fail2ban rules :( - iptables -N fail2ban-ssh - iptables -A INPUT -p tcp -m multiport --dports 22 -j fail2ban-ssh - iptables -A fail2ban-ssh -j RETURN -} - -load_nova_iptables() { - - iptables -P INPUT DROP - iptables -A INPUT -m state --state INVALID -j DROP - iptables -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - # NOTE(ja): allow localhost for everything - iptables -A INPUT -d 127.0.0.1/32 -j ACCEPT - # NOTE(ja): 22 only allowed MGMT_IP before, but we widened it to any - # address, since ssh should be listening only on internal - # before we re-add this rule we will need to add - # flexibility for RSYNC between omega/stingray - iptables -A INPUT -m tcp -p tcp --dport 22 -j ACCEPT - iptables -A INPUT -m udp -p udp --dport 123 -j ACCEPT - iptables -A INPUT -p icmp -j ACCEPT - iptables -N services - iptables -A INPUT -j services - iptables -A INPUT -p tcp -j REJECT --reject-with tcp-reset - iptables -A INPUT -j REJECT --reject-with icmp-port-unreachable - - iptables -P FORWARD DROP - iptables -A FORWARD -m state --state INVALID -j DROP - iptables -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT - iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu - - # NOTE(vish): DROP on output is too restrictive for now. We need to add - # in a bunch of more specific output rules to use it. - # iptables -P OUTPUT DROP - iptables -A OUTPUT -m state --state INVALID -j DROP - iptables -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT - - if [ -n "$GANGLIA" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 8649 -j ACCEPT - iptables -A services -m udp -p udp -d $IP --dport 8649 -j ACCEPT - fi - - # if [ -n "$WEB" ] || [ -n "$ALL" ]; then - # # NOTE(vish): This opens up ports for web access, allowing web-based - # # dashboards to work. - # iptables -A services -m tcp -p tcp -d $IP --dport 80 -j ACCEPT - # iptables -A services -m tcp -p tcp -d $IP --dport 443 -j ACCEPT - # fi - - if [ -n "$OBJECTSTORE" ] || [ -n "$ALL" ]; then - # infrastructure - iptables -A services -m tcp -p tcp -d $IP --dport 3333 -j ACCEPT - # clients - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport 3333 -j ACCEPT - fi - - if [ -n "$API" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport $API_PORT -j ACCEPT - if [ "$IP" != "$DMZ_IP" ]; then - iptables -A services -m tcp -p tcp -d $DMZ_IP --dport $API_PORT -j ACCEPT - fi - if [ "$IP" != "$MGMT_IP" ] && [ "$DMZ_IP" != "$MGMT_IP" ]; then - iptables -A services -m tcp -p tcp -d $MGMT_IP --dport $API_PORT -j ACCEPT - fi - fi - - if [ -n "$REDIS" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 6379 -j ACCEPT - fi - - if [ -n "$MYSQL" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3306 -j ACCEPT - fi - - if [ -n "$RABBITMQ" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 4369 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 5672 -j ACCEPT - iptables -A services -m tcp -p tcp -d $IP --dport 53284 -j ACCEPT - fi - - if [ -n "$DNSMASQ" ] || [ -n "$ALL" ]; then - # NOTE(vish): this could theoretically be setup per network - # for each host, but it seems like overkill - iptables -A services -m tcp -p tcp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp -s $PRIVATE_RANGE --dport 53 -j ACCEPT - iptables -A services -m udp -p udp --dport 67 -j ACCEPT - fi - - if [ -n "$LDAP" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 389 -j ACCEPT - fi - - if [ -n "$ISCSI" ] || [ -n "$ALL" ]; then - iptables -A services -m tcp -p tcp -d $IP --dport 3260 -j ACCEPT - iptables -A services -m tcp -p tcp -d 127.0.0.0/16 --dport 3260 -j ACCEPT - fi -} - - -case "$1" in - start) - echo "Starting nova-iptables: " - load_nova_iptables - ;; - stop) - echo "Clearing nova-iptables: " - clear_nova_iptables - ;; - restart) - echo "Restarting nova-iptables: " - clear_nova_iptables - load_nova_iptables - ;; - *) - echo "Usage: $NAME {start|stop|restart}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/contrib/puppet/files/production/nova-iscsi-dev.sh b/contrib/puppet/files/production/nova-iscsi-dev.sh deleted file mode 100644 index 8eda10d2e..000000000 --- a/contrib/puppet/files/production/nova-iscsi-dev.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# FILE: /etc/udev/scripts/iscsidev.sh - -BUS=${1} -HOST=${BUS%%:*} - -[ -e /sys/class/iscsi_host ] || exit 1 - -file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" - -target_name=$(cat ${file}) - -# This is not an open-scsi drive -if [ -z "${target_name}" ]; then - exit 1 -fi - -echo "${target_name##*:}" diff --git a/contrib/puppet/files/production/setup_data.sh b/contrib/puppet/files/production/setup_data.sh deleted file mode 100755 index 1fbbac41c..000000000 --- a/contrib/puppet/files/production/setup_data.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -/root/slap.sh -mysql -e "DROP DATABASE nova" -mysql -e "CREATE DATABASE nova" -mysql -e "GRANT ALL on nova.* to nova@'%' identified by 'TODO:CHANGEME:CMON'" -touch /root/installed diff --git a/contrib/puppet/files/production/slap.sh b/contrib/puppet/files/production/slap.sh deleted file mode 100755 index f8ea16949..000000000 --- a/contrib/puppet/files/production/slap.sh +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# LDAP INSTALL SCRIPT - SHOULD BE IDEMPOTENT, but it SCRUBS all USERS - -apt-get install -y slapd ldap-utils python-ldap - -cat >/etc/ldap/schema/openssh-lpk_openldap.schema < -# -# Based on the proposal of : Mark Ruijter -# - - -# octetString SYNTAX -attributetype ( 1.3.6.1.4.1.24552.500.1.1.1.13 NAME 'sshPublicKey' - DESC 'MANDATORY: OpenSSH Public key' - EQUALITY octetStringMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 ) - -# printableString SYNTAX yes|no -objectclass ( 1.3.6.1.4.1.24552.500.1.1.2.0 NAME 'ldapPublicKey' SUP top AUXILIARY - DESC 'MANDATORY: OpenSSH LPK objectclass' - MAY ( sshPublicKey $ uid ) - ) -LPK_SCHEMA_EOF - -cat >/etc/ldap/schema/nova.schema < -# -# - -# using internet experimental oid arc as per BP64 3.1 -objectidentifier novaSchema 1.3.6.1.3.1.666.666 -objectidentifier novaAttrs novaSchema:3 -objectidentifier novaOCs novaSchema:4 - -attributetype ( - novaAttrs:1 - NAME 'accessKey' - DESC 'Key for accessing data' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:2 - NAME 'secretKey' - DESC 'Secret key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 - SINGLE-VALUE - ) - -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - -objectClass ( - novaOCs:1 - NAME 'novaUser' - DESC 'access and secret keys' - AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) - ) - -NOVA_SCHEMA_EOF - -mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig -cat >/etc/ldap/slapd.conf </etc/ldap/ldap.conf </etc/ldap/base.ldif < "/usr/bin/apt-get update" } diff --git a/contrib/puppet/manifests/classes/issue.pp b/contrib/puppet/manifests/classes/issue.pp deleted file mode 100644 index 8bb37ee3f..000000000 --- a/contrib/puppet/manifests/classes/issue.pp +++ /dev/null @@ -1,14 +0,0 @@ -class issue { - file { "/etc/issue": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } - file { "/etc/issue.net": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/issue", - } -} diff --git a/contrib/puppet/manifests/classes/kern_module.pp b/contrib/puppet/manifests/classes/kern_module.pp deleted file mode 100644 index 00ec0636c..000000000 --- a/contrib/puppet/manifests/classes/kern_module.pp +++ /dev/null @@ -1,34 +0,0 @@ -# via http://projects.puppetlabs.com/projects/puppet/wiki/Kernel_Modules_Patterns - -define kern_module ($ensure) { - $modulesfile = $operatingsystem ? { ubuntu => "/etc/modules", redhat => "/etc/rc.modules" } - case $operatingsystem { - redhat: { file { "/etc/rc.modules": ensure => file, mode => 755 } } - } - case $ensure { - present: { - exec { "insert_module_${name}": - command => $operatingsystem ? { - ubuntu => "/bin/echo '${name}' >> '${modulesfile}'", - redhat => "/bin/echo '/sbin/modprobe ${name}' >> '${modulesfile}' " - }, - unless => "/bin/grep -qFx '${name}' '${modulesfile}'" - } - exec { "/sbin/modprobe ${name}": unless => "/bin/grep -q '^${name} ' '/proc/modules'" } - } - absent: { - exec { "/sbin/modprobe -r ${name}": onlyif => "/bin/grep -q '^${name} ' '/proc/modules'" } - exec { "remove_module_${name}": - command => $operatingsystem ? { - ubuntu => "/usr/bin/perl -ni -e 'print unless /^\\Q${name}\\E\$/' '${modulesfile}'", - redhat => "/usr/bin/perl -ni -e 'print unless /^\\Q/sbin/modprobe ${name}\\E\$/' '${modulesfile}'" - }, - onlyif => $operatingsystem ? { - ubuntu => "/bin/grep -qFx '${name}' '${modulesfile}'", - redhat => "/bin/grep -q '^/sbin/modprobe ${name}' '${modulesfile}'" - } - } - } - default: { err ( "unknown ensure value ${ensure}" ) } - } -} diff --git a/contrib/puppet/manifests/classes/loopback.pp b/contrib/puppet/manifests/classes/loopback.pp deleted file mode 100644 index e0fa9d541..000000000 --- a/contrib/puppet/manifests/classes/loopback.pp +++ /dev/null @@ -1,6 +0,0 @@ -define loopback($num) { - exec { "mknod -m 0660 /dev/loop${num} b 7 ${num}; chown root:disk /dev/loop${num}": - creates => "/dev/loop${num}", - path => ["/usr/bin", "/usr/sbin", "/bin"] - } -} diff --git a/contrib/puppet/manifests/classes/lvm.pp b/contrib/puppet/manifests/classes/lvm.pp deleted file mode 100644 index 5a407abcb..000000000 --- a/contrib/puppet/manifests/classes/lvm.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvm { - file { "/etc/lvm/lvm.conf": - owner => "root", - group => "root", - mode => 444, - source => "puppet://${puppet_server}/files/etc/lvm.conf", - } -} diff --git a/contrib/puppet/manifests/classes/lvmconf.pp b/contrib/puppet/manifests/classes/lvmconf.pp deleted file mode 100644 index 4aa7ddfdc..000000000 --- a/contrib/puppet/manifests/classes/lvmconf.pp +++ /dev/null @@ -1,8 +0,0 @@ -class lvmconf { - file { "/etc/lvm/lvm.conf": - owner => "root", group => "root", mode => 644, - source => "puppet://${puppet_server}/files/etc/lvm/lvm.conf", - ensure => present - } -} - diff --git a/contrib/puppet/manifests/classes/nova.pp b/contrib/puppet/manifests/classes/nova.pp deleted file mode 100644 index e942860f4..000000000 --- a/contrib/puppet/manifests/classes/nova.pp +++ /dev/null @@ -1,464 +0,0 @@ -import "kern_module" -import "apt" -import "loopback" - -#$head_node_ip = "undef" -#$rabbit_ip = "undef" -#$vpn_ip = "undef" -#$public_interface = "undef" -#$vlan_start = "5000" -#$vlan_end = "6000" -#$private_range = "10.0.0.0/16" -#$public_range = "192.168.177.0/24" - -define nova_iptables($services, $ip="", $private_range="", $mgmt_ip="", $dmz_ip="") { - file { "/etc/init.d/nova-iptables": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iptables", - } - - file { "/etc/default/nova-iptables": - owner => "root", mode => 644, - content => template("nova-iptables.erb") - } -} - -define nova_conf_pointer($name) { - file { "/etc/nova/nova-${name}.conf": - owner => "nova", mode => 400, - content => "--flagfile=/etc/nova/nova.conf" - } -} - -class novaconf { - file { "/etc/nova/nova.conf": - owner => "nova", mode => 400, - content => template("production/nova-common.conf.erb", "production/nova-${cluster_name}.conf.erb") - } - nova_conf_pointer{'manage': name => 'manage'} -} - -class novadata { - package { "rabbitmq-server": ensure => present } - - file { "/etc/rabbitmq/rabbitmq.conf": - owner => "root", mode => 644, - content => "NODENAME=rabbit@localhost", - } - - service { "rabbitmq-server": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/rabbitmq/rabbitmq.conf"], - Package["rabbitmq-server"] - ] - } - - package { "mysql-server": ensure => present } - - file { "/etc/mysql/my.cnf": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/my.cnf", - } - - service { "mysql": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/mysql/my.cnf"], - Package["mysql-server"] - ] - } - - file { "/root/slap.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/slap.sh", - } - - file { "/root/setup_data.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/setup_data.sh", - } - - # setup compute data - exec { "setup_data": - command => "/root/setup_data.sh", - path => "/usr/bin:/bin", - unless => "test -f /root/installed", - require => [ - Service["mysql"], - File["/root/slap.sh"], - File["/root/setup_data.sh"] - ] - } -} - -define nscheduler($version) { - package { "nova-scheduler": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'scheduler': name => 'scheduler'} - exec { "update-rc.d -f nova-scheduler remove; update-rc.d nova-scheduler defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-scheduler", - unless => "test -f /etc/rc2.d/S50nova-scheduler" - } - service { "nova-scheduler": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-scheduler"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-scheduler.conf"] - ] - } - -} - -define napi($version, $api_servers, $api_base_port) { - file { "/etc/boto.cfg": - owner => "root", mode => 644, - source => "puppet://${puppet_server}/files/production/boto.cfg", - } - - file { "/var/lib/nova/CA/genvpn.sh": - owner => "nova", mode => 755, - source => "puppet://${puppet_server}/files/production/genvpn.sh", - } - - package { "python-greenlet": ensure => present } - package { "nova-api": ensure => $version, require => [Exec["update-apt"], Package["python-greenlet"]] } - nova_conf_pointer{'api': name => 'api'} - - exec { "update-rc.d -f nova-api remove; update-rc.d nova-api defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-api", - unless => "test -f /etc/rc2.d/S50nova-api" - } - - service { "nova-netsync": - start => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock start", - stop => "/usr/bin/nova-netsync --pidfile=/var/run/nova/nova-netsync.pid --lockfile=/var/run/nova/nova-netsync.pid.lock stop", - ensure => running, - hasstatus => false, - pattern => "nova-netsync", - require => Service["nova-api"], - subscribe => File["/etc/nova/nova.conf"] - } - service { "nova-api": - start => "monit start all -g nova_api", - stop => "monit stop all -g nova_api", - restart => "monit restart all -g nova_api", - # ensure => running, - # hasstatus => true, - require => Service["monit"], - subscribe => [ - Package["nova-objectstore"], - File["/etc/boto.cfg"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } - - # the haproxy & monit's template use $api_servers and $api_base_port - - package { "haproxy": ensure => present } - file { "/etc/default/haproxy": - owner => "root", mode => 644, - content => "ENABLED=1", - require => Package['haproxy'] - } - file { "/etc/haproxy/haproxy.cfg": - owner => "root", mode => 644, - content => template("/srv/cloud/puppet/templates/haproxy.cfg.erb"), - require => Package['haproxy'] - } - service { "haproxy": - ensure => true, - enable => true, - hasstatus => true, - subscribe => [ - Package["haproxy"], - File["/etc/default/haproxy"], - File["/etc/haproxy/haproxy.cfg"], - ] - } - - package { "socat": ensure => present } - - file { "/usr/local/bin/gmetric_haproxy.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/ganglia/gmetric_scripts/gmetric_haproxy.sh", - } - - cron { "gmetric_haproxy": - command => "/usr/local/bin/gmetric_haproxy.sh", - user => root, - minute => "*/3", - } - - package { "monit": ensure => present } - - file { "/etc/default/monit": - owner => "root", mode => 644, - content => "startup=1", - require => Package['monit'] - } - file { "/etc/monit/monitrc": - owner => "root", mode => 600, - content => template("/srv/cloud/puppet/templates/monitrc-nova-api.erb"), - require => Package['monit'] - } - service { "monit": - ensure => true, - pattern => "sbin/monit", - subscribe => [ - Package["monit"], - File["/etc/default/monit"], - File["/etc/monit/monitrc"], - ] - } - -} - - -define nnetwork($version) { - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - # EVIL HACK: custom binary because dnsmasq 2.52 segfaulted accessing dereferenced object - file { "/usr/sbin/dnsmasq": - owner => "root", group => "root", - source => "puppet://${puppet_server}/files/production/dnsmasq", - } - - package { "nova-network": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'dhcpbridge': name => 'dhcpbridge'} - nova_conf_pointer{'network': name => "network" } - - exec { "update-rc.d -f nova-network remove; update-rc.d nova-network defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-network", - unless => "test -f /etc/rc2.d/S50nova-network" - } - service { "nova-network": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-network"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-network.conf"] - ] - } -} - -define nobjectstore($version) { - package { "nova-objectstore": ensure => $version, require => Exec["update-apt"] } - nova_conf_pointer{'objectstore': name => 'objectstore'} - exec { "update-rc.d -f nova-objectstore remove; update-rc.d nova-objectstore defaults 50": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/init.d/nova-objectstore", - unless => "test -f /etc/rc2.d/S50nova-objectstore" - } - service { "nova-objectstore": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-objectstore"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-objectstore.conf"] - ] - } -} - -define ncompute($version) { - include ganglia-python - include ganglia-compute - - # kill the default network added by the package - exec { "kill-libvirt-default-net": - command => "virsh net-destroy default; rm /etc/libvirt/qemu/networks/autostart/default.xml", - path => "/usr/bin:/bin", - onlyif => "test -f /etc/libvirt/qemu/networks/autostart/default.xml" - } - - - # LIBVIRT has to be restarted when ebtables / gawk is installed - service { "libvirt-bin": - ensure => running, - pattern => "sbin/libvirtd", - subscribe => [ - Package["ebtables"], - Kern_module["kvm_intel"] - ], - require => [ - Package["libvirt-bin"], - Package["ebtables"], - Package["gawk"], - Kern_module["kvm_intel"], - File["/dev/kvm"] - ] - } - - package { "libvirt-bin": ensure => "0.8.3-1ubuntu14~ppalucid2" } - package { "ebtables": ensure => present } - package { "gawk": ensure => present } - - # ensure proper permissions on /dev/kvm - file { "/dev/kvm": - owner => "root", - group => "kvm", - mode => 660 - } - - # require hardware virt - kern_module { "kvm_intel": - ensure => present, - } - - # increase loopback devices - file { "/etc/modprobe.d/loop.conf": - owner => "root", mode => 644, - content => "options loop max_loop=40" - } - - nova_conf_pointer{'compute': name => 'compute'} - - loopback{loop0: num => 0} - loopback{loop1: num => 1} - loopback{loop2: num => 2} - loopback{loop3: num => 3} - loopback{loop4: num => 4} - loopback{loop5: num => 5} - loopback{loop6: num => 6} - loopback{loop7: num => 7} - loopback{loop8: num => 8} - loopback{loop9: num => 9} - loopback{loop10: num => 10} - loopback{loop11: num => 11} - loopback{loop12: num => 12} - loopback{loop13: num => 13} - loopback{loop14: num => 14} - loopback{loop15: num => 15} - loopback{loop16: num => 16} - loopback{loop17: num => 17} - loopback{loop18: num => 18} - loopback{loop19: num => 19} - loopback{loop20: num => 20} - loopback{loop21: num => 21} - loopback{loop22: num => 22} - loopback{loop23: num => 23} - loopback{loop24: num => 24} - loopback{loop25: num => 25} - loopback{loop26: num => 26} - loopback{loop27: num => 27} - loopback{loop28: num => 28} - loopback{loop29: num => 29} - loopback{loop30: num => 30} - loopback{loop31: num => 31} - loopback{loop32: num => 32} - loopback{loop33: num => 33} - loopback{loop34: num => 34} - loopback{loop35: num => 35} - loopback{loop36: num => 36} - loopback{loop37: num => 37} - loopback{loop38: num => 38} - loopback{loop39: num => 39} - - package { "python-libvirt": ensure => "0.8.3-1ubuntu14~ppalucid2" } - - package { "nova-compute": - ensure => "$version", - require => Package["python-libvirt"] - } - - #file { "/usr/share/nova/libvirt.qemu.xml.template": - # owner => "nova", mode => 400, - # source => "puppet://${puppet_server}/files/production/libvirt.qemu.xml.template", - #} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-compute remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-compute" - } - service { "nova-compute": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-compute"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-compute.conf"], - #File["/usr/share/nova/libvirt.qemu.xml.template"], - Service["libvirt-bin"], - Kern_module["kvm_intel"] - ] - } -} - -define nvolume($version) { - - package { "nova-volume": ensure => $version, require => Exec["update-apt"] } - - nova_conf_pointer{'volume': name => 'volume'} - - # fix runlevels: using enable => true adds it as 20, which is too early - exec { "update-rc.d -f nova-volume remove": - path => "/usr/bin:/usr/sbin:/bin", - onlyif => "test -f /etc/rc2.d/S??nova-volume" - } - - file { "/etc/default/iscsitarget": - owner => "root", mode => 644, - content => "ISCSITARGET_ENABLE=true" - } - - package { "iscsitarget": ensure => present } - - file { "/dev/iscsi": ensure => directory } # FIXME(vish): owner / mode? - file { "/usr/sbin/nova-iscsi-dev.sh": - owner => "root", mode => 755, - source => "puppet://${puppet_server}/files/production/nova-iscsi-dev.sh" - } - file { "/etc/udev/rules.d/55-openiscsi.rules": - owner => "root", mode => 644, - content => 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/usr/sbin/nova-iscsi-dev.sh %b",SYMLINK+="iscsi/%c%n"' - } - - service { "iscsitarget": - ensure => running, - enable => true, - hasstatus => true, - require => [ - File["/etc/default/iscsitarget"], - Package["iscsitarget"] - ] - } - - service { "nova-volume": - ensure => running, - hasstatus => true, - subscribe => [ - Package["nova-volume"], - File["/etc/nova/nova.conf"], - File["/etc/nova/nova-volume.conf"] - ] - } -} - -class novaspool { - # This isn't in release yet - #cron { logspool: - # command => "/usr/bin/nova-logspool /var/log/nova.log /var/lib/nova/spool", - # user => "nova" - #} - #cron { spoolsentry: - # command => "/usr/bin/nova-spoolsentry ${sentry_url} ${sentry_key} /var/lib/nova/spool", - # user => "nova" - #} -} diff --git a/contrib/puppet/manifests/classes/swift.pp b/contrib/puppet/manifests/classes/swift.pp deleted file mode 100644 index 64ffb6fa3..000000000 --- a/contrib/puppet/manifests/classes/swift.pp +++ /dev/null @@ -1,7 +0,0 @@ -class swift { - package { "memcached": ensure => present } - service { "memcached": require => Package['memcached'] } - - package { "swift-proxy": ensure => present } -} - diff --git a/contrib/puppet/manifests/site.pp b/contrib/puppet/manifests/site.pp deleted file mode 100644 index ca07a34ad..000000000 --- a/contrib/puppet/manifests/site.pp +++ /dev/null @@ -1,120 +0,0 @@ -# site.pp - -import "templates" -import "classes/*" - -node novabase inherits default { -# $puppet_server = "192.168.0.10" - $cluster_name = "openstack001" - $ganglia_udp_send_channel = "openstack001.example.com" - $syslog = "192.168.0.10" - - # THIS STUFF ISN'T IN RELEASE YET - #$sentry_url = "http://192.168.0.19/sentry/store/" - #$sentry_key = "TODO:SENTRYPASS" - - $local_network = "192.168.0.0/16" - $vpn_ip = "192.168.0.2" - $public_interface = "eth0" - include novanode -# include nova-common - include opsmetrics - -# non-nova stuff such as nova-dash inherit from novanode -# novaspool needs a better home -# include novaspool -} - -# Builder -node "nova000.example.com" inherits novabase { - $syslog = "server" - include ntp - include syslog-server -} - -# Non-Nova nodes - -node - "blog.example.com", - "wiki.example.com" -inherits novabase { - include ganglia-python - include ganglia-apache - include ganglia-mysql -} - - -node "nova001.example.com" -inherits novabase { - include novabase - - nova_iptables { nova: - services => [ - "ganglia", - "mysql", - "rabbitmq", - "ldap", - "api", - "objectstore", - "nrpe", - ], - ip => "192.168.0.10", - } - - nobjectstore { nova: version => "0.9.0" } - nscheduler { nova: version => "0.9.0" } - napi { nova: - version => "0.9.0", - api_servers => 10, - api_base_port => 8000 - } -} - -node "nova002.example.com" -inherits novabase { - include novaconf - - nova_iptables { nova: - services => [ - "ganglia", - "dnsmasq", - "nrpe" - ], - ip => "192.168.4.2", - private_range => "192.168.0.0/16", - } - - nnetwork { nova: version => "0.9.0" } -} - -node - "nova003.example.com", - "nova004.example.com", - "nova005.example.com", - "nova006.example.com", - "nova007.example.com", - "nova008.example.com", - "nova009.example.com", - "nova010.example.com", - "nova011.example.com", - "nova012.example.com", - "nova013.example.com", - "nova014.example.com", - "nova015.example.com", - "nova016.example.com", - "nova017.example.com", - "nova018.example.com", - "nova019.example.com", -inherits novabase { - include novaconf - ncompute { nova: version => "0.9.0" } - nvolume { nova: version => "0.9.0" } -} - -#node -# "nova020.example.com" -# "nova021.example.com" -#inherits novanode { -# include novaconf - #ncompute { nova: version => "0.9.0" } -#} diff --git a/contrib/puppet/manifests/templates.pp b/contrib/puppet/manifests/templates.pp deleted file mode 100644 index 90e433013..000000000 --- a/contrib/puppet/manifests/templates.pp +++ /dev/null @@ -1,21 +0,0 @@ -# templates.pp - -import "classes/*" - -class baseclass { -# include dns-client # FIXME: missing resolv.conf.erb?? - include issue -} - -node default { - $nova_site = "undef" - $nova_ns1 = "undef" - $nova_ns2 = "undef" -# include baseclass -} - -# novanode handles the system-level requirements for Nova/Swift nodes -class novanode { - include baseclass - include lvmconf -} diff --git a/contrib/puppet/puppet.conf b/contrib/puppet/puppet.conf deleted file mode 100644 index 92af920e3..000000000 --- a/contrib/puppet/puppet.conf +++ /dev/null @@ -1,11 +0,0 @@ -[main] -logdir=/var/log/puppet -vardir=/var/lib/puppet -ssldir=/var/lib/puppet/ssl -rundir=/var/run/puppet -factpath=$vardir/lib/facter -pluginsync=false - -[puppetmasterd] -templatedir=/var/lib/nova/contrib/puppet/templates -autosign=true diff --git a/contrib/puppet/templates/haproxy.cfg.erb b/contrib/puppet/templates/haproxy.cfg.erb deleted file mode 100644 index bd9991de7..000000000 --- a/contrib/puppet/templates/haproxy.cfg.erb +++ /dev/null @@ -1,39 +0,0 @@ -# this config needs haproxy-1.1.28 or haproxy-1.2.1 - -global - log 127.0.0.1 local0 - log 127.0.0.1 local1 notice - #log loghost local0 info - maxconn 4096 - #chroot /usr/share/haproxy - stats socket /var/run/haproxy.sock - user haproxy - group haproxy - daemon - #debug - #quiet - -defaults - log global - mode http - option httplog - option dontlognull - retries 3 - option redispatch - stats enable - stats uri /haproxy - maxconn 2000 - contimeout 5000 - clitimeout 50000 - srvtimeout 50000 - - -listen nova-api 0.0.0.0:8773 - option httpchk GET / HTTP/1.0\r\nHost:\ example.com - option forwardfor - reqidel ^X-Forwarded-For:.* - balance roundrobin -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset -%> - server api_<%= port %> 127.0.0.1:<%= port %> maxconn 1 check -<% end -%> - option httpclose # disable keep-alive diff --git a/contrib/puppet/templates/monitrc-nova-api.erb b/contrib/puppet/templates/monitrc-nova-api.erb deleted file mode 100644 index fe2626327..000000000 --- a/contrib/puppet/templates/monitrc-nova-api.erb +++ /dev/null @@ -1,138 +0,0 @@ -############################################################################### -## Monit control file -############################################################################### -## -## Comments begin with a '#' and extend through the end of the line. Keywords -## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. -## -## Below you will find examples of some frequently used statements. For -## information about the control file, a complete list of statements and -## options please have a look in the monit manual. -## -## -############################################################################### -## Global section -############################################################################### -## -## Start monit in the background (run as a daemon): -# -set daemon 60 # check services at 1-minute intervals - with start delay 30 # optional: delay the first check by half a minute - # (by default check immediately after monit start) - - -## Set syslog logging with the 'daemon' facility. If the FACILITY option is -## omitted, monit will use 'user' facility by default. If you want to log to -## a stand alone log file instead, specify the path to a log file -# -set logfile syslog facility log_daemon -# -# -### Set the location of monit id file which saves the unique id specific for -### given monit. The id is generated and stored on first monit start. -### By default the file is placed in $HOME/.monit.id. -# -# set idfile /var/.monit.id -# -### Set the location of monit state file which saves the monitoring state -### on each cycle. By default the file is placed in $HOME/.monit.state. If -### state file is stored on persistent filesystem, monit will recover the -### monitoring state across reboots. If it is on temporary filesystem, the -### state will be lost on reboot. -# -# set statefile /var/.monit.state -# -## Set the list of mail servers for alert delivery. Multiple servers may be -## specified using comma separator. By default monit uses port 25 - this -## is possible to override with the PORT option. -# -# set mailserver mail.bar.baz, # primary mailserver -# backup.bar.baz port 10025, # backup mailserver on port 10025 -# localhost # fallback relay -# -# -## By default monit will drop alert events if no mail servers are available. -## If you want to keep the alerts for a later delivery retry, you can use the -## EVENTQUEUE statement. The base directory where undelivered alerts will be -## stored is specified by the BASEDIR option. You can limit the maximal queue -## size using the SLOTS option (if omitted, the queue is limited by space -## available in the back end filesystem). -# -# set eventqueue -# basedir /var/monit # set the base directory where events will be stored -# slots 100 # optionaly limit the queue size -# -# -## Send status and events to M/Monit (Monit central management: for more -## informations about M/Monit see http://www.tildeslash.com/mmonit). -# -# set mmonit http://monit:monit@192.168.1.10:8080/collector -# -# -## Monit by default uses the following alert mail format: -## -## --8<-- -## From: monit@$HOST # sender -## Subject: monit alert -- $EVENT $SERVICE # subject -## -## $EVENT Service $SERVICE # -## # -## Date: $DATE # -## Action: $ACTION # -## Host: $HOST # body -## Description: $DESCRIPTION # -## # -## Your faithful employee, # -## monit # -## --8<-- -## -## You can override this message format or parts of it, such as subject -## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. -## are expanded at runtime. For example, to override the sender: -# -# set mail-format { from: monit@foo.bar } -# -# -## You can set alert recipients here whom will receive alerts if/when a -## service defined in this file has errors. Alerts may be restricted on -## events by using a filter as in the second example below. -# -# set alert sysadm@foo.bar # receive all alerts -# set alert manager@foo.bar only on { timeout } # receive just service- -# # timeout alert -# -# -## Monit has an embedded web server which can be used to view status of -## services monitored, the current configuration, actual services parameters -## and manage services from a web interface. -# - set httpd port 2812 and - use address localhost # only accept connection from localhost - allow localhost # allow localhost to connect to the server and -# allow admin:monit # require user 'admin' with password 'monit' -# allow @monit # allow users of group 'monit' to connect (rw) -# allow @users readonly # allow users of group 'users' to connect readonly -# -# -############################################################################### -## Services -############################################################################### - -<% api_servers.to_i.times do |offset| %><% port = api_base_port.to_i + offset %> - -check process nova_api_<%= port %> with pidfile /var/run/nova/nova-api-<%= port %>.pid - group nova_api - start program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock start" - as uid nova - stop program = "/usr/bin/nova-api --flagfile=/etc/nova/nova.conf --pidfile=/var/run/nova/nova-api-<%= port %>.pid --api_listen_port=<%= port %> --lockfile=/var/run/nova/nova-api-<%= port %>.pid.lock stop" - as uid nova - if failed port <%= port %> protocol http - with timeout 15 seconds - for 4 cycles - then restart - if totalmem > 300 Mb then restart - if cpu is greater than 60% for 2 cycles then alert - if cpu > 80% for 3 cycles then restart - if 3 restarts within 5 cycles then timeout - -<% end %> diff --git a/contrib/puppet/templates/nova-iptables.erb b/contrib/puppet/templates/nova-iptables.erb deleted file mode 100644 index 2fc066305..000000000 --- a/contrib/puppet/templates/nova-iptables.erb +++ /dev/null @@ -1,10 +0,0 @@ -<% services.each do |service| -%> -<%= service.upcase %>=1 -<% end -%> -<% if ip && ip != "" %>IP="<%=ip%>"<% end %> -<% if private_range && private_range != "" %>PRIVATE_RANGE="<%=private_range%>"<% end %> -<% if mgmt_ip && mgmt_ip != "" %>MGMT_IP="<%=mgmt_ip%>"<% end %> -<% if dmz_ip && dmz_ip != "" %>DMZ_IP="<%=dmz_ip%>"<% end %> - -# warning: this file is auto-generated by puppet - diff --git a/contrib/puppet/templates/production/nova-common.conf.erb b/contrib/puppet/templates/production/nova-common.conf.erb deleted file mode 100644 index 23ee0c5e8..000000000 --- a/contrib/puppet/templates/production/nova-common.conf.erb +++ /dev/null @@ -1,55 +0,0 @@ -# global ---dmz_net=192.168.0.0 ---dmz_mask=255.255.0.0 ---dmz_cidr=192.168.0.0/16 ---ldap_user_dn=cn=Administrators,dc=example,dc=com ---ldap_user_unit=Users ---ldap_user_subtree=ou=Users,dc=example,dc=com ---ldap_project_subtree=ou=Groups,dc=example,dc=com ---role_project_subtree=ou=Groups,dc=example,dc=com ---ldap_cloudadmin=cn=NovaAdmins,ou=Groups,dc=example,dc=com ---ldap_itsec=cn=NovaSecurity,ou=Groups,dc=example,dc=com ---ldap_sysadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_netadmin=cn=Administrators,ou=Groups,dc=example,dc=com ---ldap_developer=cn=developers,ou=Groups,dc=example,dc=com ---verbose ---daemonize ---syslog ---networks_path=/var/lib/nova/networks ---instances_path=/var/lib/nova/instances ---buckets_path=/var/lib/nova/objectstore/buckets ---images_path=/var/lib/nova/objectstore/images ---scheduler_driver=nova.scheduler.simple.SimpleScheduler ---libvirt_xml_template=/usr/share/nova/libvirt.qemu.xml.template ---credentials_template=/usr/share/nova/novarc.template ---boot_script_template=/usr/share/nova/bootscript.template ---vpn_client_template=/usr/share/nova/client.ovpn.template ---max_cores=40 ---max_gigabytes=2000 ---ca_path=/var/lib/nova/CA ---keys_path=/var/lib/nova/keys ---vpn_start=11000 ---volume_group=vgdata ---volume_manager=nova.volume.manager.ISCSIManager ---volume_driver=nova.volume.driver.ISCSIDriver ---default_kernel=aki-DEFAULT ---default_ramdisk=ari-DEFAULT ---dhcpbridge=/usr/bin/nova-dhcpbridge ---vpn_image_id=ami-cloudpipe ---dhcpbridge_flagfile=/etc/nova/nova.conf ---credential_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=NOVA/CN=%s-%s ---auth_driver=nova.auth.ldapdriver.LdapDriver ---quota_cores=17 ---quota_floating_ips=5 ---quota_instances=6 ---quota_volumes=10 ---quota_gigabytes=100 ---use_nova_chains=True ---input_chain=services ---use_project_ca=True ---fixed_ip_disassociate_timeout=300 ---api_max_requests=1 ---api_listen_ip=127.0.0.1 ---user_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=%s-%s-%s ---project_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-ca-%s-%s ---vpn_cert_subject=/C=US/ST=Texas/L=Bexar/O=NovaDev/OU=Nova/CN=project-vpn-%s-%s diff --git a/contrib/puppet/templates/production/nova-nova.conf.erb b/contrib/puppet/templates/production/nova-nova.conf.erb deleted file mode 100644 index 8683fefde..000000000 --- a/contrib/puppet/templates/production/nova-nova.conf.erb +++ /dev/null @@ -1,21 +0,0 @@ ---fixed_range=192.168.0.0/16 ---iscsi_ip_prefix=192.168.4 ---floating_range=10.0.0.0/24 ---rabbit_host=192.168.0.10 ---s3_host=192.168.0.10 ---cc_host=192.168.0.10 ---cc_dmz=192.168.24.10 ---s3_dmz=192.168.24.10 ---ec2_url=http://192.168.0.1:8773/services/Cloud ---vpn_ip=192.168.0.2 ---ldap_url=ldap://192.168.0.10 ---sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---other_sql_connection=mysql://nova:TODO-MYPASS@192.168.0.10/nova ---routing_source_ip=192.168.0.2 ---bridge_dev=eth1 ---public_interface=eth0 ---vlan_start=3100 ---num_networks=700 ---rabbit_userid=TODO:RABBIT ---rabbit_password=TODO:CHANGEME ---ldap_password=TODO:CHANGEME -- cgit From 106eb47eff0551c73b2e90385b9c3a910fec8633 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 20 Feb 2011 23:16:10 -0800 Subject: fixes for various logging errors and issues --- nova/api/openstack/auth.py | 1 - nova/api/openstack/backup_schedules.py | 1 - nova/api/openstack/images.py | 2 - nova/api/openstack/servers.py | 1 - nova/api/openstack/shared_ip_groups.py | 2 - nova/api/openstack/zones.py | 1 - nova/console/manager.py | 2 +- nova/console/xvp.py | 2 +- nova/flags.py | 4 ++ nova/log.py | 117 ++++++++++++++++++++------------- nova/test.py | 2 + nova/tests/fake_flags.py | 2 + nova/tests/test_auth.py | 9 --- nova/tests/test_console.py | 2 - nova/tests/test_direct.py | 1 - nova/tests/test_localization.py | 1 - nova/tests/test_log.py | 65 +++++++++--------- nova/utils.py | 2 +- run_tests.py | 5 +- 19 files changed, 117 insertions(+), 105 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 473071738..1dfdd5318 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -19,7 +19,6 @@ import datetime import hashlib import json import time -import logging import webob.exc import webob.dec diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index 197125d86..7abb5f884 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -15,7 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging import time from webob import exc diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 9d56bc508..cf85a496f 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -15,8 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from webob import exc from nova import compute diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ce9601ecb..0bac4c64d 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -33,7 +33,6 @@ import nova.api.openstack LOG = logging.getLogger('server') -LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index bd3cc23a8..5d78f9377 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -15,8 +15,6 @@ # License for the specific language governing permissions and limitations # under the License. -import logging - from webob import exc from nova import wsgi diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 830464ffd..d5206da20 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -14,7 +14,6 @@ # under the License. import common -import logging from nova import flags from nova import wsgi diff --git a/nova/console/manager.py b/nova/console/manager.py index 5697e7cb1..57c75cf4f 100644 --- a/nova/console/manager.py +++ b/nova/console/manager.py @@ -20,11 +20,11 @@ Console Proxy Service """ import functools -import logging import socket from nova import exception from nova import flags +from nova import log as logging from nova import manager from nova import rpc from nova import utils diff --git a/nova/console/xvp.py b/nova/console/xvp.py index ee66dac46..cd257e0a6 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -20,7 +20,6 @@ XVP (Xenserver VNC Proxy) driver. """ import fcntl -import logging import os import signal import subprocess @@ -31,6 +30,7 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import log as logging from nova import utils flags.DEFINE_string('console_xvp_conf_template', diff --git a/nova/flags.py b/nova/flags.py index f64a62da9..2f3bdd675 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -38,6 +38,7 @@ class FlagValues(gflags.FlagValues): defined after the initial parsing. """ + initialized = False def __init__(self, extra_context=None): gflags.FlagValues.__init__(self) @@ -45,6 +46,8 @@ class FlagValues(gflags.FlagValues): self.__dict__['__was_already_parsed'] = False self.__dict__['__stored_argv'] = [] self.__dict__['__extra_context'] = extra_context + # NOTE(vish): force a pseudo flag to keep track of whether + # flags have been parsed already def __call__(self, argv): # We're doing some hacky stuff here so that we don't have to copy @@ -90,6 +93,7 @@ class FlagValues(gflags.FlagValues): self.__dict__['__stored_argv'] = original_argv self.__dict__['__was_already_parsed'] = True self.ClearDirty() + FlagValues.initialized = True return args def Reset(self): diff --git a/nova/log.py b/nova/log.py index 6b201ffcc..12b695a41 100644 --- a/nova/log.py +++ b/nova/log.py @@ -117,7 +117,22 @@ def _get_binary_name(): return os.path.basename(inspect.stack()[-1][1]) -def get_log_file_path(binary=None): +def _get_level_from_flags(name): + # if exactly "nova", or a child logger, honor the verbose flag + if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: + return 'DEBUG' + for pair in FLAGS.default_log_levels: + logger, _sep, level = pair.partition('=') + # NOTE(todd): if we set a.b, we want a.b.c to have the same level + # (but not a.bc, so we check the dot) + if name == logger: + return level + if name.startswith(logger) and name[len(logger)] == '.': + return level + return 'INFO' + + +def _get_log_file_path(binary=None): if FLAGS.logfile: return FLAGS.logfile if FLAGS.logdir: @@ -126,22 +141,13 @@ def get_log_file_path(binary=None): def basicConfig(): - logging.basicConfig() - for handler in logging.root.handlers: - handler.setFormatter(_formatter) - if FLAGS.verbose: - logging.root.setLevel(logging.DEBUG) - else: - logging.root.setLevel(logging.INFO) - if FLAGS.use_syslog: - syslog = SysLogHandler(address='/dev/log') - syslog.setFormatter(_formatter) - logging.root.addHandler(syslog) - logpath = get_log_file_path() - if logpath: - logfile = WatchedFileHandler(logpath) - logfile.setFormatter(_formatter) - logging.root.addHandler(logfile) + pass + + +logging.basicConfig = basicConfig +_syslog = SysLogHandler(address='/dev/log') +_filelog = None +_streamlog = StreamHandler() class NovaLogger(logging.Logger): @@ -151,23 +157,24 @@ class NovaLogger(logging.Logger): This becomes the class that is instanciated by logging.getLogger. """ def __init__(self, name, level=NOTSET): - level_name = self._get_level_from_flags(name, FLAGS) - level = globals()[level_name] logging.Logger.__init__(self, name, level) - - def _get_level_from_flags(self, name, FLAGS): - # if exactly "nova", or a child logger, honor the verbose flag - if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: - return 'DEBUG' - for pair in FLAGS.default_log_levels: - logger, _sep, level = pair.partition('=') - # NOTE(todd): if we set a.b, we want a.b.c to have the same level - # (but not a.bc, so we check the dot) - if name == logger: - return level - if name.startswith(logger) and name[len(logger)] == '.': - return level - return 'INFO' + self.initialized = False + if flags.FlagValues.initialized: + self._setup_from_flags() + + def _setup_from_flags(self): + """Setup logger from flags""" + level_name = _get_level_from_flags(self.name) + self.setLevel(globals()[level_name]) + self.initialized = True + if not logging.root.initialized: + logging.root._setup_from_flags() + + def isEnabledFor(self, level): + """Reset level after flags have been loaded""" + if not self.initialized and flags.FlagValues.initialized: + self._setup_from_flags() + return logging.Logger.isEnabledFor(self, level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" @@ -176,12 +183,12 @@ class NovaLogger(logging.Logger): if context: extra.update(_dictify_context(context)) extra.update({"nova_version": version.version_string_with_vcs()}) - logging.Logger._log(self, level, msg, args, exc_info, extra) + return logging.Logger._log(self, level, msg, args, exc_info, extra) def addHandler(self, handler): """Each handler gets our custom formatter""" handler.setFormatter(_formatter) - logging.Logger.addHandler(self, handler) + return logging.Logger.addHandler(self, handler) def audit(self, msg, *args, **kwargs): """Shortcut for our AUDIT level""" @@ -192,7 +199,7 @@ class NovaLogger(logging.Logger): """Logging.exception doesn't handle kwargs, so breaks context""" if not kwargs.get('exc_info'): kwargs['exc_info'] = 1 - self.error(msg, *args, **kwargs) + return self.error(msg, *args, **kwargs) # NOTE(todd): does this really go here, or in _log ? extra = kwargs.get('extra') if not extra: @@ -209,6 +216,8 @@ class NovaLogger(logging.Logger): def handle_exception(type, value, tb): + if len(logging.root.handlers) == 0: + logging.root.addHandler(_streamlog) logging.root.critical(str(value), exc_info=(type, value, tb)) @@ -216,15 +225,6 @@ sys.excepthook = handle_exception logging.setLoggerClass(NovaLogger) -class NovaRootLogger(NovaLogger): - pass - -if not isinstance(logging.root, NovaRootLogger): - logging.root = NovaRootLogger("nova.root", WARNING) - NovaLogger.root = logging.root - NovaLogger.manager.root = logging.root - - class NovaFormatter(logging.Formatter): """ A nova.context.RequestContext aware formatter configured through flags. @@ -271,6 +271,33 @@ class NovaFormatter(logging.Formatter): _formatter = NovaFormatter() +class NovaRootLogger(NovaLogger): + def __init__(self, name, level=NOTSET): + NovaLogger.__init__(self, name, level) + self.addHandler(_streamlog) + + def _setup_from_flags(self): + """Setup logger from flags""" + global _filelog + if FLAGS.use_syslog: + self.addHandler(_syslog) + logpath = _get_log_file_path() + if logpath: + if not _filelog: + _filelog = WatchedFileHandler(logpath) + self.addHandler(_filelog) + self.removeHandler(_streamlog) + return NovaLogger._setup_from_flags(self) + + +if not isinstance(logging.root, NovaRootLogger): + for handler in logging.root.handlers: + logging.root.removeHandler(handler) + logging.root = NovaRootLogger("nova") + NovaLogger.root = logging.root + NovaLogger.manager.root = logging.root + + def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" if len(logging.root.handlers) == 0: diff --git a/nova/test.py b/nova/test.py index a12cf9d32..a649b4c5f 100644 --- a/nova/test.py +++ b/nova/test.py @@ -60,6 +60,8 @@ class TestCase(unittest.TestCase): def setUp(self): """Run before each test method to initialize test environment""" super(TestCase, self).setUp() + # NOTE(vish): pretend like we've loaded flags from command line + flags.FlagValues.initialized = True # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 1097488ec..68b14a46e 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -41,3 +41,5 @@ FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' FLAGS.use_ipv6 = True +FLAGS.logfile = 'run_tests.err' +flags.FlagValues.initialized = True diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 35ffffb67..2a7817032 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -327,15 +327,6 @@ class AuthManagerTestCase(object): class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' - def __init__(self, *args, **kwargs): - AuthManagerTestCase.__init__(self) - test.TestCase.__init__(self, *args, **kwargs) - import nova.auth.fakeldap as fakeldap - if FLAGS.flush_db: - LOG.info("Flushing datastore") - r = fakeldap.Store.instance() - r.flushdb() - class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 85bf94458..49ff24413 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -21,7 +21,6 @@ Tests For Console proxy. """ import datetime -import logging from nova import context from nova import db @@ -38,7 +37,6 @@ FLAGS = flags.FLAGS class ConsoleTestCase(test.TestCase): """Test case for console proxy""" def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) super(ConsoleTestCase, self).setUp() self.flags(console_driver='nova.console.fake.FakeConsoleProxy', stub_compute=True) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index 8a74b2296..7656f5396 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -19,7 +19,6 @@ """Tests for Direct API.""" import json -import logging import webob diff --git a/nova/tests/test_localization.py b/nova/tests/test_localization.py index 6992773f5..393d71038 100644 --- a/nova/tests/test_localization.py +++ b/nova/tests/test_localization.py @@ -15,7 +15,6 @@ # under the License. import glob -import logging import os import re import sys diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index c2c9d7772..ada8d0a56 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -1,9 +1,12 @@ import cStringIO from nova import context +from nova import flags from nova import log from nova import test +FLAGS = flags.FLAGS + def _fake_context(): return context.RequestContext(1, 1) @@ -14,15 +17,11 @@ class RootLoggerTestCase(test.TestCase): super(RootLoggerTestCase, self).setUp() self.log = log.logging.root - def tearDown(self): - super(RootLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_is_nova_instance(self): self.assert_(isinstance(self.log, log.NovaLogger)) - def test_name_is_nova_root(self): - self.assertEqual("nova.root", self.log.name) + def test_name_is_nova(self): + self.assertEqual("nova", self.log.name) def test_handlers_have_nova_formatter(self): formatters = [] @@ -45,25 +44,38 @@ class RootLoggerTestCase(test.TestCase): log.audit("foo", context=_fake_context()) self.assert_(True) # didn't raise exception + def test_will_be_verbose_if_verbose_flag_set(self): + self.flags(verbose=True) + self.log.initialized = False + log.audit("foo", context=_fake_context()) + self.assertEqual(log.DEBUG, self.log.level) + + def test_will_not_be_verbose_if_verbose_flag_not_set(self): + self.flags(verbose=False) + self.log.initialized = False + log.audit("foo", context=_fake_context()) + self.assertEqual(log.INFO, self.log.level) + class LogHandlerTestCase(test.TestCase): def test_log_path_logdir(self): - self.flags(logdir='/some/path') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.flags(logdir='/some/path', logfile=None) + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_logfile(self): self.flags(logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') def test_log_path_none(self): - self.assertTrue(log.get_log_file_path(binary='foo-bar') is None) + self.flags(logdir=None, logfile=None) + self.assertTrue(log._get_log_file_path(binary='foo-bar') is None) def test_log_path_logfile_overrides_logdir(self): self.flags(logdir='/some/other/path', logfile='/some/path/foo-bar.log') - self.assertEquals(log.get_log_file_path(binary='foo-bar'), + self.assertEquals(log._get_log_file_path(binary='foo-bar'), '/some/path/foo-bar.log') @@ -76,13 +88,15 @@ class NovaFormatterTestCase(test.TestCase): logging_debug_format_suffix="--DBG") self.log = log.logging.root self.stream = cStringIO.StringIO() - handler = log.StreamHandler(self.stream) - self.log.addHandler(handler) + self.handler = log.StreamHandler(self.stream) + self.log.addHandler(self.handler) + self.level = self.log.level self.log.setLevel(log.DEBUG) def tearDown(self): + self.log.setLevel(self.level) + self.log.removeHandler(self.handler) super(NovaFormatterTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} def test_uncontextualized_log(self): self.log.info("foo") @@ -102,30 +116,15 @@ class NovaFormatterTestCase(test.TestCase): class NovaLoggerTestCase(test.TestCase): def setUp(self): super(NovaLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova-test=AUDIT"], verbose=False) + levels = FLAGS.default_log_levels + levels.append("nova-test=AUDIT") + self.flags(default_log_levels=levels, + verbose=True) self.log = log.getLogger('nova-test') - def tearDown(self): - super(NovaLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - def test_has_level_from_flags(self): self.assertEqual(log.AUDIT, self.log.level) def test_child_log_has_level_of_parent_flag(self): l = log.getLogger('nova-test.foo') self.assertEqual(log.AUDIT, l.level) - - -class VerboseLoggerTestCase(test.TestCase): - def setUp(self): - super(VerboseLoggerTestCase, self).setUp() - self.flags(default_log_levels=["nova.test=AUDIT"], verbose=True) - self.log = log.getLogger('nova.test') - - def tearDown(self): - super(VerboseLoggerTestCase, self).tearDown() - log.NovaLogger.manager.loggerDict = {} - - def test_will_be_verbose_if_named_nova_and_verbose_flag_set(self): - self.assertEqual(log.DEBUG, self.log.level) diff --git a/nova/utils.py b/nova/utils.py index 42efa0008..c2fd5f2ee 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -55,7 +55,7 @@ def import_class(import_str): __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError), exc: - logging.debug(_('Inner Exception: %s'), exc) + LOG.info(_('Inner Exception: %s'), exc) raise exception.NotFound(_('Class %s cannot be found') % class_str) diff --git a/run_tests.py b/run_tests.py index 24786e8ad..bf12c62c6 100644 --- a/run_tests.py +++ b/run_tests.py @@ -26,8 +26,6 @@ from nose import config from nose import result from nose import core -from nova import log as logging - class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): @@ -60,7 +58,8 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - logging.basicConfig() + if os.path.exists("nova.sqlite"): + os.unlink("nova.sqlite") c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, -- cgit From 98c2ef77e6018c0a6233e32ace435fba488175f5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 20 Feb 2011 23:18:09 -0800 Subject: move the fake initialized into fake flags --- nova/test.py | 2 -- nova/tests/fake_flags.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/test.py b/nova/test.py index a649b4c5f..a12cf9d32 100644 --- a/nova/test.py +++ b/nova/test.py @@ -60,8 +60,6 @@ class TestCase(unittest.TestCase): def setUp(self): """Run before each test method to initialize test environment""" super(TestCase, self).setUp() - # NOTE(vish): pretend like we've loaded flags from command line - flags.FlagValues.initialized = True # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 68b14a46e..0a1011d86 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -42,4 +42,5 @@ FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' FLAGS.use_ipv6 = True FLAGS.logfile = 'run_tests.err' +# NOTE(vish): pretend like we've loaded flags from command line flags.FlagValues.initialized = True -- cgit From eaddb9ea529672549af28cab4aacd2f8243c62a3 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 20 Feb 2011 23:36:36 -0800 Subject: remove extra references to logging.basicConfig --- bin/nova-api | 4 ++-- bin/nova-combined | 2 -- bin/nova-dhcpbridge | 1 - bin/nova-manage | 2 -- nova/log.py | 19 +++++++++++++------ nova/service.py | 1 - nova/twistd.py | 1 - nova/wsgi.py | 1 - 8 files changed, 15 insertions(+), 16 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 11176a021..8b3674880 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -39,7 +39,6 @@ from nova import log as logging from nova import version from nova import wsgi -logging.basicConfig() LOG = logging.getLogger('nova.api') LOG.setLevel(logging.DEBUG) @@ -71,7 +70,8 @@ def run_app(paste_config_file): return # NOTE(todd): redo logging config, verbose could be set in paste config - logging.basicConfig() + logging.root.setup_from_flags() + server = wsgi.Server() for app in apps: server.start(*app) diff --git a/bin/nova-combined b/bin/nova-combined index 913c866bf..5911d9016 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -49,7 +49,6 @@ FLAGS = flags.FLAGS if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) - logging.basicConfig() compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') @@ -73,7 +72,6 @@ if __name__ == '__main__': apps.append((app, getattr(FLAGS, "%s_port" % api), getattr(FLAGS, "%s_host" % api))) if len(apps) > 0: - logging.basicConfig() server = wsgi.Server() for app in apps: server.start(*app) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index d38ba2543..e0e6af826 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -102,7 +102,6 @@ def main(): flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) - logging.basicConfig() interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True diff --git a/bin/nova-manage b/bin/nova-manage index 6d67252b8..878a9afaa 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -77,7 +77,6 @@ from nova import crypto from nova import db from nova import exception from nova import flags -from nova import log as logging from nova import quota from nova import rpc from nova import utils @@ -87,7 +86,6 @@ from nova.cloudpipe import pipelib from nova.db import migration -logging.basicConfig() FLAGS = flags.FLAGS flags.DECLARE('fixed_range', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') diff --git a/nova/log.py b/nova/log.py index 12b695a41..0a863c921 100644 --- a/nova/log.py +++ b/nova/log.py @@ -160,20 +160,20 @@ class NovaLogger(logging.Logger): logging.Logger.__init__(self, name, level) self.initialized = False if flags.FlagValues.initialized: - self._setup_from_flags() + self.setup_from_flags() - def _setup_from_flags(self): + def setup_from_flags(self): """Setup logger from flags""" level_name = _get_level_from_flags(self.name) self.setLevel(globals()[level_name]) self.initialized = True if not logging.root.initialized: - logging.root._setup_from_flags() + logging.root.setup_from_flags() def isEnabledFor(self, level): """Reset level after flags have been loaded""" if not self.initialized and flags.FlagValues.initialized: - self._setup_from_flags() + self.setup_from_flags() return logging.Logger.isEnabledFor(self, level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): @@ -276,18 +276,24 @@ class NovaRootLogger(NovaLogger): NovaLogger.__init__(self, name, level) self.addHandler(_streamlog) - def _setup_from_flags(self): + def setup_from_flags(self): """Setup logger from flags""" global _filelog if FLAGS.use_syslog: self.addHandler(_syslog) + else: + self.removeHandler(_syslog) logpath = _get_log_file_path() if logpath: if not _filelog: _filelog = WatchedFileHandler(logpath) self.addHandler(_filelog) self.removeHandler(_streamlog) - return NovaLogger._setup_from_flags(self) + else: + self.removeHandler(_filelog) + self.addHandler(_streamlog) + + return NovaLogger.setup_from_flags(self) if not isinstance(logging.root, NovaRootLogger): @@ -296,6 +302,7 @@ if not isinstance(logging.root, NovaRootLogger): logging.root = NovaRootLogger("nova") NovaLogger.root = logging.root NovaLogger.manager.root = logging.root +root=logging.root def audit(msg, *args, **kwargs): diff --git a/nova/service.py b/nova/service.py index 59648adf2..02e86f6b3 100644 --- a/nova/service.py +++ b/nova/service.py @@ -215,7 +215,6 @@ class Service(object): def serve(*services): FLAGS(sys.argv) - logging.basicConfig() if not services: services = [Service.create()] diff --git a/nova/twistd.py b/nova/twistd.py index 60ff7879a..0e4db022c 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -258,7 +258,6 @@ def serve(filename): print 'usage: %s [options] [start|stop|restart]' % argv[0] sys.exit(1) - logging.basicConfig() logging.debug(_("Full set of FLAGS:")) for flag in FLAGS: logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) diff --git a/nova/wsgi.py b/nova/wsgi.py index e01cc1e1e..280baa80b 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -59,7 +59,6 @@ class Server(object): """Server class to manage multiple WSGI sockets and applications.""" def __init__(self, threads=1000): - logging.basicConfig() self.pool = eventlet.GreenPool(threads) def start(self, application, port, host='0.0.0.0', backlog=128): -- cgit From 2792e42a9c7da390b3db0b59b7dff357c440d3e5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 20 Feb 2011 23:45:43 -0800 Subject: clean up location of method --- nova/log.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/nova/log.py b/nova/log.py index 0a863c921..57a550a0b 100644 --- a/nova/log.py +++ b/nova/log.py @@ -117,21 +117,6 @@ def _get_binary_name(): return os.path.basename(inspect.stack()[-1][1]) -def _get_level_from_flags(name): - # if exactly "nova", or a child logger, honor the verbose flag - if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: - return 'DEBUG' - for pair in FLAGS.default_log_levels: - logger, _sep, level = pair.partition('=') - # NOTE(todd): if we set a.b, we want a.b.c to have the same level - # (but not a.bc, so we check the dot) - if name == logger: - return level - if name.startswith(logger) and name[len(logger)] == '.': - return level - return 'INFO' - - def _get_log_file_path(binary=None): if FLAGS.logfile: return FLAGS.logfile @@ -162,9 +147,24 @@ class NovaLogger(logging.Logger): if flags.FlagValues.initialized: self.setup_from_flags() + @staticmethod + def _get_level_from_flags(name): + # if exactly "nova", or a child logger, honor the verbose flag + if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: + return 'DEBUG' + for pair in FLAGS.default_log_levels: + logger, _sep, level = pair.partition('=') + # NOTE(todd): if we set a.b, we want a.b.c to have the same level + # (but not a.bc, so we check the dot) + if name == logger: + return level + if name.startswith(logger) and name[len(logger)] == '.': + return level + return 'INFO' + def setup_from_flags(self): """Setup logger from flags""" - level_name = _get_level_from_flags(self.name) + level_name = self._get_level_from_flags(self.name) self.setLevel(globals()[level_name]) self.initialized = True if not logging.root.initialized: @@ -302,7 +302,7 @@ if not isinstance(logging.root, NovaRootLogger): logging.root = NovaRootLogger("nova") NovaLogger.root = logging.root NovaLogger.manager.root = logging.root -root=logging.root +root = logging.root def audit(msg, *args, **kwargs): -- cgit From 7eab72b30cad9708e976f60e121569972b835b61 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 00:15:49 -0800 Subject: get rid of initialized flag --- nova/flags.py | 4 ++-- nova/log.py | 16 ++++------------ nova/test.py | 3 +++ nova/tests/fake_flags.py | 2 -- nova/tests/test_log.py | 6 ++---- 5 files changed, 11 insertions(+), 20 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 2f3bdd675..72d123e21 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -29,7 +29,6 @@ import sys import gflags - class FlagValues(gflags.FlagValues): """Extension of gflags.FlagValues that allows undefined and runtime flags. @@ -93,7 +92,8 @@ class FlagValues(gflags.FlagValues): self.__dict__['__stored_argv'] = original_argv self.__dict__['__was_already_parsed'] = True self.ClearDirty() - FlagValues.initialized = True + from nova import log as logging + logging.reset() return args def Reset(self): diff --git a/nova/log.py b/nova/log.py index 57a550a0b..8d240782d 100644 --- a/nova/log.py +++ b/nova/log.py @@ -143,9 +143,7 @@ class NovaLogger(logging.Logger): """ def __init__(self, name, level=NOTSET): logging.Logger.__init__(self, name, level) - self.initialized = False - if flags.FlagValues.initialized: - self.setup_from_flags() + self.setup_from_flags() @staticmethod def _get_level_from_flags(name): @@ -166,15 +164,6 @@ class NovaLogger(logging.Logger): """Setup logger from flags""" level_name = self._get_level_from_flags(self.name) self.setLevel(globals()[level_name]) - self.initialized = True - if not logging.root.initialized: - logging.root.setup_from_flags() - - def isEnabledFor(self, level): - """Reset level after flags have been loaded""" - if not self.initialized and flags.FlagValues.initialized: - self.setup_from_flags() - return logging.Logger.isEnabledFor(self, level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" @@ -304,6 +293,9 @@ if not isinstance(logging.root, NovaRootLogger): NovaLogger.manager.root = logging.root root = logging.root +def reset(): + root.setup_from_flags() + def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" diff --git a/nova/test.py b/nova/test.py index a12cf9d32..8022396cd 100644 --- a/nova/test.py +++ b/nova/test.py @@ -32,9 +32,12 @@ from nova import context from nova import db from nova import fakerabbit from nova import flags +from nova import log as logging from nova import rpc from nova.network import manager as network_manager + from nova.tests import fake_flags +logging.reset() FLAGS = flags.FLAGS diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 0a1011d86..59839b090 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -42,5 +42,3 @@ FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' FLAGS.use_ipv6 = True FLAGS.logfile = 'run_tests.err' -# NOTE(vish): pretend like we've loaded flags from command line -flags.FlagValues.initialized = True diff --git a/nova/tests/test_log.py b/nova/tests/test_log.py index ada8d0a56..122351ff6 100644 --- a/nova/tests/test_log.py +++ b/nova/tests/test_log.py @@ -46,14 +46,12 @@ class RootLoggerTestCase(test.TestCase): def test_will_be_verbose_if_verbose_flag_set(self): self.flags(verbose=True) - self.log.initialized = False - log.audit("foo", context=_fake_context()) + log.reset() self.assertEqual(log.DEBUG, self.log.level) def test_will_not_be_verbose_if_verbose_flag_not_set(self): self.flags(verbose=False) - self.log.initialized = False - log.audit("foo", context=_fake_context()) + log.reset() self.assertEqual(log.INFO, self.log.level) -- cgit From 86b202f7397b80358346e1b2a9894af81faa4f4b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 00:17:58 -0800 Subject: fix nova-api as well --- bin/nova-api | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-api b/bin/nova-api index 8b3674880..5937f7744 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -70,7 +70,7 @@ def run_app(paste_config_file): return # NOTE(todd): redo logging config, verbose could be set in paste config - logging.root.setup_from_flags() + logging.reset() server = wsgi.Server() for app in apps: -- cgit From f9af5309cf50b3b1a4ef9799c071cbaa6b1b304f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 00:22:45 -0800 Subject: removed extra comments and initialized from flags --- nova/flags.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index 72d123e21..e2f7960ec 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -37,7 +37,6 @@ class FlagValues(gflags.FlagValues): defined after the initial parsing. """ - initialized = False def __init__(self, extra_context=None): gflags.FlagValues.__init__(self) @@ -45,8 +44,6 @@ class FlagValues(gflags.FlagValues): self.__dict__['__was_already_parsed'] = False self.__dict__['__stored_argv'] = [] self.__dict__['__extra_context'] = extra_context - # NOTE(vish): force a pseudo flag to keep track of whether - # flags have been parsed already def __call__(self, argv): # We're doing some hacky stuff here so that we don't have to copy -- cgit From f28ed7d95afd17e55e1db25a75e065f9da0f06e6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 00:24:35 -0800 Subject: add docstring to reset method --- nova/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/log.py b/nova/log.py index 8d240782d..94eeecce0 100644 --- a/nova/log.py +++ b/nova/log.py @@ -294,6 +294,7 @@ if not isinstance(logging.root, NovaRootLogger): root = logging.root def reset(): + """Resets logging handlers. Should be called if FLAGS changes.""" root.setup_from_flags() -- cgit From bfba5b2cf8ade746d74485bd76f9d60238ccb2ea Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 00:48:33 -0800 Subject: reset all loggers on flag change, not just root --- nova/log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/log.py b/nova/log.py index 94eeecce0..2b43f7311 100644 --- a/nova/log.py +++ b/nova/log.py @@ -295,6 +295,9 @@ root = logging.root def reset(): """Resets logging handlers. Should be called if FLAGS changes.""" + for logger in logging.Logger.manager.loggerDict.itervalues(): + if isinstance(logger, NovaLogger): + logger.setup_from_flags() root.setup_from_flags() -- cgit From e773c16e1bce0e00b269394d1ed20d15884827ff Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 01:07:46 -0800 Subject: simplify logic for parsing log level flags --- nova/log.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/nova/log.py b/nova/log.py index 2b43f7311..0cd42d00e 100644 --- a/nova/log.py +++ b/nova/log.py @@ -145,25 +145,14 @@ class NovaLogger(logging.Logger): logging.Logger.__init__(self, name, level) self.setup_from_flags() - @staticmethod - def _get_level_from_flags(name): - # if exactly "nova", or a child logger, honor the verbose flag - if (name == "nova" or name.startswith("nova.")) and FLAGS.verbose: - return 'DEBUG' + def setup_from_flags(self): + """Setup logger from flags""" for pair in FLAGS.default_log_levels: logger, _sep, level = pair.partition('=') # NOTE(todd): if we set a.b, we want a.b.c to have the same level # (but not a.bc, so we check the dot) - if name == logger: - return level - if name.startswith(logger) and name[len(logger)] == '.': - return level - return 'INFO' - - def setup_from_flags(self): - """Setup logger from flags""" - level_name = self._get_level_from_flags(self.name) - self.setLevel(globals()[level_name]) + if self.name == logger or self.name.startswith("%s." % logger): + self.setLevel(globals()[level]) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" @@ -281,8 +270,10 @@ class NovaRootLogger(NovaLogger): else: self.removeHandler(_filelog) self.addHandler(_streamlog) - - return NovaLogger.setup_from_flags(self) + if FLAGS.verbose: + self.setLevel(DEBUG) + else: + self.setLevel(INFO) if not isinstance(logging.root, NovaRootLogger): -- cgit From 753d3a6915ad8387ea29ad1a7fb4aed74c4b71fd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 01:26:15 -0800 Subject: move exception hook into appropriate location and remove extra stuff from module namespace --- bin/nova-manage | 1 - nova/log.py | 45 +++++++++++++++++++++++---------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 878a9afaa..861798717 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -85,7 +85,6 @@ from nova.auth import manager from nova.cloudpipe import pipelib from nova.db import migration - FLAGS = flags.FLAGS flags.DECLARE('fixed_range', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') diff --git a/nova/log.py b/nova/log.py index 0cd42d00e..5ffb52cde 100644 --- a/nova/log.py +++ b/nova/log.py @@ -130,9 +130,6 @@ def basicConfig(): logging.basicConfig = basicConfig -_syslog = SysLogHandler(address='/dev/log') -_filelog = None -_streamlog = StreamHandler() class NovaLogger(logging.Logger): @@ -193,16 +190,6 @@ class NovaLogger(logging.Logger): self.error(message, **kwargs) -def handle_exception(type, value, tb): - if len(logging.root.handlers) == 0: - logging.root.addHandler(_streamlog) - logging.root.critical(str(value), exc_info=(type, value, tb)) - - -sys.excepthook = handle_exception -logging.setLoggerClass(NovaLogger) - - class NovaFormatter(logging.Formatter): """ A nova.context.RequestContext aware formatter configured through flags. @@ -251,25 +238,30 @@ _formatter = NovaFormatter() class NovaRootLogger(NovaLogger): def __init__(self, name, level=NOTSET): + self.logpath = None + self.filelog = None + self.syslog = SysLogHandler(address='/dev/log') + self.streamlog = StreamHandler() NovaLogger.__init__(self, name, level) - self.addHandler(_streamlog) def setup_from_flags(self): """Setup logger from flags""" global _filelog if FLAGS.use_syslog: - self.addHandler(_syslog) + self.addHandler(self.syslog) else: - self.removeHandler(_syslog) + self.removeHandler(self.syslog) logpath = _get_log_file_path() if logpath: - if not _filelog: - _filelog = WatchedFileHandler(logpath) - self.addHandler(_filelog) - self.removeHandler(_streamlog) + self.removeHandler(self.streamlog) + if logpath != self.logpath: + self.removeHandler(self.filelog) + self.filelog = WatchedFileHandler(logpath) + self.addHandler(self.filelog) + self.logpath = logpath else: - self.removeHandler(_filelog) - self.addHandler(_streamlog) + self.removeHandler(self.filelog) + self.addHandler(self.streamlog) if FLAGS.verbose: self.setLevel(DEBUG) else: @@ -284,6 +276,15 @@ if not isinstance(logging.root, NovaRootLogger): NovaLogger.manager.root = logging.root root = logging.root + +def handle_exception(type, value, tb): + root.critical(str(value), exc_info=(type, value, tb)) + + +sys.excepthook = handle_exception +logging.setLoggerClass(NovaLogger) + + def reset(): """Resets logging handlers. Should be called if FLAGS changes.""" for logger in logging.Logger.manager.loggerDict.itervalues(): -- cgit From 5e02518d8ae5e7f57033e28ae85067d28c1ff3d8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 10:52:28 -0800 Subject: reset to notset if level isn't in flags --- nova/log.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/log.py b/nova/log.py index 5ffb52cde..61a279882 100644 --- a/nova/log.py +++ b/nova/log.py @@ -144,12 +144,14 @@ class NovaLogger(logging.Logger): def setup_from_flags(self): """Setup logger from flags""" + level = NOTSET for pair in FLAGS.default_log_levels: - logger, _sep, level = pair.partition('=') + logger, _sep, level_name = pair.partition('=') # NOTE(todd): if we set a.b, we want a.b.c to have the same level # (but not a.bc, so we check the dot) if self.name == logger or self.name.startswith("%s." % logger): - self.setLevel(globals()[level]) + level = globals()[level_name] + self.setLevel(level) def _log(self, level, msg, args, exc_info=None, extra=None, context=None): """Extract context from any log call""" -- cgit From 8388144744849265b46d26735da01a11e35990b0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 11:07:50 -0800 Subject: cleanup from review --- nova/log.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/nova/log.py b/nova/log.py index 61a279882..3a48c97ff 100644 --- a/nova/log.py +++ b/nova/log.py @@ -125,13 +125,6 @@ def _get_log_file_path(binary=None): return '%s.log' % (os.path.join(FLAGS.logdir, binary),) -def basicConfig(): - pass - - -logging.basicConfig = basicConfig - - class NovaLogger(logging.Logger): """ NovaLogger manages request context and formatting. @@ -176,7 +169,7 @@ class NovaLogger(logging.Logger): """Logging.exception doesn't handle kwargs, so breaks context""" if not kwargs.get('exc_info'): kwargs['exc_info'] = 1 - return self.error(msg, *args, **kwargs) + self.error(msg, *args, **kwargs) # NOTE(todd): does this really go here, or in _log ? extra = kwargs.get('extra') if not extra: @@ -271,11 +264,16 @@ class NovaRootLogger(NovaLogger): if not isinstance(logging.root, NovaRootLogger): + logging._acquireLock() for handler in logging.root.handlers: logging.root.removeHandler(handler) logging.root = NovaRootLogger("nova") + for logger in NovaLogger.manager.loggerDict.itervalues(): + logger.root = logging.root NovaLogger.root = logging.root NovaLogger.manager.root = logging.root + NovaLogger.manager.loggerDict["nova"] = logging.root + logging._releaseLock() root = logging.root @@ -289,14 +287,11 @@ logging.setLoggerClass(NovaLogger) def reset(): """Resets logging handlers. Should be called if FLAGS changes.""" - for logger in logging.Logger.manager.loggerDict.itervalues(): + for logger in NovaLogger.manager.loggerDict.itervalues(): if isinstance(logger, NovaLogger): logger.setup_from_flags() - root.setup_from_flags() def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" - if len(logging.root.handlers) == 0: - basicConfig() logging.root.log(AUDIT, msg, *args, **kwargs) -- cgit From c7d83e26f7d6388857b4db4538602395b688aa7a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 11:42:46 -0800 Subject: use tests.sqlite so it doesn't conflict with running db --- nova/tests/fake_flags.py | 4 ++-- run_tests.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 59839b090..575fefff6 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,6 +39,6 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True -FLAGS.logfile = 'run_tests.err' +FLAGS.logfile = 'tests.log' diff --git a/run_tests.py b/run_tests.py index bf12c62c6..274dc4a97 100644 --- a/run_tests.py +++ b/run_tests.py @@ -58,8 +58,8 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - if os.path.exists("nova.sqlite"): - os.unlink("nova.sqlite") + if os.path.exists("tests.sqlite"): + os.unlink("tests.sqlite") c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, -- cgit From 02e196192ea1f8be22c31828266b177d14d123cd Mon Sep 17 00:00:00 2001 From: Ken Pepple Date: Mon, 21 Feb 2011 12:41:15 -0800 Subject: make sure that ec2 response times are xs:dateTime parsable --- nova/api/ec2/apirequest.py | 8 +++++++- nova/tests/test_api.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 00b527d62..2b1acba5a 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -46,6 +46,11 @@ def _underscore_to_xmlcase(str): return res[:1].lower() + res[1:] +def _database_to_isoformat(datetimeobj): + """Return a xs:dateTime parsable string from datatime""" + return datetimeobj.strftime("%Y-%m-%dT%H:%M:%SZ") + + def _try_convert(value): """Return a non-string if possible""" if value == 'None': @@ -173,7 +178,8 @@ class APIRequest(object): elif isinstance(data, bool): data_el.appendChild(xml.createTextNode(str(data).lower())) elif isinstance(data, datetime.datetime): - data_el.appendChild(xml.createTextNode(data.isoformat())) + data_el.appendChild( + xml.createTextNode(_database_to_isoformat(data))) elif data != None: data_el.appendChild(xml.createTextNode(str(data))) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index fa27825cd..d5c54a1c3 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -20,6 +20,7 @@ import boto from boto.ec2 import regioninfo +import datetime import httplib import random import StringIO @@ -127,6 +128,28 @@ class ApiEc2TestCase(test.TestCase): self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) return self.http + def test_return_valid_isoformat(self): + """ + Ensure that the ec2 api returns datetime in xs:dateTime + (which apparently isn't datetime.isoformat()) + NOTE(ken-pepple): https://bugs.launchpad.net/nova/+bug/721297 + """ + conv = apirequest._database_to_isoformat + # sqlite database representation with microseconds + time_to_convert = datetime.datetime.strptime( + "2011-02-21 20:14:10.634276", + "%Y-%m-%d %H:%M:%S.%f") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T20:14:10Z') + # mysqlite database representation + time_to_convert = datetime.datetime.strptime( + "2011-02-21 19:56:18", + "%Y-%m-%d %H:%M:%S") + self.assertEqual( + conv(time_to_convert), + '2011-02-21T19:56:18Z') + def test_xmlns_version_matches_request_version(self): self.expect_http(api_version='2010-10-30') self.mox.ReplayAll() -- cgit From 29644fe5a9cf47ae33af31b848c0edc4567f3c09 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 13:46:41 -0800 Subject: switch to explicit call to logging.setup() --- bin/nova-ajax-console-proxy | 2 +- bin/nova-api | 2 +- bin/nova-combined | 1 + bin/nova-compute | 5 ++++ bin/nova-console | 4 ++++ bin/nova-dhcpbridge | 3 ++- bin/nova-direct-api | 2 ++ bin/nova-import-canonical-imagestore | 2 ++ bin/nova-instancemonitor | 3 --- bin/nova-manage | 2 ++ bin/nova-network | 4 ++++ bin/nova-scheduler | 4 ++++ bin/nova-volume | 4 ++++ nova/log.py | 44 ++++++++++++++++++++---------------- nova/service.py | 2 -- nova/twistd.py | 1 + nova/utils.py | 2 +- run_tests.py | 2 ++ 18 files changed, 61 insertions(+), 28 deletions(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 2bc407658..392b328b1 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -25,7 +25,6 @@ from eventlet.green import urllib2 import exceptions import gettext -import logging import os import sys import time @@ -130,6 +129,7 @@ class AjaxConsoleProxy(object): if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() server = wsgi.Server() acp = AjaxConsoleProxy() acp.register_listeners() diff --git a/bin/nova-api b/bin/nova-api index 5937f7744..61a4c7402 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -40,7 +40,6 @@ from nova import version from nova import wsgi LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS @@ -80,6 +79,7 @@ def run_app(paste_config_file): if __name__ == '__main__': FLAGS(sys.argv) + logging.setup() LOG.audit(_("Starting nova-api node (version %s)"), version.version_string_with_vcs()) conf = wsgi.paste_config_file('nova-api.conf') diff --git a/bin/nova-combined b/bin/nova-combined index 5911d9016..6ae8400d1 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -49,6 +49,7 @@ FLAGS = flags.FLAGS if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') diff --git a/bin/nova-compute b/bin/nova-compute index d2d352da2..e412598c2 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -36,10 +36,15 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() + service.serve() service.wait() diff --git a/bin/nova-console b/bin/nova-console index 802cc80b6..40608b995 100755 --- a/bin/nova-console +++ b/bin/nova-console @@ -35,10 +35,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index e0e6af826..eda2dc072 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -102,6 +102,7 @@ def main(): flagfile = os.environ.get('FLAGFILE', FLAGS.dhcpbridge_flagfile) utils.default_flagfile(flagfile) argv = FLAGS(sys.argv) + logging.setup() interface = os.environ.get('DNSMASQ_INTERFACE', 'br0') if int(os.environ.get('TESTING', '0')): FLAGS.fake_rabbit = True @@ -112,7 +113,7 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', - 'nova.sqlite')) + 'tests.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] if action in ['add', 'del', 'old']: diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 173b39bdb..6c63bd26b 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -35,6 +35,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova import wsgi from nova.api import direct @@ -48,6 +49,7 @@ flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.setup() direct.register_service('compute', compute_api.API()) direct.register_service('reflect', direct.Reflection()) diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 036b41e48..404ae37f4 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -41,6 +41,7 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) from nova import flags +from nova import log as logging from nova import utils from nova.objectstore import image @@ -92,6 +93,7 @@ def main(): """Main entry point.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() images = get_images() if len(argv) == 2: diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 7dca02014..24cc9fd23 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -41,9 +41,6 @@ from nova import utils from nova import twistd from nova.compute import monitor -# TODO(todd): shouldn't this be done with flags? And what about verbose? -logging.getLogger('boto').setLevel(logging.WARN) - LOG = logging.getLogger('nova.instancemonitor') diff --git a/bin/nova-manage b/bin/nova-manage index 861798717..5189de0e1 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -77,6 +77,7 @@ from nova import crypto from nova import db from nova import exception from nova import flags +from nova import log as logging from nova import quota from nova import rpc from nova import utils @@ -707,6 +708,7 @@ def main(): """Parse options and call the appropriate class/method.""" utils.default_flagfile() argv = FLAGS(sys.argv) + logging.setup() script_name = argv.pop(0) if len(argv) < 1: diff --git a/bin/nova-network b/bin/nova-network index 0143846a7..101761ef7 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-scheduler b/bin/nova-scheduler index f4c0eaed6..0c205a80f 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index ad3ddc405..8dcdbc500 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -36,10 +36,14 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) +from nova import flags +from nova import log as logging from nova import service from nova import utils if __name__ == '__main__': utils.default_flagfile() + flags.FLAGS(sys.argv) + logging.setup() service.serve() service.wait() diff --git a/nova/log.py b/nova/log.py index 3a48c97ff..10c14d74b 100644 --- a/nova/log.py +++ b/nova/log.py @@ -65,6 +65,7 @@ flags.DEFINE_string('logging_exception_prefix', flags.DEFINE_list('default_log_levels', ['amqplib=WARN', 'sqlalchemy=WARN', + 'boto=WARN', 'eventlet.wsgi.server=WARN'], 'list of logger=LEVEL pairs') @@ -263,26 +264,8 @@ class NovaRootLogger(NovaLogger): self.setLevel(INFO) -if not isinstance(logging.root, NovaRootLogger): - logging._acquireLock() - for handler in logging.root.handlers: - logging.root.removeHandler(handler) - logging.root = NovaRootLogger("nova") - for logger in NovaLogger.manager.loggerDict.itervalues(): - logger.root = logging.root - NovaLogger.root = logging.root - NovaLogger.manager.root = logging.root - NovaLogger.manager.loggerDict["nova"] = logging.root - logging._releaseLock() -root = logging.root - - def handle_exception(type, value, tb): - root.critical(str(value), exc_info=(type, value, tb)) - - -sys.excepthook = handle_exception -logging.setLoggerClass(NovaLogger) + logging.root.critical(str(value), exc_info=(type, value, tb)) def reset(): @@ -292,6 +275,29 @@ def reset(): logger.setup_from_flags() +def setup(): + """Setup nova logging.""" + if not isinstance(logging.root, NovaRootLogger): + logging._acquireLock() + for handler in logging.root.handlers: + logging.root.removeHandler(handler) + logging.root = NovaRootLogger("nova") + NovaLogger.root = logging.root + NovaLogger.manager.root = logging.root + for logger in NovaLogger.manager.loggerDict.itervalues(): + logger.root = logging.root + if isinstance(logger, logging.Logger): + NovaLogger.manager._fixupParents(logger) + NovaLogger.manager.loggerDict["nova"] = logging.root + logging._releaseLock() + sys.excepthook = handle_exception + reset() + + +root = logging.root +logging.setLoggerClass(NovaLogger) + + def audit(msg, *args, **kwargs): """Shortcut for logging to root log with sevrity 'AUDIT'.""" logging.root.log(AUDIT, msg, *args, **kwargs) diff --git a/nova/service.py b/nova/service.py index 02e86f6b3..ddb4d7791 100644 --- a/nova/service.py +++ b/nova/service.py @@ -214,8 +214,6 @@ class Service(object): def serve(*services): - FLAGS(sys.argv) - if not services: services = [Service.create()] diff --git a/nova/twistd.py b/nova/twistd.py index 0e4db022c..c07ed991f 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -148,6 +148,7 @@ def WrapTwistedOptions(wrapped): options.insert(0, '') args = FLAGS(options) + logging.setup() argv = args[1:] # ignore subcommands diff --git a/nova/utils.py b/nova/utils.py index c2fd5f2ee..2a3acf042 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -55,7 +55,7 @@ def import_class(import_str): __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError), exc: - LOG.info(_('Inner Exception: %s'), exc) + LOG.debug(_('Inner Exception: %s'), exc) raise exception.NotFound(_('Class %s cannot be found') % class_str) diff --git a/run_tests.py b/run_tests.py index 274dc4a97..82345433a 100644 --- a/run_tests.py +++ b/run_tests.py @@ -26,6 +26,7 @@ from nose import config from nose import result from nose import core +from nova import log as logging class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): @@ -60,6 +61,7 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': if os.path.exists("tests.sqlite"): os.unlink("tests.sqlite") + logging.setup() c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, -- cgit From dbb071c8424871b6985c6b470d9eff522cdda660 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 13:59:46 -0800 Subject: fix pep8 and remove extra reference to reset --- bin/nova-compute | 1 - nova/flags.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/nova-compute b/bin/nova-compute index e412598c2..95fa393b1 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -45,6 +45,5 @@ if __name__ == '__main__': utils.default_flagfile() flags.FLAGS(sys.argv) logging.setup() - service.serve() service.wait() diff --git a/nova/flags.py b/nova/flags.py index e2f7960ec..f64a62da9 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -29,6 +29,7 @@ import sys import gflags + class FlagValues(gflags.FlagValues): """Extension of gflags.FlagValues that allows undefined and runtime flags. @@ -89,8 +90,6 @@ class FlagValues(gflags.FlagValues): self.__dict__['__stored_argv'] = original_argv self.__dict__['__was_already_parsed'] = True self.ClearDirty() - from nova import log as logging - logging.reset() return args def Reset(self): -- cgit From f797d6c6464f8ee2816d56ee771ad718418def64 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 21 Feb 2011 15:52:41 -0800 Subject: Renamed db_update to model_update, and lots more documentation --- nova/volume/driver.py | 26 ++++++++++++++++++++++---- nova/volume/manager.py | 12 ++++++------ nova/volume/san.py | 18 +++++++++--------- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index f172e2fdc..687bc99d0 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -91,7 +91,8 @@ class VolumeDriver(object): % FLAGS.volume_group) def create_volume(self, volume): - """Creates a logical volume.""" + """Creates a logical volume. Can optionally return a Dictionary of + changes to the volume object to be persisted.""" if int(volume['size']) == 0: sizestr = '100M' else: @@ -126,7 +127,8 @@ class VolumeDriver(object): raise NotImplementedError() def create_export(self, context, volume): - """Exports the volume.""" + """Exports the volume. Can optionally return a Dictionary of changes + to the volume object to be persisted.""" raise NotImplementedError() def remove_export(self, context, volume): @@ -225,7 +227,14 @@ class FakeAOEDriver(AOEDriver): class ISCSIDriver(VolumeDriver): - """Executes commands relating to ISCSI volumes.""" + """Executes commands relating to ISCSI volumes. We make use of model + provider properties as follows: + provider_location - if present, contains the iSCSI target information + in the same format as an ietadm discovery + i.e. ', ' + provider_auth - if present, contains a space-separated triple: + ' '. CHAP is the only + auth_method in use at the moment.""" def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" @@ -313,7 +322,16 @@ class ISCSIDriver(VolumeDriver): def _get_iscsi_properties(self, volume): """Gets iscsi configuration, ideally from saved information in the - volume entity, but falling back to discovery if need be.""" + volume entity, but falling back to discovery if need be. The + properties are: + target_discovered - boolean indicating whether discovery was used, + target_iqn - the IQN of the iSCSI target, + target_portal - the portal of the iSCSI target, + and auth_method, auth_username and auth_password + - the authentication details. Right now, either + auth_method is not present meaning no authentication, or + auth_method == 'CHAP' meaning use CHAP with the specified + credentials.""" properties = {} diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 7193ece14..3e8bc16b3 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -107,14 +107,14 @@ class VolumeManager(manager.Manager): vol_size = volume_ref['size'] LOG.debug(_("volume %(vol_name)s: creating lv of" " size %(vol_size)sG") % locals()) - db_update = self.driver.create_volume(volume_ref) - if db_update: - self.db.volume_update(context, volume_ref['id'], db_update) + model_update = self.driver.create_volume(volume_ref) + if model_update: + self.db.volume_update(context, volume_ref['id'], model_update) LOG.debug(_("volume %s: creating export"), volume_ref['name']) - db_update = self.driver.create_export(context, volume_ref) - if db_update: - self.db.volume_update(context, volume_ref['id'], db_update) + model_update = self.driver.create_export(context, volume_ref) + if model_update: + self.db.volume_update(context, volume_ref['id'], model_update) except Exception: self.db.volume_update(context, volume_ref['id'], {'status': 'error'}) diff --git a/nova/volume/san.py b/nova/volume/san.py index 911ad096f..09192bc9f 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -485,12 +485,12 @@ class HpSanISCSIDriver(SanISCSIDriver): cluster_vip = self._cliq_get_cluster_vip(cluster_name) iscsi_portal = cluster_vip + ":3260," + cluster_interface - db_update = {} - db_update['provider_location'] = ("%s %s" % - (iscsi_portal, - iscsi_iqn)) + model_update = {} + model_update['provider_location'] = ("%s %s" % + (iscsi_portal, + iscsi_iqn)) - return db_update + return model_update def delete_volume(self, volume): """Deletes a volume.""" @@ -517,7 +517,7 @@ class HpSanISCSIDriver(SanISCSIDriver): is_shared = 'permission.authGroup' in volume_info - db_update = {} + model_update = {} should_export = False @@ -551,10 +551,10 @@ class HpSanISCSIDriver(SanISCSIDriver): self._cliq_run_xml("assignVolumeChap", cliq_args) - db_update['provider_auth'] = ("CHAP %s %s" % - (chap_username, chap_password)) + model_update['provider_auth'] = ("CHAP %s %s" % + (chap_username, chap_password)) - return db_update + return model_update def remove_export(self, context, volume): """Removes an export for a logical volume.""" -- cgit From 90007cd9085909a1e1ac59732a0b371dd79c2557 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 15:55:50 -0800 Subject: pretty colors for logs and a few optimizations --- bin/nova-dhcpbridge | 2 + run_tests.py | 202 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 2 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index eda2dc072..04a1771f0 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -113,6 +113,8 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', + 'nova', + 'tests', 'tests.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] diff --git a/run_tests.py b/run_tests.py index 82345433a..43508394f 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + import gettext import os import unittest @@ -28,14 +51,186 @@ from nose import core from nova import log as logging +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm(fd=stream.fileno()) + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_ERROR_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_ERROR_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_ERROR_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None + self.colorizer = None + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + # NOTE(vish): nose does funky stuff with stdout, so use stderr + # to setup the colorizer + if colorizer.supported(sys.stderr): + self.colorizer = colorizer(self.stream) + break def getDescription(self, test): return str(test) + def addSuccess(self, test): + if self.showAll: + self.colorizer.write("OK", 'green') + self.stream.writeln() + elif self.dots: + self.stream.write('.') + self.stream.flush() + + def addFailure(self, test): + if self.showAll: + self.colorizer.write("FAIL", 'red') + self.stream.writeln() + elif self.dots: + self.stream.write('F') + self.stream.flush() + + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + if self.showAll: + self.colorizer.write("ERROR", 'red') + self.stream.writeln() + elif self.dots: + stream.write('E') + def startTest(self, test): unittest.TestResult.startTest(self, test) current_case = test.test.__class__.__name__ @@ -59,12 +254,15 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - if os.path.exists("tests.sqlite"): - os.unlink("tests.sqlite") logging.setup() + testdir = os.path.abspath(os.path.join("nova","tests")) + testdb = os.path.join(testdir, "tests.sqlite") + if os.path.exists(testdb): + os.unlink(testdb) c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, + workingDir=testdir, plugins=core.DefaultPluginManager()) runner = NovaTestRunner(stream=c.stream, -- cgit From 305ef6bf5f8f8926fdaa8db5f75a0680fbd8a2be Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 21 Feb 2011 16:02:38 -0800 Subject: Fixed my confusion in documenting the syntax of iSCSI discovery --- nova/volume/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 687bc99d0..22c2c2fc3 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -231,7 +231,7 @@ class ISCSIDriver(VolumeDriver): provider properties as follows: provider_location - if present, contains the iSCSI target information in the same format as an ietadm discovery - i.e. ', ' + i.e. ', ' provider_auth - if present, contains a space-separated triple: ' '. CHAP is the only auth_method in use at the moment.""" -- cgit From 71f7119910f16cb99c10f43a07ccb1e7c0ca473f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 16:05:28 -0800 Subject: remove changes to test db --- bin/nova-dhcpbridge | 2 +- nova/tests/fake_flags.py | 2 +- run_tests.py | 2 -- run_tests.sh | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index eda2dc072..35b837ca9 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -113,7 +113,7 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', - 'tests.sqlite')) + 'nova.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] if action in ['add', 'del', 'old']: diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 575fefff6..cfa65c137 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,6 +39,6 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///tests.sqlite' +FLAGS.sql_connection = 'sqlite:///nova.sqlite' FLAGS.use_ipv6 = True FLAGS.logfile = 'tests.log' diff --git a/run_tests.py b/run_tests.py index 82345433a..cb957da42 100644 --- a/run_tests.py +++ b/run_tests.py @@ -59,8 +59,6 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': - if os.path.exists("tests.sqlite"): - os.unlink("tests.sqlite") logging.setup() c = config.Config(stream=sys.stdout, env=os.environ, diff --git a/run_tests.sh b/run_tests.sh index 4e21fe945..70212cc6a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -40,7 +40,7 @@ done function run_tests { # Just run the test suites in current environment ${wrapper} rm -f nova.sqlite - ${wrapper} $NOSETESTS 2> run_tests.err.log + ${wrapper} $NOSETESTS } NOSETESTS="python run_tests.py $noseargs" -- cgit From 11c57867ec18bd61dcc6bde0dc4b459318d54e70 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 16:19:48 -0800 Subject: fixed newline and moved import fake_flags into run_tests where it makes more sense --- nova/test.py | 3 --- run_tests.py | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/test.py b/nova/test.py index 8022396cd..bff43b6c7 100644 --- a/nova/test.py +++ b/nova/test.py @@ -36,9 +36,6 @@ from nova import log as logging from nova import rpc from nova.network import manager as network_manager -from nova.tests import fake_flags -logging.reset() - FLAGS = flags.FLAGS flags.DEFINE_bool('flush_db', True, diff --git a/run_tests.py b/run_tests.py index cb957da42..6d96454b9 100644 --- a/run_tests.py +++ b/run_tests.py @@ -27,6 +27,8 @@ from nose import result from nose import core from nova import log as logging +from nova.tests import fake_flags + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): -- cgit From 0f402b72cbf80d1adde503eb532a578944fa0c79 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 16:22:09 -0800 Subject: update based on prereq branch --- nova/tests/fake_flags.py | 2 +- run_tests.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index cfa65c137..575fefff6 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -39,6 +39,6 @@ FLAGS.num_shelves = 2 FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True -FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True FLAGS.logfile = 'tests.log' diff --git a/run_tests.sh b/run_tests.sh index 70212cc6a..e8433bc06 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -39,7 +39,6 @@ done function run_tests { # Just run the test suites in current environment - ${wrapper} rm -f nova.sqlite ${wrapper} $NOSETESTS } -- cgit From 3125d978fec27608064dd3dd8d3696f2219fbf12 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 21 Feb 2011 23:26:03 -0800 Subject: use a different flag for listen port for apis --- bin/nova-api | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 11176a021..cb9b41725 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -44,6 +44,8 @@ LOG = logging.getLogger('nova.api') LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS +flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') +flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') API_ENDPOINTS = ['ec2', 'osapi'] @@ -60,10 +62,10 @@ def run_app(paste_config_file): wsgi.paste_config_to_flags(config, { "verbose": FLAGS.verbose, "%s_host" % api: config.get('host', '0.0.0.0'), - "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) + "%s_listen_port" % api: getattr(FLAGS, "%s_listen_port" % api)}) LOG.info(_("Running %s API"), api) app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_port" % api), + apps.append((app, getattr(FLAGS, "%s_listen_port" % api), getattr(FLAGS, "%s_host" % api))) if len(apps) == 0: LOG.error(_("No known API applications configured in %s."), -- cgit From 9e2942931b5381d3ba0e8cc4f9846160b003f45b Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Tue, 22 Feb 2011 17:18:04 +0100 Subject: Get rid of nova-combined, see rationale on ML --- bin/nova-combined | 83 ------------------------------------------------------- 1 file changed, 83 deletions(-) delete mode 100755 bin/nova-combined diff --git a/bin/nova-combined b/bin/nova-combined deleted file mode 100755 index 22f0d5cb7..000000000 --- a/bin/nova-combined +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Combined starter script for Nova services.""" - -import eventlet -eventlet.monkey_patch() - -import gettext -import os -import sys - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import service -from nova import utils -from nova import wsgi - - -FLAGS = flags.FLAGS - -API_ENDPOINTS = ['ec2', 'osapi'] - -for api in API_ENDPOINTS: - flags.DEFINE_string("%s_listen" % api, "0.0.0.0", - "IP address to listen to for API %s" % api) - flags.DEFINE_integer("%s_listen_port" % api, - getattr(FLAGS, "%s_port" % api), - "Port to listen to for API %s" % api) - -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) - logging.setup() - - compute = service.Service.create(binary='nova-compute') - network = service.Service.create(binary='nova-network') - volume = service.Service.create(binary='nova-volume') - scheduler = service.Service.create(binary='nova-scheduler') - #objectstore = service.Service.create(binary='nova-objectstore') - - service.serve(compute, network, volume, scheduler) - - apps = [] - paste_config_file = wsgi.paste_config_file('nova-api.conf') - for api in API_ENDPOINTS: - config = wsgi.load_paste_configuration(paste_config_file, api) - if config is None: - continue - app = wsgi.load_paste_app(paste_config_file, api) - apps.append((app, getattr(FLAGS, "%s_listen_port" % api), - getattr(FLAGS, "%s_listen" % api))) - if len(apps) > 0: - server = wsgi.Server() - for app in apps: - server.start(*app) - server.wait() -- cgit From 912e762c9baf3cb17a24bc0d9feba4b26892dbbc Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Tue, 22 Feb 2011 17:37:12 +0100 Subject: Also remove nova-combined from setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 4ab8f386b..3b48990ac 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,6 @@ DistUtilsExtra.auto.setup(name='nova', test_suite='nose.collector', scripts=['bin/nova-ajax-console-proxy', 'bin/nova-api', - 'bin/nova-combined', 'bin/nova-compute', 'bin/nova-console', 'bin/nova-dhcpbridge', -- cgit From 5b2ec209d07d7df45f9b7ca6eebfcbc9443de94e Mon Sep 17 00:00:00 2001 From: termie Date: Tue, 22 Feb 2011 17:10:34 -0800 Subject: don't make a syslog handler if we didn't ask for one --- nova/log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/log.py b/nova/log.py index 10c14d74b..591d26c63 100644 --- a/nova/log.py +++ b/nova/log.py @@ -236,16 +236,17 @@ class NovaRootLogger(NovaLogger): def __init__(self, name, level=NOTSET): self.logpath = None self.filelog = None - self.syslog = SysLogHandler(address='/dev/log') self.streamlog = StreamHandler() + self.syslog = None NovaLogger.__init__(self, name, level) def setup_from_flags(self): """Setup logger from flags""" global _filelog if FLAGS.use_syslog: + self.syslog = SysLogHandler(address='/dev/log') self.addHandler(self.syslog) - else: + elif self.syslog: self.removeHandler(self.syslog) logpath = _get_log_file_path() if logpath: -- cgit From 18793c2e184713d33bc93306d464cf443584ffd6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 17:44:07 -0800 Subject: test that shows error on filtering groups --- nova/tests/test_cloud.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 445cc6e8b..2bce64353 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -133,6 +133,20 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_security_groups(self): + """Makes sure describe_security_groups works and filters results.""" + sec = db.security_group_create(self.context, {'name': 'test'}) + result = self.cloud.describe_security_groups(self.context) + # NOTE(vish): should have the default group as well + self.assertEqual(len(result['securityGroupInfo']), 2) + result = self.cloud.describe_security_groups(self.context, + group_name=[sec['name']]) + self.assertEqual(len(result['securityGroupInfo']), 1) + self.assertEqual( + cloud.ec2_id_to_id(result['securityGroupInfo'][0]['name']), + sec['name']) + db.security_group_destroy(self.context, sec['id']) + def test_describe_volumes(self): """Makes sure describe_volumes works and filters results.""" vol1 = db.volume_create(self.context, {}) @@ -286,19 +300,6 @@ class CloudTestCase(test.TestCase): LOG.debug(_("Terminating instance %s"), instance_id) rv = self.compute.terminate_instance(instance_id) - def test_describe_instances(self): - """Makes sure describe_instances works.""" - instance1 = db.instance_create(self.context, {'host': 'host2'}) - comp1 = db.service_create(self.context, {'host': 'host2', - 'availability_zone': 'zone1', - 'topic': "compute"}) - result = self.cloud.describe_instances(self.context) - self.assertEqual(result['reservationSet'][0] - ['instancesSet'][0] - ['placement']['availabilityZone'], 'zone1') - db.instance_destroy(self.context, instance1['id']) - db.service_destroy(self.context, comp1['id']) - def test_instance_update_state(self): # TODO(termie): what is this code even testing? def instance(num): -- cgit From 2610a522d26351686612058a6da0300bce731112 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 17:49:38 -0800 Subject: fix test --- nova/tests/test_cloud.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 2bce64353..afdbb80a9 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -135,7 +135,9 @@ class CloudTestCase(test.TestCase): def test_describe_security_groups(self): """Makes sure describe_security_groups works and filters results.""" - sec = db.security_group_create(self.context, {'name': 'test'}) + sec = db.security_group_create(self.context, + {'project_id': self.context.project_id, + 'name': 'test'}) result = self.cloud.describe_security_groups(self.context) # NOTE(vish): should have the default group as well self.assertEqual(len(result['securityGroupInfo']), 2) @@ -143,7 +145,7 @@ class CloudTestCase(test.TestCase): group_name=[sec['name']]) self.assertEqual(len(result['securityGroupInfo']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['securityGroupInfo'][0]['name']), + result['securityGroupInfo'][0]['groupName'], sec['name']) db.security_group_destroy(self.context, sec['id']) -- cgit From 828e3ea3f29f57767a4e25ad40b275c886cb7968 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 18:02:25 -0800 Subject: fix and optimize security group filtering --- nova/api/ec2/cloud.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..fc9c13d91 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -318,14 +318,19 @@ class CloudController(object): def describe_security_groups(self, context, group_name=None, **kwargs): self.compute_api.ensure_default_security_group(context) - if context.is_admin: + if group_name: + groups = [] + for name in group_name: + group = db.security_group_get_by_name(context, + context.project_id, + name) + groups.append(group) + elif context.is_admin: groups = db.security_group_get_all(context) else: groups = db.security_group_get_by_project(context, context.project_id) groups = [self._format_security_group(context, g) for g in groups] - if not group_name is None: - groups = [g for g in groups if g.name in group_name] return {'securityGroupInfo': list(sorted(groups, -- cgit From 2fd33bdd50b933dc14fea065c823f5a73324129b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 18:04:08 -0800 Subject: separate out smoketests and add updated nova.sh --- contrib/nova.sh | 12 +- smoketests/base.py | 12 + smoketests/public_network_smoketests.py | 11 +- smoketests/sysadmin_smoketests.py | 293 +++++++++++++++++++++++ smoketests/user_smoketests.py | 397 -------------------------------- 5 files changed, 321 insertions(+), 404 deletions(-) create mode 100644 smoketests/sysadmin_smoketests.py delete mode 100644 smoketests/user_smoketests.py diff --git a/contrib/nova.sh b/contrib/nova.sh index 9259035ca..1187f2728 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -66,7 +66,7 @@ if [ "$CMD" == "install" ]; then sudo apt-get install -y user-mode-linux kvm libvirt-bin sudo apt-get install -y screen euca2ools vlan curl rabbitmq-server sudo apt-get install -y lvm2 iscsitarget open-iscsi - sudo apt-get install -y socat + sudo apt-get install -y socat unzip echo "ISCSITARGET_ENABLE=true" | sudo tee /etc/default/iscsitarget sudo /etc/init.d/iscsitarget restart sudo modprobe kvm @@ -111,8 +111,7 @@ if [ "$CMD" == "run" ]; then --nodaemon --dhcpbridge_flagfile=$NOVA_DIR/bin/nova.conf --network_manager=nova.network.manager.$NET_MAN ---cc_host=$HOST_IP ---routing_source_ip=$HOST_IP +--my_ip=$HOST_IP --sql_connection=$SQL_CONN --auth_driver=nova.auth.$AUTH --libvirt_type=$LIBVIRT_TYPE @@ -151,7 +150,6 @@ NOVA_CONF_EOF mkdir -p $NOVA_DIR/instances rm -rf $NOVA_DIR/networks mkdir -p $NOVA_DIR/networks - $NOVA_DIR/tools/clean-vlans if [ ! -d "$NOVA_DIR/images" ]; then ln -s $DIR/images $NOVA_DIR/images fi @@ -169,10 +167,14 @@ NOVA_CONF_EOF # create a project called 'admin' with project manager of 'admin' $NOVA_DIR/bin/nova-manage project create admin admin # export environment variables for project 'admin' and user 'admin' - $NOVA_DIR/bin/nova-manage project environment admin admin $NOVA_DIR/novarc + $NOVA_DIR/bin/nova-manage project zipfile admin admin $NOVA_DIR/nova.zip + unzip -o $NOVA_DIR/nova.zip -d $NOVA_DIR/ # create a small network $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 1 32 + # create some floating ips + $NOVA_DIR/bin/nova-manage floating create `hostname` 10.6.0.0/27 + # nova api crashes if we start it with a regular screen command, # so send the start command by forcing text into the window. screen_it api "$NOVA_DIR/bin/nova-api" diff --git a/smoketests/base.py b/smoketests/base.py index afc618074..204b4a1eb 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -28,7 +28,9 @@ from boto.ec2.regioninfo import RegionInfo from smoketests import flags +SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) boto_v6 = None @@ -173,6 +175,16 @@ class SmokeTestCase(unittest.TestCase): return True +TEST_DATA = {} + + +class UserSmokeTestCase(SmokeTestCase): + def setUp(self): + global TEST_DATA + self.conn = self.connection_for_env() + self.data = TEST_DATA + + def run_tests(suites): argv = FLAGS(sys.argv) if FLAGS.use_ipv6: diff --git a/smoketests/public_network_smoketests.py b/smoketests/public_network_smoketests.py index bfc2b20ba..5a4c67642 100644 --- a/smoketests/public_network_smoketests.py +++ b/smoketests/public_network_smoketests.py @@ -24,9 +24,16 @@ import sys import time import unittest +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + from smoketests import flags from smoketests import base -from smoketests import user_smoketests #Note that this test should run from #public network (outside of private network segments) @@ -42,7 +49,7 @@ TEST_KEY2 = '%s_key2' % TEST_PREFIX TEST_DATA = {} -class InstanceTestsFromPublic(user_smoketests.UserSmokeTestCase): +class InstanceTestsFromPublic(base.UserSmokeTestCase): def test_001_can_create_keypair(self): key = self.create_key_pair(self.conn, TEST_KEY) self.assertEqual(key.name, TEST_KEY) diff --git a/smoketests/sysadmin_smoketests.py b/smoketests/sysadmin_smoketests.py new file mode 100644 index 000000000..e3b84d3d3 --- /dev/null +++ b/smoketests/sysadmin_smoketests.py @@ -0,0 +1,293 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + + +FLAGS = flags.FLAGS +flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', + 'Local kernel file to use for bundling tests') +flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', + 'Local image file to use for bundling tests') + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX +class ImageTests(base.UserSmokeTestCase): + def test_001_can_bundle_image(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_image)) + + def test_002_can_upload_image(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) + + def test_003_can_register_image(self): + image_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_image)) + self.assert_(image_id is not None) + self.data['image_id'] = image_id + + def test_004_can_bundle_kernel(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) + + def test_005_can_upload_kernel(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) + + def test_006_can_register_kernel(self): + kernel_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_kernel)) + self.assert_(kernel_id is not None) + self.data['kernel_id'] = kernel_id + + def test_007_images_are_available_within_10_seconds(self): + for i in xrange(10): + image = self.conn.get_image(self.data['image_id']) + if image and image.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(image.type == 'machine') + + for i in xrange(10): + kernel = self.conn.get_image(self.data['kernel_id']) + if kernel and kernel.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(kernel.type == 'kernel') + + def test_008_can_describe_image_attribute(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + + def test_009_can_modify_image_launch_permission(self): + self.conn.modify_image_attribute(image_id=self.data['image_id'], + operation='add', + attribute='launchPermission', + groups='all') + image = self.conn.get_image(self.data['image_id']) + self.assertEqual(image.id, self.data['image_id']) + + def test_010_can_see_launch_permission(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + self.assert_(attrs.attrs['groups'][0], 'all') + + def test_011_user_can_deregister_kernel(self): + self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) + + def test_012_can_deregister_image(self): + self.assertTrue(self.conn.deregister_image(self.data['image_id'])) + + def test_013_can_delete_bundle(self): + self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) + + +class InstanceTests(base.UserSmokeTestCase): + def test_001_can_create_keypair(self): + key = self.create_key_pair(self.conn, TEST_KEY) + self.assertEqual(key.name, TEST_KEY) + + def test_002_can_create_instance_with_keypair(self): + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + instance_type='m1.tiny') + self.assertEqual(len(reservation.instances), 1) + self.data['instance'] = reservation.instances[0] + + def test_003_instance_runs_within_60_seconds(self): + instance = self.data['instance'] + # allow 60 seconds to exit pending with IP + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + ip = self.data['instance'].private_dns_name + self.failIf(ip == '0.0.0.0') + if FLAGS.use_ipv6: + ipv6 = self.data['instance'].dns_name_v6 + self.failIf(ipv6 is None) + + def test_004_can_ping_private_ip(self): + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + + if FLAGS.use_ipv6: + if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + self.fail('could not ping instance v6') + + def test_005_can_ssh_to_private_ip(self): + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + if FLAGS.use_ipv6: + if not self.wait_for_ssh(self.data['instance'].ip_v6, + TEST_KEY): + self.fail('could not ssh to instance v6') + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class VolumeTests(base.UserSmokeTestCase): + def setUp(self): + super(VolumeTests, self).setUp() + self.device = '/dev/vdb' + + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_create_volume(self): + volume = self.conn.create_volume(1, 'nova') + self.assertEqual(volume.size, 1) + self.data['volume'] = volume + # Give network time to find volume. + time.sleep(10) + + def test_002_can_attach_volume(self): + volume = self.data['volume'] + + for x in xrange(10): + volume.update() + if volume.status.startswith('available'): + break + time.sleep(1) + else: + self.fail('cannot attach volume with state %s' % volume.status) + + volume.attach(self.data['instance'].id, self.device) + + # wait + for x in xrange(10): + volume.update() + if volume.status.startswith('in-use'): + break + time.sleep(1) + else: + self.fail('volume never got to in use') + + self.assertTrue(volume.status.startswith('in-use')) + + # Give instance time to recognize volume. + time.sleep(10) + + def test_003_can_mount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # NOTE(vish): this will create an dev for images that don't have + # udev rules + stdin, stdout, stderr = conn.exec_command( + 'grep %s /proc/partitions | ' + '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' + % self.device.rpartition('/')[2]) + exec_list = [] + exec_list.append('mkdir -p /mnt/vol') + exec_list.append('/sbin/mke2fs %s' % self.device) + exec_list.append('mount %s /mnt/vol' % self.device) + exec_list.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) + out = stdout.read() + conn.close() + if not out.strip().endswith('success'): + self.fail('Unable to mount: %s %s' % (out, stderr.read())) + + def test_004_can_write_to_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted + stdin, stdout, stderr = conn.exec_command( + 'echo hello > /mnt/vol/test.txt') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to write to mount: %s' % (err)) + + def test_005_volume_is_correct_size(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command( + "df -h | grep %s | awk {'print $2'}" % self.device) + out = stdout.read() + conn.close() + if not out.strip() == '1007.9M': + self.fail('Volume is not the right size: %s %s' % + (out, stderr.read())) + + def test_006_me_can_umount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to unmount: %s' % (err)) + + def test_007_me_can_detach_volume(self): + result = self.conn.detach_volume(volume_id=self.data['volume'].id) + self.assertTrue(result) + time.sleep(5) + + def test_008_me_can_delete_volume(self): + result = self.conn.delete_volume(self.data['volume'].id) + self.assertTrue(result) + + def test_999_tearDown(self): + self.conn.terminate_instances([self.data['instance'].id]) + self.conn.delete_key_pair(TEST_KEY) + + +if __name__ == "__main__": + suites = {'image': unittest.makeSuite(ImageTests), + 'instance': unittest.makeSuite(InstanceTests), + 'volume': unittest.makeSuite(VolumeTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py deleted file mode 100644 index 26f6344f7..000000000 --- a/smoketests/user_smoketests.py +++ /dev/null @@ -1,397 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import commands -import os -import random -import sys -import time -import unittest - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -from smoketests import flags -from smoketests import base - - -SUITE_NAMES = '[image, instance, volume]' - -FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) -flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', - 'Local kernel file to use for bundling tests') -flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', - 'Local image file to use for bundling tests') - -TEST_PREFIX = 'test%s' % int(random.random() * 1000000) -TEST_BUCKET = '%s_bucket' % TEST_PREFIX -TEST_KEY = '%s_key' % TEST_PREFIX -TEST_GROUP = '%s_group' % TEST_PREFIX -TEST_DATA = {} - - -class UserSmokeTestCase(base.SmokeTestCase): - def setUp(self): - global TEST_DATA - self.conn = self.connection_for_env() - self.data = TEST_DATA - - -class ImageTests(UserSmokeTestCase): - def test_001_can_bundle_image(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_image)) - - def test_002_can_upload_image(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) - - def test_003_can_register_image(self): - image_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_image)) - self.assert_(image_id is not None) - self.data['image_id'] = image_id - - def test_004_can_bundle_kernel(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) - - def test_005_can_upload_kernel(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) - - def test_006_can_register_kernel(self): - kernel_id = self.conn.register_image('%s/%s.manifest.xml' % - (TEST_BUCKET, FLAGS.bundle_kernel)) - self.assert_(kernel_id is not None) - self.data['kernel_id'] = kernel_id - - def test_007_images_are_available_within_10_seconds(self): - for i in xrange(10): - image = self.conn.get_image(self.data['image_id']) - if image and image.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(image.type == 'machine') - - for i in xrange(10): - kernel = self.conn.get_image(self.data['kernel_id']) - if kernel and kernel.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(kernel.type == 'kernel') - - def test_008_can_describe_image_attribute(self): - attrs = self.conn.get_image_attribute(self.data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - - def test_009_can_modify_image_launch_permission(self): - self.conn.modify_image_attribute(image_id=self.data['image_id'], - operation='add', - attribute='launchPermission', - groups='all') - image = self.conn.get_image(self.data['image_id']) - self.assertEqual(image.id, self.data['image_id']) - - def test_010_can_see_launch_permission(self): - attrs = self.conn.get_image_attribute(self.data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - self.assert_(attrs.attrs['groups'][0], 'all') - - def test_011_user_can_deregister_kernel(self): - self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) - - def test_012_can_deregister_image(self): - self.assertTrue(self.conn.deregister_image(self.data['image_id'])) - - def test_013_can_delete_bundle(self): - self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) - - -class InstanceTests(UserSmokeTestCase): - def test_001_can_create_keypair(self): - key = self.create_key_pair(self.conn, TEST_KEY) - self.assertEqual(key.name, TEST_KEY) - - def test_002_can_create_instance_with_keypair(self): - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - instance_type='m1.tiny') - self.assertEqual(len(reservation.instances), 1) - self.data['instance'] = reservation.instances[0] - - def test_003_instance_runs_within_60_seconds(self): - instance = self.data['instance'] - # allow 60 seconds to exit pending with IP - if not self.wait_for_running(self.data['instance']): - self.fail('instance failed to start') - self.data['instance'].update() - ip = self.data['instance'].private_dns_name - self.failIf(ip == '0.0.0.0') - if FLAGS.use_ipv6: - ipv6 = self.data['instance'].dns_name_v6 - self.failIf(ipv6 is None) - - def test_004_can_ping_private_ip(self): - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - - if FLAGS.use_ipv6: - if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): - self.fail('could not ping instance v6') - - def test_005_can_ssh_to_private_ip(self): - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') - - if FLAGS.use_ipv6: - if not self.wait_for_ssh(self.data['instance'].ip_v6, - TEST_KEY): - self.fail('could not ssh to instance v6') - - def test_006_can_allocate_elastic_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - def test_007_can_associate_ip_with_instance(self): - result = self.conn.associate_address(self.data['instance'].id, - self.data['public_ip']) - self.assertTrue(result) - - def test_008_can_ssh_with_public_ip(self): - if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): - self.fail('could not ssh to public ip') - - def test_009_can_disassociate_ip_from_instance(self): - result = self.conn.disassociate_address(self.data['public_ip']) - self.assertTrue(result) - - def test_010_can_deallocate_elastic_ip(self): - result = self.conn.release_address(self.data['public_ip']) - self.assertTrue(result) - - def test_999_tearDown(self): - self.delete_key_pair(self.conn, TEST_KEY) - self.conn.terminate_instances([self.data['instance'].id]) - - -class VolumeTests(UserSmokeTestCase): - def setUp(self): - super(VolumeTests, self).setUp() - self.device = '/dev/vdb' - - def test_000_setUp(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - instance_type='m1.tiny', - key_name=TEST_KEY) - self.data['instance'] = reservation.instances[0] - if not self.wait_for_running(self.data['instance']): - self.fail('instance failed to start') - self.data['instance'].update() - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') - - def test_001_can_create_volume(self): - volume = self.conn.create_volume(1, 'nova') - self.assertEqual(volume.size, 1) - self.data['volume'] = volume - # Give network time to find volume. - time.sleep(10) - - def test_002_can_attach_volume(self): - volume = self.data['volume'] - - for x in xrange(10): - volume.update() - if volume.status.startswith('available'): - break - time.sleep(1) - else: - self.fail('cannot attach volume with state %s' % volume.status) - - volume.attach(self.data['instance'].id, self.device) - - # wait - for x in xrange(10): - volume.update() - if volume.status.startswith('in-use'): - break - time.sleep(1) - else: - self.fail('volume never got to in use') - - self.assertTrue(volume.status.startswith('in-use')) - - # Give instance time to recognize volume. - time.sleep(10) - - def test_003_can_mount_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - # NOTE(vish): this will create an dev for images that don't have - # udev rules - stdin, stdout, stderr = conn.exec_command( - 'grep %s /proc/partitions | ' - '`awk \'{print "mknod /dev/"\\$4" b "\\$1" "\\$2}\'`' - % self.device.rpartition('/')[2]) - exec_list = [] - exec_list.append('mkdir -p /mnt/vol') - exec_list.append('/sbin/mke2fs %s' % self.device) - exec_list.append('mount %s /mnt/vol' % self.device) - exec_list.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(exec_list)) - out = stdout.read() - conn.close() - if not out.strip().endswith('success'): - self.fail('Unable to mount: %s %s' % (out, stderr.read())) - - def test_004_can_write_to_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted - stdin, stdout, stderr = conn.exec_command( - 'echo hello > /mnt/vol/test.txt') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to write to mount: %s' % (err)) - - def test_005_volume_is_correct_size(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - stdin, stdout, stderr = conn.exec_command( - "df -h | grep %s | awk {'print $2'}" % self.device) - out = stdout.read() - conn.close() - if not out.strip() == '1007.9M': - self.fail('Volume is not the right size: %s %s' % - (out, stderr.read())) - - def test_006_me_can_umount_volume(self): - ip = self.data['instance'].private_dns_name - conn = self.connect_ssh(ip, TEST_KEY) - stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to unmount: %s' % (err)) - - def test_007_me_can_detach_volume(self): - result = self.conn.detach_volume(volume_id=self.data['volume'].id) - self.assertTrue(result) - time.sleep(5) - - def test_008_me_can_delete_volume(self): - result = self.conn.delete_volume(self.data['volume'].id) - self.assertTrue(result) - - def test_999_tearDown(self): - self.conn.terminate_instances([self.data['instance'].id]) - self.conn.delete_key_pair(TEST_KEY) - - -class SecurityGroupTests(UserSmokeTestCase): - - def __public_instance_is_accessible(self): - id_url = "latest/meta-data/instance-id" - options = "-s --max-time 1" - command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) - instance_id = commands.getoutput(command).strip() - if not instance_id: - return False - if instance_id != self.data['instance_id']: - raise Exception("Wrong instance id") - return True - - def test_001_can_create_security_group(self): - self.conn.create_security_group(TEST_GROUP, description='test') - - groups = self.conn.get_all_security_groups() - self.assertTrue(TEST_GROUP in [group.name for group in groups]) - - def test_002_can_launch_instance_in_security_group(self): - self.create_key_pair(self.conn, TEST_KEY) - reservation = self.conn.run_instances(FLAGS.test_image, - key_name=TEST_KEY, - security_groups=[TEST_GROUP], - instance_type='m1.tiny') - - self.data['instance_id'] = reservation.instances[0].id - - def test_003_can_authorize_security_group_ingress(self): - self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - - def test_004_can_access_instance_over_public_ip(self): - result = self.conn.allocate_address() - self.assertTrue(hasattr(result, 'public_ip')) - self.data['public_ip'] = result.public_ip - - result = self.conn.associate_address(self.data['instance_id'], - self.data['public_ip']) - start_time = time.time() - while not self.__public_instance_is_accessible(): - # 1 minute to launch - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_005_can_revoke_security_group_ingress(self): - self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, - ip_protocol='tcp', - from_port=80, - to_port=80)) - start_time = time.time() - while self.__public_instance_is_accessible(): - # 1 minute to teardown - if time.time() - start_time > 60: - raise Exception("Timeout") - time.sleep(1) - - def test_999_tearDown(self): - self.conn.delete_key_pair(TEST_KEY) - self.conn.delete_security_group(TEST_GROUP) - groups = self.conn.get_all_security_groups() - self.assertFalse(TEST_GROUP in [group.name for group in groups]) - self.conn.terminate_instances([self.data['instance_id']]) - self.assertTrue(self.conn.release_address(self.data['public_ip'])) - - -if __name__ == "__main__": - suites = {'image': unittest.makeSuite(ImageTests), - 'instance': unittest.makeSuite(InstanceTests), - #'security_group': unittest.makeSuite(SecurityGroupTests), - 'volume': unittest.makeSuite(VolumeTests) - } - sys.exit(base.run_tests(suites)) -- cgit From ef37833e6f45f99b1d16143d29685974a191c387 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 18:04:32 -0800 Subject: add netadmin smoketests --- smoketests/netadmin_smoketests.py | 194 ++++++++++++++++++++++++++++++++++++++ smoketests/proxy.sh | 22 +++++ 2 files changed, 216 insertions(+) create mode 100644 smoketests/netadmin_smoketests.py create mode 100755 smoketests/proxy.sh diff --git a/smoketests/netadmin_smoketests.py b/smoketests/netadmin_smoketests.py new file mode 100644 index 000000000..38beb8fdc --- /dev/null +++ b/smoketests/netadmin_smoketests.py @@ -0,0 +1,194 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import sys +import time +import unittest + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from smoketests import flags +from smoketests import base + + +FLAGS = flags.FLAGS + +TEST_PREFIX = 'test%s' % int(random.random() * 1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_GROUP = '%s_group' % TEST_PREFIX + + +class AddressTests(base.UserSmokeTestCase): + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_001_can_allocate_floating_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + def test_002_can_associate_ip_with_instance(self): + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + self.assertTrue(result) + + def test_003_can_ssh_with_public_ip(self): + ssh_authorized = False + groups = self.conn.get_all_security_groups(['default']) + for rule in groups[0].rules: + if (rule.ip_protocol == 'tcp' and + rule.from_port <= 22 and rule.to_port >= 22): + ssh_authorized = True + if not ssh_authorized: + self.conn.authorize_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + try: + if not self.wait_for_ssh(self.data['public_ip'], TEST_KEY): + self.fail('could not ssh to public ip') + finally: + if not ssh_authorized: + self.conn.revoke_security_group('default', + ip_protocol='tcp', + from_port=22, + to_port=22) + + def test_004_can_disassociate_ip_from_instance(self): + result = self.conn.disassociate_address(self.data['public_ip']) + self.assertTrue(result) + + def test_005_can_deallocate_floating_ip(self): + result = self.conn.release_address(self.data['public_ip']) + self.assertTrue(result) + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + self.conn.terminate_instances([self.data['instance'].id]) + + +class SecurityGroupTests(base.UserSmokeTestCase): + + def __public_instance_is_accessible(self): + id_url = "latest/meta-data/instance-id" + options = "-s --max-time 1" + command = "curl %s %s/%s" % (options, self.data['public_ip'], id_url) + instance_id = commands.getoutput(command).strip() + if not instance_id: + return False + if instance_id != self.data['instance'].id: + raise Exception("Wrong instance id") + return True + + def test_001_can_create_security_group(self): + self.conn.create_security_group(TEST_GROUP, description='test') + + groups = self.conn.get_all_security_groups() + self.assertTrue(TEST_GROUP in [group.name for group in groups]) + + def test_002_can_launch_instance_in_security_group(self): + with open("proxy.sh") as f: + user_data = f.read() + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + security_groups=[TEST_GROUP], + user_data=user_data, + instance_type='m1.tiny') + + self.data['instance'] = reservation.instances[0] + if not self.wait_for_running(self.data['instance']): + self.fail('instance failed to start') + self.data['instance'].update() + if not self.wait_for_ping(self.data['instance'].private_dns_name): + self.fail('could not ping instance') + if not self.wait_for_ssh(self.data['instance'].private_dns_name, + TEST_KEY): + self.fail('could not ssh to instance') + + def test_003_can_authorize_security_group_ingress(self): + self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + + def test_004_can_access_metadata_over_public_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + result = self.conn.associate_address(self.data['instance'].id, + self.data['public_ip']) + start_time = time.time() + try: + while not self.__public_instance_is_accessible(): + # 1 minute to launch + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + finally: + result = self.conn.disassociate_address(self.data['public_ip']) + + def test_005_can_revoke_security_group_ingress(self): + self.assertTrue(self.conn.revoke_security_group(TEST_GROUP, + ip_protocol='tcp', + from_port=80, + to_port=80)) + start_time = time.time() + while self.__public_instance_is_accessible(): + # 1 minute to teardown + if time.time() - start_time > 60: + raise Exception("Timeout") + time.sleep(1) + + def test_999_tearDown(self): + self.conn.delete_key_pair(TEST_KEY) + self.conn.delete_security_group(TEST_GROUP) + groups = self.conn.get_all_security_groups() + self.assertFalse(TEST_GROUP in [group.name for group in groups]) + self.conn.terminate_instances([self.data['instance'].id]) + self.assertTrue(self.conn.release_address(self.data['public_ip'])) + + +if __name__ == "__main__": + suites = {'address': unittest.makeSuite(AddressTests), + 'security_group': unittest.makeSuite(SecurityGroupTests) + } + sys.exit(base.run_tests(suites)) diff --git a/smoketests/proxy.sh b/smoketests/proxy.sh new file mode 100755 index 000000000..9b3f3108a --- /dev/null +++ b/smoketests/proxy.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This is a simple shell script that uses netcat to set up a proxy to the +# metadata server on port 80 and to a google ip on port 8080. This is meant +# to be passed in by a script to an instance via user data, so that +# automatic testing of network connectivity can be performed. + +# Example usage: +# euca-run-instances -t m1.tiny -f proxy.sh ami-tty + +mkfifo backpipe1 +mkfifo backpipe2 + +# NOTE(vish): proxy metadata on port 80 +while true; do + nc -l -p 80 0backpipe1 +done & + +# NOTE(vish): proxy google on port 8080 +while true; do + nc -l -p 8080 0backpipe2 +done & -- cgit From 9f169fdef93898097e33b5e1c0318f543ced672e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:41:41 -0800 Subject: Reverted change to focus on the core bug - kernel_id and ramdisk_id are optional --- nova/api/openstack/servers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 11a84687d..41b05cbb4 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -144,11 +144,13 @@ class Controller(wsgi.Controller): metadata stored in Glance as 'image_properties' """ def lookup(param): - properties = image.get('properties') - if properties: - return properties.get(param) - else: - return image.get(param) + _image_id = image_id + try: + return image['properties'][param] + except KeyError: + LOG.debug( + _("%(param)s property not found for image %(_image_id)s") % + locals()) image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From 3ef3dfc2f6c8b9cc14119793df4990432ff74ea2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 22 Feb 2011 18:42:23 -0800 Subject: Return null if no kernel_id / ramdisk_id --- nova/api/openstack/servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 41b05cbb4..d83bd34ab 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -151,6 +151,7 @@ class Controller(wsgi.Controller): LOG.debug( _("%(param)s property not found for image %(_image_id)s") % locals()) + return None image_id = str(image_id) image = self._image_service.show(req.environ['nova.context'], image_id) -- cgit From 943b863bef09a4e2b3de36c26a3fabbcc6093411 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 23:21:01 -0800 Subject: Lots of test fixing --- nova/api/ec2/cloud.py | 5 ++- nova/compute/api.py | 2 +- nova/db/sqlalchemy/api.py | 3 +- nova/tests/api/openstack/test_servers.py | 2 +- nova/tests/test_cloud.py | 75 ++++++++------------------------ nova/tests/test_network.py | 3 ++ nova/tests/test_scheduler.py | 3 ++ nova/tests/test_virt.py | 3 ++ nova/virt/fake.py | 4 +- 9 files changed, 36 insertions(+), 64 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 882cdcfc9..99b6d5cb6 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -529,8 +529,9 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - internal_id = ec2_id_to_id(ec2_id) - return self.compute_api.get_ajax_console(context, internal_id) + instance_id = ec2_id_to_id(ec2_id) + return self.compute_api.get_ajax_console(context, + instance_id=instance_id) def describe_volumes(self, context, volume_id=None, **kwargs): if volume_id: diff --git a/nova/compute/api.py b/nova/compute/api.py index 81ea6dc53..0caadc32e 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -447,7 +447,7 @@ class API(base.Base): {'method': 'authorize_ajax_console', 'args': {'token': output['token'], 'host': output['host'], 'port': output['port']}}) - return {'url': '%s?token=%s' % (FLAGS.ajax_console_proxy_url, + return {'url': '%s/?token=%s' % (FLAGS.ajax_console_proxy_url, output['token'])} def get_console_output(self, context, instance_id): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..2ab402e1c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1046,7 +1046,8 @@ def network_create_safe(context, values): @require_admin_context def network_disassociate(context, network_id): - network_update(context, network_id, {'project_id': None}) + network_update(context, network_id, {'project_id': None, + 'host': None}) @require_admin_context diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a7be0796e..589f3d3eb 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -84,7 +84,7 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "vcpus": 0, "local_gb": 0, "hostname": "", - "host": "", + "host": None, "instance_type": "", "user_data": "", "reservation_id": "", diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 1824d24bc..2c6dc5973 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -66,6 +66,7 @@ class CloudTestCase(test.TestCase): # set up services self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') self.network = self.start_service('network') self.manager = manager.AuthManager() @@ -73,8 +74,12 @@ class CloudTestCase(test.TestCase): self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, project=self.project) + host = self.network.get_network_host(self.context.elevated()) def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) self.compute.kill() @@ -201,27 +206,32 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) + greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] output = self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_ajax_console(self): + image_id = FLAGS.default_image kwargs = {'image_id': image_id} - rv = yield self.cloud.run_instances(self.context, **kwargs) + rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, - instance_id=[instance_id]) - self.assertEquals(b64decode(output['output']), - 'http://fakeajaxconsole.com/?token=FAKETOKEN') + greenthread.sleep(0.3) + output = self.cloud.get_ajax_console(context=self.context, + instance_id=[instance_id]) + self.assertEquals(output['url'], + '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) - rv = yield self.cloud.terminate_instances(self.context, [instance_id]) + rv = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -297,57 +307,6 @@ class CloudTestCase(test.TestCase): db.instance_destroy(self.context, instance1['id']) db.service_destroy(self.context, comp1['id']) - def test_instance_update_state(self): - # TODO(termie): what is this code even testing? - def instance(num): - return { - 'reservation_id': 'r-1', - 'instance_id': 'i-%s' % num, - 'image_id': 'ami-%s' % num, - 'private_dns_name': '10.0.0.%s' % num, - 'dns_name': '10.0.0%s' % num, - 'ami_launch_index': str(num), - 'instance_type': 'fake', - 'availability_zone': 'fake', - 'key_name': None, - 'kernel_id': 'fake', - 'ramdisk_id': 'fake', - 'groups': ['default'], - 'product_codes': None, - 'state': 0x01, - 'user_data': ''} - rv = self.cloud._format_describe_instances(self.context) - logging.error(str(rv)) - self.assertEqual(len(rv['reservationSet']), 0) - - # simulate launch of 5 instances - # self.cloud.instances['pending'] = {} - #for i in xrange(5): - # inst = instance(i) - # self.cloud.instances['pending'][inst['instance_id']] = inst - - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - # report 4 nodes each having 1 of the instances - #for i in xrange(4): - # self.cloud.update_state('instances', - # {('node-%s' % i): {('i-%s' % i): - # instance(i)}}) - - # one instance should be pending still - #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) - - # check that the reservations collapse - #rv = self.cloud._format_instances(self.admin) - #self.assert_(len(rv['reservationSet']) == 1) - #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) - - # check that we can get metadata for each instance - #for i in xrange(4): - # data = self.cloud.get_metadata(instance(i)['private_dns_name']) - # self.assert_(data['meta-data']['ami-id'] == 'ami-%s' % i) - @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..53cfea276 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -117,6 +117,9 @@ class NetworkTestCase(test.TestCase): utils.to_global_ipv6( network_ref['cidr_v6'], instance_ref['mac_address'])) + self._deallocate_address(0, address) + db.instance_destroy(context.get_admin_context(), + instance_ref['id']) def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 250170072..8e4a4daf5 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -118,6 +118,7 @@ class ZoneSchedulerTestCase(test.TestCase): arg = IgnoreArg() db.service_get_all_by_topic(arg, arg).AndReturn(service_list) self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + self.mox.StubOutWithMock(db, 'instance_create', use_mock_anything=True) rpc.cast(ctxt, 'compute.host1', {'method': 'run_instance', @@ -150,6 +151,7 @@ class SimpleDriverTestCase(test.TestCase): def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + super(SimpleDriverTestCase, self).tearDown() def _create_instance(self, **kwargs): """Create a test instance""" @@ -270,6 +272,7 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.driver.schedule_run_instance, self.context, instance_id) + db.instance_destroy(self.context, instance_id) for instance_id in instance_ids1: compute1.terminate_instance(self.context, instance_id) for instance_id in instance_ids2: diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..5b3247df9 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -204,6 +204,7 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) + db.instance_destroy(user_context, instance_ref['id']) def tearDown(self): super(LibvirtConnTestCase, self).tearDown() @@ -365,6 +366,7 @@ class IptablesFirewallTestCase(test.TestCase): '--dports 80:81 -j ACCEPT' % security_group_chain \ in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") + db.instance_destroy(admin_ctxt, instance_ref['id']) class NWFilterTestCase(test.TestCase): @@ -514,3 +516,4 @@ class NWFilterTestCase(test.TestCase): self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 92749f38a..4346dffc1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -319,7 +319,9 @@ class FakeConnection(object): return 'FAKE CONSOLE OUTPUT' def get_ajax_console(self, instance): - return 'http://fakeajaxconsole.com/?token=FAKETOKEN' + return {'token': 'FAKETOKEN', + 'host': 'fakeajaxconsole.com', + 'port': 6969} def get_console_pool_info(self, console_type): return {'address': '127.0.0.1', -- cgit From 015900b215805808d8cc3138b0f4deb2c0941f76 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 22 Feb 2011 23:30:52 -0800 Subject: remove unnecessary stubout --- nova/tests/test_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 8e4a4daf5..b6888c4d2 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -118,7 +118,6 @@ class ZoneSchedulerTestCase(test.TestCase): arg = IgnoreArg() db.service_get_all_by_topic(arg, arg).AndReturn(service_list) self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) - self.mox.StubOutWithMock(db, 'instance_create', use_mock_anything=True) rpc.cast(ctxt, 'compute.host1', {'method': 'run_instance', -- cgit From 60ed7c9c52306d08b1ad3e759e173931b0a495a8 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 00:59:15 -0800 Subject: fix failures --- run_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 4084d8d80..01a1cf8bc 100644 --- a/run_tests.py +++ b/run_tests.py @@ -181,6 +181,7 @@ class NovaTestResult(result.TextTestResult): return str(test) def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) if self.showAll: self.colorizer.write("OK", 'green') self.stream.writeln() @@ -188,7 +189,8 @@ class NovaTestResult(result.TextTestResult): self.stream.write('.') self.stream.flush() - def addFailure(self, test): + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) if self.showAll: self.colorizer.write("FAIL", 'red') self.stream.writeln() -- cgit From a9075d4edc126b95910258face7f00073449073d Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Wed, 23 Feb 2011 10:27:30 +0000 Subject: FlatManager.init_host now inhibits call to method in superclass. Floating IP methods have been redefined in FlatManager to raise NotImplementedError --- nova/network/manager.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nova/network/manager.py b/nova/network/manager.py index c6eba225e..a7f263daa 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -322,6 +322,17 @@ class FlatManager(NetworkManager): """ timeout_fixed_ips = False + def init_host(self): + """Do any initialization that needs to be run if this is a + standalone service. + """ + #Fix for bug 723298 - do not call init_host on superclass + #Following code has been copied for NetworkManager.init_host + ctxt = context.get_admin_context() + for network in self.db.host_get_networks(ctxt, self.host): + self._on_set_network_host(ctxt, network['id']) + + def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute @@ -406,6 +417,22 @@ class FlatManager(NetworkManager): net['dns'] = FLAGS.flat_network_dns self.db.network_update(context, network_id, net) + def allocate_floating_ip(self, context, project_id): + #Fix for bug 723298 + raise NotImplementedError() + + def associate_floating_ip(self, context, floating_address, fixed_address): + #Fix for bug 723298 + raise NotImplementedError() + + def disassociate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + + def deallocate_floating_ip(self, context, floating_address): + #Fix for bug 723298 + raise NotImplementedError() + class FlatDHCPManager(FlatManager): """Flat networking with dhcp. -- cgit From 79a4c527fbb75bc563721fa23be4ea4aa97b39ee Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Wed, 23 Feb 2011 11:49:47 +0000 Subject: Fixed pep8 errors --- nova/network/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index a7f263daa..1df193be0 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -332,7 +332,6 @@ class FlatManager(NetworkManager): for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) - def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Gets a fixed ip from the pool.""" # TODO(vish): when this is called by compute, we can associate compute -- cgit From d160455b77d7e180f252f4b412e3f65d7286b51f Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Feb 2011 08:45:27 -0800 Subject: allow users to omit 'nova.tests' with run_tests --- run_tests.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index 6d96454b9..47e3ee317 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,6 +17,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + import gettext import os import unittest @@ -62,6 +79,15 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() + # If any argument looks like a test name but doesn't have "nova.tests" in + # front of it, automatically add that so we don't have to type as much + argv = [] + for x in sys.argv: + if x.startswith('test_'): + argv.append('nova.tests.%s' % x) + else: + argv.append(x) + c = config.Config(stream=sys.stdout, env=os.environ, verbosity=3, @@ -70,4 +96,4 @@ if __name__ == '__main__': runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, config=c) - sys.exit(not core.run(config=c, testRunner=runner)) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) -- cgit From 2bbbfc5af62db57158a8d6aa26912ba234d0296e Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Feb 2011 08:46:11 -0800 Subject: dump error output directly on short import errors --- run_tests.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 70212cc6a..4e8159e7b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -40,7 +40,18 @@ done function run_tests { # Just run the test suites in current environment ${wrapper} rm -f nova.sqlite - ${wrapper} $NOSETESTS + ${wrapper} $NOSETESTS 2> run_tests.err.log + # If we get some short import error right away, print the error log directly + RESULT=$? + if [ "$RESULT" -ne "0" ]; + then + ERRSIZE=`wc -l run_tests.err.log | awk '{print \$1}'` + if [ "$ERRSIZE" -lt "40" ]; + then + cat run_tests.err.log + fi + fi + return $RESULT } NOSETESTS="python run_tests.py $noseargs" -- cgit From ef0dfb6809f31cfe8ca8056892fc9dcc2f00a0d7 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 09:40:43 -0800 Subject: Changed unit test to refer to compute API, per Todd's suggestion. Avoids needing to extend our implementation of the EC2 API. --- nova/api/ec2/cloud.py | 6 +----- nova/tests/test_quota.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 5db865b02..882cdcfc9 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -783,9 +783,6 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) - # NOTE(justinsb): the EC2 API doesn't support metadata here, but this - # is needed for the unit tests. Maybe the unit tests shouldn't be - # calling the EC2 code instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), @@ -800,8 +797,7 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone'), - metadata=kwargs.get('metadata', [])) + 'AvailabilityZone')) return self._format_run_instances(context, instances[0]['reservation_id']) diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 36ccc273e..1e42fddf3 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -16,6 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. +from nova import compute from nova import context from nova import db from nova import flags @@ -168,7 +169,7 @@ class QuotaTestCase(test.TestCase): metadata = {} for i in range(FLAGS.quota_metadata_items + 1): metadata['key%s' % i] = 'value%s' % i - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, -- cgit From 48d4054e093a2faccbd819de8e9e02c03d28cda0 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 11:26:38 -0800 Subject: fix for failing describe_instances test --- nova/api/ec2/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 1b96567eb..e219fb30c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -675,7 +675,8 @@ class CloudController(object): instances = [] for ec2_id in instance_id: internal_id = ec2_id_to_id(ec2_id) - instance = self.compute_api.get(context, internal_id) + instance = self.compute_api.get(context, + instance_id=internal_id) instances.append(instance) else: instances = self.compute_api.get_all(context, **kwargs) -- cgit From f7751eedc0e895f90d48104e2110bc2b320735fc Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 23 Feb 2011 13:53:02 -0600 Subject: Revert commit 709. This fixes issues with the Openstack API causing 'No user for access key admin' errors. --- nova/api/openstack/auth.py | 4 ++-- nova/tests/api/openstack/fakes.py | 8 ++------ nova/tests/api/openstack/test_auth.py | 6 +++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index c3fe0cc8c..1dfdd5318 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -120,8 +120,8 @@ class AuthMiddleware(wsgi.Middleware): req - webob.Request object """ ctxt = context.get_admin_context() - user = self.auth.get_user_from_access_key(username) - if user and user.secret == key: + user = self.auth.get_user_from_access_key(key) + if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, time.time())).hexdigest() token_dict = {} diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e0b7b8029..fb282f1c9 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -221,8 +221,7 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} - def add_user(self, user): - key = user.id + def add_user(self, key, user): FakeAuthManager.auth_data[key] = user def get_user(self, uid): @@ -235,10 +234,7 @@ class FakeAuthManager(object): return None def get_user_from_access_key(self, key): - for k, v in FakeAuthManager.auth_data.iteritems(): - if v.access == key: - return v - return None + return FakeAuthManager.auth_data.get(key, None) class FakeRateLimiter(object): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index eab78b50c..0dd65d321 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -48,7 +48,7 @@ class Test(unittest.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -62,7 +62,7 @@ class Test(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -144,7 +144,7 @@ class TestLimiter(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user(nova.auth.manager.User(1, 'herp', 'herp', 'derp', None)) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' -- cgit From 5283e1c131a21ea4963c702a7137536f7b894bb6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:05:49 -0800 Subject: Created mini XPath implementation, to simplify mapping logic --- nova/api/openstack/servers.py | 21 ++----- nova/tests/test_minixpath.py | 141 ++++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 45 ++++++++++++++ 3 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 nova/tests/test_minixpath.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index b54e28c0c..794705306 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -30,7 +30,7 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack - +import types LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) @@ -63,22 +63,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - fixed_ip = inst['fixed_ip'] - if fixed_ip: - # grab single private fixed ip - try: - private_ip = fixed_ip['address'] - if private_ip: - inst_dict['addresses']['private'].append(private_ip) - except KeyError: - LOG.debug(_("Failed to read private ip")) + private_ips = utils.minixpath_select(inst, 'fixed_ip/address') + inst_dict['addresses']['private'] = private_ips - # grab all public floating ips - try: - for floating in fixed_ip['floating_ips']: - inst_dict['addresses']['public'].append(floating['address']) - except KeyError: - LOG.debug(_("Failed to read public ip(s)")) + public_ips = utils.minixpath_select(inst, 'fixed_ip/floating_ips/address') + inst_dict['addresses']['public'] = public_ips inst_dict['metadata'] = {} inst_dict['hostId'] = '' diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py new file mode 100644 index 000000000..7fddcf9e9 --- /dev/null +++ b/nova/tests/test_minixpath.py @@ -0,0 +1,141 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class MiniXPathTestCase(test.TestCase): + def test_tolerates_nones(self): + xp = utils.minixpath_select + + input = [] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [None] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + def test_does_select(self): + xp = utils.minixpath_select + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], + xp(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) + + self.assertEquals([], xp(input, "a/b/c/d")) + self.assertEquals([], xp(input, "c/a/b/d")) + self.assertEquals([], xp(input, "i/r/t")) + + def test_flattens_lists(self): + xp = utils.minixpath_select + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + def test_bad_xpath(self): + xp = utils.minixpath_select + + self.assertRaises(exception.Error, xp, [], None) + self.assertRaises(exception.Error, xp, [], "") + self.assertRaises(exception.Error, xp, [], "/") + self.assertRaises(exception.Error, xp, [], "/a") + self.assertRaises(exception.Error, xp, [], "/a/") + self.assertRaises(exception.Error, xp, [], "//") + self.assertRaises(exception.Error, xp, [], "//a") + self.assertRaises(exception.Error, xp, [], "a//a") + self.assertRaises(exception.Error, xp, [], "a//a/") + self.assertRaises(exception.Error, xp, [], "a/a/") diff --git a/nova/utils.py b/nova/utils.py index 42efa0008..2f926bd82 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -34,6 +35,7 @@ import time from xml.sax import saxutils import re import netaddr +import types from eventlet import event from eventlet import greenthread @@ -499,3 +501,46 @@ def ensure_b64_encoding(val): return val except TypeError: return base64.b64encode(val) + + +def minixpath_select(items, minixpath): + """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each + item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of + the intermediate results are lists it will treat each list item + individually. A 'None' in items or any child expressions will be ignored, + this function will not throw because of None (anywhere) in items""" + + if minixpath is None: + raise exception.Error("Invalid mini_xpath") + + (first_token, sep, remainder) = minixpath.partition("/") + + if first_token == "": + raise exception.Error("Invalid mini_xpath") + + results = [] + + if items is None: + return results + + for item in items: + if item is None: + continue + get_method = getattr(item, "get", None) + if get_method is None: + continue + child = get_method(first_token) + if child is None: + continue + if isinstance(child, types.ListType): + # Flatten intermediate lists + for x in child: + results.append(x) + else: + results.append(child) + + if not sep: + # No more tokens + return results + else: + return minixpath_select(results, remainder) -- cgit From b3b005f50de54b5ef6c62e387dcec5a123f93cf6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:36:09 -0800 Subject: Cope when we pass a non-list to xpath_select - wrap it in a list --- nova/tests/test_minixpath.py | 38 ++++++++++++++++++++++++++++++++++++++ nova/utils.py | 8 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 7fddcf9e9..3b1bdf40b 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -139,3 +139,41 @@ class MiniXPathTestCase(test.TestCase): self.assertRaises(exception.Error, xp, [], "a//a") self.assertRaises(exception.Error, xp, [], "a//a/") self.assertRaises(exception.Error, xp, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + xp = utils.minixpath_select + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = xp(inst, 'fixed_ip/address') + public_ips = xp(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + xp = utils.minixpath_select + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + diff --git a/nova/utils.py b/nova/utils.py index 2f926bd82..c2cbeb2a7 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -508,7 +508,8 @@ def minixpath_select(items, minixpath): item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of the intermediate results are lists it will treat each list item individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items""" + this function will not throw because of None (anywhere) in items. The + returned list will contain no None values.""" if minixpath is None: raise exception.Error("Invalid mini_xpath") @@ -523,6 +524,10 @@ def minixpath_select(items, minixpath): if items is None: return results + if not isinstance(items, types.ListType): + # Wrap single objects in a list + items = [items] + for item in items: if item is None: continue @@ -532,6 +537,7 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue + #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: -- cgit From 21ebea24b4b77f8bd1fd42152454f1b0189843d4 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 12:54:46 -0800 Subject: fix describe_availability_zones --- nova/api/ec2/cloud.py | 5 +++-- nova/db/api.py | 4 ++-- nova/db/sqlalchemy/api.py | 9 +++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 99b6d5cb6..9e8764836 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -198,8 +198,9 @@ class CloudController(object): return self._describe_availability_zones(context, **kwargs) def _describe_availability_zones(self, context, **kwargs): - enabled_services = db.service_get_all(context) - disabled_services = db.service_get_all(context, True) + ctxt = context.elevated() + enabled_services = db.service_get_all(ctxt) + disabled_services = db.service_get_all(ctxt, True) available_zones = [] for zone in [service.availability_zone for service in enabled_services]: diff --git a/nova/db/api.py b/nova/db/api.py index d7f3746d2..0a010e727 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -85,8 +85,8 @@ def service_get(context, service_id): def service_get_all(context, disabled=False): - """Get all service.""" - return IMPL.service_get_all(context, None, disabled) + """Get all services.""" + return IMPL.service_get_all(context, disabled) def service_get_all_by_topic(context, topic): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index be29fe2a0..d8751bef4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -136,15 +136,12 @@ def service_get(context, service_id, session=None): @require_admin_context -def service_get_all(context, session=None, disabled=False): - if not session: - session = get_session() - - result = session.query(models.Service).\ +def service_get_all(context, disabled=False): + session = get_session() + return session.query(models.Service).\ filter_by(deleted=can_read_deleted(context)).\ filter_by(disabled=disabled).\ all() - return result @require_admin_context -- cgit From 89ade95d2eaabf77f9c81a8d50c7cc11aa175464 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:55:07 -0800 Subject: Fix pep8 violation (trailing whitespace) --- nova/tests/test_minixpath.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 3b1bdf40b..cc4a35ef3 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -175,5 +175,3 @@ class MiniXPathTestCase(test.TestCase): input = {'a': [1, 2, {'b': 'b_1'}]} self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) self.assertEquals(['b_1'], xp(input, "a/b")) - - -- cgit From 1183c9e11b12984b1f5007ace831864e80483712 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:07:08 -0800 Subject: Rename minixpath_select to get_from_path --- nova/api/openstack/servers.py | 5 +- nova/tests/test_minixpath.py | 177 ------------------------------------------ nova/tests/test_utils.py | 174 +++++++++++++++++++++++++++++++++++++++++ nova/utils.py | 24 +++--- 4 files changed, 188 insertions(+), 192 deletions(-) delete mode 100644 nova/tests/test_minixpath.py create mode 100644 nova/tests/test_utils.py diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 794705306..ce4a6256a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -30,7 +30,6 @@ from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack -import types LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) @@ -63,10 +62,10 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) - private_ips = utils.minixpath_select(inst, 'fixed_ip/address') + private_ips = utils.get_from_path(inst, 'fixed_ip/address') inst_dict['addresses']['private'] = private_ips - public_ips = utils.minixpath_select(inst, 'fixed_ip/floating_ips/address') + public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips inst_dict['metadata'] = {} diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py deleted file mode 100644 index cc4a35ef3..000000000 --- a/nova/tests/test_minixpath.py +++ /dev/null @@ -1,177 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import test -from nova import utils -from nova import exception - - -class MiniXPathTestCase(test.TestCase): - def test_tolerates_nones(self): - xp = utils.minixpath_select - - input = [] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [None] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': None}] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': None}}] - self.assertEquals([{'b': None}], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': None}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] - self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - def test_does_select(self): - xp = utils.minixpath_select - - input = [{'a': 'a_1'}] - self.assertEquals(['a_1'], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': 'b_1'}}] - self.assertEquals([{'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] - self.assertEquals([{'b': {'c': 'c_1'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': None}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': {'c': 'c_2'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], - xp(input, "a/b")) - self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) - - self.assertEquals([], xp(input, "a/b/c/d")) - self.assertEquals([], xp(input, "c/a/b/d")) - self.assertEquals([], xp(input, "i/r/t")) - - def test_flattens_lists(self): - xp = utils.minixpath_select - - input = [{'a': [1, 2, 3]}] - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}] - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [1, 2, {'b': 'b_1'}]}] - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - - def test_bad_xpath(self): - xp = utils.minixpath_select - - self.assertRaises(exception.Error, xp, [], None) - self.assertRaises(exception.Error, xp, [], "") - self.assertRaises(exception.Error, xp, [], "/") - self.assertRaises(exception.Error, xp, [], "/a") - self.assertRaises(exception.Error, xp, [], "/a/") - self.assertRaises(exception.Error, xp, [], "//") - self.assertRaises(exception.Error, xp, [], "//a") - self.assertRaises(exception.Error, xp, [], "a//a") - self.assertRaises(exception.Error, xp, [], "a//a/") - self.assertRaises(exception.Error, xp, [], "a/a/") - - def test_real_failure1(self): - # Real world failure case... - # We weren't coping when the input was a Dictionary instead of a List - # This led to test_accepts_dictionaries - xp = utils.minixpath_select - - inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], - 'address': '192.168.0.3'}, - 'hostname': ''} - - private_ips = xp(inst, 'fixed_ip/address') - public_ips = xp(inst, 'fixed_ip/floating_ips/address') - self.assertEquals(['192.168.0.3'], private_ips) - self.assertEquals(['1.2.3.4'], public_ips) - - def test_accepts_dictionaries(self): - xp = utils.minixpath_select - - input = {'a': [1, 2, 3]} - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': {'b': [1, 2, 3]}} - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [1, 2, {'b': 'b_1'}]} - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py new file mode 100644 index 000000000..34a407f1a --- /dev/null +++ b/nova/tests/test_utils.py @@ -0,0 +1,174 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class GetFromPathTestCase(test.TestCase): + def test_tolerates_nones(self): + f = utils.get_from_path + + input = [] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [None] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': None}], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + def test_does_select(self): + f = utils.get_from_path + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], f(input, "a")) + self.assertEquals([{'c': 'c_1'}], f(input, "a/b")) + self.assertEquals(['c_1'], f(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + f(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], f(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], f(input, "a/b/c")) + + self.assertEquals([], f(input, "a/b/c/d")) + self.assertEquals([], f(input, "c/a/b/d")) + self.assertEquals([], f(input, "i/r/t")) + + def test_flattens_lists(self): + f = utils.get_from_path + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) + + def test_bad_xpath(self): + f = utils.get_from_path + + self.assertRaises(exception.Error, f, [], None) + self.assertRaises(exception.Error, f, [], "") + self.assertRaises(exception.Error, f, [], "/") + self.assertRaises(exception.Error, f, [], "/a") + self.assertRaises(exception.Error, f, [], "/a/") + self.assertRaises(exception.Error, f, [], "//") + self.assertRaises(exception.Error, f, [], "//a") + self.assertRaises(exception.Error, f, [], "a//a") + self.assertRaises(exception.Error, f, [], "a//a/") + self.assertRaises(exception.Error, f, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + f = utils.get_from_path + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = f(inst, 'fixed_ip/address') + public_ips = f(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + f = utils.get_from_path + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], f(input, "a")) + self.assertEquals([], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], f(input, "a")) + self.assertEquals([1, 2, 3], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], f(input, "a/b")) + self.assertEquals([], f(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], f(input, "a")) + self.assertEquals(['b_1'], f(input, "a/b")) diff --git a/nova/utils.py b/nova/utils.py index c2cbeb2a7..65e28c648 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -32,10 +32,10 @@ import string import struct import sys import time +import types from xml.sax import saxutils import re import netaddr -import types from eventlet import event from eventlet import greenthread @@ -503,18 +503,19 @@ def ensure_b64_encoding(val): return base64.b64encode(val) -def minixpath_select(items, minixpath): - """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each - item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of - the intermediate results are lists it will treat each list item - individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items. The - returned list will contain no None values.""" +def get_from_path(items, path): + """ Returns a list of items matching the specified path. Takes an + XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, + looks up items[prop1][prop2][prop3]. Like XPath, if any of the + intermediate results are lists it will treat each list item individually. + A 'None' in items or any child expressions will be ignored, this function + will not throw because of None (anywhere) in items. The returned list + will contain no None values.""" - if minixpath is None: + if path is None: raise exception.Error("Invalid mini_xpath") - (first_token, sep, remainder) = minixpath.partition("/") + (first_token, sep, remainder) = path.partition("/") if first_token == "": raise exception.Error("Invalid mini_xpath") @@ -537,7 +538,6 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue - #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: @@ -549,4 +549,4 @@ def minixpath_select(items, minixpath): # No more tokens return results else: - return minixpath_select(results, remainder) + return get_from_path(results, remainder) -- cgit From a508e2dce781b98db5a719df75a451d9a2727fca Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:12:43 -0800 Subject: Make sure there are two blank links after the import --- nova/api/openstack/servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ce4a6256a..6c227d71a 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -31,6 +31,7 @@ from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack + LOG = logging.getLogger('server') LOG.setLevel(logging.DEBUG) -- cgit From 8c007b56b586257d048b6db4ecfbed8f502381fd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:16:31 -0800 Subject: Put back the comments I accidentally removed --- nova/api/openstack/servers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 6c227d71a..97323f66f 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -63,9 +63,11 @@ def _translate_detail_keys(inst): inst_dict['status'] = power_mapping[inst_dict['status']] inst_dict['addresses'] = dict(public=[], private=[]) + # grab single private fixed ip private_ips = utils.get_from_path(inst, 'fixed_ip/address') inst_dict['addresses']['private'] = private_ips + # grab all public floating ips public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') inst_dict['addresses']['public'] = public_ips -- cgit From 24090232272e0db163060e0ca32dbf97c05120c9 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Feb 2011 15:14:16 -0800 Subject: updates to nova.flags to get help working better Fixes some old bugs that were brought up on the mailing list. First step towards moving flags into the places where they belong. Also moves manager import into service's init so that we can get all the dynamically loaded flags shortly after loading. --- nova/flags.py | 49 +++++++++++++++++++++++++++++++++++++++++----- nova/service.py | 31 +++++++++++++++++------------ nova/tests/test_service.py | 7 ------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/nova/flags.py b/nova/flags.py index f64a62da9..24bca0caf 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -160,9 +160,45 @@ class StrWrapper(object): raise KeyError(name) -FLAGS = FlagValues() -gflags.FLAGS = FLAGS -gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) +# Copied from gflags with small mods to get the naming correct. +# Originally gflags checks for the first module that is not gflags that is +# in the call chain, we want to check for the first module that is not gflags +# and not this module. +def _GetCallingModule(): + """Returns the name of the module that's calling into this module. + + We generally use this function to get the name of the module calling a + DEFINE_foo... function. + """ + # Walk down the stack to find the first globals dict that's not ours. + for depth in range(1, sys.getrecursionlimit()): + if not sys._getframe(depth).f_globals is globals(): + module_name = __GetModuleName(sys._getframe(depth).f_globals) + if module_name == 'gflags': + continue + if module_name is not None: + return module_name + raise AssertionError("No module was found") + + +# Copied from gflags because it is a private function +def __GetModuleName(globals_dict): + """Given a globals dict, returns the name of the module that defines it. + + Args: + globals_dict: A dictionary that should correspond to an environment + providing the values of the globals. + + Returns: + A string (the name of the module) or None (if the module could not + be identified. + """ + for name, module in sys.modules.iteritems(): + if getattr(module, '__dict__', None) is globals_dict: + if name == '__main__': + return sys.argv[0] + return name + return None def _wrapper(func): @@ -173,6 +209,11 @@ def _wrapper(func): return _wrapped +FLAGS = FlagValues() +gflags.FLAGS = FLAGS +gflags._GetCallingModule = _GetCallingModule + + DEFINE = _wrapper(gflags.DEFINE) DEFINE_string = _wrapper(gflags.DEFINE_string) DEFINE_integer = _wrapper(gflags.DEFINE_integer) @@ -185,8 +226,6 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) DEFINE_flag = _wrapper(gflags.DEFINE_flag) - - HelpFlag = gflags.HelpFlag HelpshortFlag = gflags.HelpshortFlag HelpXMLFlag = gflags.HelpXMLFlag diff --git a/nova/service.py b/nova/service.py index cc88ac233..f47358089 100644 --- a/nova/service.py +++ b/nova/service.py @@ -45,15 +45,10 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('report_interval', 10, 'seconds between nodes reporting state to datastore', lower_bound=1) - flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) -flags.DEFINE_flag(flags.HelpFlag()) -flags.DEFINE_flag(flags.HelpshortFlag()) -flags.DEFINE_flag(flags.HelpXMLFlag()) - class Service(object): """Base class for workers that run on hosts.""" @@ -64,6 +59,8 @@ class Service(object): self.binary = binary self.topic = topic self.manager_class_name = manager + manager_class = utils.import_class(self.manager_class_name) + self.manager = manager_class(host=self.host, *args, **kwargs) self.report_interval = report_interval self.periodic_interval = periodic_interval super(Service, self).__init__(*args, **kwargs) @@ -71,9 +68,9 @@ class Service(object): self.timers = [] def start(self): - manager_class = utils.import_class(self.manager_class_name) - self.manager = manager_class(host=self.host, *self.saved_args, - **self.saved_kwargs) + vcs_string = version.version_string_with_vcs() + logging.audit(_("Starting %(topic)s node (version %(vcs_string)s)"), + {'topic': self.topic, 'vcs_string': vcs_string}) self.manager.init_host() self.model_disconnected = False ctxt = context.get_admin_context() @@ -153,9 +150,6 @@ class Service(object): report_interval = FLAGS.report_interval if not periodic_interval: periodic_interval = FLAGS.periodic_interval - vcs_string = version.version_string_with_vcs() - logging.audit(_("Starting %(topic)s node (version %(vcs_string)s)") - % locals()) service_obj = cls(host, binary, topic, manager, report_interval, periodic_interval) @@ -217,8 +211,19 @@ class Service(object): def serve(*services): - if not services: - services = [Service.create()] + try: + if not services: + services = [Service.create()] + except Exception: + logging.exception('in Service.create()') + raise + finally: + # After we've loaded up all our dynamic bits, check + # whether we should print help + flags.DEFINE_flag(flags.HelpFlag()) + flags.DEFINE_flag(flags.HelpshortFlag()) + flags.DEFINE_flag(flags.HelpXMLFlag()) + FLAGS.ParseNewFlags() name = '_'.join(x.binary for x in services) logging.debug(_("Serving %s"), name) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index a67c8d1e8..45d9afa6c 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -50,13 +50,6 @@ class ExtendedService(service.Service): class ServiceManagerTestCase(test.TestCase): """Test cases for Services""" - def test_attribute_error_for_no_manager(self): - serv = service.Service('test', - 'test', - 'test', - 'nova.tests.test_service.FakeManager') - self.assertRaises(AttributeError, getattr, serv, 'test_method') - def test_message_gets_to_manager(self): serv = service.Service('test', 'test', -- cgit From 3115a65f9981371ea8587a288b360c3c519de865 Mon Sep 17 00:00:00 2001 From: termie Date: Wed, 23 Feb 2011 15:26:52 -0800 Subject: add help back to the scripts that don't use service.py --- bin/nova-ajax-console-proxy | 4 +++- bin/nova-api | 3 +++ bin/nova-direct-api | 4 ++++ bin/nova-manage | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index 392b328b1..1e11c6d58 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -47,9 +47,11 @@ from nova import utils from nova import wsgi FLAGS = flags.FLAGS - flags.DEFINE_integer('ajax_console_idle_timeout', 300, 'Seconds before idle connection destroyed') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) LOG = logging.getLogger('nova.ajax_console_proxy') LOG.setLevel(logging.DEBUG) diff --git a/bin/nova-api b/bin/nova-api index d5efb4687..cf140570a 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -48,6 +48,9 @@ flags.DEFINE_integer('ec2_listen_port', 8773, 'port for ec2 api to listen') flags.DEFINE_string('osapi_listen', "0.0.0.0", 'IP address for OpenStack API to listen') flags.DEFINE_integer('osapi_listen_port', 8774, 'port for os api to listen') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) API_ENDPOINTS = ['ec2', 'osapi'] diff --git a/bin/nova-direct-api b/bin/nova-direct-api index 6c63bd26b..bf29d9a5e 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -45,6 +45,10 @@ from nova.compute import api as compute_api FLAGS = flags.FLAGS flags.DEFINE_integer('direct_port', 8001, 'Direct API port') flags.DEFINE_string('direct_host', '0.0.0.0', 'Direct API host') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + if __name__ == '__main__': utils.default_flagfile() diff --git a/bin/nova-manage b/bin/nova-manage index 5189de0e1..b603c8b07 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -93,6 +93,9 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) def param2id(object_id): -- cgit From 5e2f82b1487b8f8e43539d0c71466fbbfed23121 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 15:29:02 -0800 Subject: revert logfile redirection and make colors work by temporarily switching stdout --- nova/tests/fake_flags.py | 1 - run_tests.py | 16 +++++++++------- run_tests.sh | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 575fefff6..2b1919407 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -41,4 +41,3 @@ FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///tests.sqlite' FLAGS.use_ipv6 = True -FLAGS.logfile = 'tests.log' diff --git a/run_tests.py b/run_tests.py index 877849ab5..2c04a0641 100644 --- a/run_tests.py +++ b/run_tests.py @@ -97,7 +97,7 @@ class _AnsiColorizer(object): try: return curses.tigetnum("colors") > 2 except curses.error: - curses.setupterm(fd=stream.fileno()) + curses.setupterm() return curses.tigetnum("colors") > 2 except: raise @@ -122,13 +122,13 @@ class _Win32Colorizer(object): See _AnsiColorizer docstring. """ def __init__(self, stream): - from win32console import GetStdHandle, STD_ERROR_HANDLE, \ + from win32console import GetStdHandle, STD_OUT_HANDLE, \ FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ FOREGROUND_INTENSITY red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, FOREGROUND_BLUE, FOREGROUND_INTENSITY) self.stream = stream - self.screenBuffer = GetStdHandle(STD_ERROR_HANDLE) + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) self._colors = { 'normal': red | green | blue, 'red': red | bold, @@ -144,7 +144,7 @@ class _Win32Colorizer(object): try: import win32console screenBuffer = win32console.GetStdHandle( - win32console.STD_ERROR_HANDLE) + win32console.STD_OUT_HANDLE) except ImportError: return False import pywintypes @@ -185,12 +185,14 @@ class NovaTestResult(result.TextTestResult): result.TextTestResult.__init__(self, *args, **kw) self._last_case = None self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: - # NOTE(vish): nose does funky stuff with stdout, so use stderr - # to setup the colorizer - if colorizer.supported(sys.stderr): + if colorizer.supported(): self.colorizer = colorizer(self.stream) break + sys.stdout = stdout def getDescription(self, test): return str(test) diff --git a/run_tests.sh b/run_tests.sh index 7aab9ecd9..d4586a57e 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -39,12 +39,12 @@ done function run_tests { # Just run the test suites in current environment - ${wrapper} $NOSETESTS + ${wrapper} $NOSETESTS 2> run_tests.log # If we get some short import error right away, print the error log directly RESULT=$? if [ "$RESULT" -ne "0" ]; then - ERRSIZE=`wc -l run_tests.err.log | awk '{print \$1}'` + ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` if [ "$ERRSIZE" -lt "40" ]; then cat run_tests.err.log -- cgit From c27c19ea316b343f1623a7c1bf21c53cd426603b Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 15:35:30 -0800 Subject: fix pep8 --- run_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run_tests.py b/run_tests.py index 2c04a0641..eef0120bc 100644 --- a/run_tests.py +++ b/run_tests.py @@ -87,7 +87,7 @@ class _AnsiColorizer(object): coloring terminal output using this method. Returns False otherwise. """ if not stream.isatty(): - return False # auto color only on TTYs + return False # auto color only on TTYs try: import curses except ImportError: @@ -180,6 +180,7 @@ class _NullColorizer(object): def write(self, text, color): self.stream.write(text) + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): result.TextTestResult.__init__(self, *args, **kw) @@ -276,7 +277,7 @@ class NovaTestRunner(core.TextTestRunner): if __name__ == '__main__': logging.setup() - testdir = os.path.abspath(os.path.join("nova","tests")) + testdir = os.path.abspath(os.path.join("nova", "tests")) testdb = os.path.join(testdir, "tests.sqlite") if os.path.exists(testdb): os.unlink(testdb) -- cgit From 78eddce564cccf0d9be19b303cbc122966f5fa71 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 15:42:59 -0800 Subject: added comments about where code came from --- run_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tests.py b/run_tests.py index eef0120bc..8025548e5 100644 --- a/run_tests.py +++ b/run_tests.py @@ -198,6 +198,7 @@ class NovaTestResult(result.TextTestResult): def getDescription(self, test): return str(test) + # NOTE(vish): copied from unittest with edit to add color def addSuccess(self, test): unittest.TestResult.addSuccess(self, test) if self.showAll: @@ -207,6 +208,7 @@ class NovaTestResult(result.TextTestResult): self.stream.write('.') self.stream.flush() + # NOTE(vish): copied from unittest with edit to add color def addFailure(self, test, err): unittest.TestResult.addFailure(self, test, err) if self.showAll: @@ -216,6 +218,7 @@ class NovaTestResult(result.TextTestResult): self.stream.write('F') self.stream.flush() + # NOTE(vish): copied from nose with edit to add color def addError(self, test, err): """Overrides normal addError to add support for errorClasses. If the exception is a registered class, the -- cgit From 3aa0183bca5585c706b4b02e0a7542422015c693 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 23 Feb 2011 18:54:13 -0800 Subject: fix missed err.log --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index d4586a57e..7ac3ff33f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -47,7 +47,7 @@ function run_tests { ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'` if [ "$ERRSIZE" -lt "40" ]; then - cat run_tests.err.log + cat run_tests.log fi fi return $RESULT -- cgit From f86c45764d6cf62b1ded928e807e93198ed6ffd1 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 22:02:06 -0800 Subject: PEP 257 fixes --- nova/volume/driver.py | 42 +++++++++++++++++++++++------------------- nova/volume/san.py | 22 +++++++++++++++------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 22c2c2fc3..4263a6a8d 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -227,14 +227,16 @@ class FakeAOEDriver(AOEDriver): class ISCSIDriver(VolumeDriver): - """Executes commands relating to ISCSI volumes. We make use of model - provider properties as follows: - provider_location - if present, contains the iSCSI target information - in the same format as an ietadm discovery - i.e. ', ' - provider_auth - if present, contains a space-separated triple: - ' '. CHAP is the only - auth_method in use at the moment.""" + """Executes commands relating to ISCSI volumes. + + We make use of model provider properties as follows: + :provider_location: if present, contains the iSCSI target information + in the same format as an ietadm discovery + i.e. ':, ' + :provider_auth: if present, contains a space-separated triple: + ' '. + `CHAP` is the only auth_method in use at the moment. + """ def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" @@ -321,17 +323,19 @@ class ISCSIDriver(VolumeDriver): return None def _get_iscsi_properties(self, volume): - """Gets iscsi configuration, ideally from saved information in the - volume entity, but falling back to discovery if need be. The - properties are: - target_discovered - boolean indicating whether discovery was used, - target_iqn - the IQN of the iSCSI target, - target_portal - the portal of the iSCSI target, - and auth_method, auth_username and auth_password - - the authentication details. Right now, either - auth_method is not present meaning no authentication, or - auth_method == 'CHAP' meaning use CHAP with the specified - credentials.""" + """Gets iscsi configuration + + We ideally get saved information in the volume entity, but fall back + to discovery if need be. Discovery may be completely removed in future + The properties are: + :target_discovered: boolean indicating whether discovery was used + :target_iqn: the IQN of the iSCSI target, + :target_portal: the portal of the iSCSI target + :auth_method:, :auth_username:, :auth_password: + the authentication details. Right now, either auth_method is not + present meaning no authentication, or auth_method == `CHAP` + meaning use CHAP with the specified credentials. + """ properties = {} diff --git a/nova/volume/san.py b/nova/volume/san.py index 09192bc9f..d6598d16e 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -16,8 +16,9 @@ # under the License. """ Drivers for san-stored volumes. + The unique thing about a SAN is that we don't expect that we can run the volume - controller on the SAN hardware. We expect to access it over SSH or some API. +controller on the SAN hardware. We expect to access it over SSH or some API. """ import os @@ -51,7 +52,11 @@ flags.DEFINE_integer('san_ssh_port', 22, class SanISCSIDriver(ISCSIDriver): """ Base class for SAN-style storage volumes - (storage providers we access over SSH)""" + + A SAN-style storage value is 'different' because the volume controller + probably won't run on it, so we need to access is over SSH or another + remote protocol. + """ def _build_iscsi_target_name(self, volume): return "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) @@ -137,6 +142,7 @@ def _get_prefixed_values(data, prefix): class SolarisISCSIDriver(SanISCSIDriver): """Executes commands relating to Solaris-hosted ISCSI volumes. + Basic setup for a Solaris iSCSI server: pkg install storage-server SUNWiscsit svcadm enable stmf @@ -330,13 +336,14 @@ class SolarisISCSIDriver(SanISCSIDriver): class HpSanISCSIDriver(SanISCSIDriver): """Executes commands relating to HP/Lefthand SAN ISCSI volumes. + We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: - CLIQ createVolume (creates the volume) - CLIQ getVolumeInfo (to discover the IQN etc) - CLIQ getClusterInfo (to discover the iSCSI target IP address) - CLIQ assignVolumeChap (exports it with CHAP security) + :createVolume: (creates the volume) + :getVolumeInfo: (to discover the IQN etc) + :getClusterInfo: (to discover the iSCSI target IP address) + :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so normally a volume mount would need both to configure the SAN in the volume @@ -344,7 +351,8 @@ class HpSanISCSIDriver(SanISCSIDriver): not catered for at the moment in the nova architecture, so instead we share the volume using CHAP at volume creation time. Then the mount need only use those CHAP credentials, so can take place exclusively in the - compute layer""" + compute layer. + """ def _cliq_run(self, verb, cliq_args): """Runs a CLIQ command over SSH, without doing any result parsing""" -- cgit From 0a649493d21a8766f416ceca2c0122c20945ca1b Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 22:47:57 -0800 Subject: Documentation fixes so that output looks better --- nova/volume/driver.py | 9 ++++++++- nova/volume/san.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 4263a6a8d..e3744c790 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -230,9 +230,11 @@ class ISCSIDriver(VolumeDriver): """Executes commands relating to ISCSI volumes. We make use of model provider properties as follows: + :provider_location: if present, contains the iSCSI target information in the same format as an ietadm discovery i.e. ':, ' + :provider_auth: if present, contains a space-separated triple: ' '. `CHAP` is the only auth_method in use at the moment. @@ -328,10 +330,15 @@ class ISCSIDriver(VolumeDriver): We ideally get saved information in the volume entity, but fall back to discovery if need be. Discovery may be completely removed in future The properties are: + :target_discovered: boolean indicating whether discovery was used - :target_iqn: the IQN of the iSCSI target, + + :target_iqn: the IQN of the iSCSI target + :target_portal: the portal of the iSCSI target + :auth_method:, :auth_username:, :auth_password: + the authentication details. Right now, either auth_method is not present meaning no authentication, or auth_method == `CHAP` meaning use CHAP with the specified credentials. diff --git a/nova/volume/san.py b/nova/volume/san.py index d6598d16e..9532c8116 100644 --- a/nova/volume/san.py +++ b/nova/volume/san.py @@ -144,16 +144,25 @@ class SolarisISCSIDriver(SanISCSIDriver): """Executes commands relating to Solaris-hosted ISCSI volumes. Basic setup for a Solaris iSCSI server: + pkg install storage-server SUNWiscsit + svcadm enable stmf + svcadm enable -r svc:/network/iscsi/target:default + pfexec itadm create-tpg e1000g0 ${MYIP} + pfexec itadm create-target -t e1000g0 + Then grant the user that will be logging on lots of permissions. I'm not sure exactly which though: + zfs allow justinsb create,mount,destroy rpool + usermod -P'File System Management' justinsb + usermod -P'Primary Administrator' justinsb Also make sure you can login using san_login & san_password/san_privatekey @@ -340,9 +349,13 @@ class HpSanISCSIDriver(SanISCSIDriver): We use the CLIQ interface, over SSH. Rough overview of CLIQ commands used: + :createVolume: (creates the volume) + :getVolumeInfo: (to discover the IQN etc) + :getClusterInfo: (to discover the iSCSI target IP address) + :assignVolumeChap: (exports it with CHAP security) The 'trick' here is that the HP SAN enforces security by default, so -- cgit From 7e463d262339acc7c611b86471b1422dea50d1ee Mon Sep 17 00:00:00 2001 From: Salvatore Orlando Date: Thu, 24 Feb 2011 11:43:40 +0000 Subject: xenapi plugin function now checks whether /boot/guest already exists. If not, it creates the directory --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 8cb439259..61b947c25 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -73,6 +73,10 @@ def _copy_kernel_vdi(dest, copy_args): logging.debug("copying kernel/ramdisk file from %s to /boot/guest/%s", dest, vdi_uuid) filename = KERNEL_DIR + '/' + vdi_uuid + #make sure KERNEL_DIR exists, otherwise create it + if not os.path.isdir(KERNEL_DIR): + logging.debug("Creating directory %s", KERNEL_DIR) + os.makedirs(KERNEL_DIR) #read data from /dev/ and write into a file on /boot/guest of = open(filename, 'wb') f = open(dest, 'rb') -- cgit From f8640adff4beff1e7d77bbd67771a6d072d42140 Mon Sep 17 00:00:00 2001 From: Armando Migliaccio Date: Thu, 24 Feb 2011 13:58:31 +0000 Subject: Get DNS value from Flag, when working in FlatNetworking mode. Passing the flag was ineffective previously. This is an easy fix. I think we would need nova-manage to accept dns also from command line --- nova/network/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/network/manager.py b/nova/network/manager.py index 1df193be0..12a0c5018 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -369,6 +369,7 @@ class FlatManager(NetworkManager): project_net = IPy.IP(cidr) net = {} net['bridge'] = FLAGS.flat_network_bridge + net['dns'] = FLAGS.flat_network_dns net['cidr'] = cidr net['netmask'] = str(project_net.netmask()) net['gateway'] = str(project_net[1]) -- cgit From a721d2208ed9d88884925d6f1e2c8e26d7d1ea27 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Thu, 24 Feb 2011 15:19:29 +0100 Subject: Globally exclude .pyc files from tarball contents --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index f0a9cffb3..2ceed34f3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -38,3 +38,4 @@ include nova/tests/db/nova.austin.sqlite include plugins/xenapi/README include plugins/xenapi/etc/xapi.d/plugins/objectstore include plugins/xenapi/etc/xapi.d/plugins/pluginlib_nova.py +global-exclude *.pyc -- cgit